1359 files changed, 75436 insertions, 56397 deletions

diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 5a0c476361c3..026444eeb5c6 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -29,6 +29,8 @@ menuconfig DRM
 	  details.  You should also select and configure AGP
 	  (/dev/agpgart) support if it is available for your platform.
 
+if DRM
+
 config DRM_MIPI_DBI
 	tristate
 	depends on DRM
@@ -102,6 +104,38 @@ config DRM_KMS_HELPER
 	help
 	  CRTC helpers for KMS drivers.
 
+config DRM_PANIC
+	bool "Display a user-friendly message when a kernel panic occurs"
+	depends on DRM && !FRAMEBUFFER_CONSOLE
+	select DRM_KMS_HELPER
+	select FONT_SUPPORT
+	help
+	  Enable a drm panic handler, which will display a user-friendly message
+	  when a kernel panic occurs. It's useful when using a user-space
+	  console instead of fbcon.
+	  It will only work if your graphic driver supports this feature.
+	  To support Hi-DPI Display, you can enable bigger fonts like
+	  FONT_TER16x32
+
+config DRM_PANIC_FOREGROUND_COLOR
+	hex "Drm panic screen foreground color, in RGB"
+	depends on DRM_PANIC
+	default 0xffffff
+
+config DRM_PANIC_BACKGROUND_COLOR
+	hex "Drm panic screen background color, in RGB"
+	depends on DRM_PANIC
+	default 0x000000
+
+config DRM_PANIC_DEBUG
+	bool "Add a debug fs entry to trigger drm_panic"
+	depends on DRM_PANIC && DEBUG_FS
+	help
+	  Add dri/[device]/drm_panic_plane_x in the kernel debugfs, to force the
+	  panic handler to write the panic message to this plane scanout buffer.
+	  This is unsafe and should not be enabled on a production build.
+	  If in doubt, say "N".
+
 config DRM_DEBUG_DP_MST_TOPOLOGY_REFS
         bool "Enable refcount backtrace history in the DP MST helpers"
 	depends on STACKTRACE_SUPPORT
@@ -371,6 +405,8 @@ source "drivers/gpu/drm/lima/Kconfig"
 
 source "drivers/gpu/drm/panfrost/Kconfig"
 
+source "drivers/gpu/drm/panthor/Kconfig"
+
 source "drivers/gpu/drm/aspeed/Kconfig"
 
 source "drivers/gpu/drm/mcde/Kconfig"
@@ -403,10 +439,6 @@ config DRM_HYPERV
 config DRM_EXPORT_FOR_TESTS
 	bool
 
-# Separate option because drm_panel_orientation_quirks.c is shared with fbdev
-config DRM_PANEL_ORIENTATION_QUIRKS
-	tristate
-
 config DRM_LIB_RANDOM
 	bool
 	default n
@@ -414,3 +446,22 @@ config DRM_LIB_RANDOM
 config DRM_PRIVACY_SCREEN
 	bool
 	default n
+
+config DRM_WERROR
+	bool "Compile the drm subsystem with warnings as errors"
+	depends on DRM && EXPERT
+	default n
+	help
+	  A kernel build should not cause any compiler warnings, and this
+	  enables the '-Werror' flag to enforce that rule in the drm subsystem.
+
+	  The drm subsystem enables more warnings than the kernel default, so
+	  this config option is disabled by default.
+
+	  If in doubt, say N.
+
+endif
+
+# Separate option because drm_panel_orientation_quirks.c is shared with fbdev
+config DRM_PANEL_ORIENTATION_QUIRKS
+	tristate
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 104b42df2e95..f9ca4f8fa6c5 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -5,6 +5,34 @@
 
 CFLAGS-$(CONFIG_DRM_USE_DYNAMIC_DEBUG)	+= -DDYNAMIC_DEBUG_MODULE
 
+# Unconditionally enable W=1 warnings locally
+# --- begin copy-paste W=1 warnings from scripts/Makefile.extrawarn
+subdir-ccflags-y += -Wextra -Wunused -Wno-unused-parameter
+subdir-ccflags-y += $(call cc-option, -Wrestrict)
+subdir-ccflags-y += -Wmissing-format-attribute
+subdir-ccflags-y += -Wold-style-definition
+subdir-ccflags-y += -Wmissing-include-dirs
+subdir-ccflags-y += $(call cc-option, -Wunused-but-set-variable)
+subdir-ccflags-y += $(call cc-option, -Wunused-const-variable)
+subdir-ccflags-y += $(call cc-option, -Wpacked-not-aligned)
+subdir-ccflags-y += $(call cc-option, -Wformat-overflow)
+# FIXME: fix -Wformat-truncation warnings and uncomment
+#subdir-ccflags-y += $(call cc-option, -Wformat-truncation)
+subdir-ccflags-y += $(call cc-option, -Wstringop-truncation)
+# The following turn off the warnings enabled by -Wextra
+ifeq ($(findstring 2, $(KBUILD_EXTRA_WARN)),)
+subdir-ccflags-y += -Wno-missing-field-initializers
+subdir-ccflags-y += -Wno-type-limits
+subdir-ccflags-y += -Wno-shift-negative-value
+endif
+ifeq ($(findstring 3, $(KBUILD_EXTRA_WARN)),)
+subdir-ccflags-y += -Wno-sign-compare
+endif
+# --- end copy-paste
+
+# Enable -Werror in CI and development
+subdir-ccflags-$(CONFIG_DRM_WERROR) += -Werror
+
 drm-y := \
 	drm_aperture.o \
 	drm_atomic.o \
@@ -60,6 +88,7 @@ drm-$(CONFIG_DRM_PRIVACY_SCREEN) += \
 	drm_privacy_screen.o \
 	drm_privacy_screen_x86.o
 drm-$(CONFIG_DRM_ACCEL) += ../../accel/drm_accel.o
+drm-$(CONFIG_DRM_PANIC) += drm_panic.o
 obj-$(CONFIG_DRM)	+= drm.o
 
 obj-$(CONFIG_DRM_PANEL_ORIENTATION_QUIRKS) += drm_panel_orientation_quirks.o
@@ -179,6 +208,7 @@ obj-$(CONFIG_DRM_XEN) += xen/
 obj-$(CONFIG_DRM_VBOXVIDEO) += vboxvideo/
 obj-$(CONFIG_DRM_LIMA)  += lima/
 obj-$(CONFIG_DRM_PANFROST) += panfrost/
+obj-$(CONFIG_DRM_PANTHOR) += panthor/
 obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/
 obj-$(CONFIG_DRM_MCDE) += mcde/
 obj-$(CONFIG_DRM_TIDSS) += tidss/
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 4536c8ad0e11..078f588e99eb 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -23,7 +23,7 @@
 # Makefile for the drm device driver.  This driver provides support for the
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
-FULL_AMD_PATH=$(srctree)/$(src)/..
+FULL_AMD_PATH=$(src)/..
 DISPLAY_FOLDER_NAME=display
 FULL_AMD_DISPLAY_PATH = $(FULL_AMD_PATH)/$(DISPLAY_FOLDER_NAME)
 
@@ -70,7 +70,8 @@ amdgpu-y += amdgpu_device.o amdgpu_doorbell_mgr.o amdgpu_kms.o \
 	amdgpu_cs.o amdgpu_bios.o amdgpu_benchmark.o \
 	atombios_dp.o amdgpu_afmt.o amdgpu_trace_points.o \
 	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
-	amdgpu_dma_buf.o amdgpu_vm.o amdgpu_vm_pt.o amdgpu_ib.o amdgpu_pll.o \
+	amdgpu_dma_buf.o amdgpu_vm.o amdgpu_vm_pt.o amdgpu_vm_tlb_fence.o \
+	amdgpu_ib.o amdgpu_pll.o \
 	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
 	amdgpu_gtt_mgr.o amdgpu_preempt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o \
 	amdgpu_atomfirmware.o amdgpu_vf_error.o amdgpu_sched.o \
@@ -80,7 +81,7 @@ amdgpu-y += amdgpu_device.o amdgpu_doorbell_mgr.o amdgpu_kms.o \
 	amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \
 	amdgpu_fw_attestation.o amdgpu_securedisplay.o \
 	amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o \
-	amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_aca.o
+	amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_aca.o amdgpu_dev_coredump.o
 
 amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o
 
@@ -247,7 +248,8 @@ amdgpu-y += \
 	smuio_v11_0_6.o \
 	smuio_v13_0.o \
 	smuio_v13_0_3.o \
-	smuio_v13_0_6.o
+	smuio_v13_0_6.o \
+	smuio_v14_0_2.o
 
 # add reset block
 amdgpu-y += \
diff --git a/drivers/gpu/drm/amd/amdgpu/aldebaran.c b/drivers/gpu/drm/amd/amdgpu/aldebaran.c
index 576067d66bb9..d0a8da67dc2a 100644
--- a/drivers/gpu/drm/amd/amdgpu/aldebaran.c
+++ b/drivers/gpu/drm/amd/amdgpu/aldebaran.c
@@ -97,7 +97,7 @@ static int aldebaran_mode2_suspend_ip(struct amdgpu_device *adev)
 		adev->ip_blocks[i].status.hw = false;
 	}
 
-	return r;
+	return 0;
 }
 
 static int
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index b3b84647207e..f87d53e183c3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -139,6 +139,14 @@ enum amdgpu_ss {
 	AMDGPU_SS_DRV_UNLOAD
 };
 
+struct amdgpu_hwip_reg_entry {
+	u32		hwip;
+	u32		inst;
+	u32		seg;
+	u32		reg_offset;
+	const char	*reg_name;
+};
+
 struct amdgpu_watchdog_timer {
 	bool timeout_fatal_disable;
 	uint32_t period; /* maxCycles = (1 << period), the number of cycles before a timeout */
@@ -494,6 +502,7 @@ struct amdgpu_wb {
 	uint64_t		gpu_addr;
 	u32			num_wb;	/* Number of wb slots actually reserved for amdgpu. */
 	unsigned long		used[DIV_ROUND_UP(AMDGPU_MAX_WB, BITS_PER_LONG)];
+	spinlock_t		lock;
 };
 
 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb);
@@ -606,7 +615,7 @@ struct amdgpu_asic_funcs {
 	/* PCIe replay counter */
 	uint64_t (*get_pcie_replay_count)(struct amdgpu_device *adev);
 	/* device supports BACO */
-	bool (*supports_baco)(struct amdgpu_device *adev);
+	int (*supports_baco)(struct amdgpu_device *adev);
 	/* pre asic_init quirks */
 	void (*pre_asic_init)(struct amdgpu_device *adev);
 	/* enter/exit umd stable pstate */
@@ -1408,7 +1417,8 @@ bool amdgpu_device_supports_atpx(struct drm_device *dev);
 bool amdgpu_device_supports_px(struct drm_device *dev);
 bool amdgpu_device_supports_boco(struct drm_device *dev);
 bool amdgpu_device_supports_smart_shift(struct drm_device *dev);
-bool amdgpu_device_supports_baco(struct drm_device *dev);
+int amdgpu_device_supports_baco(struct drm_device *dev);
+void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev);
 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
 				      struct amdgpu_device *peer_adev);
 int amdgpu_device_baco_enter(struct drm_device *dev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c
index 493982f94649..c50202215f6b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c
@@ -28,7 +28,7 @@
 
 #define ACA_BANK_HWID(type, hwid, mcatype) [ACA_HWIP_TYPE_##type] = {hwid, mcatype}
 
-typedef int bank_handler_t(struct aca_handle *handle, struct aca_bank *bank, enum aca_error_type type, void *data);
+typedef int bank_handler_t(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type, void *data);
 
 struct aca_banks {
 	int nr_banks;
@@ -86,7 +86,7 @@ static void aca_banks_release(struct aca_banks *banks)
 	}
 }
 
-static int aca_smu_get_valid_aca_count(struct amdgpu_device *adev, enum aca_error_type type, u32 *count)
+static int aca_smu_get_valid_aca_count(struct amdgpu_device *adev, enum aca_smu_type type, u32 *count)
 {
 	struct amdgpu_aca *aca = &adev->aca;
 	const struct aca_smu_funcs *smu_funcs = aca->smu_funcs;
@@ -116,20 +116,22 @@ static struct aca_regs_dump {
 	{"CONTROL_MASK",	ACA_REG_IDX_CTL_MASK},
 };
 
-static void aca_smu_bank_dump(struct amdgpu_device *adev, int idx, int total, struct aca_bank *bank)
+static void aca_smu_bank_dump(struct amdgpu_device *adev, int idx, int total, struct aca_bank *bank,
+			      struct ras_query_context *qctx)
 {
+	u64 event_id = qctx ? qctx->event_id : 0ULL;
 	int i;
 
-	dev_info(adev->dev, HW_ERR "Accelerator Check Architecture events logged\n");
+	RAS_EVENT_LOG(adev, event_id, HW_ERR "Accelerator Check Architecture events logged\n");
 	/* plus 1 for output format, e.g: ACA[08/08]: xxxx */
 	for (i = 0; i < ARRAY_SIZE(aca_regs); i++)
-		dev_info(adev->dev, HW_ERR "ACA[%02d/%02d].%s=0x%016llx\n",
-			 idx + 1, total, aca_regs[i].name, bank->regs[aca_regs[i].reg_idx]);
+		RAS_EVENT_LOG(adev, event_id, HW_ERR "ACA[%02d/%02d].%s=0x%016llx\n",
+			      idx + 1, total, aca_regs[i].name, bank->regs[aca_regs[i].reg_idx]);
 }
 
-static int aca_smu_get_valid_aca_banks(struct amdgpu_device *adev, enum aca_error_type type,
+static int aca_smu_get_valid_aca_banks(struct amdgpu_device *adev, enum aca_smu_type type,
 				       int start, int count,
-				       struct aca_banks *banks)
+				       struct aca_banks *banks, struct ras_query_context *qctx)
 {
 	struct amdgpu_aca *aca = &adev->aca;
 	const struct aca_smu_funcs *smu_funcs = aca->smu_funcs;
@@ -143,13 +145,12 @@ static int aca_smu_get_valid_aca_banks(struct amdgpu_device *adev, enum aca_erro
 		return -EOPNOTSUPP;
 
 	switch (type) {
-	case ACA_ERROR_TYPE_UE:
+	case ACA_SMU_TYPE_UE:
 		max_count = smu_funcs->max_ue_bank_count;
 		break;
-	case ACA_ERROR_TYPE_CE:
+	case ACA_SMU_TYPE_CE:
 		max_count = smu_funcs->max_ce_bank_count;
 		break;
-	case ACA_ERROR_TYPE_DEFERRED:
 	default:
 		return -EINVAL;
 	}
@@ -164,7 +165,9 @@ static int aca_smu_get_valid_aca_banks(struct amdgpu_device *adev, enum aca_erro
 		if (ret)
 			return ret;
 
-		aca_smu_bank_dump(adev, i, count, &bank);
+		bank.type = type;
+
+		aca_smu_bank_dump(adev, i, count, &bank, qctx);
 
 		ret = aca_banks_add_bank(banks, &bank);
 		if (ret)
@@ -195,7 +198,7 @@ static bool aca_bank_hwip_is_matched(struct aca_bank *bank, enum aca_hwip_type t
 	return hwip->hwid == hwid && hwip->mcatype == mcatype;
 }
 
-static bool aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, enum aca_error_type type)
+static bool aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type)
 {
 	const struct aca_bank_ops *bank_ops = handle->bank_ops;
 
@@ -273,59 +276,49 @@ static struct aca_bank_error *get_bank_error(struct aca_error *aerr, struct aca_
 	return new_bank_error(aerr, info);
 }
 
-static int aca_log_errors(struct aca_handle *handle, enum aca_error_type type,
-			  struct aca_bank_report *report)
+int aca_error_cache_log_bank_error(struct aca_handle *handle, struct aca_bank_info *info,
+				   enum aca_error_type type, u64 count)
 {
 	struct aca_error_cache *error_cache = &handle->error_cache;
 	struct aca_bank_error *bank_error;
 	struct aca_error *aerr;
 
-	if (!handle || !report)
+	if (!handle || !info || type >= ACA_ERROR_TYPE_COUNT)
 		return -EINVAL;
 
-	if (!report->count[type])
+	if (!count)
 		return 0;
 
 	aerr = &error_cache->errors[type];
-	bank_error = get_bank_error(aerr, &report->info);
+	bank_error = get_bank_error(aerr, info);
 	if (!bank_error)
 		return -ENOMEM;
 
-	bank_error->count[type] += report->count[type];
+	bank_error->count += count;
 
 	return 0;
 }
 
-static int aca_generate_bank_report(struct aca_handle *handle, struct aca_bank *bank,
-				    enum aca_error_type type, struct aca_bank_report *report)
+static int aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type)
 {
 	const struct aca_bank_ops *bank_ops = handle->bank_ops;
 
-	if (!bank || !report)
+	if (!bank)
 		return -EINVAL;
 
-	if (!bank_ops->aca_bank_generate_report)
+	if (!bank_ops->aca_bank_parser)
 		return -EOPNOTSUPP;
 
-	memset(report, 0, sizeof(*report));
-	return bank_ops->aca_bank_generate_report(handle, bank, type,
-						  report, handle->data);
+	return bank_ops->aca_bank_parser(handle, bank, type,
+					 handle->data);
 }
 
 static int handler_aca_log_bank_error(struct aca_handle *handle, struct aca_bank *bank,
-				      enum aca_error_type type, void *data)
+				      enum aca_smu_type type, void *data)
 {
-	struct aca_bank_report report;
 	int ret;
 
-	ret = aca_generate_bank_report(handle, bank, type, &report);
-	if (ret)
-		return ret;
-
-	if (!report.count[type])
-		return 0;
-
-	ret = aca_log_errors(handle, type, &report);
+	ret = aca_bank_parser(handle, bank, type);
 	if (ret)
 		return ret;
 
@@ -333,7 +326,7 @@ static int handler_aca_log_bank_error(struct aca_handle *handle, struct aca_bank
 }
 
 static int aca_dispatch_bank(struct aca_handle_manager *mgr, struct aca_bank *bank,
-			     enum aca_error_type type, bank_handler_t handler, void *data)
+			     enum aca_smu_type type, bank_handler_t handler, void *data)
 {
 	struct aca_handle *handle;
 	int ret;
@@ -354,7 +347,7 @@ static int aca_dispatch_bank(struct aca_handle_manager *mgr, struct aca_bank *ba
 }
 
 static int aca_dispatch_banks(struct aca_handle_manager *mgr, struct aca_banks *banks,
-			      enum aca_error_type type, bank_handler_t handler, void *data)
+			      enum aca_smu_type type, bank_handler_t handler, void *data)
 {
 	struct aca_bank_node *node;
 	struct aca_bank *bank;
@@ -378,8 +371,28 @@ static int aca_dispatch_banks(struct aca_handle_manager *mgr, struct aca_banks *
 	return 0;
 }
 
-static int aca_banks_update(struct amdgpu_device *adev, enum aca_error_type type,
-			    bank_handler_t handler, void *data)
+static bool aca_bank_should_update(struct amdgpu_device *adev, enum aca_smu_type type)
+{
+	struct amdgpu_aca *aca = &adev->aca;
+	bool ret = true;
+
+	/*
+	 * Because the UE Valid MCA count will only be cleared after reset,
+	 * in order to avoid repeated counting of the error count,
+	 * the aca bank is only updated once during the gpu recovery stage.
+	 */
+	if (type == ACA_SMU_TYPE_UE) {
+		if (amdgpu_ras_intr_triggered())
+			ret = atomic_cmpxchg(&aca->ue_update_flag, 0, 1) == 0;
+		else
+			atomic_set(&aca->ue_update_flag, 0);
+	}
+
+	return ret;
+}
+
+static int aca_banks_update(struct amdgpu_device *adev, enum aca_smu_type type,
+			    bank_handler_t handler, struct ras_query_context *qctx, void *data)
 {
 	struct amdgpu_aca *aca = &adev->aca;
 	struct aca_banks banks;
@@ -389,9 +402,8 @@ static int aca_banks_update(struct amdgpu_device *adev, enum aca_error_type type
 	if (list_empty(&aca->mgr.list))
 		return 0;
 
-	/* NOTE: pmfw is only support UE and CE */
-	if (type == ACA_ERROR_TYPE_DEFERRED)
-		type = ACA_ERROR_TYPE_CE;
+	if (!aca_bank_should_update(adev, type))
+		return 0;
 
 	ret = aca_smu_get_valid_aca_count(adev, type, &count);
 	if (ret)
@@ -402,7 +414,7 @@ static int aca_banks_update(struct amdgpu_device *adev, enum aca_error_type type
 
 	aca_banks_init(&banks);
 
-	ret = aca_smu_get_valid_aca_banks(adev, type, 0, count, &banks);
+	ret = aca_smu_get_valid_aca_banks(adev, type, 0, count, &banks, qctx);
 	if (ret)
 		goto err_release_banks;
 
@@ -431,7 +443,7 @@ static int aca_log_aca_error_data(struct aca_bank_error *bank_error, enum aca_er
 	if (type >= ACA_ERROR_TYPE_COUNT)
 		return -EINVAL;
 
-	count = bank_error->count[type];
+	count = bank_error->count;
 	if (!count)
 		return 0;
 
@@ -447,6 +459,8 @@ static int aca_log_aca_error_data(struct aca_bank_error *bank_error, enum aca_er
 		amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, NULL, count);
 		break;
 	case ACA_ERROR_TYPE_DEFERRED:
+		amdgpu_ras_error_statistic_de_count(err_data, &mcm_info, NULL, count);
+		break;
 	default:
 		break;
 	}
@@ -477,12 +491,25 @@ out_unlock:
 }
 
 static int __aca_get_error_data(struct amdgpu_device *adev, struct aca_handle *handle, enum aca_error_type type,
-				struct ras_err_data *err_data)
+				struct ras_err_data *err_data, struct ras_query_context *qctx)
 {
+	enum aca_smu_type smu_type;
 	int ret;
 
+	switch (type) {
+	case ACA_ERROR_TYPE_UE:
+		smu_type = ACA_SMU_TYPE_UE;
+		break;
+	case ACA_ERROR_TYPE_CE:
+	case ACA_ERROR_TYPE_DEFERRED:
+		smu_type = ACA_SMU_TYPE_CE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
 	/* udpate aca bank to aca source error_cache first */
-	ret = aca_banks_update(adev, type, handler_aca_log_bank_error, NULL);
+	ret = aca_banks_update(adev, smu_type, handler_aca_log_bank_error, qctx, NULL);
 	if (ret)
 		return ret;
 
@@ -498,10 +525,9 @@ static bool aca_handle_is_valid(struct aca_handle *handle)
 }
 
 int amdgpu_aca_get_error_data(struct amdgpu_device *adev, struct aca_handle *handle,
-			      enum aca_error_type type, void *data)
+			      enum aca_error_type type, struct ras_err_data *err_data,
+			      struct ras_query_context *qctx)
 {
-	struct ras_err_data *err_data = (struct ras_err_data *)data;
-
 	if (!handle || !err_data)
 		return -EINVAL;
 
@@ -511,7 +537,7 @@ int amdgpu_aca_get_error_data(struct amdgpu_device *adev, struct aca_handle *han
 	if (!(BIT(type) & handle->mask))
 		return  0;
 
-	return __aca_get_error_data(adev, handle, type, err_data);
+	return __aca_get_error_data(adev, handle, type, err_data, qctx);
 }
 
 static void aca_error_init(struct aca_error *aerr, enum aca_error_type type)
@@ -668,6 +694,8 @@ int amdgpu_aca_init(struct amdgpu_device *adev)
 	struct amdgpu_aca *aca = &adev->aca;
 	int ret;
 
+	atomic_set(&aca->ue_update_flag, 0);
+
 	ret = aca_manager_init(&aca->mgr);
 	if (ret)
 		return ret;
@@ -680,6 +708,8 @@ void amdgpu_aca_fini(struct amdgpu_device *adev)
 	struct amdgpu_aca *aca = &adev->aca;
 
 	aca_manager_fini(&aca->mgr);
+
+	atomic_set(&aca->ue_update_flag, 0);
 }
 
 int amdgpu_aca_reset(struct amdgpu_device *adev)
@@ -723,23 +753,13 @@ int aca_bank_info_decode(struct aca_bank *bank, struct aca_bank_info *info)
 
 static int aca_bank_get_error_code(struct amdgpu_device *adev, struct aca_bank *bank)
 {
-	int error_code;
-
-	switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
-	case IP_VERSION(13, 0, 6):
-		if (!(adev->flags & AMD_IS_APU) && adev->pm.fw_version >= 0x00555600) {
-			error_code = ACA_REG__SYND__ERRORINFORMATION(bank->regs[ACA_REG_IDX_SYND]);
-			return error_code & 0xff;
-		}
-		break;
-	default:
-		break;
-	}
+	struct amdgpu_aca *aca = &adev->aca;
+	const struct aca_smu_funcs *smu_funcs = aca->smu_funcs;
 
-	/* NOTE: the true error code is encoded in status.errorcode[0:7] */
-	error_code = ACA_REG__STATUS__ERRORCODE(bank->regs[ACA_REG_IDX_STATUS]);
+	if (!smu_funcs || !smu_funcs->parse_error_code)
+		return -EOPNOTSUPP;
 
-	return error_code & 0xff;
+	return smu_funcs->parse_error_code(adev, bank);
 }
 
 int aca_bank_check_error_codes(struct amdgpu_device *adev, struct aca_bank *bank, int *err_codes, int size)
@@ -750,6 +770,9 @@ int aca_bank_check_error_codes(struct amdgpu_device *adev, struct aca_bank *bank
 		return -EINVAL;
 
 	error_code = aca_bank_get_error_code(adev, bank);
+	if (error_code < 0)
+		return error_code;
+
 	for (i = 0; i < size; i++) {
 		if (err_codes[i] == error_code)
 			return 0;
@@ -784,7 +807,7 @@ static int amdgpu_aca_smu_debug_mode_set(void *data, u64 val)
 	return 0;
 }
 
-static void aca_dump_entry(struct seq_file *m, struct aca_bank *bank, enum aca_error_type type, int idx)
+static void aca_dump_entry(struct seq_file *m, struct aca_bank *bank, enum aca_smu_type type, int idx)
 {
 	struct aca_bank_info info;
 	int i, ret;
@@ -793,7 +816,7 @@ static void aca_dump_entry(struct seq_file *m, struct aca_bank *bank, enum aca_e
 	if (ret)
 		return;
 
-	seq_printf(m, "aca entry[%d].type: %s\n", idx, type ==  ACA_ERROR_TYPE_UE ? "UE" : "CE");
+	seq_printf(m, "aca entry[%d].type: %s\n", idx, type ==  ACA_SMU_TYPE_UE ? "UE" : "CE");
 	seq_printf(m, "aca entry[%d].info: socketid:%d aid:%d hwid:0x%03x mcatype:0x%04x\n",
 		   idx, info.socket_id, info.die_id, info.hwid, info.mcatype);
 
@@ -807,7 +830,7 @@ struct aca_dump_context {
 };
 
 static int handler_aca_bank_dump(struct aca_handle *handle, struct aca_bank *bank,
-				 enum aca_error_type type, void *data)
+				 enum aca_smu_type type, void *data)
 {
 	struct aca_dump_context *ctx = (struct aca_dump_context *)data;
 
@@ -816,7 +839,7 @@ static int handler_aca_bank_dump(struct aca_handle *handle, struct aca_bank *ban
 	return handler_aca_log_bank_error(handle, bank, type, NULL);
 }
 
-static int aca_dump_show(struct seq_file *m, enum aca_error_type type)
+static int aca_dump_show(struct seq_file *m, enum aca_smu_type type)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
 	struct aca_dump_context context = {
@@ -824,12 +847,12 @@ static int aca_dump_show(struct seq_file *m, enum aca_error_type type)
 		.idx = 0,
 	};
 
-	return aca_banks_update(adev, type, handler_aca_bank_dump, (void *)&context);
+	return aca_banks_update(adev, type, handler_aca_bank_dump, NULL, (void *)&context);
 }
 
 static int aca_dump_ce_show(struct seq_file *m, void *unused)
 {
-	return aca_dump_show(m, ACA_ERROR_TYPE_CE);
+	return aca_dump_show(m, ACA_SMU_TYPE_CE);
 }
 
 static int aca_dump_ce_open(struct inode *inode, struct file *file)
@@ -847,7 +870,7 @@ static const struct file_operations aca_ce_dump_debug_fops = {
 
 static int aca_dump_ue_show(struct seq_file *m, void *unused)
 {
-	return aca_dump_show(m, ACA_ERROR_TYPE_UE);
+	return aca_dump_show(m, ACA_SMU_TYPE_UE);
 }
 
 static int aca_dump_ue_open(struct inode *inode, struct file *file)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.h
index 2da50e095883..5ef6b745f222 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.h
@@ -26,6 +26,9 @@
 
 #include <linux/list.h>
 
+struct ras_err_data;
+struct ras_query_context;
+
 #define ACA_MAX_REGS_COUNT	(16)
 
 #define ACA_REG_FIELD(x, h, l)			(((x) & GENMASK_ULL(h, l)) >> l)
@@ -99,7 +102,14 @@ enum aca_error_type {
 	ACA_ERROR_TYPE_COUNT
 };
 
+enum aca_smu_type {
+	ACA_SMU_TYPE_UE = 0,
+	ACA_SMU_TYPE_CE,
+	ACA_SMU_TYPE_COUNT,
+};
+
 struct aca_bank {
+	enum aca_smu_type type;
 	u64 regs[ACA_MAX_REGS_COUNT];
 };
 
@@ -115,15 +125,10 @@ struct aca_bank_info {
 	int mcatype;
 };
 
-struct aca_bank_report {
-	struct aca_bank_info info;
-	u64 count[ACA_ERROR_TYPE_COUNT];
-};
-
 struct aca_bank_error {
 	struct list_head node;
 	struct aca_bank_info info;
-	u64 count[ACA_ERROR_TYPE_COUNT];
+	u64 count;
 };
 
 struct aca_error {
@@ -157,9 +162,8 @@ struct aca_handle {
 };
 
 struct aca_bank_ops {
-	int (*aca_bank_generate_report)(struct aca_handle *handle, struct aca_bank *bank, enum aca_error_type type,
-					struct aca_bank_report *report, void *data);
-	bool (*aca_bank_is_valid)(struct aca_handle *handle, struct aca_bank *bank, enum aca_error_type type,
+	int (*aca_bank_parser)(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type, void *data);
+	bool (*aca_bank_is_valid)(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type,
 				  void *data);
 };
 
@@ -167,13 +171,15 @@ struct aca_smu_funcs {
 	int max_ue_bank_count;
 	int max_ce_bank_count;
 	int (*set_debug_mode)(struct amdgpu_device *adev, bool enable);
-	int (*get_valid_aca_count)(struct amdgpu_device *adev, enum aca_error_type type, u32 *count);
-	int (*get_valid_aca_bank)(struct amdgpu_device *adev, enum aca_error_type type, int idx, struct aca_bank *bank);
+	int (*get_valid_aca_count)(struct amdgpu_device *adev, enum aca_smu_type type, u32 *count);
+	int (*get_valid_aca_bank)(struct amdgpu_device *adev, enum aca_smu_type type, int idx, struct aca_bank *bank);
+	int (*parse_error_code)(struct amdgpu_device *adev, struct aca_bank *bank);
 };
 
 struct amdgpu_aca {
 	struct aca_handle_manager mgr;
 	const struct aca_smu_funcs *smu_funcs;
+	atomic_t ue_update_flag;
 	bool is_enabled;
 };
 
@@ -196,7 +202,10 @@ int amdgpu_aca_add_handle(struct amdgpu_device *adev, struct aca_handle *handle,
 			  const char *name, const struct aca_info *aca_info, void *data);
 void amdgpu_aca_remove_handle(struct aca_handle *handle);
 int amdgpu_aca_get_error_data(struct amdgpu_device *adev, struct aca_handle *handle,
-				     enum aca_error_type type, void *data);
+			      enum aca_error_type type, struct ras_err_data *err_data,
+			      struct ras_query_context *qctx);
 int amdgpu_aca_smu_set_debug_mode(struct amdgpu_device *adev, bool en);
 void amdgpu_aca_smu_debugfs_init(struct amdgpu_device *adev, struct dentry *root);
+int aca_error_cache_log_bank_error(struct aca_handle *handle, struct aca_bank_info *info,
+				   enum aca_error_type type, u64 count);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
index 6d72355ac492..bf6c4a0d0525 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
@@ -637,6 +637,8 @@ static const struct amd_ip_funcs acp_ip_funcs = {
 	.soft_reset = acp_soft_reset,
 	.set_clockgating_state = acp_set_clockgating_state,
 	.set_powergating_state = acp_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version acp_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 35dd6effa9a3..e3738d417245 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -455,6 +455,9 @@ void amdgpu_amdkfd_get_local_mem_info(struct amdgpu_device *adev,
 		else
 			mem_info->local_mem_size_private =
 					KFD_XCP_MEMORY_SIZE(adev, xcp->id);
+	} else if (adev->flags & AMD_IS_APU) {
+		mem_info->local_mem_size_public = (ttm_tt_pages_limit() << PAGE_SHIFT);
+		mem_info->local_mem_size_private = 0;
 	} else {
 		mem_info->local_mem_size_public = adev->gmc.visible_vram_size;
 		mem_info->local_mem_size_private = adev->gmc.real_vram_size -
@@ -747,10 +750,17 @@ bool amdgpu_amdkfd_is_fed(struct amdgpu_device *adev)
 	return amdgpu_ras_get_fed_status(adev);
 }
 
+void amdgpu_amdkfd_ras_pasid_poison_consumption_handler(struct amdgpu_device *adev,
+				enum amdgpu_ras_block block, uint16_t pasid,
+				pasid_notify pasid_fn, void *data, uint32_t reset)
+{
+	amdgpu_umc_pasid_poison_handler(adev, block, pasid, pasid_fn, data, reset);
+}
+
 void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev,
-	enum amdgpu_ras_block block, bool reset)
+	enum amdgpu_ras_block block, uint32_t reset)
 {
-	amdgpu_umc_poison_handler(adev, block, reset);
+	amdgpu_umc_pasid_poison_handler(adev, block, 0, NULL, NULL, reset);
 }
 
 int amdgpu_amdkfd_send_close_event_drain_irq(struct amdgpu_device *adev,
@@ -769,12 +779,20 @@ int amdgpu_amdkfd_send_close_event_drain_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
-bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev)
+bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev,
+			int hub_inst, int hub_type)
 {
-	if (adev->gfx.ras && adev->gfx.ras->query_utcl2_poison_status)
-		return adev->gfx.ras->query_utcl2_poison_status(adev);
-	else
-		return false;
+	if (!hub_type) {
+		if (adev->gfxhub.funcs->query_utcl2_poison_status)
+			return adev->gfxhub.funcs->query_utcl2_poison_status(adev, hub_inst);
+		else
+			return false;
+	} else {
+		if (adev->mmhub.funcs->query_utcl2_poison_status)
+			return adev->mmhub.funcs->query_utcl2_poison_status(adev, hub_inst);
+		else
+			return false;
+	}
 }
 
 int amdgpu_amdkfd_check_and_lock_kfd(struct amdgpu_device *adev)
@@ -809,6 +827,8 @@ u64 amdgpu_amdkfd_xcp_memory_size(struct amdgpu_device *adev, int xcp_id)
 		}
 		do_div(tmp, adev->xcp_mgr->num_xcp_per_mem_partition);
 		return ALIGN_DOWN(tmp, PAGE_SIZE);
+	} else if (adev->flags & AMD_IS_APU) {
+		return (ttm_tt_pages_limit() << PAGE_SHIFT);
 	} else {
 		return adev->gmc.real_vram_size;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 0ef223c2affb..1de021ebdd46 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -336,12 +336,18 @@ void amdgpu_amdkfd_debug_mem_fence(struct amdgpu_device *adev);
 int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,
 				struct tile_config *config);
 void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev,
-			enum amdgpu_ras_block block, bool reset);
+			enum amdgpu_ras_block block, uint32_t reset);
+
+void amdgpu_amdkfd_ras_pasid_poison_consumption_handler(struct amdgpu_device *adev,
+			enum amdgpu_ras_block block, uint16_t pasid,
+			pasid_notify pasid_fn, void *data, uint32_t reset);
+
 bool amdgpu_amdkfd_is_fed(struct amdgpu_device *adev);
 bool amdgpu_amdkfd_bo_mapped_to_dev(struct amdgpu_device *adev, struct kgd_mem *mem);
 void amdgpu_amdkfd_block_mmu_notifications(void *p);
 int amdgpu_amdkfd_criu_resume(void *p);
-bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev);
+bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev,
+			int hub_inst, int hub_type);
 int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 		uint64_t size, u32 alloc_flag, int8_t xcp_id);
 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
index 69810b3f1c63..3ab6c3aa0ad1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
@@ -881,6 +881,7 @@ uint32_t kgd_gfx_v10_set_wave_launch_mode(struct amdgpu_device *adev,
 }
 
 #define TCP_WATCH_STRIDE (mmTCP_WATCH1_ADDR_H - mmTCP_WATCH0_ADDR_H)
+#define SQ_WATCH_STRIDE (mmSQ_WATCH1_ADDR_H - mmSQ_WATCH0_ADDR_H)
 uint32_t kgd_gfx_v10_set_address_watch(struct amdgpu_device *adev,
 					uint64_t watch_address,
 					uint32_t watch_address_mask,
@@ -889,55 +890,93 @@ uint32_t kgd_gfx_v10_set_address_watch(struct amdgpu_device *adev,
 					uint32_t debug_vmid,
 					uint32_t inst)
 {
+	/* SQ_WATCH?_ADDR_* and TCP_WATCH?_ADDR_* are programmed with the
+	 * same values.
+	 */
 	uint32_t watch_address_high;
 	uint32_t watch_address_low;
-	uint32_t watch_address_cntl;
-
-	watch_address_cntl = 0;
+	uint32_t tcp_watch_address_cntl;
+	uint32_t sq_watch_address_cntl;
 
 	watch_address_low = lower_32_bits(watch_address);
 	watch_address_high = upper_32_bits(watch_address) & 0xffff;
 
-	watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
+	tcp_watch_address_cntl = 0;
+	tcp_watch_address_cntl = REG_SET_FIELD(tcp_watch_address_cntl,
 			TCP_WATCH0_CNTL,
 			VMID,
 			debug_vmid);
-	watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
+	tcp_watch_address_cntl = REG_SET_FIELD(tcp_watch_address_cntl,
 			TCP_WATCH0_CNTL,
 			MODE,
 			watch_mode);
-	watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
+	tcp_watch_address_cntl = REG_SET_FIELD(tcp_watch_address_cntl,
 			TCP_WATCH0_CNTL,
 			MASK,
 			watch_address_mask >> 7);
 
+	sq_watch_address_cntl = 0;
+	sq_watch_address_cntl = REG_SET_FIELD(sq_watch_address_cntl,
+			SQ_WATCH0_CNTL,
+			VMID,
+			debug_vmid);
+	sq_watch_address_cntl = REG_SET_FIELD(sq_watch_address_cntl,
+			SQ_WATCH0_CNTL,
+			MODE,
+			watch_mode);
+	sq_watch_address_cntl = REG_SET_FIELD(sq_watch_address_cntl,
+			SQ_WATCH0_CNTL,
+			MASK,
+			watch_address_mask >> 6);
+
 	/* Turning off this watch point until we set all the registers */
-	watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
+	tcp_watch_address_cntl = REG_SET_FIELD(tcp_watch_address_cntl,
 			TCP_WATCH0_CNTL,
 			VALID,
 			0);
-
 	WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_CNTL) +
 			(watch_id * TCP_WATCH_STRIDE)),
-			watch_address_cntl);
+			tcp_watch_address_cntl);
+
+	sq_watch_address_cntl = REG_SET_FIELD(sq_watch_address_cntl,
+			SQ_WATCH0_CNTL,
+			VALID,
+			0);
+	WREG32((SOC15_REG_OFFSET(GC, 0, mmSQ_WATCH0_CNTL) +
+			(watch_id * SQ_WATCH_STRIDE)),
+			sq_watch_address_cntl);
 
+	/* Program {TCP,SQ}_WATCH?_ADDR* */
 	WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_ADDR_H) +
 			(watch_id * TCP_WATCH_STRIDE)),
 			watch_address_high);
-
 	WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_ADDR_L) +
 			(watch_id * TCP_WATCH_STRIDE)),
 			watch_address_low);
 
+	WREG32((SOC15_REG_OFFSET(GC, 0, mmSQ_WATCH0_ADDR_H) +
+			(watch_id * SQ_WATCH_STRIDE)),
+			watch_address_high);
+	WREG32((SOC15_REG_OFFSET(GC, 0, mmSQ_WATCH0_ADDR_L) +
+			(watch_id * SQ_WATCH_STRIDE)),
+			watch_address_low);
+
 	/* Enable the watch point */
-	watch_address_cntl = REG_SET_FIELD(watch_address_cntl,
+	tcp_watch_address_cntl = REG_SET_FIELD(tcp_watch_address_cntl,
 			TCP_WATCH0_CNTL,
 			VALID,
 			1);
-
 	WREG32((SOC15_REG_OFFSET(GC, 0, mmTCP_WATCH0_CNTL) +
 			(watch_id * TCP_WATCH_STRIDE)),
-			watch_address_cntl);
+			tcp_watch_address_cntl);
+
+	sq_watch_address_cntl = REG_SET_FIELD(sq_watch_address_cntl,
+			SQ_WATCH0_CNTL,
+			VALID,
+			1);
+	WREG32((SOC15_REG_OFFSET(GC, 0, mmSQ_WATCH0_CNTL) +
+			(watch_id * SQ_WATCH_STRIDE)),
+			sq_watch_address_cntl);
 
 	return 0;
 }
@@ -953,8 +992,14 @@ uint32_t kgd_gfx_v10_clear_address_watch(struct amdgpu_device *adev,
 			(watch_id * TCP_WATCH_STRIDE)),
 			watch_address_cntl);
 
+	WREG32((SOC15_REG_OFFSET(GC, 0, mmSQ_WATCH0_CNTL) +
+			(watch_id * SQ_WATCH_STRIDE)),
+			watch_address_cntl);
+
 	return 0;
 }
+#undef TCP_WATCH_STRIDE
+#undef SQ_WATCH_STRIDE
 
 
 /* kgd_gfx_v10_get_iq_wait_times: Returns the mmCP_IQ_WAIT_TIME1/2 values
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index df58a6a1a67e..48ad0c04aa72 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -196,7 +196,7 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 			return -EINVAL;
 
 		vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id);
-		if (adev->gmc.is_app_apu) {
+		if (adev->flags & AMD_IS_APU) {
 			system_mem_needed = size;
 			ttm_mem_needed = size;
 		}
@@ -220,7 +220,7 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 	    (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
 	     kfd_mem_limit.max_ttm_mem_limit) ||
 	    (adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] + vram_needed >
-	     vram_size - reserved_for_pt)) {
+	     vram_size - reserved_for_pt - atomic64_read(&adev->vram_pin_size))) {
 		ret = -ENOMEM;
 		goto release;
 	}
@@ -232,7 +232,8 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 		  "adev reference can't be null when vram is used");
 	if (adev && xcp_id >= 0) {
 		adev->kfd.vram_used[xcp_id] += vram_needed;
-		adev->kfd.vram_used_aligned[xcp_id] += adev->gmc.is_app_apu ?
+		adev->kfd.vram_used_aligned[xcp_id] +=
+				(adev->flags & AMD_IS_APU) ?
 				vram_needed :
 				ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN);
 	}
@@ -260,7 +261,7 @@ void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
 
 		if (adev) {
 			adev->kfd.vram_used[xcp_id] -= size;
-			if (adev->gmc.is_app_apu) {
+			if (adev->flags & AMD_IS_APU) {
 				adev->kfd.vram_used_aligned[xcp_id] -= size;
 				kfd_mem_limit.system_mem_used -= size;
 				kfd_mem_limit.ttm_mem_used -= size;
@@ -889,7 +890,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
 	 * if peer device has large BAR. In contrast, access over xGMI is
 	 * allowed for both small and large BAR configurations of peer device
 	 */
-	if ((adev != bo_adev && !adev->gmc.is_app_apu) &&
+	if ((adev != bo_adev && !(adev->flags & AMD_IS_APU)) &&
 	    ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
 	     (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
 	     (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
@@ -1188,7 +1189,8 @@ static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
 	int ret;
 
 	ctx->sync = &mem->sync;
-	drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
+	drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
+		      DRM_EXEC_IGNORE_DUPLICATES, 0);
 	drm_exec_until_all_locked(&ctx->exec) {
 		ctx->n_vms = 0;
 		list_for_each_entry(entry, &mem->attachments, list) {
@@ -1656,7 +1658,7 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
 		- atomic64_read(&adev->vram_pin_size)
 		- reserved_for_pt;
 
-	if (adev->gmc.is_app_apu) {
+	if (adev->flags & AMD_IS_APU) {
 		system_mem_available = no_system_mem_limit ?
 					kfd_mem_limit.max_system_mem_limit :
 					kfd_mem_limit.max_system_mem_limit -
@@ -1704,7 +1706,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
 		domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;
 
-		if (adev->gmc.is_app_apu) {
+		if (adev->flags & AMD_IS_APU) {
 			domain = AMDGPU_GEM_DOMAIN_GTT;
 			alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
 			alloc_flags = 0;
@@ -1854,6 +1856,7 @@ err_node_allow:
 err_bo_create:
 	amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags, xcp_id);
 err_reserve_limit:
+	amdgpu_sync_free(&(*mem)->sync);
 	mutex_destroy(&(*mem)->lock);
 	if (gobj)
 		drm_gem_object_put(gobj);
@@ -1950,7 +1953,7 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
 	if (size) {
 		if (!is_imported &&
 		   (mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM ||
-		   (adev->gmc.is_app_apu &&
+		   ((adev->flags & AMD_IS_APU) &&
 		    mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT)))
 			*size = bo_size;
 		else
@@ -2372,8 +2375,9 @@ static int import_obj_create(struct amdgpu_device *adev,
 	(*mem)->dmabuf = dma_buf;
 	(*mem)->bo = bo;
 	(*mem)->va = va;
-	(*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) && !adev->gmc.is_app_apu ?
-		AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;
+	(*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) &&
+			 !(adev->flags & AMD_IS_APU) ?
+			 AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;
 
 	(*mem)->mapped_to_gpu_memory = 0;
 	(*mem)->process_info = avm->process_info;
@@ -2900,13 +2904,12 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu *
 
 	amdgpu_sync_create(&sync_obj);
 
-	/* Validate BOs and map them to GPUVM (update VM page tables). */
+	/* Validate BOs managed by KFD */
 	list_for_each_entry(mem, &process_info->kfd_bo_list,
 			    validate_list) {
 
 		struct amdgpu_bo *bo = mem->bo;
 		uint32_t domain = mem->domain;
-		struct kfd_mem_attachment *attachment;
 		struct dma_resv_iter cursor;
 		struct dma_fence *fence;
 
@@ -2931,6 +2934,25 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu *
 				goto validate_map_fail;
 			}
 		}
+	}
+
+	if (failed_size)
+		pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size);
+
+	/* Validate PDs, PTs and evicted DMABuf imports last. Otherwise BO
+	 * validations above would invalidate DMABuf imports again.
+	 */
+	ret = process_validate_vms(process_info, &exec.ticket);
+	if (ret) {
+		pr_debug("Validating VMs failed, ret: %d\n", ret);
+		goto validate_map_fail;
+	}
+
+	/* Update mappings managed by KFD. */
+	list_for_each_entry(mem, &process_info->kfd_bo_list,
+			    validate_list) {
+		struct kfd_mem_attachment *attachment;
+
 		list_for_each_entry(attachment, &mem->attachments, list) {
 			if (!attachment->is_mapped)
 				continue;
@@ -2947,18 +2969,6 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu *
 		}
 	}
 
-	if (failed_size)
-		pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size);
-
-	/* Validate PDs, PTs and evicted DMABuf imports last. Otherwise BO
-	 * validations above would invalidate DMABuf imports again.
-	 */
-	ret = process_validate_vms(process_info, &exec.ticket);
-	if (ret) {
-		pr_debug("Validating VMs failed, ret: %d\n", ret);
-		goto validate_map_fail;
-	}
-
 	/* Update mappings not managed by KFD */
 	list_for_each_entry(peer_vm, &process_info->vm_list_head,
 			vm_list_node) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index 6857c586ded7..108003bdf1e9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -34,6 +34,7 @@ union firmware_info {
 	struct atom_firmware_info_v3_2 v32;
 	struct atom_firmware_info_v3_3 v33;
 	struct atom_firmware_info_v3_4 v34;
+	struct atom_firmware_info_v3_5 v35;
 };
 
 /*
@@ -211,6 +212,7 @@ union igp_info {
 	struct atom_integrated_system_info_v1_11 v11;
 	struct atom_integrated_system_info_v1_12 v12;
 	struct atom_integrated_system_info_v2_1 v21;
+	struct atom_integrated_system_info_v2_3 v23;
 };
 
 union umc_info {
@@ -359,6 +361,20 @@ amdgpu_atomfirmware_get_vram_info(struct amdgpu_device *adev,
 					if (vram_type)
 						*vram_type = convert_atom_mem_type_to_vram_type(adev, mem_type);
 					break;
+				case 3:
+					mem_channel_number = igp_info->v23.umachannelnumber;
+					if (!mem_channel_number)
+						mem_channel_number = 1;
+					mem_type = igp_info->v23.memorytype;
+					if (mem_type == LpDdr5MemType)
+						mem_channel_width = 32;
+					else
+						mem_channel_width = 64;
+					if (vram_width)
+						*vram_width = mem_channel_number * mem_channel_width;
+					if (vram_type)
+						*vram_type = convert_atom_mem_type_to_vram_type(adev, mem_type);
+					break;
 				default:
 					return -EINVAL;
 				}
@@ -872,6 +888,10 @@ int amdgpu_atomfirmware_get_fw_reserved_fb_size(struct amdgpu_device *adev)
 		fw_reserved_fb_size =
 			(firmware_info->v34.fw_reserved_size_in_kb << 10);
 		break;
+	case 5:
+		fw_reserved_fb_size =
+			(firmware_info->v35.fw_reserved_size_in_kb << 10);
+		break;
 	default:
 		fw_reserved_fb_size = 0;
 		break;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
index edc6377ec5ff..199693369c7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
@@ -39,7 +39,7 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size,
 	for (i = 0; i < n; i++) {
 		struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 		r = amdgpu_copy_buffer(ring, saddr, daddr, size, NULL, &fence,
-				       false, false, false);
+				       false, false, 0);
 		if (r)
 			goto exit_do_move;
 		r = dma_fence_wait(fence, false);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 0a4b09709cfb..ec888fc6ead8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -819,7 +819,7 @@ retry:
 
 	p->bytes_moved += ctx.bytes_moved;
 	if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
-	    amdgpu_bo_in_cpu_visible_vram(bo))
+	    amdgpu_res_cpu_visible(adev, bo->tbo.resource))
 		p->bytes_moved_vis += ctx.bytes_moved;
 
 	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
index f5d0fa207a88..b62ae3c91a9d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -2065,12 +2065,13 @@ static ssize_t amdgpu_reset_dump_register_list_write(struct file *f,
 	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
 	char reg_offset[11];
 	uint32_t *new = NULL, *tmp = NULL;
-	int ret, i = 0, len = 0;
+	unsigned int len = 0;
+	int ret, i = 0;
 
 	do {
 		memset(reg_offset, 0, 11);
 		if (copy_from_user(reg_offset, buf + len,
-					min(10, ((int)size-len)))) {
+					min(10, (size-len)))) {
 			ret = -EFAULT;
 			goto error_free;
 		}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c
new file mode 100644
index 000000000000..c1cb62683695
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c
@@ -0,0 +1,360 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2024 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <generated/utsrelease.h>
+#include <linux/devcoredump.h>
+#include "amdgpu_dev_coredump.h"
+#include "atom.h"
+
+#ifndef CONFIG_DEV_COREDUMP
+void amdgpu_coredump(struct amdgpu_device *adev, bool vram_lost,
+		     struct amdgpu_reset_context *reset_context)
+{
+}
+#else
+
+const char *hw_ip_names[MAX_HWIP] = {
+	[GC_HWIP]		= "GC",
+	[HDP_HWIP]		= "HDP",
+	[SDMA0_HWIP]		= "SDMA0",
+	[SDMA1_HWIP]		= "SDMA1",
+	[SDMA2_HWIP]		= "SDMA2",
+	[SDMA3_HWIP]		= "SDMA3",
+	[SDMA4_HWIP]		= "SDMA4",
+	[SDMA5_HWIP]		= "SDMA5",
+	[SDMA6_HWIP]		= "SDMA6",
+	[SDMA7_HWIP]		= "SDMA7",
+	[LSDMA_HWIP]		= "LSDMA",
+	[MMHUB_HWIP]		= "MMHUB",
+	[ATHUB_HWIP]		= "ATHUB",
+	[NBIO_HWIP]		= "NBIO",
+	[MP0_HWIP]		= "MP0",
+	[MP1_HWIP]		= "MP1",
+	[UVD_HWIP]		= "UVD/JPEG/VCN",
+	[VCN1_HWIP]		= "VCN1",
+	[VCE_HWIP]		= "VCE",
+	[VPE_HWIP]		= "VPE",
+	[DF_HWIP]		= "DF",
+	[DCE_HWIP]		= "DCE",
+	[OSSSYS_HWIP]		= "OSSSYS",
+	[SMUIO_HWIP]		= "SMUIO",
+	[PWR_HWIP]		= "PWR",
+	[NBIF_HWIP]		= "NBIF",
+	[THM_HWIP]		= "THM",
+	[CLK_HWIP]		= "CLK",
+	[UMC_HWIP]		= "UMC",
+	[RSMU_HWIP]		= "RSMU",
+	[XGMI_HWIP]		= "XGMI",
+	[DCI_HWIP]		= "DCI",
+	[PCIE_HWIP]		= "PCIE",
+};
+
+static void amdgpu_devcoredump_fw_info(struct amdgpu_device *adev,
+				       struct drm_printer *p)
+{
+	uint32_t version;
+	uint32_t feature;
+	uint8_t smu_program, smu_major, smu_minor, smu_debug;
+	struct atom_context *ctx = adev->mode_info.atom_context;
+
+	drm_printf(p, "VCE feature version: %u, fw version: 0x%08x\n",
+		   adev->vce.fb_version, adev->vce.fw_version);
+	drm_printf(p, "UVD feature version: %u, fw version: 0x%08x\n", 0,
+		   adev->uvd.fw_version);
+	drm_printf(p, "GMC feature version: %u, fw version: 0x%08x\n", 0,
+		   adev->gmc.fw_version);
+	drm_printf(p, "ME feature version: %u, fw version: 0x%08x\n",
+		   adev->gfx.me_feature_version, adev->gfx.me_fw_version);
+	drm_printf(p, "PFP feature version: %u, fw version: 0x%08x\n",
+		   adev->gfx.pfp_feature_version, adev->gfx.pfp_fw_version);
+	drm_printf(p, "CE feature version: %u, fw version: 0x%08x\n",
+		   adev->gfx.ce_feature_version, adev->gfx.ce_fw_version);
+	drm_printf(p, "RLC feature version: %u, fw version: 0x%08x\n",
+		   adev->gfx.rlc_feature_version, adev->gfx.rlc_fw_version);
+
+	drm_printf(p, "RLC SRLC feature version: %u, fw version: 0x%08x\n",
+		   adev->gfx.rlc_srlc_feature_version,
+		   adev->gfx.rlc_srlc_fw_version);
+	drm_printf(p, "RLC SRLG feature version: %u, fw version: 0x%08x\n",
+		   adev->gfx.rlc_srlg_feature_version,
+		   adev->gfx.rlc_srlg_fw_version);
+	drm_printf(p, "RLC SRLS feature version: %u, fw version: 0x%08x\n",
+		   adev->gfx.rlc_srls_feature_version,
+		   adev->gfx.rlc_srls_fw_version);
+	drm_printf(p, "RLCP feature version: %u, fw version: 0x%08x\n",
+		   adev->gfx.rlcp_ucode_feature_version,
+		   adev->gfx.rlcp_ucode_version);
+	drm_printf(p, "RLCV feature version: %u, fw version: 0x%08x\n",
+		   adev->gfx.rlcv_ucode_feature_version,
+		   adev->gfx.rlcv_ucode_version);
+	drm_printf(p, "MEC feature version: %u, fw version: 0x%08x\n",
+		   adev->gfx.mec_feature_version, adev->gfx.mec_fw_version);
+
+	if (adev->gfx.mec2_fw)
+		drm_printf(p, "MEC2 feature version: %u, fw version: 0x%08x\n",
+			   adev->gfx.mec2_feature_version,
+			   adev->gfx.mec2_fw_version);
+
+	drm_printf(p, "IMU feature version: %u, fw version: 0x%08x\n", 0,
+		   adev->gfx.imu_fw_version);
+	drm_printf(p, "PSP SOS feature version: %u, fw version: 0x%08x\n",
+		   adev->psp.sos.feature_version, adev->psp.sos.fw_version);
+	drm_printf(p, "PSP ASD feature version: %u, fw version: 0x%08x\n",
+		   adev->psp.asd_context.bin_desc.feature_version,
+		   adev->psp.asd_context.bin_desc.fw_version);
+
+	drm_printf(p, "TA XGMI feature version: 0x%08x, fw version: 0x%08x\n",
+		   adev->psp.xgmi_context.context.bin_desc.feature_version,
+		   adev->psp.xgmi_context.context.bin_desc.fw_version);
+	drm_printf(p, "TA RAS feature version: 0x%08x, fw version: 0x%08x\n",
+		   adev->psp.ras_context.context.bin_desc.feature_version,
+		   adev->psp.ras_context.context.bin_desc.fw_version);
+	drm_printf(p, "TA HDCP feature version: 0x%08x, fw version: 0x%08x\n",
+		   adev->psp.hdcp_context.context.bin_desc.feature_version,
+		   adev->psp.hdcp_context.context.bin_desc.fw_version);
+	drm_printf(p, "TA DTM feature version: 0x%08x, fw version: 0x%08x\n",
+		   adev->psp.dtm_context.context.bin_desc.feature_version,
+		   adev->psp.dtm_context.context.bin_desc.fw_version);
+	drm_printf(p, "TA RAP feature version: 0x%08x, fw version: 0x%08x\n",
+		   adev->psp.rap_context.context.bin_desc.feature_version,
+		   adev->psp.rap_context.context.bin_desc.fw_version);
+	drm_printf(p,
+		   "TA SECURE DISPLAY feature version: 0x%08x, fw version: 0x%08x\n",
+		   adev->psp.securedisplay_context.context.bin_desc.feature_version,
+		   adev->psp.securedisplay_context.context.bin_desc.fw_version);
+
+	/* SMC firmware */
+	version = adev->pm.fw_version;
+
+	smu_program = (version >> 24) & 0xff;
+	smu_major = (version >> 16) & 0xff;
+	smu_minor = (version >> 8) & 0xff;
+	smu_debug = (version >> 0) & 0xff;
+	drm_printf(p,
+		   "SMC feature version: %u, program: %d, fw version: 0x%08x (%d.%d.%d)\n",
+		   0, smu_program, version, smu_major, smu_minor, smu_debug);
+
+	/* SDMA firmware */
+	for (int i = 0; i < adev->sdma.num_instances; i++) {
+		drm_printf(p,
+			   "SDMA%d feature version: %u, firmware version: 0x%08x\n",
+			   i, adev->sdma.instance[i].feature_version,
+			   adev->sdma.instance[i].fw_version);
+	}
+
+	drm_printf(p, "VCN feature version: %u, fw version: 0x%08x\n", 0,
+		   adev->vcn.fw_version);
+	drm_printf(p, "DMCU feature version: %u, fw version: 0x%08x\n", 0,
+		   adev->dm.dmcu_fw_version);
+	drm_printf(p, "DMCUB feature version: %u, fw version: 0x%08x\n", 0,
+		   adev->dm.dmcub_fw_version);
+	drm_printf(p, "PSP TOC feature version: %u, fw version: 0x%08x\n",
+		   adev->psp.toc.feature_version, adev->psp.toc.fw_version);
+
+	version = adev->mes.kiq_version & AMDGPU_MES_VERSION_MASK;
+	feature = (adev->mes.kiq_version & AMDGPU_MES_FEAT_VERSION_MASK) >>
+		  AMDGPU_MES_FEAT_VERSION_SHIFT;
+	drm_printf(p, "MES_KIQ feature version: %u, fw version: 0x%08x\n",
+		   feature, version);
+
+	version = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK;
+	feature = (adev->mes.sched_version & AMDGPU_MES_FEAT_VERSION_MASK) >>
+		  AMDGPU_MES_FEAT_VERSION_SHIFT;
+	drm_printf(p, "MES feature version: %u, fw version: 0x%08x\n", feature,
+		   version);
+
+	drm_printf(p, "VPE feature version: %u, fw version: 0x%08x\n",
+		   adev->vpe.feature_version, adev->vpe.fw_version);
+
+	drm_printf(p, "\nVBIOS Information\n");
+	drm_printf(p, "vbios name       : %s\n", ctx->name);
+	drm_printf(p, "vbios pn         : %s\n", ctx->vbios_pn);
+	drm_printf(p, "vbios version    : %d\n", ctx->version);
+	drm_printf(p, "vbios ver_str    : %s\n", ctx->vbios_ver_str);
+	drm_printf(p, "vbios date       : %s\n", ctx->date);
+}
+
+static ssize_t
+amdgpu_devcoredump_read(char *buffer, loff_t offset, size_t count,
+			void *data, size_t datalen)
+{
+	struct drm_printer p;
+	struct amdgpu_coredump_info *coredump = data;
+	struct drm_print_iterator iter;
+	struct amdgpu_vm_fault_info *fault_info;
+	int i, ver;
+
+	iter.data = buffer;
+	iter.offset = 0;
+	iter.start = offset;
+	iter.remain = count;
+
+	p = drm_coredump_printer(&iter);
+
+	drm_printf(&p, "**** AMDGPU Device Coredump ****\n");
+	drm_printf(&p, "version: " AMDGPU_COREDUMP_VERSION "\n");
+	drm_printf(&p, "kernel: " UTS_RELEASE "\n");
+	drm_printf(&p, "module: " KBUILD_MODNAME "\n");
+	drm_printf(&p, "time: %lld.%09ld\n", coredump->reset_time.tv_sec,
+		   coredump->reset_time.tv_nsec);
+
+	if (coredump->reset_task_info.pid)
+		drm_printf(&p, "process_name: %s PID: %d\n",
+			   coredump->reset_task_info.process_name,
+			   coredump->reset_task_info.pid);
+
+	/* GPU IP's information of the SOC */
+	drm_printf(&p, "\nIP Information\n");
+	drm_printf(&p, "SOC Family: %d\n", coredump->adev->family);
+	drm_printf(&p, "SOC Revision id: %d\n", coredump->adev->rev_id);
+	drm_printf(&p, "SOC External Revision id: %d\n", coredump->adev->external_rev_id);
+
+	for (int i = 1; i < MAX_HWIP; i++) {
+		for (int j = 0; j < HWIP_MAX_INSTANCE; j++) {
+			ver = coredump->adev->ip_versions[i][j];
+			if (ver)
+				drm_printf(&p, "HWIP: %s[%d][%d]: v%d.%d.%d.%d.%d\n",
+					   hw_ip_names[i], i, j,
+					   IP_VERSION_MAJ(ver),
+					   IP_VERSION_MIN(ver),
+					   IP_VERSION_REV(ver),
+					   IP_VERSION_VARIANT(ver),
+					   IP_VERSION_SUBREV(ver));
+		}
+	}
+
+	/* IP firmware information */
+	drm_printf(&p, "\nIP Firmwares\n");
+	amdgpu_devcoredump_fw_info(coredump->adev, &p);
+
+	if (coredump->ring) {
+		drm_printf(&p, "\nRing timed out details\n");
+		drm_printf(&p, "IP Type: %d Ring Name: %s\n",
+			   coredump->ring->funcs->type,
+			   coredump->ring->name);
+	}
+
+	/* Add page fault information */
+	fault_info = &coredump->adev->vm_manager.fault_info;
+	drm_printf(&p, "\n[%s] Page fault observed\n",
+		   fault_info->vmhub ? "mmhub" : "gfxhub");
+	drm_printf(&p, "Faulty page starting at address: 0x%016llx\n", fault_info->addr);
+	drm_printf(&p, "Protection fault status register: 0x%x\n\n", fault_info->status);
+
+	/* dump the ip state for each ip */
+	drm_printf(&p, "IP Dump\n");
+	for (int i = 0; i < coredump->adev->num_ip_blocks; i++) {
+		if (coredump->adev->ip_blocks[i].version->funcs->print_ip_state) {
+			drm_printf(&p, "IP: %s\n",
+				   coredump->adev->ip_blocks[i]
+					   .version->funcs->name);
+			coredump->adev->ip_blocks[i]
+				.version->funcs->print_ip_state(
+					(void *)coredump->adev, &p);
+			drm_printf(&p, "\n");
+		}
+	}
+
+	/* Add ring buffer information */
+	drm_printf(&p, "Ring buffer information\n");
+	for (int i = 0; i < coredump->adev->num_rings; i++) {
+		int j = 0;
+		struct amdgpu_ring *ring = coredump->adev->rings[i];
+
+		drm_printf(&p, "ring name: %s\n", ring->name);
+		drm_printf(&p, "Rptr: 0x%llx Wptr: 0x%llx RB mask: %x\n",
+			   amdgpu_ring_get_rptr(ring),
+			   amdgpu_ring_get_wptr(ring),
+			   ring->buf_mask);
+		drm_printf(&p, "Ring size in dwords: %d\n",
+			   ring->ring_size / 4);
+		drm_printf(&p, "Ring contents\n");
+		drm_printf(&p, "Offset \t Value\n");
+
+		while (j < ring->ring_size) {
+			drm_printf(&p, "0x%x \t 0x%x\n", j, ring->ring[j / 4]);
+			j += 4;
+		}
+	}
+
+	if (coredump->reset_vram_lost)
+		drm_printf(&p, "VRAM is lost due to GPU reset!\n");
+	if (coredump->adev->reset_info.num_regs) {
+		drm_printf(&p, "AMDGPU register dumps:\nOffset:     Value:\n");
+
+		for (i = 0; i < coredump->adev->reset_info.num_regs; i++)
+			drm_printf(&p, "0x%08x: 0x%08x\n",
+				   coredump->adev->reset_info.reset_dump_reg_list[i],
+				   coredump->adev->reset_info.reset_dump_reg_value[i]);
+	}
+
+	return count - iter.remain;
+}
+
+static void amdgpu_devcoredump_free(void *data)
+{
+	kfree(data);
+}
+
+void amdgpu_coredump(struct amdgpu_device *adev, bool vram_lost,
+		     struct amdgpu_reset_context *reset_context)
+{
+	struct amdgpu_coredump_info *coredump;
+	struct drm_device *dev = adev_to_drm(adev);
+	struct amdgpu_job *job = reset_context->job;
+	struct drm_sched_job *s_job;
+
+	coredump = kzalloc(sizeof(*coredump), GFP_NOWAIT);
+
+	if (!coredump) {
+		DRM_ERROR("%s: failed to allocate memory for coredump\n", __func__);
+		return;
+	}
+
+	coredump->reset_vram_lost = vram_lost;
+
+	if (reset_context->job && reset_context->job->vm) {
+		struct amdgpu_task_info *ti;
+		struct amdgpu_vm *vm = reset_context->job->vm;
+
+		ti = amdgpu_vm_get_task_info_vm(vm);
+		if (ti) {
+			coredump->reset_task_info = *ti;
+			amdgpu_vm_put_task_info(ti);
+		}
+	}
+
+	if (job) {
+		s_job = &job->base;
+		coredump->ring = to_amdgpu_ring(s_job->sched);
+	}
+
+	coredump->adev = adev;
+
+	ktime_get_ts64(&coredump->reset_time);
+
+	dev_coredumpm(dev->dev, THIS_MODULE, coredump, 0, GFP_NOWAIT,
+		      amdgpu_devcoredump_read, amdgpu_devcoredump_free);
+}
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h
new file mode 100644
index 000000000000..52459512cb2b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2024 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __AMDGPU_DEV_COREDUMP_H__
+#define __AMDGPU_DEV_COREDUMP_H__
+
+#include "amdgpu.h"
+#include "amdgpu_reset.h"
+
+#ifdef CONFIG_DEV_COREDUMP
+
+#define AMDGPU_COREDUMP_VERSION "1"
+
+struct amdgpu_coredump_info {
+	struct amdgpu_device            *adev;
+	struct amdgpu_task_info         reset_task_info;
+	struct timespec64               reset_time;
+	bool                            reset_vram_lost;
+	struct amdgpu_ring              *ring;
+};
+#endif
+
+void amdgpu_coredump(struct amdgpu_device *adev, bool vram_lost,
+		     struct amdgpu_reset_context *reset_context);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 7753a2e64d41..932dc93b2e63 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -74,6 +74,7 @@
 #include "amdgpu_fru_eeprom.h"
 #include "amdgpu_reset.h"
 #include "amdgpu_virt.h"
+#include "amdgpu_dev_coredump.h"
 
 #include <linux/suspend.h>
 #include <drm/task_barrier.h>
@@ -143,6 +144,8 @@ const char *amdgpu_asic_name[] = {
 	"LAST",
 };
 
+static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev);
+
 /**
  * DOC: pcie_replay_count
  *
@@ -335,16 +338,93 @@ bool amdgpu_device_supports_boco(struct drm_device *dev)
  *
  * @dev: drm_device pointer
  *
- * Returns true if the device supporte BACO,
- * otherwise return false.
+ * Return:
+ * 1 if the device supporte BACO;
+ * 3 if the device support MACO (only works if BACO is supported)
+ * otherwise return 0.
  */
-bool amdgpu_device_supports_baco(struct drm_device *dev)
+int amdgpu_device_supports_baco(struct drm_device *dev)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
 
 	return amdgpu_asic_supports_baco(adev);
 }
 
+void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev)
+{
+	struct drm_device *dev;
+	int bamaco_support;
+
+	dev = adev_to_drm(adev);
+
+	adev->pm.rpm_mode = AMDGPU_RUNPM_NONE;
+	bamaco_support = amdgpu_device_supports_baco(dev);
+
+	switch (amdgpu_runtime_pm) {
+	case 2:
+		if (bamaco_support & MACO_SUPPORT) {
+			adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;
+			dev_info(adev->dev, "Forcing BAMACO for runtime pm\n");
+		} else if (bamaco_support == BACO_SUPPORT) {
+			adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
+			dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n");
+		}
+		break;
+	case 1:
+		if (bamaco_support & BACO_SUPPORT) {
+			adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
+			dev_info(adev->dev, "Forcing BACO for runtime pm\n");
+		}
+		break;
+	case -1:
+	case -2:
+		if (amdgpu_device_supports_px(dev)) { /* enable PX as runtime mode */
+			adev->pm.rpm_mode = AMDGPU_RUNPM_PX;
+			dev_info(adev->dev, "Using ATPX for runtime pm\n");
+		} else if (amdgpu_device_supports_boco(dev)) { /* enable boco as runtime mode */
+			adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO;
+			dev_info(adev->dev, "Using BOCO for runtime pm\n");
+		} else {
+			if (!bamaco_support)
+				goto no_runtime_pm;
+
+			switch (adev->asic_type) {
+			case CHIP_VEGA20:
+			case CHIP_ARCTURUS:
+				/* BACO are not supported on vega20 and arctrus */
+				break;
+			case CHIP_VEGA10:
+				/* enable BACO as runpm mode if noretry=0 */
+				if (!adev->gmc.noretry)
+					adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
+				break;
+			default:
+				/* enable BACO as runpm mode on CI+ */
+				adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
+				break;
+			}
+
+			if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) {
+				if (bamaco_support & MACO_SUPPORT) {
+					adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;
+					dev_info(adev->dev, "Using BAMACO for runtime pm\n");
+				} else {
+					dev_info(adev->dev, "Using BACO for runtime pm\n");
+				}
+			}
+		}
+		break;
+	case 0:
+		dev_info(adev->dev, "runtime pm is manually disabled\n");
+		break;
+	default:
+		break;
+	}
+
+no_runtime_pm:
+	if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE)
+		dev_info(adev->dev, "Runtime PM not available\n");
+}
 /**
  * amdgpu_device_supports_smart_shift - Is the device dGPU with
  * smart shift support
@@ -1402,13 +1482,17 @@ static int amdgpu_device_wb_init(struct amdgpu_device *adev)
  */
 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
 {
-	unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
+	unsigned long flags, offset;
 
+	spin_lock_irqsave(&adev->wb.lock, flags);
+	offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
 	if (offset < adev->wb.num_wb) {
 		__set_bit(offset, adev->wb.used);
+		spin_unlock_irqrestore(&adev->wb.lock, flags);
 		*wb = offset << 3; /* convert to dw offset */
 		return 0;
 	} else {
+		spin_unlock_irqrestore(&adev->wb.lock, flags);
 		return -EINVAL;
 	}
 }
@@ -1423,9 +1507,13 @@ int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
  */
 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
 {
+	unsigned long flags;
+
 	wb >>= 3;
+	spin_lock_irqsave(&adev->wb.lock, flags);
 	if (wb < adev->wb.num_wb)
 		__clear_bit(wb, adev->wb.used);
+	spin_unlock_irqrestore(&adev->wb.lock, flags);
 }
 
 /**
@@ -1455,7 +1543,7 @@ int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
 
 	/* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */
 	if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR))
-		DRM_WARN("System can't access extended configuration space,please check!!\n");
+		DRM_WARN("System can't access extended configuration space, please check!!\n");
 
 	/* skip if the bios has already enabled large BAR */
 	if (adev->gmc.real_vram_size &&
@@ -3981,6 +4069,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	spin_lock_init(&adev->se_cac_idx_lock);
 	spin_lock_init(&adev->audio_endpt_idx_lock);
 	spin_lock_init(&adev->mm_stats.lock);
+	spin_lock_init(&adev->wb.lock);
 
 	INIT_LIST_HEAD(&adev->shadow_list);
 	mutex_init(&adev->shadow_list_lock);
@@ -4069,6 +4158,13 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	/* Enable TMZ based on IP_VERSION */
 	amdgpu_gmc_tmz_set(adev);
 
+	if (amdgpu_sriov_vf(adev) &&
+	    amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0))
+		/* VF MMIO access (except mailbox range) from CPU
+		 * will be blocked during sriov runtime
+		 */
+		adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT;
+
 	amdgpu_gmc_noretry_set(adev);
 	/* Need to get xgmi info early to decide the reset behavior*/
 	if (adev->gmc.xgmi.supported) {
@@ -4974,12 +5070,15 @@ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
 retry:
 	amdgpu_amdkfd_pre_reset(adev);
 
+	amdgpu_device_stop_pending_resets(adev);
+
 	if (from_hypervisor)
 		r = amdgpu_virt_request_full_gpu(adev, true);
 	else
 		r = amdgpu_virt_reset_gpu(adev);
 	if (r)
 		return r;
+	amdgpu_ras_set_fed(adev, false);
 	amdgpu_irq_gpu_reset_resume_helper(adev);
 
 	/* some sw clean up VF needs to do before recover */
@@ -5263,11 +5362,21 @@ int amdgpu_do_asic_reset(struct list_head *device_list_handle,
 	struct amdgpu_device *tmp_adev = NULL;
 	bool need_full_reset, skip_hw_reset, vram_lost = false;
 	int r = 0;
+	uint32_t i;
 
 	/* Try reset handler method first */
 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
 				    reset_list);
-	amdgpu_reset_reg_dumps(tmp_adev);
+
+	if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) {
+		amdgpu_reset_reg_dumps(tmp_adev);
+
+		/* Trigger ip dump before we reset the asic */
+		for (i = 0; i < tmp_adev->num_ip_blocks; i++)
+			if (tmp_adev->ip_blocks[i].version->funcs->dump_ip_state)
+				tmp_adev->ip_blocks[i].version->funcs
+				->dump_ip_state((void *)tmp_adev);
+	}
 
 	reset_context->reset_device_list = device_list_handle;
 	r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
@@ -5340,7 +5449,8 @@ int amdgpu_do_asic_reset(struct list_head *device_list_handle,
 
 				vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
 
-				amdgpu_coredump(tmp_adev, vram_lost, reset_context);
+				if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags))
+					amdgpu_coredump(tmp_adev, vram_lost, reset_context);
 
 				if (vram_lost) {
 					DRM_INFO("VRAM is lost due to GPU reset!\n");
@@ -5538,6 +5648,23 @@ static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
 
 }
 
+static int amdgpu_device_health_check(struct list_head *device_list_handle)
+{
+	struct amdgpu_device *tmp_adev;
+	int ret = 0;
+	u32 status;
+
+	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
+		pci_read_config_dword(tmp_adev->pdev, PCI_COMMAND, &status);
+		if (PCI_POSSIBLE_ERROR(status)) {
+			dev_err(tmp_adev->dev, "device lost from bus!");
+			ret = -ENODEV;
+		}
+	}
+
+	return ret;
+}
+
 /**
  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
  *
@@ -5609,6 +5736,12 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 		device_list_handle = &device_list;
 	}
 
+	if (!amdgpu_sriov_vf(adev)) {
+		r = amdgpu_device_health_check(device_list_handle);
+		if (r)
+			goto end_reset;
+	}
+
 	/* We need to lock reset domain only once both for XGMI and single device */
 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
 				    reset_list);
@@ -5691,11 +5824,12 @@ retry:	/* Rest of adevs pre asic reset from XGMI hive. */
 			tmp_adev->asic_reset_res = r;
 		}
 
-		/*
-		 * Drop all pending non scheduler resets. Scheduler resets
-		 * were already dropped during drm_sched_stop
-		 */
-		amdgpu_device_stop_pending_resets(tmp_adev);
+		if (!amdgpu_sriov_vf(tmp_adev))
+			/*
+			* Drop all pending non scheduler resets. Scheduler resets
+			* were already dropped during drm_sched_stop
+			*/
+			amdgpu_device_stop_pending_resets(tmp_adev);
 	}
 
 	/* Actual ASIC resets if needed.*/
@@ -5774,6 +5908,7 @@ skip_sched_resume:
 					    reset_list);
 	amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
 
+end_reset:
 	if (hive) {
 		mutex_unlock(&hive->hive_lock);
 		amdgpu_put_xgmi_hive(hive);
@@ -5809,13 +5944,18 @@ static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,
 	*speed = PCI_SPEED_UNKNOWN;
 	*width = PCIE_LNK_WIDTH_UNKNOWN;
 
-	while ((parent = pci_upstream_bridge(parent))) {
-		/* skip upstream/downstream switches internal to dGPU*/
-		if (parent->vendor == PCI_VENDOR_ID_ATI)
-			continue;
-		*speed = pcie_get_speed_cap(parent);
-		*width = pcie_get_width_cap(parent);
-		break;
+	if (amdgpu_device_pcie_dynamic_switching_supported(adev)) {
+		while ((parent = pci_upstream_bridge(parent))) {
+			/* skip upstream/downstream switches internal to dGPU*/
+			if (parent->vendor == PCI_VENDOR_ID_ATI)
+				continue;
+			*speed = pcie_get_speed_cap(parent);
+			*width = pcie_get_width_cap(parent);
+			break;
+		}
+	} else {
+		/* use the current speeds rather than max if switching is not supported */
+		pcie_bandwidth_available(adev->pdev, NULL, speed, width);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
index ac5bf01fe8d2..0e31bdb4b7cb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
@@ -97,6 +97,7 @@
 #include "smuio_v13_0.h"
 #include "smuio_v13_0_3.h"
 #include "smuio_v13_0_6.h"
+#include "smuio_v14_0_2.h"
 #include "vcn_v5_0_0.h"
 #include "jpeg_v5_0_0.h"
 
@@ -245,6 +246,9 @@ static int amdgpu_discovery_read_binary_from_sysmem(struct amdgpu_device *adev,
 	return -ENOENT;
 }
 
+#define IP_DISCOVERY_V2		2
+#define IP_DISCOVERY_V4		4
+
 static int amdgpu_discovery_read_binary_from_mem(struct amdgpu_device *adev,
 						 uint8_t *binary)
 {
@@ -259,14 +263,14 @@ static int amdgpu_discovery_read_binary_from_mem(struct amdgpu_device *adev,
 	 * wait for this to complete.  Once the C2PMSG is updated, we can
 	 * continue.
 	 */
-	if (dev_is_removable(&adev->pdev->dev)) {
-		for (i = 0; i < 1000; i++) {
-			msg = RREG32(mmMP0_SMN_C2PMSG_33);
-			if (msg & 0x80000000)
-				break;
-			msleep(1);
-		}
+
+	for (i = 0; i < 1000; i++) {
+		msg = RREG32(mmMP0_SMN_C2PMSG_33);
+		if (msg & 0x80000000)
+			break;
+		usleep_range(1000, 1100);
 	}
+
 	vram_size = (uint64_t)RREG32(mmRCC_CONFIG_MEMSIZE) << 20;
 
 	if (vram_size) {
@@ -1897,6 +1901,8 @@ static int amdgpu_discovery_set_smu_ip_blocks(struct amdgpu_device *adev)
 		break;
 	case IP_VERSION(14, 0, 0):
 	case IP_VERSION(14, 0, 1):
+	case IP_VERSION(14, 0, 2):
+	case IP_VERSION(14, 0, 3):
 		amdgpu_device_ip_block_add(adev, &smu_v14_0_ip_block);
 		break;
 	default:
@@ -2678,6 +2684,9 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(14, 0, 1):
 		adev->smuio.funcs = &smuio_v13_0_6_funcs;
 		break;
+	case IP_VERSION(14, 0, 2):
+		adev->smuio.funcs = &smuio_v14_0_2_funcs;
+		break;
 	default:
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index e4277298cf1a..ea14f1c8f430 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -925,7 +925,7 @@ module_param_named(freesync_video, amdgpu_freesync_vid_mode, uint, 0444);
  * GPU reset method (-1 = auto (default), 0 = legacy, 1 = mode0, 2 = mode1, 3 = mode2, 4 = baco)
  */
 MODULE_PARM_DESC(reset_method, "GPU reset method (-1 = auto (default), 0 = legacy, 1 = mode0, 2 = mode1, 3 = mode2, 4 = baco/bamaco)");
-module_param_named(reset_method, amdgpu_reset_method, int, 0444);
+module_param_named(reset_method, amdgpu_reset_method, int, 0644);
 
 /**
  * DOC: bad_page_threshold (int) Bad page threshold is specifies the
@@ -2481,6 +2481,7 @@ static void amdgpu_drv_delayed_reset_work_handler(struct work_struct *work)
 
 	/* Use a common context, just need to make sure full reset is done */
 	set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
+	set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
 	r = amdgpu_do_asic_reset(&device_list, &reset_context);
 
 	if (r) {
@@ -2744,7 +2745,8 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev)
 		drm_dev->switch_power_state = DRM_SWITCH_POWER_DYNAMIC_OFF;
 	} else if (adev->pm.rpm_mode == AMDGPU_RUNPM_BOCO) {
 		/* nothing to do */
-	} else if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) {
+	} else if ((adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) ||
+			(adev->pm.rpm_mode == AMDGPU_RUNPM_BAMACO)) {
 		amdgpu_device_baco_enter(drm_dev);
 	}
 
@@ -2784,7 +2786,8 @@ static int amdgpu_pmops_runtime_resume(struct device *dev)
 		 * PCI core handles it for _PR3.
 		 */
 		pci_set_master(pdev);
-	} else if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) {
+	} else if ((adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) ||
+			(adev->pm.rpm_mode == AMDGPU_RUNPM_BAMACO)) {
 		amdgpu_device_baco_exit(drm_dev);
 	}
 	ret = amdgpu_device_resume(drm_dev, false);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 55d5508987ff..1d955652f3ba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -1206,7 +1206,8 @@ void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev,
 		fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
 		break;
 	default:
-		break;
+		dev_err(adev->dev, "Invalid ucode id %u\n", ucode_id);
+		return;
 	}
 
 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index 8fcf889ddce9..64f197bbc866 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -259,7 +259,6 @@ struct amdgpu_cu_info {
 struct amdgpu_gfx_ras {
 	struct amdgpu_ras_block_object  ras_block;
 	void (*enable_watchdog_timer)(struct amdgpu_device *adev);
-	bool (*query_utcl2_poison_status)(struct amdgpu_device *adev);
 	int (*rlc_gc_fed_irq)(struct amdgpu_device *adev,
 				struct amdgpu_irq_src *source,
 				struct amdgpu_iv_entry *entry);
@@ -434,6 +433,10 @@ struct amdgpu_gfx {
 	uint32_t			num_xcc_per_xcp;
 	struct mutex			partition_mutex;
 	bool				mcbp; /* mid command buffer preemption */
+
+	/* IP reg dump */
+	uint32_t			*ip_dump;
+	uint32_t			reg_count;
 };
 
 struct amdgpu_gfx_ras_reg_entry {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h
index c7b44aeb671b..103a837ccc71 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h
@@ -38,6 +38,8 @@ struct amdgpu_gfxhub_funcs {
 	void (*mode2_save_regs)(struct amdgpu_device *adev);
 	void (*mode2_restore_regs)(struct amdgpu_device *adev);
 	void (*halt)(struct amdgpu_device *adev);
+	bool (*query_utcl2_poison_status)(struct amdgpu_device *adev,
+			int xcc_id);
 };
 
 struct amdgpu_gfxhub {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
index d79cb13e1aa8..00d6211e0fbf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
@@ -279,7 +279,7 @@ amdgpu_i2c_lookup(struct amdgpu_device *adev,
 	return NULL;
 }
 
-static void amdgpu_i2c_get_byte(struct amdgpu_i2c_chan *i2c_bus,
+static int amdgpu_i2c_get_byte(struct amdgpu_i2c_chan *i2c_bus,
 				 u8 slave_addr,
 				 u8 addr,
 				 u8 *val)
@@ -304,16 +304,18 @@ static void amdgpu_i2c_get_byte(struct amdgpu_i2c_chan *i2c_bus,
 	out_buf[0] = addr;
 	out_buf[1] = 0;
 
-	if (i2c_transfer(&i2c_bus->adapter, msgs, 2) == 2) {
-		*val = in_buf[0];
-		DRM_DEBUG("val = 0x%02x\n", *val);
-	} else {
-		DRM_DEBUG("i2c 0x%02x 0x%02x read failed\n",
-			  addr, *val);
+	if (i2c_transfer(&i2c_bus->adapter, msgs, 2) != 2) {
+		DRM_DEBUG("i2c 0x%02x read failed\n", addr);
+		return -EIO;
 	}
+
+	*val = in_buf[0];
+	DRM_DEBUG("val = 0x%02x\n", *val);
+
+	return 0;
 }
 
-static void amdgpu_i2c_put_byte(struct amdgpu_i2c_chan *i2c_bus,
+static int amdgpu_i2c_put_byte(struct amdgpu_i2c_chan *i2c_bus,
 				 u8 slave_addr,
 				 u8 addr,
 				 u8 val)
@@ -329,9 +331,12 @@ static void amdgpu_i2c_put_byte(struct amdgpu_i2c_chan *i2c_bus,
 	out_buf[0] = addr;
 	out_buf[1] = val;
 
-	if (i2c_transfer(&i2c_bus->adapter, &msg, 1) != 1)
-		DRM_DEBUG("i2c 0x%02x 0x%02x write failed\n",
-			  addr, val);
+	if (i2c_transfer(&i2c_bus->adapter, &msg, 1) != 1) {
+		DRM_DEBUG("i2c 0x%02x 0x%02x write failed\n", addr, val);
+		return -EIO;
+	}
+
+	return 0;
 }
 
 /* ddc router switching */
@@ -346,16 +351,18 @@ amdgpu_i2c_router_select_ddc_port(const struct amdgpu_connector *amdgpu_connecto
 	if (!amdgpu_connector->router_bus)
 		return;
 
-	amdgpu_i2c_get_byte(amdgpu_connector->router_bus,
+	if (amdgpu_i2c_get_byte(amdgpu_connector->router_bus,
 			    amdgpu_connector->router.i2c_addr,
-			    0x3, &val);
+			    0x3, &val))
+		return;
 	val &= ~amdgpu_connector->router.ddc_mux_control_pin;
 	amdgpu_i2c_put_byte(amdgpu_connector->router_bus,
 			    amdgpu_connector->router.i2c_addr,
 			    0x3, val);
-	amdgpu_i2c_get_byte(amdgpu_connector->router_bus,
+	if (amdgpu_i2c_get_byte(amdgpu_connector->router_bus,
 			    amdgpu_connector->router.i2c_addr,
-			    0x1, &val);
+			    0x1, &val))
+		return;
 	val &= ~amdgpu_connector->router.ddc_mux_control_pin;
 	val |= amdgpu_connector->router.ddc_mux_state;
 	amdgpu_i2c_put_byte(amdgpu_connector->router_bus,
@@ -375,16 +382,18 @@ amdgpu_i2c_router_select_cd_port(const struct amdgpu_connector *amdgpu_connector
 	if (!amdgpu_connector->router_bus)
 		return;
 
-	amdgpu_i2c_get_byte(amdgpu_connector->router_bus,
+	if (amdgpu_i2c_get_byte(amdgpu_connector->router_bus,
 			    amdgpu_connector->router.i2c_addr,
-			    0x3, &val);
+			    0x3, &val))
+		return;
 	val &= ~amdgpu_connector->router.cd_mux_control_pin;
 	amdgpu_i2c_put_byte(amdgpu_connector->router_bus,
 			    amdgpu_connector->router.i2c_addr,
 			    0x3, val);
-	amdgpu_i2c_get_byte(amdgpu_connector->router_bus,
+	if (amdgpu_i2c_get_byte(amdgpu_connector->router_bus,
 			    amdgpu_connector->router.i2c_addr,
-			    0x1, &val);
+			    0x1, &val))
+		return;
 	val &= ~amdgpu_connector->router.cd_mux_control_pin;
 	val |= amdgpu_connector->router.cd_mux_state;
 	amdgpu_i2c_put_byte(amdgpu_connector->router_bus,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 7e6d09730e6d..013ff373e067 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -279,7 +279,7 @@ int amdgpu_irq_init(struct amdgpu_device *adev)
 	adev->irq.msi_enabled = false;
 
 	if (!amdgpu_msi_ok(adev))
-		flags = PCI_IRQ_LEGACY;
+		flags = PCI_IRQ_INTX;
 	else
 		flags = PCI_IRQ_ALL_TYPES;
 
@@ -445,6 +445,14 @@ void amdgpu_irq_dispatch(struct amdgpu_device *adev,
 
 	entry.ih = ih;
 	entry.iv_entry = (const uint32_t *)&ih->ring[ring_index];
+
+	/*
+	 * timestamp is not supported on some legacy SOCs (cik, cz, iceland,
+	 * si and tonga), so initialize timestamp and timestamp_src to 0
+	 */
+	entry.timestamp = 0;
+	entry.timestamp_src = 0;
+
 	amdgpu_ih_decode_iv(adev, &entry);
 
 	trace_amdgpu_iv(ih - &adev->irq.ih, &entry);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index a2df3025a754..a0ea6fe8d060 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -149,38 +149,7 @@ int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags)
 		goto out;
 	}
 
-	adev->pm.rpm_mode = AMDGPU_RUNPM_NONE;
-	if (amdgpu_device_supports_px(dev) &&
-	    (amdgpu_runtime_pm != 0)) { /* enable PX as runtime mode */
-		adev->pm.rpm_mode = AMDGPU_RUNPM_PX;
-		dev_info(adev->dev, "Using ATPX for runtime pm\n");
-	} else if (amdgpu_device_supports_boco(dev) &&
-		   (amdgpu_runtime_pm != 0)) { /* enable boco as runtime mode */
-		adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO;
-		dev_info(adev->dev, "Using BOCO for runtime pm\n");
-	} else if (amdgpu_device_supports_baco(dev) &&
-		   (amdgpu_runtime_pm != 0)) {
-		switch (adev->asic_type) {
-		case CHIP_VEGA20:
-		case CHIP_ARCTURUS:
-			/* enable BACO as runpm mode if runpm=1 */
-			if (amdgpu_runtime_pm > 0)
-				adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
-			break;
-		case CHIP_VEGA10:
-			/* enable BACO as runpm mode if noretry=0 */
-			if (!adev->gmc.noretry)
-				adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
-			break;
-		default:
-			/* enable BACO as runpm mode on CI+ */
-			adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
-			break;
-		}
-
-		if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO)
-			dev_info(adev->dev, "Using BACO for runtime pm\n");
-	}
+	amdgpu_device_detect_runtime_pm_mode(adev);
 
 	/* Call ACPI methods: require modeset init
 	 * but failure is not fatal
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c
index 24ad4b97177b..0734490347db 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c
@@ -210,22 +210,26 @@ int amdgpu_mca_smu_set_debug_mode(struct amdgpu_device *adev, bool enable)
 	return -EOPNOTSUPP;
 }
 
-static void amdgpu_mca_smu_mca_bank_dump(struct amdgpu_device *adev, int idx, struct mca_bank_entry *entry)
+static void amdgpu_mca_smu_mca_bank_dump(struct amdgpu_device *adev, int idx, struct mca_bank_entry *entry,
+					 struct ras_query_context *qctx)
 {
-	dev_info(adev->dev, HW_ERR "Accelerator Check Architecture events logged\n");
-	dev_info(adev->dev, HW_ERR "aca entry[%02d].STATUS=0x%016llx\n",
-		 idx, entry->regs[MCA_REG_IDX_STATUS]);
-	dev_info(adev->dev, HW_ERR "aca entry[%02d].ADDR=0x%016llx\n",
-		 idx, entry->regs[MCA_REG_IDX_ADDR]);
-	dev_info(adev->dev, HW_ERR "aca entry[%02d].MISC0=0x%016llx\n",
-		 idx, entry->regs[MCA_REG_IDX_MISC0]);
-	dev_info(adev->dev, HW_ERR "aca entry[%02d].IPID=0x%016llx\n",
-		 idx, entry->regs[MCA_REG_IDX_IPID]);
-	dev_info(adev->dev, HW_ERR "aca entry[%02d].SYND=0x%016llx\n",
-		 idx, entry->regs[MCA_REG_IDX_SYND]);
+	u64 event_id = qctx->event_id;
+
+	RAS_EVENT_LOG(adev, event_id, HW_ERR "Accelerator Check Architecture events logged\n");
+	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].STATUS=0x%016llx\n",
+		      idx, entry->regs[MCA_REG_IDX_STATUS]);
+	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].ADDR=0x%016llx\n",
+		      idx, entry->regs[MCA_REG_IDX_ADDR]);
+	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].MISC0=0x%016llx\n",
+		      idx, entry->regs[MCA_REG_IDX_MISC0]);
+	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].IPID=0x%016llx\n",
+		      idx, entry->regs[MCA_REG_IDX_IPID]);
+	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].SYND=0x%016llx\n",
+		      idx, entry->regs[MCA_REG_IDX_SYND]);
 }
 
-int amdgpu_mca_smu_log_ras_error(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type, struct ras_err_data *err_data)
+int amdgpu_mca_smu_log_ras_error(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type,
+				 struct ras_err_data *err_data, struct ras_query_context *qctx)
 {
 	struct amdgpu_smuio_mcm_config_info mcm_info;
 	struct ras_err_addr err_addr = {0};
@@ -244,7 +248,7 @@ int amdgpu_mca_smu_log_ras_error(struct amdgpu_device *adev, enum amdgpu_ras_blo
 	list_for_each_entry(node, &mca_set.list, node) {
 		entry = &node->entry;
 
-		amdgpu_mca_smu_mca_bank_dump(adev, i++, entry);
+		amdgpu_mca_smu_mca_bank_dump(adev, i++, entry, qctx);
 
 		count = 0;
 		ret = amdgpu_mca_smu_parse_mca_error_count(adev, blk, type, entry, &count);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h
index b964110ed1e0..e5bf07ce3451 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h
@@ -169,6 +169,7 @@ void amdgpu_mca_smu_debugfs_init(struct amdgpu_device *adev, struct dentry *root
 void amdgpu_mca_bank_set_init(struct mca_bank_set *mca_set);
 int amdgpu_mca_bank_set_add_entry(struct mca_bank_set *mca_set, struct mca_bank_entry *entry);
 void amdgpu_mca_bank_set_release(struct mca_bank_set *mca_set);
-int amdgpu_mca_smu_log_ras_error(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type, struct ras_err_data *err_data);
+int amdgpu_mca_smu_log_ras_error(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type,
+				 struct ras_err_data *err_data, struct ras_query_context *qctx);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index a00cf4756ad0..5ca5c47ab54e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -32,6 +32,18 @@
 #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
 #define AMDGPU_ONE_DOORBELL_SIZE 8
 
+signed long amdgpu_mes_fence_wait_polling(u64 *fence,
+					  u64 wait_seq,
+					  signed long timeout)
+{
+
+	while ((s64)(wait_seq - *fence) > 0 && timeout > 0) {
+		udelay(2);
+		timeout -= 2;
+	}
+	return timeout > 0 ? timeout : 0;
+}
+
 int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev)
 {
 	return roundup(AMDGPU_ONE_DOORBELL_SIZE *
@@ -40,7 +52,6 @@ int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev)
 }
 
 static int amdgpu_mes_kernel_doorbell_get(struct amdgpu_device *adev,
-					 struct amdgpu_mes_process *process,
 					 int ip_type, uint64_t *doorbell_index)
 {
 	unsigned int offset, found;
@@ -65,7 +76,6 @@ static int amdgpu_mes_kernel_doorbell_get(struct amdgpu_device *adev,
 }
 
 static void amdgpu_mes_kernel_doorbell_free(struct amdgpu_device *adev,
-					   struct amdgpu_mes_process *process,
 					   uint32_t doorbell_index)
 {
 	unsigned int old, rel_index;
@@ -656,7 +666,7 @@ int amdgpu_mes_add_hw_queue(struct amdgpu_device *adev, int gang_id,
 	*queue_id = queue->queue_id = r;
 
 	/* allocate a doorbell index for the queue */
-	r = amdgpu_mes_kernel_doorbell_get(adev, gang->process,
+	r = amdgpu_mes_kernel_doorbell_get(adev,
 					  qprops->queue_type,
 					  &qprops->doorbell_off);
 	if (r)
@@ -714,8 +724,7 @@ int amdgpu_mes_add_hw_queue(struct amdgpu_device *adev, int gang_id,
 	return 0;
 
 clean_up_doorbell:
-	amdgpu_mes_kernel_doorbell_free(adev, gang->process,
-				       qprops->doorbell_off);
+	amdgpu_mes_kernel_doorbell_free(adev, qprops->doorbell_off);
 clean_up_queue_id:
 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
 	idr_remove(&adev->mes.queue_id_idr, queue->queue_id);
@@ -769,8 +778,7 @@ int amdgpu_mes_remove_hw_queue(struct amdgpu_device *adev, int queue_id)
 			  queue_id);
 
 	list_del(&queue->list);
-	amdgpu_mes_kernel_doorbell_free(adev, gang->process,
-				       queue->doorbell_off);
+	amdgpu_mes_kernel_doorbell_free(adev, queue->doorbell_off);
 	amdgpu_mes_unlock(&adev->mes);
 
 	amdgpu_mes_queue_free_mqd(queue);
@@ -778,6 +786,28 @@ int amdgpu_mes_remove_hw_queue(struct amdgpu_device *adev, int queue_id)
 	return 0;
 }
 
+int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev,
+				struct amdgpu_ring *ring)
+{
+	struct mes_map_legacy_queue_input queue_input;
+	int r;
+
+	memset(&queue_input, 0, sizeof(queue_input));
+
+	queue_input.queue_type = ring->funcs->type;
+	queue_input.doorbell_offset = ring->doorbell_index;
+	queue_input.pipe_id = ring->pipe;
+	queue_input.queue_id = ring->queue;
+	queue_input.mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
+	queue_input.wptr_addr = ring->wptr_gpu_addr;
+
+	r = adev->mes.funcs->map_legacy_queue(&adev->mes, &queue_input);
+	if (r)
+		DRM_ERROR("failed to map legacy queue\n");
+
+	return r;
+}
+
 int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev,
 				  struct amdgpu_ring *ring,
 				  enum amdgpu_unmap_queues_action action,
@@ -1132,6 +1162,7 @@ void amdgpu_mes_remove_ring(struct amdgpu_device *adev,
 		return;
 
 	amdgpu_mes_remove_hw_queue(adev, ring->hw_queue_id);
+	del_timer_sync(&ring->fence_drv.fallback_timer);
 	amdgpu_ring_fini(ring);
 	kfree(ring);
 }
@@ -1474,7 +1505,7 @@ int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe)
 	const struct mes_firmware_header_v1_0 *mes_hdr;
 	struct amdgpu_firmware_info *info;
 	char ucode_prefix[30];
-	char fw_name[40];
+	char fw_name[50];
 	bool need_retry = false;
 	int r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index 4c8fc3117ef8..df9f0404d842 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -141,6 +141,12 @@ struct amdgpu_mes {
 
 	/* ip specific functions */
 	const struct amdgpu_mes_funcs   *funcs;
+
+	/* mes resource_1 bo*/
+	struct amdgpu_bo    *resource_1;
+	uint64_t            resource_1_gpu_addr;
+	void                *resource_1_addr;
+
 };
 
 struct amdgpu_mes_process {
@@ -242,6 +248,15 @@ struct mes_remove_queue_input {
 	uint64_t	gang_context_addr;
 };
 
+struct mes_map_legacy_queue_input {
+	uint32_t                           queue_type;
+	uint32_t                           doorbell_offset;
+	uint32_t                           pipe_id;
+	uint32_t                           queue_id;
+	uint64_t                           mqd_addr;
+	uint64_t                           wptr_addr;
+};
+
 struct mes_unmap_legacy_queue_input {
 	enum amdgpu_unmap_queues_action    action;
 	uint32_t                           queue_type;
@@ -318,6 +333,9 @@ struct amdgpu_mes_funcs {
 	int (*remove_hw_queue)(struct amdgpu_mes *mes,
 			       struct mes_remove_queue_input *input);
 
+	int (*map_legacy_queue)(struct amdgpu_mes *mes,
+				struct mes_map_legacy_queue_input *input);
+
 	int (*unmap_legacy_queue)(struct amdgpu_mes *mes,
 				  struct mes_unmap_legacy_queue_input *input);
 
@@ -334,6 +352,10 @@ struct amdgpu_mes_funcs {
 #define amdgpu_mes_kiq_hw_init(adev) (adev)->mes.kiq_hw_init((adev))
 #define amdgpu_mes_kiq_hw_fini(adev) (adev)->mes.kiq_hw_fini((adev))
 
+signed long amdgpu_mes_fence_wait_polling(u64 *fence,
+					  u64 wait_seq,
+					  signed long timeout);
+
 int amdgpu_mes_ctx_get_offs(struct amdgpu_ring *ring, unsigned int id_offs);
 
 int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe);
@@ -357,6 +379,8 @@ int amdgpu_mes_add_hw_queue(struct amdgpu_device *adev, int gang_id,
 			    int *queue_id);
 int amdgpu_mes_remove_hw_queue(struct amdgpu_device *adev, int queue_id);
 
+int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev,
+				struct amdgpu_ring *ring);
 int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev,
 				  struct amdgpu_ring *ring,
 				  enum amdgpu_unmap_queues_action action,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
index 1ca9d4ed8063..95d676ee207f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
@@ -63,6 +63,8 @@ struct amdgpu_mmhub_funcs {
 				uint64_t page_table_base);
 	void (*update_power_gating)(struct amdgpu_device *adev,
                                 bool enable);
+	bool (*query_utcl2_poison_status)(struct amdgpu_device *adev,
+				int hub_inst);
 };
 
 struct amdgpu_mmhub {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 010b0cb7693c..8d8c39be6129 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -39,6 +39,7 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 #include "amdgpu_amdkfd.h"
+#include "amdgpu_vram_mgr.h"
 
 /**
  * DOC: amdgpu_object
@@ -153,8 +154,10 @@ void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 		else
 			places[c].flags |= TTM_PL_FLAG_TOPDOWN;
 
-		if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
+		if (abo->tbo.type == ttm_bo_type_kernel &&
+		    flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
 			places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
+
 		c++;
 	}
 
@@ -173,6 +176,12 @@ void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 			abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ?
 			AMDGPU_PL_PREEMPT : TTM_PL_TT;
 		places[c].flags = 0;
+		/*
+		 * When GTT is just an alternative to VRAM make sure that we
+		 * only use it as fallback and still try to fill up VRAM first.
+		 */
+		if (domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)
+			places[c].flags |= TTM_PL_FLAG_FALLBACK;
 		c++;
 	}
 
@@ -595,8 +604,7 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 	if (!amdgpu_bo_support_uswc(bo->flags))
 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
 
-	if (adev->ras_enabled)
-		bo->flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
+	bo->flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
 
 	bo->tbo.bdev = &adev->mman.bdev;
 	if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA |
@@ -605,6 +613,8 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 	else
 		amdgpu_bo_placement_from_domain(bo, bp->domain);
 	if (bp->type == ttm_bo_type_kernel)
+		bo->tbo.priority = 2;
+	else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE))
 		bo->tbo.priority = 1;
 
 	if (!bp->destroy)
@@ -617,8 +627,7 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 		return r;
 
 	if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
-	    bo->tbo.resource->mem_type == TTM_PL_VRAM &&
-	    amdgpu_bo_in_cpu_visible_vram(bo))
+	    amdgpu_res_cpu_visible(adev, bo->tbo.resource))
 		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
 					     ctx.bytes_moved);
 	else
@@ -628,7 +637,7 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 	    bo->tbo.resource->mem_type == TTM_PL_VRAM) {
 		struct dma_fence *fence;
 
-		r = amdgpu_fill_buffer(bo, 0, bo->tbo.base.resv, &fence, true);
+		r = amdgpu_ttm_clear_buffer(bo, bo->tbo.base.resv, &fence);
 		if (unlikely(r))
 			goto fail_unreserve;
 
@@ -758,7 +767,7 @@ int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, struct dma_fence **fence)
 
 	return amdgpu_copy_buffer(ring, shadow_addr, parent_addr,
 				  amdgpu_bo_size(shadow), NULL, fence,
-				  true, false, false);
+				  true, false, 0);
 }
 
 /**
@@ -960,6 +969,10 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 		if (!bo->placements[i].lpfn ||
 		    (lpfn && lpfn < bo->placements[i].lpfn))
 			bo->placements[i].lpfn = lpfn;
+
+		if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS &&
+		    bo->placements[i].mem_type == TTM_PL_VRAM)
+			bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS;
 	}
 
 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
@@ -1242,14 +1255,18 @@ int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
  * amdgpu_bo_move_notify - notification about a memory move
  * @bo: pointer to a buffer object
  * @evict: if this move is evicting the buffer from the graphics address space
+ * @new_mem: new resource for backing the BO
  *
  * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs
  * bookkeeping.
  * TTM driver callback which is called when ttm moves a buffer.
  */
-void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, bool evict)
+void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
+			   bool evict,
+			   struct ttm_resource *new_mem)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
+	struct ttm_resource *old_mem = bo->resource;
 	struct amdgpu_bo *abo;
 
 	if (!amdgpu_bo_is_amdgpu_bo(bo))
@@ -1261,34 +1278,36 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, bool evict)
 	amdgpu_bo_kunmap(abo);
 
 	if (abo->tbo.base.dma_buf && !abo->tbo.base.import_attach &&
-	    bo->resource->mem_type != TTM_PL_SYSTEM)
+	    old_mem && old_mem->mem_type != TTM_PL_SYSTEM)
 		dma_buf_move_notify(abo->tbo.base.dma_buf);
 
-	/* remember the eviction */
-	if (evict)
-		atomic64_inc(&adev->num_evictions);
+	/* move_notify is called before move happens */
+	trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1,
+			     old_mem ? old_mem->mem_type : -1);
 }
 
 void amdgpu_bo_get_memory(struct amdgpu_bo *bo,
 			  struct amdgpu_mem_stats *stats)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	struct ttm_resource *res = bo->tbo.resource;
 	uint64_t size = amdgpu_bo_size(bo);
 	struct drm_gem_object *obj;
 	unsigned int domain;
 	bool shared;
 
 	/* Abort if the BO doesn't currently have a backing store */
-	if (!bo->tbo.resource)
+	if (!res)
 		return;
 
 	obj = &bo->tbo.base;
 	shared = drm_gem_object_is_shared_for_memory_stats(obj);
 
-	domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);
+	domain = amdgpu_mem_type_to_domain(res->mem_type);
 	switch (domain) {
 	case AMDGPU_GEM_DOMAIN_VRAM:
 		stats->vram += size;
-		if (amdgpu_bo_in_cpu_visible_vram(bo))
+		if (amdgpu_res_cpu_visible(adev, bo->tbo.resource))
 			stats->visible_vram += size;
 		if (shared)
 			stats->vram_shared += size;
@@ -1359,8 +1378,9 @@ void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
 	if (WARN_ON_ONCE(!dma_resv_trylock(bo->base.resv)))
 		return;
 
-	r = amdgpu_fill_buffer(abo, AMDGPU_POISON, bo->base.resv, &fence, true);
+	r = amdgpu_fill_buffer(abo, 0, bo->base.resv, &fence, true);
 	if (!WARN_ON(r)) {
+		amdgpu_vram_mgr_set_cleared(bo->resource);
 		amdgpu_bo_fence(abo, fence, false);
 		dma_fence_put(fence);
 	}
@@ -1389,10 +1409,7 @@ vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 	/* Remember that this BO was accessed by the CPU */
 	abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 
-	if (bo->resource->mem_type != TTM_PL_VRAM)
-		return 0;
-
-	if (amdgpu_bo_in_cpu_visible_vram(abo))
+	if (amdgpu_res_cpu_visible(adev, bo->resource))
 		return 0;
 
 	/* Can't move a pinned BO to visible VRAM */
@@ -1415,7 +1432,7 @@ vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 
 	/* this should never happen */
 	if (bo->resource->mem_type == TTM_PL_VRAM &&
-	    !amdgpu_bo_in_cpu_visible_vram(abo))
+	    !amdgpu_res_cpu_visible(adev, bo->resource))
 		return VM_FAULT_SIGBUS;
 
 	ttm_bo_move_to_lru_tail_unlocked(bo);
@@ -1579,6 +1596,7 @@ uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
  */
 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	struct dma_buf_attachment *attachment;
 	struct dma_buf *dma_buf;
 	const char *placement;
@@ -1587,10 +1605,11 @@ u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
 
 	if (dma_resv_trylock(bo->tbo.base.resv)) {
 		unsigned int domain;
+
 		domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);
 		switch (domain) {
 		case AMDGPU_GEM_DOMAIN_VRAM:
-			if (amdgpu_bo_in_cpu_visible_vram(bo))
+			if (amdgpu_res_cpu_visible(adev, bo->tbo.resource))
 				placement = "VRAM VISIBLE";
 			else
 				placement = "VRAM";
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index be679c42b0b8..bc42ccbde659 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -251,28 +251,6 @@ static inline u64 amdgpu_bo_mmap_offset(struct amdgpu_bo *bo)
 }
 
 /**
- * amdgpu_bo_in_cpu_visible_vram - check if BO is (partly) in visible VRAM
- */
-static inline bool amdgpu_bo_in_cpu_visible_vram(struct amdgpu_bo *bo)
-{
-	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
-	struct amdgpu_res_cursor cursor;
-
-	if (!bo->tbo.resource || bo->tbo.resource->mem_type != TTM_PL_VRAM)
-		return false;
-
-	amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &cursor);
-	while (cursor.remaining) {
-		if (cursor.start < adev->gmc.visible_vram_size)
-			return true;
-
-		amdgpu_res_next(&cursor, cursor.size);
-	}
-
-	return false;
-}
-
-/**
  * amdgpu_bo_explicit_sync - return whether the bo is explicitly synced
  */
 static inline bool amdgpu_bo_explicit_sync(struct amdgpu_bo *bo)
@@ -350,7 +328,9 @@ int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
 			   size_t buffer_size, uint32_t *metadata_size,
 			   uint64_t *flags);
-void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, bool evict);
+void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
+			   bool evict,
+			   struct ttm_resource *new_mem);
 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo);
 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo);
 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 94b310fdb719..4bd4602d11b1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -1053,6 +1053,11 @@ static int psp_asd_initialize(struct psp_context *psp)
 	if (amdgpu_sriov_vf(psp->adev) || !psp->asd_context.bin_desc.size_bytes)
 		return 0;
 
+	/* bypass asd if display hardware is not available */
+	if (!amdgpu_device_has_display_hardware(psp->adev) &&
+	    amdgpu_ip_version(psp->adev, MP0_HWIP, 0) >= IP_VERSION(13, 0, 10))
+		return 0;
+
 	psp->asd_context.mem_context.shared_mc_addr  = 0;
 	psp->asd_context.mem_context.shared_mem_size = PSP_ASD_SHARED_MEM_SIZE;
 	psp->asd_context.ta_load_type                = GFX_CMD_ID_LOAD_ASD;
@@ -2260,6 +2265,15 @@ static int psp_hw_start(struct psp_context *psp)
 			}
 		}
 
+		if ((is_psp_fw_valid(psp->ipkeymgr_drv)) &&
+		    (psp->funcs->bootloader_load_ipkeymgr_drv != NULL)) {
+			ret = psp_bootloader_load_ipkeymgr_drv(psp);
+			if (ret) {
+				dev_err(adev->dev, "PSP load ipkeymgr_drv failed!\n");
+				return ret;
+			}
+		}
+
 		if ((is_psp_fw_valid(psp->sos)) &&
 		    (psp->funcs->bootloader_load_sos != NULL)) {
 			ret = psp_bootloader_load_sos(psp);
@@ -2617,7 +2631,8 @@ static int psp_load_p2s_table(struct psp_context *psp)
 	struct amdgpu_firmware_info *ucode =
 		&adev->firmware.ucode[AMDGPU_UCODE_ID_P2S_TABLE];
 
-	if (adev->in_runpm && (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO))
+	if (adev->in_runpm && ((adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) ||
+				(adev->pm.rpm_mode == AMDGPU_RUNPM_BAMACO)))
 		return 0;
 
 	if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6)) {
@@ -2647,7 +2662,8 @@ static int psp_load_smu_fw(struct psp_context *psp)
 	 * Skip SMU FW reloading in case of using BACO for runpm only,
 	 * as SMU is always alive.
 	 */
-	if (adev->in_runpm && (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO))
+	if (adev->in_runpm && ((adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) ||
+				(adev->pm.rpm_mode == AMDGPU_RUNPM_BAMACO)))
 		return 0;
 
 	if (!ucode->fw || amdgpu_sriov_vf(psp->adev))
@@ -3273,6 +3289,12 @@ static int parse_sos_bin_descriptor(struct psp_context *psp,
 		psp->ras_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
 		psp->ras_drv.start_addr         = ucode_start_addr;
 		break;
+	case PSP_FW_TYPE_PSP_IPKEYMGR_DRV:
+		psp->ipkeymgr_drv.fw_version         = le32_to_cpu(desc->fw_version);
+		psp->ipkeymgr_drv.feature_version    = le32_to_cpu(desc->fw_version);
+		psp->ipkeymgr_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
+		psp->ipkeymgr_drv.start_addr         = ucode_start_addr;
+		break;
 	default:
 		dev_warn(psp->adev->dev, "Unsupported PSP FW type: %d\n", desc->fw_type);
 		break;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index ee16f134ae92..3635303e6548 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -73,8 +73,10 @@ enum psp_bootloader_cmd {
 	PSP_BL__LOAD_KEY_DATABASE	= 0x80000,
 	PSP_BL__LOAD_SOCDRV             = 0xB0000,
 	PSP_BL__LOAD_DBGDRV             = 0xC0000,
+	PSP_BL__LOAD_HADDRV		= PSP_BL__LOAD_DBGDRV,
 	PSP_BL__LOAD_INTFDRV		= 0xD0000,
-	PSP_BL__LOAD_RASDRV		    = 0xE0000,
+	PSP_BL__LOAD_RASDRV		= 0xE0000,
+	PSP_BL__LOAD_IPKEYMGRDRV	= 0xF0000,
 	PSP_BL__DRAM_LONG_TRAIN		= 0x100000,
 	PSP_BL__DRAM_SHORT_TRAIN	= 0x200000,
 	PSP_BL__LOAD_TOS_SPL_TABLE	= 0x10000000,
@@ -117,6 +119,7 @@ struct psp_funcs {
 	int (*bootloader_load_intf_drv)(struct psp_context *psp);
 	int (*bootloader_load_dbg_drv)(struct psp_context *psp);
 	int (*bootloader_load_ras_drv)(struct psp_context *psp);
+	int (*bootloader_load_ipkeymgr_drv)(struct psp_context *psp);
 	int (*bootloader_load_sos)(struct psp_context *psp);
 	int (*ring_create)(struct psp_context *psp,
 			   enum psp_ring_type ring_type);
@@ -336,6 +339,7 @@ struct psp_context {
 	struct psp_bin_desc		intf_drv;
 	struct psp_bin_desc		dbg_drv;
 	struct psp_bin_desc		ras_drv;
+	struct psp_bin_desc		ipkeymgr_drv;
 
 	/* tmr buffer */
 	struct amdgpu_bo		*tmr_bo;
@@ -424,6 +428,9 @@ struct amdgpu_psp_funcs {
 #define psp_bootloader_load_ras_drv(psp) \
 		((psp)->funcs->bootloader_load_ras_drv ? \
 		(psp)->funcs->bootloader_load_ras_drv((psp)) : 0)
+#define psp_bootloader_load_ipkeymgr_drv(psp) \
+		((psp)->funcs->bootloader_load_ipkeymgr_drv ? \
+		 (psp)->funcs->bootloader_load_ipkeymgr_drv((psp)) : 0)
 #define psp_bootloader_load_sos(psp) \
 		((psp)->funcs->bootloader_load_sos ? (psp)->funcs->bootloader_load_sos((psp)) : 0)
 #define psp_smu_reload_quirk(psp) \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
index 8ebab6f22e5a..1adc81a55734 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -122,6 +122,8 @@ const char *get_ras_block_str(struct ras_common_if *ras_block)
 
 #define MAX_UMC_POISON_POLLING_TIME_ASYNC  100  //ms
 
+#define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100  //ms
+
 enum amdgpu_ras_retire_page_reservation {
 	AMDGPU_RAS_RETIRE_PAGE_RESERVED,
 	AMDGPU_RAS_RETIRE_PAGE_PENDING,
@@ -1045,6 +1047,7 @@ static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_d
 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev,
 					      struct ras_manager *ras_mgr,
 					      struct ras_err_data *err_data,
+					      struct ras_query_context *qctx,
 					      const char *blk_name,
 					      bool is_ue,
 					      bool is_de)
@@ -1052,27 +1055,28 @@ static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev,
 	struct amdgpu_smuio_mcm_config_info *mcm_info;
 	struct ras_err_node *err_node;
 	struct ras_err_info *err_info;
+	u64 event_id = qctx->event_id;
 
 	if (is_ue) {
 		for_each_ras_error(err_node, err_data) {
 			err_info = &err_node->err_info;
 			mcm_info = &err_info->mcm_info;
 			if (err_info->ue_count) {
-				dev_info(adev->dev, "socket: %d, die: %d, "
-					 "%lld new uncorrectable hardware errors detected in %s block\n",
-					 mcm_info->socket_id,
-					 mcm_info->die_id,
-					 err_info->ue_count,
-					 blk_name);
+				RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
+					      "%lld new uncorrectable hardware errors detected in %s block\n",
+					      mcm_info->socket_id,
+					      mcm_info->die_id,
+					      err_info->ue_count,
+					      blk_name);
 			}
 		}
 
 		for_each_ras_error(err_node, &ras_mgr->err_data) {
 			err_info = &err_node->err_info;
 			mcm_info = &err_info->mcm_info;
-			dev_info(adev->dev, "socket: %d, die: %d, "
-				 "%lld uncorrectable hardware errors detected in total in %s block\n",
-				 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name);
+			RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
+				      "%lld uncorrectable hardware errors detected in total in %s block\n",
+				      mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name);
 		}
 
 	} else {
@@ -1081,44 +1085,44 @@ static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev,
 				err_info = &err_node->err_info;
 				mcm_info = &err_info->mcm_info;
 				if (err_info->de_count) {
-					dev_info(adev->dev, "socket: %d, die: %d, "
-						"%lld new deferred hardware errors detected in %s block\n",
-						mcm_info->socket_id,
-						mcm_info->die_id,
-						err_info->de_count,
-						blk_name);
+					RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
+						      "%lld new deferred hardware errors detected in %s block\n",
+						      mcm_info->socket_id,
+						      mcm_info->die_id,
+						      err_info->de_count,
+						      blk_name);
 				}
 			}
 
 			for_each_ras_error(err_node, &ras_mgr->err_data) {
 				err_info = &err_node->err_info;
 				mcm_info = &err_info->mcm_info;
-				dev_info(adev->dev, "socket: %d, die: %d, "
-					"%lld deferred hardware errors detected in total in %s block\n",
-					mcm_info->socket_id, mcm_info->die_id,
-					err_info->de_count, blk_name);
+				RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
+					      "%lld deferred hardware errors detected in total in %s block\n",
+					      mcm_info->socket_id, mcm_info->die_id,
+					      err_info->de_count, blk_name);
 			}
 		} else {
 			for_each_ras_error(err_node, err_data) {
 				err_info = &err_node->err_info;
 				mcm_info = &err_info->mcm_info;
 				if (err_info->ce_count) {
-					dev_info(adev->dev, "socket: %d, die: %d, "
-						"%lld new correctable hardware errors detected in %s block\n",
-						mcm_info->socket_id,
-						mcm_info->die_id,
-						err_info->ce_count,
-						blk_name);
+					RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
+						      "%lld new correctable hardware errors detected in %s block\n",
+						      mcm_info->socket_id,
+						      mcm_info->die_id,
+						      err_info->ce_count,
+						      blk_name);
 				}
 			}
 
 			for_each_ras_error(err_node, &ras_mgr->err_data) {
 				err_info = &err_node->err_info;
 				mcm_info = &err_info->mcm_info;
-				dev_info(adev->dev, "socket: %d, die: %d, "
-					"%lld correctable hardware errors detected in total in %s block\n",
-					mcm_info->socket_id, mcm_info->die_id,
-					err_info->ce_count, blk_name);
+				RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
+					      "%lld correctable hardware errors detected in total in %s block\n",
+					      mcm_info->socket_id, mcm_info->die_id,
+					      err_info->ce_count, blk_name);
 			}
 		}
 	}
@@ -1131,77 +1135,79 @@ static inline bool err_data_has_source_info(struct ras_err_data *data)
 
 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev,
 					     struct ras_query_if *query_if,
-					     struct ras_err_data *err_data)
+					     struct ras_err_data *err_data,
+					     struct ras_query_context *qctx)
 {
 	struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head);
 	const char *blk_name = get_ras_block_str(&query_if->head);
+	u64 event_id = qctx->event_id;
 
 	if (err_data->ce_count) {
 		if (err_data_has_source_info(err_data)) {
-			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data,
+			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
 							  blk_name, false, false);
 		} else if (!adev->aid_mask &&
 			   adev->smuio.funcs &&
 			   adev->smuio.funcs->get_socket_id &&
 			   adev->smuio.funcs->get_die_id) {
-			dev_info(adev->dev, "socket: %d, die: %d "
-				 "%ld correctable hardware errors "
-				 "detected in %s block\n",
-				 adev->smuio.funcs->get_socket_id(adev),
-				 adev->smuio.funcs->get_die_id(adev),
-				 ras_mgr->err_data.ce_count,
-				 blk_name);
+			RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
+				      "%ld correctable hardware errors "
+				      "detected in %s block\n",
+				      adev->smuio.funcs->get_socket_id(adev),
+				      adev->smuio.funcs->get_die_id(adev),
+				      ras_mgr->err_data.ce_count,
+				      blk_name);
 		} else {
-			dev_info(adev->dev, "%ld correctable hardware errors "
-				 "detected in %s block\n",
-				 ras_mgr->err_data.ce_count,
-				 blk_name);
+			RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors "
+				      "detected in %s block\n",
+				      ras_mgr->err_data.ce_count,
+				      blk_name);
 		}
 	}
 
 	if (err_data->ue_count) {
 		if (err_data_has_source_info(err_data)) {
-			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data,
+			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
 							  blk_name, true, false);
 		} else if (!adev->aid_mask &&
 			   adev->smuio.funcs &&
 			   adev->smuio.funcs->get_socket_id &&
 			   adev->smuio.funcs->get_die_id) {
-			dev_info(adev->dev, "socket: %d, die: %d "
-				 "%ld uncorrectable hardware errors "
-				 "detected in %s block\n",
-				 adev->smuio.funcs->get_socket_id(adev),
-				 adev->smuio.funcs->get_die_id(adev),
-				 ras_mgr->err_data.ue_count,
-				 blk_name);
+			RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
+				      "%ld uncorrectable hardware errors "
+				      "detected in %s block\n",
+				      adev->smuio.funcs->get_socket_id(adev),
+				      adev->smuio.funcs->get_die_id(adev),
+				      ras_mgr->err_data.ue_count,
+				      blk_name);
 		} else {
-			dev_info(adev->dev, "%ld uncorrectable hardware errors "
-				 "detected in %s block\n",
-				 ras_mgr->err_data.ue_count,
-				 blk_name);
+			RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors "
+				      "detected in %s block\n",
+				      ras_mgr->err_data.ue_count,
+				      blk_name);
 		}
 	}
 
 	if (err_data->de_count) {
 		if (err_data_has_source_info(err_data)) {
-			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data,
+			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
 							  blk_name, false, true);
 		} else if (!adev->aid_mask &&
 			   adev->smuio.funcs &&
 			   adev->smuio.funcs->get_socket_id &&
 			   adev->smuio.funcs->get_die_id) {
-			dev_info(adev->dev, "socket: %d, die: %d "
-				 "%ld deferred hardware errors "
-				 "detected in %s block\n",
-				 adev->smuio.funcs->get_socket_id(adev),
-				 adev->smuio.funcs->get_die_id(adev),
-				 ras_mgr->err_data.de_count,
-				 blk_name);
+			RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
+				      "%ld deferred hardware errors "
+				      "detected in %s block\n",
+				      adev->smuio.funcs->get_socket_id(adev),
+				      adev->smuio.funcs->get_die_id(adev),
+				      ras_mgr->err_data.de_count,
+				      blk_name);
 		} else {
-			dev_info(adev->dev, "%ld deferred hardware errors "
-				 "detected in %s block\n",
-				 ras_mgr->err_data.de_count,
-				 blk_name);
+			RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors "
+				      "detected in %s block\n",
+				      ras_mgr->err_data.de_count,
+				      blk_name);
 		}
 	}
 }
@@ -1244,6 +1250,10 @@ int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
 {
 	struct ras_manager *obj;
 
+	/* in resume phase, no need to create aca fs node */
+	if (adev->in_suspend || amdgpu_in_reset(adev))
+		return 0;
+
 	obj = get_ras_manager(adev, blk);
 	if (!obj)
 		return -EINVAL;
@@ -1265,7 +1275,8 @@ int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
 }
 
 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
-					 enum aca_error_type type, struct ras_err_data *err_data)
+					 enum aca_error_type type, struct ras_err_data *err_data,
+					 struct ras_query_context *qctx)
 {
 	struct ras_manager *obj;
 
@@ -1273,7 +1284,7 @@ static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu
 	if (!obj)
 		return -EINVAL;
 
-	return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data);
+	return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx);
 }
 
 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr,
@@ -1287,13 +1298,14 @@ ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *a
 	if (amdgpu_ras_query_error_status(obj->adev, &info))
 		return -EINVAL;
 
-	return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
-			  "ce", info.ce_count);
+	return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
+			  "ce", info.ce_count, "de", info.ue_count);
 }
 
 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
 						struct ras_query_if *info,
 						struct ras_err_data *err_data,
+						struct ras_query_context *qctx,
 						unsigned int error_query_mode)
 {
 	enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT;
@@ -1329,17 +1341,21 @@ static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
 		}
 	} else {
 		if (amdgpu_aca_is_enabled(adev)) {
-			ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data);
+			ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx);
+			if (ret)
+				return ret;
+
+			ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx);
 			if (ret)
 				return ret;
 
-			ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data);
+			ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx);
 			if (ret)
 				return ret;
 		} else {
 			/* FIXME: add code to check return value later */
-			amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data);
-			amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data);
+			amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx);
+			amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx);
 		}
 	}
 
@@ -1351,6 +1367,7 @@ int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_i
 {
 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
 	struct ras_err_data err_data;
+	struct ras_query_context qctx;
 	unsigned int error_query_mode;
 	int ret;
 
@@ -1364,8 +1381,12 @@ int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_i
 	if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode))
 		return -EINVAL;
 
+	memset(&qctx, 0, sizeof(qctx));
+	qctx.event_id = amdgpu_ras_acquire_event_id(adev, amdgpu_ras_intr_triggered() ?
+						   RAS_EVENT_TYPE_ISR : RAS_EVENT_TYPE_INVALID);
 	ret = amdgpu_ras_query_error_status_helper(adev, info,
 						   &err_data,
+						   &qctx,
 						   error_query_mode);
 	if (ret)
 		goto out_fini_err_data;
@@ -1376,7 +1397,7 @@ int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_i
 	info->ce_count = obj->err_data.ce_count;
 	info->de_count = obj->err_data.de_count;
 
-	amdgpu_ras_error_generate_report(adev, info, &err_data);
+	amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx);
 
 out_fini_err_data:
 	amdgpu_ras_error_data_fini(&err_data);
@@ -2041,7 +2062,7 @@ static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *
 		}
 	}
 
-	amdgpu_umc_poison_handler(adev, obj->head.block, false);
+	amdgpu_umc_poison_handler(adev, obj->head.block, 0);
 
 	if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption)
 		poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
@@ -2061,6 +2082,17 @@ static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj
 {
 	dev_info(obj->adev->dev,
 		"Poison is created\n");
+
+	if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) {
+		struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev);
+
+		amdgpu_ras_put_poison_req(obj->adev,
+			AMDGPU_RAS_BLOCK__UMC, 0, NULL, NULL, false);
+
+		atomic_inc(&con->page_retirement_req_cnt);
+
+		wake_up(&con->page_retirement_wq);
+	}
 }
 
 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
@@ -2371,7 +2403,7 @@ static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
 			.flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
 		};
 		status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
-				data->bps[i].retired_page);
+				data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT);
 		if (status == -EBUSY)
 			(*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
 		else if (status == -ENOENT)
@@ -2384,6 +2416,19 @@ out:
 	return ret;
 }
 
+static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev,
+				   struct amdgpu_hive_info *hive, bool status)
+{
+	struct amdgpu_device *tmp_adev;
+
+	if (hive) {
+		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
+			amdgpu_ras_set_fed(tmp_adev, status);
+	} else {
+		amdgpu_ras_set_fed(adev, status);
+	}
+}
+
 static void amdgpu_ras_do_recovery(struct work_struct *work)
 {
 	struct amdgpu_ras *ras =
@@ -2393,8 +2438,21 @@ static void amdgpu_ras_do_recovery(struct work_struct *work)
 	struct list_head device_list, *device_list_handle =  NULL;
 	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
 
-	if (hive)
+	if (hive) {
 		atomic_set(&hive->ras_recovery, 1);
+
+		/* If any device which is part of the hive received RAS fatal
+		 * error interrupt, set fatal error status on all. This
+		 * condition will need a recovery, and flag will be cleared
+		 * as part of recovery.
+		 */
+		list_for_each_entry(remote_adev, &hive->device_list,
+				    gmc.xgmi.head)
+			if (amdgpu_ras_get_fed_status(remote_adev)) {
+				amdgpu_ras_set_fed_all(adev, hive, true);
+				break;
+			}
+	}
 	if (!ras->disable_ras_err_cnt_harvest) {
 
 		/* Build list of devices to query RAS related errors */
@@ -2439,18 +2497,6 @@ static void amdgpu_ras_do_recovery(struct work_struct *work)
 				ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET;
 				set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
 
-				/* For any RAS error that needs a full reset to
-				 * recover, set the fatal error status
-				 */
-				if (hive) {
-					list_for_each_entry(remote_adev,
-							    &hive->device_list,
-							    gmc.xgmi.head)
-						amdgpu_ras_set_fed(remote_adev,
-								   true);
-				} else {
-					amdgpu_ras_set_fed(adev, true);
-				}
 				psp_fatal_error_recovery_quirk(&adev->psp);
 			}
 		}
@@ -2516,9 +2562,7 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
 			goto out;
 		}
 
-		amdgpu_vram_mgr_reserve_range(&adev->mman.vram_mgr,
-			bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT,
-			AMDGPU_GPU_PAGE_SIZE);
+		amdgpu_ras_reserve_page(adev, bps[i].retired_page);
 
 		memcpy(&data->bps[data->count], &bps[i], sizeof(*data->bps));
 		data->count++;
@@ -2674,10 +2718,167 @@ static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
 	}
 }
 
+int amdgpu_ras_put_poison_req(struct amdgpu_device *adev,
+		enum amdgpu_ras_block block, uint16_t pasid,
+		pasid_notify pasid_fn, void *data, uint32_t reset)
+{
+	int ret = 0;
+	struct ras_poison_msg poison_msg;
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+
+	memset(&poison_msg, 0, sizeof(poison_msg));
+	poison_msg.block = block;
+	poison_msg.pasid = pasid;
+	poison_msg.reset = reset;
+	poison_msg.pasid_fn = pasid_fn;
+	poison_msg.data = data;
+
+	ret = kfifo_put(&con->poison_fifo, poison_msg);
+	if (!ret) {
+		dev_err(adev->dev, "Poison message fifo is full!\n");
+		return -ENOSPC;
+	}
+
+	return 0;
+}
+
+static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev,
+		struct ras_poison_msg *poison_msg)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+
+	return kfifo_get(&con->poison_fifo, poison_msg);
+}
+
+static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log)
+{
+	mutex_init(&ecc_log->lock);
+
+	/* Set any value as siphash key */
+	memset(&ecc_log->ecc_key, 0xad, sizeof(ecc_log->ecc_key));
+
+	INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL);
+	ecc_log->de_updated = false;
+}
+
+static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log)
+{
+	struct radix_tree_iter iter;
+	void __rcu **slot;
+	struct ras_ecc_err *ecc_err;
+
+	mutex_lock(&ecc_log->lock);
+	radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) {
+		ecc_err = radix_tree_deref_slot(slot);
+		kfree(ecc_err->err_pages.pfn);
+		kfree(ecc_err);
+		radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot);
+	}
+	mutex_unlock(&ecc_log->lock);
+
+	mutex_destroy(&ecc_log->lock);
+	ecc_log->de_updated = false;
+}
+
+static void amdgpu_ras_do_page_retirement(struct work_struct *work)
+{
+	struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
+					      page_retirement_dwork.work);
+	struct amdgpu_device *adev = con->adev;
+	struct ras_err_data err_data;
+
+	if (amdgpu_in_reset(adev) || atomic_read(&con->in_recovery))
+		return;
+
+	amdgpu_ras_error_data_init(&err_data);
+
+	amdgpu_umc_handle_bad_pages(adev, &err_data);
+
+	amdgpu_ras_error_data_fini(&err_data);
+
+	mutex_lock(&con->umc_ecc_log.lock);
+	if (radix_tree_tagged(&con->umc_ecc_log.de_page_tree,
+				UMC_ECC_NEW_DETECTED_TAG))
+		schedule_delayed_work(&con->page_retirement_dwork,
+			msecs_to_jiffies(AMDGPU_RAS_RETIRE_PAGE_INTERVAL));
+	mutex_unlock(&con->umc_ecc_log.lock);
+}
+
+static int amdgpu_ras_query_ecc_status(struct amdgpu_device *adev,
+			enum amdgpu_ras_block ras_block, uint32_t timeout_ms)
+{
+	int ret = 0;
+	struct ras_ecc_log_info *ecc_log;
+	struct ras_query_if info;
+	uint32_t timeout = timeout_ms;
+	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
+
+	memset(&info, 0, sizeof(info));
+	info.head.block = ras_block;
+
+	ecc_log = &ras->umc_ecc_log;
+	ecc_log->de_updated = false;
+	do {
+		ret = amdgpu_ras_query_error_status(adev, &info);
+		if (ret) {
+			dev_err(adev->dev, "Failed to query ras error! ret:%d\n", ret);
+			return ret;
+		}
+
+		if (timeout && !ecc_log->de_updated) {
+			msleep(1);
+			timeout--;
+		}
+	} while (timeout && !ecc_log->de_updated);
+
+	if (timeout_ms && !timeout) {
+		dev_warn(adev->dev, "Can't find deferred error\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static void amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev,
+					uint32_t timeout)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	int ret;
+
+	ret = amdgpu_ras_query_ecc_status(adev, AMDGPU_RAS_BLOCK__UMC, timeout);
+	if (!ret)
+		schedule_delayed_work(&con->page_retirement_dwork, 0);
+}
+
+static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev,
+			struct ras_poison_msg *poison_msg)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	uint32_t reset = poison_msg->reset;
+	uint16_t pasid = poison_msg->pasid;
+
+	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
+
+	if (poison_msg->pasid_fn)
+		poison_msg->pasid_fn(adev, pasid, poison_msg->data);
+
+	if (reset) {
+		flush_delayed_work(&con->page_retirement_dwork);
+
+		con->gpu_reset_flags |= reset;
+		amdgpu_ras_reset_gpu(adev);
+	}
+
+	return 0;
+}
+
 static int amdgpu_ras_page_retirement_thread(void *param)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)param;
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_poison_msg poison_msg;
+	enum amdgpu_ras_block ras_block;
+	bool poison_creation_is_handled = false;
 
 	while (!kthread_should_stop()) {
 
@@ -2688,13 +2889,34 @@ static int amdgpu_ras_page_retirement_thread(void *param)
 		if (kthread_should_stop())
 			break;
 
-		dev_info(adev->dev, "Start processing page retirement. request:%d\n",
-			atomic_read(&con->page_retirement_req_cnt));
-
 		atomic_dec(&con->page_retirement_req_cnt);
 
-		amdgpu_umc_bad_page_polling_timeout(adev,
-				false, MAX_UMC_POISON_POLLING_TIME_ASYNC);
+		if (!amdgpu_ras_get_poison_req(adev, &poison_msg))
+			continue;
+
+		ras_block = poison_msg.block;
+
+		dev_info(adev->dev, "Start processing ras block %s(%d)\n",
+				ras_block_str(ras_block), ras_block);
+
+		if (ras_block == AMDGPU_RAS_BLOCK__UMC) {
+			amdgpu_ras_poison_creation_handler(adev,
+				MAX_UMC_POISON_POLLING_TIME_ASYNC);
+			poison_creation_is_handled = true;
+		} else {
+			/* poison_creation_is_handled:
+			 *   false: no poison creation interrupt, but it has poison
+			 *          consumption interrupt.
+			 *   true: It has poison creation interrupt at the beginning,
+			 *         but it has no poison creation interrupt later.
+			 */
+			amdgpu_ras_poison_creation_handler(adev,
+					poison_creation_is_handled ?
+					0 : MAX_UMC_POISON_POLLING_TIME_ASYNC);
+
+			amdgpu_ras_poison_consumption_handler(adev, &poison_msg);
+			poison_creation_is_handled = false;
+		}
 	}
 
 	return 0;
@@ -2763,6 +2985,8 @@ int amdgpu_ras_recovery_init(struct amdgpu_device *adev)
 		}
 	}
 
+	mutex_init(&con->page_rsv_lock);
+	INIT_KFIFO(con->poison_fifo);
 	mutex_init(&con->page_retirement_lock);
 	init_waitqueue_head(&con->page_retirement_wq);
 	atomic_set(&con->page_retirement_req_cnt, 0);
@@ -2773,6 +2997,8 @@ int amdgpu_ras_recovery_init(struct amdgpu_device *adev)
 		dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n");
 	}
 
+	INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement);
+	amdgpu_ras_ecc_log_init(&con->umc_ecc_log);
 #ifdef CONFIG_X86_MCE_AMD
 	if ((adev->asic_type == CHIP_ALDEBARAN) &&
 	    (adev->gmc.xgmi.connected_to_cpu))
@@ -2813,8 +3039,14 @@ static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
 
 	atomic_set(&con->page_retirement_req_cnt, 0);
 
+	mutex_destroy(&con->page_rsv_lock);
+
 	cancel_work_sync(&con->recovery_work);
 
+	cancel_delayed_work_sync(&con->page_retirement_dwork);
+
+	amdgpu_ras_ecc_log_fini(&con->umc_ecc_log);
+
 	mutex_lock(&con->recovery_lock);
 	con->eh_data = NULL;
 	kfree(data->bps);
@@ -3036,6 +3268,35 @@ static int amdgpu_get_ras_schema(struct amdgpu_device *adev)
 			AMDGPU_RAS_ERROR__PARITY;
 }
 
+static void ras_event_mgr_init(struct ras_event_manager *mgr)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(mgr->seqnos); i++)
+		atomic64_set(&mgr->seqnos[i], 0);
+}
+
+static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
+	struct amdgpu_hive_info *hive;
+
+	if (!ras)
+		return;
+
+	hive = amdgpu_get_xgmi_hive(adev);
+	ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr;
+
+	/* init event manager with node 0 on xgmi system */
+	if (!amdgpu_in_reset(adev)) {
+		if (!hive || adev->gmc.xgmi.node_id == 0)
+			ras_event_mgr_init(ras->event_mgr);
+	}
+
+	if (hive)
+		amdgpu_put_xgmi_hive(hive);
+}
+
 int amdgpu_ras_init(struct amdgpu_device *adev)
 {
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
@@ -3356,6 +3617,8 @@ int amdgpu_ras_late_init(struct amdgpu_device *adev)
 	if (amdgpu_sriov_vf(adev))
 		return 0;
 
+	amdgpu_ras_event_mgr_init(adev);
+
 	if (amdgpu_aca_is_enabled(adev)) {
 		if (amdgpu_in_reset(adev))
 			r = amdgpu_aca_reset(adev);
@@ -3472,14 +3735,39 @@ void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status)
 		atomic_set(&ras->fed, !!status);
 }
 
+bool amdgpu_ras_event_id_is_valid(struct amdgpu_device *adev, u64 id)
+{
+	return !(id & BIT_ULL(63));
+}
+
+u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type)
+{
+	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
+	u64 id;
+
+	switch (type) {
+	case RAS_EVENT_TYPE_ISR:
+		id = (u64)atomic64_read(&ras->event_mgr->seqnos[type]);
+		break;
+	case RAS_EVENT_TYPE_INVALID:
+	default:
+		id = BIT_ULL(63) | 0ULL;
+		break;
+	}
+
+	return id;
+}
+
 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
 {
 	if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
 		struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
+		u64 event_id = (u64)atomic64_inc_return(&ras->event_mgr->seqnos[RAS_EVENT_TYPE_ISR]);
 
-		dev_info(adev->dev, "uncorrectable hardware error"
-			"(ERREVENT_ATHUB_INTERRUPT) detected!\n");
+		RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error"
+			      "(ERREVENT_ATHUB_INTERRUPT) detected!\n");
 
+		amdgpu_ras_set_fed(adev, true);
 		ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
 		amdgpu_ras_reset_gpu(adev);
 	}
@@ -3998,6 +4286,8 @@ void amdgpu_ras_add_mca_err_addr(struct ras_err_info *err_info, struct ras_err_a
 {
 	struct ras_err_addr *mca_err_addr;
 
+	/* This function will be retired. */
+	return;
 	mca_err_addr = kzalloc(sizeof(*mca_err_addr), GFP_KERNEL);
 	if (!mca_err_addr)
 		return;
@@ -4195,3 +4485,19 @@ void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances)
 			amdgpu_ras_boot_time_error_reporting(adev, i, boot_error);
 	}
 }
+
+int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
+	uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT;
+	int ret = 0;
+
+	mutex_lock(&con->page_rsv_lock);
+	ret = amdgpu_vram_mgr_query_page_status(mgr, start);
+	if (ret == -ENOENT)
+		ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE);
+	mutex_unlock(&con->page_rsv_lock);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
index e0f8ce9d8440..7021c4a66fb5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
@@ -26,6 +26,9 @@
 
 #include <linux/debugfs.h>
 #include <linux/list.h>
+#include <linux/kfifo.h>
+#include <linux/radix-tree.h>
+#include <linux/siphash.h>
 #include "ta_ras_if.h"
 #include "amdgpu_ras_eeprom.h"
 #include "amdgpu_smuio.h"
@@ -43,7 +46,7 @@ struct amdgpu_iv_entry;
 #define AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(x)		AMDGPU_GET_REG_FIELD(x, 7, 7)
 #define AMDGPU_RAS_GPU_ERR_SOCKET_ID(x)			AMDGPU_GET_REG_FIELD(x, 10, 8)
 #define AMDGPU_RAS_GPU_ERR_AID_ID(x)			AMDGPU_GET_REG_FIELD(x, 12, 11)
-#define AMDGPU_RAS_GPU_ERR_HBM_ID(x)			AMDGPU_GET_REG_FIELD(x, 13, 13)
+#define AMDGPU_RAS_GPU_ERR_HBM_ID(x)			AMDGPU_GET_REG_FIELD(x, 14, 13)
 #define AMDGPU_RAS_GPU_ERR_BOOT_STATUS(x)		AMDGPU_GET_REG_FIELD(x, 31, 31)
 
 #define AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT	1000
@@ -64,6 +67,14 @@ struct amdgpu_iv_entry;
 /* The high three bits indicates socketid */
 #define AMDGPU_RAS_GET_FEATURES(val)  ((val) & ~AMDGPU_RAS_FEATURES_SOCKETID_MASK)
 
+#define RAS_EVENT_LOG(_adev, _id, _fmt, ...)				\
+do {									\
+	if (amdgpu_ras_event_id_is_valid((_adev), (_id)))			\
+	    dev_info((_adev)->dev, "{%llu}" _fmt, (_id), ##__VA_ARGS__);	\
+	else								\
+	    dev_info((_adev)->dev, _fmt, ##__VA_ARGS__);			\
+} while (0)
+
 enum amdgpu_ras_block {
 	AMDGPU_RAS_BLOCK__UMC = 0,
 	AMDGPU_RAS_BLOCK__SDMA,
@@ -419,6 +430,52 @@ struct umc_ecc_info {
 	int record_ce_addr_supported;
 };
 
+enum ras_event_type {
+	RAS_EVENT_TYPE_INVALID = -1,
+	RAS_EVENT_TYPE_ISR = 0,
+	RAS_EVENT_TYPE_COUNT,
+};
+
+struct ras_event_manager {
+	atomic64_t seqnos[RAS_EVENT_TYPE_COUNT];
+};
+
+struct ras_query_context {
+	enum ras_event_type type;
+	u64 event_id;
+};
+
+typedef int (*pasid_notify)(struct amdgpu_device *adev,
+		uint16_t pasid, void *data);
+
+struct ras_poison_msg {
+	enum amdgpu_ras_block block;
+	uint16_t pasid;
+	uint32_t reset;
+	pasid_notify pasid_fn;
+	void *data;
+};
+
+struct ras_err_pages {
+	uint32_t count;
+	uint64_t *pfn;
+};
+
+struct ras_ecc_err {
+	u64 hash_index;
+	uint64_t status;
+	uint64_t ipid;
+	uint64_t addr;
+	struct ras_err_pages err_pages;
+};
+
+struct ras_ecc_log_info {
+	struct mutex lock;
+	siphash_key_t ecc_key;
+	struct radix_tree_root de_page_tree;
+	bool	de_updated;
+};
+
 struct amdgpu_ras {
 	/* ras infrastructure */
 	/* for ras itself. */
@@ -477,8 +534,18 @@ struct amdgpu_ras {
 	wait_queue_head_t page_retirement_wq;
 	struct mutex page_retirement_lock;
 	atomic_t page_retirement_req_cnt;
+	struct mutex page_rsv_lock;
+	DECLARE_KFIFO(poison_fifo, struct ras_poison_msg, 128);
+	struct ras_ecc_log_info  umc_ecc_log;
+	struct delayed_work page_retirement_dwork;
+
 	/* Fatal error detected flag */
 	atomic_t fed;
+
+	/* RAS event manager */
+	struct ras_event_manager __event_mgr;
+	struct ras_event_manager *event_mgr;
+
 };
 
 struct ras_fs_data {
@@ -512,6 +579,7 @@ struct ras_err_data {
 	unsigned long de_count;
 	unsigned long err_addr_cnt;
 	struct eeprom_table_record *err_addr;
+	unsigned long err_addr_len;
 	u32 err_list_count;
 	struct list_head err_node_list;
 };
@@ -879,4 +947,13 @@ void amdgpu_ras_del_mca_err_addr(struct ras_err_info *err_info,
 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status);
 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev);
 
+bool amdgpu_ras_event_id_is_valid(struct amdgpu_device *adev, u64 id);
+u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type);
+
+int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn);
+
+int amdgpu_ras_put_poison_req(struct amdgpu_device *adev,
+		enum amdgpu_ras_block block, uint16_t pasid,
+		pasid_notify pasid_fn, void *data, uint32_t reset);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
index b12808c0c331..06a62a8a992e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
@@ -404,6 +404,22 @@ static int amdgpu_ras_eeprom_correct_header_tag(
 	return res;
 }
 
+static void amdgpu_ras_set_eeprom_table_version(struct amdgpu_ras_eeprom_control *control)
+{
+	struct amdgpu_device *adev = to_amdgpu_device(control);
+	struct amdgpu_ras_eeprom_table_header *hdr = &control->tbl_hdr;
+
+	switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) {
+	case IP_VERSION(8, 10, 0):
+	case IP_VERSION(12, 0, 0):
+		hdr->version = RAS_TABLE_VER_V2_1;
+		return;
+	default:
+		hdr->version = RAS_TABLE_VER_V1;
+		return;
+	}
+}
+
 /**
  * amdgpu_ras_eeprom_reset_table -- Reset the RAS EEPROM table
  * @control: pointer to control structure
@@ -423,11 +439,7 @@ int amdgpu_ras_eeprom_reset_table(struct amdgpu_ras_eeprom_control *control)
 	mutex_lock(&control->ras_tbl_mutex);
 
 	hdr->header = RAS_TABLE_HDR_VAL;
-	if (adev->umc.ras &&
-	    adev->umc.ras->set_eeprom_table_version)
-		adev->umc.ras->set_eeprom_table_version(hdr);
-	else
-		hdr->version = RAS_TABLE_VER_V1;
+	amdgpu_ras_set_eeprom_table_version(control);
 
 	if (hdr->version == RAS_TABLE_VER_V2_1) {
 		hdr->first_rec_offset = RAS_RECORD_START_V2_1;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
index 381101d2bf05..50fcd86e1033 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
@@ -164,4 +164,29 @@ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size)
 	}
 }
 
+/**
+ * amdgpu_res_cleared - check if blocks are cleared
+ *
+ * @cur: the cursor to extract the block
+ *
+ * Check if the @cur block is cleared
+ */
+static inline bool amdgpu_res_cleared(struct amdgpu_res_cursor *cur)
+{
+	struct drm_buddy_block *block;
+
+	switch (cur->mem_type) {
+	case TTM_PL_VRAM:
+		block = cur->node;
+
+		if (!amdgpu_vram_mgr_is_cleared(block))
+			return false;
+		break;
+	default:
+		return false;
+	}
+
+	return true;
+}
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
index 147100c27c2d..ea4873f6ccd1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
@@ -21,9 +21,6 @@
  *
  */
 
-#include <linux/devcoredump.h>
-#include <generated/utsrelease.h>
-
 #include "amdgpu_reset.h"
 #include "aldebaran.h"
 #include "sienna_cichlid.h"
@@ -161,105 +158,3 @@ void amdgpu_device_unlock_reset_domain(struct amdgpu_reset_domain *reset_domain)
 	atomic_set(&reset_domain->in_gpu_reset, 0);
 	up_write(&reset_domain->sem);
 }
-
-#ifndef CONFIG_DEV_COREDUMP
-void amdgpu_coredump(struct amdgpu_device *adev, bool vram_lost,
-		     struct amdgpu_reset_context *reset_context)
-{
-}
-#else
-static ssize_t
-amdgpu_devcoredump_read(char *buffer, loff_t offset, size_t count,
-			void *data, size_t datalen)
-{
-	struct drm_printer p;
-	struct amdgpu_coredump_info *coredump = data;
-	struct drm_print_iterator iter;
-	int i;
-
-	iter.data = buffer;
-	iter.offset = 0;
-	iter.start = offset;
-	iter.remain = count;
-
-	p = drm_coredump_printer(&iter);
-
-	drm_printf(&p, "**** AMDGPU Device Coredump ****\n");
-	drm_printf(&p, "version: " AMDGPU_COREDUMP_VERSION "\n");
-	drm_printf(&p, "kernel: " UTS_RELEASE "\n");
-	drm_printf(&p, "module: " KBUILD_MODNAME "\n");
-	drm_printf(&p, "time: %lld.%09ld\n", coredump->reset_time.tv_sec,
-			coredump->reset_time.tv_nsec);
-
-	if (coredump->reset_task_info.pid)
-		drm_printf(&p, "process_name: %s PID: %d\n",
-			   coredump->reset_task_info.process_name,
-			   coredump->reset_task_info.pid);
-
-	if (coredump->ring) {
-		drm_printf(&p, "\nRing timed out details\n");
-		drm_printf(&p, "IP Type: %d Ring Name: %s\n",
-			   coredump->ring->funcs->type,
-			   coredump->ring->name);
-	}
-
-	if (coredump->reset_vram_lost)
-		drm_printf(&p, "VRAM is lost due to GPU reset!\n");
-	if (coredump->adev->reset_info.num_regs) {
-		drm_printf(&p, "AMDGPU register dumps:\nOffset:     Value:\n");
-
-		for (i = 0; i < coredump->adev->reset_info.num_regs; i++)
-			drm_printf(&p, "0x%08x: 0x%08x\n",
-				   coredump->adev->reset_info.reset_dump_reg_list[i],
-				   coredump->adev->reset_info.reset_dump_reg_value[i]);
-	}
-
-	return count - iter.remain;
-}
-
-static void amdgpu_devcoredump_free(void *data)
-{
-	kfree(data);
-}
-
-void amdgpu_coredump(struct amdgpu_device *adev, bool vram_lost,
-		     struct amdgpu_reset_context *reset_context)
-{
-	struct amdgpu_coredump_info *coredump;
-	struct drm_device *dev = adev_to_drm(adev);
-	struct amdgpu_job *job = reset_context->job;
-	struct drm_sched_job *s_job;
-
-	coredump = kzalloc(sizeof(*coredump), GFP_NOWAIT);
-
-	if (!coredump) {
-		DRM_ERROR("%s: failed to allocate memory for coredump\n", __func__);
-		return;
-	}
-
-	coredump->reset_vram_lost = vram_lost;
-
-	if (reset_context->job && reset_context->job->vm) {
-		struct amdgpu_task_info *ti;
-		struct amdgpu_vm *vm = reset_context->job->vm;
-
-		ti = amdgpu_vm_get_task_info_vm(vm);
-		if (ti) {
-			coredump->reset_task_info = *ti;
-			amdgpu_vm_put_task_info(ti);
-		}
-	}
-
-	if (job) {
-		s_job = &job->base;
-		coredump->ring = to_amdgpu_ring(s_job->sched);
-	}
-
-	coredump->adev = adev;
-
-	ktime_get_ts64(&coredump->reset_time);
-
-	dev_coredumpm(dev->dev, THIS_MODULE, coredump, 0, GFP_NOWAIT,
-		      amdgpu_devcoredump_read, amdgpu_devcoredump_free);
-}
-#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h
index 60522963aaca..b11d190ece53 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h
@@ -32,6 +32,7 @@ enum AMDGPU_RESET_FLAGS {
 
 	AMDGPU_NEED_FULL_RESET = 0,
 	AMDGPU_SKIP_HW_RESET = 1,
+	AMDGPU_SKIP_COREDUMP = 2,
 };
 
 struct amdgpu_reset_context {
@@ -88,19 +89,6 @@ struct amdgpu_reset_domain {
 	atomic_t reset_res;
 };
 
-#ifdef CONFIG_DEV_COREDUMP
-
-#define AMDGPU_COREDUMP_VERSION "1"
-
-struct amdgpu_coredump_info {
-	struct amdgpu_device		*adev;
-	struct amdgpu_task_info         reset_task_info;
-	struct timespec64               reset_time;
-	bool                            reset_vram_lost;
-	struct amdgpu_ring			*ring;
-};
-#endif
-
 int amdgpu_reset_init(struct amdgpu_device *adev);
 int amdgpu_reset_fini(struct amdgpu_device *adev);
 
@@ -141,9 +129,6 @@ void amdgpu_device_lock_reset_domain(struct amdgpu_reset_domain *reset_domain);
 
 void amdgpu_device_unlock_reset_domain(struct amdgpu_reset_domain *reset_domain);
 
-void amdgpu_coredump(struct amdgpu_device *adev, bool vram_lost,
-		     struct amdgpu_reset_context *reset_context);
-
 #define for_each_handler(i, handler, reset_ctl)                  \
 	for (i = 0; (i < AMDGPU_RESET_MAX_HANDLERS) &&           \
 		    (handler = (*reset_ctl->reset_handlers)[i]); \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
index 173a2a308078..b51a82e711df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
@@ -132,7 +132,7 @@ struct amdgpu_buffer_funcs {
 				 uint64_t dst_offset,
 				 /* number of byte to transfer */
 				 uint32_t byte_count,
-				 bool tmz);
+				 uint32_t copy_flags);
 
 	/* maximum bytes in a single operation */
 	uint32_t	fill_max_bytes;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h
index ff4435181055..ec9d12f85f39 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h
@@ -44,6 +44,7 @@ struct amdgpu_smuio_funcs {
 	u32 (*get_socket_id)(struct amdgpu_device *adev);
 	enum amdgpu_pkg_type (*get_pkg_type)(struct amdgpu_device *adev);
 	bool (*is_host_gpu_xgmi_supported)(struct amdgpu_device *adev);
+	u64 (*get_gpu_clock_counter)(struct amdgpu_device *adev);
 };
 
 struct amdgpu_smuio {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index f539b1d00234..7aafeb763e5d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -178,10 +178,10 @@ TRACE_EVENT(amdgpu_cs_ioctl,
 
 	    TP_fast_assign(
 			   __entry->sched_job_id = job->base.id;
-			   __assign_str(timeline, AMDGPU_JOB_GET_TIMELINE_NAME(job));
+			   __assign_str(timeline);
 			   __entry->context = job->base.s_fence->finished.context;
 			   __entry->seqno = job->base.s_fence->finished.seqno;
-			   __assign_str(ring, to_amdgpu_ring(job->base.sched)->name);
+			   __assign_str(ring);
 			   __entry->num_ibs = job->num_ibs;
 			   ),
 	    TP_printk("sched_job=%llu, timeline=%s, context=%u, seqno=%u, ring_name=%s, num_ibs=%u",
@@ -203,10 +203,10 @@ TRACE_EVENT(amdgpu_sched_run_job,
 
 	    TP_fast_assign(
 			   __entry->sched_job_id = job->base.id;
-			   __assign_str(timeline, AMDGPU_JOB_GET_TIMELINE_NAME(job));
+			   __assign_str(timeline);
 			   __entry->context = job->base.s_fence->finished.context;
 			   __entry->seqno = job->base.s_fence->finished.seqno;
-			   __assign_str(ring, to_amdgpu_ring(job->base.sched)->name);
+			   __assign_str(ring);
 			   __entry->num_ibs = job->num_ibs;
 			   ),
 	    TP_printk("sched_job=%llu, timeline=%s, context=%u, seqno=%u, ring_name=%s, num_ibs=%u",
@@ -231,7 +231,7 @@ TRACE_EVENT(amdgpu_vm_grab_id,
 
 	    TP_fast_assign(
 			   __entry->pasid = vm->pasid;
-			   __assign_str(ring, ring->name);
+			   __assign_str(ring);
 			   __entry->vmid = job->vmid;
 			   __entry->vm_hub = ring->vm_hub,
 			   __entry->pd_addr = job->vm_pd_addr;
@@ -425,7 +425,7 @@ TRACE_EVENT(amdgpu_vm_flush,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(ring, ring->name);
+			   __assign_str(ring);
 			   __entry->vmid = vmid;
 			   __entry->vm_hub = ring->vm_hub;
 			   __entry->pd_addr = pd_addr;
@@ -526,7 +526,7 @@ TRACE_EVENT(amdgpu_ib_pipe_sync,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(ring, sched_job->base.sched->name);
+			   __assign_str(ring);
 			   __entry->id = sched_job->base.id;
 			   __entry->fence = fence;
 			   __entry->ctx = fence->context;
@@ -563,7 +563,7 @@ TRACE_EVENT(amdgpu_runpm_reference_dumps,
 			     ),
 	    TP_fast_assign(
 			   __entry->index = index;
-			   __assign_str(func, func);
+			   __assign_str(func);
 			   ),
 	    TP_printk("amdgpu runpm reference dump 0x%x: 0x%s\n",
 		      __entry->index,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index fc418e670fda..e785f128411d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -133,7 +133,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 
 		} else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
 			   !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
-			   amdgpu_bo_in_cpu_visible_vram(abo)) {
+			   amdgpu_res_cpu_visible(adev, bo->resource)) {
 
 			/* Try evicting to the CPU inaccessible part of VRAM
 			 * first, but only set GTT as busy placement, so this
@@ -236,7 +236,7 @@ static int amdgpu_ttm_map_buffer(struct ttm_buffer_object *bo,
 	dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
 	dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
 	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
-				dst_addr, num_bytes, false);
+				dst_addr, num_bytes, 0);
 
 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 	WARN_ON(job->ibs[0].length_dw > num_dw);
@@ -296,6 +296,8 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 	struct dma_fence *fence = NULL;
 	int r = 0;
 
+	uint32_t copy_flags = 0;
+
 	if (!adev->mman.buffer_funcs_enabled) {
 		DRM_ERROR("Trying to move memory with ring turned off.\n");
 		return -EINVAL;
@@ -323,8 +325,11 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 		if (r)
 			goto error;
 
-		r = amdgpu_copy_buffer(ring, from, to, cur_size,
-				       resv, &next, false, true, tmz);
+		if (tmz)
+			copy_flags |= AMDGPU_COPY_FLAGS_TMZ;
+
+		r = amdgpu_copy_buffer(ring, from, to, cur_size, resv,
+				       &next, false, true, copy_flags);
 		if (r)
 			goto error;
 
@@ -378,11 +383,12 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 	    (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) {
 		struct dma_fence *wipe_fence = NULL;
 
-		r = amdgpu_fill_buffer(abo, AMDGPU_POISON, NULL, &wipe_fence,
-					false);
+		r = amdgpu_fill_buffer(abo, 0, NULL, &wipe_fence,
+				       false);
 		if (r) {
 			goto error;
 		} else if (wipe_fence) {
+			amdgpu_vram_mgr_set_cleared(bo->resource);
 			dma_fence_put(fence);
 			fence = wipe_fence;
 		}
@@ -403,40 +409,55 @@ error:
 	return r;
 }
 
-/*
- * amdgpu_mem_visible - Check that memory can be accessed by ttm_bo_move_memcpy
+/**
+ * amdgpu_res_cpu_visible - Check that resource can be accessed by CPU
+ * @adev: amdgpu device
+ * @res: the resource to check
  *
- * Called by amdgpu_bo_move()
+ * Returns: true if the full resource is CPU visible, false otherwise.
  */
-static bool amdgpu_mem_visible(struct amdgpu_device *adev,
-			       struct ttm_resource *mem)
+bool amdgpu_res_cpu_visible(struct amdgpu_device *adev,
+			    struct ttm_resource *res)
 {
-	u64 mem_size = (u64)mem->size;
 	struct amdgpu_res_cursor cursor;
-	u64 end;
 
-	if (mem->mem_type == TTM_PL_SYSTEM ||
-	    mem->mem_type == TTM_PL_TT)
+	if (!res)
+		return false;
+
+	if (res->mem_type == TTM_PL_SYSTEM || res->mem_type == TTM_PL_TT ||
+	    res->mem_type == AMDGPU_PL_PREEMPT || res->mem_type == AMDGPU_PL_DOORBELL)
 		return true;
-	if (mem->mem_type != TTM_PL_VRAM)
+
+	if (res->mem_type != TTM_PL_VRAM)
 		return false;
 
-	amdgpu_res_first(mem, 0, mem_size, &cursor);
-	end = cursor.start + cursor.size;
+	amdgpu_res_first(res, 0, res->size, &cursor);
 	while (cursor.remaining) {
+		if ((cursor.start + cursor.size) > adev->gmc.visible_vram_size)
+			return false;
 		amdgpu_res_next(&cursor, cursor.size);
+	}
 
-		if (!cursor.remaining)
-			break;
+	return true;
+}
 
-		/* ttm_resource_ioremap only supports contiguous memory */
-		if (end != cursor.start)
-			return false;
+/*
+ * amdgpu_res_copyable - Check that memory can be accessed by ttm_bo_move_memcpy
+ *
+ * Called by amdgpu_bo_move()
+ */
+static bool amdgpu_res_copyable(struct amdgpu_device *adev,
+				struct ttm_resource *mem)
+{
+	if (!amdgpu_res_cpu_visible(adev, mem))
+		return false;
 
-		end = cursor.start + cursor.size;
-	}
+	/* ttm_resource_ioremap only supports contiguous memory */
+	if (mem->mem_type == TTM_PL_VRAM &&
+	    !(mem->placement & TTM_PL_FLAG_CONTIGUOUS))
+		return false;
 
-	return end <= adev->gmc.visible_vram_size;
+	return true;
 }
 
 /*
@@ -466,14 +487,16 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 
 	if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM &&
 			 bo->ttm == NULL)) {
+		amdgpu_bo_move_notify(bo, evict, new_mem);
 		ttm_bo_move_null(bo, new_mem);
-		goto out;
+		return 0;
 	}
 	if (old_mem->mem_type == TTM_PL_SYSTEM &&
 	    (new_mem->mem_type == TTM_PL_TT ||
 	     new_mem->mem_type == AMDGPU_PL_PREEMPT)) {
+		amdgpu_bo_move_notify(bo, evict, new_mem);
 		ttm_bo_move_null(bo, new_mem);
-		goto out;
+		return 0;
 	}
 	if ((old_mem->mem_type == TTM_PL_TT ||
 	     old_mem->mem_type == AMDGPU_PL_PREEMPT) &&
@@ -483,9 +506,10 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 			return r;
 
 		amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm);
+		amdgpu_bo_move_notify(bo, evict, new_mem);
 		ttm_resource_free(bo, &bo->resource);
 		ttm_bo_assign_mem(bo, new_mem);
-		goto out;
+		return 0;
 	}
 
 	if (old_mem->mem_type == AMDGPU_PL_GDS ||
@@ -497,8 +521,9 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 	    new_mem->mem_type == AMDGPU_PL_OA ||
 	    new_mem->mem_type == AMDGPU_PL_DOORBELL) {
 		/* Nothing to save here */
+		amdgpu_bo_move_notify(bo, evict, new_mem);
 		ttm_bo_move_null(bo, new_mem);
-		goto out;
+		return 0;
 	}
 
 	if (bo->type == ttm_bo_type_device &&
@@ -510,27 +535,28 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 		abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 	}
 
-	if (adev->mman.buffer_funcs_enabled) {
-		if (((old_mem->mem_type == TTM_PL_SYSTEM &&
-		      new_mem->mem_type == TTM_PL_VRAM) ||
-		     (old_mem->mem_type == TTM_PL_VRAM &&
-		      new_mem->mem_type == TTM_PL_SYSTEM))) {
-			hop->fpfn = 0;
-			hop->lpfn = 0;
-			hop->mem_type = TTM_PL_TT;
-			hop->flags = TTM_PL_FLAG_TEMPORARY;
-			return -EMULTIHOP;
-		}
+	if (adev->mman.buffer_funcs_enabled &&
+	    ((old_mem->mem_type == TTM_PL_SYSTEM &&
+	      new_mem->mem_type == TTM_PL_VRAM) ||
+	     (old_mem->mem_type == TTM_PL_VRAM &&
+	      new_mem->mem_type == TTM_PL_SYSTEM))) {
+		hop->fpfn = 0;
+		hop->lpfn = 0;
+		hop->mem_type = TTM_PL_TT;
+		hop->flags = TTM_PL_FLAG_TEMPORARY;
+		return -EMULTIHOP;
+	}
 
+	amdgpu_bo_move_notify(bo, evict, new_mem);
+	if (adev->mman.buffer_funcs_enabled)
 		r = amdgpu_move_blit(bo, evict, new_mem, old_mem);
-	} else {
+	else
 		r = -ENODEV;
-	}
 
 	if (r) {
 		/* Check that all memory is CPU accessible */
-		if (!amdgpu_mem_visible(adev, old_mem) ||
-		    !amdgpu_mem_visible(adev, new_mem)) {
+		if (!amdgpu_res_copyable(adev, old_mem) ||
+		    !amdgpu_res_copyable(adev, new_mem)) {
 			pr_err("Move buffer fallback to memcpy unavailable\n");
 			return r;
 		}
@@ -540,11 +566,10 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 			return r;
 	}
 
-	trace_amdgpu_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
-out:
-	/* update statistics */
+	/* update statistics after the move */
+	if (evict)
+		atomic64_inc(&adev->num_evictions);
 	atomic64_add(bo->base.size, &adev->num_bytes_moved);
-	amdgpu_bo_move_notify(bo, evict);
 	return 0;
 }
 
@@ -557,7 +582,6 @@ static int amdgpu_ttm_io_mem_reserve(struct ttm_device *bdev,
 				     struct ttm_resource *mem)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
-	size_t bus_size = (size_t)mem->size;
 
 	switch (mem->mem_type) {
 	case TTM_PL_SYSTEM:
@@ -568,9 +592,6 @@ static int amdgpu_ttm_io_mem_reserve(struct ttm_device *bdev,
 		break;
 	case TTM_PL_VRAM:
 		mem->bus.offset = mem->start << PAGE_SHIFT;
-		/* check if it's visible */
-		if ((mem->bus.offset + bus_size) > adev->gmc.visible_vram_size)
-			return -EINVAL;
 
 		if (adev->mman.aper_base_kaddr &&
 		    mem->placement & TTM_PL_FLAG_CONTIGUOUS)
@@ -1477,7 +1498,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
 		swap(src_addr, dst_addr);
 
 	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, dst_addr,
-				PAGE_SIZE, false);
+				PAGE_SIZE, 0);
 
 	amdgpu_ring_pad_ib(adev->mman.buffer_funcs_ring, &job->ibs[0]);
 	WARN_ON(job->ibs[0].length_dw > num_dw);
@@ -1548,7 +1569,7 @@ static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
 static void
 amdgpu_bo_delete_mem_notify(struct ttm_buffer_object *bo)
 {
-	amdgpu_bo_move_notify(bo, false);
+	amdgpu_bo_move_notify(bo, false, NULL);
 }
 
 static struct ttm_device_funcs amdgpu_bo_driver = {
@@ -2128,7 +2149,7 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
 		       uint64_t dst_offset, uint32_t byte_count,
 		       struct dma_resv *resv,
 		       struct dma_fence **fence, bool direct_submit,
-		       bool vm_needs_flush, bool tmz)
+		       bool vm_needs_flush, uint32_t copy_flags)
 {
 	struct amdgpu_device *adev = ring->adev;
 	unsigned int num_loops, num_dw;
@@ -2154,8 +2175,7 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
 		uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
 
 		amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
-					dst_offset, cur_size_in_bytes, tmz);
-
+					dst_offset, cur_size_in_bytes, copy_flags);
 		src_offset += cur_size_in_bytes;
 		dst_offset += cur_size_in_bytes;
 		byte_count -= cur_size_in_bytes;
@@ -2215,6 +2235,71 @@ static int amdgpu_ttm_fill_mem(struct amdgpu_ring *ring, uint32_t src_data,
 	return 0;
 }
 
+/**
+ * amdgpu_ttm_clear_buffer - clear memory buffers
+ * @bo: amdgpu buffer object
+ * @resv: reservation object
+ * @fence: dma_fence associated with the operation
+ *
+ * Clear the memory buffer resource.
+ *
+ * Returns:
+ * 0 for success or a negative error code on failure.
+ */
+int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
+			    struct dma_resv *resv,
+			    struct dma_fence **fence)
+{
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+	struct amdgpu_res_cursor cursor;
+	u64 addr;
+	int r;
+
+	if (!adev->mman.buffer_funcs_enabled)
+		return -EINVAL;
+
+	if (!fence)
+		return -EINVAL;
+
+	*fence = dma_fence_get_stub();
+
+	amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &cursor);
+
+	mutex_lock(&adev->mman.gtt_window_lock);
+	while (cursor.remaining) {
+		struct dma_fence *next = NULL;
+		u64 size;
+
+		if (amdgpu_res_cleared(&cursor)) {
+			amdgpu_res_next(&cursor, cursor.size);
+			continue;
+		}
+
+		/* Never clear more than 256MiB at once to avoid timeouts */
+		size = min(cursor.size, 256ULL << 20);
+
+		r = amdgpu_ttm_map_buffer(&bo->tbo, bo->tbo.resource, &cursor,
+					  1, ring, false, &size, &addr);
+		if (r)
+			goto err;
+
+		r = amdgpu_ttm_fill_mem(ring, 0, addr, size, resv,
+					&next, true, true);
+		if (r)
+			goto err;
+
+		dma_fence_put(*fence);
+		*fence = next;
+
+		amdgpu_res_next(&cursor, size);
+	}
+err:
+	mutex_unlock(&adev->mman.gtt_window_lock);
+
+	return r;
+}
+
 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 			uint32_t src_data,
 			struct dma_resv *resv,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 65ec82141a8e..b6f53129dea3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -38,8 +38,6 @@
 #define AMDGPU_GTT_MAX_TRANSFER_SIZE	512
 #define AMDGPU_GTT_NUM_TRANSFER_WINDOWS	2
 
-#define AMDGPU_POISON	0xd0bed0be
-
 extern const struct attribute_group amdgpu_vram_mgr_attr_group;
 extern const struct attribute_group amdgpu_gtt_mgr_attr_group;
 
@@ -111,6 +109,8 @@ struct amdgpu_copy_mem {
 	unsigned long			offset;
 };
 
+#define AMDGPU_COPY_FLAGS_TMZ		(1 << 0)
+
 int amdgpu_gtt_mgr_init(struct amdgpu_device *adev, uint64_t gtt_size);
 void amdgpu_gtt_mgr_fini(struct amdgpu_device *adev);
 int amdgpu_preempt_mgr_init(struct amdgpu_device *adev);
@@ -139,6 +139,9 @@ int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr,
 int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr,
 				      uint64_t start);
 
+bool amdgpu_res_cpu_visible(struct amdgpu_device *adev,
+			    struct ttm_resource *res);
+
 int amdgpu_ttm_init(struct amdgpu_device *adev);
 void amdgpu_ttm_fini(struct amdgpu_device *adev);
 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev,
@@ -148,13 +151,16 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
 		       uint64_t dst_offset, uint32_t byte_count,
 		       struct dma_resv *resv,
 		       struct dma_fence **fence, bool direct_submit,
-		       bool vm_needs_flush, bool tmz);
+		       bool vm_needs_flush, uint32_t copy_flags);
 int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 			       const struct amdgpu_copy_mem *src,
 			       const struct amdgpu_copy_mem *dst,
 			       uint64_t size, bool tmz,
 			       struct dma_resv *resv,
 			       struct dma_fence **f);
+int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
+			    struct dma_resv *resv,
+			    struct dma_fence **fence);
 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 			uint32_t src_data,
 			struct dma_resv *resv,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
index 619445760037..105d4de0613a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
@@ -125,6 +125,7 @@ enum psp_fw_type {
 	PSP_FW_TYPE_PSP_INTF_DRV,
 	PSP_FW_TYPE_PSP_DBG_DRV,
 	PSP_FW_TYPE_PSP_RAS_DRV,
+	PSP_FW_TYPE_PSP_IPKEYMGR_DRV,
 	PSP_FW_TYPE_MAX_INDEX,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
index 20436f81856a..540e0f066b26 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
@@ -21,10 +21,13 @@
  *
  */
 
+#include <linux/sort.h>
 #include "amdgpu.h"
 #include "umc_v6_7.h"
 #define MAX_UMC_POISON_POLLING_TIME_SYNC   20  //ms
 
+#define MAX_UMC_HASH_STRING_SIZE  256
+
 static int amdgpu_umc_convert_error_address(struct amdgpu_device *adev,
 				    struct ras_err_data *err_data, uint64_t err_addr,
 				    uint32_t ch_inst, uint32_t umc_inst)
@@ -63,6 +66,8 @@ int amdgpu_umc_page_retirement_mca(struct amdgpu_device *adev,
 		goto out_fini_err_data;
 	}
 
+	err_data.err_addr_len = adev->umc.max_ras_err_cnt_per_query;
+
 	/*
 	 * Translate UMC channel address to Physical address
 	 */
@@ -86,7 +91,7 @@ out_fini_err_data:
 	return ret;
 }
 
-static void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
+void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
 			void *ras_error_status)
 {
 	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
@@ -118,6 +123,8 @@ static void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
 			if(!err_data->err_addr)
 				dev_warn(adev->dev, "Failed to alloc memory for "
 						"umc error address record!\n");
+			else
+				err_data->err_addr_len = adev->umc.max_ras_err_cnt_per_query;
 
 			/* umc query_ras_error_address is also responsible for clearing
 			 * error status
@@ -143,6 +150,8 @@ static void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
 			if(!err_data->err_addr)
 				dev_warn(adev->dev, "Failed to alloc memory for "
 						"umc error address record!\n");
+			else
+				err_data->err_addr_len = adev->umc.max_ras_err_cnt_per_query;
 
 			/* umc query_ras_error_address is also responsible for clearing
 			 * error status
@@ -170,6 +179,7 @@ static void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
 	}
 
 	kfree(err_data->err_addr);
+	err_data->err_addr = NULL;
 
 	mutex_unlock(&con->page_retirement_lock);
 }
@@ -177,7 +187,7 @@ static void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
 static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev,
 		void *ras_error_status,
 		struct amdgpu_iv_entry *entry,
-		bool reset)
+		uint32_t reset)
 {
 	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
@@ -186,9 +196,7 @@ static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev,
 	amdgpu_umc_handle_bad_pages(adev, ras_error_status);
 
 	if (err_data->ue_count && reset) {
-		/* use mode-2 reset for poison consumption */
-		if (!entry)
-			con->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE2_RESET;
+		con->gpu_reset_flags |= reset;
 		amdgpu_ras_reset_gpu(adev);
 	}
 
@@ -196,7 +204,7 @@ static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev,
 }
 
 int amdgpu_umc_bad_page_polling_timeout(struct amdgpu_device *adev,
-			bool reset, uint32_t timeout_ms)
+			uint32_t reset, uint32_t timeout_ms)
 {
 	struct ras_err_data err_data;
 	struct ras_common_if head = {
@@ -238,16 +246,16 @@ int amdgpu_umc_bad_page_polling_timeout(struct amdgpu_device *adev,
 	if (reset) {
 		struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 
-		/* use mode-2 reset for poison consumption */
-		con->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE2_RESET;
+		con->gpu_reset_flags |= reset;
 		amdgpu_ras_reset_gpu(adev);
 	}
 
 	return 0;
 }
 
-int amdgpu_umc_poison_handler(struct amdgpu_device *adev,
-			enum amdgpu_ras_block block, bool reset)
+int amdgpu_umc_pasid_poison_handler(struct amdgpu_device *adev,
+			enum amdgpu_ras_block block, uint16_t pasid,
+			pasid_notify pasid_fn, void *data, uint32_t reset)
 {
 	int ret = AMDGPU_RAS_SUCCESS;
 
@@ -285,16 +293,14 @@ int amdgpu_umc_poison_handler(struct amdgpu_device *adev,
 
 			amdgpu_ras_error_data_fini(&err_data);
 		} else {
-			if (reset) {
-				amdgpu_umc_bad_page_polling_timeout(adev,
-							reset, MAX_UMC_POISON_POLLING_TIME_SYNC);
-			} else {
 				struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 
+				amdgpu_ras_put_poison_req(adev,
+					block, pasid, pasid_fn, data, reset);
+
 				atomic_inc(&con->page_retirement_req_cnt);
 
 				wake_up(&con->page_retirement_wq);
-			}
 		}
 	} else {
 		if (adev->virt.ops && adev->virt.ops->ras_poison_handler)
@@ -307,11 +313,19 @@ int amdgpu_umc_poison_handler(struct amdgpu_device *adev,
 	return ret;
 }
 
+int amdgpu_umc_poison_handler(struct amdgpu_device *adev,
+			enum amdgpu_ras_block block, uint32_t reset)
+{
+	return amdgpu_umc_pasid_poison_handler(adev,
+				block, 0, NULL, NULL, reset);
+}
+
 int amdgpu_umc_process_ras_data_cb(struct amdgpu_device *adev,
 		void *ras_error_status,
 		struct amdgpu_iv_entry *entry)
 {
-	return amdgpu_umc_do_page_retirement(adev, ras_error_status, entry, true);
+	return amdgpu_umc_do_page_retirement(adev, ras_error_status, entry,
+				AMDGPU_RAS_GPU_RESET_MODE1_RESET);
 }
 
 int amdgpu_umc_ras_sw_init(struct amdgpu_device *adev)
@@ -388,14 +402,20 @@ int amdgpu_umc_process_ecc_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
-void amdgpu_umc_fill_error_record(struct ras_err_data *err_data,
+int amdgpu_umc_fill_error_record(struct ras_err_data *err_data,
 		uint64_t err_addr,
 		uint64_t retired_page,
 		uint32_t channel_index,
 		uint32_t umc_inst)
 {
-	struct eeprom_table_record *err_rec =
-		&err_data->err_addr[err_data->err_addr_cnt];
+	struct eeprom_table_record *err_rec;
+
+	if (!err_data ||
+	    !err_data->err_addr ||
+	    (err_data->err_addr_cnt >= err_data->err_addr_len))
+		return -EINVAL;
+
+	err_rec = &err_data->err_addr[err_data->err_addr_cnt];
 
 	err_rec->address = err_addr;
 	/* page frame address is saved */
@@ -407,6 +427,8 @@ void amdgpu_umc_fill_error_record(struct ras_err_data *err_data,
 	err_rec->mcumc_id = umc_inst;
 
 	err_data->err_addr_cnt++;
+
+	return 0;
 }
 
 int amdgpu_umc_loop_channels(struct amdgpu_device *adev,
@@ -439,3 +461,76 @@ int amdgpu_umc_loop_channels(struct amdgpu_device *adev,
 
 	return 0;
 }
+
+int amdgpu_umc_update_ecc_status(struct amdgpu_device *adev,
+				uint64_t status, uint64_t ipid, uint64_t addr)
+{
+	if (adev->umc.ras->update_ecc_status)
+		return adev->umc.ras->update_ecc_status(adev,
+					status, ipid, addr);
+	return 0;
+}
+
+static int amdgpu_umc_uint64_cmp(const void *a, const void *b)
+{
+	uint64_t *addr_a = (uint64_t *)a;
+	uint64_t *addr_b = (uint64_t *)b;
+
+	if (*addr_a > *addr_b)
+		return 1;
+	else if (*addr_a < *addr_b)
+		return -1;
+	else
+		return 0;
+}
+
+/* Use string hash to avoid logging the same bad pages repeatedly */
+int amdgpu_umc_build_pages_hash(struct amdgpu_device *adev,
+		uint64_t *pfns, int len, uint64_t *val)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	char buf[MAX_UMC_HASH_STRING_SIZE] = {0};
+	int offset = 0, i = 0;
+	uint64_t hash_val;
+
+	if (!pfns || !len)
+		return -EINVAL;
+
+	sort(pfns, len, sizeof(uint64_t), amdgpu_umc_uint64_cmp, NULL);
+
+	for (i = 0; i < len; i++)
+		offset += snprintf(&buf[offset], sizeof(buf) - offset, "%llx", pfns[i]);
+
+	hash_val = siphash(buf, offset, &con->umc_ecc_log.ecc_key);
+
+	*val = hash_val;
+
+	return 0;
+}
+
+int amdgpu_umc_logs_ecc_err(struct amdgpu_device *adev,
+		struct radix_tree_root *ecc_tree, struct ras_ecc_err *ecc_err)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_ecc_log_info *ecc_log;
+	int ret;
+
+	ecc_log = &con->umc_ecc_log;
+
+	mutex_lock(&ecc_log->lock);
+	ret = radix_tree_insert(ecc_tree, ecc_err->hash_index, ecc_err);
+	if (!ret) {
+		struct ras_err_pages *err_pages = &ecc_err->err_pages;
+		int i;
+
+		/* Reserve memory */
+		for (i = 0; i < err_pages->count; i++)
+			amdgpu_ras_reserve_page(adev, err_pages->pfn[i]);
+
+		radix_tree_tag_set(ecc_tree,
+			ecc_err->hash_index, UMC_ECC_NEW_DETECTED_TAG);
+	}
+	mutex_unlock(&ecc_log->lock);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
index 26d2ae498daf..5f50c69c3cec 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
@@ -52,6 +52,8 @@
 #define LOOP_UMC_EACH_NODE_INST_AND_CH(node_inst, umc_inst, ch_inst) \
 		LOOP_UMC_NODE_INST((node_inst)) LOOP_UMC_INST_AND_CH((umc_inst), (ch_inst))
 
+/* Page retirement tag */
+#define UMC_ECC_NEW_DETECTED_TAG       0x1
 
 typedef int (*umc_func)(struct amdgpu_device *adev, uint32_t node_inst,
 			uint32_t umc_inst, uint32_t ch_inst, void *data);
@@ -66,8 +68,8 @@ struct amdgpu_umc_ras {
 					void *ras_error_status);
 	bool (*check_ecc_err_status)(struct amdgpu_device *adev,
 			enum amdgpu_mca_error_type type, void *ras_error_status);
-	/* support different eeprom table version for different asic */
-	void (*set_eeprom_table_version)(struct amdgpu_ras_eeprom_table_header *hdr);
+	int (*update_ecc_status)(struct amdgpu_device *adev,
+			uint64_t status, uint64_t ipid, uint64_t addr);
 };
 
 struct amdgpu_umc_funcs {
@@ -103,11 +105,14 @@ struct amdgpu_umc {
 int amdgpu_umc_ras_sw_init(struct amdgpu_device *adev);
 int amdgpu_umc_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block);
 int amdgpu_umc_poison_handler(struct amdgpu_device *adev,
-			enum amdgpu_ras_block block, bool reset);
+			enum amdgpu_ras_block block, uint32_t reset);
+int amdgpu_umc_pasid_poison_handler(struct amdgpu_device *adev,
+			enum amdgpu_ras_block block, uint16_t pasid,
+			pasid_notify pasid_fn, void *data, uint32_t reset);
 int amdgpu_umc_process_ecc_irq(struct amdgpu_device *adev,
 		struct amdgpu_irq_src *source,
 		struct amdgpu_iv_entry *entry);
-void amdgpu_umc_fill_error_record(struct ras_err_data *err_data,
+int amdgpu_umc_fill_error_record(struct ras_err_data *err_data,
 		uint64_t err_addr,
 		uint64_t retired_page,
 		uint32_t channel_index,
@@ -123,5 +128,15 @@ int amdgpu_umc_loop_channels(struct amdgpu_device *adev,
 			umc_func func, void *data);
 
 int amdgpu_umc_bad_page_polling_timeout(struct amdgpu_device *adev,
-			bool reset, uint32_t timeout_ms);
+			uint32_t reset, uint32_t timeout_ms);
+
+int amdgpu_umc_update_ecc_status(struct amdgpu_device *adev,
+				uint64_t status, uint64_t ipid, uint64_t addr);
+int amdgpu_umc_build_pages_hash(struct amdgpu_device *adev,
+		uint64_t *pfns, int len, uint64_t *val);
+int amdgpu_umc_logs_ecc_err(struct amdgpu_device *adev,
+		struct radix_tree_root *ecc_tree, struct ras_ecc_err *ecc_err);
+
+void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
+			void *ras_error_status);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c
index 0df97c3e3a70..e01c1c8e64c4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c
@@ -774,6 +774,9 @@ static int umsch_mm_late_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	if (amdgpu_in_reset(adev) || adev->in_s0ix || adev->in_suspend)
+		return 0;
+
 	return umsch_mm_test(adev);
 }
 
@@ -875,6 +878,8 @@ static const struct amd_ip_funcs umsch_mm_v4_0_ip_funcs = {
 	.hw_fini = umsch_mm_hw_fini,
 	.suspend = umsch_mm_suspend,
 	.resume = umsch_mm_resume,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version umsch_mm_v4_0_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 59acf424a078..968ca2c84ef7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -743,7 +743,8 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p,
 	uint32_t created = 0;
 	uint32_t allocated = 0;
 	uint32_t tmp, handle = 0;
-	uint32_t *size = &tmp;
+	uint32_t dummy = 0xffffffff;
+	uint32_t *size = &dummy;
 	unsigned int idx;
 	int i, r = 0;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index 9c514a606a2f..677eb141554e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -93,7 +93,7 @@ static void amdgpu_vcn_idle_work_handler(struct work_struct *work);
 
 int amdgpu_vcn_early_init(struct amdgpu_device *adev)
 {
-	char ucode_prefix[30];
+	char ucode_prefix[25];
 	char fw_name[40];
 	int r, i;
 
@@ -185,7 +185,10 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev)
 	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
 		bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
 
-	if (amdgpu_ip_version(adev, UVD_HWIP, 0) >= IP_VERSION(4, 0, 0)) {
+	if (amdgpu_ip_version(adev, UVD_HWIP, 0) >= IP_VERSION(5, 0, 0)) {
+		fw_shared_size = AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn5_fw_shared));
+		log_offset = offsetof(struct amdgpu_vcn5_fw_shared, fw_log);
+	} else if (amdgpu_ip_version(adev, UVD_HWIP, 0) >= IP_VERSION(4, 0, 0)) {
 		fw_shared_size = AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared));
 		log_offset = offsetof(struct amdgpu_vcn4_fw_shared, fw_log);
 	} else {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
index a418393d89ec..9f06def236fd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
@@ -454,6 +454,16 @@ struct amdgpu_vcn_rb_metadata {
 	uint8_t pad[26];
 };
 
+struct amdgpu_vcn5_fw_shared {
+	uint32_t present_flag_0;
+	uint8_t pad[12];
+	struct amdgpu_fw_shared_unified_queue_struct sq;
+	uint8_t pad1[8];
+	struct amdgpu_fw_shared_fw_logging fw_log;
+	struct amdgpu_fw_shared_rb_setup rb_setup;
+	uint8_t pad2[4];
+};
+
 #define VCN_BLOCK_ENCODE_DISABLE_MASK 0x80
 #define VCN_BLOCK_DECODE_DISABLE_MASK 0x40
 #define VCN_BLOCK_QUEUE_DISABLE_MASK 0xC0
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 7a4eae36778a..54ab51a4ada7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -32,6 +32,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_ras.h"
+#include "amdgpu_reset.h"
 #include "vi.h"
 #include "soc15.h"
 #include "nv.h"
@@ -424,7 +425,7 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
 		return -EINVAL;
 
 	if (pf2vf_info->size > 1024) {
-		DRM_ERROR("invalid pf2vf message size\n");
+		dev_err(adev->dev, "invalid pf2vf message size: 0x%x\n", pf2vf_info->size);
 		return -EINVAL;
 	}
 
@@ -435,7 +436,9 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
 			adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size,
 			adev->virt.fw_reserve.checksum_key, checksum);
 		if (checksum != checkval) {
-			DRM_ERROR("invalid pf2vf message\n");
+			dev_err(adev->dev,
+				"invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n",
+				checksum, checkval);
 			return -EINVAL;
 		}
 
@@ -449,7 +452,9 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
 			adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size,
 			0, checksum);
 		if (checksum != checkval) {
-			DRM_ERROR("invalid pf2vf message\n");
+			dev_err(adev->dev,
+				"invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n",
+				checksum, checkval);
 			return -EINVAL;
 		}
 
@@ -485,7 +490,7 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
 			((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->uuid;
 		break;
 	default:
-		DRM_ERROR("invalid pf2vf version\n");
+		dev_err(adev->dev, "invalid pf2vf version: 0x%x\n", pf2vf_info->version);
 		return -EINVAL;
 	}
 
@@ -571,6 +576,11 @@ static int amdgpu_virt_write_vf2pf_data(struct amdgpu_device *adev)
 	vf2pf_info->decode_usage = 0;
 
 	vf2pf_info->dummy_page_addr = (uint64_t)adev->dummy_page_addr;
+	vf2pf_info->mes_info_addr = (uint64_t)adev->mes.resource_1_gpu_addr;
+
+	if (adev->mes.resource_1) {
+		vf2pf_info->mes_info_size = adev->mes.resource_1->tbo.base.size;
+	}
 	vf2pf_info->checksum =
 		amd_sriov_msg_checksum(
 		vf2pf_info, vf2pf_info->header.size, 0, 0);
@@ -584,8 +594,22 @@ static void amdgpu_virt_update_vf2pf_work_item(struct work_struct *work)
 	int ret;
 
 	ret = amdgpu_virt_read_pf2vf_data(adev);
-	if (ret)
+	if (ret) {
+		adev->virt.vf2pf_update_retry_cnt++;
+		if ((adev->virt.vf2pf_update_retry_cnt >= AMDGPU_VF2PF_UPDATE_MAX_RETRY_LIMIT) &&
+		    amdgpu_sriov_runtime(adev) && !amdgpu_in_reset(adev)) {
+			amdgpu_ras_set_fed(adev, true);
+			if (amdgpu_reset_domain_schedule(adev->reset_domain,
+							  &adev->virt.flr_work))
+				return;
+			else
+				dev_err(adev->dev, "Failed to queue work! at %s", __func__);
+		}
+
 		goto out;
+	}
+
+	adev->virt.vf2pf_update_retry_cnt = 0;
 	amdgpu_virt_write_vf2pf_data(adev);
 
 out:
@@ -606,6 +630,7 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
 	adev->virt.fw_reserve.p_pf2vf = NULL;
 	adev->virt.fw_reserve.p_vf2pf = NULL;
 	adev->virt.vf2pf_update_interval_ms = 0;
+	adev->virt.vf2pf_update_retry_cnt = 0;
 
 	if (adev->mman.fw_vram_usage_va && adev->mman.drv_vram_usage_va) {
 		DRM_WARN("Currently fw_vram and drv_vram should not have values at the same time!");
@@ -705,12 +730,6 @@ void amdgpu_detect_virtualization(struct amdgpu_device *adev)
 			adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
 	}
 
-	if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
-		/* VF MMIO access (except mailbox range) from CPU
-		 * will be blocked during sriov runtime
-		 */
-		adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT;
-
 	/* we have the ability to check now */
 	if (amdgpu_sriov_vf(adev)) {
 		switch (adev->asic_type) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 3f59b7b5523f..642f1fd287d8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -52,6 +52,8 @@
 /* tonga/fiji use this offset */
 #define mmBIF_IOV_FUNC_IDENTIFIER 0x1503
 
+#define AMDGPU_VF2PF_UPDATE_MAX_RETRY_LIMIT 5
+
 enum amdgpu_sriov_vf_mode {
 	SRIOV_VF_MODE_BARE_METAL = 0,
 	SRIOV_VF_MODE_ONE_VF,
@@ -130,6 +132,8 @@ enum AMDGIM_FEATURE_FLAG {
 	AMDGIM_FEATURE_AV1_SUPPORT = (1 << 6),
 	/* VCN RB decouple */
 	AMDGIM_FEATURE_VCN_RB_DECOUPLE = (1 << 7),
+	/* MES info */
+	AMDGIM_FEATURE_MES_INFO_ENABLE = (1 << 8),
 };
 
 enum AMDGIM_REG_ACCESS_FLAG {
@@ -257,6 +261,7 @@ struct amdgpu_virt {
 	/* vf2pf message */
 	struct delayed_work vf2pf_work;
 	uint32_t vf2pf_update_interval_ms;
+	int vf2pf_update_retry_cnt;
 
 	/* multimedia bandwidth config */
 	bool     is_mm_bw_enabled;
@@ -332,6 +337,8 @@ static inline bool is_virtual_machine(void)
 	((adev)->virt.gim_feature & AMDGIM_FEATURE_AV1_SUPPORT)
 #define amdgpu_sriov_is_vcn_rb_decouple(adev) \
 	((adev)->virt.gim_feature & AMDGIM_FEATURE_VCN_RB_DECOUPLE)
+#define amdgpu_sriov_is_mes_info_enable(adev) \
+	((adev)->virt.gim_feature & AMDGIM_FEATURE_MES_INFO_ENABLE)
 bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev);
 void amdgpu_virt_init_setting(struct amdgpu_device *adev);
 int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
index 8baa2e0935cc..e30eecd02ae1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
@@ -658,6 +658,8 @@ static const struct amd_ip_funcs amdgpu_vkms_ip_funcs = {
 	.soft_reset = amdgpu_vkms_soft_reset,
 	.set_clockgating_state = amdgpu_vkms_set_clockgating_state,
 	.set_powergating_state = amdgpu_vkms_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version amdgpu_vkms_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 4299ce386322..4e2391c83d7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -886,6 +886,44 @@ static void amdgpu_vm_tlb_seq_cb(struct dma_fence *fence,
 }
 
 /**
+ * amdgpu_vm_tlb_flush - prepare TLB flush
+ *
+ * @params: parameters for update
+ * @fence: input fence to sync TLB flush with
+ * @tlb_cb: the callback structure
+ *
+ * Increments the tlb sequence to make sure that future CS execute a VM flush.
+ */
+static void
+amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
+		    struct dma_fence **fence,
+		    struct amdgpu_vm_tlb_seq_struct *tlb_cb)
+{
+	struct amdgpu_vm *vm = params->vm;
+
+	if (!fence || !*fence)
+		return;
+
+	tlb_cb->vm = vm;
+	if (!dma_fence_add_callback(*fence, &tlb_cb->cb,
+				    amdgpu_vm_tlb_seq_cb)) {
+		dma_fence_put(vm->last_tlb_flush);
+		vm->last_tlb_flush = dma_fence_get(*fence);
+	} else {
+		amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb);
+	}
+
+	/* Prepare a TLB flush fence to be attached to PTs */
+	if (!params->unlocked && vm->is_compute_context) {
+		amdgpu_vm_tlb_fence_create(params->adev, vm, fence);
+
+		/* Makes sure no PD/PT is freed before the flush */
+		dma_resv_add_fence(vm->root.bo->tbo.base.resv, *fence,
+				   DMA_RESV_USAGE_BOOKKEEP);
+	}
+}
+
+/**
  * amdgpu_vm_update_range - update a range in the vm page table
  *
  * @adev: amdgpu_device pointer to use for commands
@@ -916,8 +954,8 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			   struct ttm_resource *res, dma_addr_t *pages_addr,
 			   struct dma_fence **fence)
 {
-	struct amdgpu_vm_update_params params;
 	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
+	struct amdgpu_vm_update_params params;
 	struct amdgpu_res_cursor cursor;
 	enum amdgpu_sync_mode sync_mode;
 	int r, idx;
@@ -927,8 +965,8 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 	tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
 	if (!tlb_cb) {
-		r = -ENOMEM;
-		goto error_unlock;
+		drm_dev_exit(idx);
+		return -ENOMEM;
 	}
 
 	/* Vega20+XGMI where PTEs get inadvertently cached in L2 texture cache,
@@ -948,7 +986,9 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	params.immediate = immediate;
 	params.pages_addr = pages_addr;
 	params.unlocked = unlocked;
+	params.needs_flush = flush_tlb;
 	params.allow_override = allow_override;
+	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
 
 	/* Implicitly sync to command submissions in the same VM before
 	 * unmapping. Sync to moving fences before mapping.
@@ -1031,24 +1071,18 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	}
 
 	r = vm->update_funcs->commit(&params, fence);
+	if (r)
+		goto error_free;
 
-	if (flush_tlb || params.table_freed) {
-		tlb_cb->vm = vm;
-		if (fence && *fence &&
-		    !dma_fence_add_callback(*fence, &tlb_cb->cb,
-					   amdgpu_vm_tlb_seq_cb)) {
-			dma_fence_put(vm->last_tlb_flush);
-			vm->last_tlb_flush = dma_fence_get(*fence);
-		} else {
-			amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb);
-		}
+	if (params.needs_flush) {
+		amdgpu_vm_tlb_flush(&params, fence, tlb_cb);
 		tlb_cb = NULL;
 	}
 
+	amdgpu_vm_pt_free_list(adev, &params);
+
 error_free:
 	kfree(tlb_cb);
-
-error_unlock:
 	amdgpu_vm_eviction_unlock(vm);
 	drm_dev_exit(idx);
 	return r;
@@ -1613,6 +1647,37 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
 	trace_amdgpu_vm_bo_map(bo_va, mapping);
 }
 
+/* Validate operation parameters to prevent potential abuse */
+static int amdgpu_vm_verify_parameters(struct amdgpu_device *adev,
+					  struct amdgpu_bo *bo,
+					  uint64_t saddr,
+					  uint64_t offset,
+					  uint64_t size)
+{
+	uint64_t tmp, lpfn;
+
+	if (saddr & AMDGPU_GPU_PAGE_MASK
+	    || offset & AMDGPU_GPU_PAGE_MASK
+	    || size & AMDGPU_GPU_PAGE_MASK)
+		return -EINVAL;
+
+	if (check_add_overflow(saddr, size, &tmp)
+	    || check_add_overflow(offset, size, &tmp)
+	    || size == 0 /* which also leads to end < begin */)
+		return -EINVAL;
+
+	/* make sure object fit at this offset */
+	if (bo && offset + size > amdgpu_bo_size(bo))
+		return -EINVAL;
+
+	/* Ensure last pfn not exceed max_pfn */
+	lpfn = (saddr + size - 1) >> AMDGPU_GPU_PAGE_SHIFT;
+	if (lpfn >= adev->vm_manager.max_pfn)
+		return -EINVAL;
+
+	return 0;
+}
+
 /**
  * amdgpu_vm_bo_map - map bo inside a vm
  *
@@ -1639,21 +1704,14 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 	struct amdgpu_bo *bo = bo_va->base.bo;
 	struct amdgpu_vm *vm = bo_va->base.vm;
 	uint64_t eaddr;
+	int r;
 
-	/* validate the parameters */
-	if (saddr & ~PAGE_MASK || offset & ~PAGE_MASK || size & ~PAGE_MASK)
-		return -EINVAL;
-	if (saddr + size <= saddr || offset + size <= offset)
-		return -EINVAL;
-
-	/* make sure object fit at this offset */
-	eaddr = saddr + size - 1;
-	if ((bo && offset + size > amdgpu_bo_size(bo)) ||
-	    (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT))
-		return -EINVAL;
+	r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size);
+	if (r)
+		return r;
 
 	saddr /= AMDGPU_GPU_PAGE_SIZE;
-	eaddr /= AMDGPU_GPU_PAGE_SIZE;
+	eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
 
 	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
 	if (tmp) {
@@ -1706,17 +1764,9 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
 	uint64_t eaddr;
 	int r;
 
-	/* validate the parameters */
-	if (saddr & ~PAGE_MASK || offset & ~PAGE_MASK || size & ~PAGE_MASK)
-		return -EINVAL;
-	if (saddr + size <= saddr || offset + size <= offset)
-		return -EINVAL;
-
-	/* make sure object fit at this offset */
-	eaddr = saddr + size - 1;
-	if ((bo && offset + size > amdgpu_bo_size(bo)) ||
-	    (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT))
-		return -EINVAL;
+	r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size);
+	if (r)
+		return r;
 
 	/* Allocate all the needed memory */
 	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
@@ -1730,7 +1780,7 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
 	}
 
 	saddr /= AMDGPU_GPU_PAGE_SIZE;
-	eaddr /= AMDGPU_GPU_PAGE_SIZE;
+	eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
 
 	mapping->start = saddr;
 	mapping->last = eaddr;
@@ -1817,10 +1867,14 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
 	struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
 	LIST_HEAD(removed);
 	uint64_t eaddr;
+	int r;
+
+	r = amdgpu_vm_verify_parameters(adev, NULL, saddr, 0, size);
+	if (r)
+		return r;
 
-	eaddr = saddr + size - 1;
 	saddr /= AMDGPU_GPU_PAGE_SIZE;
-	eaddr /= AMDGPU_GPU_PAGE_SIZE;
+	eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
 
 	/* Allocate all the needed memory */
 	before = kzalloc(sizeof(*before), GFP_KERNEL);
@@ -2391,6 +2445,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 	mutex_init(&vm->eviction_lock);
 	vm->evicting = false;
+	vm->tlb_fence_context = dma_fence_context_alloc(1);
 
 	r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
 				false, &root, xcp_id);
@@ -2924,6 +2979,14 @@ void amdgpu_vm_update_fault_cache(struct amdgpu_device *adev,
 	if (vm && status) {
 		vm->fault_info.addr = addr;
 		vm->fault_info.status = status;
+		/*
+		 * Update the fault information globally for later usage
+		 * when vm could be stale or freed.
+		 */
+		adev->vm_manager.fault_info.addr = addr;
+		adev->vm_manager.fault_info.vmhub = vmhub;
+		adev->vm_manager.fault_info.status = status;
+
 		if (AMDGPU_IS_GFXHUB(vmhub)) {
 			vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_GFX;
 			vm->fault_info.vmhub |=
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 047ec1930d12..54d7da396de0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -257,15 +257,20 @@ struct amdgpu_vm_update_params {
 	unsigned int num_dw_left;
 
 	/**
-	 * @table_freed: return true if page table is freed when updating
+	 * @needs_flush: true whenever we need to invalidate the TLB
 	 */
-	bool table_freed;
+	bool needs_flush;
 
 	/**
 	 * @allow_override: true for memory that is not uncached: allows MTYPE
 	 * to be overridden for NUMA local memory.
 	 */
 	bool allow_override;
+
+	/**
+	 * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush
+	 */
+	struct list_head tlb_flush_waitlist;
 };
 
 struct amdgpu_vm_update_funcs {
@@ -342,6 +347,7 @@ struct amdgpu_vm {
 	atomic64_t		tlb_seq;
 	struct dma_fence	*last_tlb_flush;
 	atomic64_t		kfd_last_flushed_seq;
+	uint64_t		tlb_fence_context;
 
 	/* How many times we had to re-generate the page tables */
 	uint64_t		generation;
@@ -422,6 +428,8 @@ struct amdgpu_vm_manager {
 	 * look up VM of a page fault
 	 */
 	struct xarray				pasids;
+	/* Global registration of recent page fault information */
+	struct amdgpu_vm_fault_info	fault_info;
 };
 
 struct amdgpu_bo_va_mapping;
@@ -544,6 +552,8 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 			  uint64_t start, uint64_t end,
 			  uint64_t dst, uint64_t flags);
 void amdgpu_vm_pt_free_work(struct work_struct *work);
+void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
+			    struct amdgpu_vm_update_params *params);
 
 #if defined(CONFIG_DEBUG_FS)
 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m);
@@ -609,5 +619,8 @@ void amdgpu_vm_update_fault_cache(struct amdgpu_device *adev,
 				  uint64_t addr,
 				  uint32_t status,
 				  unsigned int vmhub);
+void amdgpu_vm_tlb_fence_create(struct amdgpu_device *adev,
+				 struct amdgpu_vm *vm,
+				 struct dma_fence **fence);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
index 6e31621452de..3895bd7d176a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
@@ -108,7 +108,9 @@ static int amdgpu_vm_cpu_update(struct amdgpu_vm_update_params *p,
 static int amdgpu_vm_cpu_commit(struct amdgpu_vm_update_params *p,
 				struct dma_fence **fence)
 {
-	/* Flush HDP */
+	if (p->needs_flush)
+		atomic64_inc(&p->vm->tlb_seq);
+
 	mb();
 	amdgpu_device_flush_hdp(p->adev, NULL);
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 124389a6bf48..f07647a9a9d9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -622,40 +622,58 @@ void amdgpu_vm_pt_free_work(struct work_struct *work)
 }
 
 /**
- * amdgpu_vm_pt_free_dfs - free PD/PT levels
+ * amdgpu_vm_pt_free_list - free PD/PT levels
  *
  * @adev: amdgpu device structure
- * @vm: amdgpu vm structure
- * @start: optional cursor where to start freeing PDs/PTs
- * @unlocked: vm resv unlock status
+ * @params: see amdgpu_vm_update_params definition
  *
- * Free the page directory or page table level and all sub levels.
+ * Free the page directory objects saved in the flush list
  */
-static void amdgpu_vm_pt_free_dfs(struct amdgpu_device *adev,
-				  struct amdgpu_vm *vm,
-				  struct amdgpu_vm_pt_cursor *start,
-				  bool unlocked)
+void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
+			    struct amdgpu_vm_update_params *params)
 {
-	struct amdgpu_vm_pt_cursor cursor;
-	struct amdgpu_vm_bo_base *entry;
+	struct amdgpu_vm_bo_base *entry, *next;
+	struct amdgpu_vm *vm = params->vm;
+	bool unlocked = params->unlocked;
+
+	if (list_empty(&params->tlb_flush_waitlist))
+		return;
 
 	if (unlocked) {
 		spin_lock(&vm->status_lock);
-		for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)
-			list_move(&entry->vm_status, &vm->pt_freed);
-
-		if (start)
-			list_move(&start->entry->vm_status, &vm->pt_freed);
+		list_splice_init(&params->tlb_flush_waitlist, &vm->pt_freed);
 		spin_unlock(&vm->status_lock);
 		schedule_work(&vm->pt_free_work);
 		return;
 	}
 
-	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)
+	list_for_each_entry_safe(entry, next, &params->tlb_flush_waitlist, vm_status)
 		amdgpu_vm_pt_free(entry);
+}
 
-	if (start)
-		amdgpu_vm_pt_free(start->entry);
+/**
+ * amdgpu_vm_pt_add_list - add PD/PT level to the flush list
+ *
+ * @params: parameters for the update
+ * @cursor: first PT entry to start DF search from, non NULL
+ *
+ * This list will be freed after TLB flush.
+ */
+static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
+				  struct amdgpu_vm_pt_cursor *cursor)
+{
+	struct amdgpu_vm_pt_cursor seek;
+	struct amdgpu_vm_bo_base *entry;
+
+	spin_lock(&params->vm->status_lock);
+	for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) {
+		if (entry && entry->bo)
+			list_move(&entry->vm_status, &params->tlb_flush_waitlist);
+	}
+
+	/* enter start node now */
+	list_move(&cursor->entry->vm_status, &params->tlb_flush_waitlist);
+	spin_unlock(&params->vm->status_lock);
 }
 
 /**
@@ -667,7 +685,13 @@ static void amdgpu_vm_pt_free_dfs(struct amdgpu_device *adev,
  */
 void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 {
-	amdgpu_vm_pt_free_dfs(adev, vm, NULL, false);
+	struct amdgpu_vm_pt_cursor cursor;
+	struct amdgpu_vm_bo_base *entry;
+
+	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry) {
+		if (entry)
+			amdgpu_vm_pt_free(entry);
+	}
 }
 
 /**
@@ -682,11 +706,15 @@ int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
 			 struct amdgpu_vm_bo_base *entry)
 {
 	struct amdgpu_vm_bo_base *parent = amdgpu_vm_pt_parent(entry);
-	struct amdgpu_bo *bo = parent->bo, *pbo;
+	struct amdgpu_bo *bo, *pbo;
 	struct amdgpu_vm *vm = params->vm;
 	uint64_t pde, pt, flags;
 	unsigned int level;
 
+	if (WARN_ON(!parent))
+		return -EINVAL;
+
+	bo = parent->bo;
 	for (level = 0, pbo = bo->parent; pbo; ++level)
 		pbo = pbo->parent;
 
@@ -972,10 +1000,8 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 			while (cursor.pfn < frag_start) {
 				/* Make sure previous mapping is freed */
 				if (cursor.entry->bo) {
-					params->table_freed = true;
-					amdgpu_vm_pt_free_dfs(adev, params->vm,
-							      &cursor,
-							      params->unlocked);
+					params->needs_flush = true;
+					amdgpu_vm_pt_add_list(params, &cursor);
 				}
 				amdgpu_vm_pt_next(adev, &cursor);
 			}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
index 349416e176a1..66e8a016126b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
@@ -126,6 +126,10 @@ static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,
 
 	WARN_ON(ib->length_dw == 0);
 	amdgpu_ring_pad_ib(ring, ib);
+
+	if (p->needs_flush)
+		atomic64_inc(&p->vm->tlb_seq);
+
 	WARN_ON(ib->length_dw > p->num_dw_left);
 	f = amdgpu_job_submit(p->job);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c
new file mode 100644
index 000000000000..51cddfa3f1e8
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c
@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/dma-fence.h>
+#include <linux/workqueue.h>
+
+#include "amdgpu.h"
+#include "amdgpu_vm.h"
+#include "amdgpu_gmc.h"
+
+struct amdgpu_tlb_fence {
+	struct dma_fence	base;
+	struct amdgpu_device	*adev;
+	struct dma_fence	*dependency;
+	struct work_struct	work;
+	spinlock_t		lock;
+	uint16_t		pasid;
+
+};
+
+static const char *amdgpu_tlb_fence_get_driver_name(struct dma_fence *fence)
+{
+	return "amdgpu tlb fence";
+}
+
+static const char *amdgpu_tlb_fence_get_timeline_name(struct dma_fence *f)
+{
+	return "amdgpu tlb timeline";
+}
+
+static void amdgpu_tlb_fence_work(struct work_struct *work)
+{
+	struct amdgpu_tlb_fence *f = container_of(work, typeof(*f), work);
+	int r;
+
+	if (f->dependency) {
+		dma_fence_wait(f->dependency, false);
+		dma_fence_put(f->dependency);
+		f->dependency = NULL;
+	}
+
+	r = amdgpu_gmc_flush_gpu_tlb_pasid(f->adev, f->pasid, 2, true, 0);
+	if (r) {
+		dev_err(f->adev->dev, "TLB flush failed for PASID %d.\n",
+			f->pasid);
+		dma_fence_set_error(&f->base, r);
+	}
+
+	dma_fence_signal(&f->base);
+	dma_fence_put(&f->base);
+}
+
+static const struct dma_fence_ops amdgpu_tlb_fence_ops = {
+	.use_64bit_seqno = true,
+	.get_driver_name = amdgpu_tlb_fence_get_driver_name,
+	.get_timeline_name = amdgpu_tlb_fence_get_timeline_name
+};
+
+void amdgpu_vm_tlb_fence_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+				struct dma_fence **fence)
+{
+	struct amdgpu_tlb_fence *f;
+
+	f = kmalloc(sizeof(*f), GFP_KERNEL);
+	if (!f) {
+		/*
+		 * We can't fail since the PDEs and PTEs are already updated, so
+		 * just block for the dependency and execute the TLB flush
+		 */
+		if (*fence)
+			dma_fence_wait(*fence, false);
+
+		amdgpu_gmc_flush_gpu_tlb_pasid(adev, vm->pasid, 2, true, 0);
+		*fence = dma_fence_get_stub();
+		return;
+	}
+
+	f->adev = adev;
+	f->dependency = *fence;
+	f->pasid = vm->pasid;
+	INIT_WORK(&f->work, amdgpu_tlb_fence_work);
+	spin_lock_init(&f->lock);
+
+	dma_fence_init(&f->base, &amdgpu_tlb_fence_ops, &f->lock,
+		       vm->tlb_fence_context, atomic64_read(&vm->tlb_seq));
+
+	/* TODO: We probably need a separate wq here */
+	dma_fence_get(&f->base);
+	schedule_work(&f->work);
+
+	*fence = &f->base;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
index 6695481f870f..c23d97d34b7e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
@@ -205,7 +205,7 @@ disable_dpm:
 	dpm_ctl &= 0xfffffffe; /* Disable DPM */
 	WREG32(vpe_get_reg_offset(vpe, 0, vpe->regs.dpm_enable), dpm_ctl);
 	dev_dbg(adev->dev, "%s: disable vpe dpm\n", __func__);
-	return 0;
+	return -EINVAL;
 }
 
 int amdgpu_vpe_psp_update_sram(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 8db880244324..6c30eceec896 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -450,6 +450,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 {
 	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
 	struct amdgpu_device *adev = to_amdgpu_device(mgr);
+	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
 	u64 vis_usage = 0, max_bytes, min_block_size;
 	struct amdgpu_vram_mgr_resource *vres;
 	u64 size, remaining_size, lpfn, fpfn;
@@ -468,7 +469,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 	if (tbo->type != ttm_bo_type_kernel)
 		max_bytes -= AMDGPU_VM_RESERVED_VRAM;
 
-	if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
+	if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) {
 		pages_per_block = ~0ul;
 	} else {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -477,7 +478,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 		/* default to 2MB */
 		pages_per_block = 2UL << (20UL - PAGE_SHIFT);
 #endif
-		pages_per_block = max_t(uint32_t, pages_per_block,
+		pages_per_block = max_t(u32, pages_per_block,
 					tbo->page_alignment);
 	}
 
@@ -498,9 +499,12 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 	if (place->flags & TTM_PL_FLAG_TOPDOWN)
 		vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
 
-	if (place->flags & TTM_PL_FLAG_CONTIGUOUS)
+	if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
 		vres->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION;
 
+	if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED)
+		vres->flags |= DRM_BUDDY_CLEAR_ALLOCATION;
+
 	if (fpfn || lpfn != mgr->mm.size)
 		/* Allocate blocks in desired range */
 		vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;
@@ -514,21 +518,31 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 		else
 			min_block_size = mgr->default_page_size;
 
-		BUG_ON(min_block_size < mm->chunk_size);
-
 		/* Limit maximum size to 2GiB due to SG table limitations */
 		size = min(remaining_size, 2ULL << 30);
 
 		if ((size >= (u64)pages_per_block << PAGE_SHIFT) &&
-				!(size & (((u64)pages_per_block << PAGE_SHIFT) - 1)))
+		    !(size & (((u64)pages_per_block << PAGE_SHIFT) - 1)))
 			min_block_size = (u64)pages_per_block << PAGE_SHIFT;
 
+		BUG_ON(min_block_size < mm->chunk_size);
+
 		r = drm_buddy_alloc_blocks(mm, fpfn,
 					   lpfn,
 					   size,
 					   min_block_size,
 					   &vres->blocks,
 					   vres->flags);
+
+		if (unlikely(r == -ENOSPC) && pages_per_block == ~0ul &&
+		    !(place->flags & TTM_PL_FLAG_CONTIGUOUS)) {
+			vres->flags &= ~DRM_BUDDY_CONTIGUOUS_ALLOCATION;
+			pages_per_block = max_t(u32, 2UL << (20UL - PAGE_SHIFT),
+						tbo->page_alignment);
+
+			continue;
+		}
+
 		if (unlikely(r))
 			goto error_free_blocks;
 
@@ -571,7 +585,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 	return 0;
 
 error_free_blocks:
-	drm_buddy_free_list(mm, &vres->blocks);
+	drm_buddy_free_list(mm, &vres->blocks, 0);
 	mutex_unlock(&mgr->lock);
 error_fini:
 	ttm_resource_fini(man, &vres->base);
@@ -604,7 +618,7 @@ static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,
 
 	amdgpu_vram_mgr_do_reserve(man);
 
-	drm_buddy_free_list(mm, &vres->blocks);
+	drm_buddy_free_list(mm, &vres->blocks, vres->flags);
 	mutex_unlock(&mgr->lock);
 
 	atomic64_sub(vis_usage, &mgr->vis_usage);
@@ -912,7 +926,7 @@ void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
 		kfree(rsv);
 
 	list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) {
-		drm_buddy_free_list(&mgr->mm, &rsv->allocated);
+		drm_buddy_free_list(&mgr->mm, &rsv->allocated, 0);
 		kfree(rsv);
 	}
 	if (!adev->gmc.is_app_apu)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h
index 0e04e42cf809..b256cbc2bc27 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h
@@ -53,10 +53,20 @@ static inline u64 amdgpu_vram_mgr_block_size(struct drm_buddy_block *block)
 	return (u64)PAGE_SIZE << drm_buddy_block_order(block);
 }
 
+static inline bool amdgpu_vram_mgr_is_cleared(struct drm_buddy_block *block)
+{
+	return drm_buddy_block_is_clear(block);
+}
+
 static inline struct amdgpu_vram_mgr_resource *
 to_amdgpu_vram_mgr_resource(struct ttm_resource *res)
 {
 	return container_of(res, struct amdgpu_vram_mgr_resource, base);
 }
 
+static inline void amdgpu_vram_mgr_set_cleared(struct ttm_resource *res)
+{
+	to_amdgpu_vram_mgr_resource(res)->flags |= DRM_BUDDY_CLEARED;
+}
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index 20d51f6c9bb8..dd2ec48cf5c2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -1035,15 +1035,16 @@ int amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
 	return 0;
 }
 
-static int xgmi_v6_4_0_aca_bank_generate_report(struct aca_handle *handle, struct aca_bank *bank, enum aca_error_type type,
-						struct aca_bank_report *report, void *data)
+static int xgmi_v6_4_0_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
+				       enum aca_smu_type type, void *data)
 {
 	struct amdgpu_device *adev = handle->adev;
+	struct aca_bank_info info;
 	const char *error_str;
-	u64 status;
+	u64 status, count;
 	int ret, ext_error_code;
 
-	ret = aca_bank_info_decode(bank, &report->info);
+	ret = aca_bank_info_decode(bank, &info);
 	if (ret)
 		return ret;
 
@@ -1055,15 +1056,28 @@ static int xgmi_v6_4_0_aca_bank_generate_report(struct aca_handle *handle, struc
 	if (error_str)
 		dev_info(adev->dev, "%s detected\n", error_str);
 
-	if ((type == ACA_ERROR_TYPE_UE && ext_error_code == 0) ||
-	    (type == ACA_ERROR_TYPE_CE && ext_error_code == 6))
-		report->count[type] = ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]);
+	count = ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]);
 
-	return 0;
+	switch (type) {
+	case ACA_SMU_TYPE_UE:
+		if (ext_error_code != 0 && ext_error_code != 9)
+			count = 0ULL;
+
+		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, count);
+		break;
+	case ACA_SMU_TYPE_CE:
+		count = ext_error_code == 6 ? count : 0ULL;
+		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_CE, count);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
 }
 
 static const struct aca_bank_ops xgmi_v6_4_0_aca_bank_ops = {
-	.aca_bank_generate_report = xgmi_v6_4_0_aca_bank_generate_report,
+	.aca_bank_parser = xgmi_v6_4_0_aca_bank_parser,
 };
 
 static const struct aca_info xgmi_v6_4_0_aca_info = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index 1592c63b3099..a3bfc16de6d4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
@@ -44,6 +44,7 @@ struct amdgpu_hive_info {
 
 	struct amdgpu_reset_domain *reset_domain;
 	atomic_t ras_recovery;
+	struct ras_event_manager event_mgr;
 };
 
 struct amdgpu_pcs_ras_field {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
index 51a14f6d93bd..fb2b394bb9c5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
@@ -94,7 +94,8 @@ union amd_sriov_msg_feature_flags {
 		uint32_t reg_indirect_acc  : 1;
 		uint32_t av1_support       : 1;
 		uint32_t vcn_rb_decouple   : 1;
-		uint32_t reserved          : 24;
+		uint32_t mes_info_enable   : 1;
+		uint32_t reserved          : 23;
 	} flags;
 	uint32_t all;
 };
@@ -157,7 +158,7 @@ struct amd_sriov_msg_pf2vf_info_header {
 	uint32_t reserved[2];
 };
 
-#define AMD_SRIOV_MSG_PF2VF_INFO_FILLED_SIZE (48)
+#define AMD_SRIOV_MSG_PF2VF_INFO_FILLED_SIZE (49)
 struct amd_sriov_msg_pf2vf_info {
 	/* header contains size and version */
 	struct amd_sriov_msg_pf2vf_info_header header;
@@ -208,6 +209,8 @@ struct amd_sriov_msg_pf2vf_info {
 	struct amd_sriov_msg_uuid_info uuid_info;
 	/* PCIE atomic ops support flag */
 	uint32_t pcie_atomic_ops_support_flags;
+	/* Portion of GPU memory occupied by VF.  MAX value is 65535, but set to uint32_t to maintain alignment with reserved size */
+	uint32_t gpu_capacity;
 	/* reserved */
 	uint32_t reserved[256 - AMD_SRIOV_MSG_PF2VF_INFO_FILLED_SIZE];
 };
@@ -221,7 +224,7 @@ struct amd_sriov_msg_vf2pf_info_header {
 	uint32_t reserved[2];
 };
 
-#define AMD_SRIOV_MSG_VF2PF_INFO_FILLED_SIZE (70)
+#define AMD_SRIOV_MSG_VF2PF_INFO_FILLED_SIZE (73)
 struct amd_sriov_msg_vf2pf_info {
 	/* header contains size and version */
 	struct amd_sriov_msg_vf2pf_info_header header;
@@ -265,7 +268,9 @@ struct amd_sriov_msg_vf2pf_info {
 		uint32_t version;
 	} ucode_info[AMD_SRIOV_MSG_RESERVE_UCODE];
 	uint64_t dummy_page_addr;
-
+	/* FB allocated for guest MES to record UQ info */
+	uint64_t mes_info_addr;
+	uint32_t mes_info_size;
 	/* reserved */
 	uint32_t reserved[256 - AMD_SRIOV_MSG_VF2PF_INFO_FILLED_SIZE];
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
index fbb43ae7624f..d4e2aed2efa3 100644
--- a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
+++ b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
@@ -422,7 +422,7 @@ __aqua_vanjaram_get_auto_mode(struct amdgpu_xcp_mgr *xcp_mgr)
 
 	if (adev->gmc.num_mem_partitions == num_xcc / 2)
 		return (adev->flags & AMD_IS_APU) ? AMDGPU_TPX_PARTITION_MODE :
-						    AMDGPU_QPX_PARTITION_MODE;
+						    AMDGPU_CPX_PARTITION_MODE;
 
 	if (adev->gmc.num_mem_partitions == 2 && !(adev->flags & AMD_IS_APU))
 		return AMDGPU_DPX_PARTITION_MODE;
@@ -630,7 +630,7 @@ static int aqua_vanjaram_xcp_mgr_init(struct amdgpu_device *adev)
 
 int aqua_vanjaram_init_soc_config(struct amdgpu_device *adev)
 {
-	u32 mask, inst_mask = adev->sdma.sdma_mask;
+	u32 mask, avail_inst, inst_mask = adev->sdma.sdma_mask;
 	int ret, i;
 
 	/* generally 1 AID supports 4 instances */
@@ -642,7 +642,9 @@ int aqua_vanjaram_init_soc_config(struct amdgpu_device *adev)
 
 	for (mask = (1 << adev->sdma.num_inst_per_aid) - 1; inst_mask;
 	     inst_mask >>= adev->sdma.num_inst_per_aid, ++i) {
-		if ((inst_mask & mask) == mask)
+		avail_inst = inst_mask & mask;
+		if (avail_inst == mask || avail_inst == 0x3 ||
+		    avail_inst == 0xc)
 			adev->aid_mask |= (1 << i);
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.c b/drivers/gpu/drm/amd/amdgpu/atom.c
index 72362df352f6..d552e013354c 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.c
+++ b/drivers/gpu/drm/amd/amdgpu/atom.c
@@ -1243,6 +1243,7 @@ static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index,
 	ectx.ps_size = params_size;
 	ectx.abort = false;
 	ectx.last_jump = 0;
+	ectx.last_jump_jiffies = 0;
 	if (ws) {
 		ectx.ws = kcalloc(4, ws, GFP_KERNEL);
 		ectx.ws_size = ws;
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index a3a643254d7a..cf1d5d462b67 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -1375,14 +1375,14 @@ static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
 	return r;
 }
 
-static bool cik_asic_supports_baco(struct amdgpu_device *adev)
+static int cik_asic_supports_baco(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_BONAIRE:
 	case CHIP_HAWAII:
 		return amdgpu_dpm_is_baco_supported(adev);
 	default:
-		return false;
+		return 0;
 	}
 }
 
@@ -2210,6 +2210,8 @@ static const struct amd_ip_funcs cik_common_ip_funcs = {
 	.soft_reset = cik_common_soft_reset,
 	.set_clockgating_state = cik_common_set_clockgating_state,
 	.set_powergating_state = cik_common_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ip_block_version cik_common_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
index f24e34dc33d1..576baa9dbb0e 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
@@ -435,6 +435,8 @@ static const struct amd_ip_funcs cik_ih_ip_funcs = {
 	.soft_reset = cik_ih_soft_reset,
 	.set_clockgating_state = cik_ih_set_clockgating_state,
 	.set_powergating_state = cik_ih_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ih_funcs cik_ih_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index a3fccc4c1f43..6948ebda0fa2 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -1228,6 +1228,8 @@ static const struct amd_ip_funcs cik_sdma_ip_funcs = {
 	.soft_reset = cik_sdma_soft_reset,
 	.set_clockgating_state = cik_sdma_set_clockgating_state,
 	.set_powergating_state = cik_sdma_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = {
@@ -1290,7 +1292,7 @@ static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev)
  * @src_offset: src GPU address
  * @dst_offset: dst GPU address
  * @byte_count: number of bytes to xfer
- * @tmz: is this a secure operation
+ * @copy_flags: unused
  *
  * Copy GPU buffers using the DMA engine (CIK).
  * Used by the amdgpu ttm implementation to move pages if
@@ -1300,7 +1302,7 @@ static void cik_sdma_emit_copy_buffer(struct amdgpu_ib *ib,
 				      uint64_t src_offset,
 				      uint64_t dst_offset,
 				      uint32_t byte_count,
-				      bool tmz)
+				      uint32_t copy_flags)
 {
 	ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0);
 	ib->ptr[ib->length_dw++] = byte_count;
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
index c19681492efa..072643787384 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
@@ -433,6 +433,8 @@ static const struct amd_ip_funcs cz_ih_ip_funcs = {
 	.soft_reset = cz_ih_soft_reset,
 	.set_clockgating_state = cz_ih_set_clockgating_state,
 	.set_powergating_state = cz_ih_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ih_funcs cz_ih_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index 221af054d874..b44fce44c066 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -3333,6 +3333,8 @@ static const struct amd_ip_funcs dce_v10_0_ip_funcs = {
 	.soft_reset = dce_v10_0_soft_reset,
 	.set_clockgating_state = dce_v10_0_set_clockgating_state,
 	.set_powergating_state = dce_v10_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static void
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index 69e8b0db6cf7..80b2e7f79acf 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -3464,6 +3464,8 @@ static const struct amd_ip_funcs dce_v11_0_ip_funcs = {
 	.soft_reset = dce_v11_0_soft_reset,
 	.set_clockgating_state = dce_v11_0_set_clockgating_state,
 	.set_powergating_state = dce_v11_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static void
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index 60d40201fdd1..db20012600f5 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -3154,6 +3154,8 @@ static const struct amd_ip_funcs dce_v6_0_ip_funcs = {
 	.soft_reset = dce_v6_0_soft_reset,
 	.set_clockgating_state = dce_v6_0_set_clockgating_state,
 	.set_powergating_state = dce_v6_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static void
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 5a5fcc45e452..5b56100ec902 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -3242,6 +3242,8 @@ static const struct amd_ip_funcs dce_v8_0_ip_funcs = {
 	.soft_reset = dce_v8_0_soft_reset,
 	.set_clockgating_state = dce_v8_0_set_clockgating_state,
 	.set_powergating_state = dce_v8_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static void
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
index f90905ef32c7..536287ddd2ec 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
@@ -276,6 +276,99 @@ MODULE_FIRMWARE("amdgpu/gc_10_3_7_mec.bin");
 MODULE_FIRMWARE("amdgpu/gc_10_3_7_mec2.bin");
 MODULE_FIRMWARE("amdgpu/gc_10_3_7_rlc.bin");
 
+static const struct amdgpu_hwip_reg_entry gc_reg_list_10_1[] = {
+	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2),
+	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS3),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT1),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT2),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STALLED_STAT1),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STALLED_STAT1),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_BUSY_STAT),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_BUSY_STAT),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_BUSY_STAT),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_BUSY_STAT2),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_BUSY_STAT2),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_GFX_ERROR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_GFX_HPD_STATUS0),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_BASE),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_WPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_BASE),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_WPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_BASE),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_WPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_BASE),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_CMD_BUFSZ),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_CMD_BUFSZ),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_CMD_BUFSZ),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_CMD_BUFSZ),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_LO),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BUFSZ),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_LO),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BUFSZ),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_LO),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BUFSZ),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_LO),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BUFSZ),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCPF_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCPC_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCPG_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmGDS_PROTECTION_FAULT),
+	SOC15_REG_ENTRY_STR(GC, 0, mmGDS_VM_PROTECTION_FAULT),
+	SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_STATUS_2),
+	SOC15_REG_ENTRY_STR(GC, 0, mmPA_CL_CNTL_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRMI_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmSQC_DCACHE_UTCL0_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmSQC_ICACHE_UTCL0_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmSQG_UTCL0_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmTCP_UTCL0_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmWD_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL),
+	SOC15_REG_ENTRY_STR(GC, 0, mmGCVM_L2_PROTECTION_FAULT_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_DEBUG),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_CNTL),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MES_CNTL),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_INSTR_PNTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC1_INSTR_PNTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC2_INSTR_PNTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MES_DEBUG_INTERRUPT_INSTR_PNTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MES_INSTR_PNTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_INSTR_PNTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_INSTR_PNTR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_STAT),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_COMMAND),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_MESSAGE),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_1),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_2),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_3),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_4),
+	SOC15_REG_ENTRY_STR(GC, 0, mmSMU_RLC_RESPONSE),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SAFE_MODE),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_SAFE_MODE),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_RLCS_GPM_STAT_2),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SPP_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_RLCS_BOOTLOAD_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_INT_STAT),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_GENERAL_6),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_DEBUG_INST_A),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_DEBUG_INST_B),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_DEBUG_INST_ADDR),
+	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_LX6_CORE_PDEBUG_INST)
+};
+
 static const struct soc15_reg_golden golden_settings_gc_10_1[] = {
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_4, 0xffffffff, 0x00400014),
 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_CPF_CLK_CTRL, 0xfcff8fff, 0xf8000100),
@@ -3964,7 +4057,7 @@ static void gfx_v10_0_check_gfxoff_flag(struct amdgpu_device *adev)
 
 static int gfx_v10_0_init_microcode(struct amdgpu_device *adev)
 {
-	char fw_name[40];
+	char fw_name[53];
 	char ucode_prefix[30];
 	const char *wks = "";
 	int err;
@@ -4490,6 +4583,22 @@ static int gfx_v10_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
 			     hw_prio, NULL);
 }
 
+static void gfx_v10_0_alloc_dump_mem(struct amdgpu_device *adev)
+{
+	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_10_1);
+	uint32_t *ptr;
+
+	ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
+	if (ptr == NULL) {
+		DRM_ERROR("Failed to allocate memory for IP Dump\n");
+		adev->gfx.ip_dump = NULL;
+		adev->gfx.reg_count = 0;
+	} else {
+		adev->gfx.ip_dump = ptr;
+		adev->gfx.reg_count = reg_count;
+	}
+}
+
 static int gfx_v10_0_sw_init(void *handle)
 {
 	int i, j, k, r, ring_id = 0;
@@ -4518,7 +4627,7 @@ static int gfx_v10_0_sw_init(void *handle)
 	case IP_VERSION(10, 3, 3):
 	case IP_VERSION(10, 3, 7):
 		adev->gfx.me.num_me = 1;
-		adev->gfx.me.num_pipe_per_me = 1;
+		adev->gfx.me.num_pipe_per_me = 2;
 		adev->gfx.me.num_queue_per_pipe = 1;
 		adev->gfx.mec.num_mec = 2;
 		adev->gfx.mec.num_pipe_per_mec = 4;
@@ -4642,6 +4751,8 @@ static int gfx_v10_0_sw_init(void *handle)
 
 	gfx_v10_0_gpu_early_init(adev);
 
+	gfx_v10_0_alloc_dump_mem(adev);
+
 	return 0;
 }
 
@@ -4694,6 +4805,8 @@ static int gfx_v10_0_sw_fini(void *handle)
 
 	gfx_v10_0_free_microcode(adev);
 
+	kfree(adev->gfx.ip_dump);
+
 	return 0;
 }
 
@@ -8317,7 +8430,7 @@ static void gfx_v10_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 		}
 		reg_mem_engine = 0;
 	} else {
-		ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
+		ref_and_mask = nbio_hf_reg->ref_and_mask_cp0 << ring->pipe;
 		reg_mem_engine = 1; /* pfp */
 	}
 
@@ -9154,6 +9267,36 @@ static void gfx_v10_0_emit_mem_sync(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
 }
 
+static void gfx_v10_ip_print(void *handle, struct drm_printer *p)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	uint32_t i;
+	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_10_1);
+
+	if (!adev->gfx.ip_dump)
+		return;
+
+	for (i = 0; i < reg_count; i++)
+		drm_printf(p, "%-50s \t 0x%08x\n",
+			   gc_reg_list_10_1[i].reg_name,
+			   adev->gfx.ip_dump[i]);
+}
+
+static void gfx_v10_ip_dump(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	uint32_t i;
+	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_10_1);
+
+	if (!adev->gfx.ip_dump)
+		return;
+
+	amdgpu_gfx_off_ctrl(adev, false);
+	for (i = 0; i < reg_count; i++)
+		adev->gfx.ip_dump[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_10_1[i]));
+	amdgpu_gfx_off_ctrl(adev, true);
+}
+
 static const struct amd_ip_funcs gfx_v10_0_ip_funcs = {
 	.name = "gfx_v10_0",
 	.early_init = gfx_v10_0_early_init,
@@ -9170,6 +9313,8 @@ static const struct amd_ip_funcs gfx_v10_0_ip_funcs = {
 	.set_clockgating_state = gfx_v10_0_set_clockgating_state,
 	.set_powergating_state = gfx_v10_0_set_powergating_state,
 	.get_clockgating_state = gfx_v10_0_get_clockgating_state,
+	.dump_ip_state = gfx_v10_ip_dump,
+	.print_ip_state = gfx_v10_ip_print,
 };
 
 static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_gfx = {
@@ -9186,7 +9331,7 @@ static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_gfx = {
 		7 + /* PIPELINE_SYNC */
 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
-		2 + /* VM_FLUSH */
+		4 + /* VM_FLUSH */
 		8 + /* FENCE for VM_FLUSH */
 		20 + /* GDS switch */
 		4 + /* double SWITCH_BUFFER,
@@ -9276,7 +9421,6 @@ static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_kiq = {
 		7 + /* gfx_v10_0_ring_emit_pipeline_sync */
 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
-		2 + /* gfx_v10_0_ring_emit_vm_flush */
 		8 + 8 + 8, /* gfx_v10_0_ring_emit_fence_kiq x3 for user fence, vm fence */
 	.emit_ib_size =	7, /* gfx_v10_0_ring_emit_ib_compute */
 	.emit_ib = gfx_v10_0_ring_emit_ib_compute,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index f7325b02a191..ad6431013c73 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -510,7 +510,7 @@ static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev)
 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
 {
 	char fw_name[40];
-	char ucode_prefix[30];
+	char ucode_prefix[25];
 	int err;
 	const struct rlc_firmware_header_v2_0 *rlc_hdr;
 	uint16_t version_major;
@@ -4506,14 +4506,11 @@ static int gfx_v11_0_soft_reset(void *handle)
 
 	gfx_v11_0_set_safe_mode(adev, 0);
 
+	mutex_lock(&adev->srbm_mutex);
 	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
 		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
 			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
-				tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
-				tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, MEID, i);
-				tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, QUEUEID, j);
-				tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, k);
-				WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
+				soc21_grbm_select(adev, i, k, j, 0);
 
 				WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
 				WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
@@ -4523,16 +4520,14 @@ static int gfx_v11_0_soft_reset(void *handle)
 	for (i = 0; i < adev->gfx.me.num_me; ++i) {
 		for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
 			for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
-				tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
-				tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, MEID, i);
-				tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, QUEUEID, j);
-				tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, k);
-				WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
+				soc21_grbm_select(adev, i, k, j, 0);
 
 				WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1);
 			}
 		}
 	}
+	soc21_grbm_select(adev, 0, 0, 0, 0);
+	mutex_unlock(&adev->srbm_mutex);
 
 	/* Try to acquire the gfx mutex before access to CP_VMID_RESET */
 	r = gfx_v11_0_request_gfx_index_mutex(adev, 1);
@@ -6174,6 +6169,8 @@ static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
 	.set_clockgating_state = gfx_v11_0_set_clockgating_state,
 	.set_powergating_state = gfx_v11_0_set_powergating_state,
 	.get_clockgating_state = gfx_v11_0_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
@@ -6192,7 +6189,7 @@ static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
 		7 + /* PIPELINE_SYNC */
 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
-		2 + /* VM_FLUSH */
+		4 + /* VM_FLUSH */
 		8 + /* FENCE for VM_FLUSH */
 		20 + /* GDS switch */
 		5 + /* COND_EXEC */
@@ -6278,7 +6275,6 @@ static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = {
 		7 + /* gfx_v11_0_ring_emit_pipeline_sync */
 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
-		2 + /* gfx_v11_0_ring_emit_vm_flush */
 		8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */
 	.emit_ib_size =	7, /* gfx_v11_0_ring_emit_ib_compute */
 	.emit_ib = gfx_v11_0_ring_emit_ib_compute,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index 34f9211b2679..d0992ce9fb47 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -3457,6 +3457,8 @@ static const struct amd_ip_funcs gfx_v6_0_ip_funcs = {
 	.soft_reset = gfx_v6_0_soft_reset,
 	.set_clockgating_state = gfx_v6_0_set_clockgating_state,
 	.set_powergating_state = gfx_v6_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_gfx = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 86a4865b1ae5..541dbd70d8c7 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -4977,6 +4977,8 @@ static const struct amd_ip_funcs gfx_v7_0_ip_funcs = {
 	.soft_reset = gfx_v7_0_soft_reset,
 	.set_clockgating_state = gfx_v7_0_set_clockgating_state,
 	.set_powergating_state = gfx_v7_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 202ddda57f98..2f0e72caee1a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -6878,6 +6878,8 @@ static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
 	.set_clockgating_state = gfx_v8_0_set_clockgating_state,
 	.set_powergating_state = gfx_v8_0_set_powergating_state,
 	.get_clockgating_state = gfx_v8_0_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 6f97a6d0e6d0..3c8c5abf35ab 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -1249,7 +1249,7 @@ static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
 					  char *chip_name)
 {
-	char fw_name[30];
+	char fw_name[50];
 	int err;
 
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
@@ -1282,7 +1282,7 @@ out:
 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
 				       char *chip_name)
 {
-	char fw_name[30];
+	char fw_name[53];
 	int err;
 	const struct rlc_firmware_header_v2_0 *rlc_hdr;
 	uint16_t version_major;
@@ -1337,7 +1337,7 @@ static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
 					      char *chip_name)
 {
-	char fw_name[30];
+	char fw_name[50];
 	int err;
 
 	if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
@@ -6856,6 +6856,8 @@ static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
 	.set_clockgating_state = gfx_v9_0_set_clockgating_state,
 	.set_powergating_state = gfx_v9_0_set_powergating_state,
 	.get_clockgating_state = gfx_v9_0_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
@@ -6981,7 +6983,6 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
 		7 + /* gfx_v9_0_ring_emit_pipeline_sync */
 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
-		2 + /* gfx_v9_0_ring_emit_vm_flush */
 		8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
 		7 + /* gfx_v9_0_emit_mem_sync */
 		5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
@@ -7019,7 +7020,6 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
 		7 + /* gfx_v9_0_ring_emit_pipeline_sync */
 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
-		2 + /* gfx_v9_0_ring_emit_vm_flush */
 		8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
 	.emit_ib_size =	7, /* gfx_v9_0_ring_emit_ib_compute */
 	.emit_fence = gfx_v9_0_ring_emit_fence_kiq,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
index 065b2bd5f5a6..3f4fd2f08163 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
@@ -1909,18 +1909,7 @@ static void gfx_v9_4_2_reset_sq_timeout_status(struct amdgpu_device *adev)
 	mutex_unlock(&adev->grbm_idx_mutex);
 }
 
-static bool gfx_v9_4_2_query_uctl2_poison_status(struct amdgpu_device *adev)
-{
-	u32 status = 0;
-	struct amdgpu_vmhub *hub;
-
-	hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
-	status = RREG32(hub->vm_l2_pro_fault_status);
-	/* reset page fault status */
-	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
 
-	return REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, FED);
-}
 
 struct amdgpu_ras_block_hw_ops  gfx_v9_4_2_ras_ops = {
 		.query_ras_error_count = &gfx_v9_4_2_query_ras_error_count,
@@ -1934,5 +1923,4 @@ struct amdgpu_gfx_ras gfx_v9_4_2_ras = {
 		.hw_ops = &gfx_v9_4_2_ras_ops,
 	},
 	.enable_watchdog_timer = &gfx_v9_4_2_enable_watchdog_timer,
-	.query_utcl2_poison_status = gfx_v9_4_2_query_uctl2_poison_status,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
index b53c8fd4e8cf..7b16e8cca86a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
@@ -431,16 +431,16 @@ out:
 
 static int gfx_v9_4_3_init_microcode(struct amdgpu_device *adev)
 {
-	const char *chip_name;
+	char ucode_prefix[15];
 	int r;
 
-	chip_name = "gc_9_4_3";
+	amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
 
-	r = gfx_v9_4_3_init_rlc_microcode(adev, chip_name);
+	r = gfx_v9_4_3_init_rlc_microcode(adev, ucode_prefix);
 	if (r)
 		return r;
 
-	r = gfx_v9_4_3_init_cp_compute_microcode(adev, chip_name);
+	r = gfx_v9_4_3_init_cp_compute_microcode(adev, ucode_prefix);
 	if (r)
 		return r;
 
@@ -680,38 +680,44 @@ static const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs = {
 	.ih_node_to_logical_xcc = &gfx_v9_4_3_ih_to_xcc_inst,
 };
 
-static int gfx_v9_4_3_aca_bank_generate_report(struct aca_handle *handle,
-					       struct aca_bank *bank, enum aca_error_type type,
-					       struct aca_bank_report *report, void *data)
+static int gfx_v9_4_3_aca_bank_parser(struct aca_handle *handle,
+				      struct aca_bank *bank, enum aca_smu_type type,
+				      void *data)
 {
-	u64 status, misc0;
+	struct aca_bank_info info;
+	u64 misc0;
 	u32 instlo;
 	int ret;
 
-	status = bank->regs[ACA_REG_IDX_STATUS];
-	if ((type == ACA_ERROR_TYPE_UE &&
-	     ACA_REG__STATUS__ERRORCODEEXT(status) == ACA_EXTERROR_CODE_FAULT) ||
-	    (type == ACA_ERROR_TYPE_CE &&
-	     ACA_REG__STATUS__ERRORCODEEXT(status) == ACA_EXTERROR_CODE_CE)) {
+	ret = aca_bank_info_decode(bank, &info);
+	if (ret)
+		return ret;
 
-		ret = aca_bank_info_decode(bank, &report->info);
-		if (ret)
-			return ret;
+	/* NOTE: overwrite info.die_id with xcd id for gfx */
+	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
+	instlo &= GENMASK(31, 1);
+	info.die_id = instlo == mmSMNAID_XCD0_MCA_SMU ? 0 : 1;
 
-		/* NOTE: overwrite info.die_id with xcd id for gfx */
-		instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
-		instlo &= GENMASK(31, 1);
-		report->info.die_id = instlo == mmSMNAID_XCD0_MCA_SMU ? 0 : 1;
+	misc0 = bank->regs[ACA_REG_IDX_MISC0];
 
-		misc0 = bank->regs[ACA_REG_IDX_MISC0];
-		report->count[type] = ACA_REG__MISC0__ERRCNT(misc0);
+	switch (type) {
+	case ACA_SMU_TYPE_UE:
+		ret = aca_error_cache_log_bank_error(handle, &info,
+						     ACA_ERROR_TYPE_UE, 1ULL);
+		break;
+	case ACA_SMU_TYPE_CE:
+		ret = aca_error_cache_log_bank_error(handle, &info,
+						     ACA_ERROR_TYPE_CE, ACA_REG__MISC0__ERRCNT(misc0));
+		break;
+	default:
+		return -EINVAL;
 	}
 
-	return 0;
+	return ret;
 }
 
 static bool gfx_v9_4_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
-					 enum aca_error_type type, void *data)
+					 enum aca_smu_type type, void *data)
 {
 	u32 instlo;
 
@@ -730,7 +736,7 @@ static bool gfx_v9_4_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_b
 }
 
 static const struct aca_bank_ops gfx_v9_4_3_aca_bank_ops = {
-	.aca_bank_generate_report = gfx_v9_4_3_aca_bank_generate_report,
+	.aca_bank_parser = gfx_v9_4_3_aca_bank_parser,
 	.aca_bank_is_valid = gfx_v9_4_3_aca_bank_is_valid,
 };
 
@@ -2398,10 +2404,10 @@ gfx_v9_4_3_xcc_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
 		if (def != data)
 			WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
 
-		/* enable cgcg FSM(0x0000363F) */
+		/* CGCG Hysteresis: 400us */
 		def = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);
 
-		data = (0x36
+		data = (0x2710
 			<< RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
 		       RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
@@ -2410,10 +2416,10 @@ gfx_v9_4_3_xcc_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
 		if (def != data)
 			WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, data);
 
-		/* set IDLE_POLL_COUNT(0x00900100) */
+		/* set IDLE_POLL_COUNT(0x33450100)*/
 		def = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL);
 		data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
-			(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
+			(0x3345 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
 		if (def != data)
 			WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL, data);
 	} else {
@@ -4010,6 +4016,8 @@ static const struct amd_ip_funcs gfx_v9_4_3_ip_funcs = {
 	.set_clockgating_state = gfx_v9_4_3_set_clockgating_state,
 	.set_powergating_state = gfx_v9_4_3_set_powergating_state,
 	.get_clockgating_state = gfx_v9_4_3_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs gfx_v9_4_3_ring_funcs_compute = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
index 22175da0e16a..d200310d1731 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
@@ -443,6 +443,22 @@ static void gfxhub_v1_0_init(struct amdgpu_device *adev)
 		mmVM_INVALIDATE_ENG0_ADDR_RANGE_LO32;
 }
 
+static bool gfxhub_v1_0_query_utcl2_poison_status(struct amdgpu_device *adev,
+				int xcc_id)
+{
+	u32 status = 0;
+	struct amdgpu_vmhub *hub;
+
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))
+		return false;
+
+	hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
+	status = RREG32(hub->vm_l2_pro_fault_status);
+	/* reset page fault status */
+	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
+
+	return REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, FED);
+}
 
 const struct amdgpu_gfxhub_funcs gfxhub_v1_0_funcs = {
 	.get_mc_fb_offset = gfxhub_v1_0_get_mc_fb_offset,
@@ -452,4 +468,5 @@ const struct amdgpu_gfxhub_funcs gfxhub_v1_0_funcs = {
 	.set_fault_enable_default = gfxhub_v1_0_set_fault_enable_default,
 	.init = gfxhub_v1_0_init,
 	.get_xgmi_info = gfxhub_v1_1_get_xgmi_info,
+	.query_utcl2_poison_status = gfxhub_v1_0_query_utcl2_poison_status,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
index 49aecdcee006..77df8c9cbad2 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
@@ -620,6 +620,20 @@ static int gfxhub_v1_2_get_xgmi_info(struct amdgpu_device *adev)
 	return 0;
 }
 
+static bool gfxhub_v1_2_query_utcl2_poison_status(struct amdgpu_device *adev,
+				int xcc_id)
+{
+	u32 fed, status;
+
+	status = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regVM_L2_PROTECTION_FAULT_STATUS);
+	fed = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, FED);
+	/* reset page fault status */
+	WREG32_P(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id),
+			regVM_L2_PROTECTION_FAULT_STATUS), 1, ~1);
+
+	return fed;
+}
+
 const struct amdgpu_gfxhub_funcs gfxhub_v1_2_funcs = {
 	.get_mc_fb_offset = gfxhub_v1_2_get_mc_fb_offset,
 	.setup_vm_pt_regs = gfxhub_v1_2_setup_vm_pt_regs,
@@ -628,6 +642,7 @@ const struct amdgpu_gfxhub_funcs gfxhub_v1_2_funcs = {
 	.set_fault_enable_default = gfxhub_v1_2_set_fault_enable_default,
 	.init = gfxhub_v1_2_init,
 	.get_xgmi_info = gfxhub_v1_2_get_xgmi_info,
+	.query_utcl2_poison_status = gfxhub_v1_2_query_utcl2_poison_status,
 };
 
 static int gfxhub_v1_2_xcp_resume(void *handle, uint32_t inst_mask)
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index 23b478639921..3e38d8bfcb69 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -1115,6 +1115,8 @@ static const struct amd_ip_funcs gmc_v6_0_ip_funcs = {
 	.soft_reset = gmc_v6_0_soft_reset,
 	.set_clockgating_state = gmc_v6_0_set_clockgating_state,
 	.set_powergating_state = gmc_v6_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_gmc_funcs gmc_v6_0_gmc_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 3da7b6a2b00d..85df8fc81065 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -1354,6 +1354,8 @@ static const struct amd_ip_funcs gmc_v7_0_ip_funcs = {
 	.soft_reset = gmc_v7_0_soft_reset,
 	.set_clockgating_state = gmc_v7_0_set_clockgating_state,
 	.set_powergating_state = gmc_v7_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_gmc_funcs gmc_v7_0_gmc_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index d20e5f20ee31..fc97757e33d9 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -1717,6 +1717,8 @@ static const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
 	.set_clockgating_state = gmc_v8_0_set_clockgating_state,
 	.set_powergating_state = gmc_v8_0_set_powergating_state,
 	.get_clockgating_state = gmc_v8_0_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_gmc_funcs gmc_v8_0_gmc_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 47b63a4ce68b..c4ec1358f3aa 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -548,7 +548,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 {
 	bool retry_fault = !!(entry->src_data[1] & 0x80);
 	bool write_fault = !!(entry->src_data[1] & 0x20);
-	uint32_t status = 0, cid = 0, rw = 0;
+	uint32_t status = 0, cid = 0, rw = 0, fed = 0;
 	struct amdgpu_task_info *task_info;
 	struct amdgpu_vmhub *hub;
 	const char *mmhub_cid;
@@ -664,6 +664,13 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 	status = RREG32(hub->vm_l2_pro_fault_status);
 	cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID);
 	rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW);
+	fed = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, FED);
+
+	/* for fed error, kfd will handle it, return directly */
+	if (fed && amdgpu_ras_is_poison_mode_supported(adev) &&
+	    (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2)))
+		return 0;
+
 	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
 
 	amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub);
@@ -1450,7 +1457,6 @@ static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
 		adev->umc.channel_offs = UMC_V12_0_PER_CHANNEL_OFFSET;
 		adev->umc.active_mask = adev->aid_mask;
 		adev->umc.retire_unit = UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL;
-		adev->umc.channel_idx_tbl = &umc_v12_0_channel_idx_tbl[0][0][0];
 		if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu)
 			adev->umc.ras = &umc_v12_0_ras;
 		break;
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
index 2c02ae69883d..07984f7c3ae7 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
@@ -425,6 +425,8 @@ static const struct amd_ip_funcs iceland_ih_ip_funcs = {
 	.soft_reset = iceland_ih_soft_reset,
 	.set_clockgating_state = iceland_ih_set_clockgating_state,
 	.set_powergating_state = iceland_ih_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ih_funcs iceland_ih_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c b/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c
index ad4ad39f128f..3cb64c8f7175 100644
--- a/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c
@@ -346,6 +346,21 @@ static int ih_v6_0_irq_init(struct amdgpu_device *adev)
 			    DELAY, 3);
 	WREG32_SOC15(OSSSYS, 0, regIH_MSI_STORM_CTRL, tmp);
 
+	/* Redirect the interrupts to IH RB1 for dGPU */
+	if (adev->irq.ih1.ring_size) {
+		tmp = RREG32_SOC15(OSSSYS, 0, regIH_RING1_CLIENT_CFG_INDEX);
+		tmp = REG_SET_FIELD(tmp, IH_RING1_CLIENT_CFG_INDEX, INDEX, 0);
+		WREG32_SOC15(OSSSYS, 0, regIH_RING1_CLIENT_CFG_INDEX, tmp);
+
+		tmp = RREG32_SOC15(OSSSYS, 0, regIH_RING1_CLIENT_CFG_DATA);
+		tmp = REG_SET_FIELD(tmp, IH_RING1_CLIENT_CFG_DATA, CLIENT_ID, 0xa);
+		tmp = REG_SET_FIELD(tmp, IH_RING1_CLIENT_CFG_DATA, SOURCE_ID, 0x0);
+		tmp = REG_SET_FIELD(tmp, IH_RING1_CLIENT_CFG_DATA,
+				    SOURCE_ID_MATCH_ENABLE, 0x1);
+
+		WREG32_SOC15(OSSSYS, 0, regIH_RING1_CLIENT_CFG_DATA, tmp);
+	}
+
 	pci_set_master(adev->pdev);
 
 	/* enable interrupts */
@@ -549,8 +564,15 @@ static int ih_v6_0_sw_init(void *handle)
 	adev->irq.ih.use_doorbell = true;
 	adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
 
-	adev->irq.ih1.ring_size = 0;
-	adev->irq.ih2.ring_size = 0;
+	if (!(adev->flags & AMD_IS_APU)) {
+		r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, IH_RING_SIZE,
+					use_bus_addr);
+		if (r)
+			return r;
+
+		adev->irq.ih1.use_doorbell = true;
+		adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1;
+	}
 
 	/* initialize ih control register offset */
 	ih_v6_0_init_register_offset(adev);
@@ -748,6 +770,8 @@ static const struct amd_ip_funcs ih_v6_0_ip_funcs = {
 	.set_clockgating_state = ih_v6_0_set_clockgating_state,
 	.set_powergating_state = ih_v6_0_set_powergating_state,
 	.get_clockgating_state = ih_v6_0_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ih_funcs ih_v6_0_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c b/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c
index b8da0fc29378..0fbf5fa7b0f8 100644
--- a/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c
@@ -346,6 +346,21 @@ static int ih_v6_1_irq_init(struct amdgpu_device *adev)
 			    DELAY, 3);
 	WREG32_SOC15(OSSSYS, 0, regIH_MSI_STORM_CTRL, tmp);
 
+	/* Redirect the interrupts to IH RB1 for dGPU */
+	if (adev->irq.ih1.ring_size) {
+		tmp = RREG32_SOC15(OSSSYS, 0, regIH_RING1_CLIENT_CFG_INDEX);
+		tmp = REG_SET_FIELD(tmp, IH_RING1_CLIENT_CFG_INDEX, INDEX, 0);
+		WREG32_SOC15(OSSSYS, 0, regIH_RING1_CLIENT_CFG_INDEX, tmp);
+
+		tmp = RREG32_SOC15(OSSSYS, 0, regIH_RING1_CLIENT_CFG_DATA);
+		tmp = REG_SET_FIELD(tmp, IH_RING1_CLIENT_CFG_DATA, CLIENT_ID, 0xa);
+		tmp = REG_SET_FIELD(tmp, IH_RING1_CLIENT_CFG_DATA, SOURCE_ID, 0x0);
+		tmp = REG_SET_FIELD(tmp, IH_RING1_CLIENT_CFG_DATA,
+				    SOURCE_ID_MATCH_ENABLE, 0x1);
+
+		WREG32_SOC15(OSSSYS, 0, regIH_RING1_CLIENT_CFG_DATA, tmp);
+	}
+
 	pci_set_master(adev->pdev);
 
 	/* enable interrupts */
@@ -550,8 +565,15 @@ static int ih_v6_1_sw_init(void *handle)
 	adev->irq.ih.use_doorbell = true;
 	adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
 
-	adev->irq.ih1.ring_size = 0;
-	adev->irq.ih2.ring_size = 0;
+	if (!(adev->flags & AMD_IS_APU)) {
+		r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, IH_RING_SIZE,
+					use_bus_addr);
+		if (r)
+			return r;
+
+		adev->irq.ih1.use_doorbell = true;
+		adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1;
+	}
 
 	/* initialize ih control register offset */
 	ih_v6_1_init_register_offset(adev);
@@ -753,6 +775,8 @@ static const struct amd_ip_funcs ih_v6_1_ip_funcs = {
 	.set_clockgating_state = ih_v6_1_set_clockgating_state,
 	.set_powergating_state = ih_v6_1_set_powergating_state,
 	.get_clockgating_state = ih_v6_1_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ih_funcs ih_v6_1_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c b/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
index 7aed96fa10a9..aa6235dd4f2b 100644
--- a/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
@@ -749,6 +749,8 @@ static const struct amd_ip_funcs ih_v7_0_ip_funcs = {
 	.set_clockgating_state = ih_v7_0_set_clockgating_state,
 	.set_powergating_state = ih_v7_0_set_powergating_state,
 	.get_clockgating_state = ih_v7_0_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ih_funcs ih_v7_0_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
index 1c8116d75f63..ef3e42f6b841 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
@@ -759,6 +759,8 @@ static const struct amd_ip_funcs jpeg_v2_0_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = jpeg_v2_0_set_clockgating_state,
 	.set_powergating_state = jpeg_v2_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs jpeg_v2_0_dec_ring_vm_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
index 99cd49ee8ef6..afeaf3c64e27 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
@@ -632,6 +632,8 @@ static const struct amd_ip_funcs jpeg_v2_5_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = jpeg_v2_5_set_clockgating_state,
 	.set_powergating_state = jpeg_v2_5_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amd_ip_funcs jpeg_v2_6_ip_funcs = {
@@ -652,6 +654,8 @@ static const struct amd_ip_funcs jpeg_v2_6_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = jpeg_v2_5_set_clockgating_state,
 	.set_powergating_state = jpeg_v2_5_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs jpeg_v2_5_dec_ring_vm_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
index a92481da60cd..1c7cf4800bf7 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
@@ -557,6 +557,8 @@ static const struct amd_ip_funcs jpeg_v3_0_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = jpeg_v3_0_set_clockgating_state,
 	.set_powergating_state = jpeg_v3_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs jpeg_v3_0_dec_ring_vm_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c
index 88ea58d5c4ab..237fe5df5a8f 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c
@@ -719,6 +719,8 @@ static const struct amd_ip_funcs jpeg_v4_0_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = jpeg_v4_0_set_clockgating_state,
 	.set_powergating_state = jpeg_v4_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs jpeg_v4_0_dec_ring_vm_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
index 32caeb37cef9..d66af11aa66c 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
@@ -1053,6 +1053,8 @@ static const struct amd_ip_funcs jpeg_v4_0_3_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = jpeg_v4_0_3_set_clockgating_state,
 	.set_powergating_state = jpeg_v4_0_3_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs jpeg_v4_0_3_dec_ring_vm_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c
index edf5bcdd2bc9..da6bb9022b80 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c
@@ -762,6 +762,8 @@ static const struct amd_ip_funcs jpeg_v4_0_5_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = jpeg_v4_0_5_set_clockgating_state,
 	.set_powergating_state = jpeg_v4_0_5_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs jpeg_v4_0_5_dec_ring_vm_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c
index e70200f97555..64c856bfe0cb 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c
@@ -513,6 +513,8 @@ static const struct amd_ip_funcs jpeg_v5_0_0_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = jpeg_v5_0_0_set_clockgating_state,
 	.set_powergating_state = jpeg_v5_0_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs jpeg_v5_0_0_dec_ring_vm_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c
index 1e5ad1e08d2a..a626bf904926 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c
@@ -1176,6 +1176,8 @@ static const struct amd_ip_funcs mes_v10_1_ip_funcs = {
 	.hw_fini = mes_v10_1_hw_fini,
 	.suspend = mes_v10_1_suspend,
 	.resume = mes_v10_1_resume,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version mes_v10_1_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index 63f281a9984d..0d1407f25005 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -100,18 +100,76 @@ static const struct amdgpu_ring_funcs mes_v11_0_ring_funcs = {
 	.insert_nop = amdgpu_ring_insert_nop,
 };
 
+static const char *mes_v11_0_opcodes[] = {
+	"SET_HW_RSRC",
+	"SET_SCHEDULING_CONFIG",
+	"ADD_QUEUE",
+	"REMOVE_QUEUE",
+	"PERFORM_YIELD",
+	"SET_GANG_PRIORITY_LEVEL",
+	"SUSPEND",
+	"RESUME",
+	"RESET",
+	"SET_LOG_BUFFER",
+	"CHANGE_GANG_PRORITY",
+	"QUERY_SCHEDULER_STATUS",
+	"PROGRAM_GDS",
+	"SET_DEBUG_VMID",
+	"MISC",
+	"UPDATE_ROOT_PAGE_TABLE",
+	"AMD_LOG",
+};
+
+static const char *mes_v11_0_misc_opcodes[] = {
+	"WRITE_REG",
+	"INV_GART",
+	"QUERY_STATUS",
+	"READ_REG",
+	"WAIT_REG_MEM",
+	"SET_SHADER_DEBUGGER",
+};
+
+static const char *mes_v11_0_get_op_string(union MESAPI__MISC *x_pkt)
+{
+	const char *op_str = NULL;
+
+	if (x_pkt->header.opcode < ARRAY_SIZE(mes_v11_0_opcodes))
+		op_str = mes_v11_0_opcodes[x_pkt->header.opcode];
+
+	return op_str;
+}
+
+static const char *mes_v11_0_get_misc_op_string(union MESAPI__MISC *x_pkt)
+{
+	const char *op_str = NULL;
+
+	if ((x_pkt->header.opcode == MES_SCH_API_MISC) &&
+	    (x_pkt->opcode < ARRAY_SIZE(mes_v11_0_misc_opcodes)))
+		op_str = mes_v11_0_misc_opcodes[x_pkt->opcode];
+
+	return op_str;
+}
+
 static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 						    void *pkt, int size,
 						    int api_status_off)
 {
 	int ndw = size / 4;
 	signed long r;
-	union MESAPI__ADD_QUEUE *x_pkt = pkt;
+	union MESAPI__MISC *x_pkt = pkt;
 	struct MES_API_STATUS *api_status;
 	struct amdgpu_device *adev = mes->adev;
 	struct amdgpu_ring *ring = &mes->ring;
 	unsigned long flags;
-	signed long timeout = adev->usec_timeout;
+	signed long timeout = 3000000; /* 3000 ms */
+	const char *op_str, *misc_op_str;
+	u32 fence_offset;
+	u64 fence_gpu_addr;
+	u64 *fence_ptr;
+	int ret;
+
+	if (x_pkt->header.opcode >= MES_SCH_API_MAX)
+		return -EINVAL;
 
 	if (amdgpu_emu_mode) {
 		timeout *= 100;
@@ -121,27 +179,52 @@ static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 	}
 	BUG_ON(size % 4 != 0);
 
+	ret = amdgpu_device_wb_get(adev, &fence_offset);
+	if (ret)
+		return ret;
+	fence_gpu_addr =
+		adev->wb.gpu_addr + (fence_offset * 4);
+	fence_ptr = (u64 *)&adev->wb.wb[fence_offset];
+	*fence_ptr = 0;
+
 	spin_lock_irqsave(&mes->ring_lock, flags);
 	if (amdgpu_ring_alloc(ring, ndw)) {
 		spin_unlock_irqrestore(&mes->ring_lock, flags);
+		amdgpu_device_wb_free(adev, fence_offset);
 		return -ENOMEM;
 	}
 
 	api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);
-	api_status->api_completion_fence_addr = mes->ring.fence_drv.gpu_addr;
-	api_status->api_completion_fence_value = ++mes->ring.fence_drv.sync_seq;
+	api_status->api_completion_fence_addr = fence_gpu_addr;
+	api_status->api_completion_fence_value = 1;
 
 	amdgpu_ring_write_multiple(ring, pkt, ndw);
 	amdgpu_ring_commit(ring);
 	spin_unlock_irqrestore(&mes->ring_lock, flags);
 
-	DRM_DEBUG("MES msg=%d was emitted\n", x_pkt->header.opcode);
+	op_str = mes_v11_0_get_op_string(x_pkt);
+	misc_op_str = mes_v11_0_get_misc_op_string(x_pkt);
+
+	if (misc_op_str)
+		dev_dbg(adev->dev, "MES msg=%s (%s) was emitted\n", op_str, misc_op_str);
+	else if (op_str)
+		dev_dbg(adev->dev, "MES msg=%s was emitted\n", op_str);
+	else
+		dev_dbg(adev->dev, "MES msg=%d was emitted\n", x_pkt->header.opcode);
 
-	r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq,
-		      timeout);
+	r = amdgpu_mes_fence_wait_polling(fence_ptr, (u64)1, timeout);
+	amdgpu_device_wb_free(adev, fence_offset);
 	if (r < 1) {
-		DRM_ERROR("MES failed to response msg=%d\n",
-			  x_pkt->header.opcode);
+
+		if (misc_op_str)
+			dev_err(adev->dev, "MES failed to respond to msg=%s (%s)\n",
+				op_str, misc_op_str);
+		else if (op_str)
+			dev_err(adev->dev, "MES failed to respond to msg=%s\n",
+				op_str);
+		else
+			dev_err(adev->dev, "MES failed to respond to msg=%d\n",
+				x_pkt->header.opcode);
 
 		while (halt_if_hws_hang)
 			schedule();
@@ -422,6 +505,36 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
 			offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
 }
 
+static int mes_v11_0_set_hw_resources_1(struct amdgpu_mes *mes)
+{
+	int size = 128 * PAGE_SIZE;
+	int ret = 0;
+	struct amdgpu_device *adev = mes->adev;
+	union MESAPI_SET_HW_RESOURCES_1 mes_set_hw_res_pkt;
+	memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));
+
+	mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC_1;
+	mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+	mes_set_hw_res_pkt.enable_mes_info_ctx = 1;
+
+	ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
+				AMDGPU_GEM_DOMAIN_VRAM,
+				&mes->resource_1,
+				&mes->resource_1_gpu_addr,
+				&mes->resource_1_addr);
+	if (ret) {
+		dev_err(adev->dev, "(%d) failed to create mes resource_1 bo\n", ret);
+		return ret;
+	}
+
+	mes_set_hw_res_pkt.mes_info_ctx_mc_addr = mes->resource_1_gpu_addr;
+	mes_set_hw_res_pkt.mes_info_ctx_size = mes->resource_1->tbo.base.size;
+	return mes_v11_0_submit_pkt_and_poll_completion(mes,
+			&mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
+			offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
+}
+
 static const struct amdgpu_mes_funcs mes_v11_0_funcs = {
 	.add_hw_queue = mes_v11_0_add_hw_queue,
 	.remove_hw_queue = mes_v11_0_remove_hw_queue,
@@ -1203,6 +1316,14 @@ static int mes_v11_0_hw_init(void *handle)
 	if (r)
 		goto failure;
 
+	if (amdgpu_sriov_is_mes_info_enable(adev)) {
+		r = mes_v11_0_set_hw_resources_1(&adev->mes);
+		if (r) {
+			DRM_ERROR("failed mes_v11_0_set_hw_resources_1, r=%d\n", r);
+			goto failure;
+		}
+	}
+
 	r = mes_v11_0_query_sched_status(&adev->mes);
 	if (r) {
 		DRM_ERROR("MES is busy\n");
@@ -1226,6 +1347,11 @@ failure:
 
 static int mes_v11_0_hw_fini(void *handle)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	if (amdgpu_sriov_is_mes_info_enable(adev)) {
+		amdgpu_bo_free_kernel(&adev->mes.resource_1, &adev->mes.resource_1_gpu_addr,
+					&adev->mes.resource_1_addr);
+	}
 	return 0;
 }
 
@@ -1291,6 +1417,8 @@ static const struct amd_ip_funcs mes_v11_0_ip_funcs = {
 	.hw_fini = mes_v11_0_hw_fini,
 	.suspend = mes_v11_0_suspend,
 	.resume = mes_v11_0_resume,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version mes_v11_0_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c
index c0fc44cdd658..7a1ff298417a 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c
@@ -559,6 +559,20 @@ static void mmhub_v1_8_get_clockgating(struct amdgpu_device *adev, u64 *flags)
 
 }
 
+static bool mmhub_v1_8_query_utcl2_poison_status(struct amdgpu_device *adev,
+				int hub_inst)
+{
+	u32 fed, status;
+
+	status = RREG32_SOC15(MMHUB, hub_inst, regVM_L2_PROTECTION_FAULT_STATUS);
+	fed = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, FED);
+	/* reset page fault status */
+	WREG32_P(SOC15_REG_OFFSET(MMHUB, hub_inst,
+			regVM_L2_PROTECTION_FAULT_STATUS), 1, ~1);
+
+	return fed;
+}
+
 const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs = {
 	.get_fb_location = mmhub_v1_8_get_fb_location,
 	.init = mmhub_v1_8_init,
@@ -568,6 +582,7 @@ const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs = {
 	.setup_vm_pt_regs = mmhub_v1_8_setup_vm_pt_regs,
 	.set_clockgating = mmhub_v1_8_set_clockgating,
 	.get_clockgating = mmhub_v1_8_get_clockgating,
+	.query_utcl2_poison_status = mmhub_v1_8_query_utcl2_poison_status,
 };
 
 static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ce_reg_list[] = {
@@ -706,28 +721,32 @@ static const struct amdgpu_ras_block_hw_ops mmhub_v1_8_ras_hw_ops = {
 	.reset_ras_error_count = mmhub_v1_8_reset_ras_error_count,
 };
 
-static int mmhub_v1_8_aca_bank_generate_report(struct aca_handle *handle,
-					       struct aca_bank *bank, enum aca_error_type type,
-					       struct aca_bank_report *report, void *data)
+static int mmhub_v1_8_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
+				      enum aca_smu_type type, void *data)
 {
-	u64 status, misc0;
+	struct aca_bank_info info;
+	u64 misc0;
 	int ret;
 
-	status = bank->regs[ACA_REG_IDX_STATUS];
-	if ((type == ACA_ERROR_TYPE_UE &&
-	     ACA_REG__STATUS__ERRORCODEEXT(status) == ACA_EXTERROR_CODE_FAULT) ||
-	    (type == ACA_ERROR_TYPE_CE &&
-	     ACA_REG__STATUS__ERRORCODEEXT(status) == ACA_EXTERROR_CODE_CE)) {
-
-		ret = aca_bank_info_decode(bank, &report->info);
-		if (ret)
-			return ret;
-
-		misc0 = bank->regs[ACA_REG_IDX_MISC0];
-		report->count[type] = ACA_REG__MISC0__ERRCNT(misc0);
+	ret = aca_bank_info_decode(bank, &info);
+	if (ret)
+		return ret;
+
+	misc0 = bank->regs[ACA_REG_IDX_MISC0];
+	switch (type) {
+	case ACA_SMU_TYPE_UE:
+		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
+						     1ULL);
+		break;
+	case ACA_SMU_TYPE_CE:
+		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_CE,
+						     ACA_REG__MISC0__ERRCNT(misc0));
+		break;
+	default:
+		return -EINVAL;
 	}
 
-	return 0;
+	return ret;
 }
 
 /* reference to smu driver if header file */
@@ -741,7 +760,7 @@ static int mmhub_v1_8_err_codes[] = {
 };
 
 static bool mmhub_v1_8_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
-					 enum aca_error_type type, void *data)
+					 enum aca_smu_type type, void *data)
 {
 	u32 instlo;
 
@@ -760,7 +779,7 @@ static bool mmhub_v1_8_aca_bank_is_valid(struct aca_handle *handle, struct aca_b
 }
 
 static const struct aca_bank_ops mmhub_v1_8_aca_bank_ops = {
-	.aca_bank_generate_report = mmhub_v1_8_aca_bank_generate_report,
+	.aca_bank_parser = mmhub_v1_8_aca_bank_parser,
 	.aca_bank_is_valid = mmhub_v1_8_aca_bank_is_valid,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index a2bd2c3b1ef9..0c7275bca8f7 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
@@ -276,6 +276,8 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
 		timeout -= 10;
 	} while (timeout > 1);
 
+	dev_warn(adev->dev, "waiting IDH_FLR_NOTIFICATION_CMPL timeout\n");
+
 flr_done:
 	atomic_set(&adev->reset_domain->in_gpu_reset, 0);
 	up_write(&adev->reset_domain->sem);
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c
index 77f5b55decf9..aba00d961627 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c
@@ -309,6 +309,8 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work)
 		timeout -= 10;
 	} while (timeout > 1);
 
+	dev_warn(adev->dev, "waiting IDH_FLR_NOTIFICATION_CMPL timeout\n");
+
 flr_done:
 	atomic_set(&adev->reset_domain->in_gpu_reset, 0);
 	up_write(&adev->reset_domain->sem);
@@ -444,7 +446,6 @@ static void xgpu_nv_ras_poison_handler(struct amdgpu_device *adev,
 		amdgpu_virt_fini_data_exchange(adev);
 		xgpu_nv_send_access_requests_with_param(adev,
 					IDH_RAS_POISON,	block, 0, 0);
-		amdgpu_virt_init_data_exchange(adev);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c
index 4178f4e5dad7..b281462093f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c
@@ -713,6 +713,8 @@ static const struct amd_ip_funcs navi10_ih_ip_funcs = {
 	.set_clockgating_state = navi10_ih_set_clockgating_state,
 	.set_powergating_state = navi10_ih_set_powergating_state,
 	.get_clockgating_state = navi10_ih_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ih_funcs navi10_ih_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c
index 4d7976b77767..12e54047bf79 100644
--- a/drivers/gpu/drm/amd/amdgpu/nv.c
+++ b/drivers/gpu/drm/amd/amdgpu/nv.c
@@ -110,7 +110,7 @@ static const struct amdgpu_video_codec_info sc_video_codecs_decode_array_vcn0[]
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4096, 52)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1, 4096, 4096, 4)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)},
-	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)},
+	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 16384, 16384, 0)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_AV1, 8192, 4352, 0)},
 };
@@ -121,7 +121,7 @@ static const struct amdgpu_video_codec_info sc_video_codecs_decode_array_vcn1[]
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4096, 52)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1, 4096, 4096, 4)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)},
-	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)},
+	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 16384, 16384, 0)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)},
 };
 
@@ -199,7 +199,7 @@ static const struct amdgpu_video_codec_info yc_video_codecs_decode_array[] = {
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4096, 52)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)},
-	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)},
+	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 16384, 16384, 0)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_AV1, 8192, 4352, 0)},
 };
 
@@ -1131,4 +1131,6 @@ static const struct amd_ip_funcs nv_common_ip_funcs = {
 	.set_clockgating_state = nv_common_set_clockgating_state,
 	.set_powergating_state = nv_common_set_powergating_state,
 	.get_clockgating_state = nv_common_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c
index 78a95f8f370b..f08a32c18694 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c
@@ -169,7 +169,8 @@ static int psp_v14_0_bootloader_load_intf_drv(struct psp_context *psp)
 
 static int psp_v14_0_bootloader_load_dbg_drv(struct psp_context *psp)
 {
-	return psp_v14_0_bootloader_load_component(psp, &psp->dbg_drv, PSP_BL__LOAD_DBGDRV);
+	/* dbg_drv was renamed to had_drv in psp v14 */
+	return psp_v14_0_bootloader_load_component(psp, &psp->dbg_drv, PSP_BL__LOAD_HADDRV);
 }
 
 static int psp_v14_0_bootloader_load_ras_drv(struct psp_context *psp)
@@ -177,6 +178,10 @@ static int psp_v14_0_bootloader_load_ras_drv(struct psp_context *psp)
 	return psp_v14_0_bootloader_load_component(psp, &psp->ras_drv, PSP_BL__LOAD_RASDRV);
 }
 
+static int psp_v14_0_bootloader_load_ipkeymgr_drv(struct psp_context *psp)
+{
+	return psp_v14_0_bootloader_load_component(psp, &psp->ipkeymgr_drv, PSP_BL__LOAD_IPKEYMGRDRV);
+}
 
 static int psp_v14_0_bootloader_load_sos(struct psp_context *psp)
 {
@@ -653,6 +658,7 @@ static const struct psp_funcs psp_v14_0_funcs = {
 	.bootloader_load_intf_drv = psp_v14_0_bootloader_load_intf_drv,
 	.bootloader_load_dbg_drv = psp_v14_0_bootloader_load_dbg_drv,
 	.bootloader_load_ras_drv = psp_v14_0_bootloader_load_ras_drv,
+	.bootloader_load_ipkeymgr_drv = psp_v14_0_bootloader_load_ipkeymgr_drv,
 	.bootloader_load_sos = psp_v14_0_bootloader_load_sos,
 	.ring_create = psp_v14_0_ring_create,
 	.ring_stop = psp_v14_0_ring_stop,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index 07e19caf2bc1..ac8a9b9b3e52 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -1113,6 +1113,8 @@ static const struct amd_ip_funcs sdma_v2_4_ip_funcs = {
 	.soft_reset = sdma_v2_4_soft_reset,
 	.set_clockgating_state = sdma_v2_4_set_clockgating_state,
 	.set_powergating_state = sdma_v2_4_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs sdma_v2_4_ring_funcs = {
@@ -1176,7 +1178,7 @@ static void sdma_v2_4_set_irq_funcs(struct amdgpu_device *adev)
  * @src_offset: src GPU address
  * @dst_offset: dst GPU address
  * @byte_count: number of bytes to xfer
- * @tmz: unused
+ * @copy_flags: unused
  *
  * Copy GPU buffers using the DMA engine (VI).
  * Used by the amdgpu ttm implementation to move pages if
@@ -1186,7 +1188,7 @@ static void sdma_v2_4_emit_copy_buffer(struct amdgpu_ib *ib,
 				       uint64_t src_offset,
 				       uint64_t dst_offset,
 				       uint32_t byte_count,
-				       bool tmz)
+				       uint32_t copy_flags)
 {
 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 2ad615be4bb3..b8ebdc4ae6f6 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -1553,6 +1553,8 @@ static const struct amd_ip_funcs sdma_v3_0_ip_funcs = {
 	.set_clockgating_state = sdma_v3_0_set_clockgating_state,
 	.set_powergating_state = sdma_v3_0_set_powergating_state,
 	.get_clockgating_state = sdma_v3_0_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs sdma_v3_0_ring_funcs = {
@@ -1616,7 +1618,7 @@ static void sdma_v3_0_set_irq_funcs(struct amdgpu_device *adev)
  * @src_offset: src GPU address
  * @dst_offset: dst GPU address
  * @byte_count: number of bytes to xfer
- * @tmz: unused
+ * @copy_flags: unused
  *
  * Copy GPU buffers using the DMA engine (VI).
  * Used by the amdgpu ttm implementation to move pages if
@@ -1626,7 +1628,7 @@ static void sdma_v3_0_emit_copy_buffer(struct amdgpu_ib *ib,
 				       uint64_t src_offset,
 				       uint64_t dst_offset,
 				       uint32_t byte_count,
-				       bool tmz)
+				       uint32_t copy_flags)
 {
 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index 43775cb67ff5..101038395c3b 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -2021,6 +2021,9 @@ static int sdma_v4_0_process_trap_irq(struct amdgpu_device *adev,
 
 	DRM_DEBUG("IH: SDMA trap\n");
 	instance = sdma_v4_0_irq_id_to_seq(entry->client_id);
+	if (instance < 0)
+		return instance;
+
 	switch (entry->ring_id) {
 	case 0:
 		amdgpu_fence_process(&adev->sdma.instance[instance].ring);
@@ -2448,7 +2451,7 @@ static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev)
  * @src_offset: src GPU address
  * @dst_offset: dst GPU address
  * @byte_count: number of bytes to xfer
- * @tmz: if a secure copy should be used
+ * @copy_flags: copy flags for the buffers
  *
  * Copy GPU buffers using the DMA engine (VEGA10/12).
  * Used by the amdgpu ttm implementation to move pages if
@@ -2458,11 +2461,11 @@ static void sdma_v4_0_emit_copy_buffer(struct amdgpu_ib *ib,
 				       uint64_t src_offset,
 				       uint64_t dst_offset,
 				       uint32_t byte_count,
-				       bool tmz)
+				       uint32_t copy_flags)
 {
 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
-		SDMA_PKT_COPY_LINEAR_HEADER_TMZ(tmz ? 1 : 0);
+		SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
 	ib->ptr[ib->length_dw++] = byte_count - 1;
 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
 	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
index 82eab49be82b..341b24d8320b 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
@@ -368,7 +368,8 @@ static void sdma_v4_4_2_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 	u32 ref_and_mask = 0;
 	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
 
-	ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
+	ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0
+		       << (ring->me % adev->sdma.num_inst_per_aid);
 
 	sdma_v4_4_2_wait_reg_mem(ring, 0, 1,
 			       adev->nbio.funcs->get_hdp_flush_done_offset(adev),
@@ -1944,7 +1945,7 @@ static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev)
  * @src_offset: src GPU address
  * @dst_offset: dst GPU address
  * @byte_count: number of bytes to xfer
- * @tmz: if a secure copy should be used
+ * @copy_flags: copy flags for the buffers
  *
  * Copy GPU buffers using the DMA engine.
  * Used by the amdgpu ttm implementation to move pages if
@@ -1954,11 +1955,11 @@ static void sdma_v4_4_2_emit_copy_buffer(struct amdgpu_ib *ib,
 				       uint64_t src_offset,
 				       uint64_t dst_offset,
 				       uint32_t byte_count,
-				       bool tmz)
+				       uint32_t copy_flags)
 {
 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
-		SDMA_PKT_COPY_LINEAR_HEADER_TMZ(tmz ? 1 : 0);
+		SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
 	ib->ptr[ib->length_dw++] = byte_count - 1;
 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
 	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
@@ -2179,35 +2180,39 @@ static const struct amdgpu_ras_block_hw_ops sdma_v4_4_2_ras_hw_ops = {
 	.reset_ras_error_count = sdma_v4_4_2_reset_ras_error_count,
 };
 
-static int sdma_v4_4_2_aca_bank_generate_report(struct aca_handle *handle,
-						struct aca_bank *bank, enum aca_error_type type,
-						struct aca_bank_report *report, void *data)
+static int sdma_v4_4_2_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
+				       enum aca_smu_type type, void *data)
 {
-	u64 status, misc0;
+	struct aca_bank_info info;
+	u64 misc0;
 	int ret;
 
-	status = bank->regs[ACA_REG_IDX_STATUS];
-	if ((type == ACA_ERROR_TYPE_UE &&
-	     ACA_REG__STATUS__ERRORCODEEXT(status) == ACA_EXTERROR_CODE_FAULT) ||
-	    (type == ACA_ERROR_TYPE_CE &&
-	     ACA_REG__STATUS__ERRORCODEEXT(status) == ACA_EXTERROR_CODE_CE)) {
+	ret = aca_bank_info_decode(bank, &info);
+	if (ret)
+		return ret;
 
-		ret = aca_bank_info_decode(bank, &report->info);
-		if (ret)
-			return ret;
-
-		misc0 = bank->regs[ACA_REG_IDX_MISC0];
-		report->count[type] = ACA_REG__MISC0__ERRCNT(misc0);
+	misc0 = bank->regs[ACA_REG_IDX_MISC0];
+	switch (type) {
+	case ACA_SMU_TYPE_UE:
+		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
+						     1ULL);
+		break;
+	case ACA_SMU_TYPE_CE:
+		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_CE,
+						     ACA_REG__MISC0__ERRCNT(misc0));
+		break;
+	default:
+		return -EINVAL;
 	}
 
-	return 0;
+	return ret;
 }
 
 /* CODE_SDMA0 - CODE_SDMA4, reference to smu driver if header file */
 static int sdma_v4_4_2_err_codes[] = { 33, 34, 35, 36 };
 
 static bool sdma_v4_4_2_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
-					  enum aca_error_type type, void *data)
+					  enum aca_smu_type type, void *data)
 {
 	u32 instlo;
 
@@ -2226,7 +2231,7 @@ static bool sdma_v4_4_2_aca_bank_is_valid(struct aca_handle *handle, struct aca_
 }
 
 static const struct aca_bank_ops sdma_v4_4_2_aca_bank_ops = {
-	.aca_bank_generate_report = sdma_v4_4_2_aca_bank_generate_report,
+	.aca_bank_parser = sdma_v4_4_2_aca_bank_parser,
 	.aca_bank_is_valid = sdma_v4_4_2_aca_bank_is_valid,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
index 883e8a1b8a40..b7d33d78bce0 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
@@ -999,7 +999,8 @@ static int sdma_v5_0_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 20);
 	if (r) {
 		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_device_wb_free(adev, index);
+		if (!ring->is_mes_queue)
+			amdgpu_device_wb_free(adev, index);
 		return r;
 	}
 
@@ -1805,7 +1806,7 @@ static void sdma_v5_0_set_irq_funcs(struct amdgpu_device *adev)
  * @src_offset: src GPU address
  * @dst_offset: dst GPU address
  * @byte_count: number of bytes to xfer
- * @tmz: if a secure copy should be used
+ * @copy_flags: copy flags for the buffers
  *
  * Copy GPU buffers using the DMA engine (NAVI10).
  * Used by the amdgpu ttm implementation to move pages if
@@ -1815,11 +1816,11 @@ static void sdma_v5_0_emit_copy_buffer(struct amdgpu_ib *ib,
 				       uint64_t src_offset,
 				       uint64_t dst_offset,
 				       uint32_t byte_count,
-				       bool tmz)
+				       uint32_t copy_flags)
 {
 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
-		SDMA_PKT_COPY_LINEAR_HEADER_TMZ(tmz ? 1 : 0);
+		SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
 	ib->ptr[ib->length_dw++] = byte_count - 1;
 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
 	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
index 42f4bd250def..cc9e961f0078 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
@@ -280,17 +280,21 @@ static void sdma_v5_2_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 	u32 ref_and_mask = 0;
 	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
 
-	ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
-
-	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
-			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |
-			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
-	amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_done_offset(adev)) << 2);
-	amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_req_offset(adev)) << 2);
-	amdgpu_ring_write(ring, ref_and_mask); /* reference */
-	amdgpu_ring_write(ring, ref_and_mask); /* mask */
-	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
-			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
+	if (ring->me > 1) {
+		amdgpu_asic_flush_hdp(adev, ring);
+	} else {
+		ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
+
+		amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
+				  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |
+				  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
+		amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_done_offset(adev)) << 2);
+		amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_req_offset(adev)) << 2);
+		amdgpu_ring_write(ring, ref_and_mask); /* reference */
+		amdgpu_ring_write(ring, ref_and_mask); /* mask */
+		amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
+				  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
+	}
 }
 
 /**
@@ -835,7 +839,8 @@ static int sdma_v5_2_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 20);
 	if (r) {
 		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_device_wb_free(adev, index);
+		if (!ring->is_mes_queue)
+			amdgpu_device_wb_free(adev, index);
 		return r;
 	}
 
@@ -1747,7 +1752,7 @@ static void sdma_v5_2_set_irq_funcs(struct amdgpu_device *adev)
  * @src_offset: src GPU address
  * @dst_offset: dst GPU address
  * @byte_count: number of bytes to xfer
- * @tmz: if a secure copy should be used
+ * @copy_flags: copy flags for the buffers
  *
  * Copy GPU buffers using the DMA engine.
  * Used by the amdgpu ttm implementation to move pages if
@@ -1757,11 +1762,11 @@ static void sdma_v5_2_emit_copy_buffer(struct amdgpu_ib *ib,
 				       uint64_t src_offset,
 				       uint64_t dst_offset,
 				       uint32_t byte_count,
-				       bool tmz)
+				       uint32_t copy_flags)
 {
 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
-		SDMA_PKT_COPY_LINEAR_HEADER_TMZ(tmz ? 1 : 0);
+		SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
 	ib->ptr[ib->length_dw++] = byte_count - 1;
 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
 	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index 361835a61f2e..c833b6b8373b 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -507,6 +507,13 @@ static int sdma_v6_0_gfx_resume(struct amdgpu_device *adev)
 		/* set minor_ptr_update to 0 after wptr programed */
 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_MINOR_PTR_UPDATE), 0);
 
+		/* Set up sdma hang watchdog */
+		temp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_WATCHDOG_CNTL));
+		/* 100ms per unit */
+		temp = REG_SET_FIELD(temp, SDMA0_WATCHDOG_CNTL, QUEUE_HANG_COUNT,
+				     max(adev->usec_timeout/100000, 1));
+		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_WATCHDOG_CNTL), temp);
+
 		/* Set up RESP_MODE to non-copy addresses */
 		temp = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_UTCL1_CNTL));
 		temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, RESP_MODE, 3);
@@ -854,7 +861,8 @@ static int sdma_v6_0_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 5);
 	if (r) {
 		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_device_wb_free(adev, index);
+		if (!ring->is_mes_queue)
+			amdgpu_device_wb_free(adev, index);
 		return r;
 	}
 
@@ -1567,7 +1575,7 @@ static void sdma_v6_0_set_irq_funcs(struct amdgpu_device *adev)
  * @src_offset: src GPU address
  * @dst_offset: dst GPU address
  * @byte_count: number of bytes to xfer
- * @tmz: if a secure copy should be used
+ * @copy_flags: copy flags for the buffers
  *
  * Copy GPU buffers using the DMA engine.
  * Used by the amdgpu ttm implementation to move pages if
@@ -1577,11 +1585,11 @@ static void sdma_v6_0_emit_copy_buffer(struct amdgpu_ib *ib,
 				       uint64_t src_offset,
 				       uint64_t dst_offset,
 				       uint32_t byte_count,
-				       bool tmz)
+				       uint32_t copy_flags)
 {
 	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
 		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
-		SDMA_PKT_COPY_LINEAR_HEADER_TMZ(tmz ? 1 : 0);
+		SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
 	ib->ptr[ib->length_dw++] = byte_count - 1;
 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
 	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index 23e4ef4fff7c..85235470e872 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -1409,9 +1409,9 @@ static int si_gpu_pci_config_reset(struct amdgpu_device *adev)
 	return r;
 }
 
-static bool si_asic_supports_baco(struct amdgpu_device *adev)
+static int si_asic_supports_baco(struct amdgpu_device *adev)
 {
-	return false;
+	return 0;
 }
 
 static enum amd_reset_method
@@ -2706,6 +2706,8 @@ static const struct amd_ip_funcs si_common_ip_funcs = {
 	.soft_reset = si_common_soft_reset,
 	.set_clockgating_state = si_common_set_clockgating_state,
 	.set_powergating_state = si_common_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ip_block_version si_common_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index 9aa0e11ee673..11db5b755832 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -708,6 +708,8 @@ static const struct amd_ip_funcs si_dma_ip_funcs = {
 	.soft_reset = si_dma_soft_reset,
 	.set_clockgating_state = si_dma_set_clockgating_state,
 	.set_powergating_state = si_dma_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs si_dma_ring_funcs = {
@@ -761,7 +763,7 @@ static void si_dma_set_irq_funcs(struct amdgpu_device *adev)
  * @src_offset: src GPU address
  * @dst_offset: dst GPU address
  * @byte_count: number of bytes to xfer
- * @tmz: is this a secure operation
+ * @copy_flags: unused
  *
  * Copy GPU buffers using the DMA engine (VI).
  * Used by the amdgpu ttm implementation to move pages if
@@ -771,7 +773,7 @@ static void si_dma_emit_copy_buffer(struct amdgpu_ib *ib,
 				       uint64_t src_offset,
 				       uint64_t dst_offset,
 				       uint32_t byte_count,
-				       bool tmz)
+				       uint32_t copy_flags)
 {
 	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY,
 					      1, 0, 0, byte_count);
diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c
index cada9f300a7f..5237395e4fab 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c
@@ -296,6 +296,8 @@ static const struct amd_ip_funcs si_ih_ip_funcs = {
 	.soft_reset = si_ih_soft_reset,
 	.set_clockgating_state = si_ih_set_clockgating_state,
 	.set_powergating_state = si_ih_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ih_funcs si_ih_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c b/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c
index 93f6772d1b24..481217c32d85 100644
--- a/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c
+++ b/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c
@@ -92,7 +92,7 @@ static int sienna_cichlid_mode2_suspend_ip(struct amdgpu_device *adev)
 		adev->ip_blocks[i].status.hw = false;
 	}
 
-	return r;
+	return 0;
 }
 
 static int
diff --git a/drivers/gpu/drm/amd/amdgpu/smuio_v14_0_2.c b/drivers/gpu/drm/amd/amdgpu/smuio_v14_0_2.c
new file mode 100644
index 000000000000..2a51a70d4846
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/smuio_v14_0_2.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "smuio_v14_0_2.h"
+#include "smuio/smuio_14_0_2_offset.h"
+#include "smuio/smuio_14_0_2_sh_mask.h"
+#include <linux/preempt.h>
+
+static u32 smuio_v14_0_2_get_rom_index_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(SMUIO, 0, regROM_INDEX);
+}
+
+static u32 smuio_v14_0_2_get_rom_data_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(SMUIO, 0, regROM_DATA);
+}
+
+static u64 smuio_v14_0_2_get_gpu_clock_counter(struct amdgpu_device *adev)
+{
+	u64 clock;
+	u64 clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after;
+
+	preempt_disable();
+	clock_counter_hi_pre = (u64)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
+	clock_counter_lo = (u64)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
+	/* the clock counter may be udpated during polling the counters */
+	clock_counter_hi_after = (u64)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
+	if (clock_counter_hi_pre != clock_counter_hi_after)
+		clock_counter_lo = (u64)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
+	preempt_enable();
+
+	clock = clock_counter_lo | (clock_counter_hi_after << 32ULL);
+
+	return clock;
+}
+
+const struct amdgpu_smuio_funcs smuio_v14_0_2_funcs = {
+	.get_rom_index_offset = smuio_v14_0_2_get_rom_index_offset,
+	.get_rom_data_offset = smuio_v14_0_2_get_rom_data_offset,
+	.get_gpu_clock_counter = smuio_v14_0_2_get_gpu_clock_counter,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/smuio_v14_0_2.h b/drivers/gpu/drm/amd/amdgpu/smuio_v14_0_2.h
new file mode 100644
index 000000000000..6e617f832d90
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/smuio_v14_0_2.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SMUIO_V14_0_2_H__
+#define __SMUIO_V14_0_2_H__
+
+#include "soc15_common.h"
+
+extern const struct amdgpu_smuio_funcs smuio_v14_0_2_funcs;
+
+#endif /* __SMUIO_V14_0_2_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index dec81ccf6240..170f02e96717 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -143,7 +143,7 @@ static const struct amdgpu_video_codec_info rn_video_codecs_decode_array[] =
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4096, 52)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1, 4096, 4096, 4)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)},
-	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)},
+	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 16384, 16384, 0)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)},
 };
 
@@ -156,7 +156,7 @@ static const struct amdgpu_video_codecs rn_video_codecs_decode =
 static const struct amdgpu_video_codec_info vcn_4_0_3_video_codecs_decode_array[] = {
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4096, 52)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)},
-	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)},
+	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 16384, 16384, 0)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_AV1, 8192, 4352, 0)},
 };
@@ -502,7 +502,7 @@ static int soc15_asic_baco_reset(struct amdgpu_device *adev)
 static enum amd_reset_method
 soc15_asic_reset_method(struct amdgpu_device *adev)
 {
-	bool baco_reset = false;
+	int baco_reset = 0;
 	bool connected_to_cpu = false;
 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 
@@ -540,7 +540,7 @@ soc15_asic_reset_method(struct amdgpu_device *adev)
 			 */
 			if (ras && adev->ras_enabled &&
 			    adev->pm.fw_version <= 0x283400)
-				baco_reset = false;
+				baco_reset = 0;
 		} else {
 			baco_reset = amdgpu_dpm_is_baco_supported(adev);
 		}
@@ -620,7 +620,7 @@ static int soc15_asic_reset(struct amdgpu_device *adev)
 	}
 }
 
-static bool soc15_supports_baco(struct amdgpu_device *adev)
+static int soc15_supports_baco(struct amdgpu_device *adev)
 {
 	switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
 	case IP_VERSION(9, 0, 0):
@@ -628,13 +628,13 @@ static bool soc15_supports_baco(struct amdgpu_device *adev)
 		if (adev->asic_type == CHIP_VEGA20) {
 			if (adev->psp.sos.fw_version >= 0x80067)
 				return amdgpu_dpm_is_baco_supported(adev);
-			return false;
+			return 0;
 		} else {
 			return amdgpu_dpm_is_baco_supported(adev);
 		}
 		break;
 	default:
-		return false;
+		return 0;
 	}
 }
 
@@ -1501,4 +1501,6 @@ static const struct amd_ip_funcs soc15_common_ip_funcs = {
 	.set_clockgating_state = soc15_common_set_clockgating_state,
 	.set_powergating_state = soc15_common_set_powergating_state,
 	.get_clockgating_state= soc15_common_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.h b/drivers/gpu/drm/amd/amdgpu/soc15.h
index 1444b7765e4b..282584a48be0 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.h
@@ -88,6 +88,8 @@ struct soc15_ras_field_entry {
 };
 
 #define SOC15_REG_ENTRY(ip, inst, reg)	ip##_HWIP, inst, reg##_BASE_IDX, reg
+#define SOC15_REG_ENTRY_STR(ip, inst, reg) \
+	{ ip##_HWIP, inst, reg##_BASE_IDX, reg, #reg }
 
 #define SOC15_REG_ENTRY_OFFSET(entry)	(adev->reg_offset[entry.hwip][entry.inst][entry.seg] + entry.reg_offset)
 
diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c
index 43ca63fe85ac..fb6797467571 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc21.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc21.c
@@ -72,7 +72,7 @@ static const struct amdgpu_video_codecs vcn_4_0_0_video_codecs_encode_vcn1 = {
 static const struct amdgpu_video_codec_info vcn_4_0_0_video_codecs_decode_array_vcn0[] = {
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4096, 52)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)},
-	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)},
+	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 16384, 16384, 0)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_AV1, 8192, 4352, 0)},
 };
@@ -80,7 +80,7 @@ static const struct amdgpu_video_codec_info vcn_4_0_0_video_codecs_decode_array_
 static const struct amdgpu_video_codec_info vcn_4_0_0_video_codecs_decode_array_vcn1[] = {
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC, 4096, 4096, 52)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC, 8192, 4352, 186)},
-	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 4096, 4096, 0)},
+	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG, 16384, 16384, 0)},
 	{codec_info_build(AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9, 8192, 4352, 0)},
 };
 
@@ -985,4 +985,6 @@ static const struct amd_ip_funcs soc21_common_ip_funcs = {
 	.set_clockgating_state = soc21_common_set_clockgating_state,
 	.set_powergating_state = soc21_common_set_powergating_state,
 	.get_clockgating_state = soc21_common_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h b/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
index 056d4df8fa1f..3ac56a9645eb 100644
--- a/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
@@ -146,6 +146,7 @@ struct ta_ras_mca_addr {
 	uint32_t ch_inst;
 	uint32_t umc_inst;
 	uint32_t node_inst;
+	uint32_t socket_id;
 };
 
 struct ta_ras_phy_addr {
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index 450b6e831509..24d49d813607 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -486,6 +486,8 @@ static const struct amd_ip_funcs tonga_ih_ip_funcs = {
 	.post_soft_reset = tonga_ih_post_soft_reset,
 	.set_clockgating_state = tonga_ih_set_clockgating_state,
 	.set_powergating_state = tonga_ih_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ih_funcs tonga_ih_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c
index 77af4e25ff46..bfe61d86ee6c 100644
--- a/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c
@@ -28,27 +28,7 @@
 #include "umc/umc_12_0_0_sh_mask.h"
 #include "mp/mp_13_0_6_sh_mask.h"
 
-const uint32_t
-	umc_v12_0_channel_idx_tbl[]
-			[UMC_V12_0_UMC_INSTANCE_NUM]
-			[UMC_V12_0_CHANNEL_INSTANCE_NUM] = {
-		{{3,   7,   11,  15,  2,   6,   10,  14},  {1,   5,   9,   13,  0,   4,   8,   12},
-		 {19,  23,  27,  31,  18,  22,  26,  30},  {17,  21,  25,  29,  16,  20,  24,  28}},
-		{{47,  43,  39,  35,  46,  42,  38,  34},  {45,  41,  37,  33,  44,  40,  36,  32},
-		 {63,  59,  55,  51,  62,  58,  54,  50},  {61,  57,  53,  49,  60,  56,  52,  48}},
-		{{79,  75,  71,  67,  78,  74,  70,  66},  {77,  73,  69,  65,  76,  72,  68,  64},
-		 {95,  91,  87,  83,  94,  90,  86,  82},  {93,  89,  85,  81,  92,  88,  84,  80}},
-		{{99,  103, 107, 111, 98,  102, 106, 110}, {97,  101, 105, 109, 96,  100, 104, 108},
-		 {115, 119, 123, 127, 114, 118, 122, 126}, {113, 117, 121, 125, 112, 116, 120, 124}}
-	};
-
-/* mapping of MCA error address to normalized address */
-static const uint32_t umc_v12_0_ma2na_mapping[] = {
-	0,  5,  6,  8,  9,  14, 12, 13,
-	10, 11, 15, 16, 17, 18, 19, 20,
-	21, 22, 23, 24, 25, 26, 27, 28,
-	24, 7,  29, 30,
-};
+#define MAX_ECC_NUM_PER_RETIREMENT  32
 
 static inline uint64_t get_umc_v12_0_reg_offset(struct amdgpu_device *adev,
 					    uint32_t node_inst,
@@ -192,99 +172,74 @@ static void umc_v12_0_query_ras_error_count(struct amdgpu_device *adev,
 	umc_v12_0_reset_error_count(adev);
 }
 
-static bool umc_v12_0_bit_wise_xor(uint32_t val)
+static void umc_v12_0_convert_error_address(struct amdgpu_device *adev,
+					struct ras_err_data *err_data,
+					struct ta_ras_query_address_input *addr_in)
 {
-	bool result = 0;
-	int i;
+	uint32_t col, row, row_xor, bank, channel_index;
+	uint64_t soc_pa, retired_page, column, err_addr;
+	struct ta_ras_query_address_output addr_out;
 
-	for (i = 0; i < 32; i++)
-		result = result ^ ((val >> i) & 0x1);
+	err_addr = addr_in->ma.err_addr;
+	addr_in->addr_type = TA_RAS_MCA_TO_PA;
+	if (psp_ras_query_address(&adev->psp, addr_in, &addr_out)) {
+		dev_warn(adev->dev, "Failed to query RAS physical address for 0x%llx",
+			err_addr);
 
-	return result;
-}
+		return;
+	}
+
+	soc_pa = addr_out.pa.pa;
+	bank = addr_out.pa.bank;
+	channel_index = addr_out.pa.channel_idx;
 
-static void umc_v12_0_mca_addr_to_pa(struct amdgpu_device *adev,
-					uint64_t err_addr, uint32_t ch_inst, uint32_t umc_inst,
-					uint32_t node_inst,
-					struct ta_ras_query_address_output *addr_out)
-{
-	uint32_t channel_index, i;
-	uint64_t na, soc_pa;
-	uint32_t bank_hash0, bank_hash1, bank_hash2, bank_hash3, col, row;
-	uint32_t bank0, bank1, bank2, bank3, bank;
-
-	bank_hash0 = (err_addr >> UMC_V12_0_MCA_B0_BIT) & 0x1ULL;
-	bank_hash1 = (err_addr >> UMC_V12_0_MCA_B1_BIT) & 0x1ULL;
-	bank_hash2 = (err_addr >> UMC_V12_0_MCA_B2_BIT) & 0x1ULL;
-	bank_hash3 = (err_addr >> UMC_V12_0_MCA_B3_BIT) & 0x1ULL;
 	col = (err_addr >> 1) & 0x1fULL;
 	row = (err_addr >> 10) & 0x3fffULL;
+	row_xor = row ^ (0x1ULL << 13);
+	/* clear [C3 C2] in soc physical address */
+	soc_pa &= ~(0x3ULL << UMC_V12_0_PA_C2_BIT);
+	/* clear [C4] in soc physical address */
+	soc_pa &= ~(0x1ULL << UMC_V12_0_PA_C4_BIT);
+
+	/* loop for all possibilities of [C4 C3 C2] */
+	for (column = 0; column < UMC_V12_0_NA_MAP_PA_NUM; column++) {
+		retired_page = soc_pa | ((column & 0x3) << UMC_V12_0_PA_C2_BIT);
+		retired_page |= (((column & 0x4) >> 2) << UMC_V12_0_PA_C4_BIT);
+		/* include column bit 0 and 1 */
+		col &= 0x3;
+		col |= (column << 2);
+		dev_info(adev->dev,
+			"Error Address(PA):0x%-10llx Row:0x%-4x Col:0x%-2x Bank:0x%x Channel:0x%x\n",
+			retired_page, row, col, bank, channel_index);
+		amdgpu_umc_fill_error_record(err_data, err_addr,
+			retired_page, channel_index, addr_in->ma.umc_inst);
 
-	/* apply bank hash algorithm */
-	bank0 =
-		bank_hash0 ^ (UMC_V12_0_XOR_EN0 &
-		(umc_v12_0_bit_wise_xor(col & UMC_V12_0_COL_XOR0) ^
-		(umc_v12_0_bit_wise_xor(row & UMC_V12_0_ROW_XOR0))));
-	bank1 =
-		bank_hash1 ^ (UMC_V12_0_XOR_EN1 &
-		(umc_v12_0_bit_wise_xor(col & UMC_V12_0_COL_XOR1) ^
-		(umc_v12_0_bit_wise_xor(row & UMC_V12_0_ROW_XOR1))));
-	bank2 =
-		bank_hash2 ^ (UMC_V12_0_XOR_EN2 &
-		(umc_v12_0_bit_wise_xor(col & UMC_V12_0_COL_XOR2) ^
-		(umc_v12_0_bit_wise_xor(row & UMC_V12_0_ROW_XOR2))));
-	bank3 =
-		bank_hash3 ^ (UMC_V12_0_XOR_EN3 &
-		(umc_v12_0_bit_wise_xor(col & UMC_V12_0_COL_XOR3) ^
-		(umc_v12_0_bit_wise_xor(row & UMC_V12_0_ROW_XOR3))));
-
-	bank = bank0 | (bank1 << 1) | (bank2 << 2) | (bank3 << 3);
-	err_addr &= ~0x3c0ULL;
-	err_addr |= (bank << UMC_V12_0_MCA_B0_BIT);
-
-	na = 0x0;
-	/* convert mca error address to normalized address */
-	for (i = 1; i < ARRAY_SIZE(umc_v12_0_ma2na_mapping); i++)
-		na |= ((err_addr >> i) & 0x1ULL) << umc_v12_0_ma2na_mapping[i];
-
-	channel_index =
-		adev->umc.channel_idx_tbl[node_inst * adev->umc.umc_inst_num *
-			adev->umc.channel_inst_num +
-			umc_inst * adev->umc.channel_inst_num +
-			ch_inst];
-	/* translate umc channel address to soc pa, 3 parts are included */
-	soc_pa = ADDR_OF_32KB_BLOCK(na) |
-		ADDR_OF_256B_BLOCK(channel_index) |
-		OFFSET_IN_256B_BLOCK(na);
-
-	/* the umc channel bits are not original values, they are hashed */
-	UMC_V12_0_SET_CHANNEL_HASH(channel_index, soc_pa);
-
-	addr_out->pa.pa = soc_pa;
-	addr_out->pa.bank = bank;
-	addr_out->pa.channel_idx = channel_index;
+		/* shift R13 bit */
+		retired_page ^= (0x1ULL << UMC_V12_0_PA_R13_BIT);
+		dev_info(adev->dev,
+			"Error Address(PA):0x%-10llx Row:0x%-4x Col:0x%-2x Bank:0x%x Channel:0x%x\n",
+			retired_page, row_xor, col, bank, channel_index);
+		amdgpu_umc_fill_error_record(err_data, err_addr,
+			retired_page, channel_index, addr_in->ma.umc_inst);
+	}
 }
 
-static void umc_v12_0_convert_error_address(struct amdgpu_device *adev,
-					    struct ras_err_data *err_data, uint64_t err_addr,
-					    uint32_t ch_inst, uint32_t umc_inst,
-					    uint32_t node_inst)
+static int umc_v12_0_convert_err_addr(struct amdgpu_device *adev,
+				struct ta_ras_query_address_input *addr_in,
+				uint64_t *pfns, int len)
 {
 	uint32_t col, row, row_xor, bank, channel_index;
-	uint64_t soc_pa, retired_page, column;
-	struct ta_ras_query_address_input addr_in;
+	uint64_t soc_pa, retired_page, column, err_addr;
 	struct ta_ras_query_address_output addr_out;
+	uint32_t pos = 0;
 
-	addr_in.addr_type = TA_RAS_MCA_TO_PA;
-	addr_in.ma.err_addr = err_addr;
-	addr_in.ma.ch_inst = ch_inst;
-	addr_in.ma.umc_inst = umc_inst;
-	addr_in.ma.node_inst = node_inst;
-
-	if (psp_ras_query_address(&adev->psp, &addr_in, &addr_out))
-		/* fallback to old path if fail to get pa from psp */
-		umc_v12_0_mca_addr_to_pa(adev, err_addr, ch_inst, umc_inst,
-				node_inst, &addr_out);
+	err_addr = addr_in->ma.err_addr;
+	addr_in->addr_type = TA_RAS_MCA_TO_PA;
+	if (psp_ras_query_address(&adev->psp, addr_in, &addr_out)) {
+		dev_warn(adev->dev, "Failed to query RAS physical address for 0x%llx",
+			err_addr);
+		return 0;
+	}
 
 	soc_pa = addr_out.pa.pa;
 	bank = addr_out.pa.bank;
@@ -302,33 +257,42 @@ static void umc_v12_0_convert_error_address(struct amdgpu_device *adev,
 	for (column = 0; column < UMC_V12_0_NA_MAP_PA_NUM; column++) {
 		retired_page = soc_pa | ((column & 0x3) << UMC_V12_0_PA_C2_BIT);
 		retired_page |= (((column & 0x4) >> 2) << UMC_V12_0_PA_C4_BIT);
+
+		if (pos >= len)
+			return 0;
+		pfns[pos++] = retired_page >> AMDGPU_GPU_PAGE_SHIFT;
+
 		/* include column bit 0 and 1 */
 		col &= 0x3;
 		col |= (column << 2);
 		dev_info(adev->dev,
 			"Error Address(PA):0x%-10llx Row:0x%-4x Col:0x%-2x Bank:0x%x Channel:0x%x\n",
 			retired_page, row, col, bank, channel_index);
-		amdgpu_umc_fill_error_record(err_data, err_addr,
-			retired_page, channel_index, umc_inst);
 
 		/* shift R13 bit */
 		retired_page ^= (0x1ULL << UMC_V12_0_PA_R13_BIT);
+
+		if (pos >= len)
+			return 0;
+		pfns[pos++] = retired_page >> AMDGPU_GPU_PAGE_SHIFT;
+
 		dev_info(adev->dev,
 			"Error Address(PA):0x%-10llx Row:0x%-4x Col:0x%-2x Bank:0x%x Channel:0x%x\n",
 			retired_page, row_xor, col, bank, channel_index);
-		amdgpu_umc_fill_error_record(err_data, err_addr,
-			retired_page, channel_index, umc_inst);
 	}
+
+	return pos;
 }
 
 static int umc_v12_0_query_error_address(struct amdgpu_device *adev,
 					uint32_t node_inst, uint32_t umc_inst,
 					uint32_t ch_inst, void *data)
 {
+	struct ras_err_data *err_data = (struct ras_err_data *)data;
+	struct ta_ras_query_address_input addr_in;
 	uint64_t mc_umc_status_addr;
 	uint64_t mc_umc_status, err_addr;
 	uint64_t mc_umc_addrt0;
-	struct ras_err_data *err_data = (struct ras_err_data *)data;
 	uint64_t umc_reg_offset =
 		get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
 
@@ -357,8 +321,19 @@ static int umc_v12_0_query_error_address(struct amdgpu_device *adev,
 
 		err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
 
-		umc_v12_0_convert_error_address(adev, err_data, err_addr,
-					ch_inst, umc_inst, node_inst);
+		if (!adev->aid_mask &&
+		    adev->smuio.funcs &&
+		    adev->smuio.funcs->get_socket_id)
+			addr_in.ma.socket_id = adev->smuio.funcs->get_socket_id(adev);
+		else
+			addr_in.ma.socket_id = 0;
+
+		addr_in.ma.err_addr = err_addr;
+		addr_in.ma.ch_inst = ch_inst;
+		addr_in.ma.umc_inst = umc_inst;
+		addr_in.ma.node_inst = node_inst;
+
+		umc_v12_0_convert_error_address(adev, err_data, &addr_in);
 	}
 
 	/* clear umc status */
@@ -401,13 +376,20 @@ static int umc_v12_0_err_cnt_init_per_channel(struct amdgpu_device *adev,
 	return 0;
 }
 
+#ifdef TO_BE_REMOVED
 static void umc_v12_0_ecc_info_query_ras_error_count(struct amdgpu_device *adev,
 					void *ras_error_status)
 {
+	struct ras_query_context qctx;
+
+	memset(&qctx, 0, sizeof(qctx));
+	qctx.event_id = amdgpu_ras_acquire_event_id(adev, amdgpu_ras_intr_triggered() ?
+						    RAS_EVENT_TYPE_ISR : RAS_EVENT_TYPE_INVALID);
+
 	amdgpu_mca_smu_log_ras_error(adev,
-		AMDGPU_RAS_BLOCK__UMC, AMDGPU_MCA_ERROR_TYPE_CE, ras_error_status);
+		AMDGPU_RAS_BLOCK__UMC, AMDGPU_MCA_ERROR_TYPE_CE, ras_error_status, &qctx);
 	amdgpu_mca_smu_log_ras_error(adev,
-		AMDGPU_RAS_BLOCK__UMC, AMDGPU_MCA_ERROR_TYPE_UE, ras_error_status);
+		AMDGPU_RAS_BLOCK__UMC, AMDGPU_MCA_ERROR_TYPE_UE, ras_error_status, &qctx);
 }
 
 static void umc_v12_0_ecc_info_query_ras_error_address(struct amdgpu_device *adev,
@@ -418,12 +400,16 @@ static void umc_v12_0_ecc_info_query_ras_error_address(struct amdgpu_device *ade
 	struct ras_err_info *err_info;
 	struct ras_err_addr *mca_err_addr, *tmp;
 	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
+	struct ta_ras_query_address_input addr_in;
 
 	for_each_ras_error(err_node, err_data) {
 		err_info = &err_node->err_info;
 		if (list_empty(&err_info->err_addr_list))
 			continue;
 
+		addr_in.ma.node_inst = err_info->mcm_info.die_id;
+		addr_in.ma.socket_id = err_info->mcm_info.socket_id;
+
 		list_for_each_entry_safe(mca_err_addr, tmp, &err_info->err_addr_list, node) {
 			mc_umc_status = mca_err_addr->err_status;
 			if (mc_umc_status &&
@@ -439,6 +425,10 @@ static void umc_v12_0_ecc_info_query_ras_error_address(struct amdgpu_device *ade
 							MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
 				InstanceIdLo = REG_GET_FIELD(mca_ipid, MCMP1_IPIDT0, InstanceIdLo);
 
+				addr_in.ma.err_addr = err_addr;
+				addr_in.ma.ch_inst = MCA_IPID_LO_2_UMC_CH(InstanceIdLo);
+				addr_in.ma.umc_inst = MCA_IPID_LO_2_UMC_INST(InstanceIdLo);
+
 				dev_info(adev->dev, "UMC:IPID:0x%llx, aid:%d, inst:%d, ch:%d, err_addr:0x%llx\n",
 					mca_ipid,
 					err_info->mcm_info.die_id,
@@ -447,10 +437,7 @@ static void umc_v12_0_ecc_info_query_ras_error_address(struct amdgpu_device *ade
 					err_addr);
 
 				umc_v12_0_convert_error_address(adev,
-					err_data, err_addr,
-					MCA_IPID_LO_2_UMC_CH(InstanceIdLo),
-					MCA_IPID_LO_2_UMC_INST(InstanceIdLo),
-					err_info->mcm_info.die_id);
+					err_data, &addr_in);
 			}
 
 			/* Delete error address node from list and free memory */
@@ -458,6 +445,7 @@ static void umc_v12_0_ecc_info_query_ras_error_address(struct amdgpu_device *ade
 		}
 	}
 }
+#endif
 
 static bool umc_v12_0_check_ecc_err_status(struct amdgpu_device *adev,
 			enum amdgpu_mca_error_type type, void *ras_error_status)
@@ -498,43 +486,49 @@ const struct amdgpu_ras_block_hw_ops umc_v12_0_ras_hw_ops = {
 	.query_ras_error_address = umc_v12_0_query_ras_error_address,
 };
 
-static int umc_v12_0_aca_bank_generate_report(struct aca_handle *handle, struct aca_bank *bank, enum aca_error_type type,
-					      struct aca_bank_report *report, void *data)
+static int umc_v12_0_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
+				     enum aca_smu_type type, void *data)
 {
 	struct amdgpu_device *adev = handle->adev;
-	u64 status;
+	struct aca_bank_info info;
+	enum aca_error_type err_type;
+	u64 status, count;
+	u32 ext_error_code;
 	int ret;
 
-	ret = aca_bank_info_decode(bank, &report->info);
+	status = bank->regs[ACA_REG_IDX_STATUS];
+	if (umc_v12_0_is_deferred_error(adev, status))
+		err_type = ACA_ERROR_TYPE_DEFERRED;
+	else if (umc_v12_0_is_uncorrectable_error(adev, status))
+		err_type = ACA_ERROR_TYPE_UE;
+	else if (umc_v12_0_is_correctable_error(adev, status))
+		err_type = ACA_ERROR_TYPE_CE;
+	else
+		return 0;
+
+	ret = aca_bank_info_decode(bank, &info);
 	if (ret)
 		return ret;
 
-	status = bank->regs[ACA_REG_IDX_STATUS];
-	switch (type) {
-	case ACA_ERROR_TYPE_UE:
-		if (umc_v12_0_is_uncorrectable_error(adev, status)) {
-			report->count[type] = 1;
-		}
-		break;
-	case ACA_ERROR_TYPE_CE:
-		if (umc_v12_0_is_correctable_error(adev, status)) {
-			report->count[type] = 1;
-		}
-		break;
-	default:
-		return -EINVAL;
-	}
+	amdgpu_umc_update_ecc_status(adev,
+		bank->regs[ACA_REG_IDX_STATUS],
+		bank->regs[ACA_REG_IDX_IPID],
+		bank->regs[ACA_REG_IDX_ADDR]);
 
-	return 0;
+	ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
+	count = ext_error_code == 0 ?
+		ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]) : 1ULL;
+
+	return aca_error_cache_log_bank_error(handle, &info, err_type, count);
 }
 
 static const struct aca_bank_ops umc_v12_0_aca_bank_ops = {
-	.aca_bank_generate_report = umc_v12_0_aca_bank_generate_report,
+	.aca_bank_parser = umc_v12_0_aca_bank_parser,
 };
 
 const struct aca_info umc_v12_0_aca_info = {
 	.hwip = ACA_HWIP_TYPE_UMC,
-	.mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK,
+	.mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK | ACA_ERROR_DEFERRED_MASK,
 	.bank_ops = &umc_v12_0_aca_bank_ops,
 };
 
@@ -554,6 +548,152 @@ static int umc_v12_0_ras_late_init(struct amdgpu_device *adev, struct ras_common
 	return 0;
 }
 
+static int umc_v12_0_update_ecc_status(struct amdgpu_device *adev,
+			uint64_t status, uint64_t ipid, uint64_t addr)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	uint16_t hwid, mcatype;
+	struct ta_ras_query_address_input addr_in;
+	uint64_t page_pfn[UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL];
+	uint64_t err_addr, hash_val = 0;
+	struct ras_ecc_err *ecc_err;
+	int count;
+	int ret;
+
+	hwid = REG_GET_FIELD(ipid, MCMP1_IPIDT0, HardwareID);
+	mcatype = REG_GET_FIELD(ipid, MCMP1_IPIDT0, McaType);
+
+	if ((hwid != MCA_UMC_HWID_V12_0) || (mcatype != MCA_UMC_MCATYPE_V12_0))
+		return 0;
+
+	if (!status)
+		return 0;
+
+	if (!umc_v12_0_is_deferred_error(adev, status))
+		return 0;
+
+	err_addr = REG_GET_FIELD(addr,
+				MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
+
+	dev_info(adev->dev,
+		"UMC:IPID:0x%llx, socket:%llu, aid:%llu, inst:%llu, ch:%llu, err_addr:0x%llx\n",
+		ipid,
+		MCA_IPID_2_SOCKET_ID(ipid),
+		MCA_IPID_2_DIE_ID(ipid),
+		MCA_IPID_2_UMC_INST(ipid),
+		MCA_IPID_2_UMC_CH(ipid),
+		err_addr);
+
+	memset(page_pfn, 0, sizeof(page_pfn));
+
+	memset(&addr_in, 0, sizeof(addr_in));
+	addr_in.ma.err_addr = err_addr;
+	addr_in.ma.ch_inst = MCA_IPID_2_UMC_CH(ipid);
+	addr_in.ma.umc_inst = MCA_IPID_2_UMC_INST(ipid);
+	addr_in.ma.node_inst = MCA_IPID_2_DIE_ID(ipid);
+	addr_in.ma.socket_id = MCA_IPID_2_SOCKET_ID(ipid);
+
+	count = umc_v12_0_convert_err_addr(adev,
+				&addr_in, page_pfn, ARRAY_SIZE(page_pfn));
+	if (count <= 0) {
+		dev_warn(adev->dev, "Fail to convert error address! count:%d\n", count);
+		return 0;
+	}
+
+	ret = amdgpu_umc_build_pages_hash(adev,
+			page_pfn, count, &hash_val);
+	if (ret) {
+		dev_err(adev->dev, "Fail to build error pages hash\n");
+		return ret;
+	}
+
+	ecc_err = kzalloc(sizeof(*ecc_err), GFP_KERNEL);
+	if (!ecc_err)
+		return -ENOMEM;
+
+	ecc_err->err_pages.pfn = kcalloc(count, sizeof(*ecc_err->err_pages.pfn), GFP_KERNEL);
+	if (!ecc_err->err_pages.pfn) {
+		kfree(ecc_err);
+		return -ENOMEM;
+	}
+
+	memcpy(ecc_err->err_pages.pfn, page_pfn, count * sizeof(*ecc_err->err_pages.pfn));
+	ecc_err->err_pages.count = count;
+
+	ecc_err->hash_index = hash_val;
+	ecc_err->status = status;
+	ecc_err->ipid = ipid;
+	ecc_err->addr = addr;
+
+	ret = amdgpu_umc_logs_ecc_err(adev, &con->umc_ecc_log.de_page_tree, ecc_err);
+	if (ret) {
+		if (ret == -EEXIST)
+			con->umc_ecc_log.de_updated = true;
+		else
+			dev_err(adev->dev, "Fail to log ecc error! ret:%d\n", ret);
+
+		kfree(ecc_err->err_pages.pfn);
+		kfree(ecc_err);
+		return ret;
+	}
+
+	con->umc_ecc_log.de_updated = true;
+
+	return 0;
+}
+
+static int umc_v12_0_fill_error_record(struct amdgpu_device *adev,
+				struct ras_ecc_err *ecc_err, void *ras_error_status)
+{
+	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
+	uint32_t i = 0;
+	int ret = 0;
+
+	if (!err_data || !ecc_err)
+		return -EINVAL;
+
+	for (i = 0; i < ecc_err->err_pages.count; i++) {
+		ret = amdgpu_umc_fill_error_record(err_data,
+				ecc_err->addr,
+				ecc_err->err_pages.pfn[i] << AMDGPU_GPU_PAGE_SHIFT,
+				MCA_IPID_2_UMC_CH(ecc_err->ipid),
+				MCA_IPID_2_UMC_INST(ecc_err->ipid));
+		if (ret)
+			break;
+	}
+
+	err_data->de_count++;
+
+	return ret;
+}
+
+static void umc_v12_0_query_ras_ecc_err_addr(struct amdgpu_device *adev,
+					void *ras_error_status)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_ecc_err *entries[MAX_ECC_NUM_PER_RETIREMENT];
+	struct radix_tree_root *ecc_tree;
+	int new_detected, ret, i;
+
+	ecc_tree = &con->umc_ecc_log.de_page_tree;
+
+	mutex_lock(&con->umc_ecc_log.lock);
+	new_detected = radix_tree_gang_lookup_tag(ecc_tree, (void **)entries,
+			0, ARRAY_SIZE(entries), UMC_ECC_NEW_DETECTED_TAG);
+	for (i = 0; i < new_detected; i++) {
+		if (!entries[i])
+			continue;
+
+		ret = umc_v12_0_fill_error_record(adev, entries[i], ras_error_status);
+		if (ret) {
+			dev_err(adev->dev, "Fail to fill umc error record, ret:%d\n", ret);
+			break;
+		}
+		radix_tree_tag_clear(ecc_tree, entries[i]->hash_index, UMC_ECC_NEW_DETECTED_TAG);
+	}
+	mutex_unlock(&con->umc_ecc_log.lock);
+}
+
 struct amdgpu_umc_ras umc_v12_0_ras = {
 	.ras_block = {
 		.hw_ops = &umc_v12_0_ras_hw_ops,
@@ -561,8 +701,8 @@ struct amdgpu_umc_ras umc_v12_0_ras = {
 	},
 	.err_cnt_init = umc_v12_0_err_cnt_init,
 	.query_ras_poison_mode = umc_v12_0_query_ras_poison_mode,
-	.ecc_info_query_ras_error_count = umc_v12_0_ecc_info_query_ras_error_count,
-	.ecc_info_query_ras_error_address = umc_v12_0_ecc_info_query_ras_error_address,
+	.ecc_info_query_ras_error_address = umc_v12_0_query_ras_ecc_err_addr,
 	.check_ecc_err_status = umc_v12_0_check_ecc_err_status,
+	.update_ecc_status = umc_v12_0_update_ecc_status,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v12_0.h b/drivers/gpu/drm/amd/amdgpu/umc_v12_0.h
index 5973bfb14fce..b4974793850b 100644
--- a/drivers/gpu/drm/amd/amdgpu/umc_v12_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/umc_v12_0.h
@@ -55,83 +55,38 @@
 #define UMC_V12_0_NA_MAP_PA_NUM        8
 /* R13 bit shift should be considered, double the number */
 #define UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL (UMC_V12_0_NA_MAP_PA_NUM * 2)
-/* bank bits in MCA error address */
-#define UMC_V12_0_MCA_B0_BIT 6
-#define UMC_V12_0_MCA_B1_BIT 7
-#define UMC_V12_0_MCA_B2_BIT 8
-#define UMC_V12_0_MCA_B3_BIT 9
+
 /* column bits in SOC physical address */
 #define UMC_V12_0_PA_C2_BIT 15
 #define UMC_V12_0_PA_C4_BIT 21
 /* row bits in SOC physical address */
 #define UMC_V12_0_PA_R13_BIT 35
-/* channel index bits in SOC physical address */
-#define UMC_V12_0_PA_CH4_BIT 12
-#define UMC_V12_0_PA_CH5_BIT 13
-#define UMC_V12_0_PA_CH6_BIT 14
-
-/* bank hash settings */
-#define UMC_V12_0_XOR_EN0 1
-#define UMC_V12_0_XOR_EN1 1
-#define UMC_V12_0_XOR_EN2 1
-#define UMC_V12_0_XOR_EN3 1
-#define UMC_V12_0_COL_XOR0 0x0
-#define UMC_V12_0_COL_XOR1 0x0
-#define UMC_V12_0_COL_XOR2 0x800
-#define UMC_V12_0_COL_XOR3 0x1000
-#define UMC_V12_0_ROW_XOR0 0x11111
-#define UMC_V12_0_ROW_XOR1 0x22222
-#define UMC_V12_0_ROW_XOR2 0x4444
-#define UMC_V12_0_ROW_XOR3 0x8888
-
-/* channel hash settings */
-#define UMC_V12_0_HASH_4K 0
-#define UMC_V12_0_HASH_64K 1
-#define UMC_V12_0_HASH_2M 1
-#define UMC_V12_0_HASH_1G 1
-#define UMC_V12_0_HASH_1T 1
-
-/* XOR some bits of PA into CH4~CH6 bits (bits 12~14 of PA),
- * hash bit is only effective when related setting is enabled
- */
-#define UMC_V12_0_CHANNEL_HASH_CH4(channel_idx, pa) ((((channel_idx) >> 5) & 0x1) ^ \
-				(((pa)  >> 20) & 0x1ULL & UMC_V12_0_HASH_64K) ^ \
-				(((pa)  >> 27) & 0x1ULL & UMC_V12_0_HASH_2M) ^ \
-				(((pa)  >> 34) & 0x1ULL & UMC_V12_0_HASH_1G) ^ \
-				(((pa)  >> 41) & 0x1ULL & UMC_V12_0_HASH_1T))
-#define UMC_V12_0_CHANNEL_HASH_CH5(channel_idx, pa) ((((channel_idx) >> 6) & 0x1) ^ \
-				(((pa)  >> 21) & 0x1ULL & UMC_V12_0_HASH_64K) ^ \
-				(((pa)  >> 28) & 0x1ULL & UMC_V12_0_HASH_2M) ^ \
-				(((pa)  >> 35) & 0x1ULL & UMC_V12_0_HASH_1G) ^ \
-				(((pa)  >> 42) & 0x1ULL & UMC_V12_0_HASH_1T))
-#define UMC_V12_0_CHANNEL_HASH_CH6(channel_idx, pa) ((((channel_idx) >> 4) & 0x1) ^ \
-				(((pa)  >> 19) & 0x1ULL & UMC_V12_0_HASH_64K) ^ \
-				(((pa)  >> 26) & 0x1ULL & UMC_V12_0_HASH_2M) ^ \
-				(((pa)  >> 33) & 0x1ULL & UMC_V12_0_HASH_1G) ^ \
-				(((pa)  >> 40) & 0x1ULL & UMC_V12_0_HASH_1T) ^ \
-				(((pa)  >> 47) & 0x1ULL & UMC_V12_0_HASH_4K))
-#define UMC_V12_0_SET_CHANNEL_HASH(channel_idx, pa) do { \
-		(pa) &= ~(0x7ULL << UMC_V12_0_PA_CH4_BIT); \
-		(pa) |= (UMC_V12_0_CHANNEL_HASH_CH4(channel_idx, pa) << UMC_V12_0_PA_CH4_BIT); \
-		(pa) |= (UMC_V12_0_CHANNEL_HASH_CH5(channel_idx, pa) << UMC_V12_0_PA_CH5_BIT); \
-		(pa) |= (UMC_V12_0_CHANNEL_HASH_CH6(channel_idx, pa) << UMC_V12_0_PA_CH6_BIT); \
-	} while (0)
+
+#define MCA_UMC_HWID_V12_0     0x96
+#define MCA_UMC_MCATYPE_V12_0  0x0
 
 #define MCA_IPID_LO_2_UMC_CH(_ipid_lo) (((((_ipid_lo) >> 20) & 0x1) * 4) + \
 			(((_ipid_lo) >> 12) & 0xF))
 #define MCA_IPID_LO_2_UMC_INST(_ipid_lo) (((_ipid_lo) >> 21) & 0x7)
 
+#define MCA_IPID_2_DIE_ID(ipid)  ((REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdHi) >> 2) & 0x03)
+
+#define MCA_IPID_2_UMC_CH(ipid) \
+	(MCA_IPID_LO_2_UMC_CH(REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdLo)))
+
+#define MCA_IPID_2_UMC_INST(ipid) \
+	(MCA_IPID_LO_2_UMC_INST(REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdLo)))
+
+#define MCA_IPID_2_SOCKET_ID(ipid) \
+	(((REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdLo) & 0x1) << 2) | \
+	 (REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdHi) & 0x03))
+
 bool umc_v12_0_is_deferred_error(struct amdgpu_device *adev, uint64_t mc_umc_status);
 bool umc_v12_0_is_uncorrectable_error(struct amdgpu_device *adev, uint64_t mc_umc_status);
 bool umc_v12_0_is_correctable_error(struct amdgpu_device *adev, uint64_t mc_umc_status);
 
 typedef bool (*check_error_type_func)(struct amdgpu_device *adev, uint64_t mc_umc_status);
 
-extern const uint32_t
-	umc_v12_0_channel_idx_tbl[]
-			[UMC_V12_0_UMC_INSTANCE_NUM]
-			[UMC_V12_0_CHANNEL_INSTANCE_NUM];
-
 extern struct amdgpu_umc_ras umc_v12_0_ras;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c b/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c
index c4c77257710c..a32f87992f20 100644
--- a/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c
+++ b/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c
@@ -442,11 +442,6 @@ static void umc_v8_10_ecc_info_query_ras_error_address(struct amdgpu_device *ade
 		umc_v8_10_ecc_info_query_error_address, ras_error_status);
 }
 
-static void umc_v8_10_set_eeprom_table_version(struct amdgpu_ras_eeprom_table_header *hdr)
-{
-	hdr->version = RAS_TABLE_VER_V2_1;
-}
-
 const struct amdgpu_ras_block_hw_ops umc_v8_10_ras_hw_ops = {
 	.query_ras_error_count = umc_v8_10_query_ras_error_count,
 	.query_ras_error_address = umc_v8_10_query_ras_error_address,
@@ -460,5 +455,4 @@ struct amdgpu_umc_ras umc_v8_10_ras = {
 	.query_ras_poison_mode = umc_v8_10_query_ras_poison_mode,
 	.ecc_info_query_ras_error_count = umc_v8_10_ecc_info_query_ras_error_count,
 	.ecc_info_query_ras_error_address = umc_v8_10_ecc_info_query_ras_error_address,
-	.set_eeprom_table_version = umc_v8_10_set_eeprom_table_version,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c
index a6006f231c65..805d6662c88b 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c
@@ -819,6 +819,8 @@ static const struct amd_ip_funcs uvd_v3_1_ip_funcs = {
 	.soft_reset = uvd_v3_1_soft_reset,
 	.set_clockgating_state = uvd_v3_1_set_clockgating_state,
 	.set_powergating_state = uvd_v3_1_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version uvd_v3_1_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
index 1aa09ad7bbe3..3f19c606f4de 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
@@ -769,6 +769,8 @@ static const struct amd_ip_funcs uvd_v4_2_ip_funcs = {
 	.soft_reset = uvd_v4_2_soft_reset,
 	.set_clockgating_state = uvd_v4_2_set_clockgating_state,
 	.set_powergating_state = uvd_v4_2_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs uvd_v4_2_ring_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index f8b229b75435..efd903c21d48 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -877,6 +877,8 @@ static const struct amd_ip_funcs uvd_v5_0_ip_funcs = {
 	.set_clockgating_state = uvd_v5_0_set_clockgating_state,
 	.set_powergating_state = uvd_v5_0_set_powergating_state,
 	.get_clockgating_state = uvd_v5_0_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index a9a6880f44e3..495de5068455 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -1545,6 +1545,8 @@ static const struct amd_ip_funcs uvd_v6_0_ip_funcs = {
 	.set_clockgating_state = uvd_v6_0_set_clockgating_state,
 	.set_powergating_state = uvd_v6_0_set_powergating_state,
 	.get_clockgating_state = uvd_v6_0_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs uvd_v6_0_ring_phys_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
index a08e7abca423..66fada199bda 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
@@ -626,6 +626,8 @@ static const struct amd_ip_funcs vce_v2_0_ip_funcs = {
 	.soft_reset = vce_v2_0_soft_reset,
 	.set_clockgating_state = vce_v2_0_set_clockgating_state,
 	.set_powergating_state = vce_v2_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs vce_v2_0_ring_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index f4760748d349..32517c364cf7 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -913,6 +913,8 @@ static const struct amd_ip_funcs vce_v3_0_ip_funcs = {
 	.set_clockgating_state = vce_v3_0_set_clockgating_state,
 	.set_powergating_state = vce_v3_0_set_powergating_state,
 	.get_clockgating_state = vce_v3_0_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs vce_v3_0_ring_phys_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
index aaceecd558cf..cb253bd3a2a2 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
@@ -1902,6 +1902,8 @@ static const struct amd_ip_funcs vcn_v1_0_ip_funcs = {
 	.post_soft_reset = NULL /* vcn_v1_0_post_soft_reset */,
 	.set_clockgating_state = vcn_v1_0_set_clockgating_state,
 	.set_powergating_state = vcn_v1_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 /*
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
index e357d8cf0c01..f18fd61c435e 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
@@ -2008,6 +2008,8 @@ static const struct amd_ip_funcs vcn_v2_0_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = vcn_v2_0_set_clockgating_state,
 	.set_powergating_state = vcn_v2_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ring_funcs vcn_v2_0_dec_ring_vm_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
index 1cd8a94b0fbc..baec14bde2a2 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
@@ -1901,6 +1901,8 @@ static const struct amd_ip_funcs vcn_v2_5_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = vcn_v2_5_set_clockgating_state,
 	.set_powergating_state = vcn_v2_5_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amd_ip_funcs vcn_v2_6_ip_funcs = {
@@ -1921,6 +1923,8 @@ static const struct amd_ip_funcs vcn_v2_6_ip_funcs = {
         .post_soft_reset = NULL,
         .set_clockgating_state = vcn_v2_5_set_clockgating_state,
         .set_powergating_state = vcn_v2_5_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version vcn_v2_5_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
index 8f82fb887e9c..6b31cf4b8aac 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
@@ -359,6 +359,7 @@ static int vcn_v3_0_hw_init(void *handle)
 		}
 	}
 
+	return 0;
 done:
 	if (!r)
 		DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
@@ -2230,6 +2231,8 @@ static const struct amd_ip_funcs vcn_v3_0_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = vcn_v3_0_set_clockgating_state,
 	.set_powergating_state = vcn_v3_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version vcn_v3_0_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
index 832d15f7b5f6..ac1b8ead03b3 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
@@ -288,6 +288,7 @@ static int vcn_v4_0_hw_init(void *handle)
 		}
 	}
 
+	return 0;
 done:
 	if (!r)
 		DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
@@ -2130,6 +2131,8 @@ static const struct amd_ip_funcs vcn_v4_0_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = vcn_v4_0_set_clockgating_state,
 	.set_powergating_state = vcn_v4_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version vcn_v4_0_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
index 203fa988322b..2279d8fce03d 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
@@ -1660,6 +1660,8 @@ static const struct amd_ip_funcs vcn_v4_0_3_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = vcn_v4_0_3_set_clockgating_state,
 	.set_powergating_state = vcn_v4_0_3_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version vcn_v4_0_3_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
index 501e53e69f2a..81fb99729f37 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
@@ -237,6 +237,7 @@ static int vcn_v4_0_5_hw_init(void *handle)
 			goto done;
 	}
 
+	return 0;
 done:
 	if (!r)
 		DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
@@ -1752,6 +1753,8 @@ static const struct amd_ip_funcs vcn_v4_0_5_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = vcn_v4_0_5_set_clockgating_state,
 	.set_powergating_state = vcn_v4_0_5_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version vcn_v4_0_5_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
index bc60c554eb32..851975b5ce29 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
@@ -95,7 +95,7 @@ static int vcn_v5_0_0_sw_init(void *handle)
 		return r;
 
 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
-		volatile struct amdgpu_vcn4_fw_shared *fw_shared;
+		volatile struct amdgpu_vcn5_fw_shared *fw_shared;
 
 		if (adev->vcn.harvest_config & (1 << i))
 			continue;
@@ -154,7 +154,7 @@ static int vcn_v5_0_0_sw_fini(void *handle)
 
 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
-			volatile struct amdgpu_vcn4_fw_shared *fw_shared;
+			volatile struct amdgpu_vcn5_fw_shared *fw_shared;
 
 			if (adev->vcn.harvest_config & (1 << i))
 				continue;
@@ -203,6 +203,7 @@ static int vcn_v5_0_0_hw_init(void *handle)
 			goto done;
 	}
 
+	return 0;
 done:
 	if (!r)
 		DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
@@ -334,7 +335,7 @@ static void vcn_v5_0_0_mc_resume(struct amdgpu_device *adev, int inst)
 		upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
 	WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
 	WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_SIZE0,
-		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
+		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn5_fw_shared)));
 }
 
 /**
@@ -438,7 +439,7 @@ static void vcn_v5_0_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_i
 		VCN, inst_idx, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
 	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 		VCN, inst_idx, regUVD_VCPU_NONCACHE_SIZE0),
-		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect);
+		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn5_fw_shared)), 0, indirect);
 
 	/* VCN global tiling registers */
 	WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
@@ -615,7 +616,7 @@ static void vcn_v5_0_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
  */
 static int vcn_v5_0_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
 {
-	volatile struct amdgpu_vcn4_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
+	volatile struct amdgpu_vcn5_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
 	struct amdgpu_ring *ring;
 	uint32_t tmp;
 
@@ -712,7 +713,7 @@ static int vcn_v5_0_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, b
  */
 static int vcn_v5_0_0_start(struct amdgpu_device *adev)
 {
-	volatile struct amdgpu_vcn4_fw_shared *fw_shared;
+	volatile struct amdgpu_vcn5_fw_shared *fw_shared;
 	struct amdgpu_ring *ring;
 	uint32_t tmp;
 	int i, j, k, r;
@@ -893,7 +894,7 @@ static void vcn_v5_0_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
  */
 static int vcn_v5_0_0_stop(struct amdgpu_device *adev)
 {
-	volatile struct amdgpu_vcn4_fw_shared *fw_shared;
+	volatile struct amdgpu_vcn5_fw_shared *fw_shared;
 	uint32_t tmp;
 	int i, r = 0;
 
@@ -1328,6 +1329,8 @@ static const struct amd_ip_funcs vcn_v5_0_0_ip_funcs = {
 	.post_soft_reset = NULL,
 	.set_clockgating_state = vcn_v5_0_0_set_clockgating_state,
 	.set_powergating_state = vcn_v5_0_0_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version vcn_v5_0_0_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 1a98812981f4..d39c670f6220 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -897,7 +897,7 @@ static int vi_asic_pci_config_reset(struct amdgpu_device *adev)
 	return r;
 }
 
-static bool vi_asic_supports_baco(struct amdgpu_device *adev)
+static int vi_asic_supports_baco(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_FIJI:
@@ -908,14 +908,14 @@ static bool vi_asic_supports_baco(struct amdgpu_device *adev)
 	case CHIP_TOPAZ:
 		return amdgpu_dpm_is_baco_supported(adev);
 	default:
-		return false;
+		return 0;
 	}
 }
 
 static enum amd_reset_method
 vi_asic_reset_method(struct amdgpu_device *adev)
 {
-	bool baco_reset;
+	int baco_reset;
 
 	if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
 	    amdgpu_reset_method == AMD_RESET_METHOD_BACO)
@@ -935,7 +935,7 @@ vi_asic_reset_method(struct amdgpu_device *adev)
 		baco_reset = amdgpu_dpm_is_baco_supported(adev);
 		break;
 	default:
-		baco_reset = false;
+		baco_reset = 0;
 		break;
 	}
 
@@ -2058,6 +2058,8 @@ static const struct amd_ip_funcs vi_common_ip_funcs = {
 	.set_clockgating_state = vi_common_set_clockgating_state,
 	.set_powergating_state = vi_common_set_powergating_state,
 	.get_clockgating_state = vi_common_get_clockgating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 static const struct amdgpu_ip_block_version vi_common_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/vpe_v6_1.c b/drivers/gpu/drm/amd/amdgpu/vpe_v6_1.c
index 769eb8f7bb3c..09315dd5a1ec 100644
--- a/drivers/gpu/drm/amd/amdgpu/vpe_v6_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/vpe_v6_1.c
@@ -144,6 +144,12 @@ static int vpe_v6_1_load_microcode(struct amdgpu_vpe *vpe)
 			WREG32(vpe_get_reg_offset(vpe, j, regVPEC_CNTL), ret);
 	}
 
+	/* setup collaborate mode */
+	vpe_v6_1_set_collaborate_mode(vpe, true);
+	/* setup DPM */
+	if (amdgpu_vpe_configure_dpm(vpe))
+		dev_warn(adev->dev, "VPE failed to enable DPM\n");
+
 	/*
 	 * For VPE 6.1.1, still only need to add master's offset, and psp will apply it to slave as well.
 	 * Here use instance 0 as master.
@@ -159,11 +165,7 @@ static int vpe_v6_1_load_microcode(struct amdgpu_vpe *vpe)
 		adev->vpe.cmdbuf_cpu_addr[0] = f32_offset;
 		adev->vpe.cmdbuf_cpu_addr[1] = f32_cntl;
 
-		amdgpu_vpe_psp_update_sram(adev);
-		vpe_v6_1_set_collaborate_mode(vpe, true);
-		amdgpu_vpe_configure_dpm(vpe);
-
-		return 0;
+		return amdgpu_vpe_psp_update_sram(adev);
 	}
 
 	vpe_hdr = (const struct vpe_firmware_header_v1_0 *)adev->vpe.fw->data;
@@ -196,8 +198,6 @@ static int vpe_v6_1_load_microcode(struct amdgpu_vpe *vpe)
 	}
 
 	vpe_v6_1_halt(vpe, false);
-	vpe_v6_1_set_collaborate_mode(vpe, true);
-	amdgpu_vpe_configure_dpm(vpe);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 55aa74cbc532..fdf171ad4a3c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -371,6 +371,11 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 			err = -EINVAL;
 			goto err_wptr_map_gart;
 		}
+		if (dev->adev != amdgpu_ttm_adev(wptr_bo->tbo.bdev)) {
+			pr_err("Queue memory allocated to wrong device\n");
+			err = -EINVAL;
+			goto err_wptr_map_gart;
+		}
 
 		err = amdgpu_amdkfd_map_gtt_bo_to_gart(wptr_bo);
 		if (err) {
@@ -1139,7 +1144,7 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
 			goto err_unlock;
 		}
 		offset = dev->adev->rmmio_remap.bus_addr;
-		if (!offset) {
+		if (!offset || (PAGE_SIZE > 4096)) {
 			err = -ENOMEM;
 			goto err_unlock;
 		}
@@ -2307,7 +2312,7 @@ static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
 			return -EINVAL;
 		}
 		offset = pdd->dev->adev->rmmio_remap.bus_addr;
-		if (!offset) {
+		if (!offset || (PAGE_SIZE > 4096)) {
 			pr_err("amdgpu_amdkfd_get_mmio_remap_phys_addr failed\n");
 			return -ENOMEM;
 		}
@@ -3349,6 +3354,9 @@ static int kfd_mmio_mmap(struct kfd_node *dev, struct kfd_process *process,
 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
 		return -EINVAL;
 
+	if (PAGE_SIZE > 4096)
+		return -EINVAL;
+
 	address = dev->adev->rmmio_remap.bus_addr;
 
 	vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 719d6d365e15..afc57df421cd 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -408,15 +408,8 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
 			f2g = &gfx_v11_kfd2kgd;
 			break;
 		case IP_VERSION(11, 0, 3):
-			if ((adev->pdev->device == 0x7460 &&
-			     adev->pdev->revision == 0x00) ||
-			    (adev->pdev->device == 0x7461 &&
-			     adev->pdev->revision == 0x00))
-				/* Note: Compiler version is 11.0.5 while HW version is 11.0.3 */
-				gfx_target_version = 110005;
-			else
-				/* Note: Compiler version is 11.0.1 while HW version is 11.0.3 */
-				gfx_target_version = 110001;
+			/* Note: Compiler version is 11.0.1 while HW version is 11.0.3 */
+			gfx_target_version = 110001;
 			f2g = &gfx_v11_kfd2kgd;
 			break;
 		case IP_VERSION(11, 5, 0):
@@ -435,12 +428,12 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
 
 	if (!f2g) {
 		if (amdgpu_ip_version(adev, GC_HWIP, 0))
-			dev_err(kfd_device,
+			dev_info(kfd_device,
 				"GC IP %06x %s not supported in kfd\n",
 				amdgpu_ip_version(adev, GC_HWIP, 0),
 				vf ? "VF" : "");
 		else
-			dev_err(kfd_device, "%s %s not supported in kfd\n",
+			dev_info(kfd_device, "%s %s not supported in kfd\n",
 				amdgpu_asic_name[adev->asic_type], vf ? "VF" : "");
 		return NULL;
 	}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 0b655555e167..c08b6ee25289 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -1997,8 +1997,7 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm,
 	 * check those fields
 	 */
 	mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];
-	if (mqd_mgr->read_doorbell_id(dqm->packet_mgr.priv_queue->queue->mqd)) {
-		dev_err(dev, "HIQ MQD's queue_doorbell_id0 is not 0, Queue preemption time out\n");
+	if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd)) {
 		while (halt_if_hws_hang)
 			schedule();
 		kfd_hws_hang(dqm);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c
index 40a21be6c07c..8e0d0356e810 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c
@@ -134,6 +134,7 @@ static void event_interrupt_poison_consumption(struct kfd_node *dev,
 {
 	enum amdgpu_ras_block block = 0;
 	int old_poison, ret = -EINVAL;
+	uint32_t reset = 0;
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
 
 	if (!p)
@@ -153,6 +154,8 @@ static void event_interrupt_poison_consumption(struct kfd_node *dev,
 	case SOC15_IH_CLIENTID_UTCL2:
 		ret = kfd_dqm_evict_pasid(dev->dqm, pasid);
 		block = AMDGPU_RAS_BLOCK__GFX;
+		if (ret)
+			reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
 		break;
 	case SOC15_IH_CLIENTID_SDMA0:
 	case SOC15_IH_CLIENTID_SDMA1:
@@ -160,6 +163,7 @@ static void event_interrupt_poison_consumption(struct kfd_node *dev,
 	case SOC15_IH_CLIENTID_SDMA3:
 	case SOC15_IH_CLIENTID_SDMA4:
 		block = AMDGPU_RAS_BLOCK__SDMA;
+		reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
 		break;
 	default:
 		break;
@@ -170,17 +174,16 @@ static void event_interrupt_poison_consumption(struct kfd_node *dev,
 	/* resetting queue passes, do page retirement without gpu reset
 	 * resetting queue fails, fallback to gpu reset solution
 	 */
-	if (!ret) {
+	if (!ret)
 		dev_warn(dev->adev->dev,
 			"RAS poison consumption, unmap queue flow succeeded: client id %d\n",
 			client_id);
-		amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, block, false);
-	} else {
+	else
 		dev_warn(dev->adev->dev,
 			"RAS poison consumption, fall back to gpu reset flow: client id %d\n",
 			client_id);
-		amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, block, true);
-	}
+
+	amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, block, reset);
 }
 
 static bool event_interrupt_isr_v10(struct kfd_node *dev,
@@ -368,10 +371,25 @@ static void event_interrupt_wq_v10(struct kfd_node *dev,
 		   client_id == SOC15_IH_CLIENTID_UTCL2) {
 		struct kfd_vm_fault_info info = {0};
 		uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
+		uint32_t node_id = SOC15_NODEID_FROM_IH_ENTRY(ih_ring_entry);
+		uint32_t vmid_type = SOC15_VMID_TYPE_FROM_IH_ENTRY(ih_ring_entry);
+		int hub_inst = 0;
 		struct kfd_hsa_memory_exception_data exception_data;
 
-		if (client_id == SOC15_IH_CLIENTID_UTCL2 &&
-				amdgpu_amdkfd_ras_query_utcl2_poison_status(dev->adev)) {
+		/* gfxhub */
+		if (!vmid_type && dev->adev->gfx.funcs->ih_node_to_logical_xcc) {
+			hub_inst = dev->adev->gfx.funcs->ih_node_to_logical_xcc(dev->adev,
+				node_id);
+			if (hub_inst < 0)
+				hub_inst = 0;
+		}
+
+		/* mmhub */
+		if (vmid_type && client_id == SOC15_IH_CLIENTID_VMC)
+			hub_inst = node_id / 4;
+
+		if (amdgpu_amdkfd_ras_query_utcl2_poison_status(dev->adev,
+					hub_inst, vmid_type)) {
 			event_interrupt_poison_consumption(dev, pasid, client_id);
 			return;
 		}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
index fe2ad0c0de95..f524a55eee11 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
@@ -193,6 +193,7 @@ static void event_interrupt_poison_consumption_v11(struct kfd_node *dev,
 {
 	enum amdgpu_ras_block block = 0;
 	int ret = -EINVAL;
+	uint32_t reset = 0;
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
 
 	if (!p)
@@ -212,10 +213,13 @@ static void event_interrupt_poison_consumption_v11(struct kfd_node *dev,
 		if (dev->dqm->ops.reset_queues)
 			ret = dev->dqm->ops.reset_queues(dev->dqm, pasid);
 		block = AMDGPU_RAS_BLOCK__GFX;
+		if (ret)
+			reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
 		break;
 	case SOC21_INTSRC_SDMA_ECC:
 	default:
 		block = AMDGPU_RAS_BLOCK__GFX;
+		reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
 		break;
 	}
 
@@ -223,10 +227,7 @@ static void event_interrupt_poison_consumption_v11(struct kfd_node *dev,
 
 	/* resetting queue passes, do page retirement without gpu reset
 	   resetting queue fails, fallback to gpu reset solution */
-	if (!ret)
-		amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, block, false);
-	else
-		amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, block, true);
+	amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, block, reset);
 }
 
 static bool event_interrupt_isr_v11(struct kfd_node *dev,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
index c4c6a29052ac..e1c21d250611 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
@@ -144,7 +144,8 @@ static void event_interrupt_poison_consumption_v9(struct kfd_node *dev,
 				uint16_t pasid, uint16_t client_id)
 {
 	enum amdgpu_ras_block block = 0;
-	int old_poison, ret = -EINVAL;
+	int old_poison;
+	uint32_t reset = 0;
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
 
 	if (!p)
@@ -162,8 +163,13 @@ static void event_interrupt_poison_consumption_v9(struct kfd_node *dev,
 	case SOC15_IH_CLIENTID_SE2SH:
 	case SOC15_IH_CLIENTID_SE3SH:
 	case SOC15_IH_CLIENTID_UTCL2:
-		ret = kfd_dqm_evict_pasid(dev->dqm, pasid);
 		block = AMDGPU_RAS_BLOCK__GFX;
+		reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
+		break;
+	case SOC15_IH_CLIENTID_VMC:
+	case SOC15_IH_CLIENTID_VMC1:
+		block = AMDGPU_RAS_BLOCK__MMHUB;
+		reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET;
 		break;
 	case SOC15_IH_CLIENTID_SDMA0:
 	case SOC15_IH_CLIENTID_SDMA1:
@@ -171,27 +177,21 @@ static void event_interrupt_poison_consumption_v9(struct kfd_node *dev,
 	case SOC15_IH_CLIENTID_SDMA3:
 	case SOC15_IH_CLIENTID_SDMA4:
 		block = AMDGPU_RAS_BLOCK__SDMA;
+		reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
 		break;
 	default:
-		break;
+		dev_warn(dev->adev->dev,
+			 "client %d does not support poison consumption\n", client_id);
+		return;
 	}
 
 	kfd_signal_poison_consumed_event(dev, pasid);
 
-	/* resetting queue passes, do page retirement without gpu reset
-	 * resetting queue fails, fallback to gpu reset solution
-	 */
-	if (!ret) {
-		dev_warn(dev->adev->dev,
-			"RAS poison consumption, unmap queue flow succeeded: client id %d\n",
-			client_id);
-		amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, block, false);
-	} else {
-		dev_warn(dev->adev->dev,
-			"RAS poison consumption, fall back to gpu reset flow: client id %d\n",
-			client_id);
-		amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, block, true);
-	}
+	dev_warn(dev->adev->dev,
+		 "poison is consumed by client %d, kick off gpu reset flow\n", client_id);
+
+	amdgpu_amdkfd_ras_pasid_poison_consumption_handler(dev->adev,
+		block, pasid, NULL, NULL, reset);
 }
 
 static bool context_id_expected(struct kfd_dev *dev)
@@ -414,10 +414,25 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
 		   client_id == SOC15_IH_CLIENTID_UTCL2) {
 		struct kfd_vm_fault_info info = {0};
 		uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
+		uint32_t node_id = SOC15_NODEID_FROM_IH_ENTRY(ih_ring_entry);
+		uint32_t vmid_type = SOC15_VMID_TYPE_FROM_IH_ENTRY(ih_ring_entry);
+		int hub_inst = 0;
 		struct kfd_hsa_memory_exception_data exception_data;
 
-		if (client_id == SOC15_IH_CLIENTID_UTCL2 &&
-		    amdgpu_amdkfd_ras_query_utcl2_poison_status(dev->adev)) {
+		/* gfxhub */
+		if (!vmid_type && dev->adev->gfx.funcs->ih_node_to_logical_xcc) {
+			hub_inst = dev->adev->gfx.funcs->ih_node_to_logical_xcc(dev->adev,
+				node_id);
+			if (hub_inst < 0)
+				hub_inst = 0;
+		}
+
+		/* mmhub */
+		if (vmid_type && client_id == SOC15_IH_CLIENTID_VMC)
+			hub_inst = node_id / 4;
+
+		if (amdgpu_amdkfd_ras_query_utcl2_poison_status(dev->adev,
+					hub_inst, vmid_type)) {
 			event_interrupt_poison_consumption_v9(dev, pasid, client_id);
 			return;
 		}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
index dd3c43c1ad70..9b6b6e882593 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
@@ -104,6 +104,8 @@ void kfd_interrupt_exit(struct kfd_node *node)
 	 */
 	flush_workqueue(node->ih_wq);
 
+	destroy_workqueue(node->ih_wq);
+
 	kfifo_free(&node->ih_fifo);
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index bdc01ca9609a..8ee3d07ffbdf 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -77,7 +77,7 @@ svm_migrate_gart_map(struct amdgpu_ring *ring, uint64_t npages,
 
 	dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
 	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
-				dst_addr, num_bytes, false);
+				dst_addr, num_bytes, 0);
 
 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 	WARN_ON(job->ibs[0].length_dw > num_dw);
@@ -153,7 +153,7 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
 		}
 
 		r = amdgpu_copy_buffer(ring, gart_s, gart_d, size * PAGE_SIZE,
-				       NULL, &next, false, true, false);
+				       NULL, &next, false, true, 0);
 		if (r) {
 			dev_err(adev->dev, "fail %d to copy memory\n", r);
 			goto out_unlock;
@@ -509,10 +509,19 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
 	start = start_mgr << PAGE_SHIFT;
 	end = (last_mgr + 1) << PAGE_SHIFT;
 
+	r = amdgpu_amdkfd_reserve_mem_limit(node->adev,
+					prange->npages * PAGE_SIZE,
+					KFD_IOC_ALLOC_MEM_FLAGS_VRAM,
+					node->xcp ? node->xcp->id : 0);
+	if (r) {
+		dev_dbg(node->adev->dev, "failed to reserve VRAM, r: %ld\n", r);
+		return -ENOSPC;
+	}
+
 	r = svm_range_vram_node_new(node, prange, true);
 	if (r) {
 		dev_dbg(node->adev->dev, "fail %ld to alloc vram\n", r);
-		return r;
+		goto out;
 	}
 	ttm_res_offset = (start_mgr - prange->start + prange->offset) << PAGE_SHIFT;
 
@@ -545,6 +554,11 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
 		svm_range_vram_node_free(prange);
 	}
 
+out:
+	amdgpu_amdkfd_unreserve_mem_limit(node->adev,
+					prange->npages * PAGE_SIZE,
+					KFD_IOC_ALLOC_MEM_FLAGS_VRAM,
+					node->xcp ? node->xcp->id : 0);
 	return r < 0 ? r : 0;
 }
 
@@ -1009,7 +1023,7 @@ int kgd2kfd_init_zone_device(struct amdgpu_device *adev)
 	if (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(9, 0, 1))
 		return -EINVAL;
 
-	if (adev->gmc.is_app_apu)
+	if (adev->flags & AMD_IS_APU)
 		return 0;
 
 	pgmap = &kfddev->pgmap;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
index 050a6936ff84..8746a61a852d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
@@ -290,3 +290,21 @@ uint64_t kfd_mqd_stride(struct mqd_manager *mm,
 {
 	return mm->mqd_size;
 }
+
+bool kfd_check_hiq_mqd_doorbell_id(struct kfd_node *node, uint32_t doorbell_id,
+				   uint32_t inst)
+{
+	if (doorbell_id) {
+		struct device *dev = node->adev->dev;
+
+		if (node->adev->xcp_mgr && node->adev->xcp_mgr->num_xcps > 0)
+			dev_err(dev, "XCC %d: Queue preemption failed for queue with doorbell_id: %x\n",
+							inst, doorbell_id);
+		else
+			dev_err(dev, "Queue preemption failed for queue with doorbell_id: %x\n",
+							doorbell_id);
+		return true;
+	}
+
+	return false;
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index e5cc697a3ca8..17cc1f25c8d0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -119,7 +119,7 @@ struct mqd_manager {
 #if defined(CONFIG_DEBUG_FS)
 	int	(*debugfs_show_mqd)(struct seq_file *m, void *data);
 #endif
-	uint32_t (*read_doorbell_id)(void *mqd);
+	bool (*check_preemption_failed)(struct mqd_manager *mm, void *mqd);
 	uint64_t (*mqd_stride)(struct mqd_manager *mm,
 				struct queue_properties *p);
 
@@ -198,4 +198,6 @@ void kfd_get_hiq_xcc_mqd(struct kfd_node *dev,
 uint64_t kfd_hiq_mqd_stride(struct kfd_node *dev);
 uint64_t kfd_mqd_stride(struct mqd_manager *mm,
 			struct queue_properties *q);
+bool kfd_check_hiq_mqd_doorbell_id(struct kfd_node *node, uint32_t doorbell_id,
+				   uint32_t inst);
 #endif /* KFD_MQD_MANAGER_H_ */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index 1a4a69943c71..05f3ac2eaef9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -206,11 +206,11 @@ static void __update_mqd(struct mqd_manager *mm, void *mqd,
 	q->is_active = QUEUE_IS_ACTIVE(*q);
 }
 
-static uint32_t read_doorbell_id(void *mqd)
+static bool check_preemption_failed(struct mqd_manager *mm, void *mqd)
 {
 	struct cik_mqd *m = (struct cik_mqd *)mqd;
 
-	return m->queue_doorbell_id0;
+	return kfd_check_hiq_mqd_doorbell_id(mm->dev, m->queue_doorbell_id0, 0);
 }
 
 static void update_mqd(struct mqd_manager *mm, void *mqd,
@@ -423,7 +423,7 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
 #if defined(CONFIG_DEBUG_FS)
 		mqd->debugfs_show_mqd = debugfs_show_mqd;
 #endif
-		mqd->read_doorbell_id = read_doorbell_id;
+		mqd->check_preemption_failed = check_preemption_failed;
 		break;
 	case KFD_MQD_TYPE_DIQ:
 		mqd->allocate_mqd = allocate_mqd;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
index 22cbfa1bdadd..2eff37aaf827 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
@@ -224,11 +224,11 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
 	q->is_active = QUEUE_IS_ACTIVE(*q);
 }
 
-static uint32_t read_doorbell_id(void *mqd)
+static bool check_preemption_failed(struct mqd_manager *mm, void *mqd)
 {
 	struct v10_compute_mqd *m = (struct v10_compute_mqd *)mqd;
 
-	return m->queue_doorbell_id0;
+	return kfd_check_hiq_mqd_doorbell_id(mm->dev, m->queue_doorbell_id0, 0);
 }
 
 static int get_wave_state(struct mqd_manager *mm, void *mqd,
@@ -488,7 +488,7 @@ struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
 #if defined(CONFIG_DEBUG_FS)
 		mqd->debugfs_show_mqd = debugfs_show_mqd;
 #endif
-		mqd->read_doorbell_id = read_doorbell_id;
+		mqd->check_preemption_failed = check_preemption_failed;
 		pr_debug("%s@%i\n", __func__, __LINE__);
 		break;
 	case KFD_MQD_TYPE_DIQ:
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
index 826bc4f6c8a7..68dbc0399c87 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
@@ -278,11 +278,11 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
 	q->is_active = QUEUE_IS_ACTIVE(*q);
 }
 
-static uint32_t read_doorbell_id(void *mqd)
+static bool check_preemption_failed(struct mqd_manager *mm, void *mqd)
 {
 	struct v11_compute_mqd *m = (struct v11_compute_mqd *)mqd;
 
-	return m->queue_doorbell_id0;
+	return kfd_check_hiq_mqd_doorbell_id(mm->dev, m->queue_doorbell_id0, 0);
 }
 
 static int get_wave_state(struct mqd_manager *mm, void *mqd,
@@ -517,7 +517,7 @@ struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,
 #if defined(CONFIG_DEBUG_FS)
 		mqd->debugfs_show_mqd = debugfs_show_mqd;
 #endif
-		mqd->read_doorbell_id = read_doorbell_id;
+		mqd->check_preemption_failed = check_preemption_failed;
 		pr_debug("%s@%i\n", __func__, __LINE__);
 		break;
 	case KFD_MQD_TYPE_DIQ:
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index 697b6d530d12..6bddc16808d7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -316,11 +316,11 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
 }
 
 
-static uint32_t read_doorbell_id(void *mqd)
+static bool check_preemption_failed(struct mqd_manager *mm, void *mqd)
 {
 	struct v9_mqd *m = (struct v9_mqd *)mqd;
 
-	return m->queue_doorbell_id0;
+	return kfd_check_hiq_mqd_doorbell_id(mm->dev, m->queue_doorbell_id0, 0);
 }
 
 static int get_wave_state(struct mqd_manager *mm, void *mqd,
@@ -607,6 +607,24 @@ static int destroy_hiq_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
 	return err;
 }
 
+static bool check_preemption_failed_v9_4_3(struct mqd_manager *mm, void *mqd)
+{
+	uint64_t hiq_mqd_size = kfd_hiq_mqd_stride(mm->dev);
+	uint32_t xcc_mask = mm->dev->xcc_mask;
+	int inst = 0, xcc_id;
+	struct v9_mqd *m;
+	bool ret = false;
+
+	for_each_inst(xcc_id, xcc_mask) {
+		m = get_mqd(mqd + hiq_mqd_size * inst);
+		ret |= kfd_check_hiq_mqd_doorbell_id(mm->dev,
+					m->queue_doorbell_id0, inst);
+		++inst;
+	}
+
+	return ret;
+}
+
 static void get_xcc_mqd(struct kfd_mem_obj *mqd_mem_obj,
 			       struct kfd_mem_obj *xcc_mqd_mem_obj,
 			       uint64_t offset)
@@ -881,15 +899,16 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
 #if defined(CONFIG_DEBUG_FS)
 		mqd->debugfs_show_mqd = debugfs_show_mqd;
 #endif
-		mqd->read_doorbell_id = read_doorbell_id;
 		if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3)) {
 			mqd->init_mqd = init_mqd_hiq_v9_4_3;
 			mqd->load_mqd = hiq_load_mqd_kiq_v9_4_3;
 			mqd->destroy_mqd = destroy_hiq_mqd_v9_4_3;
+			mqd->check_preemption_failed = check_preemption_failed_v9_4_3;
 		} else {
 			mqd->init_mqd = init_mqd_hiq;
 			mqd->load_mqd = kfd_hiq_load_mqd_kiq;
 			mqd->destroy_mqd = destroy_hiq_mqd;
+			mqd->check_preemption_failed = check_preemption_failed;
 		}
 		break;
 	case KFD_MQD_TYPE_DIQ:
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 3e1a574d4ea6..c1fafc502515 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -237,11 +237,11 @@ static void __update_mqd(struct mqd_manager *mm, void *mqd,
 	q->is_active = QUEUE_IS_ACTIVE(*q);
 }
 
-static uint32_t read_doorbell_id(void *mqd)
+static bool check_preemption_failed(struct mqd_manager *mm, void *mqd)
 {
 	struct vi_mqd *m = (struct vi_mqd *)mqd;
 
-	return m->queue_doorbell_id0;
+	return kfd_check_hiq_mqd_doorbell_id(mm->dev, m->queue_doorbell_id0, 0);
 }
 
 static void update_mqd(struct mqd_manager *mm, void *mqd,
@@ -482,7 +482,7 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 #if defined(CONFIG_DEBUG_FS)
 		mqd->debugfs_show_mqd = debugfs_show_mqd;
 #endif
-		mqd->read_doorbell_id = read_doorbell_id;
+		mqd->check_preemption_failed = check_preemption_failed;
 		break;
 	case KFD_MQD_TYPE_DIQ:
 		mqd->allocate_mqd = allocate_mqd;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 717a60d7a4ea..451bb058cc62 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -819,9 +819,9 @@ struct kfd_process *kfd_create_process(struct task_struct *thread)
 	mutex_lock(&kfd_processes_mutex);
 
 	if (kfd_is_locked()) {
-		mutex_unlock(&kfd_processes_mutex);
 		pr_debug("KFD is locked! Cannot create process");
-		return ERR_PTR(-EINVAL);
+		process = ERR_PTR(-EINVAL);
+		goto out;
 	}
 
 	/* A prior open of /dev/kfd could have already created the process. */
@@ -829,6 +829,14 @@ struct kfd_process *kfd_create_process(struct task_struct *thread)
 	if (process) {
 		pr_debug("Process already found\n");
 	} else {
+		/* If the process just called exec(3), it is possible that the
+		 * cleanup of the kfd_process (following the release of the mm
+		 * of the old process image) is still in the cleanup work queue.
+		 * Make sure to drain any job before trying to recreate any
+		 * resource for this process.
+		 */
+		flush_workqueue(kfd_process_wq);
+
 		process = create_process(thread);
 		if (IS_ERR(process))
 			goto out;
@@ -1922,6 +1930,8 @@ static int signal_eviction_fence(struct kfd_process *p)
 	rcu_read_lock();
 	ef = dma_fence_get_rcu_safe(&p->ef);
 	rcu_read_unlock();
+	if (!ef)
+		return -EINVAL;
 
 	ret = dma_fence_signal(ef);
 	dma_fence_put(ef);
@@ -1949,10 +1959,9 @@ static void evict_process_worker(struct work_struct *work)
 		 * they are responsible stopping the queues and scheduling
 		 * the restore work.
 		 */
-		if (!signal_eviction_fence(p))
-			queue_delayed_work(kfd_restore_wq, &p->restore_work,
-				msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
-		else
+		if (signal_eviction_fence(p) ||
+		    mod_delayed_work(kfd_restore_wq, &p->restore_work,
+				     msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
 			kfd_process_restore_queues(p);
 
 		pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
@@ -2011,9 +2020,9 @@ static void restore_process_worker(struct work_struct *work)
 	if (ret) {
 		pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
 			 p->pasid, PROCESS_BACK_OFF_TIME_MS);
-		ret = queue_delayed_work(kfd_restore_wq, &p->restore_work,
-				msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
-		WARN(!ret, "reschedule restore work failed\n");
+		if (mod_delayed_work(kfd_restore_wq, &p->restore_work,
+				     msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
+			kfd_process_restore_queues(p);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index f0f7f48af413..31e500859ab0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -2619,7 +2619,7 @@ svm_range_best_restore_location(struct svm_range *prange,
 		return -1;
 	}
 
-	if (node->adev->gmc.is_app_apu)
+	if (node->adev->flags & AMD_IS_APU)
 		return 0;
 
 	if (prange->preferred_loc == gpuid ||
@@ -3337,7 +3337,7 @@ svm_range_best_prefetch_location(struct svm_range *prange)
 		goto out;
 	}
 
-	if (bo_node->adev->gmc.is_app_apu) {
+	if (bo_node->adev->flags & AMD_IS_APU) {
 		best_loc = 0;
 		goto out;
 	}
@@ -3426,7 +3426,7 @@ svm_range_trigger_migration(struct mm_struct *mm, struct svm_range *prange,
 				mm, KFD_MIGRATE_TRIGGER_PREFETCH);
 	*migrated = !r;
 
-	return r;
+	return 0;
 }
 
 int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index 026863a0abcd..70c1776611c4 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -201,7 +201,7 @@ void svm_range_list_lock_and_flush_work(struct svm_range_list *svms, struct mm_s
  * is initialized to not 0 when page migration register device memory.
  */
 #define KFD_IS_SVM_API_SUPPORTED(adev) ((adev)->kfd.pgmap.type != 0 ||\
-					(adev)->gmc.is_app_apu)
+					((adev)->flags & AMD_IS_APU))
 
 void svm_range_bo_unref_async(struct svm_range_bo *svm_bo);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index c51f131eaa2f..bc9eb847ecfe 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1997,9 +1997,8 @@ int kfd_topology_add_device(struct kfd_node *gpu)
 			HSA_CAP_ASIC_REVISION_MASK);
 
 	dev->node_props.location_id = pci_dev_id(gpu->adev->pdev);
-	/* On multi-partition nodes, node id = location_id[31:28] */
-	if (gpu->kfd->num_nodes > 1)
-		dev->node_props.location_id |= (dev->gpu->node_id << 28);
+	if (KFD_GC_VERSION(dev->gpu->kfd) == IP_VERSION(9, 4, 3))
+		dev->node_props.location_id |= dev->gpu->node_id;
 
 	dev->node_props.domain = pci_domain_nr(gpu->adev->pdev->bus);
 	dev->node_props.max_engine_clk_fcompute =
diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig
index 901d1961b739..5fcd4f778dc3 100644
--- a/drivers/gpu/drm/amd/display/Kconfig
+++ b/drivers/gpu/drm/amd/display/Kconfig
@@ -8,7 +8,7 @@ config DRM_AMD_DC
 	depends on BROKEN || !CC_IS_CLANG || ARM64 || RISCV || SPARC64 || X86_64
 	select SND_HDA_COMPONENT if SND_HDA_CORE
 	# !CC_IS_CLANG: https://github.com/ClangBuiltLinux/linux/issues/1752
-	select DRM_AMD_DC_FP if (X86 || LOONGARCH || (PPC64 && ALTIVEC) || (ARM64 && KERNEL_MODE_NEON && !CC_IS_CLANG))
+	select DRM_AMD_DC_FP if ARCH_HAS_KERNEL_FPU_SUPPORT && (!ARM64 || !CC_IS_CLANG)
 	help
 	  Choose this option if you want to use the new display engine
 	  support for AMDGPU. This adds required support for Vega and
diff --git a/drivers/gpu/drm/amd/display/Makefile b/drivers/gpu/drm/amd/display/Makefile
index 92a5c5efcf92..9a5bcafbf730 100644
--- a/drivers/gpu/drm/amd/display/Makefile
+++ b/drivers/gpu/drm/amd/display/Makefile
@@ -33,6 +33,7 @@ subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/hwss
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/resource
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/dsc
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/optc
+subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/dpp
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/modules/inc
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/modules/freesync
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/modules/color
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index 6d2f60c61dec..f1d67c6f4b98 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -274,7 +274,7 @@ static u32 dm_vblank_get_counter(struct amdgpu_device *adev, int crtc)
 static int dm_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc,
 				  u32 *vbl, u32 *position)
 {
-	u32 v_blank_start, v_blank_end, h_position, v_position;
+	u32 v_blank_start = 0, v_blank_end = 0, h_position = 0, v_position = 0;
 	struct amdgpu_crtc *acrtc = NULL;
 	struct dc *dc = adev->dm.dc;
 
@@ -848,7 +848,7 @@ static void dm_handle_hpd_work(struct work_struct *work)
  */
 static void dm_dmub_outbox1_low_irq(void *interrupt_params)
 {
-	struct dmub_notification notify;
+	struct dmub_notification notify = {0};
 	struct common_irq_params *irq_params = interrupt_params;
 	struct amdgpu_device *adev = irq_params->adev;
 	struct amdgpu_display_manager *dm = &adev->dm;
@@ -1230,6 +1230,15 @@ static int dm_dmub_hw_init(struct amdgpu_device *adev)
 		break;
 	}
 
+	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+	case IP_VERSION(3, 5, 0):
+	case IP_VERSION(3, 5, 1):
+		hw_params.ips_sequential_ono = adev->external_rev_id > 0x10;
+		break;
+	default:
+		break;
+	}
+
 	status = dmub_srv_hw_init(dmub_srv, &hw_params);
 	if (status != DMUB_STATUS_OK) {
 		DRM_ERROR("Error initializing DMUB HW: %d\n", status);
@@ -1726,8 +1735,10 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 
 	if (amdgpu_dc_debug_mask & DC_DISABLE_IPS)
 		init_data.flags.disable_ips = DMUB_IPS_DISABLE_ALL;
+	else
+		init_data.flags.disable_ips = DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF;
 
-	init_data.flags.disable_ips_in_vpb = 1;
+	init_data.flags.disable_ips_in_vpb = 0;
 
 	/* Enable DWB for tested platforms only */
 	if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0))
@@ -2629,6 +2640,7 @@ static enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc)
 	int i;
 	struct dc_stream_state *del_streams[MAX_PIPES];
 	int del_streams_count = 0;
+	struct dc_commit_streams_params params = {};
 
 	memset(del_streams, 0, sizeof(del_streams));
 
@@ -2655,7 +2667,9 @@ static enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc)
 			goto fail;
 	}
 
-	res = dc_commit_streams(dc, context->streams, context->stream_count);
+	params.streams = context->streams;
+	params.stream_count = context->stream_count;
+	res = dc_commit_streams(dc, &params);
 
 fail:
 	dc_state_release(context);
@@ -2877,6 +2891,7 @@ static int dm_resume(void *handle)
 	struct dc_state *dc_state;
 	int i, r, j, ret;
 	bool need_hotplug = false;
+	struct dc_commit_streams_params commit_params = {};
 
 	if (dm->dc->caps.ips_support) {
 		dc_dmub_srv_apply_idle_power_optimizations(dm->dc, false);
@@ -2926,7 +2941,9 @@ static int dm_resume(void *handle)
 			dc_enable_dmub_outbox(adev->dm.dc);
 		}
 
-		WARN_ON(!dc_commit_streams(dm->dc, dc_state->streams, dc_state->stream_count));
+		commit_params.streams = dc_state->streams;
+		commit_params.stream_count = dc_state->stream_count;
+		WARN_ON(!dc_commit_streams(dm->dc, &commit_params));
 
 		dm_gpureset_commit_state(dm->cached_dc_state, dm);
 
@@ -2943,7 +2960,7 @@ static int dm_resume(void *handle)
 	}
 	/* Recreate dc_state - DC invalidates it when setting power state to S3. */
 	dc_state_release(dm_state->context);
-	dm_state->context = dc_state_create(dm->dc);
+	dm_state->context = dc_state_create(dm->dc, NULL);
 	/* TODO: Remove dc_state->dccg, use dc->dccg directly. */
 
 	/* Before powering on DC we need to re-initialize DMUB. */
@@ -3029,6 +3046,7 @@ static int dm_resume(void *handle)
 			dc_stream_release(dm_new_crtc_state->stream);
 			dm_new_crtc_state->stream = NULL;
 		}
+		dm_new_crtc_state->base.color_mgmt_changed = true;
 	}
 
 	for_each_new_plane_in_state(dm->cached_state, plane, new_plane_state, i) {
@@ -3103,6 +3121,8 @@ static const struct amd_ip_funcs amdgpu_dm_funcs = {
 	.soft_reset = dm_soft_reset,
 	.set_clockgating_state = dm_set_clockgating_state,
 	.set_powergating_state = dm_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version dm_ip_block = {
@@ -4536,15 +4556,18 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 	/* Determine whether to enable Replay support by default. */
 	if (!(amdgpu_dc_debug_mask & DC_DISABLE_REPLAY)) {
 		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-		case IP_VERSION(3, 1, 4):
-		case IP_VERSION(3, 1, 5):
-		case IP_VERSION(3, 1, 6):
-		case IP_VERSION(3, 2, 0):
-		case IP_VERSION(3, 2, 1):
-		case IP_VERSION(3, 5, 0):
-		case IP_VERSION(3, 5, 1):
-			replay_feature_enabled = true;
-			break;
+/*
+ * Disabled by default due to https://gitlab.freedesktop.org/drm/amd/-/issues/3344
+ *		case IP_VERSION(3, 1, 4):
+ *		case IP_VERSION(3, 1, 5):
+ *		case IP_VERSION(3, 1, 6):
+ *		case IP_VERSION(3, 2, 0):
+ *		case IP_VERSION(3, 2, 1):
+ *		case IP_VERSION(3, 5, 0):
+ *		case IP_VERSION(3, 5, 1):
+ *			replay_feature_enabled = true;
+ *			break;
+ */
 		default:
 			replay_feature_enabled = amdgpu_dc_feature_mask & DC_REPLAY_MASK;
 			break;
@@ -5709,8 +5732,8 @@ static void fill_stream_properties_from_drm_display_mode(
 
 	timing_out->aspect_ratio = get_aspect_ratio(mode_in);
 
-	stream->out_transfer_func->type = TF_TYPE_PREDEFINED;
-	stream->out_transfer_func->tf = TRANSFER_FUNCTION_SRGB;
+	stream->out_transfer_func.type = TF_TYPE_PREDEFINED;
+	stream->out_transfer_func.tf = TRANSFER_FUNCTION_SRGB;
 	if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) {
 		if (!adjust_colour_depth_from_display_info(timing_out, info) &&
 		    drm_mode_is_420_also(info, mode_in) &&
@@ -6328,7 +6351,7 @@ create_stream_for_sink(struct drm_connector *connector,
 		stream->use_vsc_sdp_for_colorimetry = stream->link->dpcd_caps.dpcd_rev.raw >= 0x14 &&
 						      stream->link->dpcd_caps.dprx_feature.bits.VSC_SDP_COLORIMETRY_SUPPORTED;
 
-		if (stream->out_transfer_func->tf == TRANSFER_FUNCTION_GAMMA22)
+		if (stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22)
 			tf = TRANSFER_FUNC_GAMMA_22;
 		mod_build_vsc_infopacket(stream, &stream->vsc_infopacket, stream->output_color_space, tf);
 		aconnector->psr_skip_count = AMDGPU_DM_PSR_ENTRY_DELAY;
@@ -6799,7 +6822,7 @@ static enum dc_status dm_validate_stream_and_context(struct dc *dc,
 	if (!dc_plane_state)
 		goto cleanup;
 
-	dc_state = dc_state_create(dc);
+	dc_state = dc_state_create(dc, NULL);
 	if (!dc_state)
 		goto cleanup;
 
@@ -7188,7 +7211,7 @@ static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_state *state,
 	struct amdgpu_dm_connector *aconnector;
 	struct dm_connector_state *dm_conn_state;
 	int i, j, ret;
-	int vcpi, pbn_div, pbn, slot_num = 0;
+	int vcpi, pbn_div, pbn = 0, slot_num = 0;
 
 	for_each_new_connector_in_state(state, connector, new_con_state, i) {
 
@@ -8401,13 +8424,13 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 
 		bundle->surface_updates[planes_count].surface = dc_plane;
 		if (new_pcrtc_state->color_mgmt_changed) {
-			bundle->surface_updates[planes_count].gamma = dc_plane->gamma_correction;
-			bundle->surface_updates[planes_count].in_transfer_func = dc_plane->in_transfer_func;
+			bundle->surface_updates[planes_count].gamma = &dc_plane->gamma_correction;
+			bundle->surface_updates[planes_count].in_transfer_func = &dc_plane->in_transfer_func;
 			bundle->surface_updates[planes_count].gamut_remap_matrix = &dc_plane->gamut_remap_matrix;
 			bundle->surface_updates[planes_count].hdr_mult = dc_plane->hdr_mult;
-			bundle->surface_updates[planes_count].func_shaper = dc_plane->in_shaper_func;
-			bundle->surface_updates[planes_count].lut3d_func = dc_plane->lut3d_func;
-			bundle->surface_updates[planes_count].blend_tf = dc_plane->blend_tf;
+			bundle->surface_updates[planes_count].func_shaper = &dc_plane->in_shaper_func;
+			bundle->surface_updates[planes_count].lut3d_func = &dc_plane->lut3d_func;
+			bundle->surface_updates[planes_count].blend_tf = &dc_plane->blend_tf;
 		}
 
 		amdgpu_dm_plane_fill_dc_scaling_info(dm->adev, new_plane_state,
@@ -8620,7 +8643,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 			bundle->stream_update.output_csc_transform =
 				&acrtc_state->stream->csc_color_matrix;
 			bundle->stream_update.out_transfer_func =
-				acrtc_state->stream->out_transfer_func;
+				&acrtc_state->stream->out_transfer_func;
 			bundle->stream_update.lut3d_func =
 				(struct dc_3dlut *) acrtc_state->stream->lut3d_func;
 			bundle->stream_update.func_shaper =
@@ -8854,6 +8877,7 @@ static void amdgpu_dm_commit_streams(struct drm_atomic_state *state,
 	struct drm_connector *connector;
 	bool mode_set_reset_required = false;
 	u32 i;
+	struct dc_commit_streams_params params = {dc_state->streams, dc_state->stream_count};
 
 	/* Disable writeback */
 	for_each_old_connector_in_state(state, connector, old_con_state, i) {
@@ -8990,7 +9014,7 @@ static void amdgpu_dm_commit_streams(struct drm_atomic_state *state,
 
 	dm_enable_per_frame_crtc_master_sync(dc_state);
 	mutex_lock(&dm->dc_lock);
-	WARN_ON(!dc_commit_streams(dm->dc, dc_state->streams, dc_state->stream_count));
+	WARN_ON(!dc_commit_streams(dm->dc, &params));
 
 	/* Allow idle optimization when vblank count is 0 for display off */
 	if (dm->active_vblank_irq_count == 0)
@@ -10594,7 +10618,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 	struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state;
 	struct drm_dp_mst_topology_mgr *mgr;
 	struct drm_dp_mst_topology_state *mst_state;
-	struct dsc_mst_fairness_vars vars[MAX_PIPES];
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {0};
 
 	trace_amdgpu_dm_atomic_check_begin(state);
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
index c87b64e464ed..ebabfe3a512f 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
@@ -571,7 +571,7 @@ static int amdgpu_dm_set_atomic_regamma(struct dc_stream_state *stream,
 					uint32_t regamma_size, bool has_rom,
 					enum dc_transfer_func_predefined tf)
 {
-	struct dc_transfer_func *out_tf = stream->out_transfer_func;
+	struct dc_transfer_func *out_tf = &stream->out_transfer_func;
 	int ret = 0;
 
 	if (regamma_size || tf != TRANSFER_FUNCTION_LINEAR) {
@@ -954,8 +954,8 @@ int amdgpu_dm_update_crtc_color_mgmt(struct dm_crtc_state *crtc)
 		 * inverse color ramp in legacy userspace.
 		 */
 		crtc->cm_is_degamma_srgb = true;
-		stream->out_transfer_func->type = TF_TYPE_DISTRIBUTED_POINTS;
-		stream->out_transfer_func->tf = TRANSFER_FUNCTION_SRGB;
+		stream->out_transfer_func.type = TF_TYPE_DISTRIBUTED_POINTS;
+		stream->out_transfer_func.tf = TRANSFER_FUNCTION_SRGB;
 		/*
 		 * Note: although we pass has_rom as parameter here, we never
 		 * actually use ROM because the color module only takes the ROM
@@ -963,7 +963,7 @@ int amdgpu_dm_update_crtc_color_mgmt(struct dm_crtc_state *crtc)
 		 *
 		 * See more in mod_color_calculate_regamma_params()
 		 */
-		r = __set_legacy_tf(stream->out_transfer_func, regamma_lut,
+		r = __set_legacy_tf(&stream->out_transfer_func, regamma_lut,
 				    regamma_size, has_rom);
 		if (r)
 			return r;
@@ -1034,7 +1034,7 @@ map_crtc_degamma_to_dc_plane(struct dm_crtc_state *crtc,
 						 &degamma_size);
 		ASSERT(degamma_size == MAX_COLOR_LUT_ENTRIES);
 
-		dc_plane_state->in_transfer_func->type = TF_TYPE_DISTRIBUTED_POINTS;
+		dc_plane_state->in_transfer_func.type = TF_TYPE_DISTRIBUTED_POINTS;
 
 		/*
 		 * This case isn't fully correct, but also fairly
@@ -1061,12 +1061,12 @@ map_crtc_degamma_to_dc_plane(struct dm_crtc_state *crtc,
 		 * map these to the atomic one instead.
 		 */
 		if (crtc->cm_is_degamma_srgb)
-			dc_plane_state->in_transfer_func->tf = tf;
+			dc_plane_state->in_transfer_func.tf = tf;
 		else
-			dc_plane_state->in_transfer_func->tf =
+			dc_plane_state->in_transfer_func.tf =
 				TRANSFER_FUNCTION_LINEAR;
 
-		r = __set_input_tf(caps, dc_plane_state->in_transfer_func,
+		r = __set_input_tf(caps, &dc_plane_state->in_transfer_func,
 				   degamma_lut, degamma_size);
 		if (r)
 			return r;
@@ -1075,12 +1075,12 @@ map_crtc_degamma_to_dc_plane(struct dm_crtc_state *crtc,
 		 * For legacy gamma support we need the regamma input
 		 * in linear space. Assume that the input is sRGB.
 		 */
-		dc_plane_state->in_transfer_func->type = TF_TYPE_PREDEFINED;
-		dc_plane_state->in_transfer_func->tf = tf;
+		dc_plane_state->in_transfer_func.type = TF_TYPE_PREDEFINED;
+		dc_plane_state->in_transfer_func.tf = tf;
 
 		if (tf != TRANSFER_FUNCTION_SRGB &&
 		    !mod_color_calculate_degamma_params(caps,
-							dc_plane_state->in_transfer_func,
+							&dc_plane_state->in_transfer_func,
 							NULL, false))
 			return -ENOMEM;
 	}
@@ -1114,24 +1114,24 @@ __set_dm_plane_degamma(struct drm_plane_state *plane_state,
 	if (!has_degamma_lut && tf == AMDGPU_TRANSFER_FUNCTION_DEFAULT)
 		return -EINVAL;
 
-	dc_plane_state->in_transfer_func->tf = amdgpu_tf_to_dc_tf(tf);
+	dc_plane_state->in_transfer_func.tf = amdgpu_tf_to_dc_tf(tf);
 
 	if (has_degamma_lut) {
 		ASSERT(degamma_size == MAX_COLOR_LUT_ENTRIES);
 
-		dc_plane_state->in_transfer_func->type =
+		dc_plane_state->in_transfer_func.type =
 			TF_TYPE_DISTRIBUTED_POINTS;
 
-		ret = __set_input_tf(color_caps, dc_plane_state->in_transfer_func,
+		ret = __set_input_tf(color_caps, &dc_plane_state->in_transfer_func,
 				     degamma_lut, degamma_size);
 		if (ret)
 			return ret;
        } else {
-		dc_plane_state->in_transfer_func->type =
+		dc_plane_state->in_transfer_func.type =
 			TF_TYPE_PREDEFINED;
 
 		if (!mod_color_calculate_degamma_params(color_caps,
-		    dc_plane_state->in_transfer_func, NULL, false))
+		    &dc_plane_state->in_transfer_func, NULL, false))
 			return -ENOMEM;
 	}
 	return 0;
@@ -1156,11 +1156,11 @@ amdgpu_dm_plane_set_color_properties(struct drm_plane_state *plane_state,
 	lut3d = __extract_blob_lut(dm_plane_state->lut3d, &lut3d_size);
 	lut3d_size = lut3d != NULL ? lut3d_size : 0;
 
-	amdgpu_dm_atomic_lut3d(lut3d, lut3d_size, dc_plane_state->lut3d_func);
+	amdgpu_dm_atomic_lut3d(lut3d, lut3d_size, &dc_plane_state->lut3d_func);
 	ret = amdgpu_dm_atomic_shaper_lut(shaper_lut, false,
 					  amdgpu_tf_to_dc_tf(shaper_tf),
 					  shaper_size,
-					  dc_plane_state->in_shaper_func);
+					  &dc_plane_state->in_shaper_func);
 	if (ret) {
 		drm_dbg_kms(plane_state->plane->dev,
 			    "setting plane %d shaper LUT failed.\n",
@@ -1175,7 +1175,7 @@ amdgpu_dm_plane_set_color_properties(struct drm_plane_state *plane_state,
 
 	ret = amdgpu_dm_atomic_blend_lut(blend_lut, false,
 					 amdgpu_tf_to_dc_tf(blend_tf),
-					 blend_size, dc_plane_state->blend_tf);
+					 blend_size, &dc_plane_state->blend_tf);
 	if (ret) {
 		drm_dbg_kms(plane_state->plane->dev,
 			    "setting plane %d gamma lut failed.\n",
@@ -1221,8 +1221,8 @@ int amdgpu_dm_update_plane_color_mgmt(struct dm_crtc_state *crtc,
 		color_caps = &dc_plane_state->ctx->dc->caps.color;
 
 	/* Initially, we can just bypass the DGM block. */
-	dc_plane_state->in_transfer_func->type = TF_TYPE_BYPASS;
-	dc_plane_state->in_transfer_func->tf = TRANSFER_FUNCTION_LINEAR;
+	dc_plane_state->in_transfer_func.type = TF_TYPE_BYPASS;
+	dc_plane_state->in_transfer_func.tf = TRANSFER_FUNCTION_LINEAR;
 
 	/* After, we start to update values according to color props */
 	has_crtc_cm_degamma = (crtc->cm_has_degamma || crtc->cm_is_degamma_srgb);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
index eee4945653e2..4d7a5d470b1e 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
@@ -1249,7 +1249,7 @@ static ssize_t dp_sdp_message_debugfs_write(struct file *f, const char __user *b
 				 size_t size, loff_t *pos)
 {
 	int r;
-	uint8_t data[36];
+	uint8_t data[36] = {0};
 	struct amdgpu_dm_connector *connector = file_inode(f)->i_private;
 	struct dm_crtc_state *acrtc_state;
 	uint32_t write_size = 36;
@@ -1495,7 +1495,9 @@ static ssize_t dp_dsc_clock_en_read(struct file *f, char __user *buf,
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx->stream &&
-		    pipe_ctx->stream->link == aconnector->dc_link)
+		    pipe_ctx->stream->link == aconnector->dc_link &&
+		    pipe_ctx->stream->sink &&
+		    pipe_ctx->stream->sink == aconnector->dc_sink)
 			break;
 	}
 
@@ -1596,7 +1598,9 @@ static ssize_t dp_dsc_clock_en_write(struct file *f, const char __user *buf,
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx->stream &&
-		    pipe_ctx->stream->link == aconnector->dc_link)
+		    pipe_ctx->stream->link == aconnector->dc_link &&
+		    pipe_ctx->stream->sink &&
+		    pipe_ctx->stream->sink == aconnector->dc_sink)
 			break;
 	}
 
@@ -1681,7 +1685,9 @@ static ssize_t dp_dsc_slice_width_read(struct file *f, char __user *buf,
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx->stream &&
-		    pipe_ctx->stream->link == aconnector->dc_link)
+		    pipe_ctx->stream->link == aconnector->dc_link &&
+		    pipe_ctx->stream->sink &&
+		    pipe_ctx->stream->sink == aconnector->dc_sink)
 			break;
 	}
 
@@ -1780,7 +1786,9 @@ static ssize_t dp_dsc_slice_width_write(struct file *f, const char __user *buf,
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx->stream &&
-		    pipe_ctx->stream->link == aconnector->dc_link)
+		    pipe_ctx->stream->link == aconnector->dc_link &&
+		    pipe_ctx->stream->sink &&
+		    pipe_ctx->stream->sink == aconnector->dc_sink)
 			break;
 	}
 
@@ -1865,7 +1873,9 @@ static ssize_t dp_dsc_slice_height_read(struct file *f, char __user *buf,
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx->stream &&
-		    pipe_ctx->stream->link == aconnector->dc_link)
+		    pipe_ctx->stream->link == aconnector->dc_link &&
+		    pipe_ctx->stream->sink &&
+		    pipe_ctx->stream->sink == aconnector->dc_sink)
 			break;
 	}
 
@@ -1964,7 +1974,9 @@ static ssize_t dp_dsc_slice_height_write(struct file *f, const char __user *buf,
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx->stream &&
-		    pipe_ctx->stream->link == aconnector->dc_link)
+		    pipe_ctx->stream->link == aconnector->dc_link &&
+		    pipe_ctx->stream->sink &&
+		    pipe_ctx->stream->sink == aconnector->dc_sink)
 			break;
 	}
 
@@ -2045,7 +2057,9 @@ static ssize_t dp_dsc_bits_per_pixel_read(struct file *f, char __user *buf,
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx->stream &&
-		    pipe_ctx->stream->link == aconnector->dc_link)
+		    pipe_ctx->stream->link == aconnector->dc_link &&
+		    pipe_ctx->stream->sink &&
+		    pipe_ctx->stream->sink == aconnector->dc_sink)
 			break;
 	}
 
@@ -2141,7 +2155,9 @@ static ssize_t dp_dsc_bits_per_pixel_write(struct file *f, const char __user *bu
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx->stream &&
-		    pipe_ctx->stream->link == aconnector->dc_link)
+		    pipe_ctx->stream->link == aconnector->dc_link &&
+		    pipe_ctx->stream->sink &&
+		    pipe_ctx->stream->sink == aconnector->dc_sink)
 			break;
 	}
 
@@ -2220,7 +2236,9 @@ static ssize_t dp_dsc_pic_width_read(struct file *f, char __user *buf,
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx->stream &&
-		    pipe_ctx->stream->link == aconnector->dc_link)
+		    pipe_ctx->stream->link == aconnector->dc_link &&
+		    pipe_ctx->stream->sink &&
+		    pipe_ctx->stream->sink == aconnector->dc_sink)
 			break;
 	}
 
@@ -2276,7 +2294,9 @@ static ssize_t dp_dsc_pic_height_read(struct file *f, char __user *buf,
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx->stream &&
-		    pipe_ctx->stream->link == aconnector->dc_link)
+		    pipe_ctx->stream->link == aconnector->dc_link &&
+		    pipe_ctx->stream->sink &&
+		    pipe_ctx->stream->sink == aconnector->dc_sink)
 			break;
 	}
 
@@ -2347,7 +2367,9 @@ static ssize_t dp_dsc_chunk_size_read(struct file *f, char __user *buf,
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx->stream &&
-		    pipe_ctx->stream->link == aconnector->dc_link)
+		    pipe_ctx->stream->link == aconnector->dc_link &&
+		    pipe_ctx->stream->sink &&
+		    pipe_ctx->stream->sink == aconnector->dc_sink)
 			break;
 	}
 
@@ -2418,7 +2440,9 @@ static ssize_t dp_dsc_slice_bpg_offset_read(struct file *f, char __user *buf,
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &aconnector->dc_link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx->stream &&
-		    pipe_ctx->stream->link == aconnector->dc_link)
+		    pipe_ctx->stream->link == aconnector->dc_link &&
+		    pipe_ctx->stream->sink &&
+		    pipe_ctx->stream->sink == aconnector->dc_sink)
 			break;
 	}
 
@@ -2936,7 +2960,7 @@ static int psr_read_residency(void *data, u64 *val)
 {
 	struct amdgpu_dm_connector *connector = data;
 	struct dc_link *link = connector->dc_link;
-	u32 residency;
+	u32 residency = 0;
 
 	link->dc->link_srv->edp_get_psr_residency(link, &residency);
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index c27063305a13..2c36f3d00ca2 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -363,7 +363,7 @@ void dm_helpers_dp_mst_send_payload_allocation(
 	mst_state = to_drm_dp_mst_topology_state(mst_mgr->base.state);
 	new_payload = drm_atomic_get_mst_payload_state(mst_state, aconnector->mst_output_port);
 
-	ret = drm_dp_add_payload_part2(mst_mgr, mst_state->base.state, new_payload);
+	ret = drm_dp_add_payload_part2(mst_mgr, new_payload);
 
 	if (ret) {
 		amdgpu_dm_set_mst_status(&aconnector->mst_status,
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
index 941e96f100f4..a5e1a93ddaea 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
@@ -613,6 +613,9 @@ dm_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
 		&connector->base,
 		dev->mode_config.tile_property,
 		0);
+	connector->colorspace_property = master->base.colorspace_property;
+	if (connector->colorspace_property)
+		drm_connector_attach_colorspace_property(connector);
 
 	drm_connector_set_path_property(connector, pathprop);
 
@@ -791,25 +794,12 @@ struct dsc_mst_fairness_params {
 	struct amdgpu_dm_connector *aconnector;
 };
 
-static uint16_t get_fec_overhead_multiplier(struct dc_link *dc_link)
-{
-	u8 link_coding_cap;
-	uint16_t fec_overhead_multiplier_x1000 = PBN_FEC_OVERHEAD_MULTIPLIER_8B_10B;
-
-	link_coding_cap = dc_link_dp_mst_decide_link_encoding_format(dc_link);
-	if (link_coding_cap == DP_128b_132b_ENCODING)
-		fec_overhead_multiplier_x1000 = PBN_FEC_OVERHEAD_MULTIPLIER_128B_132B;
-
-	return fec_overhead_multiplier_x1000;
-}
-
-static int kbps_to_peak_pbn(int kbps, uint16_t fec_overhead_multiplier_x1000)
+static int kbps_to_peak_pbn(int kbps)
 {
 	u64 peak_kbps = kbps;
 
 	peak_kbps *= 1006;
-	peak_kbps *= fec_overhead_multiplier_x1000;
-	peak_kbps = div_u64(peak_kbps, 1000 * 1000);
+	peak_kbps = div_u64(peak_kbps, 1000);
 	return (int) DIV64_U64_ROUND_UP(peak_kbps * 64, (54 * 8 * 1000));
 }
 
@@ -910,12 +900,11 @@ static int increase_dsc_bpp(struct drm_atomic_state *state,
 	int link_timeslots_used;
 	int fair_pbn_alloc;
 	int ret = 0;
-	uint16_t fec_overhead_multiplier_x1000 = get_fec_overhead_multiplier(dc_link);
 
 	for (i = 0; i < count; i++) {
 		if (vars[i + k].dsc_enabled) {
 			initial_slack[i] =
-			kbps_to_peak_pbn(params[i].bw_range.max_kbps, fec_overhead_multiplier_x1000) - vars[i + k].pbn;
+			kbps_to_peak_pbn(params[i].bw_range.max_kbps) - vars[i + k].pbn;
 			bpp_increased[i] = false;
 			remaining_to_increase += 1;
 		} else {
@@ -1011,7 +1000,6 @@ static int try_disable_dsc(struct drm_atomic_state *state,
 	int next_index;
 	int remaining_to_try = 0;
 	int ret;
-	uint16_t fec_overhead_multiplier_x1000 = get_fec_overhead_multiplier(dc_link);
 
 	for (i = 0; i < count; i++) {
 		if (vars[i + k].dsc_enabled
@@ -1041,7 +1029,7 @@ static int try_disable_dsc(struct drm_atomic_state *state,
 		if (next_index == -1)
 			break;
 
-		vars[next_index].pbn = kbps_to_peak_pbn(params[next_index].bw_range.stream_kbps, fec_overhead_multiplier_x1000);
+		vars[next_index].pbn = kbps_to_peak_pbn(params[next_index].bw_range.stream_kbps);
 		ret = drm_dp_atomic_find_time_slots(state,
 						    params[next_index].port->mgr,
 						    params[next_index].port,
@@ -1054,7 +1042,8 @@ static int try_disable_dsc(struct drm_atomic_state *state,
 			vars[next_index].dsc_enabled = false;
 			vars[next_index].bpp_x16 = 0;
 		} else {
-			vars[next_index].pbn = kbps_to_peak_pbn(params[next_index].bw_range.max_kbps, fec_overhead_multiplier_x1000);
+			vars[next_index].pbn = kbps_to_peak_pbn(
+				params[next_index].bw_range.max_kbps);
 			ret = drm_dp_atomic_find_time_slots(state,
 							    params[next_index].port->mgr,
 							    params[next_index].port,
@@ -1083,7 +1072,6 @@ static int compute_mst_dsc_configs_for_link(struct drm_atomic_state *state,
 	int count = 0;
 	int i, k, ret;
 	bool debugfs_overwrite = false;
-	uint16_t fec_overhead_multiplier_x1000 = get_fec_overhead_multiplier(dc_link);
 
 	memset(params, 0, sizeof(params));
 
@@ -1148,7 +1136,7 @@ static int compute_mst_dsc_configs_for_link(struct drm_atomic_state *state,
 	/* Try no compression */
 	for (i = 0; i < count; i++) {
 		vars[i + k].aconnector = params[i].aconnector;
-		vars[i + k].pbn = kbps_to_peak_pbn(params[i].bw_range.stream_kbps, fec_overhead_multiplier_x1000);
+		vars[i + k].pbn = kbps_to_peak_pbn(params[i].bw_range.stream_kbps);
 		vars[i + k].dsc_enabled = false;
 		vars[i + k].bpp_x16 = 0;
 		ret = drm_dp_atomic_find_time_slots(state, params[i].port->mgr, params[i].port,
@@ -1167,7 +1155,7 @@ static int compute_mst_dsc_configs_for_link(struct drm_atomic_state *state,
 	/* Try max compression */
 	for (i = 0; i < count; i++) {
 		if (params[i].compression_possible && params[i].clock_force_enable != DSC_CLK_FORCE_DISABLE) {
-			vars[i + k].pbn = kbps_to_peak_pbn(params[i].bw_range.min_kbps, fec_overhead_multiplier_x1000);
+			vars[i + k].pbn = kbps_to_peak_pbn(params[i].bw_range.min_kbps);
 			vars[i + k].dsc_enabled = true;
 			vars[i + k].bpp_x16 = params[i].bw_range.min_target_bpp_x16;
 			ret = drm_dp_atomic_find_time_slots(state, params[i].port->mgr,
@@ -1175,7 +1163,7 @@ static int compute_mst_dsc_configs_for_link(struct drm_atomic_state *state,
 			if (ret < 0)
 				return ret;
 		} else {
-			vars[i + k].pbn = kbps_to_peak_pbn(params[i].bw_range.stream_kbps, fec_overhead_multiplier_x1000);
+			vars[i + k].pbn = kbps_to_peak_pbn(params[i].bw_range.stream_kbps);
 			vars[i + k].dsc_enabled = false;
 			vars[i + k].bpp_x16 = 0;
 			ret = drm_dp_atomic_find_time_slots(state, params[i].port->mgr,
@@ -1219,8 +1207,10 @@ static bool is_dsc_need_re_compute(
 	if (dc_link->type != dc_connection_mst_branch)
 		return false;
 
-	if (!(dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT ||
-		dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_PASSTHROUGH_SUPPORT))
+	/* add a check for older MST DSC with no virtual DPCDs */
+	if (needs_dsc_aux_workaround(dc_link)  &&
+		(!(dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT ||
+		dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_PASSTHROUGH_SUPPORT)))
 		return false;
 
 	for (i = 0; i < MAX_PIPES; i++)
@@ -1240,7 +1230,15 @@ static bool is_dsc_need_re_compute(
 			continue;
 
 		aconnector = (struct amdgpu_dm_connector *) stream->dm_stream_context;
-		if (!aconnector)
+		if (!aconnector || !aconnector->dsc_aux)
+			continue;
+
+		/*
+		 *	check if cached virtual MST DSC caps are available and DSC is supported
+		 *	as per specifications in their Virtual DPCD registers.
+		*/
+		if (!(aconnector->dc_sink->dsc_caps.dsc_dec_caps.is_dsc_supported ||
+			aconnector->dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_PASSTHROUGH_SUPPORT))
 			continue;
 
 		stream_on_link[new_stream_on_link_num] = aconnector;
@@ -1601,7 +1599,7 @@ enum dc_status dm_dp_mst_is_port_support_mode(
 	struct amdgpu_dm_connector *aconnector,
 	struct dc_stream_state *stream)
 {
-	int bpp, pbn, branch_max_throughput_mps = 0;
+	int pbn, branch_max_throughput_mps = 0;
 	struct dc_link_settings cur_link_settings;
 	unsigned int end_to_end_bw_in_kbps = 0;
 	unsigned int upper_link_bw_in_kbps = 0, down_link_bw_in_kbps = 0;
@@ -1651,11 +1649,34 @@ enum dc_status dm_dp_mst_is_port_support_mode(
 			}
 		}
 	} else {
-		/* check if mode could be supported within full_pbn */
-		bpp = convert_dc_color_depth_into_bpc(stream->timing.display_color_depth) * 3;
-		pbn = drm_dp_calc_pbn_mode(stream->timing.pix_clk_100hz / 10, bpp << 4);
-		if (pbn > aconnector->mst_output_port->full_pbn)
+		/* Check if mode could be supported within max slot
+		 * number of current mst link and full_pbn of mst links.
+		 */
+		int pbn_div, slot_num, max_slot_num;
+		enum dc_link_encoding_format link_encoding;
+		uint32_t stream_kbps =
+			dc_bandwidth_in_kbps_from_timing(&stream->timing,
+				dc_link_get_highest_encoding_format(stream->link));
+
+		pbn = kbps_to_peak_pbn(stream_kbps);
+		pbn_div = dm_mst_get_pbn_divider(stream->link);
+		slot_num = DIV_ROUND_UP(pbn, pbn_div);
+
+		link_encoding = dc_link_get_highest_encoding_format(stream->link);
+		if (link_encoding == DC_LINK_ENCODING_DP_8b_10b)
+			max_slot_num = 63;
+		else if (link_encoding == DC_LINK_ENCODING_DP_128b_132b)
+			max_slot_num = 64;
+		else {
+			DRM_DEBUG_DRIVER("Invalid link encoding format\n");
 			return DC_FAIL_BANDWIDTH_VALIDATE;
+		}
+
+		if (slot_num > max_slot_num ||
+			pbn > aconnector->mst_output_port->full_pbn) {
+			DRM_DEBUG_DRIVER("Mode can not be supported within mst links!");
+			return DC_FAIL_BANDWIDTH_VALIDATE;
+		}
 	}
 
 	/* check is mst dsc output bandwidth branch_overall_throughput_0_mps */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h
index 37c820ab0fdb..fa84d34b7373 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h
@@ -46,9 +46,6 @@
 #define SYNAPTICS_CASCADED_HUB_ID  0x5A
 #define IS_SYNAPTICS_CASCADED_PANAMERA(devName, data) ((IS_SYNAPTICS_PANAMERA(devName) && ((int)data[2] == SYNAPTICS_CASCADED_HUB_ID)) ? 1 : 0)
 
-#define PBN_FEC_OVERHEAD_MULTIPLIER_8B_10B	1031
-#define PBN_FEC_OVERHEAD_MULTIPLIER_128B_132B	1000
-
 enum mst_msg_ready_type {
 	NONE_MSG_RDY_EVENT = 0,
 	DOWN_REP_MSG_RDY_EVENT = 1,
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
index 286ecd28cc6e..bfa090432ce2 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
@@ -212,7 +212,7 @@ bool amdgpu_dm_psr_disable(struct dc_stream_state *stream)
 }
 
 /*
- * amdgpu_dm_psr_disable() - disable psr f/w
+ * amdgpu_dm_psr_disable_all() - disable psr f/w for all streams
  * if psr is enabled on any stream
  *
  * Return: true if success
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
index d9e33c6bccd9..0005f5f8f34f 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
@@ -52,4 +52,12 @@ void dm_perf_trace_timestamp(const char *func_name, unsigned int line, struct dc
 				    func_name, line);
 }
 
+void dm_trace_smu_msg(uint32_t msg_id, uint32_t param_in, struct dc_context *ctx)
+{
+}
+
+void dm_trace_smu_delay(uint32_t delay, struct dc_context *ctx)
+{
+}
+
 /**** power component interfaces ****/
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h
index 133af994a08c..4686d4b0cbad 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h
@@ -87,7 +87,7 @@ TRACE_EVENT(amdgpu_dc_performance,
 			__entry->writes = write_count;
 			__entry->read_delta = read_count - *last_read;
 			__entry->write_delta = write_count - *last_write;
-			__assign_str(func, func);
+			__assign_str(func);
 			__entry->line = line;
 			*last_read = read_count;
 			*last_write = write_count;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c
index 4ae4720535a5..e46f8ce41d87 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c
@@ -26,16 +26,7 @@
 
 #include "dc_trace.h"
 
-#if defined(CONFIG_X86)
-#include <asm/fpu/api.h>
-#elif defined(CONFIG_PPC64)
-#include <asm/switch_to.h>
-#include <asm/cputable.h>
-#elif defined(CONFIG_ARM64)
-#include <asm/neon.h>
-#elif defined(CONFIG_LOONGARCH)
-#include <asm/fpu.h>
-#endif
+#include <linux/fpu.h>
 
 /**
  * DOC: DC FPU manipulation overview
@@ -87,20 +78,9 @@ void dc_fpu_begin(const char *function_name, const int line)
 	WARN_ON_ONCE(!in_task());
 	preempt_disable();
 	depth = __this_cpu_inc_return(fpu_recursion_depth);
-
 	if (depth == 1) {
-#if defined(CONFIG_X86) || defined(CONFIG_LOONGARCH)
+		BUG_ON(!kernel_fpu_available());
 		kernel_fpu_begin();
-#elif defined(CONFIG_PPC64)
-		if (cpu_has_feature(CPU_FTR_VSX_COMP))
-			enable_kernel_vsx();
-		else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP))
-			enable_kernel_altivec();
-		else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE))
-			enable_kernel_fp();
-#elif defined(CONFIG_ARM64)
-		kernel_neon_begin();
-#endif
 	}
 
 	TRACE_DCN_FPU(true, function_name, line, depth);
@@ -122,18 +102,7 @@ void dc_fpu_end(const char *function_name, const int line)
 
 	depth = __this_cpu_dec_return(fpu_recursion_depth);
 	if (depth == 0) {
-#if defined(CONFIG_X86) || defined(CONFIG_LOONGARCH)
 		kernel_fpu_end();
-#elif defined(CONFIG_PPC64)
-		if (cpu_has_feature(CPU_FTR_VSX_COMP))
-			disable_kernel_vsx();
-		else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP))
-			disable_kernel_altivec();
-		else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE))
-			disable_kernel_fp();
-#elif defined(CONFIG_ARM64)
-		kernel_neon_end();
-#endif
 	} else {
 		WARN_ON_ONCE(depth < 0);
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/Makefile b/drivers/gpu/drm/amd/display/dc/Makefile
index 7991ae468f75..4e9fb1742877 100644
--- a/drivers/gpu/drm/amd/display/dc/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/Makefile
@@ -22,7 +22,7 @@
 #
 # Makefile for Display Core (dc) component.
 
-DC_LIBS = basics bios dml clk_mgr dce gpio hwss irq link virtual dsc resource optc
+DC_LIBS = basics bios dml clk_mgr dce gpio hwss irq link virtual dsc resource optc dpp
 
 ifdef CONFIG_DRM_AMD_DC_FP
 
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
index 6450853fea94..bc16db69a663 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
@@ -44,8 +44,6 @@
 
 #include "bios_parser_common.h"
 
-#include "dc.h"
-
 #define THREE_PERCENT_OF_10000 300
 
 #define LAST_RECORD_TYPE 0xff
@@ -1731,6 +1729,7 @@ static uint32_t get_ss_entry_number_from_internal_ss_info_tbl_v2_1(
 
 	return 0;
 }
+
 /**
  * get_ss_entry_number_from_internal_ss_info_tbl_V3_1
  * Get Number of SpreadSpectrum Entry from the ASIC_InternalSS_Info table of
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
index 05f392501c0a..9fe0020bcb9c 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
@@ -1594,8 +1594,6 @@ static bool bios_parser_is_device_id_supported(
 		return (le16_to_cpu(bp->object_info_tbl.v1_5->supporteddevices) & mask) != 0;
 		break;
 	}
-
-	return false;
 }
 
 static uint32_t bios_parser_get_ss_entry_number(
@@ -2948,6 +2946,7 @@ static enum bp_result construct_integrated_info(
 				result = get_integrated_info_v2_1(bp, info);
 				break;
 			case 2:
+			case 3:
 				result = get_integrated_info_v2_2(bp, info);
 				break;
 			default:
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table.c b/drivers/gpu/drm/amd/display/dc/bios/command_table.c
index 86f9198e7501..2bcae0643e61 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table.c
@@ -399,7 +399,7 @@ static enum bp_result transmitter_control_v1_6(
 static void init_transmitter_control(struct bios_parser *bp)
 {
 	uint8_t frev;
-	uint8_t crev;
+	uint8_t crev = 0;
 
 	if (BIOS_CMD_TABLE_REVISION(UNIPHYTransmitterControl,
 			frev, crev) == false)
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
index cbae1be7b009..cc000833d300 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
@@ -225,7 +225,7 @@ static enum bp_result transmitter_control_fallback(
 static void init_transmitter_control(struct bios_parser *bp)
 {
 	uint8_t frev;
-	uint8_t crev;
+	uint8_t crev = 0;
 
 	BIOS_CMD_TABLE_REVISION(dig1transmittercontrol, frev, crev);
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
index 9f0f25aee426..a2b4ff2cff16 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
@@ -272,7 +272,7 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p
 			dcn3_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg);
 			return &clk_mgr->base;
 		}
-		if (asic_id.chip_id == DEVICE_ID_NV_13FE) {
+		if (ctx->dce_version == DCN_VERSION_2_01) {
 			dcn201_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg);
 			return &clk_mgr->base;
 		}
@@ -329,15 +329,14 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p
 	}
 		break;
 	case AMDGPU_FAMILY_GC_11_0_0: {
-	    struct clk_mgr_internal *clk_mgr = kzalloc(sizeof(*clk_mgr), GFP_KERNEL);
-
-	    if (clk_mgr == NULL) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	    }
+		struct clk_mgr_internal *clk_mgr = kzalloc(sizeof(*clk_mgr), GFP_KERNEL);
 
-	    dcn32_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg);
-	    return &clk_mgr->base;
+		if (clk_mgr == NULL) {
+			BREAK_TO_DEBUGGER();
+			return NULL;
+		}
+		dcn32_clk_mgr_construct(ctx, clk_mgr, pp_smu, dccg);
+		return &clk_mgr->base;
 	}
 
 	case AMDGPU_FAMILY_GC_11_0_1: {
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c
index b77804cfde0f..2a5dd3a296b2 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c
@@ -131,8 +131,8 @@ int dce_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base)
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	int dprefclk_wdivider;
 	int dprefclk_src_sel;
-	int dp_ref_clk_khz;
-	int target_div = 600000;
+	int dp_ref_clk_khz = 600000;
+	int target_div;
 
 	/* ASSERT DP Reference Clock source is from DFS*/
 	REG_GET(DPREFCLK_CNTL, DPREFCLK_SRC_SEL, &dprefclk_src_sel);
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c
index 2a74e2d74909..369421e46c52 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c
@@ -23,7 +23,6 @@
  *
  */
 
-#include "reg_helper.h"
 #include "core_types.h"
 #include "clk_mgr_internal.h"
 #include "rv1_clk_mgr.h"
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_vbios_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_vbios_smu.c
index 89b79dd39628..19897fa52e7e 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_vbios_smu.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr_vbios_smu.c
@@ -26,7 +26,6 @@
 #include "core_types.h"
 #include "clk_mgr_internal.h"
 #include "reg_helper.h"
-#include <linux/delay.h>
 
 #include "rv1_clk_mgr_vbios_smu.h"
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c
index 5ee87965a078..bb4f3bd7532e 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c
@@ -503,7 +503,7 @@ static void dcn2_notify_link_rate_change(struct clk_mgr *clk_mgr_base, struct dc
 
 	clk_mgr->cur_phyclk_req_table[link->link_index] = link->cur_link_settings.link_rate * LINK_RATE_REF_FREQ_IN_KHZ;
 
-	for (i = 0; i < MAX_PIPES * 2; i++) {
+	for (i = 0; i < MAX_LINKS; i++) {
 		if (clk_mgr->cur_phyclk_req_table[i] > max_phyclk_req)
 			max_phyclk_req = clk_mgr->cur_phyclk_req_table[i];
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c
index 9c90090e7351..f77840dd051e 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c
@@ -100,7 +100,15 @@ static void dcn201_update_clocks(struct clk_mgr *clk_mgr_base,
 
 	if (clk_mgr_base->clks.dispclk_khz == 0 ||
 	    dc->debug.force_clock_mode & 0x1) {
+		/* this is from resume or boot up, if forced_clock cfg option
+		 * used, we bypass program dispclk and DPPCLK, but need set them
+		 * for S3.
+		 */
+
 		force_reset = true;
+		/* force_clock_mode 0x1:  force reset the clock even it is the
+		 * same clock as long as it is in Passive level.
+		 */
 
 		dcn2_read_clocks_from_hw_dentist(clk_mgr_base);
 	}
@@ -150,11 +158,14 @@ static void dcn201_update_clocks(struct clk_mgr *clk_mgr_base,
 
 	if (dc->config.forced_clocks == false || (force_reset && safe_to_lower)) {
 		if (dpp_clock_lowered) {
+			// if clock is being lowered, increase DTO before lowering refclk
 			dcn20_update_clocks_update_dpp_dto(clk_mgr, context, safe_to_lower);
 			dcn20_update_clocks_update_dentist(clk_mgr, context);
 		} else {
+			// if clock is being raised, increase refclk before lowering DTO
 			if (update_dppclk || update_dispclk)
 				dcn20_update_clocks_update_dentist(clk_mgr, context);
+			// always update dtos unless clock is lowered and not safe to lower
 			dcn20_update_clocks_update_dpp_dto(clk_mgr, context, safe_to_lower);
 		}
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c
index e3e1940198a9..5ef0879f6ad9 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c
@@ -548,7 +548,7 @@ static void rn_notify_link_rate_change(struct clk_mgr *clk_mgr_base, struct dc_l
 
 	clk_mgr->cur_phyclk_req_table[link->link_index] = link->cur_link_settings.link_rate * LINK_RATE_REF_FREQ_IN_KHZ;
 
-	for (i = 0; i < MAX_PIPES * 2; i++) {
+	for (i = 0; i < MAX_LINKS; i++) {
 		if (clk_mgr->cur_phyclk_req_table[i] > max_phyclk_req)
 			max_phyclk_req = clk_mgr->cur_phyclk_req_table[i];
 	}
@@ -642,7 +642,8 @@ static void rn_clk_mgr_helper_populate_bw_params(struct clk_bw_params *bw_params
 
 	j = -1;
 
-	ASSERT(PP_SMU_NUM_FCLK_DPM_LEVELS <= MAX_NUM_DPM_LVL);
+	static_assert(PP_SMU_NUM_FCLK_DPM_LEVELS <= MAX_NUM_DPM_LVL,
+		"number of reported FCLK DPM levels exceed maximum");
 
 	/* Find lowest DPM, FCLK is filled in reverse order*/
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
index 3271c8c7905d..8083a553c60e 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
@@ -474,7 +474,7 @@ static void dcn30_notify_link_rate_change(struct clk_mgr *clk_mgr_base, struct d
 
 	clk_mgr->cur_phyclk_req_table[link->link_index] = link->cur_link_settings.link_rate * LINK_RATE_REF_FREQ_IN_KHZ;
 
-	for (i = 0; i < MAX_PIPES * 2; i++) {
+	for (i = 0; i < MAX_LINKS; i++) {
 		if (clk_mgr->cur_phyclk_req_table[i] > max_phyclk_req)
 			max_phyclk_req = clk_mgr->cur_phyclk_req_table[i];
 	}
@@ -560,11 +560,19 @@ void dcn3_clk_mgr_construct(
 	dce_clock_read_ss_info(clk_mgr);
 
 	clk_mgr->base.bw_params = kzalloc(sizeof(*clk_mgr->base.bw_params), GFP_KERNEL);
+	if (!clk_mgr->base.bw_params) {
+		BREAK_TO_DEBUGGER();
+		return;
+	}
 
 	/* need physical address of table to give to PMFW */
 	clk_mgr->wm_range_table = dm_helpers_allocate_gpu_mem(clk_mgr->base.ctx,
 			DC_MEM_ALLOC_TYPE_GART, sizeof(WatermarksExternal_t),
 			&clk_mgr->wm_range_table_addr);
+	if (!clk_mgr->wm_range_table) {
+		BREAK_TO_DEBUGGER();
+		return;
+	}
 }
 
 void dcn3_clk_mgr_destroy(struct clk_mgr_internal *clk_mgr)
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.c
index bdbf18306698..3253115a153d 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.c
@@ -23,7 +23,6 @@
  *
  */
 
-#include <linux/delay.h>
 #include "dcn30_clk_mgr_smu_msg.h"
 
 #include "clk_mgr_internal.h"
@@ -54,6 +53,7 @@
  */
 static uint32_t dcn30_smu_wait_for_response(struct clk_mgr_internal *clk_mgr, unsigned int delay_us, unsigned int max_retries)
 {
+	const uint32_t initial_max_retries = max_retries;
 	uint32_t reg = 0;
 
 	do {
@@ -69,7 +69,7 @@ static uint32_t dcn30_smu_wait_for_response(struct clk_mgr_internal *clk_mgr, un
 
 	/* handle DALSMC_Result_CmdRejectedBusy? */
 
-	/* Log? */
+	TRACE_SMU_DELAY(delay_us * (initial_max_retries - max_retries), clk_mgr->base.ctx);
 
 	return reg;
 }
@@ -89,6 +89,8 @@ static bool dcn30_smu_send_msg_with_param(struct clk_mgr_internal *clk_mgr, uint
 	/* Trigger the message transaction by writing the message ID */
 	REG_WRITE(DAL_MSG_REG, msg_id);
 
+	TRACE_SMU_MSG(msg_id, param_in, clk_mgr->base.ctx);
+
 	result = dcn30_smu_wait_for_response(clk_mgr, 10, 200000);
 
 	if (IS_SMU_TIMEOUT(result)) {
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c
index aa9fd1dc550a..191d8b969d19 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c
@@ -566,7 +566,8 @@ static void vg_clk_mgr_helper_populate_bw_params(
 
 	j = -1;
 
-	ASSERT(VG_NUM_FCLK_DPM_LEVELS <= MAX_NUM_DPM_LVL);
+	static_assert(VG_NUM_FCLK_DPM_LEVELS <= MAX_NUM_DPM_LVL,
+		"number of reported FCLK DPM levels exceeds maximum");
 
 	/* Find lowest DPM, FCLK is filled in reverse order*/
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
index ce1386e22576..12a7752758b8 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
@@ -562,7 +562,8 @@ static void dcn31_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *clk
 
 	j = -1;
 
-	ASSERT(NUM_DF_PSTATE_LEVELS <= MAX_NUM_DPM_LVL);
+	static_assert(NUM_DF_PSTATE_LEVELS <= MAX_NUM_DPM_LVL,
+		"number of reported pstate levels exceeds maximum");
 
 	/* Find lowest DPM, FCLK is filled in reverse order*/
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_smu.c
index 6904e95113c1..f201628e4e98 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_smu.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_smu.c
@@ -23,7 +23,6 @@
  *
  */
 
-#include <linux/delay.h>
 #include "core_types.h"
 #include "clk_mgr_internal.h"
 #include "reg_helper.h"
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_smu.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_smu.h
index 047d19ea919c..78ca1e5c5e9e 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_smu.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_smu.h
@@ -37,34 +37,34 @@ typedef enum {
 } WCK_RATIO_e;
 
 typedef struct {
-  uint32_t FClk;
-  uint32_t MemClk;
-  uint32_t Voltage;
-  uint8_t  WckRatio;
-  uint8_t  Spare[3];
+	uint32_t FClk;
+	uint32_t MemClk;
+	uint32_t Voltage;
+	uint8_t  WckRatio;
+	uint8_t  Spare[3];
 } DfPstateTable314_t;
 
 //Freq in MHz
 //Voltage in milli volts with 2 fractional bits
 typedef struct {
-  uint32_t DcfClocks[NUM_DCFCLK_DPM_LEVELS];
-  uint32_t DispClocks[NUM_DISPCLK_DPM_LEVELS];
-  uint32_t DppClocks[NUM_DPPCLK_DPM_LEVELS];
-  uint32_t SocClocks[NUM_SOCCLK_DPM_LEVELS];
-  uint32_t VClocks[NUM_VCN_DPM_LEVELS];
-  uint32_t DClocks[NUM_VCN_DPM_LEVELS];
-  uint32_t SocVoltage[NUM_SOC_VOLTAGE_LEVELS];
-  DfPstateTable314_t DfPstateTable[NUM_DF_PSTATE_LEVELS];
+	uint32_t DcfClocks[NUM_DCFCLK_DPM_LEVELS];
+	uint32_t DispClocks[NUM_DISPCLK_DPM_LEVELS];
+	uint32_t DppClocks[NUM_DPPCLK_DPM_LEVELS];
+	uint32_t SocClocks[NUM_SOCCLK_DPM_LEVELS];
+	uint32_t VClocks[NUM_VCN_DPM_LEVELS];
+	uint32_t DClocks[NUM_VCN_DPM_LEVELS];
+	uint32_t SocVoltage[NUM_SOC_VOLTAGE_LEVELS];
+	DfPstateTable314_t DfPstateTable[NUM_DF_PSTATE_LEVELS];
 
-  uint8_t  NumDcfClkLevelsEnabled;
-  uint8_t  NumDispClkLevelsEnabled; //Applies to both Dispclk and Dppclk
-  uint8_t  NumSocClkLevelsEnabled;
-  uint8_t  VcnClkLevelsEnabled;     //Applies to both Vclk and Dclk
-  uint8_t  NumDfPstatesEnabled;
-  uint8_t  spare[3];
+	uint8_t  NumDcfClkLevelsEnabled;
+	uint8_t  NumDispClkLevelsEnabled; //Applies to both Dispclk and Dppclk
+	uint8_t  NumSocClkLevelsEnabled;
+	uint8_t  VcnClkLevelsEnabled;     //Applies to both Vclk and Dclk
+	uint8_t  NumDfPstatesEnabled;
+	uint8_t  spare[3];
 
-  uint32_t MinGfxClk;
-  uint32_t MaxGfxClk;
+	uint32_t MinGfxClk;
+	uint32_t MaxGfxClk;
 } DpmClocks314_t;
 
 struct dcn314_watermarks {
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
index 644da4637320..5506cf9b3672 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
@@ -145,6 +145,10 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base,
 	 */
 	clk_mgr_base->clks.zstate_support = new_clocks->zstate_support;
 	if (safe_to_lower) {
+		if (clk_mgr_base->clks.dtbclk_en && !new_clocks->dtbclk_en) {
+			dcn315_smu_set_dtbclk(clk_mgr, false);
+			clk_mgr_base->clks.dtbclk_en = new_clocks->dtbclk_en;
+		}
 		/* check that we're not already in lower */
 		if (clk_mgr_base->clks.pwr_state != DCN_PWR_STATE_LOW_POWER) {
 			display_count = dcn315_get_active_display_cnt_wa(dc, context);
@@ -160,6 +164,10 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base,
 			}
 		}
 	} else {
+		if (!clk_mgr_base->clks.dtbclk_en && new_clocks->dtbclk_en) {
+			dcn315_smu_set_dtbclk(clk_mgr, true);
+			clk_mgr_base->clks.dtbclk_en = new_clocks->dtbclk_en;
+		}
 		/* check that we're not already in D0 */
 		if (clk_mgr_base->clks.pwr_state != DCN_PWR_STATE_MISSION_MODE) {
 			union display_idle_optimization_u idle_info = { 0 };
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_smu.c
index 879f1494c4cd..2d14346b680e 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_smu.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_smu.c
@@ -29,6 +29,7 @@
 #include "dm_helpers.h"
 #include "dcn315_smu.h"
 #include "mp/mp_13_0_5_offset.h"
+#include "logger_types.h"
 
 #define MAX_INSTANCE                                        6
 #define MAX_SEGMENT                                         6
@@ -69,7 +70,6 @@ static const struct IP_BASE NBIO_BASE = { { { { 0x00000000, 0x00000014, 0x00000D
 #define REG_NBIO(reg_name) \
 	(NBIO_BASE.instance[0].segment[regBIF_BX_PF2_ ## reg_name ## _BASE_IDX] + regBIF_BX_PF2_ ## reg_name)
 
-#include "logger_types.h"
 #undef DC_LOGGER
 #define DC_LOGGER \
 	CTX->logger
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c
index 6ad4f4efec5d..20ca7afa9cb4 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c
@@ -485,7 +485,8 @@ static void dcn316_clk_mgr_helper_populate_bw_params(
 
 	j = -1;
 
-	ASSERT(NUM_DF_PSTATE_LEVELS <= MAX_NUM_DPM_LVL);
+	static_assert(NUM_DF_PSTATE_LEVELS <= MAX_NUM_DPM_LVL,
+		"number of reported pstate levels exceeds maximum");
 
 	/* Find lowest DPM, FCLK is filled in reverse order*/
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
index bec252e1dd27..ff5fdc7b1198 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
@@ -29,6 +29,7 @@
 #include "dcn20/dcn20_clk_mgr.h"
 #include "dce100/dce_clk_mgr.h"
 #include "dcn31/dcn31_clk_mgr.h"
+#include "dcn32/dcn32_clk_mgr.h"
 #include "reg_helper.h"
 #include "core_types.h"
 #include "dm_helpers.h"
@@ -40,7 +41,6 @@
 #include "dcn/dcn_3_2_0_offset.h"
 #include "dcn/dcn_3_2_0_sh_mask.h"
 
-#include "dcn32/dcn32_clk_mgr.h"
 #include "dml/dcn32/dcn32_fpu.h"
 
 #define DCN_BASE__INST0_SEG1                       0x000000C0
@@ -712,8 +712,12 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base,
 					 * since we calculate mode support based on softmax being the max UCLK
 					 * frequency.
 					 */
-					dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
-							dc->clk_mgr->bw_params->dc_mode_softmax_memclk);
+					if (dc->debug.disable_dc_mode_overwrite) {
+						dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK, dc->clk_mgr->bw_params->max_memclk_mhz);
+						dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, dc->clk_mgr->bw_params->max_memclk_mhz);
+					} else
+						dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
+								dc->clk_mgr->bw_params->dc_mode_softmax_memclk);
 				} else {
 					dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, dc->clk_mgr->bw_params->max_memclk_mhz);
 				}
@@ -746,8 +750,13 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base,
 		/* set UCLK to requested value if P-State switching is supported, or to re-enable P-State switching */
 		if (clk_mgr_base->clks.p_state_change_support &&
 				(update_uclk || !clk_mgr_base->clks.prev_p_state_change_support) &&
-				!dc->work_arounds.clock_update_disable_mask.uclk)
+				!dc->work_arounds.clock_update_disable_mask.uclk) {
+			if (dc->clk_mgr->dc_mode_softmax_enabled && dc->debug.disable_dc_mode_overwrite)
+				dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK,
+						max((int)dc->clk_mgr->bw_params->dc_mode_softmax_memclk, khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz)));
+
 			dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz));
+		}
 
 		if (clk_mgr_base->clks.num_ways != new_clocks->num_ways &&
 				clk_mgr_base->clks.num_ways > new_clocks->num_ways) {
@@ -829,7 +838,7 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base,
 		dmcu->funcs->set_psr_wait_loop(dmcu,
 				clk_mgr_base->clks.dispclk_khz / 1000 / 7);
 
-	if (dc->config.enable_auto_dpm_test_logs && safe_to_lower) {
+	if (dc->config.enable_auto_dpm_test_logs) {
 	    dcn32_auto_dpm_test_log(new_clocks, clk_mgr, context);
 	}
 }
@@ -1199,11 +1208,19 @@ void dcn32_clk_mgr_construct(
 	clk_mgr->smu_present = false;
 
 	clk_mgr->base.bw_params = kzalloc(sizeof(*clk_mgr->base.bw_params), GFP_KERNEL);
+	if (!clk_mgr->base.bw_params) {
+		BREAK_TO_DEBUGGER();
+		return;
+	}
 
 	/* need physical address of table to give to PMFW */
 	clk_mgr->wm_range_table = dm_helpers_allocate_gpu_mem(clk_mgr->base.ctx,
 			DC_MEM_ALLOC_TYPE_GART, sizeof(WatermarksExternal_t),
 			&clk_mgr->wm_range_table_addr);
+	if (!clk_mgr->wm_range_table) {
+		BREAK_TO_DEBUGGER();
+		return;
+	}
 }
 
 void dcn32_clk_mgr_destroy(struct clk_mgr_internal *clk_mgr)
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr_smu_msg.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr_smu_msg.c
index df244b175fdb..f2f60478b1a6 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr_smu_msg.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr_smu_msg.c
@@ -49,6 +49,7 @@
  */
 static uint32_t dcn32_smu_wait_for_response(struct clk_mgr_internal *clk_mgr, unsigned int delay_us, unsigned int max_retries)
 {
+	const uint32_t initial_max_retries = max_retries;
 	uint32_t reg = 0;
 
 	do {
@@ -62,6 +63,8 @@ static uint32_t dcn32_smu_wait_for_response(struct clk_mgr_internal *clk_mgr, un
 			udelay(delay_us);
 	} while (max_retries--);
 
+	TRACE_SMU_DELAY(delay_us * (initial_max_retries - max_retries), clk_mgr->base.ctx);
+
 	return reg;
 }
 
@@ -79,6 +82,8 @@ static bool dcn32_smu_send_msg_with_param(struct clk_mgr_internal *clk_mgr, uint
 	/* Trigger the message transaction by writing the message ID */
 	REG_WRITE(DAL_MSG_REG, msg_id);
 
+	TRACE_SMU_MSG(msg_id, param_in, clk_mgr->base.ctx);
+
 	/* Wait for response */
 	if (dcn32_smu_wait_for_response(clk_mgr, 10, 200000) == DALSMC_Result_OK) {
 		if (param_out)
@@ -115,6 +120,8 @@ static uint32_t dcn32_smu_wait_for_response_delay(struct clk_mgr_internal *clk_m
 		*total_delay_us += delay_us;
 	} while (max_retries--);
 
+	TRACE_SMU_DELAY(*total_delay_us, clk_mgr->base.ctx);
+
 	return reg;
 }
 
@@ -135,6 +142,8 @@ static bool dcn32_smu_send_msg_with_param_delay(struct clk_mgr_internal *clk_mgr
 	/* Trigger the message transaction by writing the message ID */
 	REG_WRITE(DAL_MSG_REG, msg_id);
 
+	TRACE_SMU_MSG(msg_id, param_in, clk_mgr->base.ctx);
+
 	/* Wait for response */
 	if (dcn32_smu_wait_for_response_delay(clk_mgr, 10, 200000, &delay2_us) == DALSMC_Result_OK) {
 		if (param_out)
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr_smu_msg.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr_smu_msg.h
index c76352a817de..5c44ab0e8667 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr_smu_msg.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr_smu_msg.h
@@ -37,10 +37,9 @@
 #define DALSMC_Result_OK				0x1
 
 void dcn32_smu_send_fclk_pstate_message(struct clk_mgr_internal *clk_mgr, bool enable);
-void dcn32_smu_transfer_wm_table_dram_2_smu(struct clk_mgr_internal *clk_mgr);
-void dcn32_smu_set_pme_workaround(struct clk_mgr_internal *clk_mgr);
 void dcn32_smu_send_cab_for_uclk_message(struct clk_mgr_internal *clk_mgr, unsigned int num_ways);
 void dcn32_smu_transfer_wm_table_dram_2_smu(struct clk_mgr_internal *clk_mgr);
+void dcn32_smu_set_pme_workaround(struct clk_mgr_internal *clk_mgr);
 unsigned int dcn32_smu_set_hard_min_by_freq(struct clk_mgr_internal *clk_mgr, uint32_t clk, uint16_t freq_mhz);
 void dcn32_smu_wait_for_dmub_ack_mclk(struct clk_mgr_internal *clk_mgr, bool enable);
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
index d9c5692c86c2..6c9b4e6491a5 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
@@ -252,7 +252,8 @@ void dcn35_update_clocks(struct clk_mgr *clk_mgr_base,
 		}
 
 		if (clk_mgr_base->clks.dtbclk_en && !new_clocks->dtbclk_en) {
-			dcn35_smu_set_dtbclk(clk_mgr, false);
+			if (clk_mgr->base.ctx->dc->config.allow_0_dtb_clk)
+				dcn35_smu_set_dtbclk(clk_mgr, false);
 			clk_mgr_base->clks.dtbclk_en = new_clocks->dtbclk_en;
 		}
 		/* check that we're not already in lower */
@@ -889,35 +890,6 @@ static void dcn35_set_low_power_state(struct clk_mgr *clk_mgr_base)
 	}
 }
 
-static void dcn35_set_ips_idle_state(struct clk_mgr *clk_mgr_base, bool allow_idle)
-{
-	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
-	struct dc *dc = clk_mgr_base->ctx->dc;
-	uint32_t val = dcn35_smu_read_ips_scratch(clk_mgr);
-
-	if (dc->config.disable_ips == DMUB_IPS_ENABLE ||
-		dc->config.disable_ips == DMUB_IPS_DISABLE_DYNAMIC) {
-		val = val & ~DMUB_IPS1_ALLOW_MASK;
-		val = val & ~DMUB_IPS2_ALLOW_MASK;
-	} else if (dc->config.disable_ips == DMUB_IPS_DISABLE_IPS1) {
-		val |= DMUB_IPS1_ALLOW_MASK;
-		val |= DMUB_IPS2_ALLOW_MASK;
-	} else if (dc->config.disable_ips == DMUB_IPS_DISABLE_IPS2) {
-		val = val & ~DMUB_IPS1_ALLOW_MASK;
-		val |= DMUB_IPS2_ALLOW_MASK;
-	} else if (dc->config.disable_ips == DMUB_IPS_DISABLE_IPS2_Z10) {
-		val = val & ~DMUB_IPS1_ALLOW_MASK;
-		val = val & ~DMUB_IPS2_ALLOW_MASK;
-	}
-
-	if (!allow_idle) {
-		val |= DMUB_IPS1_ALLOW_MASK;
-		val |= DMUB_IPS2_ALLOW_MASK;
-	}
-
-	dcn35_smu_write_ips_scratch(clk_mgr, val);
-}
-
 static void dcn35_exit_low_power_state(struct clk_mgr *clk_mgr_base)
 {
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
@@ -937,13 +909,6 @@ static bool dcn35_is_ips_supported(struct clk_mgr *clk_mgr_base)
 	return ips_supported;
 }
 
-static uint32_t dcn35_get_ips_idle_state(struct clk_mgr *clk_mgr_base)
-{
-	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
-
-	return dcn35_smu_read_ips_scratch(clk_mgr);
-}
-
 static void dcn35_init_clocks_fpga(struct clk_mgr *clk_mgr)
 {
 	init_clk_states(clk_mgr);
@@ -1031,8 +996,6 @@ static struct clk_mgr_funcs dcn35_funcs = {
 	.set_low_power_state = dcn35_set_low_power_state,
 	.exit_low_power_state = dcn35_exit_low_power_state,
 	.is_ips_supported = dcn35_is_ips_supported,
-	.set_idle_state = dcn35_set_ips_idle_state,
-	.get_idle_state = dcn35_get_ips_idle_state
 };
 
 struct clk_mgr_funcs dcn35_fpga_funcs = {
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c
index 9e588c56c570..1399b41dfd1c 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c
@@ -487,24 +487,3 @@ int dcn35_smu_get_ips_supported(struct clk_mgr_internal *clk_mgr)
 	//smu_print("%s: VBIOSSMC_MSG_QueryIPS2Support return = %x\n", __func__, retv);
 	return retv;
 }
-
-void dcn35_smu_write_ips_scratch(struct clk_mgr_internal *clk_mgr, uint32_t param)
-{
-	if (!clk_mgr->smu_present)
-		return;
-
-	REG_WRITE(MP1_SMN_C2PMSG_71, param);
-	//smu_print("%s: write_ips_scratch = %x\n", __func__, param);
-}
-
-uint32_t dcn35_smu_read_ips_scratch(struct clk_mgr_internal *clk_mgr)
-{
-	uint32_t retv;
-
-	if (!clk_mgr->smu_present)
-		return 0;
-
-	retv = REG_READ(MP1_SMN_C2PMSG_71);
-	//smu_print("%s: dcn35_smu_read_ips_scratch = %x\n",  __func__, retv);
-	return retv;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.h
index 2b8e6959a03d..06cd3cc6d36e 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.h
@@ -198,6 +198,4 @@ int dcn35_smu_exit_low_power_state(struct clk_mgr_internal *clk_mgr);
 int dcn35_smu_get_ips_supported(struct clk_mgr_internal *clk_mgr);
 int dcn35_smu_get_dtbclk(struct clk_mgr_internal *clk_mgr);
 int dcn35_smu_get_dprefclk(struct clk_mgr_internal *clk_mgr);
-void dcn35_smu_write_ips_scratch(struct clk_mgr_internal *clk_mgr, uint32_t param);
-uint32_t dcn35_smu_read_ips_scratch(struct clk_mgr_internal *clk_mgr);
 #endif /* DAL_DC_35_SMU_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 03b554e912a2..236876d95185 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -36,6 +36,7 @@
 #include "resource.h"
 #include "dc_state.h"
 #include "dc_state_priv.h"
+#include "dc_plane_priv.h"
 
 #include "gpio_service_interface.h"
 #include "clk_mgr.h"
@@ -212,7 +213,8 @@ static bool create_links(
 		connectors_num,
 		num_virtual_links);
 
-	for (i = 0; i < connectors_num; i++) {
+	// condition loop on link_count to allow skipping invalid indices
+	for (i = 0; dc->link_count < connectors_num && i < MAX_LINKS; i++) {
 		struct link_init_data link_init_params = {0};
 		struct dc_link *link;
 
@@ -386,6 +388,30 @@ static void dc_perf_trace_destroy(struct dc_perf_trace **perf_trace)
 	*perf_trace = NULL;
 }
 
+static bool set_long_vtotal(struct dc *dc, struct dc_stream_state *stream, struct dc_crtc_timing_adjust *adjust)
+{
+	if (!dc || !stream || !adjust)
+		return false;
+
+	if (!dc->current_state)
+		return false;
+
+	int i;
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream == stream && pipe->stream_res.tg) {
+			if (dc->hwss.set_long_vtotal)
+				dc->hwss.set_long_vtotal(&pipe, 1, adjust->v_total_min, adjust->v_total_max);
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
 /**
  *  dc_stream_adjust_vmin_vmax - look up pipe context & update parts of DRR
  *  @dc:     dc reference
@@ -420,6 +446,15 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc,
 	stream->adjust.v_total_mid = adjust->v_total_mid;
 	stream->adjust.v_total_mid_frame_num = adjust->v_total_mid_frame_num;
 	stream->adjust.v_total_min = adjust->v_total_min;
+	stream->adjust.allow_otg_v_count_halt = adjust->allow_otg_v_count_halt;
+
+	if (dc->caps.max_v_total != 0 &&
+		(adjust->v_total_max > dc->caps.max_v_total || adjust->v_total_min > dc->caps.max_v_total)) {
+		if (adjust->allow_otg_v_count_halt)
+			return set_long_vtotal(dc, stream, adjust);
+		else
+			return false;
+	}
 
 	for (i = 0; i < MAX_PIPES; i++) {
 		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
@@ -1055,8 +1090,7 @@ static bool dc_construct(struct dc *dc,
 	 * is initialized in dc_create_resource_pool because
 	 * on creation it copies the contents of dc->dml
 	 */
-
-	dc->current_state = dc_state_create(dc);
+	dc->current_state = dc_state_create(dc, NULL);
 
 	if (!dc->current_state) {
 		dm_error("%s: failed to create validate ctx\n", __func__);
@@ -1272,7 +1306,7 @@ static void disable_vbios_mode_if_required(
 
 		if (link != NULL && link->link_enc->funcs->is_dig_enabled(link->link_enc)) {
 			unsigned int enc_inst, tg_inst = 0;
-			unsigned int pix_clk_100hz;
+			unsigned int pix_clk_100hz = 0;
 
 			enc_inst = link->link_enc->funcs->get_dig_frontend(link->link_enc);
 			if (enc_inst != ENGINE_ID_UNKNOWN) {
@@ -1759,7 +1793,7 @@ bool dc_validate_boot_timing(const struct dc *dc,
 		return false;
 
 	if (dc_is_dp_signal(link->connector_signal)) {
-		unsigned int pix_clk_100hz;
+		unsigned int pix_clk_100hz = 0;
 		uint32_t numOdmPipes = 1;
 		uint32_t id_src[4] = {0};
 
@@ -1801,6 +1835,9 @@ bool dc_validate_boot_timing(const struct dc *dc,
 		return false;
 	}
 
+	if (link->dpcd_caps.channel_coding_cap.bits.DP_128b_132b_SUPPORTED)
+		return false;
+
 	if (dc->link_srv->edp_is_ilr_optimization_required(link, crtc_timing)) {
 		DC_LOG_EVENT_LINK_TRAINING("Seamless boot disabled to optimize eDP link rate\n");
 		return false;
@@ -2085,15 +2122,14 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 	return result;
 }
 
-static bool commit_minimal_transition_state_legacy(struct dc *dc,
+static bool commit_minimal_transition_state(struct dc *dc,
 		struct dc_state *transition_base_context);
 
 /**
  * dc_commit_streams - Commit current stream state
  *
  * @dc: DC object with the commit state to be configured in the hardware
- * @streams: Array with a list of stream state
- * @stream_count: Total of streams
+ * @params: Parameters for the commit, including the streams to be committed
  *
  * Function responsible for commit streams change to the hardware.
  *
@@ -2101,9 +2137,7 @@ static bool commit_minimal_transition_state_legacy(struct dc *dc,
  * Return DC_OK if everything work as expected, otherwise, return a dc_status
  * code.
  */
-enum dc_status dc_commit_streams(struct dc *dc,
-				 struct dc_stream_state *streams[],
-				 uint8_t stream_count)
+enum dc_status dc_commit_streams(struct dc *dc, struct dc_commit_streams_params *params)
 {
 	int i, j;
 	struct dc_state *context;
@@ -2112,18 +2146,22 @@ enum dc_status dc_commit_streams(struct dc *dc,
 	struct pipe_ctx *pipe;
 	bool handle_exit_odm2to1 = false;
 
+	if (!params)
+		return DC_ERROR_UNEXPECTED;
+
 	if (dc->ctx->dce_environment == DCE_ENV_VIRTUAL_HW)
 		return res;
 
-	if (!streams_changed(dc, streams, stream_count))
+	if (!streams_changed(dc, params->streams, params->stream_count) &&
+			dc->current_state->power_source == params->power_source)
 		return res;
 
 	dc_exit_ips_for_hw_access(dc);
 
-	DC_LOG_DC("%s: %d streams\n", __func__, stream_count);
+	DC_LOG_DC("%s: %d streams\n", __func__, params->stream_count);
 
-	for (i = 0; i < stream_count; i++) {
-		struct dc_stream_state *stream = streams[i];
+	for (i = 0; i < params->stream_count; i++) {
+		struct dc_stream_state *stream = params->streams[i];
 		struct dc_stream_status *status = dc_stream_get_status(stream);
 
 		dc_stream_log(dc, stream);
@@ -2141,7 +2179,7 @@ enum dc_status dc_commit_streams(struct dc *dc,
 	 * scenario, it uses extra pipes than needed to reduce power consumption
 	 * We need to switch off this feature to make room for new streams.
 	 */
-	if (stream_count > dc->current_state->stream_count &&
+	if (params->stream_count > dc->current_state->stream_count &&
 			dc->current_state->stream_count == 1) {
 		for (i = 0; i < dc->res_pool->pipe_count; i++) {
 			pipe = &dc->current_state->res_ctx.pipe_ctx[i];
@@ -2151,13 +2189,15 @@ enum dc_status dc_commit_streams(struct dc *dc,
 	}
 
 	if (handle_exit_odm2to1)
-		res = commit_minimal_transition_state_legacy(dc, dc->current_state);
+		res = commit_minimal_transition_state(dc, dc->current_state);
 
 	context = dc_state_create_current_copy(dc);
 	if (!context)
 		goto context_alloc_fail;
 
-	res = dc_validate_with_context(dc, set, stream_count, context, false);
+	context->power_source = params->power_source;
+
+	res = dc_validate_with_context(dc, set, params->stream_count, context, false);
 	if (res != DC_OK) {
 		BREAK_TO_DEBUGGER();
 		goto fail;
@@ -2165,16 +2205,16 @@ enum dc_status dc_commit_streams(struct dc *dc,
 
 	res = dc_commit_state_no_check(dc, context);
 
-	for (i = 0; i < stream_count; i++) {
+	for (i = 0; i < params->stream_count; i++) {
 		for (j = 0; j < context->stream_count; j++) {
-			if (streams[i]->stream_id == context->streams[j]->stream_id)
-				streams[i]->out.otg_offset = context->stream_status[j].primary_otg_inst;
+			if (params->streams[i]->stream_id == context->streams[j]->stream_id)
+				params->streams[i]->out.otg_offset = context->stream_status[j].primary_otg_inst;
 
-			if (dc_is_embedded_signal(streams[i]->signal)) {
-				struct dc_stream_status *status = dc_state_get_stream_status(context, streams[i]);
+			if (dc_is_embedded_signal(params->streams[i]->signal)) {
+				struct dc_stream_status *status = dc_state_get_stream_status(context, params->streams[i]);
 
 				if (dc->hwss.is_abm_supported)
-					status->is_abm_supported = dc->hwss.is_abm_supported(dc, context, streams[i]);
+					status->is_abm_supported = dc->hwss.is_abm_supported(dc, context, params->streams[i]);
 				else
 					status->is_abm_supported = true;
 			}
@@ -2818,55 +2858,45 @@ static void copy_surface_update_to_plane(
 				srf_update->plane_info->layer_index;
 	}
 
-	if (srf_update->gamma &&
-			(surface->gamma_correction !=
-					srf_update->gamma)) {
-		memcpy(&surface->gamma_correction->entries,
+	if (srf_update->gamma) {
+		memcpy(&surface->gamma_correction.entries,
 			&srf_update->gamma->entries,
 			sizeof(struct dc_gamma_entries));
-		surface->gamma_correction->is_identity =
+		surface->gamma_correction.is_identity =
 			srf_update->gamma->is_identity;
-		surface->gamma_correction->num_entries =
+		surface->gamma_correction.num_entries =
 			srf_update->gamma->num_entries;
-		surface->gamma_correction->type =
+		surface->gamma_correction.type =
 			srf_update->gamma->type;
 	}
 
-	if (srf_update->in_transfer_func &&
-			(surface->in_transfer_func !=
-				srf_update->in_transfer_func)) {
-		surface->in_transfer_func->sdr_ref_white_level =
+	if (srf_update->in_transfer_func) {
+		surface->in_transfer_func.sdr_ref_white_level =
 			srf_update->in_transfer_func->sdr_ref_white_level;
-		surface->in_transfer_func->tf =
+		surface->in_transfer_func.tf =
 			srf_update->in_transfer_func->tf;
-		surface->in_transfer_func->type =
+		surface->in_transfer_func.type =
 			srf_update->in_transfer_func->type;
-		memcpy(&surface->in_transfer_func->tf_pts,
+		memcpy(&surface->in_transfer_func.tf_pts,
 			&srf_update->in_transfer_func->tf_pts,
 			sizeof(struct dc_transfer_func_distributed_points));
 	}
 
-	if (srf_update->func_shaper &&
-			(surface->in_shaper_func !=
-			srf_update->func_shaper))
-		memcpy(surface->in_shaper_func, srf_update->func_shaper,
-		sizeof(*surface->in_shaper_func));
+	if (srf_update->func_shaper)
+		memcpy(&surface->in_shaper_func, srf_update->func_shaper,
+		sizeof(surface->in_shaper_func));
 
-	if (srf_update->lut3d_func &&
-			(surface->lut3d_func !=
-			srf_update->lut3d_func))
-		memcpy(surface->lut3d_func, srf_update->lut3d_func,
-		sizeof(*surface->lut3d_func));
+	if (srf_update->lut3d_func)
+		memcpy(&surface->lut3d_func, srf_update->lut3d_func,
+		sizeof(surface->lut3d_func));
 
 	if (srf_update->hdr_mult.value)
 		surface->hdr_mult =
 				srf_update->hdr_mult;
 
-	if (srf_update->blend_tf &&
-			(surface->blend_tf !=
-			srf_update->blend_tf))
-		memcpy(surface->blend_tf, srf_update->blend_tf,
-		sizeof(*surface->blend_tf));
+	if (srf_update->blend_tf)
+		memcpy(&surface->blend_tf, srf_update->blend_tf,
+		sizeof(surface->blend_tf));
 
 	if (srf_update->input_csc_color_matrix)
 		surface->input_csc_color_matrix =
@@ -2897,14 +2927,13 @@ static void copy_stream_update_to_stream(struct dc *dc,
 	if (update->dst.height && update->dst.width)
 		stream->dst = update->dst;
 
-	if (update->out_transfer_func &&
-	    stream->out_transfer_func != update->out_transfer_func) {
-		stream->out_transfer_func->sdr_ref_white_level =
+	if (update->out_transfer_func) {
+		stream->out_transfer_func.sdr_ref_white_level =
 			update->out_transfer_func->sdr_ref_white_level;
-		stream->out_transfer_func->tf = update->out_transfer_func->tf;
-		stream->out_transfer_func->type =
+		stream->out_transfer_func.tf = update->out_transfer_func->tf;
+		stream->out_transfer_func.type =
 			update->out_transfer_func->type;
-		memcpy(&stream->out_transfer_func->tf_pts,
+		memcpy(&stream->out_transfer_func.tf_pts,
 		       &update->out_transfer_func->tf_pts,
 		       sizeof(struct dc_transfer_func_distributed_points));
 	}
@@ -3017,15 +3046,8 @@ static void backup_planes_and_stream_state(
 
 	for (i = 0; i < status->plane_count; i++) {
 		scratch->plane_states[i] = *status->plane_states[i];
-		scratch->gamma_correction[i] = *status->plane_states[i]->gamma_correction;
-		scratch->in_transfer_func[i] = *status->plane_states[i]->in_transfer_func;
-		scratch->lut3d_func[i] = *status->plane_states[i]->lut3d_func;
-		scratch->in_shaper_func[i] = *status->plane_states[i]->in_shaper_func;
-		scratch->blend_tf[i] = *status->plane_states[i]->blend_tf;
 	}
 	scratch->stream_state = *stream;
-	if (stream->out_transfer_func)
-		scratch->out_transfer_func = *stream->out_transfer_func;
 }
 
 static void restore_planes_and_stream_state(
@@ -3040,17 +3062,67 @@ static void restore_planes_and_stream_state(
 
 	for (i = 0; i < status->plane_count; i++) {
 		*status->plane_states[i] = scratch->plane_states[i];
-		*status->plane_states[i]->gamma_correction = scratch->gamma_correction[i];
-		*status->plane_states[i]->in_transfer_func = scratch->in_transfer_func[i];
-		*status->plane_states[i]->lut3d_func = scratch->lut3d_func[i];
-		*status->plane_states[i]->in_shaper_func = scratch->in_shaper_func[i];
-		*status->plane_states[i]->blend_tf = scratch->blend_tf[i];
 	}
 	*stream = scratch->stream_state;
-	if (stream->out_transfer_func)
-		*stream->out_transfer_func = scratch->out_transfer_func;
 }
 
+/**
+ * update_seamless_boot_flags() - Helper function for updating seamless boot flags
+ *
+ * @dc: Current DC state
+ * @context: New DC state to be programmed
+ * @surface_count: Number of surfaces that have an updated
+ * @stream: Corresponding stream to be updated in the current flip
+ *
+ * Updating seamless boot flags do not need to be part of the commit sequence. This
+ * helper function will update the seamless boot flags on each flip (if required)
+ * outside of the HW commit sequence (fast or slow).
+ *
+ * Return: void
+ */
+static void update_seamless_boot_flags(struct dc *dc,
+		struct dc_state *context,
+		int surface_count,
+		struct dc_stream_state *stream)
+{
+	if (get_seamless_boot_stream_count(context) > 0 && surface_count > 0) {
+		/* Optimize seamless boot flag keeps clocks and watermarks high until
+		 * first flip. After first flip, optimization is required to lower
+		 * bandwidth. Important to note that it is expected UEFI will
+		 * only light up a single display on POST, therefore we only expect
+		 * one stream with seamless boot flag set.
+		 */
+		if (stream->apply_seamless_boot_optimization) {
+			stream->apply_seamless_boot_optimization = false;
+
+			if (get_seamless_boot_stream_count(context) == 0)
+				dc->optimized_required = true;
+		}
+	}
+}
+
+/**
+ * update_planes_and_stream_state() - The function takes planes and stream
+ * updates as inputs and determines the appropriate update type. If update type
+ * is FULL, the function allocates a new context, populates and validates it.
+ * Otherwise, it updates current dc context. The function will return both
+ * new_context and new_update_type back to the caller. The function also backs
+ * up both current and new contexts into corresponding dc state scratch memory.
+ * TODO: The function does too many things, and even conditionally allocates dc
+ * context memory implicitly. We should consider to break it down.
+ *
+ * @dc: Current DC state
+ * @srf_updates: an array of surface updates
+ * @surface_count: surface update count
+ * @stream: Corresponding stream to be updated
+ * @stream_update: stream update
+ * @new_update_type: [out] determined update type by the function
+ * @new_context: [out] new context allocated and validated if update type is
+ * FULL, reference to current context if update type is less than FULL.
+ *
+ * Return: true if a valid update is populated into new_context, false
+ * otherwise.
+ */
 static bool update_planes_and_stream_state(struct dc *dc,
 		struct dc_surface_update *srf_updates, int surface_count,
 		struct dc_stream_state *stream,
@@ -3074,9 +3146,10 @@ static bool update_planes_and_stream_state(struct dc *dc,
 	}
 
 	context = dc->current_state;
-	backup_planes_and_stream_state(&dc->current_state->scratch, stream);
 	update_type = dc_check_update_surfaces_for_stream(
 			dc, srf_updates, surface_count, stream_update, stream_status);
+	if (update_type == UPDATE_TYPE_FULL)
+		backup_planes_and_stream_state(&dc->scratch.current_state, stream);
 
 	/* update current stream with the new updates */
 	copy_stream_update_to_stream(dc, context, stream, stream_update);
@@ -3145,7 +3218,10 @@ static bool update_planes_and_stream_state(struct dc *dc,
 	for (i = 0; i < surface_count; i++) {
 		struct dc_plane_state *surface = srf_updates[i].surface;
 
-		if (update_type >= UPDATE_TYPE_MED) {
+		if (update_type != UPDATE_TYPE_MED)
+			continue;
+		if (surface->update_flags.bits.clip_size_change ||
+				surface->update_flags.bits.position_change) {
 			for (j = 0; j < dc->res_pool->pipe_count; j++) {
 				struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
 
@@ -3162,19 +3238,13 @@ static bool update_planes_and_stream_state(struct dc *dc,
 			BREAK_TO_DEBUGGER();
 			goto fail;
 		}
-
-		for (i = 0; i < context->stream_count; i++) {
-			struct pipe_ctx *otg_master = resource_get_otg_master_for_stream(&context->res_ctx,
-					context->streams[i]);
-
-			if (otg_master && otg_master->stream->test_pattern.type != DP_TEST_PATTERN_VIDEO_MODE)
-				resource_build_test_pattern_params(&context->res_ctx, otg_master);
-		}
 	}
+	update_seamless_boot_flags(dc, context, surface_count, stream);
 
 	*new_context = context;
 	*new_update_type = update_type;
-	backup_planes_and_stream_state(&context->scratch, stream);
+	if (update_type == UPDATE_TYPE_FULL)
+		backup_planes_and_stream_state(&dc->scratch.new_state, stream);
 
 	return true;
 
@@ -3263,12 +3333,26 @@ static void commit_planes_do_stream_update(struct dc *dc,
  			}
 
 			if (stream_update->pending_test_pattern) {
-				dc_link_dp_set_test_pattern(stream->link,
+				/*
+				 * test pattern params depends on ODM topology
+				 * changes that we could be applying to front
+				 * end. Since at the current stage front end
+				 * changes are not yet applied. We can only
+				 * apply test pattern in hw based on current
+				 * state and populate the final test pattern
+				 * params in new state. If current and new test
+				 * pattern params are different as result of
+				 * different ODM topology being used, it will be
+				 * detected and handle during front end
+				 * programming update.
+				 */
+				dc->link_srv->dp_set_test_pattern(stream->link,
 					stream->test_pattern.type,
 					stream->test_pattern.color_space,
 					stream->test_pattern.p_link_settings,
 					stream->test_pattern.p_custom_pattern,
 					stream->test_pattern.cust_pattern_size);
+				resource_build_test_pattern_params(&context->res_ctx, pipe_ctx);
 			}
 
 			if (stream_update->dpms_off) {
@@ -3365,6 +3449,7 @@ void dc_dmub_update_dirty_rect(struct dc *dc,
 		if (srf_updates[i].surface->flip_immediate)
 			continue;
 
+		update_dirty_rect->cmd_version = DMUB_CMD_PSR_CONTROL_VERSION_1;
 		update_dirty_rect->dirty_rect_count = flip_addr->dirty_rect_count;
 		memcpy(update_dirty_rect->src_dirty_rects, flip_addr->dirty_rects,
 				sizeof(flip_addr->dirty_rects));
@@ -3481,6 +3566,7 @@ static void commit_planes_for_stream_fast(struct dc *dc,
 	int i, j;
 	struct pipe_ctx *top_pipe_to_program = NULL;
 	struct dc_stream_status *stream_status = NULL;
+
 	dc_exit_ips_for_hw_access(dc);
 
 	dc_z10_restore(dc);
@@ -3538,7 +3624,8 @@ static void commit_planes_for_stream_fast(struct dc *dc,
 			context->block_sequence,
 			&(context->block_sequence_steps),
 			top_pipe_to_program,
-			stream_status);
+			stream_status,
+			context);
 	hwss_execute_sequence(dc,
 			context->block_sequence,
 			context->block_sequence_steps);
@@ -4067,24 +4154,14 @@ struct pipe_split_policy_backup {
 	bool dynamic_odm_policy;
 	bool subvp_policy;
 	enum pipe_split_policy mpc_policy;
+	char force_odm[MAX_PIPES];
 };
 
-static void release_minimal_transition_state(struct dc *dc,
-		struct dc_state *context, struct pipe_split_policy_backup *policy)
-{
-	dc_state_release(context);
-	/* restore previous pipe split and odm policy */
-	if (!dc->config.is_vmin_only_asic)
-		dc->debug.pipe_split_policy = policy->mpc_policy;
-	dc->debug.enable_single_display_2to1_odm_policy = policy->dynamic_odm_policy;
-	dc->debug.force_disable_subvp = policy->subvp_policy;
-}
-
-static struct dc_state *create_minimal_transition_state(struct dc *dc,
-		struct dc_state *base_context, struct pipe_split_policy_backup *policy)
+static void backup_and_set_minimal_pipe_split_policy(struct dc *dc,
+		struct dc_state *context,
+		struct pipe_split_policy_backup *policy)
 {
-	struct dc_state *minimal_transition_context = NULL;
-	unsigned int i, j;
+	int i;
 
 	if (!dc->config.is_vmin_only_asic) {
 		policy->mpc_policy = dc->debug.pipe_split_policy;
@@ -4094,97 +4171,257 @@ static struct dc_state *create_minimal_transition_state(struct dc *dc,
 	dc->debug.enable_single_display_2to1_odm_policy = false;
 	policy->subvp_policy = dc->debug.force_disable_subvp;
 	dc->debug.force_disable_subvp = true;
+	for (i = 0; i < context->stream_count; i++) {
+		policy->force_odm[i] = context->streams[i]->debug.force_odm_combine_segments;
+		context->streams[i]->debug.force_odm_combine_segments = 0;
+	}
+}
+
+static void restore_minimal_pipe_split_policy(struct dc *dc,
+		struct dc_state *context,
+		struct pipe_split_policy_backup *policy)
+{
+	uint8_t i;
+
+	if (!dc->config.is_vmin_only_asic)
+		dc->debug.pipe_split_policy = policy->mpc_policy;
+	dc->debug.enable_single_display_2to1_odm_policy =
+			policy->dynamic_odm_policy;
+	dc->debug.force_disable_subvp = policy->subvp_policy;
+	for (i = 0; i < context->stream_count; i++)
+		context->streams[i]->debug.force_odm_combine_segments = policy->force_odm[i];
+}
+
+static void release_minimal_transition_state(struct dc *dc,
+		struct dc_state *minimal_transition_context,
+		struct dc_state *base_context,
+		struct pipe_split_policy_backup *policy)
+{
+	restore_minimal_pipe_split_policy(dc, base_context, policy);
+	dc_state_release(minimal_transition_context);
+}
+
+static void force_vsync_flip_in_minimal_transition_context(struct dc_state *context)
+{
+	uint8_t i;
+	int j;
+	struct dc_stream_status *stream_status;
+
+	for (i = 0; i < context->stream_count; i++) {
+		stream_status = &context->stream_status[i];
+
+		for (j = 0; j < stream_status->plane_count; j++)
+			stream_status->plane_states[j]->flip_immediate = false;
+	}
+}
+
+static struct dc_state *create_minimal_transition_state(struct dc *dc,
+		struct dc_state *base_context, struct pipe_split_policy_backup *policy)
+{
+	struct dc_state *minimal_transition_context = NULL;
 
 	minimal_transition_context = dc_state_create_copy(base_context);
 	if (!minimal_transition_context)
 		return NULL;
 
+	backup_and_set_minimal_pipe_split_policy(dc, base_context, policy);
 	/* commit minimal state */
 	if (dc->res_pool->funcs->validate_bandwidth(dc, minimal_transition_context, false)) {
-		for (i = 0; i < minimal_transition_context->stream_count; i++) {
-			struct dc_stream_status *stream_status = &minimal_transition_context->stream_status[i];
-
-			for (j = 0; j < stream_status->plane_count; j++) {
-				struct dc_plane_state *plane_state = stream_status->plane_states[j];
-
-				/* force vsync flip when reconfiguring pipes to prevent underflow
-				 * and corruption
-				 */
-				plane_state->flip_immediate = false;
-			}
-		}
+		/* prevent underflow and corruption when reconfiguring pipes */
+		force_vsync_flip_in_minimal_transition_context(minimal_transition_context);
 	} else {
-		/* this should never happen */
-		release_minimal_transition_state(dc, minimal_transition_context, policy);
+		/*
+		 * This should never happen, minimal transition state should
+		 * always be validated first before adding pipe split features.
+		 */
+		release_minimal_transition_state(dc, minimal_transition_context, base_context, policy);
 		BREAK_TO_DEBUGGER();
 		minimal_transition_context = NULL;
 	}
 	return minimal_transition_context;
 }
 
+static bool is_pipe_topology_transition_seamless_with_intermediate_step(
+		struct dc *dc,
+		struct dc_state *initial_state,
+		struct dc_state *intermediate_state,
+		struct dc_state *final_state)
+{
+	return dc->hwss.is_pipe_topology_transition_seamless(dc, initial_state,
+			intermediate_state) &&
+			dc->hwss.is_pipe_topology_transition_seamless(dc,
+					intermediate_state, final_state);
+}
+
+static void swap_and_release_current_context(struct dc *dc,
+		struct dc_state *new_context, struct dc_stream_state *stream)
+{
+
+	int i;
+	struct dc_state *old = dc->current_state;
+	struct pipe_ctx *pipe_ctx;
+
+	/* Since memory free requires elevated IRQ, an interrupt
+	 * request is generated by mem free. If this happens
+	 * between freeing and reassigning the context, our vsync
+	 * interrupt will call into dc and cause a memory
+	 * corruption. Hence, we first reassign the context,
+	 * then free the old context.
+	 */
+	dc->current_state = new_context;
+	dc_state_release(old);
+
+	// clear any forced full updates
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		pipe_ctx = &new_context->res_ctx.pipe_ctx[i];
+
+		if (pipe_ctx->plane_state && pipe_ctx->stream == stream)
+			pipe_ctx->plane_state->force_full_update = false;
+	}
+}
+
+static int initialize_empty_surface_updates(
+		struct dc_stream_state *stream,
+		struct dc_surface_update *srf_updates)
+{
+	struct dc_stream_status *status = dc_stream_get_status(stream);
+	int i;
+
+	if (!status)
+		return 0;
+
+	for (i = 0; i < status->plane_count; i++)
+		srf_updates[i].surface = status->plane_states[i];
+
+	return status->plane_count;
+}
+
+static bool commit_minimal_transition_based_on_new_context(struct dc *dc,
+		struct dc_state *new_context,
+		struct dc_stream_state *stream,
+		struct dc_surface_update *srf_updates,
+		int surface_count)
+{
+	bool success = false;
+	struct pipe_split_policy_backup policy;
+	struct dc_state *intermediate_context =
+			create_minimal_transition_state(dc, new_context,
+					&policy);
+
+	if (intermediate_context) {
+		if (is_pipe_topology_transition_seamless_with_intermediate_step(
+				dc,
+				dc->current_state,
+				intermediate_context,
+				new_context)) {
+			DC_LOG_DC("commit minimal transition state: base = new state\n");
+			commit_planes_for_stream(dc, srf_updates,
+					surface_count, stream, NULL,
+					UPDATE_TYPE_FULL, intermediate_context);
+			swap_and_release_current_context(
+					dc, intermediate_context, stream);
+			dc_state_retain(dc->current_state);
+			success = true;
+		}
+		release_minimal_transition_state(
+				dc, intermediate_context, new_context, &policy);
+	}
+	return success;
+}
+
+static bool commit_minimal_transition_based_on_current_context(struct dc *dc,
+		struct dc_state *new_context, struct dc_stream_state *stream)
+{
+	bool success = false;
+	struct pipe_split_policy_backup policy;
+	struct dc_state *intermediate_context;
+	struct dc_state *old_current_state = dc->current_state;
+	struct dc_surface_update srf_updates[MAX_SURFACE_NUM] = {0};
+	int surface_count;
+
+	/*
+	 * Both current and new contexts share the same stream and plane state
+	 * pointers. When new context is validated, stream and planes get
+	 * populated with new updates such as new plane addresses. This makes
+	 * the current context no longer valid because stream and planes are
+	 * modified from the original. We backup current stream and plane states
+	 * into scratch space whenever we are populating new context. So we can
+	 * restore the original values back by calling the restore function now.
+	 * This restores back the original stream and plane states associated
+	 * with the current state.
+	 */
+	restore_planes_and_stream_state(&dc->scratch.current_state, stream);
+	dc_state_retain(old_current_state);
+	intermediate_context = create_minimal_transition_state(dc,
+			old_current_state, &policy);
+
+	if (intermediate_context) {
+		if (is_pipe_topology_transition_seamless_with_intermediate_step(
+				dc,
+				dc->current_state,
+				intermediate_context,
+				new_context)) {
+			DC_LOG_DC("commit minimal transition state: base = current state\n");
+			surface_count = initialize_empty_surface_updates(
+					stream, srf_updates);
+			commit_planes_for_stream(dc, srf_updates,
+					surface_count, stream, NULL,
+					UPDATE_TYPE_FULL, intermediate_context);
+			swap_and_release_current_context(
+					dc, intermediate_context, stream);
+			dc_state_retain(dc->current_state);
+			success = true;
+		}
+		release_minimal_transition_state(dc, intermediate_context,
+				old_current_state, &policy);
+	}
+	dc_state_release(old_current_state);
+	/*
+	 * Restore stream and plane states back to the values associated with
+	 * new context.
+	 */
+	restore_planes_and_stream_state(&dc->scratch.new_state, stream);
+	return success;
+}
 
 /**
- * commit_minimal_transition_state - Commit a minimal state based on current or new context
+ * commit_minimal_transition_state_in_dc_update - Commit a minimal state based
+ * on current or new context
  *
  * @dc: DC structure, used to get the current state
- * @context: New context
+ * @new_context: New context
  * @stream: Stream getting the update for the flip
+ * @srf_updates: Surface updates
+ * @surface_count: Number of surfaces
  *
- * The function takes in current state and new state and determine a minimal transition state
- * as the intermediate step which could make the transition between current and new states
- * seamless. If found, it will commit the minimal transition state and update current state to
- * this minimal transition state and return true, if not, it will return false.
+ * The function takes in current state and new state and determine a minimal
+ * transition state as the intermediate step which could make the transition
+ * between current and new states seamless. If found, it will commit the minimal
+ * transition state and update current state to this minimal transition state
+ * and return true, if not, it will return false.
  *
  * Return:
  * Return True if the minimal transition succeeded, false otherwise
  */
-static bool commit_minimal_transition_state(struct dc *dc,
-		struct dc_state *context,
-		struct dc_stream_state *stream)
-{
-	bool success = false;
-	struct dc_state *minimal_transition_context;
-	struct pipe_split_policy_backup policy;
-
-	/* commit based on new context */
-	minimal_transition_context = create_minimal_transition_state(dc,
-			context, &policy);
-	if (minimal_transition_context) {
-		if (dc->hwss.is_pipe_topology_transition_seamless(
-					dc, dc->current_state, minimal_transition_context) &&
-			dc->hwss.is_pipe_topology_transition_seamless(
-					dc, minimal_transition_context, context)) {
-			DC_LOG_DC("%s base = new state\n", __func__);
-
-			success = dc_commit_state_no_check(dc, minimal_transition_context) == DC_OK;
-		}
-		release_minimal_transition_state(dc, minimal_transition_context, &policy);
-	}
-
-	if (!success) {
-		/* commit based on current context */
-		restore_planes_and_stream_state(&dc->current_state->scratch, stream);
-		minimal_transition_context = create_minimal_transition_state(dc,
-				dc->current_state, &policy);
-		if (minimal_transition_context) {
-			if (dc->hwss.is_pipe_topology_transition_seamless(
-					dc, dc->current_state, minimal_transition_context) &&
-				dc->hwss.is_pipe_topology_transition_seamless(
-						dc, minimal_transition_context, context)) {
-				DC_LOG_DC("%s base = current state\n", __func__);
-				success = dc_commit_state_no_check(dc, minimal_transition_context) == DC_OK;
-			}
-			release_minimal_transition_state(dc, minimal_transition_context, &policy);
-		}
-		restore_planes_and_stream_state(&context->scratch, stream);
-	}
-
-	ASSERT(success);
+static bool commit_minimal_transition_state_in_dc_update(struct dc *dc,
+		struct dc_state *new_context,
+		struct dc_stream_state *stream,
+		struct dc_surface_update *srf_updates,
+		int surface_count)
+{
+	bool success = commit_minimal_transition_based_on_new_context(
+				dc, new_context, stream, srf_updates,
+				surface_count);
+	if (!success)
+		success = commit_minimal_transition_based_on_current_context(dc,
+				new_context, stream);
+	if (!success)
+		DC_LOG_ERROR("Fail to commit a seamless minimal transition state between current and new states.\nThis pipe topology update is non-seamless!\n");
 	return success;
 }
 
 /**
- * commit_minimal_transition_state_legacy - Create a transition pipe split state
+ * commit_minimal_transition_state - Create a transition pipe split state
  *
  * @dc: Used to get the current state status
  * @transition_base_context: New transition state
@@ -4201,7 +4438,7 @@ static bool commit_minimal_transition_state(struct dc *dc,
  * Return:
  * Return false if something is wrong in the transition state.
  */
-static bool commit_minimal_transition_state_legacy(struct dc *dc,
+static bool commit_minimal_transition_state(struct dc *dc,
 		struct dc_state *transition_base_context)
 {
 	struct dc_state *transition_context;
@@ -4262,12 +4499,14 @@ static bool commit_minimal_transition_state_legacy(struct dc *dc,
 			dc->debug.pipe_split_policy != MPC_SPLIT_AVOID ? "MPC in Use" :
 			"Unknown");
 
+	dc_state_retain(transition_base_context);
 	transition_context = create_minimal_transition_state(dc,
 			transition_base_context, &policy);
 	if (transition_context) {
 		ret = dc_commit_state_no_check(dc, transition_context);
-		release_minimal_transition_state(dc, transition_context, &policy);
+		release_minimal_transition_state(dc, transition_context, transition_base_context, &policy);
 	}
+	dc_state_release(transition_base_context);
 
 	if (ret != DC_OK) {
 		/* this should never happen */
@@ -4285,41 +4524,6 @@ static bool commit_minimal_transition_state_legacy(struct dc *dc,
 	return true;
 }
 
-/**
- * update_seamless_boot_flags() - Helper function for updating seamless boot flags
- *
- * @dc: Current DC state
- * @context: New DC state to be programmed
- * @surface_count: Number of surfaces that have an updated
- * @stream: Corresponding stream to be updated in the current flip
- *
- * Updating seamless boot flags do not need to be part of the commit sequence. This
- * helper function will update the seamless boot flags on each flip (if required)
- * outside of the HW commit sequence (fast or slow).
- *
- * Return: void
- */
-static void update_seamless_boot_flags(struct dc *dc,
-		struct dc_state *context,
-		int surface_count,
-		struct dc_stream_state *stream)
-{
-	if (get_seamless_boot_stream_count(context) > 0 && surface_count > 0) {
-		/* Optimize seamless boot flag keeps clocks and watermarks high until
-		 * first flip. After first flip, optimization is required to lower
-		 * bandwidth. Important to note that it is expected UEFI will
-		 * only light up a single display on POST, therefore we only expect
-		 * one stream with seamless boot flag set.
-		 */
-		if (stream->apply_seamless_boot_optimization) {
-			stream->apply_seamless_boot_optimization = false;
-
-			if (get_seamless_boot_stream_count(context) == 0)
-				dc->optimized_required = true;
-		}
-	}
-}
-
 static void populate_fast_updates(struct dc_fast_update *fast_update,
 		struct dc_surface_update *srf_updates,
 		int surface_count,
@@ -4439,123 +4643,9 @@ static bool fast_update_only(struct dc *dc,
 			&& !full_update_required(dc, srf_updates, surface_count, stream_update, stream);
 }
 
-bool dc_update_planes_and_stream(struct dc *dc,
+static bool update_planes_and_stream_v1(struct dc *dc,
 		struct dc_surface_update *srf_updates, int surface_count,
 		struct dc_stream_state *stream,
-		struct dc_stream_update *stream_update)
-{
-	struct dc_state *context;
-	enum surface_update_type update_type;
-	int i;
-	struct dc_fast_update fast_update[MAX_SURFACES] = {0};
-
-	/* In cases where MPO and split or ODM are used transitions can
-	 * cause underflow. Apply stream configuration with minimal pipe
-	 * split first to avoid unsupported transitions for active pipes.
-	 */
-	bool force_minimal_pipe_splitting = 0;
-	bool is_plane_addition = 0;
-	bool is_fast_update_only;
-
-	dc_exit_ips_for_hw_access(dc);
-
-	populate_fast_updates(fast_update, srf_updates, surface_count, stream_update);
-	is_fast_update_only = fast_update_only(dc, fast_update, srf_updates,
-			surface_count, stream_update, stream);
-	force_minimal_pipe_splitting = could_mpcc_tree_change_for_active_pipes(
-			dc,
-			stream,
-			srf_updates,
-			surface_count,
-			&is_plane_addition);
-
-	/* on plane addition, minimal state is the current one */
-	if (force_minimal_pipe_splitting && is_plane_addition &&
-		!commit_minimal_transition_state_legacy(dc, dc->current_state))
-				return false;
-
-	if (!update_planes_and_stream_state(
-			dc,
-			srf_updates,
-			surface_count,
-			stream,
-			stream_update,
-			&update_type,
-			&context))
-		return false;
-
-	/* on plane removal, minimal state is the new one */
-	if (force_minimal_pipe_splitting && !is_plane_addition) {
-		if (!commit_minimal_transition_state_legacy(dc, context)) {
-			dc_state_release(context);
-			return false;
-		}
-		update_type = UPDATE_TYPE_FULL;
-	}
-
-	if (dc->hwss.is_pipe_topology_transition_seamless &&
-			!dc->hwss.is_pipe_topology_transition_seamless(
-					dc, dc->current_state, context)) {
-		commit_minimal_transition_state(dc,
-				context, stream);
-	}
-	update_seamless_boot_flags(dc, context, surface_count, stream);
-	if (is_fast_update_only && !dc->debug.enable_legacy_fast_update) {
-		commit_planes_for_stream_fast(dc,
-				srf_updates,
-				surface_count,
-				stream,
-				stream_update,
-				update_type,
-				context);
-	} else {
-		if (!stream_update &&
-				dc->hwss.is_pipe_topology_transition_seamless &&
-				!dc->hwss.is_pipe_topology_transition_seamless(
-						dc, dc->current_state, context)) {
-			DC_LOG_ERROR("performing non-seamless pipe topology transition with surface only update!\n");
-			BREAK_TO_DEBUGGER();
-		}
-		commit_planes_for_stream(
-				dc,
-				srf_updates,
-				surface_count,
-				stream,
-				stream_update,
-				update_type,
-				context);
-	}
-
-	if (dc->current_state != context) {
-
-		/* Since memory free requires elevated IRQL, an interrupt
-		 * request is generated by mem free. If this happens
-		 * between freeing and reassigning the context, our vsync
-		 * interrupt will call into dc and cause a memory
-		 * corruption BSOD. Hence, we first reassign the context,
-		 * then free the old context.
-		 */
-
-		struct dc_state *old = dc->current_state;
-
-		dc->current_state = context;
-		dc_state_release(old);
-
-		// clear any forced full updates
-		for (i = 0; i < dc->res_pool->pipe_count; i++) {
-			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
-
-			if (pipe_ctx->plane_state && pipe_ctx->stream == stream)
-				pipe_ctx->plane_state->force_full_update = false;
-		}
-	}
-	return true;
-}
-
-void dc_commit_updates_for_stream(struct dc *dc,
-		struct dc_surface_update *srf_updates,
-		int surface_count,
-		struct dc_stream_state *stream,
 		struct dc_stream_update *stream_update,
 		struct dc_state *state)
 {
@@ -4575,35 +4665,13 @@ void dc_commit_updates_for_stream(struct dc *dc,
 	update_type = dc_check_update_surfaces_for_stream(
 				dc, srf_updates, surface_count, stream_update, stream_status);
 
-	/* TODO: Since change commit sequence can have a huge impact,
-	 * we decided to only enable it for DCN3x. However, as soon as
-	 * we get more confident about this change we'll need to enable
-	 * the new sequence for all ASICs.
-	 */
-	if (dc->ctx->dce_version >= DCN_VERSION_3_2) {
-		/*
-		 * Previous frame finished and HW is ready for optimization.
-		 */
-		if (update_type == UPDATE_TYPE_FAST)
-			dc_post_update_surfaces_to_stream(dc);
-
-		dc_update_planes_and_stream(dc, srf_updates,
-					    surface_count, stream,
-					    stream_update);
-		return;
-	}
-
-	if (update_type >= update_surface_trace_level)
-		update_surface_trace(dc, srf_updates, surface_count);
-
-
 	if (update_type >= UPDATE_TYPE_FULL) {
 
 		/* initialize scratch memory for building context */
 		context = dc_state_create_copy(state);
 		if (context == NULL) {
 			DC_ERROR("Failed to allocate new validate context!\n");
-			return;
+			return false;
 		}
 
 		for (i = 0; i < dc->res_pool->pipe_count; i++) {
@@ -4620,7 +4688,6 @@ void dc_commit_updates_for_stream(struct dc *dc,
 		dc_post_update_surfaces_to_stream(dc);
 	}
 
-
 	for (i = 0; i < surface_count; i++) {
 		struct dc_plane_state *surface = srf_updates[i].surface;
 
@@ -4645,13 +4712,12 @@ void dc_commit_updates_for_stream(struct dc *dc,
 		if (!dc->res_pool->funcs->validate_bandwidth(dc, context, false)) {
 			DC_ERROR("Mode validation failed for stream update!\n");
 			dc_state_release(context);
-			return;
+			return false;
 		}
 	}
 
 	TRACE_DC_PIPE_STATE(pipe_ctx, i, MAX_PIPES);
 
-	update_seamless_boot_flags(dc, context, surface_count, stream);
 	if (fast_update_only(dc, fast_update, srf_updates, surface_count, stream_update, stream) &&
 			!dc->debug.enable_legacy_fast_update) {
 		commit_planes_for_stream_fast(dc,
@@ -4692,9 +4758,252 @@ void dc_commit_updates_for_stream(struct dc *dc,
 		dc_post_update_surfaces_to_stream(dc);
 		TRACE_DCE_CLOCK_STATE(&context->bw_ctx.bw.dce);
 	}
+	return true;
+}
+
+static bool update_planes_and_stream_v2(struct dc *dc,
+		struct dc_surface_update *srf_updates, int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update)
+{
+	struct dc_state *context;
+	enum surface_update_type update_type;
+	struct dc_fast_update fast_update[MAX_SURFACES] = {0};
+
+	/* In cases where MPO and split or ODM are used transitions can
+	 * cause underflow. Apply stream configuration with minimal pipe
+	 * split first to avoid unsupported transitions for active pipes.
+	 */
+	bool force_minimal_pipe_splitting = 0;
+	bool is_plane_addition = 0;
+	bool is_fast_update_only;
+
+	populate_fast_updates(fast_update, srf_updates, surface_count, stream_update);
+	is_fast_update_only = fast_update_only(dc, fast_update, srf_updates,
+			surface_count, stream_update, stream);
+	force_minimal_pipe_splitting = could_mpcc_tree_change_for_active_pipes(
+			dc,
+			stream,
+			srf_updates,
+			surface_count,
+			&is_plane_addition);
+
+	/* on plane addition, minimal state is the current one */
+	if (force_minimal_pipe_splitting && is_plane_addition &&
+		!commit_minimal_transition_state(dc, dc->current_state))
+		return false;
+
+	if (!update_planes_and_stream_state(
+			dc,
+			srf_updates,
+			surface_count,
+			stream,
+			stream_update,
+			&update_type,
+			&context))
+		return false;
+
+	/* on plane removal, minimal state is the new one */
+	if (force_minimal_pipe_splitting && !is_plane_addition) {
+		if (!commit_minimal_transition_state(dc, context)) {
+			dc_state_release(context);
+			return false;
+		}
+		update_type = UPDATE_TYPE_FULL;
+	}
 
-	return;
+	if (dc->hwss.is_pipe_topology_transition_seamless &&
+			!dc->hwss.is_pipe_topology_transition_seamless(
+					dc, dc->current_state, context))
+		commit_minimal_transition_state_in_dc_update(dc, context, stream,
+				srf_updates, surface_count);
 
+	if (is_fast_update_only && !dc->debug.enable_legacy_fast_update) {
+		commit_planes_for_stream_fast(dc,
+				srf_updates,
+				surface_count,
+				stream,
+				stream_update,
+				update_type,
+				context);
+	} else {
+		if (!stream_update &&
+				dc->hwss.is_pipe_topology_transition_seamless &&
+				!dc->hwss.is_pipe_topology_transition_seamless(
+						dc, dc->current_state, context)) {
+			DC_LOG_ERROR("performing non-seamless pipe topology transition with surface only update!\n");
+			BREAK_TO_DEBUGGER();
+		}
+		commit_planes_for_stream(
+				dc,
+				srf_updates,
+				surface_count,
+				stream,
+				stream_update,
+				update_type,
+				context);
+	}
+	if (dc->current_state != context)
+		swap_and_release_current_context(dc, context, stream);
+	return true;
+}
+
+static void commit_planes_and_stream_update_on_current_context(struct dc *dc,
+		struct dc_surface_update *srf_updates, int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		enum surface_update_type update_type)
+{
+	struct dc_fast_update fast_update[MAX_SURFACES] = {0};
+
+	ASSERT(update_type < UPDATE_TYPE_FULL);
+	populate_fast_updates(fast_update, srf_updates, surface_count,
+			stream_update);
+	if (fast_update_only(dc, fast_update, srf_updates, surface_count,
+			stream_update, stream) &&
+			!dc->debug.enable_legacy_fast_update)
+		commit_planes_for_stream_fast(dc,
+				srf_updates,
+				surface_count,
+				stream,
+				stream_update,
+				update_type,
+				dc->current_state);
+	else
+		commit_planes_for_stream(
+				dc,
+				srf_updates,
+				surface_count,
+				stream,
+				stream_update,
+				update_type,
+				dc->current_state);
+}
+
+static void commit_planes_and_stream_update_with_new_context(struct dc *dc,
+		struct dc_surface_update *srf_updates, int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		enum surface_update_type update_type,
+		struct dc_state *new_context)
+{
+	ASSERT(update_type >= UPDATE_TYPE_FULL);
+	if (!dc->hwss.is_pipe_topology_transition_seamless(dc,
+			dc->current_state, new_context))
+		/*
+		 * It is required by the feature design that all pipe topologies
+		 * using extra free pipes for power saving purposes such as
+		 * dynamic ODM or SubVp shall only be enabled when it can be
+		 * transitioned seamlessly to AND from its minimal transition
+		 * state. A minimal transition state is defined as the same dc
+		 * state but with all power saving features disabled. So it uses
+		 * the minimum pipe topology. When we can't seamlessly
+		 * transition from state A to state B, we will insert the
+		 * minimal transition state A' or B' in between so seamless
+		 * transition between A and B can be made possible.
+		 */
+		commit_minimal_transition_state_in_dc_update(dc, new_context,
+				stream, srf_updates, surface_count);
+
+	commit_planes_for_stream(
+			dc,
+			srf_updates,
+			surface_count,
+			stream,
+			stream_update,
+			update_type,
+			new_context);
+}
+
+static bool update_planes_and_stream_v3(struct dc *dc,
+		struct dc_surface_update *srf_updates, int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update)
+{
+	struct dc_state *new_context;
+	enum surface_update_type update_type;
+
+	/*
+	 * When this function returns true and new_context is not equal to
+	 * current state, the function allocates and validates a new dc state
+	 * and assigns it to new_context. The function expects that the caller
+	 * is responsible to free this memory when new_context is no longer
+	 * used. We swap current with new context and free current instead. So
+	 * new_context's memory will live until the next full update after it is
+	 * replaced by a newer context. Refer to the use of
+	 * swap_and_free_current_context below.
+	 */
+	if (!update_planes_and_stream_state(dc, srf_updates, surface_count,
+				stream, stream_update, &update_type,
+				&new_context))
+		return false;
+
+	if (new_context == dc->current_state) {
+		commit_planes_and_stream_update_on_current_context(dc,
+				srf_updates, surface_count, stream,
+				stream_update, update_type);
+	} else {
+		commit_planes_and_stream_update_with_new_context(dc,
+				srf_updates, surface_count, stream,
+				stream_update, update_type, new_context);
+		swap_and_release_current_context(dc, new_context, stream);
+	}
+
+	return true;
+}
+
+bool dc_update_planes_and_stream(struct dc *dc,
+		struct dc_surface_update *srf_updates, int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update)
+{
+	dc_exit_ips_for_hw_access(dc);
+	/*
+	 * update planes and stream version 3 separates FULL and FAST updates
+	 * to their own sequences. It aims to clean up frequent checks for
+	 * update type resulting unnecessary branching in logic flow. It also
+	 * adds a new commit minimal transition sequence, which detects the need
+	 * for minimal transition based on the actual comparison of current and
+	 * new states instead of "predicting" it based on per feature software
+	 * policy.i.e could_mpcc_tree_change_for_active_pipes. The new commit
+	 * minimal transition sequence is made universal to any power saving
+	 * features that would use extra free pipes such as Dynamic ODM/MPC
+	 * Combine, MPO or SubVp. Therefore there is no longer a need to
+	 * specially handle compatibility problems with transitions among those
+	 * features as they are now transparent to the new sequence.
+	 */
+	if (dc->ctx->dce_version > DCN_VERSION_3_51)
+		return update_planes_and_stream_v3(dc, srf_updates,
+				surface_count, stream, stream_update);
+	return update_planes_and_stream_v2(dc, srf_updates,
+			surface_count, stream, stream_update);
+}
+
+void dc_commit_updates_for_stream(struct dc *dc,
+		struct dc_surface_update *srf_updates,
+		int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		struct dc_state *state)
+{
+	dc_exit_ips_for_hw_access(dc);
+	/* TODO: Since change commit sequence can have a huge impact,
+	 * we decided to only enable it for DCN3x. However, as soon as
+	 * we get more confident about this change we'll need to enable
+	 * the new sequence for all ASICs.
+	 */
+	if (dc->ctx->dce_version > DCN_VERSION_3_51) {
+		update_planes_and_stream_v3(dc, srf_updates, surface_count,
+				stream, stream_update);
+		return;
+	}
+	if (dc->ctx->dce_version >= DCN_VERSION_3_2) {
+		update_planes_and_stream_v2(dc, srf_updates, surface_count,
+				stream, stream_update);
+		return;
+	}
+	update_planes_and_stream_v1(dc, srf_updates, surface_count, stream,
+			stream_update, state);
 }
 
 uint8_t dc_get_current_stream_count(struct dc *dc)
@@ -4737,8 +5046,13 @@ void dc_interrupt_ack(struct dc *dc, enum dc_irq_source src)
 void dc_power_down_on_boot(struct dc *dc)
 {
 	if (dc->ctx->dce_environment != DCE_ENV_VIRTUAL_HW &&
-			dc->hwss.power_down_on_boot)
+			dc->hwss.power_down_on_boot) {
+
+			if (dc->caps.ips_support)
+				dc_exit_ips_for_hw_access(dc);
+
 		dc->hwss.power_down_on_boot(dc);
+	}
 }
 
 void dc_set_power_state(
@@ -4876,11 +5190,15 @@ bool dc_set_replay_allow_active(struct dc *dc, bool active)
 	return true;
 }
 
-void dc_allow_idle_optimizations(struct dc *dc, bool allow)
+void dc_allow_idle_optimizations_internal(struct dc *dc, bool allow, char const *caller_name)
 {
 	if (dc->debug.disable_idle_power_optimizations)
 		return;
 
+	if (allow != dc->idle_optimizations_allowed)
+		DC_LOG_IPS("%s: allow_idle old=%d new=%d (caller=%s)\n", __func__,
+			   dc->idle_optimizations_allowed, allow, caller_name);
+
 	if (dc->caps.ips_support && (dc->config.disable_ips == DMUB_IPS_DISABLE_ALL))
 		return;
 
@@ -4895,10 +5213,10 @@ void dc_allow_idle_optimizations(struct dc *dc, bool allow)
 		dc->idle_optimizations_allowed = allow;
 }
 
-void dc_exit_ips_for_hw_access(struct dc *dc)
+void dc_exit_ips_for_hw_access_internal(struct dc *dc, const char *caller_name)
 {
 	if (dc->caps.ips_support)
-		dc_allow_idle_optimizations(dc, false);
+		dc_allow_idle_optimizations_internal(dc, false, caller_name);
 }
 
 bool dc_dmub_is_ips_idle_state(struct dc *dc)
@@ -5032,10 +5350,13 @@ void dc_enable_dcmode_clk_limit(struct dc *dc, bool enable)
 	}
 	dc->clk_mgr->dc_mode_softmax_enabled = enable;
 }
-bool dc_is_plane_eligible_for_idle_optimizations(struct dc *dc, struct dc_plane_state *plane,
+bool dc_is_plane_eligible_for_idle_optimizations(struct dc *dc,
+		unsigned int pitch,
+		unsigned int height,
+		enum surface_pixel_format format,
 		struct dc_cursor_attributes *cursor_attr)
 {
-	if (dc->hwss.does_plane_fit_in_mall && dc->hwss.does_plane_fit_in_mall(dc, plane, cursor_attr))
+	if (dc->hwss.does_plane_fit_in_mall && dc->hwss.does_plane_fit_in_mall(dc, pitch, height, format, cursor_attr))
 		return true;
 	return false;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
index 9c05b1a07142..5c1d3017aefd 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
@@ -392,10 +392,10 @@ void get_hdr_visual_confirm_color(
 
 	switch (top_pipe_ctx->plane_res.scl_data.format) {
 	case PIXEL_FORMAT_ARGB2101010:
-		if (top_pipe_ctx->stream->out_transfer_func->tf == TRANSFER_FUNCTION_PQ) {
+		if (top_pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_PQ) {
 			/* HDR10, ARGB2101010 - set border color to red */
 			color->color_r_cr = color_value;
-		} else if (top_pipe_ctx->stream->out_transfer_func->tf == TRANSFER_FUNCTION_GAMMA22) {
+		} else if (top_pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22) {
 			/* FreeSync 2 ARGB2101010 - set border color to pink */
 			color->color_r_cr = color_value;
 			color->color_b_cb = color_value;
@@ -403,10 +403,10 @@ void get_hdr_visual_confirm_color(
 			is_sdr = true;
 		break;
 	case PIXEL_FORMAT_FP16:
-		if (top_pipe_ctx->stream->out_transfer_func->tf == TRANSFER_FUNCTION_PQ) {
+		if (top_pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_PQ) {
 			/* HDR10, FP16 - set border color to blue */
 			color->color_b_cb = color_value;
-		} else if (top_pipe_ctx->stream->out_transfer_func->tf == TRANSFER_FUNCTION_GAMMA22) {
+		} else if (top_pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22) {
 			/* FreeSync 2 HDR - set border color to green */
 			color->color_g_y = color_value;
 		} else
@@ -558,9 +558,10 @@ void hwss_build_fast_sequence(struct dc *dc,
 		struct dc_dmub_cmd *dc_dmub_cmd,
 		unsigned int dmub_cmd_count,
 		struct block_sequence block_sequence[],
-		int *num_steps,
+		unsigned int *num_steps,
 		struct pipe_ctx *pipe_ctx,
-		struct dc_stream_status *stream_status)
+		struct dc_stream_status *stream_status,
+		struct dc_state *context)
 {
 	struct dc_plane_state *plane = pipe_ctx->plane_state;
 	struct dc_stream_state *stream = pipe_ctx->stream;
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index ec4bf9432bdb..15819416a2f3 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -340,7 +340,7 @@ struct resource_pool *dc_create_resource_pool(struct dc  *dc,
 	return res_pool;
 }
 
-void dc_destroy_resource_pool(struct dc  *dc)
+void dc_destroy_resource_pool(struct dc *dc)
 {
 	if (dc) {
 		if (dc->res_pool)
@@ -1457,6 +1457,9 @@ void resource_build_test_pattern_params(struct resource_context *res_ctx,
 	controller_color_space = convert_dp_to_controller_color_space(
 			otg_master->stream->test_pattern.color_space);
 
+	if (controller_test_pattern == CONTROLLER_DP_TEST_PATTERN_VIDEOMODE)
+		return;
+
 	odm_cnt = resource_get_opp_heads_for_otg_master(otg_master, res_ctx, opp_heads);
 
 	odm_slice_width = h_active / odm_cnt;
@@ -1485,6 +1488,7 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx)
 	struct dc_crtc_timing *timing = &pipe_ctx->stream->timing;
 	const struct rect odm_slice_rec = calculate_odm_slice_in_timing_active(pipe_ctx);
 	bool res = false;
+
 	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
 
 	/* Invalid input */
@@ -1496,9 +1500,6 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx)
 		return false;
 	}
 
-	pipe_ctx->plane_res.scl_data.format = convert_pixel_format_to_dalsurface(
-			pipe_ctx->plane_state->format);
-
 	/* Timing borders are part of vactive that we are also supposed to skip in addition
 	 * to any stream dst offset. Since dm logic assumes dst is in addressable
 	 * space we need to add the left and top borders to dst offsets temporarily.
@@ -1510,6 +1511,8 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx)
 	/* Calculate H and V active size */
 	pipe_ctx->plane_res.scl_data.h_active = odm_slice_rec.width;
 	pipe_ctx->plane_res.scl_data.v_active = odm_slice_rec.height;
+	pipe_ctx->plane_res.scl_data.format = convert_pixel_format_to_dalsurface(
+			pipe_ctx->plane_state->format);
 
 	/* depends on h_active */
 	calculate_recout(pipe_ctx);
@@ -1794,6 +1797,30 @@ int recource_find_free_pipe_used_as_otg_master_in_cur_res_ctx(
 	return free_pipe_idx;
 }
 
+int resource_find_free_pipe_used_as_cur_sec_dpp(
+		const struct resource_context *cur_res_ctx,
+		struct resource_context *new_res_ctx,
+		const struct resource_pool *pool)
+{
+	int free_pipe_idx = FREE_PIPE_INDEX_NOT_FOUND;
+	const struct pipe_ctx *new_pipe, *cur_pipe;
+	int i;
+
+	for (i = 0; i < pool->pipe_count; i++) {
+		cur_pipe = &cur_res_ctx->pipe_ctx[i];
+		new_pipe = &new_res_ctx->pipe_ctx[i];
+
+		if (resource_is_pipe_type(cur_pipe, DPP_PIPE) &&
+				!resource_is_pipe_type(cur_pipe, OPP_HEAD) &&
+				resource_is_pipe_type(new_pipe, FREE_PIPE)) {
+			free_pipe_idx = i;
+			break;
+		}
+	}
+
+	return free_pipe_idx;
+}
+
 int resource_find_free_pipe_used_as_cur_sec_dpp_in_mpcc_combine(
 		const struct resource_context *cur_res_ctx,
 		struct resource_context *new_res_ctx,
@@ -2168,50 +2195,91 @@ static void resource_log_pipe(struct dc *dc, struct pipe_ctx *pipe,
 	}
 }
 
-void resource_log_pipe_topology_update(struct dc *dc, struct dc_state *state)
+static void resource_log_pipe_for_stream(struct dc *dc, struct dc_state *state,
+		struct pipe_ctx *otg_master, int stream_idx)
 {
-	struct pipe_ctx *otg_master;
 	struct pipe_ctx *opp_heads[MAX_PIPES];
 	struct pipe_ctx *dpp_pipes[MAX_PIPES];
 
-	int stream_idx, slice_idx, dpp_idx, plane_idx, slice_count, dpp_count;
+	int slice_idx, dpp_idx, plane_idx, slice_count, dpp_count;
 	bool is_primary;
 	DC_LOGGER_INIT(dc->ctx->logger);
 
+	slice_count = resource_get_opp_heads_for_otg_master(otg_master,
+			&state->res_ctx, opp_heads);
+	for (slice_idx = 0; slice_idx < slice_count; slice_idx++) {
+		plane_idx = -1;
+		if (opp_heads[slice_idx]->plane_state) {
+			dpp_count = resource_get_dpp_pipes_for_opp_head(
+					opp_heads[slice_idx],
+					&state->res_ctx,
+					dpp_pipes);
+			for (dpp_idx = 0; dpp_idx < dpp_count; dpp_idx++) {
+				is_primary = !dpp_pipes[dpp_idx]->top_pipe ||
+						dpp_pipes[dpp_idx]->top_pipe->plane_state != dpp_pipes[dpp_idx]->plane_state;
+				if (is_primary)
+					plane_idx++;
+				resource_log_pipe(dc, dpp_pipes[dpp_idx],
+						stream_idx, slice_idx,
+						plane_idx, slice_count,
+						is_primary);
+			}
+		} else {
+			resource_log_pipe(dc, opp_heads[slice_idx],
+					stream_idx, slice_idx, plane_idx,
+					slice_count, true);
+		}
+
+	}
+}
+
+static int resource_stream_to_stream_idx(struct dc_state *state,
+		struct dc_stream_state *stream)
+{
+	int i, stream_idx = -1;
+
+	for (i = 0; i < state->stream_count; i++)
+		if (state->streams[i] == stream) {
+			stream_idx = i;
+			break;
+		}
+
+	/* never return negative array index */
+	if (stream_idx == -1) {
+		ASSERT(0);
+		return 0;
+	}
+
+	return stream_idx;
+}
+
+void resource_log_pipe_topology_update(struct dc *dc, struct dc_state *state)
+{
+	struct pipe_ctx *otg_master;
+	int stream_idx, phantom_stream_idx;
+	DC_LOGGER_INIT(dc->ctx->logger);
+
 	DC_LOG_DC("    pipe topology update");
 	DC_LOG_DC("  ________________________");
 	for (stream_idx = 0; stream_idx < state->stream_count; stream_idx++) {
+		if (state->streams[stream_idx]->is_phantom)
+			continue;
+
 		otg_master = resource_get_otg_master_for_stream(
 				&state->res_ctx, state->streams[stream_idx]);
-		if (!otg_master	|| otg_master->stream_res.tg == NULL) {
-			DC_LOG_DC("topology update: otg_master NULL stream_idx %d!\n", stream_idx);
-			return;
-		}
-		slice_count = resource_get_opp_heads_for_otg_master(otg_master,
-				&state->res_ctx, opp_heads);
-		for (slice_idx = 0; slice_idx < slice_count; slice_idx++) {
-			plane_idx = -1;
-			if (opp_heads[slice_idx]->plane_state) {
-				dpp_count = resource_get_dpp_pipes_for_opp_head(
-						opp_heads[slice_idx],
-						&state->res_ctx,
-						dpp_pipes);
-				for (dpp_idx = 0; dpp_idx < dpp_count; dpp_idx++) {
-					is_primary = !dpp_pipes[dpp_idx]->top_pipe ||
-							dpp_pipes[dpp_idx]->top_pipe->plane_state != dpp_pipes[dpp_idx]->plane_state;
-					if (is_primary)
-						plane_idx++;
-					resource_log_pipe(dc, dpp_pipes[dpp_idx],
-							stream_idx, slice_idx,
-							plane_idx, slice_count,
-							is_primary);
-				}
-			} else {
-				resource_log_pipe(dc, opp_heads[slice_idx],
-						stream_idx, slice_idx, plane_idx,
-						slice_count, true);
-			}
+		resource_log_pipe_for_stream(dc, state, otg_master, stream_idx);
+	}
+	if (state->phantom_stream_count > 0) {
+		DC_LOG_DC(" |    (phantom pipes)     |");
+		for (stream_idx = 0; stream_idx < state->stream_count; stream_idx++) {
+			if (state->stream_status[stream_idx].mall_stream_config.type != SUBVP_MAIN)
+				continue;
 
+			phantom_stream_idx = resource_stream_to_stream_idx(state,
+					state->stream_status[stream_idx].mall_stream_config.paired_stream);
+			otg_master = resource_get_otg_master_for_stream(
+					&state->res_ctx, state->streams[phantom_stream_idx]);
+			resource_log_pipe_for_stream(dc, state, otg_master, stream_idx);
 		}
 	}
 	DC_LOG_DC(" |________________________|\n");
@@ -2266,6 +2334,9 @@ static bool update_pipe_params_after_odm_slice_count_change(
 
 	if (pool->funcs->build_pipe_pix_clk_params)
 		pool->funcs->build_pipe_pix_clk_params(otg_master);
+
+	resource_build_test_pattern_params(&context->res_ctx, otg_master);
+
 	return result;
 }
 
@@ -2624,13 +2695,19 @@ bool resource_append_dpp_pipes_for_plane_composition(
 		struct pipe_ctx *otg_master_pipe,
 		struct dc_plane_state *plane_state)
 {
+	bool success;
 	if (otg_master_pipe->plane_state == NULL)
-		return add_plane_to_opp_head_pipes(otg_master_pipe,
+		success = add_plane_to_opp_head_pipes(otg_master_pipe,
 				plane_state, new_ctx);
 	else
-		return acquire_secondary_dpp_pipes_and_add_plane(
+		success = acquire_secondary_dpp_pipes_and_add_plane(
 				otg_master_pipe, plane_state, new_ctx,
 				cur_ctx, pool);
+	if (success)
+		/* when appending a plane mpc slice count changes from 0 to 1 */
+		success = update_pipe_params_after_mpc_slice_count_change(
+				plane_state, new_ctx, pool);
+	return success;
 }
 
 void resource_remove_dpp_pipes_for_plane_composition(
@@ -2965,7 +3042,7 @@ bool resource_update_pipes_for_plane_with_slice_count(
 	int i;
 	int dpp_pipe_count;
 	int cur_slice_count;
-	struct pipe_ctx *dpp_pipes[MAX_PIPES];
+	struct pipe_ctx *dpp_pipes[MAX_PIPES] = {0};
 	bool result = true;
 
 	dpp_pipe_count = resource_get_dpp_pipes_for_plane(plane,
@@ -3117,6 +3194,9 @@ static struct audio *find_first_free_audio(
 {
 	int i, available_audio_count;
 
+	if (id == ENGINE_ID_UNKNOWN)
+		return NULL;
+
 	available_audio_count = pool->audio_count;
 
 	for (i = 0; i < available_audio_count; i++) {
@@ -3371,11 +3451,31 @@ static bool acquire_otg_master_pipe_for_stream(
 	 * any free pipes already used in current context as this could tear
 	 * down exiting ODM/MPC/MPO configuration unnecessarily.
 	 */
+
+	/*
+	 * Try to acquire the same OTG master already in use. This is not
+	 * optimal because resetting an enabled OTG master pipe for a new stream
+	 * requires an extra frame of wait. However there are test automation
+	 * and eDP assumptions that rely on reusing the same OTG master pipe
+	 * during mode change. We have to keep this logic as is for now.
+	 */
 	pipe_idx = recource_find_free_pipe_used_as_otg_master_in_cur_res_ctx(
 			&cur_ctx->res_ctx, &new_ctx->res_ctx, pool);
+	/*
+	 * Try to acquire a pipe not used in current resource context to avoid
+	 * pipe swapping.
+	 */
 	if (pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
 		pipe_idx = recource_find_free_pipe_not_used_in_cur_res_ctx(
 				&cur_ctx->res_ctx, &new_ctx->res_ctx, pool);
+	/*
+	 * If pipe swapping is unavoidable, try to acquire pipe used as
+	 * secondary DPP pipe in current state as we prioritize to support more
+	 * streams over supporting MPO planes.
+	 */
+	if (pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
+		pipe_idx = resource_find_free_pipe_used_as_cur_sec_dpp(
+				&cur_ctx->res_ctx, &new_ctx->res_ctx, pool);
 	if (pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
 		pipe_idx = resource_find_any_free_pipe(&new_ctx->res_ctx, pool);
 	if (pipe_idx != FREE_PIPE_INDEX_NOT_FOUND) {
@@ -3990,7 +4090,7 @@ static void set_avi_info_frame(
 	}
 
 	if (pixel_encoding && color_space == COLOR_SPACE_2020_YCBCR &&
-			stream->out_transfer_func->tf == TRANSFER_FUNCTION_GAMMA22) {
+			stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22) {
 		hdmi_info.bits.EC0_EC2 = 0;
 		hdmi_info.bits.C0_C1 = COLORIMETRY_ITU709;
 	}
@@ -4992,3 +5092,39 @@ bool check_subvp_sw_cursor_fallback_req(const struct dc *dc, struct dc_stream_st
 
 	return false;
 }
+
+void resource_init_common_dml2_callbacks(struct dc *dc, struct dml2_configuration_options *dml2_options)
+{
+	dml2_options->callbacks.dc = dc;
+	dml2_options->callbacks.build_scaling_params = &resource_build_scaling_params;
+	dml2_options->callbacks.build_test_pattern_params = &resource_build_test_pattern_params;
+	dml2_options->callbacks.acquire_secondary_pipe_for_mpc_odm = &dc_resource_acquire_secondary_pipe_for_mpc_odm_legacy;
+	dml2_options->callbacks.update_pipes_for_stream_with_slice_count = &resource_update_pipes_for_stream_with_slice_count;
+	dml2_options->callbacks.update_pipes_for_plane_with_slice_count = &resource_update_pipes_for_plane_with_slice_count;
+	dml2_options->callbacks.get_mpc_slice_index = &resource_get_mpc_slice_index;
+	dml2_options->callbacks.get_mpc_slice_count = &resource_get_mpc_slice_count;
+	dml2_options->callbacks.get_odm_slice_index = &resource_get_odm_slice_index;
+	dml2_options->callbacks.get_odm_slice_count = &resource_get_odm_slice_count;
+	dml2_options->callbacks.get_opp_head = &resource_get_opp_head;
+	dml2_options->callbacks.get_otg_master_for_stream = &resource_get_otg_master_for_stream;
+	dml2_options->callbacks.get_opp_heads_for_otg_master = &resource_get_opp_heads_for_otg_master;
+	dml2_options->callbacks.get_dpp_pipes_for_plane = &resource_get_dpp_pipes_for_plane;
+	dml2_options->callbacks.get_stream_status = &dc_state_get_stream_status;
+	dml2_options->callbacks.get_stream_from_id = &dc_state_get_stream_from_id;
+
+	dml2_options->svp_pstate.callbacks.dc = dc;
+	dml2_options->svp_pstate.callbacks.add_phantom_plane = &dc_state_add_phantom_plane;
+	dml2_options->svp_pstate.callbacks.add_phantom_stream = &dc_state_add_phantom_stream;
+	dml2_options->svp_pstate.callbacks.build_scaling_params = &resource_build_scaling_params;
+	dml2_options->svp_pstate.callbacks.create_phantom_plane = &dc_state_create_phantom_plane;
+	dml2_options->svp_pstate.callbacks.remove_phantom_plane = &dc_state_remove_phantom_plane;
+	dml2_options->svp_pstate.callbacks.remove_phantom_stream = &dc_state_remove_phantom_stream;
+	dml2_options->svp_pstate.callbacks.create_phantom_stream = &dc_state_create_phantom_stream;
+	dml2_options->svp_pstate.callbacks.release_phantom_plane = &dc_state_release_phantom_plane;
+	dml2_options->svp_pstate.callbacks.release_phantom_stream = &dc_state_release_phantom_stream;
+	dml2_options->svp_pstate.callbacks.get_pipe_subvp_type = &dc_state_get_pipe_subvp_type;
+	dml2_options->svp_pstate.callbacks.get_stream_subvp_type = &dc_state_get_stream_subvp_type;
+	dml2_options->svp_pstate.callbacks.get_paired_subvp_stream = &dc_state_get_paired_subvp_stream;
+	dml2_options->svp_pstate.callbacks.remove_phantom_streams_and_planes = &dc_state_remove_phantom_streams_and_planes;
+	dml2_options->svp_pstate.callbacks.release_phantom_streams_and_planes = &dc_state_release_phantom_streams_and_planes;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stat.c b/drivers/gpu/drm/amd/display/dc/core/dc_stat.c
index 5f6392ae31a6..cd6570a1e20e 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stat.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stat.c
@@ -61,7 +61,7 @@ void dc_stat_get_dmub_notification(const struct dc *dc, struct dmub_notification
 	/* For HPD/HPD RX, convert dpia port index into link index */
 	if (notify->type == DMUB_NOTIFICATION_HPD ||
 	    notify->type == DMUB_NOTIFICATION_HPD_IRQ ||
-		notify->type == DMUB_NOTIFICATION_DPIA_NOTIFICATION ||
+	    notify->type == DMUB_NOTIFICATION_DPIA_NOTIFICATION ||
 	    notify->type == DMUB_NOTIFICATION_SET_CONFIG_REPLY) {
 		notify->link_index =
 			get_link_index_from_dpia_port_index(dc, notify->link_index);
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_state.c b/drivers/gpu/drm/amd/display/dc/core/dc_state.c
index 61986e5cb491..76bb05f4d6bf 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_state.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_state.c
@@ -188,8 +188,11 @@ static void init_state(struct dc *dc, struct dc_state *state)
 }
 
 /* Public dc_state functions */
-struct dc_state *dc_state_create(struct dc *dc)
+struct dc_state *dc_state_create(struct dc *dc, struct dc_state_create_params *params)
 {
+#ifdef CONFIG_DRM_AMD_DC_FP
+	struct dml2_configuration_options *dml2_opt = &dc->dml2_options;
+#endif
 	struct dc_state *state = kvzalloc(sizeof(struct dc_state),
 			GFP_KERNEL);
 
@@ -198,10 +201,16 @@ struct dc_state *dc_state_create(struct dc *dc)
 
 	init_state(dc, state);
 	dc_state_construct(dc, state);
+	state->power_source = params ? params->power_source : DC_POWER_SOURCE_AC;
 
 #ifdef CONFIG_DRM_AMD_DC_FP
-	if (dc->debug.using_dml2)
-		dml2_create(dc, &dc->dml2_options, &state->bw_ctx.dml2);
+	if (dc->debug.using_dml2) {
+		dml2_opt->use_clock_dc_limits = false;
+		dml2_create(dc, dml2_opt, &state->bw_ctx.dml2);
+
+		dml2_opt->use_clock_dc_limits = true;
+		dml2_create(dc, dml2_opt, &state->bw_ctx.dml2_dc_power_source);
+	}
 #endif
 
 	kref_init(&state->refcount);
@@ -214,6 +223,7 @@ void dc_state_copy(struct dc_state *dst_state, struct dc_state *src_state)
 	struct kref refcount = dst_state->refcount;
 #ifdef CONFIG_DRM_AMD_DC_FP
 	struct dml2_context *dst_dml2 = dst_state->bw_ctx.dml2;
+	struct dml2_context *dst_dml2_dc_power_source = dst_state->bw_ctx.dml2_dc_power_source;
 #endif
 
 	dc_state_copy_internal(dst_state, src_state);
@@ -222,6 +232,10 @@ void dc_state_copy(struct dc_state *dst_state, struct dc_state *src_state)
 	dst_state->bw_ctx.dml2 = dst_dml2;
 	if (src_state->bw_ctx.dml2)
 		dml2_copy(dst_state->bw_ctx.dml2, src_state->bw_ctx.dml2);
+
+	dst_state->bw_ctx.dml2_dc_power_source = dst_dml2_dc_power_source;
+	if (src_state->bw_ctx.dml2_dc_power_source)
+		dml2_copy(dst_state->bw_ctx.dml2_dc_power_source, src_state->bw_ctx.dml2_dc_power_source);
 #endif
 
 	/* context refcount should not be overridden */
@@ -245,6 +259,12 @@ struct dc_state *dc_state_create_copy(struct dc_state *src_state)
 		dc_state_release(new_state);
 		return NULL;
 	}
+
+	if (src_state->bw_ctx.dml2_dc_power_source &&
+			!dml2_create_copy(&new_state->bw_ctx.dml2_dc_power_source, src_state->bw_ctx.dml2_dc_power_source)) {
+		dc_state_release(new_state);
+		return NULL;
+	}
 #endif
 
 	kref_init(&new_state->refcount);
@@ -310,7 +330,6 @@ void dc_state_destruct(struct dc_state *state)
 	memset(state->dc_dmub_cmd, 0, sizeof(state->dc_dmub_cmd));
 	state->dmub_cmd_count = 0;
 	memset(&state->perf_params, 0, sizeof(state->perf_params));
-	memset(&state->scratch, 0, sizeof(state->scratch));
 }
 
 void dc_state_retain(struct dc_state *state)
@@ -327,6 +346,9 @@ static void dc_state_free(struct kref *kref)
 #ifdef CONFIG_DRM_AMD_DC_FP
 	dml2_destroy(state->bw_ctx.dml2);
 	state->bw_ctx.dml2 = 0;
+
+	dml2_destroy(state->bw_ctx.dml2_dc_power_source);
+	state->bw_ctx.dml2_dc_power_source = 0;
 #endif
 
 	kvfree(state);
@@ -341,7 +363,7 @@ void dc_state_release(struct dc_state *state)
  * dc_state_add_stream() - Add a new dc_stream_state to a dc_state.
  */
 enum dc_status dc_state_add_stream(
-		struct dc *dc,
+		const struct dc *dc,
 		struct dc_state *state,
 		struct dc_stream_state *stream)
 {
@@ -370,7 +392,7 @@ enum dc_status dc_state_add_stream(
  * dc_state_remove_stream() - Remove a stream from a dc_state.
  */
 enum dc_status dc_state_remove_stream(
-		struct dc *dc,
+		const struct dc *dc,
 		struct dc_state *state,
 		struct dc_stream_state *stream)
 {
@@ -595,7 +617,7 @@ bool dc_state_add_all_planes_for_stream(
  */
 struct dc_stream_status *dc_state_get_stream_status(
 		struct dc_state *state,
-		struct dc_stream_state *stream)
+		const struct dc_stream_state *stream)
 {
 	uint8_t i;
 
@@ -689,7 +711,7 @@ void dc_state_release_phantom_stream(const struct dc *dc,
 	dc_stream_release(phantom_stream);
 }
 
-struct dc_plane_state *dc_state_create_phantom_plane(struct dc *dc,
+struct dc_plane_state *dc_state_create_phantom_plane(const struct dc *dc,
 		struct dc_state *state,
 		struct dc_plane_state *main_plane)
 {
@@ -725,7 +747,7 @@ void dc_state_release_phantom_plane(const struct dc *dc,
 }
 
 /* add phantom streams to context and generate correct meta inside dc_state */
-enum dc_status dc_state_add_phantom_stream(struct dc *dc,
+enum dc_status dc_state_add_phantom_stream(const struct dc *dc,
 		struct dc_state *state,
 		struct dc_stream_state *phantom_stream,
 		struct dc_stream_state *main_stream)
@@ -751,7 +773,7 @@ enum dc_status dc_state_add_phantom_stream(struct dc *dc,
 	return res;
 }
 
-enum dc_status dc_state_remove_phantom_stream(struct dc *dc,
+enum dc_status dc_state_remove_phantom_stream(const struct dc *dc,
 		struct dc_state *state,
 		struct dc_stream_state *phantom_stream)
 {
@@ -845,7 +867,7 @@ bool dc_state_add_all_phantom_planes_for_stream(
 }
 
 bool dc_state_remove_phantom_streams_and_planes(
-	struct dc *dc,
+	const struct dc *dc,
 	struct dc_state *state)
 {
 	int i;
@@ -867,7 +889,7 @@ bool dc_state_remove_phantom_streams_and_planes(
 }
 
 void dc_state_release_phantom_streams_and_planes(
-		struct dc *dc,
+		const struct dc *dc,
 		struct dc_state *state)
 {
 	int i;
@@ -878,3 +900,19 @@ void dc_state_release_phantom_streams_and_planes(
 	for (i = 0; i < state->phantom_plane_count; i++)
 		dc_state_release_phantom_plane(dc, state, state->phantom_planes[i]);
 }
+
+struct dc_stream_state *dc_state_get_stream_from_id(const struct dc_state *state, unsigned int id)
+{
+	struct dc_stream_state *stream = NULL;
+	int i;
+
+	for (i = 0; i < state->stream_count; i++) {
+		if (state->streams[i] && state->streams[i]->stream_id == id) {
+			stream = state->streams[i];
+			break;
+		}
+	}
+
+	return stream;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
index 51a970fcb5d0..5c7e4884cac2 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
@@ -116,12 +116,7 @@ bool dc_stream_construct(struct dc_stream_state *stream,
 
 	update_stream_signal(stream, dc_sink_data);
 
-	stream->out_transfer_func = dc_create_transfer_func();
-	if (stream->out_transfer_func == NULL) {
-		dc_sink_release(dc_sink_data);
-		return false;
-	}
-	stream->out_transfer_func->type = TF_TYPE_BYPASS;
+	stream->out_transfer_func.type = TF_TYPE_BYPASS;
 
 	dc_stream_assign_stream_id(stream);
 
@@ -131,10 +126,6 @@ bool dc_stream_construct(struct dc_stream_state *stream,
 void dc_stream_destruct(struct dc_stream_state *stream)
 {
 	dc_sink_release(stream->sink);
-	if (stream->out_transfer_func != NULL) {
-		dc_transfer_func_release(stream->out_transfer_func);
-		stream->out_transfer_func = NULL;
-	}
 }
 
 void dc_stream_assign_stream_id(struct dc_stream_state *stream)
@@ -201,9 +192,6 @@ struct dc_stream_state *dc_copy_stream(const struct dc_stream_state *stream)
 	if (new_stream->sink)
 		dc_sink_retain(new_stream->sink);
 
-	if (new_stream->out_transfer_func)
-		dc_transfer_func_retain(new_stream->out_transfer_func);
-
 	dc_stream_assign_stream_id(new_stream);
 
 	/* If using dynamic encoder assignment, wait till stream committed to assign encoder. */
@@ -319,7 +307,7 @@ bool dc_stream_set_cursor_attributes(
 	program_cursor_attributes(dc, stream, attributes);
 
 	/* re-enable idle optimizations if necessary */
-	if (reset_idle_optimizations)
+	if (reset_idle_optimizations && !dc->debug.disable_dmub_reallow_idle)
 		dc_allow_idle_optimizations(dc, true);
 
 	return true;
@@ -394,7 +382,7 @@ bool dc_stream_set_cursor_position(
 
 	program_cursor_position(dc, stream, position);
 	/* re-enable idle optimizations if necessary */
-	if (reset_idle_optimizations)
+	if (reset_idle_optimizations && !dc->debug.disable_dmub_reallow_idle)
 		dc_allow_idle_optimizations(dc, true);
 
 	return true;
@@ -425,7 +413,7 @@ bool dc_stream_add_writeback(struct dc *dc,
 
 	dc_exit_ips_for_hw_access(dc);
 
-	wb_info->dwb_params.out_transfer_func = stream->out_transfer_func;
+	wb_info->dwb_params.out_transfer_func = &stream->out_transfer_func;
 
 	dwb = dc->res_pool->dwbc[wb_info->dwb_pipe_inst];
 	dwb->dwb_is_drc = false;
@@ -507,7 +495,7 @@ bool dc_stream_remove_writeback(struct dc *dc,
 		struct dc_stream_state *stream,
 		uint32_t dwb_pipe_inst)
 {
-	int i = 0, j = 0;
+	unsigned int i, j;
 	if (stream == NULL) {
 		dm_error("DC: dc_stream is NULL!\n");
 		return false;
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
index 19140fb65787..067f6555cfdf 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
@@ -41,25 +41,15 @@ void dc_plane_construct(struct dc_context *ctx, struct dc_plane_state *plane_sta
 {
 	plane_state->ctx = ctx;
 
-	plane_state->gamma_correction = dc_create_gamma();
-	if (plane_state->gamma_correction != NULL)
-		plane_state->gamma_correction->is_identity = true;
+	plane_state->gamma_correction.is_identity = true;
 
-	plane_state->in_transfer_func = dc_create_transfer_func();
-	if (plane_state->in_transfer_func != NULL) {
-		plane_state->in_transfer_func->type = TF_TYPE_BYPASS;
-	}
-	plane_state->in_shaper_func = dc_create_transfer_func();
-	if (plane_state->in_shaper_func != NULL) {
-		plane_state->in_shaper_func->type = TF_TYPE_BYPASS;
-	}
+	plane_state->in_transfer_func.type = TF_TYPE_BYPASS;
 
-	plane_state->lut3d_func = dc_create_3dlut_func();
+	plane_state->in_shaper_func.type = TF_TYPE_BYPASS;
 
-	plane_state->blend_tf = dc_create_transfer_func();
-	if (plane_state->blend_tf != NULL) {
-		plane_state->blend_tf->type = TF_TYPE_BYPASS;
-	}
+	plane_state->lut3d_func.state.raw = 0;
+
+	plane_state->blend_tf.type = TF_TYPE_BYPASS;
 
 	plane_state->pre_multiplied_alpha = true;
 
@@ -67,30 +57,27 @@ void dc_plane_construct(struct dc_context *ctx, struct dc_plane_state *plane_sta
 
 void dc_plane_destruct(struct dc_plane_state *plane_state)
 {
-	if (plane_state->gamma_correction != NULL) {
-		dc_gamma_release(&plane_state->gamma_correction);
-	}
-	if (plane_state->in_transfer_func != NULL) {
-		dc_transfer_func_release(
-				plane_state->in_transfer_func);
-		plane_state->in_transfer_func = NULL;
-	}
-	if (plane_state->in_shaper_func != NULL) {
-		dc_transfer_func_release(
-				plane_state->in_shaper_func);
-		plane_state->in_shaper_func = NULL;
-	}
-	if (plane_state->lut3d_func != NULL) {
-		dc_3dlut_func_release(
-				plane_state->lut3d_func);
-		plane_state->lut3d_func = NULL;
-	}
-	if (plane_state->blend_tf != NULL) {
-		dc_transfer_func_release(
-				plane_state->blend_tf);
-		plane_state->blend_tf = NULL;
+	// no more pointers to free within dc_plane_state
+}
+
+
+/* dc_state is passed in separately since it may differ from the current dc state accessible from plane_state e.g.
+ * if the driver is doing an update from an old context to a new one and the caller wants the pipe mask for the new
+ * context rather than the existing one
+ */
+uint8_t  dc_plane_get_pipe_mask(struct dc_state *dc_state, const struct dc_plane_state *plane_state)
+{
+	uint8_t pipe_mask = 0;
+	int i;
+
+	for (i = 0; i < plane_state->ctx->dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &dc_state->res_ctx.pipe_ctx[i];
+
+		if (pipe_ctx->plane_state == plane_state && pipe_ctx->plane_res.hubp)
+			pipe_mask |= 1 << pipe_ctx->plane_res.hubp->inst;
 	}
 
+	return pipe_mask;
 }
 
 /*******************************************************************************
@@ -103,7 +90,7 @@ void enable_surface_flip_reporting(struct dc_plane_state *plane_state,
 	/*register_flip_interrupt(surface);*/
 }
 
-struct dc_plane_state *dc_create_plane_state(struct dc *dc)
+struct dc_plane_state *dc_create_plane_state(const struct dc *dc)
 {
 	struct dc_plane_state *plane_state = kvzalloc(sizeof(*plane_state),
 							GFP_KERNEL);
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index ee8453bf958f..3c33c3bcbe2c 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -44,6 +44,8 @@
 
 #include "dml2/dml2_wrapper.h"
 
+#include "dmub/inc/dmub_cmd.h"
+
 struct abm_save_restore;
 
 /* forward declaration */
@@ -51,7 +53,7 @@ struct aux_payload;
 struct set_config_cmd_payload;
 struct dmub_notification;
 
-#define DC_VER "3.2.273"
+#define DC_VER "3.2.281"
 
 #define MAX_SURFACES 3
 #define MAX_PLANES 6
@@ -219,6 +221,7 @@ struct dc_dmub_caps {
 	bool mclk_sw;
 	bool subvp_psr;
 	bool gecc_enable;
+	uint8_t fams_ver;
 };
 
 struct dc_caps {
@@ -306,12 +309,12 @@ struct dc_dcc_setting {
 	unsigned int max_compressed_blk_size;
 	unsigned int max_uncompressed_blk_size;
 	bool independent_64b_blks;
-	//These bitfields to be used starting with DCN
+	//These bitfields to be used starting with DCN 3.0
 	struct {
-		uint32_t dcc_256_64_64 : 1;//available in ASICs before DCN (the worst compression case)
-		uint32_t dcc_128_128_uncontrained : 1;  //available in ASICs before DCN
-		uint32_t dcc_256_128_128 : 1;		//available starting with DCN
-		uint32_t dcc_256_256_unconstrained : 1;  //available in ASICs before DCN (the best compression case)
+		uint32_t dcc_256_64_64 : 1;//available in ASICs before DCN 3.0 (the worst compression case)
+		uint32_t dcc_128_128_uncontrained : 1;  //available in ASICs before DCN 3.0
+		uint32_t dcc_256_128_128 : 1;		//available starting with DCN 3.0
+		uint32_t dcc_256_256_unconstrained : 1;  //available in ASICs before DCN 3.0 (the best compression case)
 	} dcc_controls;
 };
 
@@ -435,6 +438,9 @@ struct dc_config {
 	unsigned int disable_ips;
 	unsigned int disable_ips_in_vpb;
 	bool usb4_bw_alloc_support;
+	bool allow_0_dtb_clk;
+	bool use_assr_psp_message;
+	bool support_edp0_on_dp1;
 };
 
 enum visual_confirm {
@@ -693,6 +699,8 @@ enum pg_hw_pipe_resources {
 	PG_MPCC,
 	PG_OPP,
 	PG_OPTC,
+	PG_DPSTREAM,
+	PG_HDMISTREAM,
 	PG_HW_PIPE_RESOURCES_NUM_ELEMENT
 };
 
@@ -987,14 +995,17 @@ struct dc_debug_options {
 	bool psp_disabled_wa;
 	unsigned int ips2_eval_delay_us;
 	unsigned int ips2_entry_delay_us;
+	bool optimize_ips_handshake;
 	bool disable_dmub_reallow_idle;
 	bool disable_timeout;
 	bool disable_extblankadj;
+	bool enable_idle_reg_checks;
 	unsigned int static_screen_wait_frames;
 	bool force_chroma_subsampling_1tap;
+	bool disable_422_left_edge_pixel;
+	unsigned int force_cositing;
 };
 
-struct gpu_info_soc_bounding_box_v1_0;
 
 /* Generic structure that can be used to query properties of DC. More fields
  * can be added as required.
@@ -1003,76 +1014,6 @@ struct dc_current_properties {
 	unsigned int cursor_size_limit;
 };
 
-struct dc {
-	struct dc_debug_options debug;
-	struct dc_versions versions;
-	struct dc_caps caps;
-	struct dc_cap_funcs cap_funcs;
-	struct dc_config config;
-	struct dc_bounding_box_overrides bb_overrides;
-	struct dc_bug_wa work_arounds;
-	struct dc_context *ctx;
-	struct dc_phy_addr_space_config vm_pa_config;
-
-	uint8_t link_count;
-	struct dc_link *links[MAX_PIPES * 2];
-	struct link_service *link_srv;
-
-	struct dc_state *current_state;
-	struct resource_pool *res_pool;
-
-	struct clk_mgr *clk_mgr;
-
-	/* Display Engine Clock levels */
-	struct dm_pp_clock_levels sclk_lvls;
-
-	/* Inputs into BW and WM calculations. */
-	struct bw_calcs_dceip *bw_dceip;
-	struct bw_calcs_vbios *bw_vbios;
-	struct dcn_soc_bounding_box *dcn_soc;
-	struct dcn_ip_params *dcn_ip;
-	struct display_mode_lib dml;
-
-	/* HW functions */
-	struct hw_sequencer_funcs hwss;
-	struct dce_hwseq *hwseq;
-
-	/* Require to optimize clocks and bandwidth for added/removed planes */
-	bool optimized_required;
-	bool wm_optimized_required;
-	bool idle_optimizations_allowed;
-	bool enable_c20_dtm_b0;
-
-	/* Require to maintain clocks and bandwidth for UEFI enabled HW */
-
-	/* FBC compressor */
-	struct compressor *fbc_compressor;
-
-	struct dc_debug_data debug_data;
-	struct dpcd_vendor_signature vendor_signature;
-
-	const char *build_id;
-	struct vm_helper *vm_helper;
-
-	uint32_t *dcn_reg_offsets;
-	uint32_t *nbio_reg_offsets;
-	uint32_t *clk_reg_offsets;
-
-	/* Scratch memory */
-	struct {
-		struct {
-			/*
-			 * For matching clock_limits table in driver with table
-			 * from PMFW.
-			 */
-			struct _vcs_dpi_voltage_scaling_st clock_limits[DC__VOLTAGE_STATES];
-		} update_bw_bounding_box;
-	} scratch;
-
-	struct dml2_configuration_options dml2_options;
-	enum dc_acpi_cm_power_state power_state;
-};
-
 enum frame_buffer_mode {
 	FRAME_BUFFER_MODE_LOCAL_ONLY = 0,
 	FRAME_BUFFER_MODE_ZFB_ONLY,
@@ -1277,6 +1218,8 @@ union surface_update_flags {
 	uint32_t raw;
 };
 
+#define DC_REMOVE_PLANE_POINTERS 1
+
 struct dc_plane_state {
 	struct dc_plane_address address;
 	struct dc_plane_flip_time time;
@@ -1291,8 +1234,8 @@ struct dc_plane_state {
 
 	struct dc_plane_dcc_param dcc;
 
-	struct dc_gamma *gamma_correction;
-	struct dc_transfer_func *in_transfer_func;
+	struct dc_gamma gamma_correction;
+	struct dc_transfer_func in_transfer_func;
 	struct dc_bias_and_scale *bias_and_scale;
 	struct dc_csc_transform input_csc_color_matrix;
 	struct fixed31_32 coeff_reduction_factor;
@@ -1304,9 +1247,9 @@ struct dc_plane_state {
 
 	enum dc_color_space color_space;
 
-	struct dc_3dlut *lut3d_func;
-	struct dc_transfer_func *in_shaper_func;
-	struct dc_transfer_func *blend_tf;
+	struct dc_3dlut lut3d_func;
+	struct dc_transfer_func in_shaper_func;
+	struct dc_transfer_func blend_tf;
 
 	struct dc_transfer_func *gamcor_tf;
 	enum surface_pixel_format format;
@@ -1342,6 +1285,7 @@ struct dc_plane_state {
 	struct tg_color visual_confirm_color;
 
 	bool is_statically_allocated;
+	enum chroma_cositing cositing;
 };
 
 struct dc_plane_info {
@@ -1360,6 +1304,96 @@ struct dc_plane_info {
 	int  global_alpha_value;
 	bool input_csc_enabled;
 	int layer_index;
+	enum chroma_cositing cositing;
+};
+
+#include "dc_stream.h"
+
+struct dc_scratch_space {
+	/* used to temporarily backup plane states of a stream during
+	 * dc update. The reason is that plane states are overwritten
+	 * with surface updates in dc update. Once they are overwritten
+	 * current state is no longer valid. We want to temporarily
+	 * store current value in plane states so we can still recover
+	 * a valid current state during dc update.
+	 */
+	struct dc_plane_state plane_states[MAX_SURFACE_NUM];
+
+	struct dc_stream_state stream_state;
+};
+
+struct dc {
+	struct dc_debug_options debug;
+	struct dc_versions versions;
+	struct dc_caps caps;
+	struct dc_cap_funcs cap_funcs;
+	struct dc_config config;
+	struct dc_bounding_box_overrides bb_overrides;
+	struct dc_bug_wa work_arounds;
+	struct dc_context *ctx;
+	struct dc_phy_addr_space_config vm_pa_config;
+
+	uint8_t link_count;
+	struct dc_link *links[MAX_LINKS];
+	struct link_service *link_srv;
+
+	struct dc_state *current_state;
+	struct resource_pool *res_pool;
+
+	struct clk_mgr *clk_mgr;
+
+	/* Display Engine Clock levels */
+	struct dm_pp_clock_levels sclk_lvls;
+
+	/* Inputs into BW and WM calculations. */
+	struct bw_calcs_dceip *bw_dceip;
+	struct bw_calcs_vbios *bw_vbios;
+	struct dcn_soc_bounding_box *dcn_soc;
+	struct dcn_ip_params *dcn_ip;
+	struct display_mode_lib dml;
+
+	/* HW functions */
+	struct hw_sequencer_funcs hwss;
+	struct dce_hwseq *hwseq;
+
+	/* Require to optimize clocks and bandwidth for added/removed planes */
+	bool optimized_required;
+	bool wm_optimized_required;
+	bool idle_optimizations_allowed;
+	bool enable_c20_dtm_b0;
+
+	/* Require to maintain clocks and bandwidth for UEFI enabled HW */
+
+	/* FBC compressor */
+	struct compressor *fbc_compressor;
+
+	struct dc_debug_data debug_data;
+	struct dpcd_vendor_signature vendor_signature;
+
+	const char *build_id;
+	struct vm_helper *vm_helper;
+
+	uint32_t *dcn_reg_offsets;
+	uint32_t *nbio_reg_offsets;
+	uint32_t *clk_reg_offsets;
+
+	/* Scratch memory */
+	struct {
+		struct {
+			/*
+			 * For matching clock_limits table in driver with table
+			 * from PMFW.
+			 */
+			struct _vcs_dpi_voltage_scaling_st clock_limits[DC__VOLTAGE_STATES];
+		} update_bw_bounding_box;
+		struct dc_scratch_space current_state;
+		struct dc_scratch_space new_state;
+		struct dc_stream_state temp_stream; // Used so we don't need to allocate stream on the stack
+	} scratch;
+
+	struct dml2_configuration_options dml2_options;
+	enum dc_acpi_cm_power_state power_state;
+
 };
 
 struct dc_scaling_info {
@@ -1476,10 +1510,15 @@ bool dc_acquire_release_mpc_3dlut(
 bool dc_resource_is_dsc_encoding_supported(const struct dc *dc);
 void get_audio_check(struct audio_info *aud_modes,
 	struct audio_check *aud_chk);
-
-enum dc_status dc_commit_streams(struct dc *dc,
-				 struct dc_stream_state *streams[],
-				 uint8_t stream_count);
+/*
+ * Set up streams and links associated to drive sinks
+ * The streams parameter is an absolute set of all active streams.
+ *
+ * After this call:
+ *   Phy, Encoder, Timing Generator are programmed and enabled.
+ *   New streams are enabled with blank stream; no memory read.
+ */
+enum dc_status dc_commit_streams(struct dc *dc, struct dc_commit_streams_params *params);
 
 
 struct dc_plane_state *dc_get_surface_for_mpcc(struct dc *dc,
@@ -2335,11 +2374,17 @@ bool dc_is_dmcu_initialized(struct dc *dc);
 enum dc_status dc_set_clock(struct dc *dc, enum dc_clock_type clock_type, uint32_t clk_khz, uint32_t stepping);
 void dc_get_clock(struct dc *dc, enum dc_clock_type clock_type, struct dc_clock_config *clock_cfg);
 
-bool dc_is_plane_eligible_for_idle_optimizations(struct dc *dc, struct dc_plane_state *plane,
-				struct dc_cursor_attributes *cursor_attr);
+bool dc_is_plane_eligible_for_idle_optimizations(struct dc *dc,
+		unsigned int pitch,
+		unsigned int height,
+		enum surface_pixel_format format,
+		struct dc_cursor_attributes *cursor_attr);
+
+#define dc_allow_idle_optimizations(dc, allow) dc_allow_idle_optimizations_internal(dc, allow, __func__)
+#define dc_exit_ips_for_hw_access(dc) dc_exit_ips_for_hw_access_internal(dc, __func__)
 
-void dc_allow_idle_optimizations(struct dc *dc, bool allow);
-void dc_exit_ips_for_hw_access(struct dc *dc);
+void dc_allow_idle_optimizations_internal(struct dc *dc, bool allow, const char *caller_name);
+void dc_exit_ips_for_hw_access_internal(struct dc *dc, const char *caller_name);
 bool dc_dmub_is_ips_idle_state(struct dc *dc);
 
 /* set min and max memory clock to lowest and highest DPM level, respectively */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
index 6083b1dcf050..2293a92df3be 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
@@ -23,6 +23,7 @@
  *
  */
 
+#include "dm_services.h"
 #include "dc.h"
 #include "dc_dmub_srv.h"
 #include "../dmub/dmub_srv.h"
@@ -34,6 +35,7 @@
 #include "resource.h"
 #include "clk_mgr.h"
 #include "dc_state_priv.h"
+#include "dc_plane_priv.h"
 
 #define CTX dc_dmub_srv->ctx
 #define DC_LOGGER CTX->logger
@@ -198,6 +200,11 @@ bool dc_dmub_srv_wait_for_idle(struct dc_dmub_srv *dc_dmub_srv,
 
 		if (status != DMUB_STATUS_OK) {
 			DC_LOG_DEBUG("No reply for DMUB command: status=%d\n", status);
+			if (!dmub->debug.timeout_occured) {
+				dmub->debug.timeout_occured = true;
+				dmub->debug.timeout_cmd = *cmd_list;
+				dmub->debug.timestamp = dm_get_timestamp(dc_dmub_srv->ctx);
+			}
 			dc_dmub_srv_log_diagnostic_data(dc_dmub_srv);
 			return false;
 		}
@@ -904,12 +911,15 @@ bool dc_dmub_srv_get_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv, struct dmu
 void dc_dmub_srv_log_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv)
 {
 	struct dmub_diagnostic_data diag_data = {0};
+	uint32_t i;
 
 	if (!dc_dmub_srv || !dc_dmub_srv->dmub) {
 		DC_LOG_ERROR("%s: invalid parameters.", __func__);
 		return;
 	}
 
+	DC_LOG_ERROR("%s: DMCUB error - collecting diagnostic data\n", __func__);
+
 	if (!dc_dmub_srv_get_diagnostic_data(dc_dmub_srv, &diag_data)) {
 		DC_LOG_ERROR("%s: dc_dmub_srv_get_diagnostic_data failed.", __func__);
 		return;
@@ -933,7 +943,8 @@ void dc_dmub_srv_log_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv)
 	DC_LOG_DEBUG("    scratch [13]       : %08x", diag_data.scratch[13]);
 	DC_LOG_DEBUG("    scratch [14]       : %08x", diag_data.scratch[14]);
 	DC_LOG_DEBUG("    scratch [15]       : %08x", diag_data.scratch[15]);
-	DC_LOG_DEBUG("    pc                 : %08x", diag_data.pc);
+	for (i = 0; i < DMUB_PC_SNAPSHOT_COUNT; i++)
+		DC_LOG_DEBUG("    pc[%d]             : %08x", i, diag_data.pc[i]);
 	DC_LOG_DEBUG("    unk_fault_addr     : %08x", diag_data.undefined_address_fault_addr);
 	DC_LOG_DEBUG("    inst_fault_addr    : %08x", diag_data.inst_fetch_fault_addr);
 	DC_LOG_DEBUG("    data_fault_addr    : %08x", diag_data.data_write_fault_addr);
@@ -1199,8 +1210,23 @@ bool dc_dmub_srv_is_hw_pwr_up(struct dc_dmub_srv *dc_dmub_srv, bool wait)
 	return true;
 }
 
+static int count_active_streams(const struct dc *dc)
+{
+	int i, count = 0;
+
+	for (i = 0; i < dc->current_state->stream_count; ++i) {
+		struct dc_stream_state *stream = dc->current_state->streams[i];
+
+		if (stream && !stream->dpms_off)
+			count += 1;
+	}
+
+	return count;
+}
+
 static void dc_dmub_srv_notify_idle(const struct dc *dc, bool allow_idle)
 {
+	volatile const struct dmub_shared_state_ips_fw *ips_fw;
 	struct dc_dmub_srv *dc_dmub_srv;
 	union dmub_rb_cmd cmd = {0};
 
@@ -1211,6 +1237,7 @@ static void dc_dmub_srv_notify_idle(const struct dc *dc, bool allow_idle)
 		return;
 
 	dc_dmub_srv = dc->ctx->dmub_srv;
+	ips_fw = &dc_dmub_srv->dmub->shared_state[DMUB_SHARED_SHARE_FEATURE__IPS_FW].data.ips_fw;
 
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.idle_opt_notify_idle.header.type = DMUB_CMD__IDLE_OPT;
@@ -1226,6 +1253,12 @@ static void dc_dmub_srv_notify_idle(const struct dc *dc, bool allow_idle)
 			&dc_dmub_srv->dmub->shared_state[DMUB_SHARED_SHARE_FEATURE__IPS_DRIVER].data.ips_driver;
 		union dmub_shared_state_ips_driver_signals new_signals;
 
+		DC_LOG_IPS(
+			"%s wait idle (ips1_commit=%d ips2_commit=%d)",
+			__func__,
+			ips_fw->signals.bits.ips1_commit,
+			ips_fw->signals.bits.ips2_commit);
+
 		dc_dmub_srv_wait_idle(dc->ctx->dmub_srv);
 
 		memset(&new_signals, 0, sizeof(new_signals));
@@ -1245,19 +1278,46 @@ static void dc_dmub_srv_notify_idle(const struct dc *dc, bool allow_idle)
 			new_signals.bits.allow_pg = 1;
 			new_signals.bits.allow_ips1 = 1;
 			new_signals.bits.allow_ips2 = 1;
+		} else if (dc->config.disable_ips == DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF) {
+			/* TODO: Move this logic out to hwseq */
+			if (count_active_streams(dc) == 0) {
+				/* IPS2 - Display off */
+				new_signals.bits.allow_pg = 1;
+				new_signals.bits.allow_ips1 = 1;
+				new_signals.bits.allow_ips2 = 1;
+				new_signals.bits.allow_z10 = 1;
+			} else {
+				/* RCG only */
+				new_signals.bits.allow_pg = 0;
+				new_signals.bits.allow_ips1 = 1;
+				new_signals.bits.allow_ips2 = 0;
+				new_signals.bits.allow_z10 = 0;
+			}
 		}
 
 		ips_driver->signals = new_signals;
 	}
 
+	DC_LOG_IPS(
+		"%s send allow_idle=%d (ips1_commit=%d ips2_commit=%d)",
+		__func__,
+		allow_idle,
+		ips_fw->signals.bits.ips1_commit,
+		ips_fw->signals.bits.ips2_commit);
+
 	/* NOTE: This does not use the "wake" interface since this is part of the wake path. */
 	/* We also do not perform a wait since DMCUB could enter idle after the notification. */
 	dm_execute_dmub_cmd(dc->ctx, &cmd, allow_idle ? DM_DMUB_WAIT_TYPE_NO_WAIT : DM_DMUB_WAIT_TYPE_WAIT);
+
+	/* Register access should stop at this point. */
+	if (allow_idle)
+		dc_dmub_srv->needs_idle_wake = true;
 }
 
 static void dc_dmub_srv_exit_low_power_state(const struct dc *dc)
 {
 	struct dc_dmub_srv *dc_dmub_srv;
+	uint32_t rcg_exit_count = 0, ips1_exit_count = 0, ips2_exit_count = 0;
 
 	if (dc->debug.dmcub_emulation)
 		return;
@@ -1274,40 +1334,113 @@ static void dc_dmub_srv_exit_low_power_state(const struct dc *dc)
 			&dc_dmub_srv->dmub->shared_state[DMUB_SHARED_SHARE_FEATURE__IPS_DRIVER].data.ips_driver;
 		union dmub_shared_state_ips_driver_signals prev_driver_signals = ips_driver->signals;
 
+		rcg_exit_count = ips_fw->rcg_exit_count;
+		ips1_exit_count = ips_fw->ips1_exit_count;
+		ips2_exit_count = ips_fw->ips2_exit_count;
+
 		ips_driver->signals.all = 0;
 
-		if (prev_driver_signals.bits.allow_ips2) {
-			udelay(dc->debug.ips2_eval_delay_us);
+		DC_LOG_IPS(
+			"%s (allow ips1=%d ips2=%d) (commit ips1=%d ips2=%d) (count rcg=%d ips1=%d ips2=%d)",
+			__func__,
+			ips_driver->signals.bits.allow_ips1,
+			ips_driver->signals.bits.allow_ips2,
+			ips_fw->signals.bits.ips1_commit,
+			ips_fw->signals.bits.ips2_commit,
+			ips_fw->rcg_entry_count,
+			ips_fw->ips1_entry_count,
+			ips_fw->ips2_entry_count);
+
+		/* Note: register access has technically not resumed for DCN here, but we
+		 * need to be message PMFW through our standard register interface.
+		 */
+		dc_dmub_srv->needs_idle_wake = false;
+
+		if (prev_driver_signals.bits.allow_ips2 &&
+		    (!dc->debug.optimize_ips_handshake ||
+		     ips_fw->signals.bits.ips2_commit || !ips_fw->signals.bits.in_idle)) {
+			DC_LOG_IPS(
+				"wait IPS2 eval (ips1_commit=%d ips2_commit=%d)",
+				ips_fw->signals.bits.ips1_commit,
+				ips_fw->signals.bits.ips2_commit);
+
+			if (!dc->debug.optimize_ips_handshake || !ips_fw->signals.bits.ips2_commit)
+				udelay(dc->debug.ips2_eval_delay_us);
 
 			if (ips_fw->signals.bits.ips2_commit) {
+				DC_LOG_IPS(
+					"exit IPS2 #1 (ips1_commit=%d ips2_commit=%d)",
+					ips_fw->signals.bits.ips1_commit,
+					ips_fw->signals.bits.ips2_commit);
+
 				// Tell PMFW to exit low power state
 				dc->clk_mgr->funcs->exit_low_power_state(dc->clk_mgr);
 
+				DC_LOG_IPS(
+					"wait IPS2 entry delay (ips1_commit=%d ips2_commit=%d)",
+					ips_fw->signals.bits.ips1_commit,
+					ips_fw->signals.bits.ips2_commit);
+
 				// Wait for IPS2 entry upper bound
 				udelay(dc->debug.ips2_entry_delay_us);
 
+				DC_LOG_IPS(
+					"exit IPS2 #2 (ips1_commit=%d ips2_commit=%d)",
+					ips_fw->signals.bits.ips1_commit,
+					ips_fw->signals.bits.ips2_commit);
+
 				dc->clk_mgr->funcs->exit_low_power_state(dc->clk_mgr);
 
+				DC_LOG_IPS(
+					"wait IPS2 commit clear (ips1_commit=%d ips2_commit=%d)",
+					ips_fw->signals.bits.ips1_commit,
+					ips_fw->signals.bits.ips2_commit);
+
 				while (ips_fw->signals.bits.ips2_commit)
 					udelay(1);
 
+				DC_LOG_IPS(
+					"wait hw_pwr_up (ips1_commit=%d ips2_commit=%d)",
+					ips_fw->signals.bits.ips1_commit,
+					ips_fw->signals.bits.ips2_commit);
+
 				if (!dc_dmub_srv_is_hw_pwr_up(dc->ctx->dmub_srv, true))
 					ASSERT(0);
 
+				DC_LOG_IPS(
+					"resync inbox1 (ips1_commit=%d ips2_commit=%d)",
+					ips_fw->signals.bits.ips1_commit,
+					ips_fw->signals.bits.ips2_commit);
+
 				dmub_srv_sync_inbox1(dc->ctx->dmub_srv->dmub);
 			}
 		}
 
 		dc_dmub_srv_notify_idle(dc, false);
 		if (prev_driver_signals.bits.allow_ips1) {
+			DC_LOG_IPS(
+				"wait for IPS1 commit clear (ips1_commit=%d ips2_commit=%d)",
+				ips_fw->signals.bits.ips1_commit,
+				ips_fw->signals.bits.ips2_commit);
+
 			while (ips_fw->signals.bits.ips1_commit)
 				udelay(1);
 
+			DC_LOG_IPS(
+				"wait for IPS1 commit clear done (ips1_commit=%d ips2_commit=%d)",
+				ips_fw->signals.bits.ips1_commit,
+				ips_fw->signals.bits.ips2_commit);
 		}
 	}
 
 	if (!dc_dmub_srv_is_hw_pwr_up(dc->ctx->dmub_srv, true))
 		ASSERT(0);
+
+	DC_LOG_IPS("%s exit (count rcg=%d ips1=%d ips2=%d)",
+		__func__,
+		rcg_exit_count,
+		ips1_exit_count,
+		ips2_exit_count);
 }
 
 void dc_dmub_srv_set_power_state(struct dc_dmub_srv *dc_dmub_srv, enum dc_acpi_cm_power_state powerState)
@@ -1335,21 +1468,42 @@ void dc_dmub_srv_apply_idle_power_optimizations(const struct dc *dc, bool allow_
 	if (dc_dmub_srv->idle_allowed == allow_idle)
 		return;
 
+	DC_LOG_IPS("%s state change: old=%d new=%d", __func__, dc_dmub_srv->idle_allowed, allow_idle);
+
 	/*
 	 * Entering a low power state requires a driver notification.
 	 * Powering up the hardware requires notifying PMFW and DMCUB.
 	 * Clearing the driver idle allow requires a DMCUB command.
 	 * DMCUB commands requires the DMCUB to be powered up and restored.
-	 *
-	 * Exit out early to prevent an infinite loop of DMCUB commands
-	 * triggering exit low power - use software state to track this.
 	 */
-	dc_dmub_srv->idle_allowed = allow_idle;
 
-	if (!allow_idle)
+	if (!allow_idle) {
+		dc_dmub_srv->idle_exit_counter += 1;
+
 		dc_dmub_srv_exit_low_power_state(dc);
-	else
+		/*
+		 * Idle is considered fully exited only after the sequence above
+		 * fully completes. If we have a race of two threads exiting
+		 * at the same time then it's safe to perform the sequence
+		 * twice as long as we're not re-entering.
+		 *
+		 * Infinite command submission is avoided by using the
+		 * dm_execute_dmub_cmd submission instead of the "wake" helpers.
+		 */
+		dc_dmub_srv->idle_allowed = false;
+
+		dc_dmub_srv->idle_exit_counter -= 1;
+		if (dc_dmub_srv->idle_exit_counter < 0) {
+			ASSERT(0);
+			dc_dmub_srv->idle_exit_counter = 0;
+		}
+	} else {
+		/* Consider idle as notified prior to the actual submission to
+		 * prevent multiple entries. */
+		dc_dmub_srv->idle_allowed = true;
+
 		dc_dmub_srv_notify_idle(dc, allow_idle);
+	}
 }
 
 bool dc_wake_and_execute_dmub_cmd(const struct dc_context *ctx, union dmub_rb_cmd *cmd,
@@ -1384,7 +1538,8 @@ bool dc_wake_and_execute_dmub_cmd_list(const struct dc_context *ctx, unsigned in
 	else
 		result = dm_execute_dmub_cmd(ctx, cmd, wait_type);
 
-	if (result && reallow_idle && !ctx->dc->debug.disable_dmub_reallow_idle)
+	if (result && reallow_idle && dc_dmub_srv->idle_exit_counter == 0 &&
+	    !ctx->dc->debug.disable_dmub_reallow_idle)
 		dc_dmub_srv_apply_idle_power_optimizations(ctx->dc, true);
 
 	return result;
@@ -1433,8 +1588,10 @@ bool dc_wake_and_execute_gpint(const struct dc_context *ctx, enum dmub_gpint_com
 
 	result = dc_dmub_execute_gpint(ctx, command_code, param, response, wait_type);
 
-	if (result && reallow_idle && !ctx->dc->debug.disable_dmub_reallow_idle)
+	if (result && reallow_idle && dc_dmub_srv->idle_exit_counter == 0 &&
+	    !ctx->dc->debug.disable_dmub_reallow_idle)
 		dc_dmub_srv_apply_idle_power_optimizations(ctx->dc, true);
 
 	return result;
 }
+
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
index 952bfb368886..2c5866211f60 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
@@ -35,6 +35,7 @@ struct pipe_ctx;
 struct dc_crtc_timing_adjust;
 struct dc_crtc_timing;
 struct dc_state;
+struct dc_surface_update;
 
 struct dc_reg_helper_state {
 	bool gather_in_progress;
@@ -51,7 +52,9 @@ struct dc_dmub_srv {
 	struct dc_context *ctx;
 	void *dm;
 
+	int32_t idle_exit_counter;
 	bool idle_allowed;
+	bool needs_idle_wake;
 };
 
 void dc_dmub_srv_wait_idle(struct dc_dmub_srv *dc_dmub_srv);
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
index 1cb7765f593a..519c3df78ee5 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
@@ -137,8 +137,13 @@ enum dp_link_encoding {
 
 enum dp_test_link_rate {
 	DP_TEST_LINK_RATE_RBR		= 0x06,
+	DP_TEST_LINK_RATE_RATE_2    = 0x08,	// Rate_2        - 2.16 Gbps/Lane
+	DP_TEST_LINK_RATE_RATE_3    = 0x09,	// Rate_3        - 2.43 Gbps/Lane
 	DP_TEST_LINK_RATE_HBR		= 0x0A,
+	DP_TEST_LINK_RATE_RBR2      = 0x0C,	// Rate_5 (RBR2) - 3.24 Gbps/Lane
+	DP_TEST_LINK_RATE_RATE_6    = 0x10,	// Rate_6        - 4.32 Gbps/Lane
 	DP_TEST_LINK_RATE_HBR2		= 0x14,
+	DP_TEST_LINK_RATE_RATE_8    = 0x19,	// Rate_8        - 6.75 Gbps/Lane
 	DP_TEST_LINK_RATE_HBR3		= 0x1E,
 	DP_TEST_LINK_RATE_UHBR10	= 0x01,
 	DP_TEST_LINK_RATE_UHBR20	= 0x02,
@@ -917,16 +922,6 @@ struct dpcd_usb4_dp_tunneling_info {
 	uint8_t usb4_topology_id[DPCD_USB4_TOPOLOGY_ID_LEN];
 };
 
-#ifndef DP_DFP_CAPABILITY_EXTENSION_SUPPORT
-#define DP_DFP_CAPABILITY_EXTENSION_SUPPORT		0x0A3
-#endif
-#ifndef DP_TEST_264BIT_CUSTOM_PATTERN_7_0
-#define DP_TEST_264BIT_CUSTOM_PATTERN_7_0		0X2230
-#endif
-#ifndef DP_TEST_264BIT_CUSTOM_PATTERN_263_256
-#define DP_TEST_264BIT_CUSTOM_PATTERN_263_256		0X2250
-#endif
-
 union dp_main_line_channel_coding_cap {
 	struct {
 		uint8_t DP_8b_10b_SUPPORTED	:1;
@@ -1232,8 +1227,7 @@ union replay_enable_and_configuration {
 		unsigned char FREESYNC_PANEL_REPLAY_MODE              :1;
 		unsigned char TIMING_DESYNC_ERROR_VERIFICATION        :1;
 		unsigned char STATE_TRANSITION_ERROR_DETECTION        :1;
-		unsigned char RESERVED0                               :1;
-		unsigned char RESERVED1                               :4;
+		unsigned char RESERVED                                :5;
 	} bits;
 	unsigned char raw;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
index aae2f3a2660d..2ad7f60805f5 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
@@ -738,6 +738,13 @@ enum scanning_type {
 	SCANNING_TYPE_UNDEFINED
 };
 
+enum chroma_cositing {
+	CHROMA_COSITING_NONE,
+	CHROMA_COSITING_LEFT,
+	CHROMA_COSITING_TOPLEFT,
+	CHROMA_COSITING_COUNT
+};
+
 struct dc_crtc_timing_flags {
 	uint32_t INTERLACE :1;
 	uint32_t HSYNC_POSITIVE_POLARITY :1; /* when set to 1,
@@ -974,6 +981,7 @@ struct dc_crtc_timing_adjust {
 	uint32_t v_total_max;
 	uint32_t v_total_mid;
 	uint32_t v_total_mid_frame_num;
+	uint32_t allow_otg_v_count_halt;
 };
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc_plane.h b/drivers/gpu/drm/amd/display/dc/dc_plane.h
index ef380cae816a..44afcd989224 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_plane.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_plane.h
@@ -29,7 +29,7 @@
 #include "dc.h"
 #include "dc_hw_types.h"
 
-struct dc_plane_state *dc_create_plane_state(struct dc *dc);
+struct dc_plane_state *dc_create_plane_state(const struct dc *dc);
 const struct dc_plane_status *dc_plane_get_status(
 		const struct dc_plane_state *plane_state);
 void dc_plane_state_retain(struct dc_plane_state *plane_state);
diff --git a/drivers/gpu/drm/amd/display/dc/dc_plane_priv.h b/drivers/gpu/drm/amd/display/dc/dc_plane_priv.h
index 9ee184c1df00..ab13335f1d01 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_plane_priv.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_plane_priv.h
@@ -30,5 +30,6 @@
 
 void dc_plane_construct(struct dc_context *ctx, struct dc_plane_state *plane_state);
 void dc_plane_destruct(struct dc_plane_state *plane_state);
+uint8_t dc_plane_get_pipe_mask(struct dc_state *dc_state, const struct dc_plane_state *plane_state);
 
 #endif /* _DC_PLANE_PRIV_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_state.h b/drivers/gpu/drm/amd/display/dc/dc_state.h
index d167fdbfa8a9..caa45db50232 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_state.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_state.h
@@ -29,7 +29,7 @@
 #include "dc.h"
 #include "inc/core_status.h"
 
-struct dc_state *dc_state_create(struct dc *dc);
+struct dc_state *dc_state_create(struct dc *dc, struct dc_state_create_params *params);
 void dc_state_copy(struct dc_state *dst_state, struct dc_state *src_state);
 struct dc_state *dc_state_create_copy(struct dc_state *src_state);
 void dc_state_copy_current(struct dc *dc, struct dc_state *dst_state);
@@ -39,12 +39,12 @@ void dc_state_destruct(struct dc_state *state);
 void dc_state_retain(struct dc_state *state);
 void dc_state_release(struct dc_state *state);
 
-enum dc_status dc_state_add_stream(struct dc *dc,
+enum dc_status dc_state_add_stream(const struct dc *dc,
 				    struct dc_state *state,
 				    struct dc_stream_state *stream);
 
 enum dc_status dc_state_remove_stream(
-		struct dc *dc,
+		const struct dc *dc,
 		struct dc_state *state,
 		struct dc_stream_state *stream);
 
@@ -74,5 +74,5 @@ bool dc_state_add_all_planes_for_stream(
 
 struct dc_stream_status *dc_state_get_stream_status(
 	struct dc_state *state,
-	struct dc_stream_state *stream);
+	const struct dc_stream_state *stream);
 #endif /* _DC_STATE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_state_priv.h b/drivers/gpu/drm/amd/display/dc/dc_state_priv.h
index c1f44e09a6c1..615086d74d32 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_state_priv.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_state_priv.h
@@ -29,6 +29,8 @@
 #include "dc_state.h"
 #include "dc_stream.h"
 
+struct dc_stream_state *dc_state_get_stream_from_id(const struct dc_state *state, unsigned int id);
+
 /* Get the type of the provided resource (none, phantom, main) based on the provided
  * context. If the context is unavailable, determine only if phantom or not.
  */
@@ -45,7 +47,7 @@ struct dc_stream_state *dc_state_get_paired_subvp_stream(const struct dc_state *
 struct dc_stream_state *dc_state_create_phantom_stream(const struct dc *dc,
 		struct dc_state *state,
 		struct dc_stream_state *main_stream);
-struct dc_plane_state *dc_state_create_phantom_plane(struct dc *dc,
+struct dc_plane_state *dc_state_create_phantom_plane(const struct dc *dc,
 		struct dc_state *state,
 		struct dc_plane_state *main_plane);
 
@@ -58,11 +60,11 @@ void dc_state_release_phantom_plane(const struct dc *dc,
 		struct dc_plane_state *phantom_plane);
 
 /* add/remove phantom stream to context and generate subvp meta data */
-enum dc_status dc_state_add_phantom_stream(struct dc *dc,
+enum dc_status dc_state_add_phantom_stream(const struct dc *dc,
 		struct dc_state *state,
 		struct dc_stream_state *phantom_stream,
 		struct dc_stream_state *main_stream);
-enum dc_status dc_state_remove_phantom_stream(struct dc *dc,
+enum dc_status dc_state_remove_phantom_stream(const struct dc *dc,
 		struct dc_state *state,
 		struct dc_stream_state *phantom_stream);
 
@@ -92,11 +94,11 @@ bool dc_state_add_all_phantom_planes_for_stream(
 		struct dc_state *state);
 
 bool dc_state_remove_phantom_streams_and_planes(
-		struct dc *dc,
+		const struct dc *dc,
 		struct dc_state *state);
 
 void dc_state_release_phantom_streams_and_planes(
-		struct dc *dc,
+		const struct dc *dc,
 		struct dc_state *state);
 
 #endif /* _DC_STATE_PRIV_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
index ee10941caa59..e5dbbc6089a5 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_stream.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -190,7 +190,7 @@ struct dc_stream_state {
 	PHYSICAL_ADDRESS_LOC dmdata_address;
 	bool   use_dynamic_meta;
 
-	struct dc_transfer_func *out_transfer_func;
+	struct dc_transfer_func out_transfer_func;
 	struct colorspace_transform gamut_remap_matrix;
 	struct dc_csc_transform csc_color_matrix;
 
@@ -428,14 +428,6 @@ bool dc_stream_set_dynamic_metadata(struct dc *dc,
 enum dc_status dc_validate_stream(struct dc *dc, struct dc_stream_state *stream);
 
 /*
- * Set up streams and links associated to drive sinks
- * The streams parameter is an absolute set of all active streams.
- *
- * After this call:
- *   Phy, Encoder, Timing Generator are programmed and enabled.
- *   New streams are enabled with blank stream; no memory read.
- */
-/*
  * Enable stereo when commit_streams is not required,
  * for example, frame alternate.
  */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h
index be2ac5c442a4..0f66d00ef80f 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_types.h
@@ -422,7 +422,7 @@ struct dc_dwb_params {
 	enum dwb_capture_rate		capture_rate;	/* controls the frame capture rate */
 	struct scaling_taps 		scaler_taps;	/* Scaling taps */
 	enum dwb_subsample_position	subsample_position;
-	struct dc_transfer_func *out_transfer_func;
+	const struct dc_transfer_func *out_transfer_func;
 };
 
 /* audio*/
@@ -1050,6 +1050,8 @@ union replay_error_status {
 struct replay_config {
 	/* Replay feature is supported */
 	bool replay_supported;
+	/* Replay caps support DPCD & EDID caps*/
+	bool replay_cap_support;
 	/* Power opt flags that are supported */
 	unsigned int replay_power_opt_supported;
 	/* SMU optimization is supported */
@@ -1175,4 +1177,20 @@ enum mall_stream_type {
 	SUBVP_MAIN, // subvp in use, this stream is main stream
 	SUBVP_PHANTOM, // subvp in use, this stream is a phantom stream
 };
+
+enum dc_power_source_type {
+	DC_POWER_SOURCE_AC, // wall power
+	DC_POWER_SOURCE_DC, // battery power
+};
+
+struct dc_state_create_params {
+	enum dc_power_source_type power_source;
+};
+
+struct dc_commit_streams_params {
+	struct dc_stream_state **streams;
+	uint8_t stream_count;
+	enum dc_power_source_type power_source;
+};
+
 #endif /* DC_TYPES_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c
index a2f48d46d199..ee601a6897a1 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c
@@ -22,9 +22,6 @@
  * Authors: AMD
  *
  */
-
-#include <linux/delay.h>
-
 #include "resource.h"
 #include "dce_i2c.h"
 #include "dce_i2c_hw.h"
@@ -315,9 +312,6 @@ static bool setup_engine(
 	/* we have checked I2c not used by DMCU, set SW use I2C REQ to 1 to indicate SW using it*/
 	REG_UPDATE(DC_I2C_ARBITRATION, DC_I2C_SW_USE_I2C_REG_REQ, 1);
 
-	/* we have checked I2c not used by DMCU, set SW use I2C REQ to 1 to indicate SW using it*/
-	REG_UPDATE(DC_I2C_ARBITRATION, DC_I2C_SW_USE_I2C_REG_REQ, 1);
-
 	/*set SW requested I2c speed to default, if API calls in it will be override later*/
 	set_speed(dce_i2c_hw, dce_i2c_hw->ctx->dc->caps.i2c_speed_in_khz);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h
index f98400efdd9b..e34e445a4013 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_mem_input.h
@@ -181,6 +181,7 @@ struct dce_mem_input_registers {
 	SFB(blk, GRPH_ENABLE, GRPH_ENABLE, mask_sh),\
 	SFB(blk, GRPH_CONTROL, GRPH_DEPTH, mask_sh),\
 	SFB(blk, GRPH_CONTROL, GRPH_FORMAT, mask_sh),\
+	SFB(blk, GRPH_CONTROL, GRPH_NUM_BANKS, mask_sh),\
 	SFB(blk, GRPH_X_START, GRPH_X_START, mask_sh),\
 	SFB(blk, GRPH_Y_START, GRPH_Y_START, mask_sh),\
 	SFB(blk, GRPH_X_END, GRPH_X_END, mask_sh),\
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h
index bf1ffc3629c7..3d9be87aae45 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h
@@ -111,6 +111,7 @@ enum dce110_opp_reg_type {
 	OPP_SF(FMT_DITHER_RAND_R_SEED, FMT_RAND_R_SEED, mask_sh),\
 	OPP_SF(FMT_DITHER_RAND_G_SEED, FMT_RAND_G_SEED, mask_sh),\
 	OPP_SF(FMT_DITHER_RAND_B_SEED, FMT_RAND_B_SEED, mask_sh),\
+	OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_EN, mask_sh),\
 	OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_RESET, mask_sh),\
 	OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_OFFSET, mask_sh),\
 	OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_DEPTH, mask_sh),\
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
index 670d5ab9d998..2b1673d69ea8 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
@@ -1408,7 +1408,7 @@ void dce110_opp_set_csc_default(
 static void program_pwl(struct dce_transform *xfm_dce,
 			const struct pwl_params *params)
 {
-	int retval;
+	uint32_t retval;
 	uint8_t max_tries = 10;
 	uint8_t counter = 0;
 	uint32_t i = 0;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c
index f9d6a181164a..b851fc65f5b7 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c
@@ -34,11 +34,7 @@
 #include "reg_helper.h"
 #include "fixed31_32.h"
 
-#ifdef _WIN32
-#include "atombios.h"
-#else
 #include "atom.h"
-#endif
 
 #define TO_DMUB_ABM(abm)\
 	container_of(abm, struct dce_abm, base)
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c
index b010814706fe..4f559a025cf0 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c
@@ -244,7 +244,7 @@ static void dmub_replay_residency(struct dmub_replay *dmub, uint8_t panel_inst,
 	uint16_t param = (uint16_t)(panel_inst << 8);
 
 	if (is_alpm)
-		param |= REPLAY_RESIDENCY_MODE_ALPM;
+		param |= REPLAY_RESIDENCY_FIELD_MODE_ALPM;
 
 	if (is_start)
 		param |= REPLAY_RESIDENCY_ENABLE;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/Makefile b/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
index ae6a131be71b..8dc7938c36d8 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
@@ -24,9 +24,9 @@
 
 DCN10 = dcn10_ipp.o \
 		dcn10_hw_sequencer_debug.o \
-		dcn10_dpp.o dcn10_opp.o \
+		dcn10_opp.o \
 		dcn10_hubp.o dcn10_mpc.o \
-		dcn10_dpp_dscl.o dcn10_dpp_cm.o dcn10_cm_common.o \
+		dcn10_cm_common.o \
 		dcn10_hubbub.o dcn10_stream_encoder.o dcn10_link_encoder.o
 
 AMD_DAL_DCN10 = $(addprefix $(AMDDALPATH)/dc/dcn10/,$(DCN10))
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c
index b7e57aa27361..0b49362f71b0 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c
@@ -24,7 +24,7 @@
  */
 #include "dc.h"
 #include "reg_helper.h"
-#include "dcn10_dpp.h"
+#include "dcn10/dcn10_dpp.h"
 
 #include "dcn10_cm_common.h"
 #include "custom_float.h"
@@ -402,6 +402,11 @@ bool cm_helper_translate_curve_to_hw_format(struct dc_context *ctx,
 				i += increment) {
 			if (j == hw_points - 1)
 				break;
+			if (i >= TRANSFER_FUNC_POINTS) {
+				DC_LOG_ERROR("Index out of bounds: i=%d, TRANSFER_FUNC_POINTS=%d\n",
+					     i, TRANSFER_FUNC_POINTS);
+				return false;
+			}
 			rgb_resulted[j].red = output_tf->tf_pts.red[i];
 			rgb_resulted[j].green = output_tf->tf_pts.green[i];
 			rgb_resulted[j].blue = output_tf->tf_pts.blue[i];
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
index d51f1ce02874..6dd355a03033 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
@@ -130,7 +130,7 @@ bool hubbub1_verify_allow_pstate_change_high(
 	static unsigned int max_sampled_pstate_wait_us; /* data collection */
 	static bool forced_pstate_allow; /* help with revert wa */
 
-	unsigned int debug_data;
+	unsigned int debug_data = 0;
 	unsigned int i;
 
 	if (forced_pstate_allow) {
@@ -242,7 +242,7 @@ void hubbub1_wm_change_req_wa(struct hubbub *hubbub)
 
 bool hubbub1_program_urgent_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -356,7 +356,7 @@ bool hubbub1_program_urgent_watermarks(
 
 bool hubbub1_program_stutter_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -501,7 +501,7 @@ bool hubbub1_program_stutter_watermarks(
 
 bool hubbub1_program_pstate_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -582,7 +582,7 @@ bool hubbub1_program_pstate_watermarks(
 
 bool hubbub1_program_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
index 4201b7627030..d1f9e63944c8 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
@@ -409,7 +409,7 @@ struct dcn10_hubbub {
 	const struct dcn_hubbub_shift *shifts;
 	const struct dcn_hubbub_mask *masks;
 	unsigned int debug_test_index_pstate;
-	struct dcn_watermark_set watermarks;
+	union dcn_watermark_set watermarks;
 };
 
 void hubbub1_update_dchub(
@@ -423,7 +423,7 @@ void hubbub1_wm_change_req_wa(struct hubbub *hubbub);
 
 bool hubbub1_program_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 
@@ -446,17 +446,17 @@ void hubbub1_construct(struct hubbub *hubbub,
 
 bool hubbub1_program_urgent_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 bool hubbub1_program_stutter_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 bool hubbub1_program_pstate_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
index 09784222cc03..69119b2fdce2 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
@@ -692,6 +692,7 @@ struct dcn_hubp_state {
 	uint32_t primary_meta_addr_hi;
 	uint32_t uclk_pstate_force;
 	uint32_t hubp_cntl;
+	uint32_t flip_control;
 };
 
 struct dcn10_hubp {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c
index 9033b39e0e0c..c51b717e5622 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c
@@ -392,7 +392,7 @@ static unsigned int dcn10_get_mpcc_states(struct dc *dc, char *pBuf, unsigned in
 	remaining_buffer -= chars_printed;
 	pBuf += chars_printed;
 
-	for (i = 0; i < pool->pipe_count; i++) {
+	for (i = 0; i < pool->mpcc_count; i++) {
 		struct mpcc_state s = {0};
 
 		pool->mpc->funcs->read_mpcc_state(pool->mpc, i, &s);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
index 377f1ba1a81b..4d0eed7598b2 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
@@ -1439,7 +1439,6 @@ enum signal_type dcn10_get_dig_mode(
 	default:
 		return SIGNAL_TYPE_NONE;
 	}
-	return SIGNAL_TYPE_NONE;
 }
 
 void dcn10_link_encoder_get_max_link_cap(struct link_encoder *enc,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h
index d980e6bd6c66..b7a89c39f445 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h
@@ -167,7 +167,6 @@ struct dcn10_link_enc_registers {
 	uint32_t DIO_LINKD_CNTL;
 	uint32_t DIO_LINKE_CNTL;
 	uint32_t DIO_LINKF_CNTL;
-	uint32_t DIG_FIFO_CTRL0;
 	uint32_t DIO_CLK_CNTL;
 	uint32_t DIG_BE_CLK_CNTL;
 };
@@ -475,9 +474,6 @@ struct dcn10_link_enc_registers {
 	type HPO_DP_ENC_SEL;\
 	type HPO_HDMI_ENC_SEL
 
-#define DCN32_LINK_ENCODER_REG_FIELD_LIST(type) \
-	type DIG_FIFO_OUTPUT_PIXEL_MODE
-
 #define DCN35_LINK_ENCODER_REG_FIELD_LIST(type) \
 	type DIG_BE_ENABLE;\
 	type DIG_RB_SWITCH_EN;\
@@ -512,7 +508,6 @@ struct dcn10_link_enc_shift {
 	DCN20_LINK_ENCODER_REG_FIELD_LIST(uint8_t);
 	DCN30_LINK_ENCODER_REG_FIELD_LIST(uint8_t);
 	DCN31_LINK_ENCODER_REG_FIELD_LIST(uint8_t);
-	DCN32_LINK_ENCODER_REG_FIELD_LIST(uint8_t);
 	DCN35_LINK_ENCODER_REG_FIELD_LIST(uint8_t);
 };
 
@@ -521,7 +516,6 @@ struct dcn10_link_enc_mask {
 	DCN20_LINK_ENCODER_REG_FIELD_LIST(uint32_t);
 	DCN30_LINK_ENCODER_REG_FIELD_LIST(uint32_t);
 	DCN31_LINK_ENCODER_REG_FIELD_LIST(uint32_t);
-	DCN32_LINK_ENCODER_REG_FIELD_LIST(uint32_t);
 	DCN35_LINK_ENCODER_REG_FIELD_LIST(uint32_t);
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c
index 5838a11efd00..71e9288d60ed 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c
@@ -168,6 +168,10 @@ static void opp1_set_pixel_encoding(
 
 	case PIXEL_ENCODING_RGB:
 	case PIXEL_ENCODING_YCBCR444:
+		REG_UPDATE_3(FMT_CONTROL,
+				FMT_PIXEL_ENCODING, 0,
+				FMT_SUBSAMPLING_MODE, 0,
+				FMT_CBCR_BIT_REDUCTION_BYPASS, 0);
 		REG_UPDATE(FMT_CONTROL, FMT_PIXEL_ENCODING, 0);
 		break;
 	case PIXEL_ENCODING_YCBCR422:
@@ -177,7 +181,10 @@ static void opp1_set_pixel_encoding(
 				FMT_CBCR_BIT_REDUCTION_BYPASS, 0);
 		break;
 	case PIXEL_ENCODING_YCBCR420:
-		REG_UPDATE(FMT_CONTROL, FMT_PIXEL_ENCODING, 2);
+		REG_UPDATE_3(FMT_CONTROL,
+				FMT_PIXEL_ENCODING, 2,
+				FMT_SUBSAMPLING_MODE, 2,
+				FMT_CBCR_BIT_REDUCTION_BYPASS, 1);
 		break;
 	default:
 		break;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.h
index 2c0ecfa5a643..c87de68a509e 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.h
@@ -79,6 +79,8 @@
 	OPP_SF(FMT0_FMT_CONTROL, FMT_SPATIAL_DITHER_FRAME_COUNTER_MAX, mask_sh), \
 	OPP_SF(FMT0_FMT_CONTROL, FMT_SPATIAL_DITHER_FRAME_COUNTER_BIT_SWAP, mask_sh), \
 	OPP_SF(FMT0_FMT_CONTROL, FMT_PIXEL_ENCODING, mask_sh), \
+	OPP_SF(FMT0_FMT_CONTROL, FMT_SUBSAMPLING_MODE, mask_sh), \
+	OPP_SF(FMT0_FMT_CONTROL, FMT_CBCR_BIT_REDUCTION_BYPASS, mask_sh), \
 	OPP_SF(FMT0_FMT_CONTROL, FMT_STEREOSYNC_OVERRIDE, mask_sh), \
 	OPP_SF(FMT0_FMT_DITHER_RAND_R_SEED, FMT_RAND_R_SEED, mask_sh), \
 	OPP_SF(FMT0_FMT_DITHER_RAND_G_SEED, FMT_RAND_G_SEED, mask_sh), \
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h
index c429590f1298..1b96972b9d0f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h
@@ -127,7 +127,6 @@ struct dcn10_stream_enc_registers {
 	uint32_t AFMT_60958_1;
 	uint32_t AFMT_60958_2;
 	uint32_t DIG_FE_CNTL;
-	uint32_t DIG_FE_CNTL2;
 	uint32_t DIG_FIFO_STATUS;
 	uint32_t DP_MSE_RATE_CNTL;
 	uint32_t DP_MSE_RATE_UPDATE;
@@ -570,7 +569,7 @@ struct dcn10_stream_enc_registers {
 	type DP_SEC_GSP11_ENABLE;\
 	type DP_SEC_GSP11_LINE_NUM
 
-#define SE_REG_FIELD_LIST_DCN3_2(type) \
+#define SE_REG_FIELD_LIST_DCN3_1_COMMON(type) \
 	type DIG_FIFO_OUTPUT_PIXEL_MODE;\
 	type DP_PIXEL_PER_CYCLE_PROCESSING_MODE;\
 	type DIG_SYMCLK_FE_ON;\
@@ -599,7 +598,7 @@ struct dcn10_stream_encoder_shift {
 	uint8_t HDMI_ACP_SEND;
 	SE_REG_FIELD_LIST_DCN2_0(uint8_t);
 	SE_REG_FIELD_LIST_DCN3_0(uint8_t);
-	SE_REG_FIELD_LIST_DCN3_2(uint8_t);
+	SE_REG_FIELD_LIST_DCN3_1_COMMON(uint8_t);
 	SE_REG_FIELD_LIST_DCN3_5_COMMON(uint8_t);
 };
 
@@ -608,7 +607,7 @@ struct dcn10_stream_encoder_mask {
 	uint32_t HDMI_ACP_SEND;
 	SE_REG_FIELD_LIST_DCN2_0(uint32_t);
 	SE_REG_FIELD_LIST_DCN3_0(uint32_t);
-	SE_REG_FIELD_LIST_DCN3_2(uint32_t);
+	SE_REG_FIELD_LIST_DCN3_1_COMMON(uint32_t);
 	SE_REG_FIELD_LIST_DCN3_5_COMMON(uint32_t);
 };
 
@@ -667,9 +666,6 @@ void enc1_stream_encoder_send_immediate_sdp_message(
 void enc1_stream_encoder_stop_dp_info_packets(
 	struct stream_encoder *enc);
 
-void enc1_stream_encoder_reset_fifo(
-	struct stream_encoder *enc);
-
 void enc1_stream_encoder_dp_blank(
 	struct dc_link *link,
 	struct stream_encoder *enc);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/Makefile b/drivers/gpu/drm/amd/display/dc/dcn20/Makefile
index 3dae3943b056..9b6070c99794 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/Makefile
@@ -2,7 +2,7 @@
 #
 # Makefile for DCN.
 
-DCN20 = dcn20_dpp.o dcn20_dpp_cm.o dcn20_hubp.o \
+DCN20 = dcn20_hubp.o \
 		dcn20_mpc.o dcn20_opp.o dcn20_hubbub.o dcn20_mmhubbub.o \
 		dcn20_stream_encoder.o dcn20_link_encoder.o dcn20_dccg.o \
 		dcn20_vmid.o dcn20_dwb.o dcn20_dwb_scl.o
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.c
index f8667be57046..80779e85e2c5 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb.c
@@ -299,6 +299,17 @@ void dwb2_set_scaler(struct dwbc *dwbc, struct dc_dwb_params *params)
 		}
 	}
 
+
+	if (dwbc20->dwbc_mask->WBSCL_COEF_RAM_SEL) {
+		/* Swap double buffered coefficient set */
+		uint32_t wbscl_mode = REG_READ(WBSCL_MODE);
+		bool coef_ram_current = get_reg_field_value_ex(
+			wbscl_mode, dwbc20->dwbc_mask->WBSCL_COEF_RAM_SEL_CURRENT,
+			dwbc20->dwbc_shift->WBSCL_COEF_RAM_SEL_CURRENT);
+
+		REG_UPDATE(WBSCL_MODE, WBSCL_COEF_RAM_SEL, !coef_ram_current);
+	}
+
 }
 
 static const struct dwbc_funcs dcn20_dwbc_funcs = {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.c
index 6eebcb22e317..c6f859871d11 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.c
@@ -570,7 +570,7 @@ void hubbub2_get_dchub_ref_freq(struct hubbub *hubbub,
 
 static bool hubbub2_program_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.h
index 2f6146bf1d32..24a9c45988ed 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubbub.h
@@ -85,7 +85,7 @@ struct dcn20_hubbub {
 	const struct dcn_hubbub_shift *shifts;
 	const struct dcn_hubbub_mask *masks;
 	unsigned int debug_test_index_pstate;
-	struct dcn_watermark_set watermarks;
+	union dcn_watermark_set watermarks;
 	int num_vmid;
 	struct dcn20_vmid vmid[16];
 	unsigned int detile_buf_size;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c
index 89c3bf0fe0c9..6bba020ad6fb 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c
@@ -1331,6 +1331,12 @@ void hubp2_read_state(struct hubp *hubp)
 		SWATH_HEIGHT_C, &rq_regs->rq_regs_c.swath_height,
 		PTE_ROW_HEIGHT_LINEAR_C, &rq_regs->rq_regs_c.pte_row_height_linear);
 
+	if (REG(DCHUBP_CNTL))
+		s->hubp_cntl = REG_READ(DCHUBP_CNTL);
+
+	if (REG(DCSURF_FLIP_CONTROL))
+		s->flip_control = REG_READ(DCSURF_FLIP_CONTROL);
+
 }
 
 static void hubp2_validate_dml_output(struct hubp *hubp,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.h
index efa2adf4f83d..8da3084d933f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.h
@@ -147,7 +147,7 @@
 	uint32_t DCN_CUR1_TTU_CNTL1;\
 	uint32_t VMID_SETTINGS_0
 
-
+/*shared with dcn3.x*/
 #define DCN21_HUBP_REG_COMMON_VARIABLE_LIST \
 	DCN2_HUBP_REG_COMMON_VARIABLE_LIST; \
 	uint32_t FLIP_PARAMETERS_3;\
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.h
index b2b266953d18..c34e04cac9a0 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.h
@@ -147,7 +147,8 @@
 	LE_SF(DPCSTX0_DPCSTX_TX_CNTL, DPCS_TX_DATA_SWAP, mask_sh),\
 	LE_SF(DPCSTX0_DPCSTX_TX_CNTL, DPCS_TX_DATA_ORDER_INVERT, mask_sh),\
 	LE_SF(DPCSTX0_DPCSTX_TX_CNTL, DPCS_TX_FIFO_EN, mask_sh),\
-	LE_SF(DPCSTX0_DPCSTX_TX_CNTL, DPCS_TX_FIFO_RD_START_DELAY, mask_sh)
+	LE_SF(DPCSTX0_DPCSTX_TX_CNTL, DPCS_TX_FIFO_RD_START_DELAY, mask_sh),\
+	LE_SF(DPCSTX0_DPCSTX_DEBUG_CONFIG, DPCS_DBG_CBUS_DIS, mask_sh)
 
 #define DPCS_DCN2_MASK_SH_LIST(mask_sh)\
 	DPCS_MASK_SH_LIST(mask_sh),\
@@ -231,6 +232,8 @@
 	SRI(RDPCSTX_PHY_FUSE3, RDPCSTX, id), \
 	SRI(DPCSTX_TX_CLOCK_CNTL, DPCSTX, id), \
 	SRI(DPCSTX_TX_CNTL, DPCSTX, id), \
+	SRI(DPCSTX_DEBUG_CONFIG, DPCSTX, id), \
+	SRI(RDPCSTX_DEBUG_CONFIG, RDPCSTX, id), \
 	SR(RDPCSTX0_RDPCSTX_SCRATCH)
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.c
index 16b5ff208d14..ea73473b970a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mpc.c
@@ -395,9 +395,12 @@ static void mpc20_program_ogam_pwl(
 				MPCC_OGAM_LUT_DATA, rgb[i].delta_green_reg);
 		REG_SET(MPCC_OGAM_LUT_DATA[mpcc_id], 0,
 				MPCC_OGAM_LUT_DATA, rgb[i].delta_blue_reg);
-
 	}
 
+	REG_SEQ_SUBMIT();
+	PERF_TRACE();
+	REG_SEQ_WAIT_DONE();
+	PERF_TRACE();
 }
 
 static void apply_DEDCN20_305_wa(struct mpc *mpc, int mpcc_id,
@@ -501,11 +504,6 @@ void mpc2_assert_mpcc_idle_before_connect(struct mpc *mpc, int mpcc_id)
 		ASSERT(!mpc_disabled);
 		ASSERT(!mpc_idle);
 	}
-
-	REG_SEQ_SUBMIT();
-	PERF_TRACE();
-	REG_SEQ_WAIT_DONE();
-	PERF_TRACE();
 }
 
 static void mpc2_init_mpcc(struct mpcc *mpcc, int mpcc_inst)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/Makefile b/drivers/gpu/drm/amd/display/dc/dcn201/Makefile
index 2b0b4f32e13b..3880db59e457 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn201/Makefile
@@ -2,7 +2,7 @@
 #
 # Makefile for DCN.
 DCN201 = dcn201_hubbub.o\
-	dcn201_mpc.o dcn201_hubp.o dcn201_opp.o dcn201_dpp.o \
+	dcn201_mpc.o dcn201_hubp.o dcn201_opp.o \
 	dcn201_dccg.o dcn201_link_encoder.o
 
 AMD_DAL_DCN201 = $(addprefix $(AMDDALPATH)/dc/dcn201/,$(DCN201))
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hubbub.c
index 037d265431c6..63798132ed95 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hubbub.c
@@ -52,7 +52,7 @@
 
 static bool hubbub201_program_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -103,5 +103,5 @@ void hubbub201_construct(struct dcn20_hubbub *hubbub,
 	hubbub->masks = hubbub_mask;
 
 	hubbub->debug_test_index_pstate = 0xB;
-	hubbub->detile_buf_size = 164 * 1024;
+	hubbub->detile_buf_size = 164 * 1024; /* 164KB for DCN2.0 */
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hubp.c
index 35dd4bac242a..cd2bfcc51276 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hubp.c
@@ -77,6 +77,7 @@ static void hubp201_program_requestor(struct hubp *hubp,
 			MRQ_EXPANSION_MODE, rq_regs->mrq_expansion_mode,
 			CRQ_EXPANSION_MODE, rq_regs->crq_expansion_mode);
 
+	/* no need to program PTE */
 	REG_SET_5(DCHUBP_REQ_SIZE_CONFIG, 0,
 		CHUNK_SIZE, rq_regs->rq_regs_l.chunk_size,
 		MIN_CHUNK_SIZE, rq_regs->rq_regs_l.min_chunk_size,
@@ -99,6 +100,10 @@ static void hubp201_setup(
 		struct _vcs_dpi_display_rq_regs_st *rq_regs,
 		struct _vcs_dpi_display_pipe_dest_params_st *pipe_dest)
 {
+	/*
+	 * otg is locked when this func is called. Register are double buffered.
+	 * disable the requestors is not needed
+	 */
 	hubp2_vready_at_or_After_vsync(hubp, pipe_dest);
 	hubp201_program_requestor(hubp, rq_regs);
 	hubp201_program_deadline(hubp, dlg_attr, ttu_attr);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_link_encoder.h
index 8b95ef251332..be25e8dc0636 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_link_encoder.h
@@ -30,6 +30,10 @@
 
 #define DPCS_DCN201_MASK_SH_LIST(mask_sh)\
 	DPCS_MASK_SH_LIST(mask_sh),\
+	LE_SF(DPCSSYS_CR0_RAWLANE0_DIG_PCS_XF_RX_OVRD_IN_2, VCO_LD_VAL_OVRD, mask_sh),\
+	LE_SF(DPCSSYS_CR0_RAWLANE0_DIG_PCS_XF_RX_OVRD_IN_2, VCO_LD_VAL_OVRD_EN, mask_sh),\
+	LE_SF(DPCSSYS_CR0_RAWLANE0_DIG_PCS_XF_RX_OVRD_IN_3, REF_LD_VAL_OVRD, mask_sh),\
+	LE_SF(DPCSSYS_CR0_RAWLANE0_DIG_PCS_XF_RX_OVRD_IN_3, REF_LD_VAL_OVRD_EN, mask_sh),\
 	LE_SF(RDPCSTX0_RDPCSTX_PHY_CNTL2, RDPCS_PHY_DPALT_DISABLE_ACK, mask_sh),\
 	LE_SF(RDPCSTX0_RDPCSTX_PHY_CNTL2, RDPCS_PHY_DPALT_DISABLE, mask_sh),\
 	LE_SF(RDPCSTX0_RDPCSTX_PHY_CNTL2, RDPCS_PHY_DPALT_DP4, mask_sh),\
@@ -44,7 +48,15 @@
 	LE_SF(RDPCSTX0_RDPCSTX_PHY_CNTL11, RDPCS_PHY_DP_REF_CLK_EN, mask_sh)
 
 #define DPCS_DCN201_REG_LIST(id) \
-	DPCS_DCN2_CMN_REG_LIST(id)
+	DPCS_DCN2_CMN_REG_LIST(id), \
+	SRI_IX(RAWLANE0_DIG_PCS_XF_RX_OVRD_IN_2, DPCSSYS_CR, id), \
+	SRI_IX(RAWLANE0_DIG_PCS_XF_RX_OVRD_IN_3, DPCSSYS_CR, id), \
+	SRI_IX(RAWLANE1_DIG_PCS_XF_RX_OVRD_IN_2, DPCSSYS_CR, id), \
+	SRI_IX(RAWLANE1_DIG_PCS_XF_RX_OVRD_IN_3, DPCSSYS_CR, id), \
+	SRI_IX(RAWLANE2_DIG_PCS_XF_RX_OVRD_IN_2, DPCSSYS_CR, id), \
+	SRI_IX(RAWLANE2_DIG_PCS_XF_RX_OVRD_IN_3, DPCSSYS_CR, id), \
+	SRI_IX(RAWLANE3_DIG_PCS_XF_RX_OVRD_IN_2, DPCSSYS_CR, id), \
+	SRI_IX(RAWLANE3_DIG_PCS_XF_RX_OVRD_IN_3, DPCSSYS_CR, id)
 
 void dcn201_link_encoder_construct(
 	struct dcn20_link_encoder *enc20,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.c
index aeb0e0d9b70a..2546224b326a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.c
@@ -140,7 +140,7 @@ int hubbub21_init_dchub(struct hubbub *hubbub,
 
 bool hubbub21_program_urgent_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -334,7 +334,7 @@ bool hubbub21_program_urgent_watermarks(
 
 bool hubbub21_program_stutter_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -487,7 +487,7 @@ bool hubbub21_program_stutter_watermarks(
 
 bool hubbub21_program_pstate_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -573,7 +573,7 @@ bool hubbub21_program_pstate_watermarks(
 
 bool hubbub21_program_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.h
index d8eb2bb7282c..ab2ce0313529 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.h
@@ -127,22 +127,22 @@ int hubbub21_init_dchub(struct hubbub *hubbub,
 		struct dcn_hubbub_phys_addr_config *pa_config);
 bool hubbub21_program_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 bool hubbub21_program_urgent_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 bool hubbub21_program_stutter_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 bool hubbub21_program_pstate_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/Makefile b/drivers/gpu/drm/amd/display/dc/dcn30/Makefile
index b5b2aa3b3783..c6ca70f3c061 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/Makefile
@@ -25,13 +25,11 @@
 
 DCN30 := dcn30_hubbub.o \
 	dcn30_hubp.o \
-	dcn30_dpp.o \
 	dcn30_dccg.o \
 	dcn30_mpc.o dcn30_vpg.o \
 	dcn30_afmt.o \
 	dcn30_dio_stream_encoder.o \
 	dcn30_dwb.o \
-	dcn30_dpp_cm.o \
 	dcn30_dwb_cm.o \
 	dcn30_cm_common.o \
 	dcn30_mmhubbub.o \
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c
index ddb344056d40..b8327237ed44 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c
@@ -26,7 +26,7 @@
 #include "dm_services.h"
 #include "core_types.h"
 #include "reg_helper.h"
-#include "dcn30_dpp.h"
+#include "dcn30/dcn30_dpp.h"
 #include "basics/conversion.h"
 #include "dcn30_cm_common.h"
 #include "custom_float.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dccg.h
index 35a613bb08bf..3f1da7f3a91c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dccg.h
@@ -29,15 +29,9 @@
 #include "dcn20/dcn20_dccg.h"
 
 
-#define DCCG_REG_LIST_DCN3AG() \
-	DCCG_COMMON_REG_LIST_DCN_BASE(),\
-	SR(PHYASYMCLK_CLOCK_CNTL),\
-	SR(PHYBSYMCLK_CLOCK_CNTL),\
-	SR(PHYCSYMCLK_CLOCK_CNTL)
-
-
 #define DCCG_REG_LIST_DCN30() \
 	DCCG_REG_LIST_DCN2(),\
+	DCCG_SRII(CLOCK_CNTL, HDMICHARCLK, 0),\
 	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 2),\
 	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 3),\
 	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 4),\
@@ -46,19 +40,10 @@
 	SR(PHYBSYMCLK_CLOCK_CNTL),\
 	SR(PHYCSYMCLK_CLOCK_CNTL)
 
-#define DCCG_MASK_SH_LIST_DCN3AG(mask_sh) \
-	DCCG_MASK_SH_LIST_DCN2_1(mask_sh),\
-	DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_EN, mask_sh),\
-	DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_SRC_SEL, mask_sh),\
-	DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_EN, mask_sh),\
-	DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_SRC_SEL, mask_sh),\
-	DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_FORCE_EN, mask_sh),\
-	DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_FORCE_SRC_SEL, mask_sh),\
-	DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_FORCE_EN, mask_sh),\
-	DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_FORCE_SRC_SEL, mask_sh)
-
 #define DCCG_MASK_SH_LIST_DCN3(mask_sh) \
 	DCCG_MASK_SH_LIST_DCN2(mask_sh),\
+	DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_EN, mask_sh),\
+	DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_SRC_SEL, mask_sh),\
 	DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_EN, mask_sh),\
 	DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_SRC_SEL, mask_sh),\
 	DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_FORCE_EN, mask_sh),\
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_link_encoder.c
index 1fb8fd7afc95..b8e31b5ea114 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_link_encoder.c
@@ -30,8 +30,6 @@
 #include "dcn30_dio_link_encoder.h"
 #include "stream_encoder.h"
 #include "dc_bios_types.h"
-/* #include "dcn3ag/dcn3ag_phy_fw.h" */
-
 #include "gpio_service_interface.h"
 
 #define CTX \
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_link_encoder.h
index f2d90f2b8bf1..5b6177c2ae98 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_link_encoder.h
@@ -55,7 +55,8 @@
 	SRI(DP_DPHY_HBR2_PATTERN_CONTROL, DP, id)
 
 #define LINK_ENCODER_MASK_SH_LIST_DCN30(mask_sh) \
-	LINK_ENCODER_MASK_SH_LIST_DCN20(mask_sh)
+	LINK_ENCODER_MASK_SH_LIST_DCN20(mask_sh),\
+	LE_SF(DIG0_TMDS_DCBALANCER_CONTROL, TMDS_SYNC_DCBAL_EN, mask_sh)
 
 #define DPCS_DCN3_MASK_SH_LIST(mask_sh)\
 	DPCS_DCN2_MASK_SH_LIST(mask_sh),\
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_stream_encoder.c
index 005dbe099a7a..425b830b88d2 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_stream_encoder.c
@@ -29,9 +29,6 @@
 #include "reg_helper.h"
 #include "hw_shared.h"
 #include "dc.h"
-#include "core_types.h"
-#include <linux/delay.h>
-
 
 #define DC_LOGGER \
 		enc1->base.ctx->logger
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb.c
index 1b9d9495f76d..fae98cf52020 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb.c
@@ -251,9 +251,7 @@ static const struct dwbc_funcs dcn30_dwbc_funcs = {
 	.set_fc_enable		= dwb3_set_fc_enable,
 	.set_stereo		= dwb3_set_stereo,
 	.set_new_content	= dwb3_set_new_content,
-	.dwb_program_output_csc	= NULL,
 	.dwb_ogam_set_input_transfer_func	= dwb3_ogam_set_input_transfer_func, //TODO: rename
-	.dwb_set_scaler		= NULL,
 };
 
 void dcn30_dwbc_construct(struct dcn30_dwbc *dwbc30,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb.h
index 332634b76aac..0f3f7c5fbaec 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dwb.h
@@ -217,6 +217,7 @@
 	SF_DWB2(DWB_OGAM_LUT_DATA, DWBCP, 0, DWB_OGAM_LUT_DATA, mask_sh),\
 	SF_DWB2(DWB_OGAM_LUT_CONTROL, DWBCP, 0, DWB_OGAM_LUT_WRITE_COLOR_MASK, mask_sh),\
 	SF_DWB2(DWB_OGAM_LUT_CONTROL, DWBCP, 0, DWB_OGAM_LUT_READ_COLOR_SEL, mask_sh),\
+	SF_DWB2(DWB_OGAM_LUT_CONTROL, DWBCP, 0, DWB_OGAM_LUT_READ_DBG, mask_sh),\
 	SF_DWB2(DWB_OGAM_LUT_CONTROL, DWBCP, 0, DWB_OGAM_LUT_HOST_SEL, mask_sh),\
 	SF_DWB2(DWB_OGAM_LUT_CONTROL, DWBCP, 0, DWB_OGAM_LUT_CONFIG_MODE, mask_sh),\
 	SF_DWB2(DWB_OGAM_RAMA_START_CNTL_B, DWBCP, 0, DWB_OGAM_RAMA_EXP_REGION_START_B, mask_sh),\
@@ -524,6 +525,7 @@
 	type DWB_OGAM_LUT_DATA;\
 	type DWB_OGAM_LUT_WRITE_COLOR_MASK;\
 	type DWB_OGAM_LUT_READ_COLOR_SEL;\
+	type DWB_OGAM_LUT_READ_DBG;\
 	type DWB_OGAM_LUT_HOST_SEL;\
 	type DWB_OGAM_LUT_CONFIG_MODE;\
 	type DWB_OGAM_LUT_STATUS;\
@@ -710,7 +712,7 @@
 	type DWB_OGAM_RAMB_EXP_REGION32_LUT_OFFSET;\
 	type DWB_OGAM_RAMB_EXP_REGION32_NUM_SEGMENTS;\
 	type DWB_OGAM_RAMB_EXP_REGION33_LUT_OFFSET;\
-	type DWB_OGAM_RAMB_EXP_REGION33_NUM_SEGMENTS;
+	type DWB_OGAM_RAMB_EXP_REGION33_NUM_SEGMENTS
 
 struct dcn30_dwbc_registers {
 	/* DCN3AG */
@@ -733,6 +735,10 @@ struct dcn30_dwbc_registers {
 	uint32_t DWB_MMHUBBUB_BACKPRESSURE_CNT;
 	uint32_t DWB_HOST_READ_CONTROL;
 	uint32_t DWB_SOFT_RESET;
+	uint32_t DWB_DEBUG_CTRL;
+	uint32_t DWB_DEBUG;
+	uint32_t DWB_TEST_DEBUG_INDEX;
+	uint32_t DWB_TEST_DEBUG_DATA;
 
 	/* DWBSCL */
 	uint32_t DWBSCL_COEF_RAM_TAP_SELECT;
@@ -747,6 +753,9 @@ struct dcn30_dwbc_registers {
 	uint32_t DWBSCL_DEST_SIZE;
 	uint32_t DWBSCL_OVERFLOW_STATUS;
 	uint32_t DWBSCL_OVERFLOW_COUNTER;
+	uint32_t DWBSCL_DEBUG;
+	uint32_t DWBSCL_TEST_DEBUG_INDEX;
+	uint32_t DWBSCL_TEST_DEBUG_DATA;
 
 	/* DWBCP */
 	uint32_t DWB_HDR_MULT_COEF;
@@ -838,6 +847,9 @@ struct dcn30_dwbc_registers {
 	uint32_t DWB_OGAM_RAMB_REGION_28_29;
 	uint32_t DWB_OGAM_RAMB_REGION_30_31;
 	uint32_t DWB_OGAM_RAMB_REGION_32_33;
+	uint32_t DWBCP_DEBUG;
+	uint32_t DWBCP_TEST_DEBUG_INDEX;
+	uint32_t DWBCP_TEST_DEBUG_DATA;
 };
 
 /* Internal enums / structs */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubbub.c
index 152c9c5733f1..6a5af3da4b45 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubbub.c
@@ -95,7 +95,7 @@ int hubbub3_init_dchub_sys_ctx(struct hubbub *hubbub,
 
 bool hubbub3_program_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubbub.h
index 7b597908b937..ca6233e8f1f4 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubbub.h
@@ -124,7 +124,7 @@ bool hubbub3_get_dcc_compression_cap(struct hubbub *hubbub,
 
 bool hubbub3_program_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.c
index 75547ce86c09..60a64d290352 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.c
@@ -455,6 +455,9 @@ void hubp3_read_state(struct hubp *hubp)
 	if (REG(DCHUBP_CNTL))
 		s->hubp_cntl = REG_READ(DCHUBP_CNTL);
 
+	if (REG(DCSURF_FLIP_CONTROL))
+		s->flip_control = REG_READ(DCSURF_FLIP_CONTROL);
+
 }
 
 void hubp3_setup(
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c
index 5ebb57303130..fca94e50ae93 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c
@@ -1183,7 +1183,7 @@ void mpc3_get_gamut_remap(struct mpc *mpc,
 			  struct mpc_grph_gamut_adjustment *adjust)
 {
 	struct dcn30_mpc *mpc30 = TO_DCN30_MPC(mpc);
-	uint16_t arr_reg_val[12];
+	uint16_t arr_reg_val[12] = {0};
 	int select;
 
 	read_gamut_remap(mpc30, mpcc_id, arr_reg_val, &select);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_vpg.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_vpg.h
index ed9a5549c389..466ba20b9c61 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_vpg.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_vpg.h
@@ -26,6 +26,7 @@
 #ifndef __DAL_DCN30_VPG_H__
 #define __DAL_DCN30_VPG_H__
 
+#include "vpg.h"
 
 #define DCN30_VPG_FROM_VPG(vpg)\
 	container_of(vpg, struct dcn30_vpg, base)
@@ -132,28 +133,6 @@ struct dcn30_vpg_mask {
 	VPG_DCN3_REG_FIELD_LIST(uint32_t);
 };
 
-struct vpg;
-
-struct vpg_funcs {
-	void (*update_generic_info_packet)(
-		struct vpg *vpg,
-		uint32_t packet_index,
-		const struct dc_info_packet *info_packet,
-		bool immediate_update);
-
-	void (*vpg_poweron)(
-		struct vpg *vpg);
-
-	void (*vpg_powerdown)(
-		struct vpg *vpg);
-};
-
-struct vpg {
-	const struct vpg_funcs *funcs;
-	struct dc_context *ctx;
-	int inst;
-};
-
 struct dcn30_vpg {
 	struct vpg base;
 	const struct dcn30_vpg_registers *regs;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_dccg.h
index 73db962dbc03..067e49cb238e 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_dccg.h
@@ -56,10 +56,4 @@ struct dccg *dccg301_create(
 	const struct dccg_shift *dccg_shift,
 	const struct dccg_mask *dccg_mask);
 
-struct dccg *dccg301_create(
-	struct dc_context *ctx,
-	const struct dccg_registers *regs,
-	const struct dccg_shift *dccg_shift,
-	const struct dccg_mask *dccg_mask);
-
 #endif //__DCN301_DCCG_H__
diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_hubbub.c
index a046664e2031..c1959672df50 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_hubbub.c
@@ -63,6 +63,7 @@ static const struct hubbub_funcs hubbub301_funcs = {
 	.verify_allow_pstate_change_high = hubbub1_verify_allow_pstate_change_high,
 	.force_wm_propagate_to_pipes = hubbub3_force_wm_propagate_to_pipes,
 	.force_pstate_change_control = hubbub3_force_pstate_change_control,
+	.init_watermarks = hubbub3_init_watermarks,
 	.hubbub_read_state = hubbub2_read_state,
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dccg.h
index e3caaacf7493..e3be0bab4007 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dccg.h
@@ -34,12 +34,14 @@
 	DCCG_SRII(DTO_PARAM, DPPCLK, 1),\
 	DCCG_SRII(DTO_PARAM, DPPCLK, 2),\
 	DCCG_SRII(DTO_PARAM, DPPCLK, 3),\
+	DCCG_SRII(CLOCK_CNTL, HDMICHARCLK, 0),\
 	SR(PHYASYMCLK_CLOCK_CNTL),\
 	SR(PHYBSYMCLK_CLOCK_CNTL),\
 	SR(PHYCSYMCLK_CLOCK_CNTL),\
 	SR(PHYDSYMCLK_CLOCK_CNTL),\
 	SR(PHYESYMCLK_CLOCK_CNTL),\
 	SR(DPSTREAMCLK_CNTL),\
+	SR(HDMISTREAMCLK_CNTL),\
 	SR(SYMCLK32_SE_CNTL),\
 	SR(SYMCLK32_LE_CNTL),\
 	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 0),\
@@ -78,6 +80,8 @@
 	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 3, mask_sh),\
 	DCCG_SF(DPPCLK0_DTO_PARAM, DPPCLK0_DTO_PHASE, mask_sh),\
 	DCCG_SF(DPPCLK0_DTO_PARAM, DPPCLK0_DTO_MODULO, mask_sh),\
+	DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_EN, mask_sh),\
+	DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_SRC_SEL, mask_sh),\
 	DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_EN, mask_sh),\
 	DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_SRC_SEL, mask_sh),\
 	DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_FORCE_EN, mask_sh),\
@@ -92,6 +96,8 @@
 	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK_PIPE1_EN, mask_sh),\
 	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK_PIPE2_EN, mask_sh),\
 	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK_PIPE3_EN, mask_sh),\
+	DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_SRC_SEL, mask_sh),\
+	DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_DTO_FORCE_DIS, mask_sh),\
 	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE0_SRC_SEL, mask_sh),\
 	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE1_SRC_SEL, mask_sh),\
 	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE2_SRC_SEL, mask_sh),\
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dio_link_encoder.c
index 26be5fee7411..b2cea59ba5d4 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dio_link_encoder.c
@@ -205,7 +205,7 @@ void dcn31_link_encoder_set_dio_phy_mux(
 	}
 }
 
-static void enc31_hw_init(struct link_encoder *enc)
+void enc31_hw_init(struct link_encoder *enc)
 {
 	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dio_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dio_link_encoder.h
index 221671563a0b..ee78ba80797c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dio_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_dio_link_encoder.h
@@ -89,6 +89,7 @@
 	SRI(RDPCSTX_PHY_FUSE1, RDPCSTX, id), \
 	SRI(RDPCSTX_PHY_FUSE2, RDPCSTX, id), \
 	SRI(RDPCSTX_PHY_FUSE3, RDPCSTX, id), \
+	SRI(RDPCSTX_DEBUG_CONFIG, RDPCSTX, id), \
 	SR(RDPCSTX0_RDPCSTX_SCRATCH), \
 	SRI(RDPCSTX_PHY_RX_LD_VAL, RDPCSTX, id),\
 	SRI(RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG, RDPCSTX, id)
@@ -222,6 +223,7 @@
 	SRI(RDPCSTX_PHY_FUSE1, RDPCSTX, id), \
 	SRI(RDPCSTX_PHY_FUSE2, RDPCSTX, id), \
 	SRI(RDPCSTX_PHY_FUSE3, RDPCSTX, id), \
+	SRI(RDPCSTX_DEBUG_CONFIG, RDPCSTX, id), \
 	SR(RDPCSTX0_RDPCSTX_SCRATCH), \
 	SRI(RDPCSTX_PHY_RX_LD_VAL, RDPCSTX, id),\
 	SRI(RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG, RDPCSTX, id)
@@ -283,4 +285,6 @@ bool dcn31_link_encoder_is_in_alt_mode(
 void dcn31_link_encoder_get_max_link_cap(struct link_encoder *enc,
 	struct dc_link_settings *link_settings);
 
+void enc31_hw_init(struct link_encoder *enc);
+
 #endif /* __DC_LINK_ENCODER__DCN31_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_link_encoder.c
index 5b7ad38f85e0..03b4ac2f1991 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_link_encoder.c
@@ -377,7 +377,7 @@ void dcn31_hpo_dp_link_enc_update_stream_allocation_table(
 	 */
 	REG_WAIT(DP_DPHY_SYM32_STATUS,
 			SAT_UPDATE_PENDING, 0,
-			10, DP_SAT_UPDATE_MAX_RETRY);
+			100, DP_SAT_UPDATE_MAX_RETRY);
 }
 
 void dcn31_hpo_dp_link_enc_set_throttled_vcp_size(
@@ -395,6 +395,12 @@ void dcn31_hpo_dp_link_enc_set_throttled_vcp_size(
 				x),
 			25));
 
+	// If y rounds up to integer, carry it over to x.
+	if (y >> 25) {
+		x += 1;
+		y = 0;
+	}
+
 	switch (stream_encoder_inst) {
 	case 0:
 		REG_SET_2(DP_DPHY_SYM32_VC_RATE_CNTL0, 0,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c
index 45143459eedd..678db949cfe3 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c
@@ -474,6 +474,10 @@ static void dcn31_hpo_dp_stream_enc_update_dp_info_packets(
 				&info_frame->hdrsmd,
 				true);
 
+	/* packetIndex 4 is used for send immediate sdp message, and please
+	 * use other packetIndex (such as 5,6) for other info packet
+	 */
+
 	if (info_frame->adaptive_sync.valid)
 		enc->vpg->funcs->update_generic_info_packet(
 				enc->vpg,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c
index 5b5b5e0775fa..b906db6e7355 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c
@@ -172,7 +172,7 @@ static uint32_t convert_and_clamp(
 
 static bool hubbub31_program_urgent_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -362,7 +362,7 @@ static bool hubbub31_program_urgent_watermarks(
 
 static bool hubbub31_program_stutter_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -635,7 +635,7 @@ static bool hubbub31_program_stutter_watermarks(
 
 static bool hubbub31_program_pstate_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -717,7 +717,7 @@ static bool hubbub31_program_pstate_watermarks(
 
 static bool hubbub31_program_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_panel_cntl.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_panel_cntl.c
index 281be20b1a10..20c6fe48567f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_panel_cntl.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_panel_cntl.c
@@ -173,5 +173,12 @@ void dcn31_panel_cntl_construct(
 		break;
 	}
 
-	dcn31_panel_cntl->base.pwrseq_inst = pwrseq_inst;
+	if (dcn31_panel_cntl->base.ctx->dc->config.support_edp0_on_dp1)
+		//If supported, power sequencer mapping shall follow the DIG instance
+		dcn31_panel_cntl->base.pwrseq_inst = pwrseq_inst;
+	else
+		/* If not supported, pwrseq will be assigned in order,
+		 * so first pwrseq will be assigned to first panel instance (legacy behavior)
+		 */
+		dcn31_panel_cntl->base.pwrseq_inst = dcn31_panel_cntl->base.inst;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_vpg.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_vpg.c
index f1deb1c3c363..cfb923d85630 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_vpg.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_vpg.c
@@ -63,7 +63,12 @@ void vpg31_poweron(struct vpg *vpg)
 {
 	struct dcn31_vpg *vpg31 = DCN31_VPG_FROM_VPG(vpg);
 
-	if (vpg->ctx->dc->debug.enable_mem_low_power.bits.vpg == false)
+	uint32_t vpg_gsp_mem_pwr_state;
+
+	REG_GET(VPG_MEM_PWR, VPG_GSP_MEM_PWR_STATE, &vpg_gsp_mem_pwr_state);
+
+	if (vpg->ctx->dc->debug.enable_mem_low_power.bits.vpg == false &&
+			vpg_gsp_mem_pwr_state == 0)
 		return;
 
 	REG_UPDATE_2(VPG_MEM_PWR, VPG_GSP_MEM_LIGHT_SLEEP_DIS, 1, VPG_GSP_LIGHT_SLEEP_FORCE, 0);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_vpg.h b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_vpg.h
index 0e76eabce441..609e58dbd056 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_vpg.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_vpg.h
@@ -26,6 +26,7 @@
 #ifndef __DAL_DCN31_VPG_H__
 #define __DAL_DCN31_VPG_H__
 
+#include "vpg.h"
 
 #define DCN31_VPG_FROM_VPG(vpg)\
 	container_of(vpg, struct dcn31_vpg, base)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/Makefile b/drivers/gpu/drm/amd/display/dc/dcn32/Makefile
index 5314770fff1c..a58c37165f5a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/Makefile
@@ -11,7 +11,7 @@
 # Makefile for dcn32.
 
 DCN32 = dcn32_hubbub.o dcn32_dccg.o \
-		dcn32_mmhubbub.o dcn32_dpp.o dcn32_hubp.o dcn32_mpc.o \
+		dcn32_mmhubbub.o dcn32_hubp.o dcn32_mpc.o \
 		dcn32_dio_stream_encoder.o dcn32_dio_link_encoder.o dcn32_resource_helpers.o \
 		dcn32_hpo_dp_link_encoder.o
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.c
index e224a028d68a..d9ff95cd2dbd 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.c
@@ -251,8 +251,6 @@ void dcn32_link_encoder_construct(
 	enc10->base.preferred_engine = ENGINE_ID_UNKNOWN;
 
 	enc10->base.features = *enc_features;
-	if (enc10->base.connector.id == CONNECTOR_ID_USBC)
-		enc10->base.features.flags.bits.DP_IS_USB_C = 1;
 
 	if (enc10->base.connector.id == CONNECTOR_ID_USBC)
 		enc10->base.features.flags.bits.DP_IS_USB_C = 1;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.h
index 2d5f25290ed1..35d23d9db45e 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.h
@@ -26,15 +26,7 @@
 #ifndef __DC_LINK_ENCODER__DCN32_H__
 #define __DC_LINK_ENCODER__DCN32_H__
 
-#include "dcn31/dcn31_dio_link_encoder.h"
-
-#define LE_DCN32_REG_LIST(id)\
-	LE_DCN31_REG_LIST(id),\
-	SRI(DIG_FIFO_CTRL0, DIG, id)
-
-#define LINK_ENCODER_MASK_SH_LIST_DCN32(mask_sh) \
-	LINK_ENCODER_MASK_SH_LIST_DCN31(mask_sh),\
-	LE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_OUTPUT_PIXEL_MODE, mask_sh)
+#include "dcn30/dcn30_dio_link_encoder.h"
 
 void dcn32_link_encoder_construct(
 	struct dcn20_link_encoder *enc20,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.h
index 1be5410cce97..ca53d39561d2 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.h
@@ -177,11 +177,12 @@
 	SE_SF(DIG0_DIG_FE_CNTL, DIG_SYMCLK_FE_ON, mask_sh),\
 	SE_SF(DP0_DP_SEC_FRAMING4, DP_SST_SDP_SPLITTING, mask_sh),\
 	SE_SF(DIG0_DIG_CLOCK_PATTERN, DIG_CLOCK_PATTERN, mask_sh),\
+	SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_OUTPUT_PIXEL_MODE, mask_sh),\
 	SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_READ_START_LEVEL, mask_sh),\
 	SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_ENABLE, mask_sh),\
 	SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_RESET, mask_sh),\
-	SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, mask_sh),\
-	SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_OUTPUT_PIXEL_MODE, mask_sh)
+	SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, mask_sh)
+
 
 void dcn32_dio_stream_encoder_construct(
 	struct dcn10_stream_encoder *enc1,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c
index 88dfc907553d..515c4c2b4c21 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c
@@ -167,7 +167,7 @@ static uint32_t convert_and_clamp(
 
 bool hubbub32_program_urgent_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -357,7 +357,7 @@ bool hubbub32_program_urgent_watermarks(
 
 bool hubbub32_program_stutter_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -503,7 +503,7 @@ bool hubbub32_program_stutter_watermarks(
 
 bool hubbub32_program_pstate_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -656,7 +656,7 @@ bool hubbub32_program_pstate_watermarks(
 
 bool hubbub32_program_usr_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -750,7 +750,7 @@ void hubbub32_force_usr_retraining_allow(struct hubbub *hubbub, bool allow)
 
 static bool hubbub32_program_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h
index f073839a4b6d..e439ba0fa30f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h
@@ -118,25 +118,25 @@
 
 bool hubbub32_program_urgent_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 
 bool hubbub32_program_stutter_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 
 bool hubbub32_program_pstate_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 
 bool hubbub32_program_usr_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c
index f98def6c8c2d..fbcd6f7bc993 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c
@@ -35,25 +35,6 @@ static bool is_dual_plane(enum surface_pixel_format format)
 	return format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN || format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA;
 }
 
-
-uint32_t dcn32_helper_mall_bytes_to_ways(
-		struct dc *dc,
-		uint32_t total_size_in_mall_bytes)
-{
-	uint32_t cache_lines_used, lines_per_way, total_cache_lines, num_ways;
-
-	/* add 2 lines for worst case alignment */
-	cache_lines_used = total_size_in_mall_bytes / dc->caps.cache_line_size + 2;
-
-	total_cache_lines = dc->caps.max_cab_allocation_bytes / dc->caps.cache_line_size;
-	lines_per_way = total_cache_lines / dc->caps.cache_num_ways;
-	num_ways = cache_lines_used / lines_per_way;
-	if (cache_lines_used % lines_per_way > 0)
-		num_ways++;
-
-	return num_ways;
-}
-
 uint32_t dcn32_helper_calculate_mall_bytes_for_cursor(
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
@@ -112,8 +93,10 @@ uint32_t dcn32_helper_calculate_num_ways_for_subvp(
 	if (context->bw_ctx.bw.dcn.mall_subvp_size_bytes > 0) {
 		if (dc->debug.force_subvp_num_ways) {
 			return dc->debug.force_subvp_num_ways;
+		} else if (dc->res_pool->funcs->calculate_mall_ways_from_bytes) {
+			return dc->res_pool->funcs->calculate_mall_ways_from_bytes(dc, context->bw_ctx.bw.dcn.mall_subvp_size_bytes);
 		} else {
-			return dcn32_helper_mall_bytes_to_ways(dc, context->bw_ctx.bw.dcn.mall_subvp_size_bytes);
+			return 0;
 		}
 	} else {
 		return 0;
@@ -399,7 +382,7 @@ void dcn32_set_det_allocations(struct dc *dc, struct dc_state *context,
 {
 	int i, pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
-	struct pipe_ctx *pipe;
+	struct pipe_ctx *pipe = 0;
 	bool disable_unbounded_requesting = dc->debug.disable_z9_mpc || dc->debug.disable_unbounded_requesting;
 
 	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_dio_link_encoder.c
index 13be5f06d987..05783daa62ac 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_dio_link_encoder.c
@@ -127,11 +127,6 @@ void dcn321_link_encoder_construct(
 	 * while doing the DP sink detect
 	 */
 
-/*	if (dal_adapter_service_is_feature_supported(as,
-		FEATURE_DP_SINK_DETECT_POLL_DATA_PIN))
-		enc10->base.features.flags.bits.
-			DP_SINK_DETECT_POLL_DATA_PIN = true;*/
-
 	enc10->base.output_signals =
 		SIGNAL_TYPE_DVI_SINGLE_LINK |
 		SIGNAL_TYPE_DVI_DUAL_LINK |
@@ -191,7 +186,6 @@ void dcn321_link_encoder_construct(
 				__func__,
 				result);
 	}
-	if (enc10->base.ctx->dc->debug.hdmi20_disable) {
+	if (enc10->base.ctx->dc->debug.hdmi20_disable)
 		enc10->base.features.flags.bits.HDMI_6GB_EN = 0;
-	}
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/Makefile b/drivers/gpu/drm/amd/display/dc/dcn35/Makefile
index 0e317e0c36a0..d5b4533d2f62 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn35/Makefile
@@ -13,7 +13,7 @@
 DCN35 = dcn35_dio_stream_encoder.o \
 	dcn35_dio_link_encoder.o dcn35_dccg.o \
 	dcn35_hubp.o dcn35_hubbub.o \
-	dcn35_mmhubbub.o dcn35_opp.o dcn35_dpp.o dcn35_pg_cntl.o dcn35_dwb.o
+	dcn35_mmhubbub.o dcn35_opp.o dcn35_pg_cntl.o dcn35_dwb.o
 
 AMD_DAL_DCN35 = $(addprefix $(AMDDALPATH)/dc/dcn35/,$(DCN35))
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.c b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.c
index f1ba7bb792ea..58dd3c5bbff0 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.c
@@ -49,15 +49,23 @@ static void dcn35_set_dppclk_enable(struct dccg *dccg,
 	switch (dpp_inst) {
 	case 0:
 		REG_UPDATE(DPPCLK_CTRL, DPPCLK0_EN, enable);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpp)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DPPCLK0_ROOT_GATE_DISABLE, enable);
 		break;
 	case 1:
 		REG_UPDATE(DPPCLK_CTRL, DPPCLK1_EN, enable);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpp)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DPPCLK1_ROOT_GATE_DISABLE, enable);
 		break;
 	case 2:
 		REG_UPDATE(DPPCLK_CTRL, DPPCLK2_EN, enable);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpp)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DPPCLK2_ROOT_GATE_DISABLE, enable);
 		break;
 	case 3:
 		REG_UPDATE(DPPCLK_CTRL, DPPCLK3_EN, enable);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpp)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DPPCLK3_ROOT_GATE_DISABLE, enable);
 		break;
 	default:
 		break;
@@ -100,6 +108,32 @@ static void dccg35_update_dpp_dto(struct dccg *dccg, int dpp_inst,
 	dccg->pipe_dppclk_khz[dpp_inst] = req_dppclk;
 }
 
+static void dccg35_set_dppclk_root_clock_gating(struct dccg *dccg,
+		 uint32_t dpp_inst, uint32_t enable)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+	if (!dccg->ctx->dc->debug.root_clock_optimization.bits.dpp)
+		return;
+
+	switch (dpp_inst) {
+	case 0:
+		REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DPPCLK0_ROOT_GATE_DISABLE, enable);
+		break;
+	case 1:
+		REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DPPCLK1_ROOT_GATE_DISABLE, enable);
+		break;
+	case 2:
+		REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DPPCLK2_ROOT_GATE_DISABLE, enable);
+		break;
+	case 3:
+		REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DPPCLK3_ROOT_GATE_DISABLE, enable);
+		break;
+	default:
+		break;
+	}
+}
+
 static void dccg35_get_pixel_rate_div(
 		struct dccg *dccg,
 		uint32_t otg_inst,
@@ -333,21 +367,67 @@ static void dccg35_set_dpstreamclk(
 	/* enabled to select one of the DTBCLKs for pipe */
 	switch (dp_hpo_inst) {
 	case 0:
-		REG_UPDATE_2(DPSTREAMCLK_CNTL,
-				DPSTREAMCLK0_EN,
+		REG_UPDATE_2(DPSTREAMCLK_CNTL, DPSTREAMCLK0_EN,
 				(src == REFCLK) ? 0 : 1, DPSTREAMCLK0_SRC_SEL, otg_inst);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpstream)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK0_ROOT_GATE_DISABLE, (src == REFCLK) ? 0 : 1);
 		break;
 	case 1:
 		REG_UPDATE_2(DPSTREAMCLK_CNTL, DPSTREAMCLK1_EN,
 				(src == REFCLK) ? 0 : 1, DPSTREAMCLK1_SRC_SEL, otg_inst);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpstream)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK1_ROOT_GATE_DISABLE, (src == REFCLK) ? 0 : 1);
 		break;
 	case 2:
 		REG_UPDATE_2(DPSTREAMCLK_CNTL, DPSTREAMCLK2_EN,
 				(src == REFCLK) ? 0 : 1, DPSTREAMCLK2_SRC_SEL, otg_inst);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpstream)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK2_ROOT_GATE_DISABLE, (src == REFCLK) ? 0 : 1);
 		break;
 	case 3:
 		REG_UPDATE_2(DPSTREAMCLK_CNTL, DPSTREAMCLK3_EN,
 				(src == REFCLK) ? 0 : 1, DPSTREAMCLK3_SRC_SEL, otg_inst);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpstream)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK3_ROOT_GATE_DISABLE, (src == REFCLK) ? 0 : 1);
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		return;
+	}
+}
+
+
+static void dccg35_set_dpstreamclk_root_clock_gating(
+		struct dccg *dccg,
+		int dp_hpo_inst,
+		bool enable)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+	switch (dp_hpo_inst) {
+	case 0:
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpstream) {
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK0_ROOT_GATE_DISABLE, enable ? 1 : 0);
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK0_GATE_DISABLE, enable ? 1 : 0);
+		}
+		break;
+	case 1:
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpstream) {
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK1_ROOT_GATE_DISABLE, enable ? 1 : 0);
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK1_GATE_DISABLE, enable ? 1 : 0);
+		}
+		break;
+	case 2:
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpstream) {
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK2_ROOT_GATE_DISABLE, enable ? 1 : 0);
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK2_GATE_DISABLE, enable ? 1 : 0);
+		}
+		break;
+	case 3:
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpstream) {
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK3_ROOT_GATE_DISABLE, enable ? 1 : 0);
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK3_GATE_DISABLE, enable ? 1 : 0);
+		}
 		break;
 	default:
 		BREAK_TO_DEBUGGER();
@@ -355,6 +435,8 @@ static void dccg35_set_dpstreamclk(
 	}
 }
 
+
+
 static void dccg35_set_physymclk_root_clock_gating(
 		struct dccg *dccg,
 		int phy_inst,
@@ -369,22 +451,32 @@ static void dccg35_set_physymclk_root_clock_gating(
 	case 0:
 		REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
 				PHYASYMCLK_ROOT_GATE_DISABLE, enable ? 1 : 0);
+//		REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//				PHYA_REFCLK_ROOT_GATE_DISABLE, enable ? 1 : 0);
 		break;
 	case 1:
 		REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
 				PHYBSYMCLK_ROOT_GATE_DISABLE, enable ? 1 : 0);
+//		REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//				PHYB_REFCLK_ROOT_GATE_DISABLE, enable ? 1 : 0);
 		break;
 	case 2:
 		REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
 				PHYCSYMCLK_ROOT_GATE_DISABLE, enable ? 1 : 0);
+//		REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//				PHYC_REFCLK_ROOT_GATE_DISABLE, enable ? 1 : 0);
 		break;
 	case 3:
 		REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
 				PHYDSYMCLK_ROOT_GATE_DISABLE, enable ? 1 : 0);
+//		REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//				PHYD_REFCLK_ROOT_GATE_DISABLE, enable ? 1 : 0);
 		break;
 	case 4:
 		REG_UPDATE(DCCG_GATE_DISABLE_CNTL2,
 				PHYESYMCLK_ROOT_GATE_DISABLE, enable ? 1 : 0);
+//		REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//				PHYE_REFCLK_ROOT_GATE_DISABLE, enable ? 1 : 0);
 		break;
 	default:
 		BREAK_TO_DEBUGGER();
@@ -407,10 +499,16 @@ static void dccg35_set_physymclk(
 			REG_UPDATE_2(PHYASYMCLK_CLOCK_CNTL,
 					PHYASYMCLK_EN, 1,
 					PHYASYMCLK_SRC_SEL, clk_src);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//						PHYA_REFCLK_ROOT_GATE_DISABLE, 0);
 		} else {
 			REG_UPDATE_2(PHYASYMCLK_CLOCK_CNTL,
 					PHYASYMCLK_EN, 0,
 					PHYASYMCLK_SRC_SEL, 0);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//						PHYA_REFCLK_ROOT_GATE_DISABLE, 1);
 		}
 		break;
 	case 1:
@@ -418,10 +516,16 @@ static void dccg35_set_physymclk(
 			REG_UPDATE_2(PHYBSYMCLK_CLOCK_CNTL,
 					PHYBSYMCLK_EN, 1,
 					PHYBSYMCLK_SRC_SEL, clk_src);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//						PHYB_REFCLK_ROOT_GATE_DISABLE, 0);
 		} else {
 			REG_UPDATE_2(PHYBSYMCLK_CLOCK_CNTL,
 					PHYBSYMCLK_EN, 0,
 					PHYBSYMCLK_SRC_SEL, 0);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//						PHYB_REFCLK_ROOT_GATE_DISABLE, 1);
 		}
 		break;
 	case 2:
@@ -429,10 +533,16 @@ static void dccg35_set_physymclk(
 			REG_UPDATE_2(PHYCSYMCLK_CLOCK_CNTL,
 					PHYCSYMCLK_EN, 1,
 					PHYCSYMCLK_SRC_SEL, clk_src);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//						PHYC_REFCLK_ROOT_GATE_DISABLE, 0);
 		} else {
 			REG_UPDATE_2(PHYCSYMCLK_CLOCK_CNTL,
 					PHYCSYMCLK_EN, 0,
 					PHYCSYMCLK_SRC_SEL, 0);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//						PHYC_REFCLK_ROOT_GATE_DISABLE, 1);
 		}
 		break;
 	case 3:
@@ -440,10 +550,16 @@ static void dccg35_set_physymclk(
 			REG_UPDATE_2(PHYDSYMCLK_CLOCK_CNTL,
 					PHYDSYMCLK_EN, 1,
 					PHYDSYMCLK_SRC_SEL, clk_src);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//						PHYD_REFCLK_ROOT_GATE_DISABLE, 0);
 		} else {
 			REG_UPDATE_2(PHYDSYMCLK_CLOCK_CNTL,
 					PHYDSYMCLK_EN, 0,
 					PHYDSYMCLK_SRC_SEL, 0);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//						PHYD_REFCLK_ROOT_GATE_DISABLE, 1);
 		}
 		break;
 	case 4:
@@ -451,10 +567,16 @@ static void dccg35_set_physymclk(
 			REG_UPDATE_2(PHYESYMCLK_CLOCK_CNTL,
 					PHYESYMCLK_EN, 1,
 					PHYESYMCLK_SRC_SEL, clk_src);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//						PHYE_REFCLK_ROOT_GATE_DISABLE, 0);
 		} else {
 			REG_UPDATE_2(PHYESYMCLK_CLOCK_CNTL,
 					PHYESYMCLK_EN, 0,
 					PHYESYMCLK_SRC_SEL, 0);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL4,
+//						PHYE_REFCLK_ROOT_GATE_DISABLE, 1);
 		}
 		break;
 	default:
@@ -491,12 +613,12 @@ static void dccg35_dpp_root_clock_control(
 
 	if (clock_on) {
 		/* turn off the DTO and leave phase/modulo at max */
-		dcn35_set_dppclk_enable(dccg, dpp_inst, 0);
+		dcn35_set_dppclk_enable(dccg, dpp_inst, 1);
 		REG_SET_2(DPPCLK_DTO_PARAM[dpp_inst], 0,
 			  DPPCLK0_DTO_PHASE, 0xFF,
 			  DPPCLK0_DTO_MODULO, 0xFF);
 	} else {
-		dcn35_set_dppclk_enable(dccg, dpp_inst, 1);
+		dcn35_set_dppclk_enable(dccg, dpp_inst, 0);
 		/* turn on the DTO to generate a 0hz clock */
 		REG_SET_2(DPPCLK_DTO_PARAM[dpp_inst], 0,
 			  DPPCLK0_DTO_PHASE, 0,
@@ -575,18 +697,32 @@ void dccg35_init(struct dccg *dccg)
 		dccg35_disable_symclk32_se(dccg, otg_inst);
 
 	if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_le)
-		for (otg_inst = 0; otg_inst < 2; otg_inst++)
+		for (otg_inst = 0; otg_inst < 2; otg_inst++) {
 			dccg31_disable_symclk32_le(dccg, otg_inst);
+			dccg31_set_symclk32_le_root_clock_gating(dccg, otg_inst, false);
+		}
+
+//	if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+//		for (otg_inst = 0; otg_inst < 4; otg_inst++)
+//			dccg35_disable_symclk_se(dccg, otg_inst, otg_inst);
+
 
 	if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpstream)
-		for (otg_inst = 0; otg_inst < 4; otg_inst++)
-			dccg314_set_dpstreamclk(dccg, REFCLK, otg_inst,
+		for (otg_inst = 0; otg_inst < 4; otg_inst++) {
+			dccg35_set_dpstreamclk(dccg, REFCLK, otg_inst,
 						otg_inst);
+			dccg35_set_dpstreamclk_root_clock_gating(dccg, otg_inst, false);
+		}
 
 	if (dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk)
 		for (otg_inst = 0; otg_inst < 5; otg_inst++)
 			dccg35_set_physymclk_root_clock_gating(dccg, otg_inst,
 					false);
+
+	if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpp)
+		for (otg_inst = 0; otg_inst < 4; otg_inst++)
+			dccg35_set_dppclk_root_clock_gating(dccg, otg_inst, 0);
+
 /*
 	dccg35_enable_global_fgcg_rep(
 		dccg, dccg->ctx->dc->debug.enable_fine_grain_clock_gating.bits
@@ -611,24 +747,32 @@ static void dccg35_enable_dscclk(struct dccg *dccg, int inst)
 				DSCCLK0_DTO_PHASE, 0,
 				DSCCLK0_DTO_MODULO, 0);
 		REG_UPDATE(DSCCLK_DTO_CTRL,	DSCCLK0_EN, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dsc)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DSCCLK0_ROOT_GATE_DISABLE, 1);
 		break;
 	case 1:
 		REG_UPDATE_2(DSCCLK1_DTO_PARAM,
 				DSCCLK1_DTO_PHASE, 0,
 				DSCCLK1_DTO_MODULO, 0);
 		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK1_EN, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dsc)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DSCCLK1_ROOT_GATE_DISABLE, 1);
 		break;
 	case 2:
 		REG_UPDATE_2(DSCCLK2_DTO_PARAM,
 				DSCCLK2_DTO_PHASE, 0,
 				DSCCLK2_DTO_MODULO, 0);
 		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK2_EN, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dsc)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DSCCLK2_ROOT_GATE_DISABLE, 1);
 		break;
 	case 3:
 		REG_UPDATE_2(DSCCLK3_DTO_PARAM,
 				DSCCLK3_DTO_PHASE, 0,
 				DSCCLK3_DTO_MODULO, 0);
 		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK3_EN, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dsc)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DSCCLK3_ROOT_GATE_DISABLE, 1);
 		break;
 	default:
 		BREAK_TO_DEBUGGER();
@@ -650,24 +794,32 @@ static void dccg35_disable_dscclk(struct dccg *dccg,
 		REG_UPDATE_2(DSCCLK0_DTO_PARAM,
 				DSCCLK0_DTO_PHASE, 0,
 				DSCCLK0_DTO_MODULO, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dsc)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DSCCLK0_ROOT_GATE_DISABLE, 0);
 		break;
 	case 1:
 		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK1_EN, 0);
 		REG_UPDATE_2(DSCCLK1_DTO_PARAM,
 				DSCCLK1_DTO_PHASE, 0,
 				DSCCLK1_DTO_MODULO, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dsc)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DSCCLK1_ROOT_GATE_DISABLE, 0);
 		break;
 	case 2:
 		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK2_EN, 0);
 		REG_UPDATE_2(DSCCLK2_DTO_PARAM,
 				DSCCLK2_DTO_PHASE, 0,
 				DSCCLK2_DTO_MODULO, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dsc)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DSCCLK2_ROOT_GATE_DISABLE, 0);
 		break;
 	case 3:
 		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK3_EN, 0);
 		REG_UPDATE_2(DSCCLK3_DTO_PARAM,
 				DSCCLK3_DTO_PHASE, 0,
 				DSCCLK3_DTO_MODULO, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dsc)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DSCCLK3_ROOT_GATE_DISABLE, 0);
 		break;
 	default:
 		return;
@@ -682,22 +834,32 @@ static void dccg35_enable_symclk_se(struct dccg *dccg, uint32_t stream_enc_inst,
 	case 0:
 		REG_UPDATE(SYMCLKA_CLOCK_ENABLE,
 				SYMCLKA_CLOCK_ENABLE, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKA_ROOT_GATE_DISABLE, 1);
 		break;
 	case 1:
 		REG_UPDATE(SYMCLKB_CLOCK_ENABLE,
 				SYMCLKB_CLOCK_ENABLE, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKB_ROOT_GATE_DISABLE, 1);
 		break;
 	case 2:
 		REG_UPDATE(SYMCLKC_CLOCK_ENABLE,
 				SYMCLKC_CLOCK_ENABLE, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKC_ROOT_GATE_DISABLE, 1);
 		break;
 	case 3:
 		REG_UPDATE(SYMCLKD_CLOCK_ENABLE,
 				SYMCLKD_CLOCK_ENABLE, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKD_ROOT_GATE_DISABLE, 1);
 		break;
 	case 4:
 		REG_UPDATE(SYMCLKE_CLOCK_ENABLE,
 				SYMCLKE_CLOCK_ENABLE, 1);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKE_ROOT_GATE_DISABLE, 1);
 		break;
 	}
 
@@ -706,26 +868,36 @@ static void dccg35_enable_symclk_se(struct dccg *dccg, uint32_t stream_enc_inst,
 		REG_UPDATE_2(SYMCLKA_CLOCK_ENABLE,
 				SYMCLKA_FE_EN, 1,
 				SYMCLKA_FE_SRC_SEL, link_enc_inst);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKA_FE_ROOT_GATE_DISABLE, 1);
 		break;
 	case 1:
 		REG_UPDATE_2(SYMCLKB_CLOCK_ENABLE,
 				SYMCLKB_FE_EN, 1,
 				SYMCLKB_FE_SRC_SEL, link_enc_inst);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKB_FE_ROOT_GATE_DISABLE, 1);
 		break;
 	case 2:
 		REG_UPDATE_2(SYMCLKC_CLOCK_ENABLE,
 				SYMCLKC_FE_EN, 1,
 				SYMCLKC_FE_SRC_SEL, link_enc_inst);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKC_FE_ROOT_GATE_DISABLE, 1);
 		break;
 	case 3:
 		REG_UPDATE_2(SYMCLKD_CLOCK_ENABLE,
 				SYMCLKD_FE_EN, 1,
 				SYMCLKD_FE_SRC_SEL, link_enc_inst);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKD_FE_ROOT_GATE_DISABLE, 1);
 		break;
 	case 4:
 		REG_UPDATE_2(SYMCLKE_CLOCK_ENABLE,
 				SYMCLKE_FE_EN, 1,
 				SYMCLKE_FE_SRC_SEL, link_enc_inst);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKE_FE_ROOT_GATE_DISABLE, 1);
 		break;
 	}
 }
@@ -786,26 +958,36 @@ static void dccg35_disable_symclk_se(struct dccg *dccg, uint32_t stream_enc_inst
 		REG_UPDATE_2(SYMCLKA_CLOCK_ENABLE,
 				SYMCLKA_FE_EN, 0,
 				SYMCLKA_FE_SRC_SEL, 0);
+//		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+//			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKA_FE_ROOT_GATE_DISABLE, 0);
 		break;
 	case 1:
 		REG_UPDATE_2(SYMCLKB_CLOCK_ENABLE,
 				SYMCLKB_FE_EN, 0,
 				SYMCLKB_FE_SRC_SEL, 0);
+//		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+//			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKB_FE_ROOT_GATE_DISABLE, 0);
 		break;
 	case 2:
 		REG_UPDATE_2(SYMCLKC_CLOCK_ENABLE,
 				SYMCLKC_FE_EN, 0,
 				SYMCLKC_FE_SRC_SEL, 0);
+//		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+//			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKC_FE_ROOT_GATE_DISABLE, 0);
 		break;
 	case 3:
 		REG_UPDATE_2(SYMCLKD_CLOCK_ENABLE,
 				SYMCLKD_FE_EN, 0,
 				SYMCLKD_FE_SRC_SEL, 0);
+//		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+//			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKD_FE_ROOT_GATE_DISABLE, 0);
 		break;
 	case 4:
 		REG_UPDATE_2(SYMCLKE_CLOCK_ENABLE,
 				SYMCLKE_FE_EN, 0,
 				SYMCLKE_FE_SRC_SEL, 0);
+//		if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_se)
+//			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKE_FE_ROOT_GATE_DISABLE, 0);
 		break;
 	}
 
@@ -818,22 +1000,32 @@ static void dccg35_disable_symclk_se(struct dccg *dccg, uint32_t stream_enc_inst
 		case 0:
 			REG_UPDATE(SYMCLKA_CLOCK_ENABLE,
 					SYMCLKA_CLOCK_ENABLE, 0);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_le)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKA_ROOT_GATE_DISABLE, 0);
 			break;
 		case 1:
 			REG_UPDATE(SYMCLKB_CLOCK_ENABLE,
 					SYMCLKB_CLOCK_ENABLE, 0);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_le)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKB_ROOT_GATE_DISABLE, 0);
 			break;
 		case 2:
 			REG_UPDATE(SYMCLKC_CLOCK_ENABLE,
 					SYMCLKC_CLOCK_ENABLE, 0);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_le)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKC_ROOT_GATE_DISABLE, 0);
 			break;
 		case 3:
 			REG_UPDATE(SYMCLKD_CLOCK_ENABLE,
 					SYMCLKD_CLOCK_ENABLE, 0);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_le)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKD_ROOT_GATE_DISABLE, 0);
 			break;
 		case 4:
 			REG_UPDATE(SYMCLKE_CLOCK_ENABLE,
 					SYMCLKE_CLOCK_ENABLE, 0);
+//			if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_le)
+//				REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, SYMCLKE_ROOT_GATE_DISABLE, 0);
 			break;
 		}
 	}
@@ -845,6 +1037,7 @@ static const struct dccg_funcs dccg35_funcs = {
 	.get_dccg_ref_freq = dccg31_get_dccg_ref_freq,
 	.dccg_init = dccg35_init,
 	.set_dpstreamclk = dccg35_set_dpstreamclk,
+	.set_dpstreamclk_root_clock_gating = dccg35_set_dpstreamclk_root_clock_gating,
 	.enable_symclk32_se = dccg31_enable_symclk32_se,
 	.disable_symclk32_se = dccg35_disable_symclk32_se,
 	.enable_symclk32_le = dccg31_enable_symclk32_le,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dio_link_encoder.c
index 81e349d5835b..20f810a6646c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dio_link_encoder.c
@@ -80,7 +80,6 @@ enum signal_type dcn35_get_dig_mode(
 	default:
 		return SIGNAL_TYPE_NONE;
 	}
-	return SIGNAL_TYPE_NONE;
 }
 
 void dcn35_link_encoder_setup(
@@ -119,7 +118,7 @@ void dcn35_link_encoder_setup(
 
 void dcn35_link_encoder_init(struct link_encoder *enc)
 {
-	enc32_hw_init(enc);
+	enc31_hw_init(enc);
 	dcn35_link_encoder_set_fgcg(enc, enc->ctx->dc->debug.enable_fine_grain_clock_gating.bits.dio);
 }
 
@@ -184,6 +183,8 @@ void dcn35_link_encoder_construct(
 	enc10->base.hpd_source = init_data->hpd_source;
 	enc10->base.connector = init_data->connector;
 
+	if (enc10->base.connector.id == CONNECTOR_ID_USBC)
+		enc10->base.features.flags.bits.DP_IS_USB_C = 1;
 
 	enc10->base.preferred_engine = ENGINE_ID_UNKNOWN;
 
@@ -238,8 +239,6 @@ void dcn35_link_encoder_construct(
 	}
 
 	enc10->base.features.flags.bits.HDMI_6GB_EN = 1;
-	if (enc10->base.connector.id == CONNECTOR_ID_USBC)
-		enc10->base.features.flags.bits.DP_IS_USB_C = 1;
 
 	if (bp_funcs->get_connector_speed_cap_info)
 		result = bp_funcs->get_connector_speed_cap_info(enc10->base.ctx->dc_bios,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dio_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dio_link_encoder.h
index e1e560732a9d..d546a3676304 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dio_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dio_link_encoder.h
@@ -37,7 +37,9 @@
 	LE_SF(DIG0_DIG_BE_CLK_CNTL, DIG_BE_MODE, mask_sh),\
 	LE_SF(DIG0_DIG_BE_CLK_CNTL, DIG_BE_CLK_EN, mask_sh),\
 	LE_SF(DIG0_DIG_BE_CLK_CNTL, DIG_BE_SOFT_RESET, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CLK_CNTL, HDCP_SOFT_RESET, mask_sh),\
 	LE_SF(DIG0_DIG_BE_CLK_CNTL, DIG_BE_SYMCLK_G_CLOCK_ON, mask_sh),\
+	LE_SF(DIG0_DIG_BE_CLK_CNTL, DIG_BE_SYMCLK_G_HDCP_CLOCK_ON, mask_sh),\
 	LE_SF(DIG0_DIG_BE_CLK_CNTL, DIG_BE_SYMCLK_G_TMDS_CLOCK_ON, mask_sh),\
 	LE_SF(DIG0_DIG_CLOCK_PATTERN, DIG_CLOCK_PATTERN, mask_sh),\
 	LE_SF(DIG0_TMDS_CTL_BITS, TMDS_CTL0, mask_sh), \
@@ -114,7 +116,15 @@
 	LE_SF(DIO_CLK_CNTL, SYMCLK_FE_G_GATE_DIS, mask_sh),\
 	LE_SF(DIO_CLK_CNTL, SYMCLK_R_GATE_DIS, mask_sh),\
 	LE_SF(DIO_CLK_CNTL, SYMCLK_G_GATE_DIS, mask_sh),\
-	LE_SF(DIO_CLK_CNTL, DIO_FGCG_REP_DIS, mask_sh)
+	LE_SF(DIO_CLK_CNTL, DIO_FGCG_REP_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, DISPCLK_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKA_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKB_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKC_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKD_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKE_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKF_G_HDCP_GATE_DIS, mask_sh),\
+	LE_SF(DIO_CLK_CNTL, SYMCLKG_G_HDCP_GATE_DIS, mask_sh)
 
 
 void dcn35_link_encoder_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dio_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dio_stream_encoder.h
index 499052329ebb..1212fcee38f2 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dio_stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dio_stream_encoder.h
@@ -28,7 +28,6 @@
 #include "dcn30/dcn30_vpg.h"
 #include "dcn30/dcn30_afmt.h"
 #include "stream_encoder.h"
-#include "dcn10/dcn10_link_encoder.h"
 #include "dcn20/dcn20_stream_encoder.h"
 
 /* Register bit field name change */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_hubbub.c
index 339bf0c722dd..6293173ba2b9 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_hubbub.c
@@ -111,7 +111,7 @@ static uint32_t convert_and_clamp(
 
 static bool hubbub35_program_stutter_z8_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
@@ -297,7 +297,7 @@ static void hubbub35_get_dchub_ref_freq(struct hubbub *hubbub,
 
 static bool hubbub35_program_watermarks(
 		struct hubbub *hubbub,
-		struct dcn_watermark_set *watermarks,
+		union dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
 		bool safe_to_lower)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dm_helpers.h b/drivers/gpu/drm/amd/display/dc/dm_helpers.h
index 6d7a15dcf8a7..34adae7ab6e8 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_helpers.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_helpers.h
@@ -36,6 +36,7 @@
 struct dc_dp_mst_stream_allocation_table;
 struct aux_payload;
 enum aux_return_code_type;
+enum set_config_status;
 
 /*
  * Allocate memory accessible by the GPU
@@ -200,7 +201,7 @@ int dm_helper_dmub_aux_transfer_sync(
 		const struct dc_link *link,
 		struct aux_payload *payload,
 		enum aux_return_code_type *operation_result);
-enum set_config_status;
+
 int dm_helpers_dmub_set_config_sync(struct dc_context *ctx,
 		const struct dc_link *link,
 		struct set_config_cmd_payload *payload,
diff --git a/drivers/gpu/drm/amd/display/dc/dm_services.h b/drivers/gpu/drm/amd/display/dc/dm_services.h
index d0eed3b4771e..9405c47ee2a9 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_services.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_services.h
@@ -275,6 +275,16 @@ void dm_perf_trace_timestamp(const char *func_name, unsigned int line, struct dc
 #define PERF_TRACE_CTX(__CTX)	dm_perf_trace_timestamp(__func__, __LINE__, __CTX)
 
 /*
+ * SMU message tracing
+ */
+void dm_trace_smu_msg(uint32_t msg_id, uint32_t param_in, struct dc_context *ctx);
+void dm_trace_smu_delay(uint32_t delay, struct dc_context *ctx);
+
+#define TRACE_SMU_MSG(msg_id, param_in, ctx)	dm_trace_smu_msg(msg_id, param_in, ctx)
+#define TRACE_SMU_DELAY(response_delay, ctx)	dm_trace_smu_delay(response_delay, ctx)
+
+
+/*
  * DMUB Interfaces
  */
 bool dm_execute_dmub_cmd(const struct dc_context *ctx, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type);
diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile
index c4a5efd2dda5..a94b6d546cd1 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile
@@ -25,40 +25,8 @@
 # It provides the general basic services required by other DAL
 # subcomponents.
 
-ifdef CONFIG_X86
-dml_ccflags-$(CONFIG_CC_IS_GCC) := -mhard-float
-dml_ccflags := $(dml_ccflags-y) -msse
-endif
-
-ifdef CONFIG_PPC64
-dml_ccflags := -mhard-float -maltivec
-endif
-
-ifdef CONFIG_ARM64
-dml_rcflags := -mgeneral-regs-only
-endif
-
-ifdef CONFIG_LOONGARCH
-dml_ccflags := -mfpu=64
-dml_rcflags := -msoft-float
-endif
-
-ifdef CONFIG_CC_IS_GCC
-ifneq ($(call gcc-min-version, 70100),y)
-IS_OLD_GCC = 1
-endif
-endif
-
-ifdef CONFIG_X86
-ifdef IS_OLD_GCC
-# Stack alignment mismatch, proceed with caution.
-# GCC < 7.1 cannot compile code using `double` and -mpreferred-stack-boundary=3
-# (8B stack alignment).
-dml_ccflags += -mpreferred-stack-boundary=4
-else
-dml_ccflags += -msse2
-endif
-endif
+dml_ccflags := $(CC_FLAGS_FPU)
+dml_rcflags := $(CC_FLAGS_NO_FPU)
 
 ifneq ($(CONFIG_FRAME_WARN),0)
 ifeq ($(filter y,$(CONFIG_KASAN)$(CONFIG_KCSAN)),y)
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c
index 38ab9ad60ef8..74da9ebda016 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c
@@ -1085,6 +1085,9 @@ static enum dcn_zstate_support_state  decide_zstate_support(struct dc *dc, struc
 		int minmum_z8_residency = dc->debug.minimum_z8_residency_time > 0 ? dc->debug.minimum_z8_residency_time : 1000;
 		bool allow_z8 = context->bw_ctx.dml.vba.StutterPeriod > (double)minmum_z8_residency;
 		bool is_pwrseq0 = link->link_index == 0;
+		bool is_psr = (link && (link->psr_settings.psr_version == DC_PSR_VERSION_1 ||
+						link->psr_settings.psr_version == DC_PSR_VERSION_SU_1) && !link->panel_config.psr.disable_psr);
+		bool is_replay = link && link->replay_settings.replay_feature_enabled;
 
 		/* Don't support multi-plane configurations */
 		if (stream_status->plane_count > 1)
@@ -1092,8 +1095,8 @@ static enum dcn_zstate_support_state  decide_zstate_support(struct dc *dc, struc
 
 		if (is_pwrseq0 && context->bw_ctx.dml.vba.StutterPeriod > 5000.0)
 			return DCN_ZSTATE_SUPPORT_ALLOW;
-		else if (is_pwrseq0 && link->psr_settings.psr_version == DC_PSR_VERSION_1 && !link->panel_config.psr.disable_psr)
-			return allow_z8 ? DCN_ZSTATE_SUPPORT_ALLOW_Z8_Z10_ONLY : DCN_ZSTATE_SUPPORT_ALLOW_Z10_ONLY;
+		else if (is_pwrseq0 && (is_psr || is_replay))
+			return DCN_ZSTATE_SUPPORT_ALLOW_Z8_Z10_ONLY;
 		else
 			return allow_z8 ? DCN_ZSTATE_SUPPORT_ALLOW_Z8_ONLY : DCN_ZSTATE_SUPPORT_DISALLOW;
 	} else {
@@ -2369,7 +2372,7 @@ validate_out:
 
 static struct _vcs_dpi_voltage_scaling_st construct_low_pstate_lvl(struct clk_limit_table *clk_table, unsigned int high_voltage_lvl)
 {
-	struct _vcs_dpi_voltage_scaling_st low_pstate_lvl;
+	struct _vcs_dpi_voltage_scaling_st low_pstate_lvl = {0};
 	int i;
 
 	low_pstate_lvl.state = 1;
@@ -2474,7 +2477,7 @@ void dcn201_populate_dml_writeback_from_context_fpu(struct dc *dc,
 	int pipe_cnt, i, j;
 	double max_calc_writeback_dispclk;
 	double writeback_dispclk;
-	struct writeback_st dout_wb;
+	struct writeback_st dout_wb = {0};
 
 	dc_assert_fp_enabled();
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c
index ccb4ad78f667..81f7b90849ce 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c
@@ -260,7 +260,7 @@ void dcn30_fpu_populate_dml_writeback_from_context(
 	int pipe_cnt, i, j;
 	double max_calc_writeback_dispclk;
 	double writeback_dispclk;
-	struct writeback_st dout_wb;
+	struct writeback_st dout_wb = {0};
 
 	dc_assert_fp_enabled();
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
index e7f4a2d491cc..e0b52db2c210 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
@@ -3535,7 +3535,6 @@ static double TruncToValidBPP(
 			return DesiredBPP;
 		}
 	}
-	return BPP_INVALID;
 }
 
 void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_lib)
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c
index deb6d162a2d5..94317b2e4a85 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c
@@ -291,6 +291,7 @@ static struct _vcs_dpi_soc_bounding_box_st dcn3_15_soc = {
 	.do_urgent_latency_adjustment = false,
 	.urgent_latency_adjustment_fabric_clock_component_us = 0,
 	.urgent_latency_adjustment_fabric_clock_reference_mhz = 0,
+	.dispclk_dppclk_vco_speed_mhz = 2400.0,
 	.num_chans = 4,
 	.dummy_pstate_latency_us = 10.0
 };
@@ -438,6 +439,7 @@ static struct _vcs_dpi_soc_bounding_box_st dcn3_16_soc = {
 	.do_urgent_latency_adjustment = false,
 	.urgent_latency_adjustment_fabric_clock_component_us = 0,
 	.urgent_latency_adjustment_fabric_clock_reference_mhz = 0,
+	.dispclk_dppclk_vco_speed_mhz = 2500.0,
 };
 
 void dcn31_zero_pipe_dcc_fraction(display_e2e_pipe_params_st *pipes,
@@ -485,6 +487,7 @@ void dcn31_calculate_wm_and_dlg_fp(
 {
 	int i, pipe_idx, total_det = 0, active_hubp_count = 0;
 	double dcfclk = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
+	uint32_t cstate_enter_plus_exit_z8_ns;
 
 	dc_assert_fp_enabled();
 
@@ -504,6 +507,13 @@ void dcn31_calculate_wm_and_dlg_fp(
 	pipes[0].clks_cfg.dcfclk_mhz = dcfclk;
 	pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].socclk_mhz;
 
+	cstate_enter_plus_exit_z8_ns =
+		get_wm_z8_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+
+	if (get_stutter_period(&context->bw_ctx.dml, pipes, pipe_cnt) < dc->debug.minimum_z8_residency_time &&
+			cstate_enter_plus_exit_z8_ns < dc->debug.minimum_z8_residency_time * 1000)
+		cstate_enter_plus_exit_z8_ns = dc->debug.minimum_z8_residency_time * 1000;
+
 	/* Set A:
 	 * All clocks min required
 	 *
@@ -514,7 +524,7 @@ void dcn31_calculate_wm_and_dlg_fp(
 	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
 	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
 	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_z8_ns = get_wm_z8_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_z8_ns = cstate_enter_plus_exit_z8_ns;
 	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_z8_ns = get_wm_z8_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
 	context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
 	context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.h b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.h
index 8f9c8faed260..d2ae43a82ba5 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.h
@@ -30,6 +30,7 @@
 #define DCN3_15_DEFAULT_DET_SIZE 192
 #define DCN3_15_MIN_COMPBUF_SIZE_KB 128
 #define DCN3_16_DEFAULT_DET_SIZE 192
+#define DCN3_16_MIN_COMPBUF_SIZE_KB 128
 
 void dcn31_zero_pipe_dcc_fraction(display_e2e_pipe_params_st *pipes,
 				  int pipe_cnt);
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
index adea459e7d36..33cf824c5da1 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
@@ -3679,7 +3679,6 @@ static double TruncToValidBPP(
 			return DesiredBPP;
 		}
 	}
-	return BPP_INVALID;
 }
 
 static noinline void CalculatePrefetchSchedulePerPlane(
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c
index fb21572750e8..21f637ae4add 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c
@@ -310,7 +310,7 @@ int dcn314_populate_dml_pipes_from_context_fpu(struct dc *dc, struct dc_state *c
 {
 	int i, pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
-	struct pipe_ctx *pipe;
+	struct pipe_ctx *pipe = 0;
 	bool upscaled = false;
 	const unsigned int max_allowed_vblank_nom = 1023;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
index 88e56889a68c..3242957d00c5 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
@@ -3788,7 +3788,6 @@ static double TruncToValidBPP(
 			return DesiredBPP;
 		}
 	}
-	return BPP_INVALID;
 }
 
 static noinline void CalculatePrefetchSchedulePerPlane(
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c
index a0a65e099104..f6fe0a64beac 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c
@@ -180,6 +180,9 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_2_soc = {
 	.urgent_latency_adjustment_fabric_clock_reference_mhz = 3000,
 };
 
+static bool dcn32_apply_merge_split_flags_helper(struct dc *dc, struct dc_state *context,
+	bool *repopulate_pipes, int *split, bool *merge);
+
 void dcn32_build_wm_range_table_fpu(struct clk_mgr_internal *clk_mgr)
 {
 	/* defaults */
@@ -622,7 +625,7 @@ static bool dcn32_assign_subvp_pipe(struct dc *dc,
 		 *   to combine this with SubVP can cause issues with the scheduling).
 		 * - Not TMZ surface
 		 */
-		if (pipe->plane_state && !pipe->top_pipe && !dcn32_is_center_timing(pipe) &&
+		if (pipe->plane_state && !pipe->top_pipe && !pipe->prev_odm_pipe && !dcn32_is_center_timing(pipe) &&
 				!(pipe->stream->timing.pix_clk_100hz / 10000 > DCN3_2_MAX_SUBVP_PIXEL_RATE_MHZ) &&
 				(!dcn32_is_psr_capable(pipe) || (context->stream_count == 1 && dc->caps.dmub_caps.subvp_psr)) &&
 				dc_state_get_pipe_subvp_type(context, pipe) == SUBVP_NONE &&
@@ -720,7 +723,7 @@ static bool dcn32_enough_pipes_for_subvp(struct dc *dc, struct dc_state *context
  */
 static bool subvp_subvp_schedulable(struct dc *dc, struct dc_state *context)
 {
-	struct pipe_ctx *subvp_pipes[2];
+	struct pipe_ctx *subvp_pipes[2] = {0};
 	struct dc_stream_state *phantom = NULL;
 	uint32_t microschedule_lines = 0;
 	uint32_t index = 0;
@@ -1425,13 +1428,14 @@ static bool is_test_pattern_enabled(
 	return false;
 }
 
-static void dcn32_full_validate_bw_helper(struct dc *dc,
+static bool dcn32_full_validate_bw_helper(struct dc *dc,
 				   struct dc_state *context,
 				   display_e2e_pipe_params_st *pipes,
 				   int *vlevel,
 				   int *split,
 				   bool *merge,
-				   int *pipe_cnt)
+				   int *pipe_cnt,
+				   bool *repopulate_pipes)
 {
 	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
 	unsigned int dc_pipe_idx = 0;
@@ -1461,6 +1465,12 @@ static void dcn32_full_validate_bw_helper(struct dc *dc,
 		vba->VoltageLevel = *vlevel;
 	}
 
+	/* Apply split and merge flags before checking for subvp */
+	if (!dcn32_apply_merge_split_flags_helper(dc, context, repopulate_pipes, split, merge))
+		return false;
+	memset(split, 0, MAX_PIPES * sizeof(int));
+	memset(merge, 0, MAX_PIPES * sizeof(bool));
+
 	/* Conditions for setting up phantom pipes for SubVP:
 	 * 1. Not force disable SubVP
 	 * 2. Full update (i.e. !fast_validate)
@@ -1475,19 +1485,7 @@ static void dcn32_full_validate_bw_helper(struct dc *dc,
 	    vba->DRAMClockChangeSupport[*vlevel][vba->maxMpcComb] == dm_dram_clock_change_unsupported ||
 	    dc->debug.force_subvp_mclk_switch)) {
 
-		dcn32_merge_pipes_for_subvp(dc, context);
-		memset(merge, 0, MAX_PIPES * sizeof(bool));
-
 		vlevel_temp = *vlevel;
-		/* to re-initialize viewport after the pipe merge */
-		for (i = 0; i < dc->res_pool->pipe_count; i++) {
-			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
-
-			if (!pipe_ctx->plane_state || !pipe_ctx->stream)
-				continue;
-
-			resource_build_scaling_params(pipe_ctx);
-		}
 
 		while (!found_supported_config && dcn32_enough_pipes_for_subvp(dc, context) &&
 			dcn32_assign_subvp_pipe(dc, context, &dc_pipe_idx)) {
@@ -1576,8 +1574,6 @@ static void dcn32_full_validate_bw_helper(struct dc *dc,
 			 * add phantom pipes. If pipe split (ODM / MPC) is required, both the main
 			 * and phantom pipes will be split in the regular pipe splitting sequence.
 			 */
-			memset(split, 0, MAX_PIPES * sizeof(int));
-			memset(merge, 0, MAX_PIPES * sizeof(bool));
 			*vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, *vlevel, split, merge);
 			vba->VoltageLevel = *vlevel;
 			// Note: We can't apply the phantom pipes to hardware at this time. We have to wait
@@ -1590,6 +1586,7 @@ static void dcn32_full_validate_bw_helper(struct dc *dc,
 		try_odm_power_optimization_and_revalidate(
 				dc, context, pipes, split, merge, vlevel, *pipe_cnt);
 
+	return true;
 }
 
 static bool is_dtbclk_required(struct dc *dc, struct dc_state *context)
@@ -1929,106 +1926,23 @@ static bool dcn32_split_stream_for_mpc_or_odm(
 	return true;
 }
 
-bool dcn32_internal_validate_bw(struct dc *dc,
-				struct dc_state *context,
-				display_e2e_pipe_params_st *pipes,
-				int *pipe_cnt_out,
-				int *vlevel_out,
-				bool fast_validate)
+static bool dcn32_apply_merge_split_flags_helper(
+		struct dc *dc,
+		struct dc_state *context,
+		bool *repopulate_pipes,
+		int *split,
+		bool *merge)
 {
-	bool out = false;
-	bool repopulate_pipes = false;
-	int split[MAX_PIPES] = { 0 };
-	bool merge[MAX_PIPES] = { false };
+	int i, pipe_idx;
 	bool newly_split[MAX_PIPES] = { false };
-	int pipe_cnt, i, pipe_idx;
-	int vlevel = context->bw_ctx.dml.soc.num_states;
 	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
 
-	dc_assert_fp_enabled();
-
-	ASSERT(pipes);
-	if (!pipes)
-		return false;
-
-	// For each full update, remove all existing phantom pipes first
-	dc_state_remove_phantom_streams_and_planes(dc, context);
-	dc_state_release_phantom_streams_and_planes(dc, context);
-
-	dc->res_pool->funcs->update_soc_for_wm_a(dc, context);
-
-	pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
-
-	if (!pipe_cnt) {
-		out = true;
-		goto validate_out;
-	}
-
-	dml_log_pipe_params(&context->bw_ctx.dml, pipes, pipe_cnt);
-	context->bw_ctx.dml.soc.max_vratio_pre = dcn32_determine_max_vratio_prefetch(dc, context);
-
-	if (!fast_validate)
-		dcn32_full_validate_bw_helper(dc, context, pipes, &vlevel, split, merge, &pipe_cnt);
-
-	if (fast_validate ||
-			(dc->debug.dml_disallow_alternate_prefetch_modes &&
-			(vlevel == context->bw_ctx.dml.soc.num_states ||
-				vba->DRAMClockChangeSupport[vlevel][vba->maxMpcComb] == dm_dram_clock_change_unsupported))) {
-		/*
-		 * If dml_disallow_alternate_prefetch_modes is false, then we have already
-		 * tried alternate prefetch modes during full validation.
-		 *
-		 * If mode is unsupported or there is no p-state support, then
-		 * fall back to favouring voltage.
-		 *
-		 * If Prefetch mode 0 failed for this config, or passed with Max UCLK, then try
-		 * to support with Prefetch mode 1 (dm_prefetch_support_fclk_and_stutter == 2)
-		 */
-		context->bw_ctx.dml.soc.allow_for_pstate_or_stutter_in_vblank_final =
-			dm_prefetch_support_none;
-
-		context->bw_ctx.dml.validate_max_state = fast_validate;
-		vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
-
-		context->bw_ctx.dml.validate_max_state = false;
-
-		if (vlevel < context->bw_ctx.dml.soc.num_states) {
-			memset(split, 0, sizeof(split));
-			memset(merge, 0, sizeof(merge));
-			vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, merge);
-			// dcn20_validate_apply_pipe_split_flags can modify voltage level outside of DML
-			vba->VoltageLevel = vlevel;
-		}
-	}
-
-	dml_log_mode_support_params(&context->bw_ctx.dml);
-
-	if (vlevel == context->bw_ctx.dml.soc.num_states)
-		goto validate_fail;
-
-	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *mpo_pipe = pipe->bottom_pipe;
-
-		if (!pipe->stream)
-			continue;
-
-		if (vba->ODMCombineEnabled[vba->pipe_plane[pipe_idx]] != dm_odm_combine_mode_disabled
-				&& !dc->config.enable_windowed_mpo_odm
-				&& pipe->plane_state && mpo_pipe
-				&& memcmp(&mpo_pipe->plane_state->clip_rect,
-						&pipe->stream->src,
-						sizeof(struct rect)) != 0) {
-			ASSERT(mpo_pipe->plane_state != pipe->plane_state);
-			goto validate_fail;
-		}
-		pipe_idx++;
-	}
-
 	if (dc->config.enable_windowed_mpo_odm) {
-		repopulate_pipes = update_pipes_with_split_flags(
-				dc, context, vba, split, merge);
+		if (update_pipes_with_split_flags(
+			dc, context, vba, split, merge))
+			*repopulate_pipes = true;
 	} else {
+
 		/* the code below will be removed once windowed mpo odm is fully
 		 * enabled.
 		 */
@@ -2085,7 +1999,7 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 				memset(&pipe->plane_res, 0, sizeof(pipe->plane_res));
 				memset(&pipe->stream_res, 0, sizeof(pipe->stream_res));
 				memset(&pipe->link_res, 0, sizeof(pipe->link_res));
-				repopulate_pipes = true;
+				*repopulate_pipes = true;
 			} else if (pipe->top_pipe && pipe->top_pipe->plane_state == pipe->plane_state) {
 				struct pipe_ctx *top_pipe = pipe->top_pipe;
 				struct pipe_ctx *bottom_pipe = pipe->bottom_pipe;
@@ -2101,7 +2015,7 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 				memset(&pipe->plane_res, 0, sizeof(pipe->plane_res));
 				memset(&pipe->stream_res, 0, sizeof(pipe->stream_res));
 				memset(&pipe->link_res, 0, sizeof(pipe->link_res));
-				repopulate_pipes = true;
+				*repopulate_pipes = true;
 			} else
 				ASSERT(0); /* Should never try to merge master pipe */
 
@@ -2140,15 +2054,15 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 				hsplit_pipe = dcn32_find_split_pipe(dc, context, old_index);
 				ASSERT(hsplit_pipe);
 				if (!hsplit_pipe)
-					goto validate_fail;
+					return false;
 
 				if (!dcn32_split_stream_for_mpc_or_odm(
 						dc, &context->res_ctx,
 						pipe, hsplit_pipe, odm))
-					goto validate_fail;
+					return false;
 
 				newly_split[hsplit_pipe->pipe_idx] = true;
-				repopulate_pipes = true;
+				*repopulate_pipes = true;
 			}
 			if (split[i] == 4) {
 				struct pipe_ctx *pipe_4to1;
@@ -2163,11 +2077,11 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 				pipe_4to1 = dcn32_find_split_pipe(dc, context, old_index);
 				ASSERT(pipe_4to1);
 				if (!pipe_4to1)
-					goto validate_fail;
+					return false;
 				if (!dcn32_split_stream_for_mpc_or_odm(
 						dc, &context->res_ctx,
 						pipe, pipe_4to1, odm))
-					goto validate_fail;
+					return false;
 				newly_split[pipe_4to1->pipe_idx] = true;
 
 				if (odm && old_pipe->next_odm_pipe && old_pipe->next_odm_pipe->next_odm_pipe
@@ -2182,11 +2096,11 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 				pipe_4to1 = dcn32_find_split_pipe(dc, context, old_index);
 				ASSERT(pipe_4to1);
 				if (!pipe_4to1)
-					goto validate_fail;
+					return false;
 				if (!dcn32_split_stream_for_mpc_or_odm(
 						dc, &context->res_ctx,
 						hsplit_pipe, pipe_4to1, odm))
-					goto validate_fail;
+					return false;
 				newly_split[pipe_4to1->pipe_idx] = true;
 			}
 			if (odm)
@@ -2198,11 +2112,122 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 
 			if (pipe->plane_state) {
 				if (!resource_build_scaling_params(pipe))
-					goto validate_fail;
+					return false;
 			}
 		}
+
+		for (i = 0; i < context->stream_count; i++) {
+			struct pipe_ctx *otg_master = resource_get_otg_master_for_stream(&context->res_ctx,
+					context->streams[i]);
+
+			if (otg_master)
+				resource_build_test_pattern_params(&context->res_ctx, otg_master);
+		}
+	}
+	return true;
+}
+
+bool dcn32_internal_validate_bw(struct dc *dc,
+				struct dc_state *context,
+				display_e2e_pipe_params_st *pipes,
+				int *pipe_cnt_out,
+				int *vlevel_out,
+				bool fast_validate)
+{
+	bool out = false;
+	bool repopulate_pipes = false;
+	int split[MAX_PIPES] = { 0 };
+	bool merge[MAX_PIPES] = { false };
+	int pipe_cnt, i, pipe_idx;
+	int vlevel = context->bw_ctx.dml.soc.num_states;
+	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
+
+	dc_assert_fp_enabled();
+
+	ASSERT(pipes);
+	if (!pipes)
+		return false;
+
+	/* For each full update, remove all existing phantom pipes first */
+	dc_state_remove_phantom_streams_and_planes(dc, context);
+	dc_state_release_phantom_streams_and_planes(dc, context);
+
+	dc->res_pool->funcs->update_soc_for_wm_a(dc, context);
+
+	pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
+
+	if (!pipe_cnt) {
+		out = true;
+		goto validate_out;
+	}
+
+	dml_log_pipe_params(&context->bw_ctx.dml, pipes, pipe_cnt);
+	context->bw_ctx.dml.soc.max_vratio_pre = dcn32_determine_max_vratio_prefetch(dc, context);
+
+	if (!fast_validate) {
+		if (!dcn32_full_validate_bw_helper(dc, context, pipes, &vlevel, split, merge,
+			&pipe_cnt, &repopulate_pipes))
+			goto validate_fail;
+	}
+
+	if (fast_validate ||
+			(dc->debug.dml_disallow_alternate_prefetch_modes &&
+			(vlevel == context->bw_ctx.dml.soc.num_states ||
+				vba->DRAMClockChangeSupport[vlevel][vba->maxMpcComb] == dm_dram_clock_change_unsupported))) {
+		/*
+		 * If dml_disallow_alternate_prefetch_modes is false, then we have already
+		 * tried alternate prefetch modes during full validation.
+		 *
+		 * If mode is unsupported or there is no p-state support, then
+		 * fall back to favouring voltage.
+		 *
+		 * If Prefetch mode 0 failed for this config, or passed with Max UCLK, then try
+		 * to support with Prefetch mode 1 (dm_prefetch_support_fclk_and_stutter == 2)
+		 */
+		context->bw_ctx.dml.soc.allow_for_pstate_or_stutter_in_vblank_final =
+			dm_prefetch_support_none;
+
+		context->bw_ctx.dml.validate_max_state = fast_validate;
+		vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
+
+		context->bw_ctx.dml.validate_max_state = false;
+
+		if (vlevel < context->bw_ctx.dml.soc.num_states) {
+			memset(split, 0, sizeof(split));
+			memset(merge, 0, sizeof(merge));
+			vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, merge);
+			/* dcn20_validate_apply_pipe_split_flags can modify voltage level outside of DML */
+			vba->VoltageLevel = vlevel;
+		}
 	}
 
+	dml_log_mode_support_params(&context->bw_ctx.dml);
+
+	if (vlevel == context->bw_ctx.dml.soc.num_states)
+		goto validate_fail;
+
+	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *mpo_pipe = pipe->bottom_pipe;
+
+		if (!pipe->stream)
+			continue;
+
+		if (vba->ODMCombineEnabled[vba->pipe_plane[pipe_idx]] != dm_odm_combine_mode_disabled
+				&& !dc->config.enable_windowed_mpo_odm
+				&& pipe->plane_state && mpo_pipe
+				&& memcmp(&mpo_pipe->plane_state->clip_rect,
+						&pipe->stream->src,
+						sizeof(struct rect)) != 0) {
+			ASSERT(mpo_pipe->plane_state != pipe->plane_state);
+			goto validate_fail;
+		}
+		pipe_idx++;
+	}
+
+	if (!dcn32_apply_merge_split_flags_helper(dc, context, &repopulate_pipes, split, merge))
+		goto validate_fail;
+
 	/* Actual dsc count per stream dsc validation*/
 	if (!dcn20_validate_dsc(dc, context)) {
 		vba->ValidationStatus[vba->soc.num_states] = DML_FAIL_DSC_VALIDATION_FAILURE;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
index 80fccd4999a5..ba1310c8fd77 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
@@ -1650,6 +1650,8 @@ double dml32_TruncToValidBPP(
 			MaxLinkBPP = 2 * MaxLinkBPP;
 	}
 
+	*RequiredSlots = dml_ceil(DesiredBPP / MaxLinkBPP * 64, 1);
+
 	if (DesiredBPP == 0) {
 		if (DSCEnable) {
 			if (MaxLinkBPP < MinDSCBPP)
@@ -1676,10 +1678,6 @@ double dml32_TruncToValidBPP(
 		else
 			return DesiredBPP;
 	}
-
-	*RequiredSlots = dml_ceil(DesiredBPP / MaxLinkBPP * 64, 1);
-
-	return BPP_INVALID;
 } // TruncToValidBPP
 
 double dml32_RequiredDTBCLK(
@@ -1975,8 +1973,8 @@ void dml32_CalculateVMRowAndSwath(
 	unsigned int PTEBufferSizeInRequestsForChroma[DC__NUM_DPP__MAX];
 	unsigned int PDEAndMetaPTEBytesFrameY;
 	unsigned int PDEAndMetaPTEBytesFrameC;
-	unsigned int MetaRowByteY[DC__NUM_DPP__MAX];
-	unsigned int MetaRowByteC[DC__NUM_DPP__MAX];
+	unsigned int MetaRowByteY[DC__NUM_DPP__MAX] = {0};
+	unsigned int MetaRowByteC[DC__NUM_DPP__MAX] = {0};
 	unsigned int PixelPTEBytesPerRowY[DC__NUM_DPP__MAX];
 	unsigned int PixelPTEBytesPerRowC[DC__NUM_DPP__MAX];
 	unsigned int PixelPTEBytesPerRowY_one_row_per_frame[DC__NUM_DPP__MAX];
@@ -4291,7 +4289,7 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport(
 	unsigned int i, j, k;
 	unsigned int SurfaceWithMinActiveFCLKChangeMargin = 0;
 	unsigned int DRAMClockChangeSupportNumber = 0;
-	unsigned int LastSurfaceWithoutMargin;
+	unsigned int LastSurfaceWithoutMargin = 0;
 	unsigned int DRAMClockChangeMethod = 0;
 	bool FoundFirstSurfaceWithMinActiveFCLKChangeMargin = false;
 	double MinActiveFCLKChangeMargin = 0.;
@@ -5656,9 +5654,9 @@ void dml32_CalculateStutterEfficiency(
 	double LastZ8StutterPeriod = 0.0;
 	double LastStutterPeriod = 0.0;
 	unsigned int TotalNumberOfActiveOTG = 0;
-	double doublePixelClock;
-	unsigned int doubleHTotal;
-	unsigned int doubleVTotal;
+	double doublePixelClock = 0;
+	unsigned int doubleHTotal = 0;
+	unsigned int doubleVTotal = 0;
 	bool SameTiming = true;
 	double DETBufferingTimeY;
 	double SwathWidthYCriticalSurface = 0.0;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c
index 21e0eef3269b..60f251cf973b 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c
@@ -195,9 +195,9 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_5_soc = {
 	.dcn_downspread_percent = 0.5,
 	.gpuvm_min_page_size_bytes = 4096,
 	.hostvm_min_page_size_bytes = 4096,
-	.do_urgent_latency_adjustment = 0,
+	.do_urgent_latency_adjustment = 1,
 	.urgent_latency_adjustment_fabric_clock_component_us = 0,
-	.urgent_latency_adjustment_fabric_clock_reference_mhz = 0,
+	.urgent_latency_adjustment_fabric_clock_reference_mhz = 3000,
 };
 
 void dcn35_build_wm_range_table_fpu(struct clk_mgr *clk_mgr)
@@ -439,7 +439,7 @@ int dcn35_populate_dml_pipes_from_context_fpu(struct dc *dc,
 {
 	int i, pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
-	struct pipe_ctx *pipe;
+	struct pipe_ctx *pipe = 0;
 	bool upscaled = false;
 	const unsigned int max_allowed_vblank_nom = 1023;
 
@@ -577,6 +577,7 @@ void dcn35_decide_zstate_support(struct dc *dc, struct dc_state *context)
 {
 	enum dcn_zstate_support_state support = DCN_ZSTATE_SUPPORT_DISALLOW;
 	unsigned int i, plane_count = 0;
+	DC_LOGGER_INIT(dc->ctx->logger);
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		if (context->res_ctx.pipe_ctx[i].plane_state)
@@ -602,11 +603,14 @@ void dcn35_decide_zstate_support(struct dc *dc, struct dc_state *context)
 		if (is_pwrseq0 && allow_z10)
 			support = DCN_ZSTATE_SUPPORT_ALLOW;
 		else if (is_pwrseq0 && (is_psr || is_replay))
-			support = allow_z8 ? DCN_ZSTATE_SUPPORT_ALLOW_Z8_Z10_ONLY : DCN_ZSTATE_SUPPORT_ALLOW_Z10_ONLY;
+			support = DCN_ZSTATE_SUPPORT_ALLOW_Z8_Z10_ONLY;
 		else if (allow_z8)
 			support = DCN_ZSTATE_SUPPORT_ALLOW_Z8_ONLY;
 
 	}
 
+	DC_LOG_SMU("zstate_support: %d, StutterPeriod: %d\n", support,
+		   (int)context->bw_ctx.dml.vba.StutterPeriod);
+
 	context->bw_ctx.bw.dcn.clk.zstate_support = support;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn351/dcn351_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn351/dcn351_fpu.c
index b3ffab77cf88..e4f333d4fb54 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn351/dcn351_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn351/dcn351_fpu.c
@@ -237,7 +237,6 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_51_soc = {
 	.urgent_latency_adjustment_fabric_clock_component_us = 0,
 	.urgent_latency_adjustment_fabric_clock_reference_mhz = 0,
 	.num_chans = 4,
-	.dram_clock_change_latency_us = 11.72,
 	.dispclk_dppclk_vco_speed_mhz = 2400.0,
 };
 
@@ -474,7 +473,7 @@ int dcn351_populate_dml_pipes_from_context_fpu(struct dc *dc,
 {
 	int i, pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
-	struct pipe_ctx *pipe;
+	struct pipe_ctx *pipe = 0;
 	bool upscaled = false;
 	const unsigned int max_allowed_vblank_nom = 1023;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2/Makefile b/drivers/gpu/drm/amd/display/dc/dml2/Makefile
index acff3449b8d7..c576bb0c780f 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dml2/Makefile
@@ -24,40 +24,8 @@
 #
 # Makefile for dml2.
 
-ifdef CONFIG_X86
-dml2_ccflags-$(CONFIG_CC_IS_GCC) := -mhard-float
-dml2_ccflags := $(dml2_ccflags-y) -msse
-endif
-
-ifdef CONFIG_PPC64
-dml2_ccflags := -mhard-float -maltivec
-endif
-
-ifdef CONFIG_ARM64
-dml2_rcflags := -mgeneral-regs-only
-endif
-
-ifdef CONFIG_LOONGARCH
-dml2_ccflags := -mfpu=64
-dml2_rcflags := -msoft-float
-endif
-
-ifdef CONFIG_CC_IS_GCC
-ifeq ($(call cc-ifversion, -lt, 0701, y), y)
-IS_OLD_GCC = 1
-endif
-endif
-
-ifdef CONFIG_X86
-ifdef IS_OLD_GCC
-# Stack alignment mismatch, proceed with caution.
-# GCC < 7.1 cannot compile code using `double` and -mpreferred-stack-boundary=3
-# (8B stack alignment).
-dml2_ccflags += -mpreferred-stack-boundary=4
-else
-dml2_ccflags += -msse2
-endif
-endif
+dml2_ccflags := $(CC_FLAGS_FPU)
+dml2_rcflags := $(CC_FLAGS_NO_FPU)
 
 ifneq ($(CONFIG_FRAME_WARN),0)
 ifeq ($(filter y,$(CONFIG_KASAN)$(CONFIG_KCSAN)),y)
@@ -67,6 +35,7 @@ frame_warn_flag := -Wframe-larger-than=2048
 endif
 endif
 
+subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/dml2
 CFLAGS_$(AMDDALPATH)/dc/dml2/display_mode_core.o := $(dml2_ccflags) $(frame_warn_flag)
 CFLAGS_$(AMDDALPATH)/dc/dml2/display_mode_util.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2/dml2_wrapper.o := $(dml2_ccflags)
diff --git a/drivers/gpu/drm/amd/display/dc/dml2/display_mode_core.c b/drivers/gpu/drm/amd/display/dc/dml2/display_mode_core.c
index 9be5ebf3a8c0..3e919f5c00ca 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2/display_mode_core.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2/display_mode_core.c
@@ -31,6 +31,8 @@
 #include "dml_assert.h"
 
 #define DML2_MAX_FMT_420_BUFFER_WIDTH 4096
+#define TB_BORROWED_MAX 400
+
 // ---------------------------
 //  Declaration Begins
 // ---------------------------
@@ -2782,6 +2784,8 @@ static dml_float_t TruncToValidBPP(
 		}
 	}
 
+	*RequiredSlots = (dml_uint_t)(dml_ceil(DesiredBPP / MaxLinkBPP * 64, 1));
+
 	if (DesiredBPP == 0) {
 		if (DSCEnable) {
 			if (MaxLinkBPP < MinDSCBPP) {
@@ -2810,10 +2814,6 @@ static dml_float_t TruncToValidBPP(
 			return DesiredBPP;
 		}
 	}
-
-	*RequiredSlots = (dml_uint_t)(dml_ceil(DesiredBPP / MaxLinkBPP * 64, 1));
-
-	return __DML_DPP_INVALID__;
 } // TruncToValidBPP
 
 static void CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport(
@@ -3790,9 +3790,9 @@ static void CalculateStutterEfficiency(struct display_mode_lib_scratch_st *scrat
 	dml_bool_t FoundCriticalSurface = false;
 
 	dml_uint_t TotalNumberOfActiveOTG = 0;
-	dml_float_t SinglePixelClock;
-	dml_uint_t SingleHTotal;
-	dml_uint_t SingleVTotal;
+	dml_float_t SinglePixelClock = 0;
+	dml_uint_t SingleHTotal = 0;
+	dml_uint_t SingleVTotal = 0;
 	dml_bool_t SameTiming = true;
 
 	dml_float_t LastStutterPeriod = 0.0;
@@ -9460,8 +9460,10 @@ void dml_core_mode_programming(struct display_mode_lib_st *mode_lib, const struc
 
 		/* Copy the calculated watermarks to mp.Watermark as the getter functions are
 		 * implemented by the DML team to copy the calculated values from the mp.Watermark interface.
+		 * &mode_lib->mp.Watermark and &locals->Watermark are the same address, memcpy may lead to
+		 * unexpected behavior. memmove should be used.
 		 */
-		memcpy(&mode_lib->mp.Watermark, CalculateWatermarks_params->Watermark, sizeof(struct Watermarks));
+		memmove(&mode_lib->mp.Watermark, CalculateWatermarks_params->Watermark, sizeof(struct Watermarks));
 
 		for (k = 0; k < mode_lib->ms.num_active_planes; ++k) {
 			if (mode_lib->ms.cache_display_cfg.writeback.WritebackEnable[k] == true) {
@@ -10214,6 +10216,7 @@ dml_get_var_func(fraction_of_urgent_bandwidth_imm_flip, dml_float_t, mode_lib->m
 dml_get_var_func(urgent_latency, dml_float_t, mode_lib->mp.UrgentLatency);
 dml_get_var_func(clk_dcf_deepsleep, dml_float_t, mode_lib->mp.DCFCLKDeepSleep);
 dml_get_var_func(wm_writeback_dram_clock_change, dml_float_t, mode_lib->mp.Watermark.WritebackDRAMClockChangeWatermark);
+dml_get_var_func(wm_writeback_urgent, dml_float_t, mode_lib->mp.Watermark.WritebackUrgentWatermark);
 dml_get_var_func(stutter_efficiency, dml_float_t, mode_lib->mp.StutterEfficiency);
 dml_get_var_func(stutter_efficiency_no_vblank, dml_float_t, mode_lib->mp.StutterEfficiencyNotIncludingVBlank);
 dml_get_var_func(stutter_efficiency_z8, dml_float_t, mode_lib->mp.Z8StutterEfficiency);
diff --git a/drivers/gpu/drm/amd/display/dc/dml2/display_mode_core.h b/drivers/gpu/drm/amd/display/dc/dml2/display_mode_core.h
index 8452485684f5..3116b88e99dc 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2/display_mode_core.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2/display_mode_core.h
@@ -94,6 +94,7 @@ dml_get_var_decl(wm_usr_retraining, dml_float_t);
 dml_get_var_decl(urgent_latency, dml_float_t);
 
 dml_get_var_decl(wm_writeback_dram_clock_change, dml_float_t);
+dml_get_var_decl(wm_writeback_urgent, dml_float_t);
 dml_get_var_decl(stutter_efficiency_no_vblank, dml_float_t);
 dml_get_var_decl(stutter_efficiency, dml_float_t);
 dml_get_var_decl(stutter_efficiency_z8, dml_float_t);
diff --git a/drivers/gpu/drm/amd/display/dc/dml2/display_mode_lib_defines.h b/drivers/gpu/drm/amd/display/dc/dml2/display_mode_lib_defines.h
index de63364be01d..14d389525296 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2/display_mode_lib_defines.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2/display_mode_lib_defines.h
@@ -41,6 +41,7 @@
 #define DCN_DML__VM_PRESENT__1 1
 #define DCN_DML__HOST_VM_PRESENT 1
 #define DCN_DML__HOST_VM_PRESENT__1 1
+#define DCN_DML__DWB 1
 
 #include "dml_depedencies.h"
 
@@ -59,6 +60,7 @@
 #define __DML_NUM_PLANES__                              DCN_DML__NUM_PLANE
 #define __DML_NUM_CURSORS__                             DCN_DML__NUM_CURSOR
 #define __DML_DPP_INVALID__                             0
+#define __DML_NUM_DMB__                                 DCN_DML__DWB
 #define __DML_PIPE_NO_PLANE__                           99
 
 #define __DML_MAX_STATE_ARRAY_SIZE__        DCN_DML__NUM_PWR_STATE
diff --git a/drivers/gpu/drm/amd/display/dc/dml2/dml2_dc_resource_mgmt.c b/drivers/gpu/drm/amd/display/dc/dml2/dml2_dc_resource_mgmt.c
index a52c594e1ba4..ad2a6b4769fe 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2/dml2_dc_resource_mgmt.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2/dml2_dc_resource_mgmt.c
@@ -88,7 +88,8 @@ static int find_disp_cfg_idx_by_plane_id(struct dml2_dml_to_dc_pipe_mapping *map
 			return  i;
 	}
 
-	return -1;
+	ASSERT(false);
+	return __DML2_WRAPPER_MAX_STREAMS_PLANES__;
 }
 
 static int find_disp_cfg_idx_by_stream_id(struct dml2_dml_to_dc_pipe_mapping *mapping, unsigned int stream_id)
@@ -100,7 +101,8 @@ static int find_disp_cfg_idx_by_stream_id(struct dml2_dml_to_dc_pipe_mapping *ma
 			return  i;
 	}
 
-	return -1;
+	ASSERT(false);
+	return __DML2_WRAPPER_MAX_STREAMS_PLANES__;
 }
 
 // The master pipe of a stream is defined as the top pipe in odm slice 0
@@ -793,8 +795,8 @@ static void map_pipes_for_plane(struct dml2_context *ctx, struct dc_state *state
 	free_unused_pipes_for_plane(ctx, state, plane, &scratch->pipe_pool, stream->stream_id, plane_index);
 }
 
-static unsigned int get_mpc_factor(struct dml2_context *ctx,
-		const struct dc_state *state,
+static unsigned int get_target_mpc_factor(struct dml2_context *ctx,
+		struct dc_state *state,
 		const struct dml_display_cfg_st *disp_cfg,
 		struct dml2_dml_to_dc_pipe_mapping *mapping,
 		const struct dc_stream_status *status,
@@ -805,10 +807,10 @@ static unsigned int get_mpc_factor(struct dml2_context *ctx,
 	unsigned int cfg_idx;
 	unsigned int mpc_factor;
 
-	get_plane_id(ctx, state, status->plane_states[plane_idx],
-			stream->stream_id, plane_idx, &plane_id);
-	cfg_idx = find_disp_cfg_idx_by_plane_id(mapping, plane_id);
 	if (ctx->architecture == dml2_architecture_20) {
+		get_plane_id(ctx, state, status->plane_states[plane_idx],
+				stream->stream_id, plane_idx, &plane_id);
+		cfg_idx = find_disp_cfg_idx_by_plane_id(mapping, plane_id);
 		mpc_factor = (unsigned int)disp_cfg->hw.DPPPerSurface[cfg_idx];
 	} else {
 		mpc_factor = 1;
@@ -822,16 +824,18 @@ static unsigned int get_mpc_factor(struct dml2_context *ctx,
 	return mpc_factor;
 }
 
-static unsigned int get_odm_factor(
+static unsigned int get_target_odm_factor(
 		const struct dml2_context *ctx,
+		struct dc_state *state,
 		const struct dml_display_cfg_st *disp_cfg,
 		struct dml2_dml_to_dc_pipe_mapping *mapping,
 		const struct dc_stream_state *stream)
 {
-	unsigned int cfg_idx = find_disp_cfg_idx_by_stream_id(
-			mapping, stream->stream_id);
+	unsigned int cfg_idx;
 
-	if (ctx->architecture == dml2_architecture_20)
+	if (ctx->architecture == dml2_architecture_20) {
+		cfg_idx = find_disp_cfg_idx_by_stream_id(
+				mapping, stream->stream_id);
 		switch (disp_cfg->hw.ODMMode[cfg_idx]) {
 		case dml_odm_mode_bypass:
 			return 1;
@@ -842,83 +846,122 @@ static unsigned int get_odm_factor(
 		default:
 			break;
 		}
+	}
 	ASSERT(false);
 	return 1;
 }
 
+static unsigned int get_source_odm_factor(const struct dml2_context *ctx,
+		struct dc_state *state,
+		const struct dc_stream_state *stream)
+{
+	struct pipe_ctx *otg_master = ctx->config.callbacks.get_otg_master_for_stream(&state->res_ctx, stream);
+
+	return ctx->config.callbacks.get_odm_slice_count(otg_master);
+}
+
+static unsigned int get_source_mpc_factor(const struct dml2_context *ctx,
+		struct dc_state *state,
+		const struct dc_plane_state *plane)
+{
+	struct pipe_ctx *dpp_pipes[MAX_PIPES] = {0};
+	int dpp_pipe_count = ctx->config.callbacks.get_dpp_pipes_for_plane(plane,
+			&state->res_ctx, dpp_pipes);
+
+	ASSERT(dpp_pipe_count > 0);
+	return ctx->config.callbacks.get_mpc_slice_count(dpp_pipes[0]);
+}
+
+
 static void populate_mpc_factors_for_stream(
 		struct dml2_context *ctx,
 		const struct dml_display_cfg_st *disp_cfg,
 		struct dml2_dml_to_dc_pipe_mapping *mapping,
-		const struct dc_state *state,
+		struct dc_state *state,
 		unsigned int stream_idx,
-		unsigned int odm_factor,
-		unsigned int mpc_factors[MAX_PIPES])
+		struct dml2_pipe_combine_factor odm_factor,
+		struct dml2_pipe_combine_factor mpc_factors[MAX_PIPES])
 {
 	const struct dc_stream_status *status = &state->stream_status[stream_idx];
 	int i;
 
-	for (i = 0; i < status->plane_count; i++)
-		if (odm_factor == 1)
-			mpc_factors[i] = get_mpc_factor(
-					ctx, state, disp_cfg, mapping, status,
-					state->streams[stream_idx], i);
-		else
-			mpc_factors[i] = 1;
+	for (i = 0; i < status->plane_count; i++) {
+		mpc_factors[i].source = get_source_mpc_factor(ctx, state, status->plane_states[i]);
+		mpc_factors[i].target = (odm_factor.target == 1) ?
+				get_target_mpc_factor(ctx, state, disp_cfg, mapping, status, state->streams[stream_idx], i) : 1;
+	}
 }
 
 static void populate_odm_factors(const struct dml2_context *ctx,
 		const struct dml_display_cfg_st *disp_cfg,
 		struct dml2_dml_to_dc_pipe_mapping *mapping,
-		const struct dc_state *state,
-		unsigned int odm_factors[MAX_PIPES])
+		struct dc_state *state,
+		struct dml2_pipe_combine_factor odm_factors[MAX_PIPES])
 {
 	int i;
 
-	for (i = 0; i < state->stream_count; i++)
-		odm_factors[i] = get_odm_factor(
-				ctx, disp_cfg, mapping, state->streams[i]);
+	for (i = 0; i < state->stream_count; i++) {
+		odm_factors[i].source = get_source_odm_factor(ctx, state, state->streams[i]);
+		odm_factors[i].target = get_target_odm_factor(
+				ctx, state, disp_cfg, mapping, state->streams[i]);
+	}
 }
 
-static bool map_dc_pipes_for_stream(struct dml2_context *ctx,
+static bool unmap_dc_pipes_for_stream(struct dml2_context *ctx,
 		struct dc_state *state,
 		const struct dc_state *existing_state,
 		const struct dc_stream_state *stream,
 		const struct dc_stream_status *status,
-		unsigned int odm_factor,
-		unsigned int mpc_factors[MAX_PIPES])
+		struct dml2_pipe_combine_factor odm_factor,
+		struct dml2_pipe_combine_factor mpc_factors[MAX_PIPES])
 {
 	int plane_idx;
 	bool result = true;
 
-	if (odm_factor == 1)
-		/*
-		 * ODM and MPC combines are by DML design mutually exclusive.
-		 * ODM factor of 1 means MPC factors may be greater than 1.
-		 * In this case, we want to set ODM factor to 1 first to free up
-		 * pipe resources from previous ODM configuration before setting
-		 * up MPC combine to acquire more pipe resources.
-		 */
+	for (plane_idx = 0; plane_idx < status->plane_count; plane_idx++)
+		if (mpc_factors[plane_idx].target < mpc_factors[plane_idx].source)
+			result &= ctx->config.callbacks.update_pipes_for_plane_with_slice_count(
+					state,
+					existing_state,
+					ctx->config.callbacks.dc->res_pool,
+					status->plane_states[plane_idx],
+					mpc_factors[plane_idx].target);
+	if (odm_factor.target < odm_factor.source)
 		result &= ctx->config.callbacks.update_pipes_for_stream_with_slice_count(
 				state,
 				existing_state,
 				ctx->config.callbacks.dc->res_pool,
 				stream,
-				odm_factor);
+				odm_factor.target);
+	return result;
+}
+
+static bool map_dc_pipes_for_stream(struct dml2_context *ctx,
+		struct dc_state *state,
+		const struct dc_state *existing_state,
+		const struct dc_stream_state *stream,
+		const struct dc_stream_status *status,
+		struct dml2_pipe_combine_factor odm_factor,
+		struct dml2_pipe_combine_factor mpc_factors[MAX_PIPES])
+{
+	int plane_idx;
+	bool result = true;
+
 	for (plane_idx = 0; plane_idx < status->plane_count; plane_idx++)
-		result &= ctx->config.callbacks.update_pipes_for_plane_with_slice_count(
-				state,
-				existing_state,
-				ctx->config.callbacks.dc->res_pool,
-				status->plane_states[plane_idx],
-				mpc_factors[plane_idx]);
-	if (odm_factor > 1)
+		if (mpc_factors[plane_idx].target > mpc_factors[plane_idx].source)
+			result &= ctx->config.callbacks.update_pipes_for_plane_with_slice_count(
+					state,
+					existing_state,
+					ctx->config.callbacks.dc->res_pool,
+					status->plane_states[plane_idx],
+					mpc_factors[plane_idx].target);
+	if (odm_factor.target > odm_factor.source)
 		result &= ctx->config.callbacks.update_pipes_for_stream_with_slice_count(
 				state,
 				existing_state,
 				ctx->config.callbacks.dc->res_pool,
 				stream,
-				odm_factor);
+				odm_factor.target);
 	return result;
 }
 
@@ -928,20 +971,20 @@ static bool map_dc_pipes_with_callbacks(struct dml2_context *ctx,
 		struct dml2_dml_to_dc_pipe_mapping *mapping,
 		const struct dc_state *existing_state)
 {
-	unsigned int odm_factors[MAX_PIPES];
-	unsigned int mpc_factors_for_stream[MAX_PIPES];
 	int i;
 	bool result = true;
 
-	populate_odm_factors(ctx, disp_cfg, mapping, state, odm_factors);
-	for (i = 0; i < state->stream_count; i++) {
+	populate_odm_factors(ctx, disp_cfg, mapping, state, ctx->pipe_combine_scratch.odm_factors);
+	for (i = 0; i < state->stream_count; i++)
 		populate_mpc_factors_for_stream(ctx, disp_cfg, mapping, state,
-				i, odm_factors[i], mpc_factors_for_stream);
-		result &= map_dc_pipes_for_stream(ctx, state, existing_state,
-				state->streams[i],
-				&state->stream_status[i],
-				odm_factors[i], mpc_factors_for_stream);
-	}
+				i, ctx->pipe_combine_scratch.odm_factors[i], ctx->pipe_combine_scratch.mpc_factors[i]);
+	for (i = 0; i < state->stream_count; i++)
+		result &= unmap_dc_pipes_for_stream(ctx, state, existing_state, state->streams[i],
+				&state->stream_status[i], ctx->pipe_combine_scratch.odm_factors[i], ctx->pipe_combine_scratch.mpc_factors[i]);
+	for (i = 0; i < state->stream_count; i++)
+		result &= map_dc_pipes_for_stream(ctx, state, existing_state, state->streams[i],
+				&state->stream_status[i], ctx->pipe_combine_scratch.odm_factors[i], ctx->pipe_combine_scratch.mpc_factors[i]);
+
 	return result;
 }
 
@@ -1037,6 +1080,12 @@ bool dml2_map_dc_pipes(struct dml2_context *ctx, struct dc_state *state, const s
 				ASSERT(false);
 			}
 		}
+
+		if (ctx->config.callbacks.build_test_pattern_params &&
+				pipe->stream &&
+				pipe->prev_odm_pipe == NULL &&
+				pipe->top_pipe == NULL)
+			ctx->config.callbacks.build_test_pattern_params(&state->res_ctx, pipe);
 	}
 
 	return true;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2/dml2_dc_resource_mgmt.h b/drivers/gpu/drm/amd/display/dc/dml2/dml2_dc_resource_mgmt.h
index 2f91244a7b01..1538b708d8be 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2/dml2_dc_resource_mgmt.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2/dml2_dc_resource_mgmt.h
@@ -30,6 +30,8 @@
 #include "dml2_dc_types.h"
 
 struct dml2_context;
+struct dml2_dml_to_dc_pipe_mapping;
+struct dml_display_cfg_st;
 
 /*
  * dml2_map_dc_pipes - Creates a pipe linkage in dc_state based on current display config.
diff --git a/drivers/gpu/drm/amd/display/dc/dml2/dml2_internal_types.h b/drivers/gpu/drm/amd/display/dc/dml2/dml2_internal_types.h
index 1cf8a884c0fb..9dab4e43c511 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2/dml2_internal_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2/dml2_internal_types.h
@@ -109,10 +109,21 @@ enum dml2_architecture {
 	dml2_architecture_20,
 };
 
+struct dml2_pipe_combine_factor {
+	unsigned int source;
+	unsigned int target;
+};
+
+struct dml2_pipe_combine_scratch {
+	struct dml2_pipe_combine_factor odm_factors[MAX_PIPES];
+	struct dml2_pipe_combine_factor mpc_factors[MAX_PIPES][MAX_PIPES];
+};
+
 struct dml2_context {
 	enum dml2_architecture architecture;
 	struct dml2_configuration_options config;
 	struct dml2_helper_det_policy_scratch det_helper_scratch;
+	struct dml2_pipe_combine_scratch pipe_combine_scratch;
 	union {
 		struct {
 			struct display_mode_lib_st dml_core_ctx;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2/dml2_translation_helper.c b/drivers/gpu/drm/amd/display/dc/dml2/dml2_translation_helper.c
index a20f28a5d2e7..a41812598ce8 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2/dml2_translation_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2/dml2_translation_helper.c
@@ -29,6 +29,7 @@
 #include "dml2_translation_helper.h"
 
 #define NUM_DCFCLK_STAS 5
+#define NUM_DCFCLK_STAS_NEW 8
 
 void dml2_init_ip_params(struct dml2_context *dml2, const struct dc *in_dc, struct ip_params_st *out)
 {
@@ -249,12 +250,21 @@ void dml2_init_soc_states(struct dml2_context *dml2, const struct dc *in_dc,
 {
 	struct dml2_policy_build_synthetic_soc_states_scratch *s = &dml2->v20.scratch.create_scratch.build_synthetic_socbb_scratch;
 	struct dml2_policy_build_synthetic_soc_states_params *p = &dml2->v20.scratch.build_synthetic_socbb_params;
-	unsigned int dcfclk_stas_mhz[NUM_DCFCLK_STAS];
+	unsigned int dcfclk_stas_mhz[NUM_DCFCLK_STAS] = {0};
+	unsigned int dcfclk_stas_mhz_new[NUM_DCFCLK_STAS_NEW] = {0};
+	unsigned int dml_project = dml2->v20.dml_core_ctx.project;
+
 	unsigned int i = 0;
 	unsigned int transactions_per_mem_clock = 16; // project specific, depends on used Memory type
 
-	p->dcfclk_stas_mhz = dcfclk_stas_mhz;
-	p->num_dcfclk_stas = NUM_DCFCLK_STAS;
+	if (dml_project == dml_project_dcn351) {
+		p->dcfclk_stas_mhz = dcfclk_stas_mhz_new;
+		p->num_dcfclk_stas = NUM_DCFCLK_STAS_NEW;
+	} else {
+		p->dcfclk_stas_mhz = dcfclk_stas_mhz;
+		p->num_dcfclk_stas = NUM_DCFCLK_STAS;
+	}
+
 	p->in_bbox = in_bbox;
 	p->out_states = out;
 	p->in_states = &dml2->v20.scratch.create_scratch.in_states;
@@ -432,8 +442,7 @@ void dml2_init_soc_states(struct dml2_context *dml2, const struct dc *in_dc,
 	}
 
 	dml2_policy_build_synthetic_soc_states(s, p);
-	if (dml2->v20.dml_core_ctx.project == dml_project_dcn35 ||
-		dml2->v20.dml_core_ctx.project == dml_project_dcn351) {
+	if (dml2->v20.dml_core_ctx.project == dml_project_dcn35) {
 		// Override last out_state with data from last in_state
 		// This will ensure that out_state contains max fclk
 		memcpy(&p->out_states->state_array[p->out_states->num_states - 1],
@@ -1052,7 +1061,46 @@ static void dml2_populate_pipe_to_plane_index_mapping(struct dml2_context *dml2,
 		plane_index = 0;
 	}
 }
-
+static void populate_dml_writeback_cfg_from_stream_state(struct dml_writeback_cfg_st *out,
+		unsigned int location, const struct dc_stream_state *in)
+{
+	if (in->num_wb_info > 0) {
+		for (int i = 0; i < __DML_NUM_DMB__; i++) {
+			const struct dc_writeback_info *wb_info = &in->writeback_info[i];
+			/*current dml support 1 dwb per stream, limitation*/
+			if (wb_info->wb_enabled) {
+				out->WritebackEnable[location] = wb_info->wb_enabled;
+				out->ActiveWritebacksPerSurface[location] = wb_info->dwb_params.cnv_params.src_width;
+				out->WritebackDestinationWidth[location] = wb_info->dwb_params.dest_width;
+				out->WritebackDestinationHeight[location] = wb_info->dwb_params.dest_height;
+
+				out->WritebackSourceWidth[location] = wb_info->dwb_params.cnv_params.crop_en ?
+					wb_info->dwb_params.cnv_params.crop_width :
+					wb_info->dwb_params.cnv_params.src_width;
+
+				out->WritebackSourceHeight[location] = wb_info->dwb_params.cnv_params.crop_en ?
+					wb_info->dwb_params.cnv_params.crop_height :
+					wb_info->dwb_params.cnv_params.src_height;
+				/*current design does not have chroma scaling, need to follow up*/
+				out->WritebackHTaps[location] = wb_info->dwb_params.scaler_taps.h_taps > 0 ?
+					wb_info->dwb_params.scaler_taps.h_taps : 1;
+				out->WritebackVTaps[location] = wb_info->dwb_params.scaler_taps.v_taps > 0 ?
+					wb_info->dwb_params.scaler_taps.v_taps : 1;
+
+				out->WritebackHRatio[location] = wb_info->dwb_params.cnv_params.crop_en ?
+					(double)wb_info->dwb_params.cnv_params.crop_width /
+						(double)wb_info->dwb_params.dest_width :
+					(double)wb_info->dwb_params.cnv_params.src_width /
+						(double)wb_info->dwb_params.dest_width;
+				out->WritebackVRatio[location] = wb_info->dwb_params.cnv_params.crop_en ?
+					(double)wb_info->dwb_params.cnv_params.crop_height /
+						(double)wb_info->dwb_params.dest_height :
+					(double)wb_info->dwb_params.cnv_params.src_height /
+						(double)wb_info->dwb_params.dest_height;
+			}
+		}
+	}
+}
 void map_dc_state_into_dml_display_cfg(struct dml2_context *dml2, struct dc_state *context, struct dml_display_cfg_st *dml_dispcfg)
 {
 	int i = 0, j = 0, k = 0;
@@ -1097,6 +1145,10 @@ void map_dc_state_into_dml_display_cfg(struct dml2_context *dml2, struct dc_stat
 
 		populate_dml_timing_cfg_from_stream_state(&dml_dispcfg->timing, disp_cfg_stream_location, context->streams[i]);
 		populate_dml_output_cfg_from_stream_state(&dml_dispcfg->output, disp_cfg_stream_location, context->streams[i], current_pipe_context);
+		/*Call site for populate_dml_writeback_cfg_from_stream_state*/
+		populate_dml_writeback_cfg_from_stream_state(&dml_dispcfg->writeback,
+			disp_cfg_stream_location, context->streams[i]);
+
 		switch (context->streams[i]->debug.force_odm_combine_segments) {
 		case 2:
 			dml2->v20.dml_core_ctx.policy.ODMUse[disp_cfg_stream_location] = dml_odm_use_policy_combine_2to1;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2/dml2_utils.c b/drivers/gpu/drm/amd/display/dc/dml2/dml2_utils.c
index f15d1dbad6a9..0f8b3336e26d 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2/dml2_utils.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2/dml2_utils.c
@@ -224,7 +224,7 @@ static int find_dml_pipe_idx_by_plane_id(struct dml2_context *ctx, unsigned int
 static bool get_plane_id(struct dml2_context *dml2, const struct dc_state *state, const struct dc_plane_state *plane,
 	unsigned int stream_id, unsigned int plane_index, unsigned int *plane_id)
 {
-	int i, j;
+	unsigned int i, j;
 	bool is_plane_duplicate = dml2->v20.scratch.plane_duplicate_exists;
 
 	if (!plane_id)
@@ -327,6 +327,8 @@ void dml2_calculate_rq_and_dlg_params(const struct dc *dc, struct dc_state *cont
 			dml_pipe_idx = dml2_helper_find_dml_pipe_idx_by_stream_id(in_ctx, context->res_ctx.pipe_ctx[dc_pipe_ctx_index].stream->stream_id);
 		}
 
+		if (dml_pipe_idx == 0xFFFFFFFF)
+			continue;
 		ASSERT(in_ctx->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_stream_id_valid[dml_pipe_idx]);
 		ASSERT(in_ctx->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_stream_id[dml_pipe_idx] == context->res_ctx.pipe_ctx[dc_pipe_ctx_index].stream->stream_id);
 
@@ -374,10 +376,16 @@ void dml2_calculate_rq_and_dlg_params(const struct dc *dc, struct dc_state *cont
 
 	context->bw_ctx.bw.dcn.clk.bw_dppclk_khz = context->bw_ctx.bw.dcn.clk.dppclk_khz;
 	context->bw_ctx.bw.dcn.clk.bw_dispclk_khz = context->bw_ctx.bw.dcn.clk.dispclk_khz;
+
 	context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz = in_ctx->v20.dml_core_ctx.states.state_array[in_ctx->v20.scratch.mode_support_params.out_lowest_state_idx].dppclk_mhz
 		* 1000;
 	context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz = in_ctx->v20.dml_core_ctx.states.state_array[in_ctx->v20.scratch.mode_support_params.out_lowest_state_idx].dispclk_mhz
 		* 1000;
+
+	if (dc->config.forced_clocks || dc->debug.max_disp_clk) {
+		context->bw_ctx.bw.dcn.clk.bw_dispclk_khz = context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz;
+		context->bw_ctx.bw.dcn.clk.bw_dppclk_khz = context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz ;
+	}
 }
 
 void dml2_extract_watermark_set(struct dcn_watermarks *watermark, struct display_mode_lib_st *dml_core_ctx)
@@ -396,6 +404,71 @@ void dml2_extract_watermark_set(struct dcn_watermarks *watermark, struct display
 	watermark->cstate_pstate.cstate_exit_z8_ns = dml_get_wm_z8_stutter(dml_core_ctx) * 1000;
 }
 
+unsigned int dml2_calc_max_scaled_time(
+		unsigned int time_per_pixel,
+		enum mmhubbub_wbif_mode mode,
+		unsigned int urgent_watermark)
+{
+	unsigned int time_per_byte = 0;
+	unsigned int total_free_entry = 0xb40;
+	unsigned int buf_lh_capability;
+	unsigned int max_scaled_time;
+
+	if (mode == PACKED_444) /* packed mode 32 bpp */
+		time_per_byte = time_per_pixel/4;
+	else if (mode == PACKED_444_FP16) /* packed mode 64 bpp */
+		time_per_byte = time_per_pixel/8;
+
+	if (time_per_byte == 0)
+		time_per_byte = 1;
+
+	buf_lh_capability = (total_free_entry*time_per_byte*32) >> 6; /* time_per_byte is in u6.6*/
+	max_scaled_time   = buf_lh_capability - urgent_watermark;
+	return max_scaled_time;
+}
+
+void dml2_extract_writeback_wm(struct dc_state *context, struct display_mode_lib_st *dml_core_ctx)
+{
+	int i, j = 0;;
+	struct mcif_arb_params *wb_arb_params = NULL;
+	struct dcn_bw_writeback *bw_writeback = NULL;
+	enum mmhubbub_wbif_mode wbif_mode = PACKED_444_FP16; /*for now*/
+
+	if (context->stream_count != 0) {
+		for (i = 0; i < context->stream_count; i++) {
+			if (context->streams[i]->num_wb_info != 0)
+				j++;
+		}
+	}
+	if (j == 0) /*no dwb */
+		return;
+	for (i = 0; i < __DML_NUM_DMB__; i++) {
+		bw_writeback = &context->bw_ctx.bw.dcn.bw_writeback;
+		wb_arb_params = &context->bw_ctx.bw.dcn.bw_writeback.mcif_wb_arb[i];
+
+		for (j = 0 ; j < 4; j++) {
+			/*current dml only has one set of watermark, need to follow up*/
+			bw_writeback->mcif_wb_arb[i].cli_watermark[j] =
+					dml_get_wm_writeback_urgent(dml_core_ctx) * 1000;
+			bw_writeback->mcif_wb_arb[i].pstate_watermark[j] =
+					dml_get_wm_writeback_dram_clock_change(dml_core_ctx) * 1000;
+		}
+		if (context->res_ctx.pipe_ctx[i].stream->phy_pix_clk != 0) {
+			/* time_per_pixel should be in u6.6 format */
+			bw_writeback->mcif_wb_arb[i].time_per_pixel =
+				(1000000 << 6) / context->res_ctx.pipe_ctx[i].stream->phy_pix_clk;
+		}
+		bw_writeback->mcif_wb_arb[i].slice_lines = 32;
+		bw_writeback->mcif_wb_arb[i].arbitration_slice = 2;
+		bw_writeback->mcif_wb_arb[i].max_scaled_time =
+			dml2_calc_max_scaled_time(wb_arb_params->time_per_pixel,
+					wbif_mode, 	wb_arb_params->cli_watermark[0]);
+		/*not required any more*/
+		bw_writeback->mcif_wb_arb[i].dram_speed_change_duration =
+			dml_get_wm_writeback_dram_clock_change(dml_core_ctx) * 1000;
+
+	}
+}
 void dml2_initialize_det_scratch(struct dml2_context *in_ctx)
 {
 	int i;
@@ -468,6 +541,9 @@ bool dml2_verify_det_buffer_configuration(struct dml2_context *in_ctx, struct dc
 			dml_pipe_idx = find_dml_pipe_idx_by_plane_id(in_ctx, plane_id);
 		else
 			dml_pipe_idx = dml2_helper_find_dml_pipe_idx_by_stream_id(in_ctx, display_state->res_ctx.pipe_ctx[i].stream->stream_id);
+
+		if (dml_pipe_idx == 0xFFFFFFFF)
+			continue;
 		total_det_allocated += dml_get_det_buffer_size_kbytes(&in_ctx->v20.dml_core_ctx, dml_pipe_idx);
 		if (total_det_allocated > max_det_size) {
 			need_recalculation = true;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2/dml2_utils.h b/drivers/gpu/drm/amd/display/dc/dml2/dml2_utils.h
index 5842d6d3c4b6..04fcfe637119 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2/dml2_utils.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2/dml2_utils.h
@@ -40,9 +40,14 @@ void dml2_util_copy_dml_output(struct dml_output_cfg_st *dml_output_array, unsig
 unsigned int dml2_util_get_maximum_odm_combine_for_output(bool force_odm_4to1, enum dml_output_encoder_class encoder, bool dsc_enabled);
 void dml2_copy_clocks_to_dc_state(struct dml2_dcn_clocks *out_clks, struct dc_state *context);
 void dml2_extract_watermark_set(struct dcn_watermarks *watermark, struct display_mode_lib_st *dml_core_ctx);
+void dml2_extract_writeback_wm(struct dc_state *context, struct display_mode_lib_st *dml_core_ctx);
 int dml2_helper_find_dml_pipe_idx_by_stream_id(struct dml2_context *ctx, unsigned int stream_id);
 bool is_dtbclk_required(const struct dc *dc, struct dc_state *context);
 bool dml2_is_stereo_timing(const struct dc_stream_state *stream);
+unsigned int dml2_calc_max_scaled_time(
+		unsigned int time_per_pixel,
+		enum mmhubbub_wbif_mode mode,
+		unsigned int urgent_watermark);
 
 /*
  * dml2_dc_construct_pipes - This function will determine if we need additional pipes based
diff --git a/drivers/gpu/drm/amd/display/dc/dml2/dml2_wrapper.c b/drivers/gpu/drm/amd/display/dc/dml2/dml2_wrapper.c
index 72cca367062e..9412d5384a41 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2/dml2_wrapper.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2/dml2_wrapper.c
@@ -570,6 +570,7 @@ static bool dml2_validate_and_build_resource(const struct dc *in_dc, struct dc_s
 	struct dml2_dcn_clocks out_clks;
 	unsigned int result = 0;
 	bool need_recalculation = false;
+	uint32_t cstate_enter_plus_exit_z8_ns;
 
 	if (!context || context->stream_count == 0)
 		return true;
@@ -639,8 +640,17 @@ static bool dml2_validate_and_build_resource(const struct dc *in_dc, struct dc_s
 		dml2_extract_watermark_set(&context->bw_ctx.bw.dcn.watermarks.b, &dml2->v20.dml_core_ctx);
 		memcpy(&context->bw_ctx.bw.dcn.watermarks.c, &dml2->v20.g6_temp_read_watermark_set, sizeof(context->bw_ctx.bw.dcn.watermarks.c));
 		dml2_extract_watermark_set(&context->bw_ctx.bw.dcn.watermarks.d, &dml2->v20.dml_core_ctx);
+		dml2_extract_writeback_wm(context, &dml2->v20.dml_core_ctx);
 		//copy for deciding zstate use
 		context->bw_ctx.dml.vba.StutterPeriod = context->bw_ctx.dml2->v20.dml_core_ctx.mp.StutterPeriod;
+
+		cstate_enter_plus_exit_z8_ns = context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_z8_ns;
+
+		if (context->bw_ctx.dml.vba.StutterPeriod < in_dc->debug.minimum_z8_residency_time &&
+				cstate_enter_plus_exit_z8_ns < in_dc->debug.minimum_z8_residency_time * 1000)
+			cstate_enter_plus_exit_z8_ns = in_dc->debug.minimum_z8_residency_time * 1000;
+
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_z8_ns = cstate_enter_plus_exit_z8_ns;
 	}
 
 	return result;
@@ -681,13 +691,13 @@ static void dml2_apply_debug_options(const struct dc *dc, struct dml2_context *d
 	}
 }
 
-bool dml2_validate(const struct dc *in_dc, struct dc_state *context, bool fast_validate)
+bool dml2_validate(const struct dc *in_dc, struct dc_state *context, struct dml2_context *dml2, bool fast_validate)
 {
 	bool out = false;
 
-	if (!(context->bw_ctx.dml2))
+	if (!dml2)
 		return false;
-	dml2_apply_debug_options(in_dc, context->bw_ctx.dml2);
+	dml2_apply_debug_options(in_dc, dml2);
 
 
 	/* Use dml_validate_only for fast_validate path */
diff --git a/drivers/gpu/drm/amd/display/dc/dml2/dml2_wrapper.h b/drivers/gpu/drm/amd/display/dc/dml2/dml2_wrapper.h
index cc662d682fd4..4a8bd2f4195e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2/dml2_wrapper.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2/dml2_wrapper.h
@@ -71,6 +71,7 @@ struct dml2_dcn_clocks {
 struct dml2_dc_callbacks {
 	struct dc *dc;
 	bool (*build_scaling_params)(struct pipe_ctx *pipe_ctx);
+	void (*build_test_pattern_params)(struct resource_context *res_ctx, struct pipe_ctx *otg_master);
 	bool (*can_support_mclk_switch_using_fw_based_vblank_stretch)(struct dc *dc, struct dc_state *context);
 	bool (*acquire_secondary_pipe_for_mpc_odm)(const struct dc *dc, struct dc_state *state, struct pipe_ctx *pri_pipe, struct pipe_ctx *sec_pipe, bool odm);
 	bool (*update_pipes_for_stream_with_slice_count)(
@@ -86,8 +87,23 @@ struct dml2_dc_callbacks {
 			const struct dc_plane_state *plane,
 			int slice_count);
 	int (*get_odm_slice_index)(const struct pipe_ctx *opp_head);
+	int (*get_odm_slice_count)(const struct pipe_ctx *opp_head);
 	int (*get_mpc_slice_index)(const struct pipe_ctx *dpp_pipe);
+	int (*get_mpc_slice_count)(const struct pipe_ctx *dpp_pipe);
 	struct pipe_ctx *(*get_opp_head)(const struct pipe_ctx *pipe_ctx);
+	struct pipe_ctx *(*get_otg_master_for_stream)(
+		struct resource_context *res_ctx,
+		const struct dc_stream_state *stream);
+	int (*get_opp_heads_for_otg_master)(const struct pipe_ctx *otg_master,
+		struct resource_context *res_ctx,
+		struct pipe_ctx *opp_heads[MAX_PIPES]);
+	int (*get_dpp_pipes_for_plane)(const struct dc_plane_state *plane,
+			struct resource_context *res_ctx,
+			struct pipe_ctx *dpp_pipes[MAX_PIPES]);
+	struct dc_stream_status *(*get_stream_status)(
+		struct dc_state *state,
+		const struct dc_stream_state *stream);
+	struct dc_stream_state *(*get_stream_from_id)(const struct dc_state *state, unsigned int id);
 };
 
 struct dml2_dc_svp_callbacks {
@@ -96,10 +112,10 @@ struct dml2_dc_svp_callbacks {
 	struct dc_stream_state* (*create_phantom_stream)(const struct dc *dc,
 			struct dc_state *state,
 			struct dc_stream_state *main_stream);
-	struct dc_plane_state* (*create_phantom_plane)(struct dc *dc,
+	struct dc_plane_state* (*create_phantom_plane)(const struct dc *dc,
 			struct dc_state *state,
 			struct dc_plane_state *main_plane);
-	enum dc_status (*add_phantom_stream)(struct dc *dc,
+	enum dc_status (*add_phantom_stream)(const struct dc *dc,
 			struct dc_state *state,
 			struct dc_stream_state *phantom_stream,
 			struct dc_stream_state *main_stream);
@@ -108,7 +124,7 @@ struct dml2_dc_svp_callbacks {
 			struct dc_stream_state *stream,
 			struct dc_plane_state *plane_state,
 			struct dc_state *context);
-	enum dc_status (*remove_phantom_stream)(struct dc *dc,
+	enum dc_status (*remove_phantom_stream)(const struct dc *dc,
 			struct dc_state *state,
 			struct dc_stream_state *stream);
 	void (*release_phantom_plane)(const struct dc *dc,
@@ -121,6 +137,15 @@ struct dml2_dc_svp_callbacks {
 	enum mall_stream_type (*get_pipe_subvp_type)(const struct dc_state *state, const struct pipe_ctx *pipe_ctx);
 	enum mall_stream_type (*get_stream_subvp_type)(const struct dc_state *state, const struct dc_stream_state *stream);
 	struct dc_stream_state *(*get_paired_subvp_stream)(const struct dc_state *state, const struct dc_stream_state *stream);
+	bool (*remove_phantom_streams_and_planes)(
+			const struct dc *dc,
+			struct dc_state *state);
+	void (*release_phantom_streams_and_planes)(
+			const struct dc *dc,
+			struct dc_state *state);
+	unsigned int (*calculate_mall_ways_from_bytes)(
+				const struct dc *dc,
+				unsigned int total_size_in_mall_bytes);
 };
 
 struct dml2_clks_table_entry {
@@ -191,6 +216,8 @@ struct dml2_configuration_options {
 	unsigned int max_segments_per_hubp;
 	unsigned int det_segment_size;
 	bool map_dc_pipes_with_callbacks;
+
+	bool use_clock_dc_limits;
 };
 
 /*
@@ -244,6 +271,7 @@ void dml2_reinit(const struct dc *in_dc,
  */
 bool dml2_validate(const struct dc *in_dc,
 				   struct dc_state *context,
+				   struct dml2_context *dml2,
 				   bool fast_validate);
 
 /*
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/Makefile b/drivers/gpu/drm/amd/display/dc/dpp/Makefile
new file mode 100644
index 000000000000..99bd36073561
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dpp/Makefile
@@ -0,0 +1,77 @@
+
+# Copyright 2022 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# Makefile for the 'dpp' sub-component of DAL.
+#
+ifdef CONFIG_DRM_AMD_DC_FP
+###############################################################################
+# DCN
+###############################################################################
+
+DPP_DCN10 = dcn10_dpp.o dcn10_dpp_dscl.o dcn10_dpp_cm.o
+
+AMD_DAL_DPP_DCN10 = $(addprefix $(AMDDALPATH)/dc/dpp/dcn10/,$(DPP_DCN10))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_DPP_DCN10)
+
+###############################################################################
+
+DPP_DCN20 = dcn20_dpp.o dcn20_dpp_cm.o
+
+AMD_DAL_DPP_DCN20 = $(addprefix $(AMDDALPATH)/dc/dpp/dcn20/,$(DPP_DCN20))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_DPP_DCN20)
+
+###############################################################################
+
+DPP_DCN201 = dcn201_dpp.o
+
+AMD_DAL_DPP_DCN201 = $(addprefix $(AMDDALPATH)/dc/dpp/dcn201/,$(DPP_DCN201))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_DPP_DCN201)
+
+###############################################################################
+
+DPP_DCN30 = dcn30_dpp.o dcn30_dpp_cm.o
+
+AMD_DAL_DPP_DCN30 = $(addprefix $(AMDDALPATH)/dc/dpp/dcn30/,$(DPP_DCN30))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_DPP_DCN30)
+
+###############################################################################
+
+DPP_DCN32 = dcn32_dpp.o
+
+AMD_DAL_DPP_DCN32 = $(addprefix $(AMDDALPATH)/dc/dpp/dcn32/,$(DPP_DCN32))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_DPP_DCN32)
+
+###############################################################################
+
+DPP_DCN35 = dcn35_dpp.o
+
+AMD_DAL_DPP_DCN35 = $(addprefix $(AMDDALPATH)/dc/dpp/dcn35/,$(DPP_DCN35))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_DPP_DCN35)
+
+###############################################################################
+
+endif
\ No newline at end of file
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/CMakeLists.txt b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/CMakeLists.txt
new file mode 100644
index 000000000000..1318c6fba3e7
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/CMakeLists.txt
@@ -0,0 +1,6 @@
+dal3_subdirectory_sources(
+  dcn10_dpp.c
+  dcn10_dpp_cm.c
+  dcn10_dpp_dscl.c
+  dcn10_dpp.h
+)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.c
index 4e391fd1d71c..e1da48b05d00 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.c
@@ -28,7 +28,7 @@
 #include "core_types.h"
 
 #include "reg_helper.h"
-#include "dcn10_dpp.h"
+#include "dcn10/dcn10_dpp.h"
 #include "basics/conversion.h"
 
 #define NUM_PHASES    64
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.h
index a039eedc7c24..c48139bed11f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.h
@@ -1090,7 +1090,8 @@
 	type DPP_CLOCK_ENABLE; \
 	type CM_HDR_MULT_COEF; \
 	type CUR0_FP_BIAS; \
-	type CUR0_FP_SCALE;
+	type CUR0_FP_SCALE;\
+	type DISPCLK_R_GATE_DISABLE;
 
 struct dcn_dpp_shift {
 	TF_REG_FIELD_LIST(uint8_t)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_cm.c
index 2f994a3a0b9c..006e23842016 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_cm.c
@@ -28,9 +28,9 @@
 #include "core_types.h"
 
 #include "reg_helper.h"
-#include "dcn10_dpp.h"
+#include "dcn10/dcn10_dpp.h"
 #include "basics/conversion.h"
-#include "dcn10_cm_common.h"
+#include "dcn10/dcn10_cm_common.h"
 
 #define NUM_PHASES    64
 #define HORZ_MAX_TAPS 8
@@ -234,7 +234,7 @@ void dpp1_cm_get_gamut_remap(struct dpp *dpp_base,
 			     struct dpp_grph_csc_adjustment *adjust)
 {
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
-	uint16_t arr_reg_val[12];
+	uint16_t arr_reg_val[12] = {0};
 	enum gamut_remap_select select;
 
 	read_gamut_remap(dpp, arr_reg_val, &select);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_dscl.c
index 5ca9ab8a76e8..808bca9fb804 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_dscl.c
@@ -28,7 +28,7 @@
 #include "core_types.h"
 
 #include "reg_helper.h"
-#include "dcn10_dpp.h"
+#include "dcn10/dcn10_dpp.h"
 #include "basics/conversion.h"
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn20/CMakeLists.txt b/drivers/gpu/drm/amd/display/dc/dpp/dcn20/CMakeLists.txt
new file mode 100644
index 000000000000..9c2d7096348e
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn20/CMakeLists.txt
@@ -0,0 +1,5 @@
+dal3_subdirectory_sources(
+  dcn20_dpp.c
+  dcn20_dpp_cm.c
+  dcn20_dpp.h
+)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn20/dcn20_dpp.c
index 1516c0a48726..56ebd7164dd7 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn20/dcn20_dpp.c
@@ -28,7 +28,7 @@
 #include "core_types.h"
 
 #include "reg_helper.h"
-#include "dcn20_dpp.h"
+#include "dcn20/dcn20_dpp.h"
 #include "basics/conversion.h"
 
 #define NUM_PHASES    64
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn20/dcn20_dpp.h
index 672cde46c4b9..49cb25c9cb36 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn20/dcn20_dpp.h
@@ -736,7 +736,7 @@ bool dpp20_program_shaper(
 
 bool dpp20_program_3dlut(
 		struct dpp *dpp_base,
-		struct tetrahedral_params *params);
+		const struct tetrahedral_params *params);
 
 void dpp2_cnv_set_alpha_keyer(
 			struct dpp *dpp_base,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn20/dcn20_dpp_cm.c
index 58dc69926e8a..31613372e214 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn20/dcn20_dpp_cm.c
@@ -28,7 +28,7 @@
 #include "core_types.h"
 
 #include "reg_helper.h"
-#include "dcn20_dpp.h"
+#include "dcn20/dcn20_dpp.h"
 #include "basics/conversion.h"
 
 #include "dcn10/dcn10_cm_common.h"
@@ -274,7 +274,7 @@ void dpp2_cm_get_gamut_remap(struct dpp *dpp_base,
 			     struct dpp_grph_csc_adjustment *adjust)
 {
 	struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
-	uint16_t arr_reg_val[12];
+	uint16_t arr_reg_val[12] = {0};
 	enum dcn20_gamut_remap_select select;
 
 	read_gamut_remap(dpp, arr_reg_val, &select);
@@ -1114,15 +1114,15 @@ static void dpp20_select_3dlut_ram_mask(
 
 bool dpp20_program_3dlut(
 		struct dpp *dpp_base,
-		struct tetrahedral_params *params)
+		const struct tetrahedral_params *params)
 {
 	enum dc_lut_mode mode;
 	bool is_17x17x17;
 	bool is_12bits_color_channel;
-	struct dc_rgb *lut0;
-	struct dc_rgb *lut1;
-	struct dc_rgb *lut2;
-	struct dc_rgb *lut3;
+	const struct dc_rgb *lut0;
+	const struct dc_rgb *lut1;
+	const struct dc_rgb *lut2;
+	const struct dc_rgb *lut3;
 	int lut_size0;
 	int lut_size;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn201/CMakeLists.txt b/drivers/gpu/drm/amd/display/dc/dpp/dcn201/CMakeLists.txt
new file mode 100644
index 000000000000..7711cd3c47a7
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn201/CMakeLists.txt
@@ -0,0 +1,4 @@
+dal3_subdirectory_sources(
+    dcn201_dpp.c
+    dcn201_dpp.h
+)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn201/dcn201_dpp.c
index f809a7d21033..345202fee40f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn201/dcn201_dpp.c
@@ -28,7 +28,7 @@
 #include "core_types.h"
 
 #include "reg_helper.h"
-#include "dcn201_dpp.h"
+#include "dcn201/dcn201_dpp.h"
 #include "basics/conversion.h"
 
 #define REG(reg)\
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn201/dcn201_dpp.h
index cbd5b47b4acf..cbd5b47b4acf 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn201/dcn201_dpp.h
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/CMakeLists.txt b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/CMakeLists.txt
new file mode 100644
index 000000000000..0faee2a1e32b
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/CMakeLists.txt
@@ -0,0 +1,5 @@
+dal3_subdirectory_sources(
+  dcn30_dpp.c
+  dcn30_dpp_cm.c
+  dcn30_dpp.h
+)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c
index a3a769aad042..f8c0cee34080 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c
@@ -26,9 +26,9 @@
 #include "dm_services.h"
 #include "core_types.h"
 #include "reg_helper.h"
-#include "dcn30_dpp.h"
+#include "dcn30/dcn30_dpp.h"
 #include "basics/conversion.h"
-#include "dcn30_cm_common.h"
+#include "dcn30/dcn30_cm_common.h"
 
 #define REG(reg)\
 	dpp->tf_regs->reg
@@ -293,9 +293,11 @@ void dpp3_cnv_setup (
 		break;
 	case SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FIX:
 		pixel_format = 112;
+		alpha_en = 0;
 		break;
 	case SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FIX:
 		pixel_format = 113;
+		alpha_en = 0;
 		break;
 	case SURFACE_PIXEL_FORMAT_VIDEO_ACrYCb2101010:
 		pixel_format = 114;
@@ -319,9 +321,11 @@ void dpp3_cnv_setup (
 		break;
 	case SURFACE_PIXEL_FORMAT_GRPH_RGB111110_FLOAT:
 		pixel_format = 118;
+		alpha_en = 0;
 		break;
 	case SURFACE_PIXEL_FORMAT_GRPH_BGR101111_FLOAT:
 		pixel_format = 119;
+		alpha_en = 0;
 		break;
 	default:
 		break;
@@ -1384,15 +1388,15 @@ static void dpp3_select_3dlut_ram_mask(
 }
 
 static bool dpp3_program_3dlut(struct dpp *dpp_base,
-			       struct tetrahedral_params *params)
+			       const struct tetrahedral_params *params)
 {
 	enum dc_lut_mode mode;
 	bool is_17x17x17;
 	bool is_12bits_color_channel;
-	struct dc_rgb *lut0;
-	struct dc_rgb *lut1;
-	struct dc_rgb *lut2;
-	struct dc_rgb *lut3;
+	const struct dc_rgb *lut0;
+	const struct dc_rgb *lut1;
+	const struct dc_rgb *lut2;
+	const struct dc_rgb *lut3;
 	int lut_size0;
 	int lut_size;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h
index 2ac8045a87a1..269f437c1633 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h
@@ -132,6 +132,8 @@
 	SRI(CM_POST_CSC_B_C33_C34, CM, id), \
 	SRI(CM_MEM_PWR_CTRL, CM, id), \
 	SRI(CM_CONTROL, CM, id), \
+	SRI(CM_TEST_DEBUG_INDEX, CM, id), \
+	SRI(CM_TEST_DEBUG_DATA, CM, id), \
 	SRI(FORMAT_CONTROL, CNVC_CFG, id), \
 	SRI(CNVC_SURFACE_PIXEL_FORMAT, CNVC_CFG, id), \
 	SRI(CURSOR0_CONTROL, CNVC_CUR, id), \
@@ -294,6 +296,7 @@
 	TF_SF(CM0_CM_POST_CSC_C11_C12, CM_POST_CSC_C12, mask_sh), \
 	TF_SF(CM0_CM_POST_CSC_C33_C34, CM_POST_CSC_C33, mask_sh), \
 	TF_SF(CM0_CM_POST_CSC_C33_C34, CM_POST_CSC_C34, mask_sh), \
+	TF_SF(CM0_CM_TEST_DEBUG_INDEX, CM_TEST_DEBUG_INDEX, mask_sh), \
 	TF_SF(CNVC_CFG0_FORMAT_CONTROL, CNVC_BYPASS, mask_sh), \
 	TF2_SF(CNVC_CFG0, FORMAT_CONTROL__ALPHA_EN, mask_sh), \
 	TF_SF(CNVC_CFG0_FORMAT_CONTROL, FORMAT_EXPANSION_MODE, mask_sh), \
@@ -426,6 +429,7 @@
 	type CM_GAMCOR_LUT_DATA; \
 	type CM_GAMCOR_LUT_WRITE_COLOR_MASK; \
 	type CM_GAMCOR_LUT_READ_COLOR_SEL; \
+	type CM_GAMCOR_LUT_READ_DBG; \
 	type CM_GAMCOR_LUT_HOST_SEL; \
 	type CM_GAMCOR_LUT_CONFIG_MODE; \
 	type CM_GAMCOR_LUT_STATUS; \
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c
index 2f5b3fbd3507..82eca0e7b7d0 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c
@@ -26,9 +26,9 @@
 #include "dm_services.h"
 #include "core_types.h"
 #include "reg_helper.h"
-#include "dcn30_dpp.h"
+#include "dcn30/dcn30_dpp.h"
 #include "basics/conversion.h"
-#include "dcn30_cm_common.h"
+#include "dcn30/dcn30_cm_common.h"
 
 #define REG(reg)\
 	dpp->tf_regs->reg
@@ -445,7 +445,7 @@ void dpp3_cm_get_gamut_remap(struct dpp *dpp_base,
 			     struct dpp_grph_csc_adjustment *adjust)
 {
 	struct dcn3_dpp *dpp = TO_DCN30_DPP(dpp_base);
-	uint16_t arr_reg_val[12];
+	uint16_t arr_reg_val[12] = {0};
 	int select;
 
 	read_gamut_remap(dpp, arr_reg_val, &select);
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn32/CMakeLists.txt b/drivers/gpu/drm/amd/display/dc/dpp/dcn32/CMakeLists.txt
new file mode 100644
index 000000000000..7743edc4599f
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn32/CMakeLists.txt
@@ -0,0 +1,4 @@
+dal3_subdirectory_sources(
+  dcn32_dpp.c
+  dcn32_dpp.h
+)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn32/dcn32_dpp.c
index 681e75c6dbaf..41679997b44d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn32/dcn32_dpp.c
@@ -26,7 +26,7 @@
 #include "dm_services.h"
 #include "core_types.h"
 #include "reg_helper.h"
-#include "dcn32_dpp.h"
+#include "dcn32/dcn32_dpp.h"
 #include "basics/conversion.h"
 #include "dcn30/dcn30_cm_common.h"
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn32/dcn32_dpp.h
index 572958d287eb..572958d287eb 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn32/dcn32_dpp.h
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn35/CMakeLists.txt b/drivers/gpu/drm/amd/display/dc/dpp/dcn35/CMakeLists.txt
new file mode 100644
index 000000000000..91df5db26435
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn35/CMakeLists.txt
@@ -0,0 +1,4 @@
+dal3_subdirectory_sources(
+  dcn35_dpp.c
+  dcn35_dpp.h
+)
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn35/dcn35_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn35/dcn35_dpp.c
new file mode 100644
index 000000000000..e16274fee31d
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn35/dcn35_dpp.c
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "core_types.h"
+#include "dcn35/dcn35_dpp.h"
+#include "reg_helper.h"
+
+#define REG(reg) dpp->tf_regs->reg
+
+#define CTX dpp->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name)                                       \
+	((const struct dcn35_dpp_shift *)(dpp->tf_shift))->field_name, \
+	((const struct dcn35_dpp_mask *)(dpp->tf_mask))->field_name
+
+void dpp35_dppclk_control(
+		struct dpp *dpp_base,
+		bool dppclk_div,
+		bool enable)
+{
+	struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
+
+	if (enable) {
+		if (dpp->tf_mask->DPPCLK_RATE_CONTROL)
+			REG_UPDATE_2(DPP_CONTROL,
+				DPPCLK_RATE_CONTROL, dppclk_div,
+				DPP_CLOCK_ENABLE, 1);
+		else
+			REG_UPDATE_2(DPP_CONTROL,
+					DPP_CLOCK_ENABLE, 1,
+					DISPCLK_R_GATE_DISABLE, 1);
+	} else
+		REG_UPDATE_2(DPP_CONTROL,
+				DPP_CLOCK_ENABLE, 0,
+				DISPCLK_R_GATE_DISABLE, 0);
+}
+
+static struct dpp_funcs dcn35_dpp_funcs = {
+	.dpp_program_gamcor_lut		= dpp3_program_gamcor_lut,
+	.dpp_read_state				= dpp30_read_state,
+	.dpp_reset					= dpp_reset,
+	.dpp_set_scaler				= dpp1_dscl_set_scaler_manual_scale,
+	.dpp_get_optimal_number_of_taps	= dpp3_get_optimal_number_of_taps,
+	.dpp_set_gamut_remap		= dpp3_cm_set_gamut_remap,
+	.dpp_set_csc_adjustment		= NULL,
+	.dpp_set_csc_default		= NULL,
+	.dpp_program_regamma_pwl	= NULL,
+	.dpp_set_pre_degam			= dpp3_set_pre_degam,
+	.dpp_program_input_lut		= NULL,
+	.dpp_full_bypass			= dpp1_full_bypass,
+	.dpp_setup					= dpp3_cnv_setup,
+	.dpp_program_degamma_pwl	= NULL,
+	.dpp_program_cm_dealpha		= dpp3_program_cm_dealpha,
+	.dpp_program_cm_bias		= dpp3_program_cm_bias,
+
+	.dpp_program_blnd_lut		= NULL, // BLNDGAM is removed completely in DCN3.2 DPP
+	.dpp_program_shaper_lut		= NULL, // CM SHAPER block is removed in DCN3.2 DPP, (it is in MPCC, programmable before or after BLND)
+	.dpp_program_3dlut			= NULL, // CM 3DLUT block is removed in DCN3.2 DPP, (it is in MPCC, programmable before or after BLND)
+
+	.dpp_program_bias_and_scale	= NULL,
+	.dpp_cnv_set_alpha_keyer	= dpp2_cnv_set_alpha_keyer,
+	.set_cursor_attributes		= dpp3_set_cursor_attributes,
+	.set_cursor_position		= dpp1_set_cursor_position,
+	.set_optional_cursor_attributes	= dpp1_cnv_set_optional_cursor_attributes,
+	.dpp_dppclk_control			= dpp35_dppclk_control,
+	.dpp_set_hdr_multiplier		= dpp3_set_hdr_multiplier,
+	.dpp_get_gamut_remap		= dpp3_cm_get_gamut_remap,
+};
+
+
+bool dpp35_construct(
+	struct dcn3_dpp *dpp, struct dc_context *ctx,
+	uint32_t inst, const struct dcn3_dpp_registers *tf_regs,
+	const struct dcn35_dpp_shift *tf_shift,
+	const struct dcn35_dpp_mask *tf_mask)
+{
+	bool ret = dpp32_construct(dpp, ctx, inst, tf_regs,
+			      (const struct dcn3_dpp_shift *)(tf_shift),
+			      (const struct dcn3_dpp_mask *)(tf_mask));
+
+	dpp->base.funcs = &dcn35_dpp_funcs;
+	return ret;
+}
+
+void dpp35_set_fgcg(struct dcn3_dpp *dpp, bool enable)
+{
+	REG_UPDATE(DPP_CONTROL, DPP_FGCG_REP_DIS, !enable);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn35/dcn35_dpp.h
index 09b84307cd9e..135872d88219 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn35/dcn35_dpp.h
@@ -31,7 +31,9 @@
 
 #define DPP_REG_LIST_SH_MASK_DCN35(mask_sh)  \
 	DPP_REG_LIST_SH_MASK_DCN30_COMMON(mask_sh), \
-		TF_SF(DPP_TOP0_DPP_CONTROL, DPP_FGCG_REP_DIS, mask_sh)
+		TF_SF(DPP_TOP0_DPP_CONTROL, DPP_FGCG_REP_DIS, mask_sh), \
+		TF_SF(DPP_TOP0_DPP_CONTROL, DPP_FGCG_REP_DIS, mask_sh), \
+		TF_SF(DPP_TOP0_DPP_CONTROL, DISPCLK_R_GATE_DISABLE, mask_sh)
 
 #define DPP_REG_FIELD_LIST_DCN35(type)         \
 	struct {                               \
@@ -47,6 +49,11 @@ struct dcn35_dpp_mask {
 	DPP_REG_FIELD_LIST_DCN35(uint32_t);
 };
 
+void dpp35_dppclk_control(
+		struct dpp *dpp_base,
+		bool dppclk_div,
+		bool enable);
+
 bool dpp35_construct(struct dcn3_dpp *dpp3, struct dc_context *ctx,
 		     uint32_t inst, const struct dcn3_dpp_registers *tf_regs,
 		     const struct dcn35_dpp_shift *tf_shift,
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
index ac41f9c0a283..150ef23440a2 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
@@ -458,7 +458,7 @@ bool dc_dsc_compute_bandwidth_range(
 	bool is_dsc_possible = false;
 	struct dsc_enc_caps dsc_enc_caps;
 	struct dsc_enc_caps dsc_common_caps;
-	struct dc_dsc_config config;
+	struct dc_dsc_config config = {0};
 	struct dc_dsc_config_options options = {0};
 
 	options.dsc_min_slice_height_override = dsc_min_slice_height_override;
@@ -868,9 +868,9 @@ static bool setup_dsc_config(
 		struct dc_dsc_config *dsc_cfg)
 {
 	struct dsc_enc_caps dsc_common_caps;
-	int max_slices_h;
-	int min_slices_h;
-	int num_slices_h;
+	int max_slices_h = 0;
+	int min_slices_h = 0;
+	int num_slices_h = 0;
 	int pic_width;
 	int slice_width;
 	int target_bpp;
@@ -1055,7 +1055,12 @@ static bool setup_dsc_config(
 	if (!is_dsc_possible)
 		goto done;
 
-	dsc_cfg->num_slices_v = pic_height/slice_height;
+	if (slice_height > 0) {
+		dsc_cfg->num_slices_v = pic_height / slice_height;
+	} else {
+		is_dsc_possible = false;
+		goto done;
+	}
 
 	if (target_bandwidth_kbps > 0) {
 		is_dsc_possible = decide_dsc_target_bpp_x16(
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
index 36d6c1646a51..59864130cf83 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
@@ -101,7 +101,6 @@ int dscc_compute_dsc_parameters(const struct drm_dsc_config *pps,
 {
 	int              ret;
 	struct drm_dsc_config   dsc_cfg;
-	unsigned long long tmp;
 
 	dsc_params->pps = *pps;
 	dsc_params->pps.initial_scale_value = 8 * rc->rc_model_size / (rc->rc_model_size - rc->initial_fullness_offset);
@@ -112,9 +111,9 @@ int dscc_compute_dsc_parameters(const struct drm_dsc_config *pps,
 	dsc_cfg.mux_word_size = dsc_params->pps.bits_per_component <= 10 ? 48 : 64;
 
 	ret = drm_dsc_compute_rc_parameters(&dsc_cfg);
-	tmp = (unsigned long long)dsc_cfg.slice_chunk_size * 0x10000000 + (dsc_cfg.slice_width - 1);
-	do_div(tmp, (uint32_t)dsc_cfg.slice_width);  //ROUND-UP
-	dsc_params->bytes_per_pixel = (uint32_t)tmp;
+	dsc_params->bytes_per_pixel =
+			(uint32_t)(div_u64(((uint64_t)dsc_cfg.slice_chunk_size * 0x10000000 + (dsc_cfg.slice_width - 1)),
+							(uint32_t)dsc_cfg.slice_width));  /* Round-up */
 
 	copy_pps_fields(&dsc_params->pps, &dsc_cfg);
 	dsc_params->rc_buffer_model_size = dsc_cfg.rc_bits;
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn21/hw_translate_dcn21.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn21/hw_translate_dcn21.c
index d734e3a134d1..2840ed5c57d8 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dcn21/hw_translate_dcn21.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn21/hw_translate_dcn21.c
@@ -95,10 +95,6 @@ static bool offset_to_id(
 			return true;
 		default:
 			ASSERT_CRITICAL(false);
-#ifdef PALLADIUM_SUPPORTED
-		*en = GPIO_DDC_LINE_DDC1;
-		return true;
-#endif
 			return false;
 		}
 	break;
@@ -184,11 +180,6 @@ static bool offset_to_id(
 	/* UNEXPECTED */
 	default:
 /*	case REG(DC_GPIO_SYNCA_A): not exista */
-#ifdef PALLADIUM_SUPPORTED
-		*id = GPIO_ID_HPD;
-		*en = GPIO_DDC_LINE_DDC1;
-		return true;
-#endif
 		ASSERT_CRITICAL(false);
 		return false;
 	}
@@ -308,10 +299,6 @@ static bool id_to_offset(
 		break;
 		default:
 			ASSERT_CRITICAL(false);
-#ifdef PALLADIUM_SUPPORTED
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK;
-			result = true;
-#endif
 			result = false;
 		}
 	break;
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c b/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c
index 3ede6e02c3a7..663c17f52779 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c
@@ -128,7 +128,7 @@ struct gpio *dal_gpio_service_create_irq(
 	uint32_t offset,
 	uint32_t mask)
 {
-	enum gpio_id id;
+	enum gpio_id id = 0;
 	uint32_t en;
 
 	if (!service->translate.funcs->offset_to_id(offset, mask, &id, &en)) {
@@ -144,7 +144,7 @@ struct gpio *dal_gpio_service_create_generic_mux(
 	uint32_t offset,
 	uint32_t mask)
 {
-	enum gpio_id id;
+	enum gpio_id id = 0;
 	uint32_t en;
 	struct gpio *generic;
 
@@ -178,7 +178,7 @@ struct gpio_pin_info dal_gpio_get_generic_pin_info(
 	enum gpio_id id,
 	uint32_t en)
 {
-	struct gpio_pin_info pin;
+	struct gpio_pin_info pin = {0};
 
 	if (service->translate.funcs->id_to_offset) {
 		service->translate.funcs->id_to_offset(id, en, &pin);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/Makefile b/drivers/gpu/drm/amd/display/dc/hwss/Makefile
index 9e8e9de51a92..cf8aa23b4415 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/hwss/Makefile
@@ -180,7 +180,7 @@ AMD_DISPLAY_FILES += $(AMD_DAL_HWSS_DCN35)
 
 ###############################################################################
 
-HWSS_DCN351 = dcn351_init.o
+HWSS_DCN351 = dcn351_hwseq.o dcn351_init.o
 
 AMD_DAL_HWSS_DCN351 = $(addprefix $(AMDDALPATH)/dc/hwss/dcn351/,$(HWSS_DCN351))
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
index 0ba1feaf96c0..0d3ea291eeee 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
@@ -249,7 +249,7 @@ static bool dce110_enable_display_power_gating(
 		return false;
 }
 
-static void build_prescale_params(struct ipp_prescale_params *prescale_params,
+static void dce110_prescale_params(struct ipp_prescale_params *prescale_params,
 		const struct dc_plane_state *plane_state)
 {
 	prescale_params->mode = IPP_PRESCALE_MODE_FIXED_UNSIGNED;
@@ -289,16 +289,14 @@ dce110_set_input_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
 	if (ipp == NULL)
 		return false;
 
-	if (plane_state->in_transfer_func)
-		tf = plane_state->in_transfer_func;
+	tf = &plane_state->in_transfer_func;
 
-	build_prescale_params(&prescale_params, plane_state);
+	dce110_prescale_params(&prescale_params, plane_state);
 	ipp->funcs->ipp_program_prescale(ipp, &prescale_params);
 
-	if (plane_state->gamma_correction &&
-			!plane_state->gamma_correction->is_identity &&
+	if (!plane_state->gamma_correction.is_identity &&
 			dce_use_lut(plane_state->format))
-		ipp->funcs->ipp_program_input_lut(ipp, plane_state->gamma_correction);
+		ipp->funcs->ipp_program_input_lut(ipp, &plane_state->gamma_correction);
 
 	if (tf == NULL) {
 		/* Default case if no input transfer function specified */
@@ -614,11 +612,10 @@ dce110_set_output_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
 	xfm->funcs->opp_power_on_regamma_lut(xfm, true);
 	xfm->regamma_params.hw_points_num = GAMMA_HW_POINTS_NUM;
 
-	if (stream->out_transfer_func &&
-	    stream->out_transfer_func->type == TF_TYPE_PREDEFINED &&
-	    stream->out_transfer_func->tf == TRANSFER_FUNCTION_SRGB) {
+	if (stream->out_transfer_func.type == TF_TYPE_PREDEFINED &&
+	    stream->out_transfer_func.tf == TRANSFER_FUNCTION_SRGB) {
 		xfm->funcs->opp_set_regamma_mode(xfm, OPP_REGAMMA_SRGB);
-	} else if (dce110_translate_regamma_to_hw_format(stream->out_transfer_func,
+	} else if (dce110_translate_regamma_to_hw_format(&stream->out_transfer_func,
 							 &xfm->regamma_params)) {
 		xfm->funcs->opp_program_regamma_pwl(xfm, &xfm->regamma_params);
 		xfm->funcs->opp_set_regamma_mode(xfm, OPP_REGAMMA_USER);
@@ -1192,16 +1189,6 @@ void dce110_disable_stream(struct pipe_ctx *pipe_ctx)
 		dccg->funcs->disable_symclk_se(dccg, stream_enc->stream_enc_inst,
 					       link_enc->transmitter - TRANSMITTER_UNIPHY_A);
 	}
-
-	if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
-		/* TODO: This looks like a bug to me as we are disabling HPO IO when
-		 * we are just disabling a single HPO stream. Shouldn't we disable HPO
-		 * HW control only when HPOs for all streams are disabled?
-		 */
-		if (pipe_ctx->stream->ctx->dc->hwseq->funcs.setup_hpo_hw_control)
-			pipe_ctx->stream->ctx->dc->hwseq->funcs.setup_hpo_hw_control(
-					pipe_ctx->stream->ctx->dc->hwseq, false);
-	}
 }
 
 void dce110_unblank_stream(struct pipe_ctx *pipe_ctx,
@@ -1550,7 +1537,7 @@ enum dc_status dce110_apply_single_controller_ctx_to_hw(
 	}
 
 	if (pipe_ctx->stream_res.audio != NULL) {
-		struct audio_output audio_output;
+		struct audio_output audio_output = {0};
 
 		build_audio_output(context, pipe_ctx, &audio_output);
 
@@ -2201,7 +2188,7 @@ static void dce110_setup_audio_dto(
 		struct dc *dc,
 		struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 
 	/* program audio wall clock. use HDMI as clock source if HDMI
 	 * audio active. Otherwise, use DP as clock source
@@ -2273,7 +2260,7 @@ static void dce110_setup_audio_dto(
 				continue;
 
 			if (pipe_ctx->stream_res.audio != NULL) {
-				struct audio_output audio_output;
+				struct audio_output audio_output = {0};
 
 				build_audio_output(context, pipe_ctx, &audio_output);
 
@@ -2288,6 +2275,19 @@ static void dce110_setup_audio_dto(
 	}
 }
 
+static bool dce110_is_hpo_enabled(struct dc_state *context)
+{
+	int i;
+
+	for (i = 0; i < MAX_HPO_DP2_ENCODERS; i++) {
+		if (context->res_ctx.is_hpo_dp_stream_enc_acquired[i]) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
 enum dc_status dce110_apply_ctx_to_hw(
 		struct dc *dc,
 		struct dc_state *context)
@@ -2296,6 +2296,8 @@ enum dc_status dce110_apply_ctx_to_hw(
 	struct dc_bios *dcb = dc->ctx->dc_bios;
 	enum dc_status status;
 	int i;
+	bool was_hpo_enabled = dce110_is_hpo_enabled(dc->current_state);
+	bool is_hpo_enabled = dce110_is_hpo_enabled(context);
 
 	/* reset syncd pipes from disabled pipes */
 	if (dc->config.use_pipe_ctx_sync_logic)
@@ -2338,6 +2340,10 @@ enum dc_status dce110_apply_ctx_to_hw(
 
 	dce110_setup_audio_dto(dc, context);
 
+	if (dc->hwseq->funcs.setup_hpo_hw_control && was_hpo_enabled != is_hpo_enabled) {
+		dc->hwseq->funcs.setup_hpo_hw_control(dc->hwseq, is_hpo_enabled);
+	}
+
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx_old =
 					&dc->current_state->res_ctx.pipe_ctx[i];
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
index 314798400b16..0c4aef8ffe2c 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
@@ -367,7 +367,7 @@ static void dcn10_log_color_state(struct dc *dc,
 		 dc->caps.color.dpp.ocsc);
 
 	DTN_INFO("MPCC:  OPP  DPP  MPCCBOT  MODE  ALPHA_MODE  PREMULT  OVERLAP_ONLY  IDLE\n");
-	for (i = 0; i < pool->pipe_count; i++) {
+	for (i = 0; i < pool->mpcc_count; i++) {
 		struct mpcc_state s = {0};
 
 		pool->mpc->funcs->read_mpcc_state(pool->mpc, i, &s);
@@ -1366,6 +1366,7 @@ void dcn10_init_pipes(struct dc *dc, struct dc_state *context)
 	struct dce_hwseq *hws = dc->hwseq;
 	struct hubbub *hubbub = dc->res_pool->hubbub;
 	bool can_apply_seamless_boot = false;
+	bool tg_enabled[MAX_PIPES] = {false};
 
 	for (i = 0; i < context->stream_count; i++) {
 		if (context->streams[i]->apply_seamless_boot_optimization) {
@@ -1447,6 +1448,7 @@ void dcn10_init_pipes(struct dc *dc, struct dc_state *context)
 			// requesting data while in PSR.
 			tg->funcs->tg_init(tg);
 			hubp->power_gated = true;
+			tg_enabled[i] = true;
 			continue;
 		}
 
@@ -1488,6 +1490,20 @@ void dcn10_init_pipes(struct dc *dc, struct dc_state *context)
 		tg->funcs->tg_init(tg);
 	}
 
+	/* Clean up MPC tree */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (tg_enabled[i]) {
+			if (dc->res_pool->opps[i]->mpc_tree_params.opp_list) {
+				if (dc->res_pool->opps[i]->mpc_tree_params.opp_list->mpcc_bot) {
+					int bot_id = dc->res_pool->opps[i]->mpc_tree_params.opp_list->mpcc_bot->mpcc_id;
+
+					if ((bot_id < MAX_MPCC) && (bot_id < MAX_PIPES) && (!tg_enabled[bot_id]))
+						dc->res_pool->opps[i]->mpc_tree_params.opp_list = NULL;
+				}
+			}
+		}
+	}
+
 	/* Power gate DSCs */
 	if (hws->funcs.dsc_pg_control != NULL) {
 		uint32_t num_opps = 0;
@@ -1813,14 +1829,12 @@ bool dcn10_set_input_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
 	if (dpp_base == NULL)
 		return false;
 
-	if (plane_state->in_transfer_func)
-		tf = plane_state->in_transfer_func;
+	tf = &plane_state->in_transfer_func;
 
-	if (plane_state->gamma_correction &&
-		!dpp_base->ctx->dc->debug.always_use_regamma
-		&& !plane_state->gamma_correction->is_identity
+	if (!dpp_base->ctx->dc->debug.always_use_regamma
+		&& !plane_state->gamma_correction.is_identity
 			&& dce_use_lut(plane_state->format))
-		dpp_base->funcs->dpp_program_input_lut(dpp_base, plane_state->gamma_correction);
+		dpp_base->funcs->dpp_program_input_lut(dpp_base, &plane_state->gamma_correction);
 
 	if (tf == NULL)
 		dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_BYPASS);
@@ -1861,7 +1875,7 @@ bool dcn10_set_input_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
 #define MAX_NUM_HW_POINTS 0x200
 
 static void log_tf(struct dc_context *ctx,
-				struct dc_transfer_func *tf, uint32_t hw_points_num)
+				const struct dc_transfer_func *tf, uint32_t hw_points_num)
 {
 	// DC_LOG_GAMMA is default logging of all hw points
 	// DC_LOG_ALL_GAMMA logs all points, not only hw points
@@ -1898,16 +1912,15 @@ bool dcn10_set_output_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
 
 	dpp->regamma_params.hw_points_num = GAMMA_HW_POINTS_NUM;
 
-	if (stream->out_transfer_func &&
-	    stream->out_transfer_func->type == TF_TYPE_PREDEFINED &&
-	    stream->out_transfer_func->tf == TRANSFER_FUNCTION_SRGB)
+	if (stream->out_transfer_func.type == TF_TYPE_PREDEFINED &&
+	    stream->out_transfer_func.tf == TRANSFER_FUNCTION_SRGB)
 		dpp->funcs->dpp_program_regamma_pwl(dpp, NULL, OPP_REGAMMA_SRGB);
 
 	/* dcn10_translate_regamma_to_hw_format takes 750us, only do it when full
 	 * update.
 	 */
 	else if (cm_helper_translate_curve_to_hw_format(dc->ctx,
-			stream->out_transfer_func,
+			&stream->out_transfer_func,
 			&dpp->regamma_params, false)) {
 		dpp->funcs->dpp_program_regamma_pwl(
 				dpp,
@@ -1915,10 +1928,9 @@ bool dcn10_set_output_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
 	} else
 		dpp->funcs->dpp_program_regamma_pwl(dpp, NULL, OPP_REGAMMA_BYPASS);
 
-	if (stream->ctx &&
-	    stream->out_transfer_func) {
+	if (stream->ctx) {
 		log_tf(stream->ctx,
-				stream->out_transfer_func,
+				&stream->out_transfer_func,
 				dpp->regamma_params.hw_points_num);
 	}
 
@@ -2173,7 +2185,7 @@ static int dcn10_align_pixel_clocks(struct dc *dc, int group_size,
 	struct dc_crtc_timing *hw_crtc_timing;
 	uint64_t phase[MAX_PIPES];
 	uint64_t modulo[MAX_PIPES];
-	unsigned int pclk;
+	unsigned int pclk = 0;
 
 	uint32_t embedded_pix_clk_100hz;
 	uint16_t embedded_h_total;
@@ -2264,7 +2276,7 @@ void dcn10_enable_vblanks_synchronization(
 	struct dc_context *dc_ctx = dc->ctx;
 	struct output_pixel_processor *opp;
 	struct timing_generator *tg;
-	int i, width, height, master;
+	int i, width = 0, height = 0, master;
 
 	DC_LOGGER_INIT(dc_ctx->logger);
 
@@ -2330,7 +2342,7 @@ void dcn10_enable_timing_synchronization(
 	struct dc_context *dc_ctx = dc->ctx;
 	struct output_pixel_processor *opp;
 	struct timing_generator *tg;
-	int i, width, height;
+	int i, width = 0, height = 0;
 
 	DC_LOGGER_INIT(dc_ctx->logger);
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
index 8b3536c380b8..7d833fa6dd77 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
@@ -155,7 +155,7 @@ void dcn20_log_color_state(struct dc *dc,
 	DTN_INFO("MPCC:  OPP  DPP  MPCCBOT  MODE  ALPHA_MODE  PREMULT  OVERLAP_ONLY  IDLE"
 		 "  OGAM mode\n");
 
-	for (i = 0; i < pool->pipe_count; i++) {
+	for (i = 0; i < pool->mpcc_count; i++) {
 		struct mpcc_state s = {0};
 
 		pool->mpc->funcs->read_mpcc_state(pool->mpc, i, &s);
@@ -403,7 +403,7 @@ void dcn20_init_blank(
 	struct output_pixel_processor *opp = NULL;
 	struct output_pixel_processor *bottom_opp = NULL;
 	uint32_t num_opps, opp_id_src0, opp_id_src1;
-	uint32_t otg_active_width, otg_active_height;
+	uint32_t otg_active_width = 0, otg_active_height = 0;
 
 	/* program opp dpg blank color */
 	color_space = COLOR_SPACE_SRGB;
@@ -873,6 +873,22 @@ enum dc_status dcn20_enable_stream_timing(
 		return DC_ERROR_UNEXPECTED;
 	}
 
+	if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
+		struct dccg *dccg = dc->res_pool->dccg;
+		struct timing_generator *tg = pipe_ctx->stream_res.tg;
+		struct dtbclk_dto_params dto_params = {0};
+
+		if (dccg->funcs->set_dtbclk_p_src)
+			dccg->funcs->set_dtbclk_p_src(dccg, DTBCLK0, tg->inst);
+
+		dto_params.otg_inst = tg->inst;
+		dto_params.pixclk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10;
+		dto_params.num_odm_segments = get_odm_segment_count(pipe_ctx);
+		dto_params.timing = &pipe_ctx->stream->timing;
+		dto_params.ref_dtbclk_khz = dc->clk_mgr->funcs->get_dtb_ref_clk_frequency(dc->clk_mgr);
+		dccg->funcs->set_dtbclk_dto(dccg, &dto_params);
+	}
+
 	if (dc_is_hdmi_tmds_signal(stream->signal)) {
 		stream->link->phy_state.symclk_ref_cnts.otg = 1;
 		if (stream->link->phy_state.symclk_state == SYMCLK_OFF_TX_OFF)
@@ -959,22 +975,6 @@ enum dc_status dcn20_enable_stream_timing(
 			pipe_ctx->stream_res.tg->funcs->phantom_crtc_post_enable(pipe_ctx->stream_res.tg);
 	}
 
-	if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
-		struct dccg *dccg = dc->res_pool->dccg;
-		struct timing_generator *tg = pipe_ctx->stream_res.tg;
-		struct dtbclk_dto_params dto_params = {0};
-
-		if (dccg->funcs->set_dtbclk_p_src)
-			dccg->funcs->set_dtbclk_p_src(dccg, DTBCLK0, tg->inst);
-
-		dto_params.otg_inst = tg->inst;
-		dto_params.pixclk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10;
-		dto_params.num_odm_segments = get_odm_segment_count(pipe_ctx);
-		dto_params.timing = &pipe_ctx->stream->timing;
-		dto_params.ref_dtbclk_khz = dc->clk_mgr->funcs->get_dtb_ref_clk_frequency(dc->clk_mgr);
-		dccg->funcs->set_dtbclk_dto(dccg, &dto_params);
-	}
-
 	return DC_OK;
 }
 
@@ -1011,7 +1011,7 @@ bool dcn20_set_output_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
 {
 	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
 	struct mpc *mpc = pipe_ctx->stream_res.opp->ctx->dc->res_pool->mpc;
-	struct pwl_params *params = NULL;
+	const struct pwl_params *params = NULL;
 	/*
 	 * program OGAM only for the top pipe
 	 * if there is a pipe split then fix diagnostic is required:
@@ -1022,19 +1022,19 @@ bool dcn20_set_output_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
 	if (mpc->funcs->power_on_mpc_mem_pwr)
 		mpc->funcs->power_on_mpc_mem_pwr(mpc, mpcc_id, true);
 	if (pipe_ctx->top_pipe == NULL
-			&& mpc->funcs->set_output_gamma && stream->out_transfer_func) {
-		if (stream->out_transfer_func->type == TF_TYPE_HWPWL)
-			params = &stream->out_transfer_func->pwl;
-		else if (pipe_ctx->stream->out_transfer_func->type ==
+			&& mpc->funcs->set_output_gamma) {
+		if (stream->out_transfer_func.type == TF_TYPE_HWPWL)
+			params = &stream->out_transfer_func.pwl;
+		else if (pipe_ctx->stream->out_transfer_func.type ==
 			TF_TYPE_DISTRIBUTED_POINTS &&
 			cm_helper_translate_curve_to_hw_format(dc->ctx,
-			stream->out_transfer_func,
+			&stream->out_transfer_func,
 			&mpc->blender_params, false))
 			params = &mpc->blender_params;
 		/*
 		 * there is no ROM
 		 */
-		if (stream->out_transfer_func->type == TF_TYPE_PREDEFINED)
+		if (stream->out_transfer_func.type == TF_TYPE_PREDEFINED)
 			BREAK_TO_DEBUGGER();
 	}
 	/*
@@ -1050,17 +1050,15 @@ bool dcn20_set_blend_lut(
 {
 	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
 	bool result = true;
-	struct pwl_params *blend_lut = NULL;
-
-	if (plane_state->blend_tf) {
-		if (plane_state->blend_tf->type == TF_TYPE_HWPWL)
-			blend_lut = &plane_state->blend_tf->pwl;
-		else if (plane_state->blend_tf->type == TF_TYPE_DISTRIBUTED_POINTS) {
-			cm_helper_translate_curve_to_hw_format(plane_state->ctx,
-					plane_state->blend_tf,
-					&dpp_base->regamma_params, false);
-			blend_lut = &dpp_base->regamma_params;
-		}
+	const struct pwl_params *blend_lut = NULL;
+
+	if (plane_state->blend_tf.type == TF_TYPE_HWPWL)
+		blend_lut = &plane_state->blend_tf.pwl;
+	else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		cm_helper_translate_curve_to_hw_format(plane_state->ctx,
+				&plane_state->blend_tf,
+				&dpp_base->regamma_params, false);
+		blend_lut = &dpp_base->regamma_params;
 	}
 	result = dpp_base->funcs->dpp_program_blnd_lut(dpp_base, blend_lut);
 
@@ -1072,24 +1070,21 @@ bool dcn20_set_shaper_3dlut(
 {
 	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
 	bool result = true;
-	struct pwl_params *shaper_lut = NULL;
-
-	if (plane_state->in_shaper_func) {
-		if (plane_state->in_shaper_func->type == TF_TYPE_HWPWL)
-			shaper_lut = &plane_state->in_shaper_func->pwl;
-		else if (plane_state->in_shaper_func->type == TF_TYPE_DISTRIBUTED_POINTS) {
-			cm_helper_translate_curve_to_hw_format(plane_state->ctx,
-					plane_state->in_shaper_func,
-					&dpp_base->shaper_params, true);
-			shaper_lut = &dpp_base->shaper_params;
-		}
+	const struct pwl_params *shaper_lut = NULL;
+
+	if (plane_state->in_shaper_func.type == TF_TYPE_HWPWL)
+		shaper_lut = &plane_state->in_shaper_func.pwl;
+	else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		cm_helper_translate_curve_to_hw_format(plane_state->ctx,
+				&plane_state->in_shaper_func,
+				&dpp_base->shaper_params, true);
+		shaper_lut = &dpp_base->shaper_params;
 	}
 
 	result = dpp_base->funcs->dpp_program_shaper_lut(dpp_base, shaper_lut);
-	if (plane_state->lut3d_func &&
-		plane_state->lut3d_func->state.bits.initialized == 1)
+	if (plane_state->lut3d_func.state.bits.initialized == 1)
 		result = dpp_base->funcs->dpp_program_3dlut(dpp_base,
-								&plane_state->lut3d_func->lut_3d);
+								&plane_state->lut3d_func.lut_3d);
 	else
 		result = dpp_base->funcs->dpp_program_3dlut(dpp_base, NULL);
 
@@ -1112,15 +1107,7 @@ bool dcn20_set_input_transfer_func(struct dc *dc,
 	hws->funcs.set_shaper_3dlut(pipe_ctx, plane_state);
 	hws->funcs.set_blend_lut(pipe_ctx, plane_state);
 
-	if (plane_state->in_transfer_func)
-		tf = plane_state->in_transfer_func;
-
-
-	if (tf == NULL) {
-		dpp_base->funcs->dpp_set_degamma(dpp_base,
-				IPP_DEGAMMA_MODE_BYPASS);
-		return true;
-	}
+	tf = &plane_state->in_transfer_func;
 
 	if (tf->type == TF_TYPE_HWPWL || tf->type == TF_TYPE_DISTRIBUTED_POINTS)
 		use_degamma_ram = true;
@@ -1917,9 +1904,11 @@ static void dcn20_program_pipe(
 			dc->res_pool->hubbub->funcs->force_wm_propagate_to_pipes(dc->res_pool->hubbub);
 	}
 
-	if (dc->res_pool->hubbub->funcs->program_det_size && pipe_ctx->update_flags.bits.det_size)
-		dc->res_pool->hubbub->funcs->program_det_size(
-			dc->res_pool->hubbub, pipe_ctx->plane_res.hubp->inst, pipe_ctx->det_buffer_size_kb);
+	if (pipe_ctx->update_flags.bits.det_size) {
+		if (dc->res_pool->hubbub->funcs->program_det_size)
+			dc->res_pool->hubbub->funcs->program_det_size(
+				dc->res_pool->hubbub, pipe_ctx->plane_res.hubp->inst, pipe_ctx->det_buffer_size_kb);
+	}
 
 	if (pipe_ctx->update_flags.raw || pipe_ctx->plane_state->update_flags.raw || pipe_ctx->stream->update_flags.raw)
 		dcn20_update_dchubp_dpp(dc, pipe_ctx, context);
@@ -2080,9 +2069,11 @@ void dcn20_program_front_end_for_ctx(
 			 * turned on (i.e. in an MCLK switch) which can come in too late and cause issues with
 			 * DET allocation.
 			 */
-			if (hubbub->funcs->program_det_size && (context->res_ctx.pipe_ctx[i].update_flags.bits.disable ||
-					(context->res_ctx.pipe_ctx[i].plane_state && dc_state_get_pipe_subvp_type(context, &context->res_ctx.pipe_ctx[i]) == SUBVP_PHANTOM)))
-				hubbub->funcs->program_det_size(hubbub, dc->current_state->res_ctx.pipe_ctx[i].plane_res.hubp->inst, 0);
+			if ((context->res_ctx.pipe_ctx[i].update_flags.bits.disable ||
+					(context->res_ctx.pipe_ctx[i].plane_state && dc_state_get_pipe_subvp_type(context, &context->res_ctx.pipe_ctx[i]) == SUBVP_PHANTOM))) {
+				if (hubbub->funcs->program_det_size)
+					hubbub->funcs->program_det_size(hubbub, dc->current_state->res_ctx.pipe_ctx[i].plane_res.hubp->inst, 0);
+			}
 			hws->funcs.plane_atomic_disconnect(dc, dc->current_state, &dc->current_state->res_ctx.pipe_ctx[i]);
 			DC_LOG_DC("Reset mpcc for pipe %d\n", dc->current_state->res_ctx.pipe_ctx[i].pipe_idx);
 		}
@@ -2893,11 +2884,6 @@ void dcn20_enable_stream(struct pipe_ctx *pipe_ctx)
 	struct stream_encoder *stream_enc = pipe_ctx->stream_res.stream_enc;
 
 	if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
-		if (dc->hwseq->funcs.setup_hpo_hw_control)
-			dc->hwseq->funcs.setup_hpo_hw_control(dc->hwseq, true);
-	}
-
-	if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) {
 		dto_params.otg_inst = tg->inst;
 		dto_params.pixclk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10;
 		dto_params.num_odm_segments = get_odm_segment_count(pipe_ctx);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c
index 884e3e323338..ef6488165b8f 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c
@@ -67,6 +67,7 @@ static const struct hw_sequencer_funcs dcn20_funcs = {
 	.setup_stereo = dcn10_setup_stereo,
 	.set_avmute = dce110_set_avmute,
 	.log_hw_state = dcn10_log_hw_state,
+	.log_color_state = dcn20_log_color_state,
 	.get_hw_state = dcn10_get_hw_state,
 	.clear_status_bits = dcn10_clear_status_bits,
 	.wait_for_mpcc_disconnect = dcn10_wait_for_mpcc_disconnect,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c
index d5769f38874f..6be846635a79 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c
@@ -167,7 +167,7 @@ void dcn201_init_blank(
 	struct tg_color black_color = {0};
 	struct output_pixel_processor *opp = NULL;
 	uint32_t num_opps, opp_id_src0, opp_id_src1;
-	uint32_t otg_active_width, otg_active_height;
+	uint32_t otg_active_width = 0, otg_active_height = 0;
 
 	/* program opp dpg blank color */
 	color_space = COLOR_SPACE_SRGB;
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c
index 7252f5f781f0..804be977ea47 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c
@@ -66,7 +66,7 @@ static void mmhub_update_page_table_config(struct dcn_hubbub_phys_addr_config *c
 
 int dcn21_init_sys_ctx(struct dce_hwseq *hws, struct dc *dc, struct dc_phy_addr_space_config *pa_config)
 {
-	struct dcn_hubbub_phys_addr_config config;
+	struct dcn_hubbub_phys_addr_config config = {0};
 
 	config.system_aperture.fb_top = pa_config->system_aperture.fb_top;
 	config.system_aperture.fb_offset = pa_config->system_aperture.fb_offset;
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c
index 8bc3d01537bb..ed9141a67db3 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c
@@ -166,7 +166,7 @@ void dcn30_log_color_state(struct dc *dc,
 		 "C21        C22        C23        C24        "
 		 "C31        C32        C33        C34        \n");
 
-	for (i = 0; i < pool->pipe_count; i++) {
+	for (i = 0; i < pool->mpcc_count; i++) {
 		struct mpcc_state s = {0};
 
 		pool->mpc->funcs->read_mpcc_state(pool->mpc, i, &s);
@@ -223,16 +223,14 @@ bool dcn30_set_blend_lut(
 {
 	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
 	bool result = true;
-	struct pwl_params *blend_lut = NULL;
-
-	if (plane_state->blend_tf) {
-		if (plane_state->blend_tf->type == TF_TYPE_HWPWL)
-			blend_lut = &plane_state->blend_tf->pwl;
-		else if (plane_state->blend_tf->type == TF_TYPE_DISTRIBUTED_POINTS) {
-			cm3_helper_translate_curve_to_hw_format(
-					plane_state->blend_tf, &dpp_base->regamma_params, false);
-			blend_lut = &dpp_base->regamma_params;
-		}
+	const struct pwl_params *blend_lut = NULL;
+
+	if (plane_state->blend_tf.type == TF_TYPE_HWPWL)
+		blend_lut = &plane_state->blend_tf.pwl;
+	else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		cm3_helper_translate_curve_to_hw_format(
+				&plane_state->blend_tf, &dpp_base->regamma_params, false);
+		blend_lut = &dpp_base->regamma_params;
 	}
 	result = dpp_base->funcs->dpp_program_blnd_lut(dpp_base, blend_lut);
 
@@ -300,27 +298,24 @@ bool dcn30_set_input_transfer_func(struct dc *dc,
 	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
 	enum dc_transfer_func_predefined tf;
 	bool result = true;
-	struct pwl_params *params = NULL;
+	const struct pwl_params *params = NULL;
 
 	if (dpp_base == NULL || plane_state == NULL)
 		return false;
 
 	tf = TRANSFER_FUNCTION_UNITY;
 
-	if (plane_state->in_transfer_func &&
-		plane_state->in_transfer_func->type == TF_TYPE_PREDEFINED)
-		tf = plane_state->in_transfer_func->tf;
+	if (plane_state->in_transfer_func.type == TF_TYPE_PREDEFINED)
+		tf = plane_state->in_transfer_func.tf;
 
 	dpp_base->funcs->dpp_set_pre_degam(dpp_base, tf);
 
-	if (plane_state->in_transfer_func) {
-		if (plane_state->in_transfer_func->type == TF_TYPE_HWPWL)
-			params = &plane_state->in_transfer_func->pwl;
-		else if (plane_state->in_transfer_func->type == TF_TYPE_DISTRIBUTED_POINTS &&
-			cm3_helper_translate_curve_to_hw_format(plane_state->in_transfer_func,
-					&dpp_base->degamma_params, false))
-			params = &dpp_base->degamma_params;
-	}
+	if (plane_state->in_transfer_func.type == TF_TYPE_HWPWL)
+		params = &plane_state->in_transfer_func.pwl;
+	else if (plane_state->in_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS &&
+		cm3_helper_translate_curve_to_hw_format(&plane_state->in_transfer_func,
+				&dpp_base->degamma_params, false))
+		params = &dpp_base->degamma_params;
 
 	result = dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params);
 
@@ -378,24 +373,24 @@ bool dcn30_set_output_transfer_func(struct dc *dc,
 {
 	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
 	struct mpc *mpc = pipe_ctx->stream_res.opp->ctx->dc->res_pool->mpc;
-	struct pwl_params *params = NULL;
+	const struct pwl_params *params = NULL;
 	bool ret = false;
 
 	/* program OGAM or 3DLUT only for the top pipe*/
 	if (pipe_ctx->top_pipe == NULL) {
 		/*program rmu shaper and 3dlut in MPC*/
 		ret = dcn30_set_mpc_shaper_3dlut(pipe_ctx, stream);
-		if (ret == false && mpc->funcs->set_output_gamma && stream->out_transfer_func) {
-			if (stream->out_transfer_func->type == TF_TYPE_HWPWL)
-				params = &stream->out_transfer_func->pwl;
-			else if (pipe_ctx->stream->out_transfer_func->type ==
+		if (ret == false && mpc->funcs->set_output_gamma) {
+			if (stream->out_transfer_func.type == TF_TYPE_HWPWL)
+				params = &stream->out_transfer_func.pwl;
+			else if (pipe_ctx->stream->out_transfer_func.type ==
 					TF_TYPE_DISTRIBUTED_POINTS &&
 					cm3_helper_translate_curve_to_hw_format(
-					stream->out_transfer_func,
+					&stream->out_transfer_func,
 					&mpc->blender_params, false))
 				params = &mpc->blender_params;
 			 /* there are no ROM LUTs in OUTGAM */
-			if (stream->out_transfer_func->type == TF_TYPE_PREDEFINED)
+			if (stream->out_transfer_func.type == TF_TYPE_PREDEFINED)
 				BREAK_TO_DEBUGGER();
 		}
 	}
@@ -804,7 +799,7 @@ void dcn30_init_hw(struct dc *dc)
 	// Get DMCUB capabilities
 	dc_dmub_srv_query_caps_cmd(dc->ctx->dmub_srv);
 	dc->caps.dmub_caps.psr = dc->ctx->dmub_srv->dmub->feature_caps.psr;
-	dc->caps.dmub_caps.mclk_sw = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch;
+	dc->caps.dmub_caps.mclk_sw = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch_ver;
 }
 
 void dcn30_set_avmute(struct pipe_ctx *pipe_ctx, bool enable)
@@ -818,7 +813,7 @@ void dcn30_set_avmute(struct pipe_ctx *pipe_ctx, bool enable)
 				enable);
 
 		/* Wait for two frame to make sure AV mute is sent out */
-		if (enable) {
+		if (enable && pipe_ctx->stream_res.tg->funcs->is_tg_enabled(pipe_ctx->stream_res.tg)) {
 			pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE);
 			pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VBLANK);
 			pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE);
@@ -890,7 +885,7 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable)
 {
 	union dmub_rb_cmd cmd;
 	uint32_t tmr_delay = 0, tmr_scale = 0;
-	struct dc_cursor_attributes cursor_attr;
+	struct dc_cursor_attributes cursor_attr = {0};
 	bool cursor_cache_enable = false;
 	struct dc_stream_state *stream = NULL;
 	struct dc_plane_state *plane = NULL;
@@ -946,7 +941,8 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable)
 					plane->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB8888 &&
 					plane->address.page_table_base.quad_part == 0 &&
 					dc->hwss.does_plane_fit_in_mall &&
-					dc->hwss.does_plane_fit_in_mall(dc, plane,
+					dc->hwss.does_plane_fit_in_mall(dc, plane->plane_size.surface_pitch,
+							plane->plane_size.surface_size.height, plane->format,
 							cursor_cache_enable ? &cursor_attr : NULL)) {
 				unsigned int v_total = stream->adjust.v_total_max ?
 						stream->adjust.v_total_max : stream->timing.v_total;
@@ -1076,11 +1072,15 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable)
 	return true;
 }
 
-bool dcn30_does_plane_fit_in_mall(struct dc *dc, struct dc_plane_state *plane, struct dc_cursor_attributes *cursor_attr)
+bool dcn30_does_plane_fit_in_mall(struct dc *dc,
+		unsigned int pitch,
+		unsigned int height,
+		enum surface_pixel_format format,
+		struct dc_cursor_attributes *cursor_attr)
 {
 	// add meta size?
-	unsigned int surface_size = plane->plane_size.surface_pitch * plane->plane_size.surface_size.height *
-			(plane->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 ? 8 : 4);
+	unsigned int surface_size = pitch * height *
+			(format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 ? 8 : 4);
 	unsigned int mall_size = dc->caps.mall_size_total;
 	unsigned int cursor_size = 0;
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.h
index 638f018a3cb5..76b16839486a 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.h
@@ -71,7 +71,10 @@ void dcn30_set_avmute(struct pipe_ctx *pipe_ctx, bool enable);
 void dcn30_update_info_frame(struct pipe_ctx *pipe_ctx);
 void dcn30_program_dmdata_engine(struct pipe_ctx *pipe_ctx);
 
-bool dcn30_does_plane_fit_in_mall(struct dc *dc, struct dc_plane_state *plane,
+bool dcn30_does_plane_fit_in_mall(struct dc *dc,
+		unsigned int pitch,
+		unsigned int height,
+		enum surface_pixel_format format,
 		struct dc_cursor_attributes *cursor_attr);
 
 bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
index a760f0c6fe98..1c8abb417b6e 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
@@ -273,7 +273,7 @@ void dcn31_init_hw(struct dc *dc)
 	// Get DMCUB capabilities
 	dc_dmub_srv_query_caps_cmd(dc->ctx->dmub_srv);
 	dc->caps.dmub_caps.psr = dc->ctx->dmub_srv->dmub->feature_caps.psr;
-	dc->caps.dmub_caps.mclk_sw = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch;
+	dc->caps.dmub_caps.mclk_sw = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch_ver;
 }
 
 void dcn31_dsc_pg_control(
@@ -479,7 +479,7 @@ void dcn31_hubp_pg_control(struct dce_hwseq *hws, unsigned int hubp_inst, bool p
 
 int dcn31_init_sys_ctx(struct dce_hwseq *hws, struct dc *dc, struct dc_phy_addr_space_config *pa_config)
 {
-	struct dcn_hubbub_phys_addr_config config;
+	struct dcn_hubbub_phys_addr_config config = {0};
 
 	config.system_aperture.fb_top = pa_config->system_aperture.fb_top;
 	config.system_aperture.fb_offset = pa_config->system_aperture.fb_offset;
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
index 093f4387553c..0d8a05cf8b1a 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
@@ -82,7 +82,7 @@ static void update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 
 	if (enable) {
 		struct dsc_config dsc_cfg;
-		struct dsc_optc_config dsc_optc_cfg;
+		struct dsc_optc_config dsc_optc_cfg = {0};
 		enum optc_dsc_mode optc_dsc_mode;
 
 		/* Enable DSC hw block */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
index 7668229438da..b8e884368dc6 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
@@ -239,8 +239,10 @@ static uint32_t dcn32_calculate_cab_allocation(struct dc *dc, struct dc_state *c
 	// Convert number of cache lines required to number of ways
 	if (dc->debug.force_mall_ss_num_ways > 0) {
 		num_ways = dc->debug.force_mall_ss_num_ways;
+	} else if (dc->res_pool->funcs->calculate_mall_ways_from_bytes) {
+		num_ways = dc->res_pool->funcs->calculate_mall_ways_from_bytes(dc, mall_ss_size_bytes);
 	} else {
-		num_ways = dcn32_helper_mall_bytes_to_ways(dc, mall_ss_size_bytes);
+		num_ways = 0;
 	}
 
 	return num_ways;
@@ -261,7 +263,9 @@ bool dcn32_apply_idle_power_optimizations(struct dc *dc, bool enable)
 	for (i = 0; i < dc->current_state->stream_count; i++) {
 		/* MALL SS messaging is not supported with PSR at this time */
 		if (dc->current_state->streams[i] != NULL &&
-				dc->current_state->streams[i]->link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED)
+				dc->current_state->streams[i]->link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED &&
+				(dc->current_state->stream_count > 1 || (!dc->current_state->streams[i]->dpms_off &&
+						dc->current_state->stream_status[i].plane_count > 0)))
 			return false;
 	}
 
@@ -475,39 +479,35 @@ bool dcn32_set_mcm_luts(
 	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
 	struct mpc *mpc = pipe_ctx->stream_res.opp->ctx->dc->res_pool->mpc;
 	bool result = true;
-	struct pwl_params *lut_params = NULL;
+	const struct pwl_params *lut_params = NULL;
 
 	// 1D LUT
-	if (plane_state->blend_tf) {
-		if (plane_state->blend_tf->type == TF_TYPE_HWPWL)
-			lut_params = &plane_state->blend_tf->pwl;
-		else if (plane_state->blend_tf->type == TF_TYPE_DISTRIBUTED_POINTS) {
-			cm3_helper_translate_curve_to_hw_format(plane_state->blend_tf,
-					&dpp_base->regamma_params, false);
-			lut_params = &dpp_base->regamma_params;
-		}
+	if (plane_state->blend_tf.type == TF_TYPE_HWPWL)
+		lut_params = &plane_state->blend_tf.pwl;
+	else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		cm3_helper_translate_curve_to_hw_format(&plane_state->blend_tf,
+				&dpp_base->regamma_params, false);
+		lut_params = &dpp_base->regamma_params;
 	}
 	result = mpc->funcs->program_1dlut(mpc, lut_params, mpcc_id);
 	lut_params = NULL;
 
 	// Shaper
-	if (plane_state->in_shaper_func) {
-		if (plane_state->in_shaper_func->type == TF_TYPE_HWPWL)
-			lut_params = &plane_state->in_shaper_func->pwl;
-		else if (plane_state->in_shaper_func->type == TF_TYPE_DISTRIBUTED_POINTS) {
-			// TODO: dpp_base replace
-			ASSERT(false);
-			cm3_helper_translate_curve_to_hw_format(plane_state->in_shaper_func,
-					&dpp_base->shaper_params, true);
-			lut_params = &dpp_base->shaper_params;
-		}
+	if (plane_state->in_shaper_func.type == TF_TYPE_HWPWL)
+		lut_params = &plane_state->in_shaper_func.pwl;
+	else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		// TODO: dpp_base replace
+		ASSERT(false);
+		cm3_helper_translate_curve_to_hw_format(&plane_state->in_shaper_func,
+				&dpp_base->shaper_params, true);
+		lut_params = &dpp_base->shaper_params;
 	}
 
 	result = mpc->funcs->program_shaper(mpc, lut_params, mpcc_id);
 
 	// 3D
-	if (plane_state->lut3d_func && plane_state->lut3d_func->state.bits.initialized == 1)
-		result = mpc->funcs->program_3dlut(mpc, &plane_state->lut3d_func->lut_3d, mpcc_id);
+	if (plane_state->lut3d_func.state.bits.initialized == 1)
+		result = mpc->funcs->program_3dlut(mpc, &plane_state->lut3d_func.lut_3d, mpcc_id);
 	else
 		result = mpc->funcs->program_3dlut(mpc, NULL, mpcc_id);
 
@@ -524,27 +524,24 @@ bool dcn32_set_input_transfer_func(struct dc *dc,
 
 	enum dc_transfer_func_predefined tf;
 	bool result = true;
-	struct pwl_params *params = NULL;
+	const struct pwl_params *params = NULL;
 
 	if (mpc == NULL || plane_state == NULL)
 		return false;
 
 	tf = TRANSFER_FUNCTION_UNITY;
 
-	if (plane_state->in_transfer_func &&
-		plane_state->in_transfer_func->type == TF_TYPE_PREDEFINED)
-		tf = plane_state->in_transfer_func->tf;
+	if (plane_state->in_transfer_func.type == TF_TYPE_PREDEFINED)
+		tf = plane_state->in_transfer_func.tf;
 
 	dpp_base->funcs->dpp_set_pre_degam(dpp_base, tf);
 
-	if (plane_state->in_transfer_func) {
-		if (plane_state->in_transfer_func->type == TF_TYPE_HWPWL)
-			params = &plane_state->in_transfer_func->pwl;
-		else if (plane_state->in_transfer_func->type == TF_TYPE_DISTRIBUTED_POINTS &&
-			cm3_helper_translate_curve_to_hw_format(plane_state->in_transfer_func,
-					&dpp_base->degamma_params, false))
-			params = &dpp_base->degamma_params;
-	}
+	if (plane_state->in_transfer_func.type == TF_TYPE_HWPWL)
+		params = &plane_state->in_transfer_func.pwl;
+	else if (plane_state->in_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS &&
+		cm3_helper_translate_curve_to_hw_format(&plane_state->in_transfer_func,
+				&dpp_base->degamma_params, false))
+		params = &dpp_base->degamma_params;
 
 	dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params);
 
@@ -562,24 +559,24 @@ bool dcn32_set_output_transfer_func(struct dc *dc,
 {
 	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
 	struct mpc *mpc = pipe_ctx->stream_res.opp->ctx->dc->res_pool->mpc;
-	struct pwl_params *params = NULL;
+	const struct pwl_params *params = NULL;
 	bool ret = false;
 
 	/* program OGAM or 3DLUT only for the top pipe*/
 	if (resource_is_pipe_type(pipe_ctx, OPP_HEAD)) {
 		/*program shaper and 3dlut in MPC*/
 		ret = dcn32_set_mpc_shaper_3dlut(pipe_ctx, stream);
-		if (ret == false && mpc->funcs->set_output_gamma && stream->out_transfer_func) {
-			if (stream->out_transfer_func->type == TF_TYPE_HWPWL)
-				params = &stream->out_transfer_func->pwl;
-			else if (pipe_ctx->stream->out_transfer_func->type ==
+		if (ret == false && mpc->funcs->set_output_gamma) {
+			if (stream->out_transfer_func.type == TF_TYPE_HWPWL)
+				params = &stream->out_transfer_func.pwl;
+			else if (pipe_ctx->stream->out_transfer_func.type ==
 					TF_TYPE_DISTRIBUTED_POINTS &&
 					cm3_helper_translate_curve_to_hw_format(
-					stream->out_transfer_func,
+					&stream->out_transfer_func,
 					&mpc->blender_params, false))
 				params = &mpc->blender_params;
 			/* there are no ROM LUTs in OUTGAM */
-			if (stream->out_transfer_func->type == TF_TYPE_PREDEFINED)
+			if (stream->out_transfer_func.type == TF_TYPE_PREDEFINED)
 				BREAK_TO_DEBUGGER();
 		}
 	}
@@ -956,10 +953,10 @@ void dcn32_init_hw(struct dc *dc)
 		dc->caps.dmub_caps.psr = dc->ctx->dmub_srv->dmub->feature_caps.psr;
 		dc->caps.dmub_caps.subvp_psr = dc->ctx->dmub_srv->dmub->feature_caps.subvp_psr_support;
 		dc->caps.dmub_caps.gecc_enable = dc->ctx->dmub_srv->dmub->feature_caps.gecc_enable;
-		dc->caps.dmub_caps.mclk_sw = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch;
+		dc->caps.dmub_caps.mclk_sw = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch_ver;
 
 		if (dc->ctx->dmub_srv->dmub->fw_version <
-		    DMUB_FW_VERSION(7, 0, 35)) {
+				DMUB_FW_VERSION(7, 0, 35)) {
 			dc->debug.force_disable_subvp = true;
 			dc->debug.disable_fpo_optimizations = true;
 		}
@@ -992,7 +989,7 @@ static void update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 
 	if (enable) {
 		struct dsc_config dsc_cfg;
-		struct dsc_optc_config dsc_optc_cfg;
+		struct dsc_optc_config dsc_optc_cfg = {0};
 		enum optc_dsc_mode optc_dsc_mode;
 
 		/* Enable DSC hw block */
@@ -1545,7 +1542,7 @@ void dcn32_init_blank(
 	struct output_pixel_processor *opp = NULL;
 	struct output_pixel_processor *bottom_opp = NULL;
 	uint32_t num_opps, opp_id_src0, opp_id_src1;
-	uint32_t otg_active_width, otg_active_height;
+	uint32_t otg_active_width = 0, otg_active_height = 0;
 	uint32_t i;
 
 	/* program opp dpg blank color */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
index a5560b3fc39b..5295f52e4fc8 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
@@ -349,7 +349,7 @@ void dcn35_init_hw(struct dc *dc)
 	if (dc->ctx->dmub_srv) {
 		dc_dmub_srv_query_caps_cmd(dc->ctx->dmub_srv);
 		dc->caps.dmub_caps.psr = dc->ctx->dmub_srv->dmub->feature_caps.psr;
-		dc->caps.dmub_caps.mclk_sw = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch;
+		dc->caps.dmub_caps.mclk_sw = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch_ver;
 	}
 
 	if (dc->res_pool->pg_cntl) {
@@ -373,7 +373,7 @@ static void update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 
 	if (enable) {
 		struct dsc_config dsc_cfg;
-		struct dsc_optc_config dsc_optc_cfg;
+		struct dsc_optc_config dsc_optc_cfg = {0};
 		enum optc_dsc_mode optc_dsc_mode;
 
 		/* Enable DSC hw block */
@@ -495,6 +495,17 @@ void dcn35_dpp_root_clock_control(struct dce_hwseq *hws, unsigned int dpp_inst,
 	}
 }
 
+void dcn35_dpstream_root_clock_control(struct dce_hwseq *hws, unsigned int dp_hpo_inst, bool clock_on)
+{
+	if (!hws->ctx->dc->debug.root_clock_optimization.bits.dpstream)
+		return;
+
+	if (hws->ctx->dc->res_pool->dccg->funcs->set_dpstreamclk_root_clock_gating) {
+		hws->ctx->dc->res_pool->dccg->funcs->set_dpstreamclk_root_clock_gating(
+			hws->ctx->dc->res_pool->dccg, dp_hpo_inst, clock_on);
+	}
+}
+
 void dcn35_dsc_pg_control(
 		struct dce_hwseq *hws,
 		unsigned int dsc_inst,
@@ -638,22 +649,43 @@ void dcn35_power_down_on_boot(struct dc *dc)
 
 bool dcn35_apply_idle_power_optimizations(struct dc *dc, bool enable)
 {
-	struct dc_link *edp_links[MAX_NUM_EDP];
-	int i, edp_num;
 	if (dc->debug.dmcub_emulation)
 		return true;
 
 	if (enable) {
-		dc_get_edp_links(dc, edp_links, &edp_num);
-		if (edp_num == 0 || edp_num > 1)
-			return false;
+		uint32_t num_active_edp = 0;
+		int i;
 
 		for (i = 0; i < dc->current_state->stream_count; ++i) {
 			struct dc_stream_state *stream = dc->current_state->streams[i];
+			struct dc_link *link = stream->link;
+			bool is_psr = link && !link->panel_config.psr.disable_psr &&
+				      (link->psr_settings.psr_version == DC_PSR_VERSION_1 ||
+				       link->psr_settings.psr_version == DC_PSR_VERSION_SU_1);
+			bool is_replay = link && link->replay_settings.replay_feature_enabled;
+
+			/* Ignore streams that disabled. */
+			if (stream->dpms_off)
+				continue;
+
+			/* Active external displays block idle optimizations. */
+			if (!dc_is_embedded_signal(stream->signal))
+				return false;
+
+			/* If not PWRSEQ0 can't enter idle optimizations */
+			if (link && link->link_index != 0)
+				return false;
 
-			if (!stream->dpms_off && !dc_is_embedded_signal(stream->signal))
+			/* Check for panel power features required for idle optimizations. */
+			if (!is_psr && !is_replay)
 				return false;
+
+			num_active_edp += 1;
 		}
+
+		/* If more than one active eDP then disallow. */
+		if (num_active_edp > 1)
+			return false;
 	}
 
 	// TODO: review other cases when idle optimization is allowed
@@ -679,6 +711,7 @@ void dcn35_init_pipes(struct dc *dc, struct dc_state *context)
 	struct hubbub *hubbub = dc->res_pool->hubbub;
 	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
 	bool can_apply_seamless_boot = false;
+	bool tg_enabled[MAX_PIPES] = {false};
 
 	for (i = 0; i < context->stream_count; i++) {
 		if (context->streams[i]->apply_seamless_boot_optimization) {
@@ -760,6 +793,7 @@ void dcn35_init_pipes(struct dc *dc, struct dc_state *context)
 			// requesting data while in PSR.
 			tg->funcs->tg_init(tg);
 			hubp->power_gated = true;
+			tg_enabled[i] = true;
 			continue;
 		}
 
@@ -801,6 +835,20 @@ void dcn35_init_pipes(struct dc *dc, struct dc_state *context)
 		tg->funcs->tg_init(tg);
 	}
 
+	/* Clean up MPC tree */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (tg_enabled[i]) {
+			if (dc->res_pool->opps[i]->mpc_tree_params.opp_list) {
+				if (dc->res_pool->opps[i]->mpc_tree_params.opp_list->mpcc_bot) {
+					int bot_id = dc->res_pool->opps[i]->mpc_tree_params.opp_list->mpcc_bot->mpcc_id;
+
+					if ((bot_id < MAX_MPCC) && (bot_id < MAX_PIPES) && (!tg_enabled[bot_id]))
+						dc->res_pool->opps[i]->mpc_tree_params.opp_list = NULL;
+				}
+			}
+		}
+	}
+
 	if (pg_cntl != NULL) {
 		if (pg_cntl->funcs->dsc_pg_control != NULL) {
 			uint32_t num_opps = 0;
@@ -961,6 +1009,9 @@ void dcn35_calc_blocks_to_gate(struct dc *dc, struct dc_state *context,
 	if (!hpo_frl_stream_enc_acquired && !hpo_dp_stream_enc_acquired)
 		update_state->pg_res_update[PG_HPO] = true;
 
+	if (hpo_frl_stream_enc_acquired)
+		update_state->pg_pipe_res_update[PG_HDMISTREAM][0] = true;
+
 	update_state->pg_res_update[PG_DWB] = true;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
@@ -978,8 +1029,7 @@ void dcn35_calc_blocks_to_gate(struct dc *dc, struct dc_state *context,
 		if (pipe_ctx->plane_res.dpp)
 			update_state->pg_pipe_res_update[PG_DPP][pipe_ctx->plane_res.hubp->inst] = false;
 
-		if ((pipe_ctx->plane_res.dpp || pipe_ctx->stream_res.opp) &&
-			pipe_ctx->plane_res.mpcc_inst >= 0)
+		if (pipe_ctx->plane_res.dpp || pipe_ctx->stream_res.opp)
 			update_state->pg_pipe_res_update[PG_MPCC][pipe_ctx->plane_res.mpcc_inst] = false;
 
 		if (pipe_ctx->stream_res.dsc)
@@ -987,6 +1037,9 @@ void dcn35_calc_blocks_to_gate(struct dc *dc, struct dc_state *context,
 
 		if (pipe_ctx->stream_res.opp)
 			update_state->pg_pipe_res_update[PG_OPP][pipe_ctx->stream_res.opp->inst] = false;
+
+		if (pipe_ctx->stream_res.hpo_dp_stream_enc)
+			update_state->pg_pipe_res_update[PG_DPSTREAM][pipe_ctx->stream_res.hpo_dp_stream_enc->inst] = false;
 	}
 	/*domain24 controls all the otg, mpc, opp, as long as one otg is still up, avoid enabling OTG PG*/
 	for (i = 0; i < dc->res_pool->timing_generator_count; i++) {
@@ -1044,6 +1097,9 @@ void dcn35_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
 
 				if (j == PG_OPTC && new_pipe->stream_res.tg)
 					update_state->pg_pipe_res_update[j][new_pipe->stream_res.tg->inst] = true;
+
+				if (j == PG_DPSTREAM && new_pipe->stream_res.hpo_dp_stream_enc)
+					update_state->pg_pipe_res_update[j][new_pipe->stream_res.hpo_dp_stream_enc->inst] = true;
 			}
 		} else if (cur_pipe->plane_state == new_pipe->plane_state ||
 				cur_pipe == new_pipe) {
@@ -1073,6 +1129,11 @@ void dcn35_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
 					cur_pipe->stream_res.tg != new_pipe->stream_res.tg &&
 					new_pipe->stream_res.tg)
 					update_state->pg_pipe_res_update[j][new_pipe->stream_res.tg->inst] = true;
+
+				if (j == PG_DPSTREAM &&
+					cur_pipe->stream_res.hpo_dp_stream_enc != new_pipe->stream_res.hpo_dp_stream_enc &&
+					new_pipe->stream_res.hpo_dp_stream_enc)
+					update_state->pg_pipe_res_update[j][new_pipe->stream_res.hpo_dp_stream_enc->inst] = true;
 			}
 		}
 	}
@@ -1088,6 +1149,9 @@ void dcn35_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
 	if (hpo_frl_stream_enc_acquired || hpo_dp_stream_enc_acquired)
 		update_state->pg_res_update[PG_HPO] = true;
 
+	if (hpo_frl_stream_enc_acquired)
+		update_state->pg_pipe_res_update[PG_HDMISTREAM][0] = true;
+
 }
 
 /**
@@ -1212,14 +1276,19 @@ void dcn35_root_clock_control(struct dc *dc,
 	if (!pg_cntl)
 		return;
 	/*enable root clock first when power up*/
-	if (power_on)
+	if (power_on) {
 		for (i = 0; i < dc->res_pool->pipe_count; i++) {
 			if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
 				update_state->pg_pipe_res_update[PG_DPP][i]) {
 				if (dc->hwseq->funcs.dpp_root_clock_control)
 					dc->hwseq->funcs.dpp_root_clock_control(dc->hwseq, i, power_on);
 			}
+			if (update_state->pg_pipe_res_update[PG_DPSTREAM][i])
+				if (dc->hwseq->funcs.dpstream_root_clock_control)
+					dc->hwseq->funcs.dpstream_root_clock_control(dc->hwseq, i, power_on);
 		}
+
+	}
 	for (i = 0; i < dc->res_pool->res_cap->num_dsc; i++) {
 		if (update_state->pg_pipe_res_update[PG_DSC][i]) {
 			if (power_on) {
@@ -1232,14 +1301,19 @@ void dcn35_root_clock_control(struct dc *dc,
 		}
 	}
 	/*disable root clock first when power down*/
-	if (!power_on)
+	if (!power_on) {
 		for (i = 0; i < dc->res_pool->pipe_count; i++) {
 			if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
 				update_state->pg_pipe_res_update[PG_DPP][i]) {
 				if (dc->hwseq->funcs.dpp_root_clock_control)
 					dc->hwseq->funcs.dpp_root_clock_control(dc->hwseq, i, power_on);
 			}
+			if (update_state->pg_pipe_res_update[PG_DPSTREAM][i])
+				if (dc->hwseq->funcs.dpstream_root_clock_control)
+					dc->hwseq->funcs.dpstream_root_clock_control(dc->hwseq, i, power_on);
 		}
+
+	}
 }
 
 void dcn35_prepare_bandwidth(
@@ -1280,22 +1354,6 @@ void dcn35_optimize_bandwidth(
 	}
 }
 
-void dcn35_set_idle_state(const struct dc *dc, bool allow_idle)
-{
-	// TODO: Find a more suitable communcation
-	if (dc->clk_mgr->funcs->set_idle_state)
-		dc->clk_mgr->funcs->set_idle_state(dc->clk_mgr, allow_idle);
-}
-
-uint32_t dcn35_get_idle_state(const struct dc *dc)
-{
-	// TODO: Find a more suitable communcation
-	if (dc->clk_mgr->funcs->get_idle_state)
-		return dc->clk_mgr->funcs->get_idle_state(dc->clk_mgr);
-
-	return 0;
-}
-
 void dcn35_set_drr(struct pipe_ctx **pipe_ctx,
 		int num_pipes, struct dc_crtc_timing_adjust adjust)
 {
@@ -1353,3 +1411,31 @@ void dcn35_set_static_screen_control(struct pipe_ctx **pipe_ctx,
 			set_static_screen_control(pipe_ctx[i]->stream_res.tg,
 					triggers, params->num_frames);
 }
+
+void dcn35_set_long_vblank(struct pipe_ctx **pipe_ctx,
+		int num_pipes, uint32_t v_total_min, uint32_t v_total_max)
+{
+	int i = 0;
+	struct long_vtotal_params params = {0};
+
+	params.vertical_total_max = v_total_max;
+	params.vertical_total_min = v_total_min;
+
+	for (i = 0; i < num_pipes; i++) {
+		if (!pipe_ctx[i])
+			continue;
+
+		if (pipe_ctx[i]->stream) {
+			struct dc_crtc_timing *timing = &pipe_ctx[i]->stream->timing;
+
+			if (timing)
+				params.vertical_blank_start = timing->v_total - timing->v_front_porch;
+			else
+				params.vertical_blank_start = 0;
+
+			if ((pipe_ctx[i]->stream_res.tg != NULL) && pipe_ctx[i]->stream_res.tg->funcs &&
+				pipe_ctx[i]->stream_res.tg->funcs->set_long_vtotal)
+				pipe_ctx[i]->stream_res.tg->funcs->set_long_vtotal(pipe_ctx[i]->stream_res.tg, &params);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h
index c354efa6c1b2..a731c8880d60 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h
@@ -37,6 +37,8 @@ void dcn35_dsc_pg_control(struct dce_hwseq *hws, unsigned int dsc_inst, bool pow
 
 void dcn35_dpp_root_clock_control(struct dce_hwseq *hws, unsigned int dpp_inst, bool clock_on);
 
+void dcn35_dpstream_root_clock_control(struct dce_hwseq *hws, unsigned int dp_hpo_inst, bool clock_on);
+
 void dcn35_enable_power_gating_plane(struct dce_hwseq *hws, bool enable);
 
 void dcn35_set_dmu_fgcg(struct dce_hwseq *hws, bool enable);
@@ -84,13 +86,13 @@ void dcn35_dsc_pg_control(
 		unsigned int dsc_inst,
 		bool power_on);
 
-void dcn35_set_idle_state(const struct dc *dc, bool allow_idle);
-uint32_t dcn35_get_idle_state(const struct dc *dc);
-
 void dcn35_set_drr(struct pipe_ctx **pipe_ctx,
 		int num_pipes, struct dc_crtc_timing_adjust adjust);
 
 void dcn35_set_static_screen_control(struct pipe_ctx **pipe_ctx,
 		int num_pipes, const struct dc_static_screen_params *params);
 
+void dcn35_set_long_vblank(struct pipe_ctx **pipe_ctx,
+		int num_pipes, uint32_t v_total_min, uint32_t v_total_max);
+
 #endif /* __DC_HWSS_DCN35_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c
index a93073055e7b..df3bf77f3fb4 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c
@@ -121,8 +121,7 @@ static const struct hw_sequencer_funcs dcn35_funcs = {
 	.hw_block_power_up = dcn35_hw_block_power_up,
 	.hw_block_power_down = dcn35_hw_block_power_down,
 	.root_clock_control = dcn35_root_clock_control,
-	.set_idle_state = dcn35_set_idle_state,
-	.get_idle_state = dcn35_get_idle_state
+	.set_long_vtotal = dcn35_set_long_vblank,
 };
 
 static const struct hwseq_private_funcs dcn35_private_funcs = {
@@ -148,6 +147,7 @@ static const struct hwseq_private_funcs dcn35_private_funcs = {
 	//.hubp_pg_control = dcn35_hubp_pg_control,
 	.enable_power_gating_plane = dcn35_enable_power_gating_plane,
 	.dpp_root_clock_control = dcn35_dpp_root_clock_control,
+	.dpstream_root_clock_control = dcn35_dpstream_root_clock_control,
 	.program_all_writeback_pipes_in_tree = dcn30_program_all_writeback_pipes_in_tree,
 	.update_odm = dcn35_update_odm,
 	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn351/Makefile b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/Makefile
index b24ad27fe6ef..a4b3c1e99ec6 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn351/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/Makefile
@@ -1,16 +1,27 @@
 #
-# (c) Copyright 2022 Advanced Micro Devices, Inc. All the rights reserved
+# Copyright (c) 2022-2024 Advanced Micro Devices, Inc.
 #
-#  All rights reserved.  This notice is intended as a precaution against
-#  inadvertent publication and does not imply publication or any waiver
-#  of confidentiality.  The year included in the foregoing notice is the
-#  year of creation of the work.
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
 #
-#  Authors: AMD
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
 #
 # Makefile for DCN351.
 
-DCN351 = dcn351_init.o
+DCN351 = dcn351_hwseq.o dcn351_init.o
 
 AMD_DAL_DCN351 = $(addprefix $(AMDDALPATH)/dc/dcn351/,$(DCN351))
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_hwseq.c
new file mode 100644
index 000000000000..93fe5b262a3d
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_hwseq.c
@@ -0,0 +1,182 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2024 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "core_types.h"
+#include "resource.h"
+#include "dcn351_hwseq.h"
+#include "dcn35/dcn35_hwseq.h"
+
+#define DC_LOGGER_INIT(logger) \
+	struct dal_logger *dc_logger = logger
+
+#define DC_LOGGER \
+	dc_logger
+
+void dcn351_calc_blocks_to_gate(struct dc *dc, struct dc_state *context,
+	struct pg_block_update *update_state)
+{
+	int i, j;
+
+	dcn35_calc_blocks_to_gate(dc, context, update_state);
+
+	for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
+		if (!update_state->pg_pipe_res_update[PG_HUBP][i] &&
+			!update_state->pg_pipe_res_update[PG_DPP][i]) {
+			for (j = i - 1; j >= 0; j--) {
+				update_state->pg_pipe_res_update[PG_HUBP][j] = false;
+				update_state->pg_pipe_res_update[PG_DPP][j] = false;
+			}
+
+			break;
+		}
+	}
+}
+
+void dcn351_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
+	struct pg_block_update *update_state)
+{
+	int i, j;
+
+	dcn35_calc_blocks_to_ungate(dc, context, update_state);
+
+	for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
+		if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
+			update_state->pg_pipe_res_update[PG_DPP][i]) {
+			for (j = i - 1; j >= 0; j--) {
+				update_state->pg_pipe_res_update[PG_HUBP][j] = true;
+				update_state->pg_pipe_res_update[PG_DPP][j] = true;
+			}
+
+			break;
+		}
+	}
+}
+
+/**
+ * dcn351_hw_block_power_down() - power down sequence
+ *
+ * The following sequence describes the ON-OFF (ONO) for power down:
+ *
+ *	ONO Region 11, DCPG 19: dsc3
+ *	ONO Region 10, DCPG 3: dchubp3, dpp3
+ *	ONO Region 9, DCPG 18: dsc2
+ *	ONO Region 8, DCPG 2: dchubp2, dpp2
+ *	ONO Region 7, DCPG 17: dsc1
+ *	ONO Region 6, DCPG 1: dchubp1, dpp1
+ *	ONO Region 5, DCPG 16: dsc0
+ *	ONO Region 4, DCPG 0: dchubp0, dpp0
+ *	ONO Region 3, DCPG 25: hpo - SKIPPED. Should be kept on
+ *	ONO Region 2, DCPG 24: mpc opp optc dwb
+ *	ONO Region 1, DCPG 23: dchubbub dchvm dchubbubmem - SKIPPED. PMFW will pwr dwn at IPS2 entry
+ *	ONO Region 0, DCPG 22: dccg dio dcio - SKIPPED. will be pwr dwn after lono timer is armed
+ *
+ * @dc: Current DC state
+ * @update_state: update PG sequence states for HW block
+ */
+void dcn351_hw_block_power_down(struct dc *dc,
+	struct pg_block_update *update_state)
+{
+	int i = 0;
+	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
+
+	if (!pg_cntl || dc->debug.ignore_pg)
+		return;
+
+	for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
+		if (update_state->pg_pipe_res_update[PG_DSC][i]) {
+			if (pg_cntl->funcs->dsc_pg_control)
+				pg_cntl->funcs->dsc_pg_control(pg_cntl, i, false);
+		}
+
+		if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
+			update_state->pg_pipe_res_update[PG_DPP][i]) {
+			if (pg_cntl->funcs->hubp_dpp_pg_control)
+				pg_cntl->funcs->hubp_dpp_pg_control(pg_cntl, i, false);
+		}
+	}
+
+	// domain25 currently always on.
+
+	/* this will need all the clients to unregister optc interrupts, let dmubfw handle this */
+	if (pg_cntl->funcs->plane_otg_pg_control)
+		pg_cntl->funcs->plane_otg_pg_control(pg_cntl, false);
+
+	// domain23 currently always on.
+	// domain22 currently always on.
+}
+
+/**
+ * dcn351_hw_block_power_up() - power up sequence
+ *
+ * The following sequence describes the ON-OFF (ONO) for power up:
+ *
+ *	ONO Region 0, DCPG 22: dccg dio dcio - SKIPPED
+ *	ONO Region 1, DCPG 23: dchubbub dchvm dchubbubmem - SKIPPED. PMFW will power up at IPS2 exit
+ *	ONO Region 2, DCPG 24: mpc opp optc dwb
+ *	ONO Region 3, DCPG 25: hpo - SKIPPED
+ *	ONO Region 4, DCPG 0: dchubp0, dpp0
+ *	ONO Region 5, DCPG 16: dsc0
+ *	ONO Region 6, DCPG 1: dchubp1, dpp1
+ *	ONO Region 7, DCPG 17: dsc1
+ *	ONO Region 8, DCPG 2: dchubp2, dpp2
+ *	ONO Region 9, DCPG 18: dsc2
+ *	ONO Region 10, DCPG 3: dchubp3, dpp3
+ *	ONO Region 11, DCPG 19: dsc3
+ *
+ * @dc: Current DC state
+ * @update_state: update PG sequence states for HW block
+ */
+void dcn351_hw_block_power_up(struct dc *dc,
+	struct pg_block_update *update_state)
+{
+	int i = 0;
+	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
+
+	if (!pg_cntl || dc->debug.ignore_pg)
+		return;
+
+	// domain22 currently always on.
+	// domain23 currently always on.
+
+	/* this will need all the clients to unregister optc interrupts, let dmubfw handle this */
+	if (pg_cntl->funcs->plane_otg_pg_control)
+		pg_cntl->funcs->plane_otg_pg_control(pg_cntl, true);
+
+	// domain25 currently always on.
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
+			update_state->pg_pipe_res_update[PG_DPP][i]) {
+			if (pg_cntl->funcs->hubp_dpp_pg_control)
+				pg_cntl->funcs->hubp_dpp_pg_control(pg_cntl, i, true);
+		}
+
+		if (update_state->pg_pipe_res_update[PG_DSC][i]) {
+			if (pg_cntl->funcs->dsc_pg_control)
+				pg_cntl->funcs->dsc_pg_control(pg_cntl, i, true);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dpp.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_hwseq.h
index 3341ef71009b..6d8f3bfb668e 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_hwseq.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT */
 /*
- * Copyright 2023 Advanced Micro Devices, Inc.
+ * Copyright 2024 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -24,30 +24,18 @@
  *
  */
 
-#include "core_types.h"
-#include "dcn35_dpp.h"
-#include "reg_helper.h"
+#ifndef __DC_HWSS_DCN351_H__
+#define __DC_HWSS_DCN351_H__
 
-#define REG(reg) dpp->tf_regs->reg
+#include "hw_sequencer_private.h"
 
-#define CTX dpp->base.ctx
+void dcn351_calc_blocks_to_gate(struct dc *dc, struct dc_state *context,
+	struct pg_block_update *update_state);
+void dcn351_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
+	struct pg_block_update *update_state);
+void dcn351_hw_block_power_up(struct dc *dc,
+	struct pg_block_update *update_state);
+void dcn351_hw_block_power_down(struct dc *dc,
+	struct pg_block_update *update_state);
 
-#undef FN
-#define FN(reg_name, field_name)                                       \
-	((const struct dcn35_dpp_shift *)(dpp->tf_shift))->field_name, \
-	((const struct dcn35_dpp_mask *)(dpp->tf_mask))->field_name
-
-bool dpp35_construct(struct dcn3_dpp *dpp, struct dc_context *ctx,
-		     uint32_t inst, const struct dcn3_dpp_registers *tf_regs,
-		     const struct dcn35_dpp_shift *tf_shift,
-		     const struct dcn35_dpp_mask *tf_mask)
-{
-	return dpp32_construct(dpp, ctx, inst, tf_regs,
-			      (const struct dcn3_dpp_shift *)(tf_shift),
-			      (const struct dcn3_dpp_mask *)(tf_mask));
-}
-
-void dpp35_set_fgcg(struct dcn3_dpp *dpp, bool enable)
-{
-	REG_UPDATE(DPP_CONTROL, DPP_FGCG_REP_DIS, !enable);
-}
+#endif /* __DC_HWSS_DCN351_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c
index 670255c9bc82..a53092cd619b 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c
@@ -32,6 +32,7 @@
 #include "dcn31/dcn31_hwseq.h"
 #include "dcn32/dcn32_hwseq.h"
 #include "dcn35/dcn35_hwseq.h"
+#include "dcn351/dcn351_hwseq.h"
 
 #include "dcn351_init.h"
 
@@ -120,8 +121,6 @@ static const struct hw_sequencer_funcs dcn351_funcs = {
 	.hw_block_power_up = dcn35_hw_block_power_up,
 	.hw_block_power_down = dcn35_hw_block_power_down,
 	.root_clock_control = dcn35_root_clock_control,
-	.set_idle_state = dcn35_set_idle_state,
-	.get_idle_state = dcn35_get_idle_state
 };
 
 static const struct hwseq_private_funcs dcn351_private_funcs = {
@@ -147,6 +146,7 @@ static const struct hwseq_private_funcs dcn351_private_funcs = {
 	//.hubp_pg_control = dcn35_hubp_pg_control,
 	.enable_power_gating_plane = dcn35_enable_power_gating_plane,
 	.dpp_root_clock_control = dcn35_dpp_root_clock_control,
+	.dpstream_root_clock_control = dcn35_dpstream_root_clock_control,
 	.program_all_writeback_pipes_in_tree = dcn30_program_all_writeback_pipes_in_tree,
 	.update_odm = dcn35_update_odm,
 	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
index f89f205e42a1..7c339e7e7117 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
@@ -377,7 +377,10 @@ struct hw_sequencer_funcs {
 	/* Idle Optimization Related */
 	bool (*apply_idle_power_optimizations)(struct dc *dc, bool enable);
 
-	bool (*does_plane_fit_in_mall)(struct dc *dc, struct dc_plane_state *plane,
+	bool (*does_plane_fit_in_mall)(struct dc *dc,
+			unsigned int pitch,
+			unsigned int height,
+			enum surface_pixel_format format,
 			struct dc_cursor_attributes *cursor_attr);
 	void (*commit_subvp_config)(struct dc *dc, struct dc_state *context);
 	void (*enable_phantom_streams)(struct dc *dc, struct dc_state *context);
@@ -424,11 +427,10 @@ struct hw_sequencer_funcs {
 		struct pg_block_update *update_state);
 	void (*root_clock_control)(struct dc *dc,
 		struct pg_block_update *update_state, bool power_on);
-	void (*set_idle_state)(const struct dc *dc, bool allow_idle);
-	uint32_t (*get_idle_state)(const struct dc *dc);
 	bool (*is_pipe_topology_transition_seamless)(struct dc *dc,
 			const struct dc_state *cur_ctx,
 			const struct dc_state *new_ctx);
+	void (*set_long_vtotal)(struct pipe_ctx **pipe_ctx, int num_pipes, uint32_t v_total_min, uint32_t v_total_max);
 };
 
 void color_space_to_black_color(
@@ -478,9 +480,10 @@ void hwss_build_fast_sequence(struct dc *dc,
 		struct dc_dmub_cmd *dc_dmub_cmd,
 		unsigned int dmub_cmd_count,
 		struct block_sequence block_sequence[],
-		int *num_steps,
+		unsigned int *num_steps,
 		struct pipe_ctx *pipe_ctx,
-		struct dc_stream_status *stream_status);
+		struct dc_stream_status *stream_status,
+		struct dc_state *context);
 
 void hwss_send_dmcub_cmd(union block_sequence_params *params);
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
index 554cfab5ab24..341219cf4144 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
@@ -120,6 +120,10 @@ struct hwseq_private_funcs {
 			struct dce_hwseq *hws,
 			unsigned int dpp_inst,
 			bool clock_on);
+	void (*dpstream_root_clock_control)(
+			struct dce_hwseq *hws,
+			unsigned int dpp_inst,
+			bool clock_on);
 	void (*dpp_pg_control)(struct dce_hwseq *hws,
 			unsigned int dpp_inst,
 			bool power_on);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
index b1b72e688f74..028b2f971e36 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
@@ -90,6 +90,9 @@ struct resource_funcs {
 	void (*update_soc_for_wm_a)(
 				struct dc *dc, struct dc_state *context);
 
+	unsigned int (*calculate_mall_ways_from_bytes)(
+				const struct dc *dc,
+				unsigned int total_size_in_mall_bytes);
 	/**
 	 * @populate_dml_pipes - Populate pipe data struct
 	 *
@@ -336,7 +339,9 @@ struct stream_resource {
 };
 
 struct plane_resource {
+	/* scl_data is scratch space required to program a plane */
 	struct scaler_data scl_data;
+	/* Below pointers to hw objects are required to enable the plane */
 	struct hubp *hubp;
 	struct mem_input *mi;
 	struct input_pixel_processor *ipp;
@@ -496,7 +501,7 @@ struct dcn_bw_writeback {
 
 struct dcn_bw_output {
 	struct dc_clocks clk;
-	struct dcn_watermark_set watermarks;
+	union dcn_watermark_set watermarks;
 	struct dcn_bw_writeback bw_writeback;
 	int compbuf_size_kb;
 	unsigned int mall_ss_size_bytes;
@@ -515,6 +520,7 @@ struct bw_context {
 	union bw_output bw;
 	struct display_mode_lib dml;
 	struct dml2_context *dml2;
+	struct dml2_context *dml2_dc_power_source;
 };
 
 struct dc_dmub_cmd {
@@ -522,25 +528,6 @@ struct dc_dmub_cmd {
 	enum dm_dmub_wait_type wait_type;
 };
 
-struct dc_scratch_space {
-	/* used to temporarily backup plane states of a stream during
-	 * dc update. The reason is that plane states are overwritten
-	 * with surface updates in dc update. Once they are overwritten
-	 * current state is no longer valid. We want to temporarily
-	 * store current value in plane states so we can still recover
-	 * a valid current state during dc update.
-	 */
-	struct dc_plane_state plane_states[MAX_SURFACE_NUM];
-	struct dc_gamma gamma_correction[MAX_SURFACE_NUM];
-	struct dc_transfer_func in_transfer_func[MAX_SURFACE_NUM];
-	struct dc_3dlut lut3d_func[MAX_SURFACE_NUM];
-	struct dc_transfer_func in_shaper_func[MAX_SURFACE_NUM];
-	struct dc_transfer_func blend_tf[MAX_SURFACE_NUM];
-
-	struct dc_stream_state stream_state;
-	struct dc_transfer_func out_transfer_func;
-};
-
 /**
  * struct dc_state - The full description of a state requested by users
  */
@@ -623,8 +610,7 @@ struct dc_state {
 		unsigned int stutter_period_us;
 	} perf_params;
 
-
-	struct dc_scratch_space scratch;
+	enum dc_power_source_type power_source;
 };
 
 struct replay_context {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h b/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
index 9e4ddc985240..55529c5f471c 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
@@ -31,7 +31,7 @@
 #define __DCN_CALCS_H__
 
 #include "bw_fixed.h"
-#include "../dml/display_mode_lib.h"
+#include "dml/display_mode_lib.h"
 
 
 struct dc;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
index 17e014d3bdc8..4f7480f60c85 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
@@ -281,8 +281,6 @@ struct clk_mgr_funcs {
 	void (*set_low_power_state)(struct clk_mgr *clk_mgr);
 	void (*exit_low_power_state)(struct clk_mgr *clk_mgr);
 	bool (*is_ips_supported)(struct clk_mgr *clk_mgr);
-	void (*set_idle_state)(struct clk_mgr *clk_mgr, bool allow_idle);
-	uint32_t (*get_idle_state)(struct clk_mgr *clk_mgr);
 
 	void (*init_clocks)(struct clk_mgr *clk_mgr);
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h
index f4d4a68c91dc..4ba18ea57aad 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h
@@ -349,7 +349,7 @@ struct clk_mgr_internal {
 	enum dm_pp_clocks_state cur_min_clks_state;
 	bool periodic_retraining_disabled;
 
-	unsigned int cur_phyclk_req_table[MAX_PIPES * 2];
+	unsigned int cur_phyclk_req_table[MAX_LINKS];
 
 	bool smu_present;
 	void *wm_range_table;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
index b9a06bf84cc9..d4c7885fc916 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
@@ -59,6 +59,7 @@ enum dentist_dispclk_change_mode {
 struct dp_dto_params {
 	int otg_inst;
 	enum signal_type signal;
+	enum streamclk_source clk_src;
 	uint64_t pixclk_hz;
 	uint64_t refclk_hz;
 };
@@ -105,6 +106,10 @@ struct dccg_funcs {
 	void (*otg_drop_pixel)(struct dccg *dccg,
 			uint32_t otg_inst);
 	void (*dccg_init)(struct dccg *dccg);
+	void (*set_dpstreamclk_root_clock_gating)(
+			struct dccg *dccg,
+			int dp_hpo_inst,
+			bool enable);
 
 	void (*set_dpstreamclk)(
 			struct dccg *dccg,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
index 2ae7484d18af..305fdc127bfc 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
@@ -160,7 +160,7 @@ struct hubbub_funcs {
 
 	bool (*program_watermarks)(
 			struct hubbub *hubbub,
-			struct dcn_watermark_set *watermarks,
+			union dcn_watermark_set *watermarks,
 			unsigned int refclk_mhz,
 			bool safe_to_lower);
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
index 0f24afbf4388..ca8de345d039 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
@@ -322,7 +322,7 @@ struct dpp_funcs {
 			const struct pwl_params *params);
 	bool (*dpp_program_3dlut)(
 			struct dpp *dpp,
-			struct tetrahedral_params *params);
+			const struct tetrahedral_params *params);
 	void (*dpp_cnv_set_alpha_keyer)(
 			struct dpp *dpp_base,
 			struct cnv_color_keyer_params *color_keyer);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dwb.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dwb.h
index 729ca0064e94..063efc8128a7 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dwb.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dwb.h
@@ -147,9 +147,10 @@ struct dwb_caps {
 		unsigned int support_ogam	:1;
 		unsigned int support_wbscl	:1;
 		unsigned int support_ocsc	:1;
-		unsigned int support_stereo :1;
+		unsigned int support_stereo	:1;
+		unsigned int support_4k_120p	:1;
 	} caps;
-	unsigned int	 reserved2[9];	/* Reserved for future use, MUST BE 0. */
+	unsigned int	 reserved2[10];	/* Reserved for future use, MUST BE 0. */
 };
 
 struct dwbc {
@@ -166,8 +167,9 @@ struct dwbc {
 	bool dwb_is_drc;
 	int wb_src_plane_inst;/*hubp, mpcc, inst*/
 	uint32_t mask_id;
-    int otg_inst;
-    bool mvc_cfg;
+	int otg_inst;
+	bool mvc_cfg;
+	struct dc_dwb_params params;
 };
 
 struct dwbc_funcs {
@@ -192,6 +194,10 @@ struct dwbc_funcs {
 		struct dwbc *dwbc,
 		enum dwb_frame_capture_enable enable);
 
+	void (*dwb_set_scaler)(
+		struct dwbc *dwbc,
+		struct dc_dwb_params *params);
+
 	void (*set_stereo)(
 		struct dwbc *dwbc,
 		struct dwb_stereo_params *stereo_params);
@@ -205,9 +211,11 @@ struct dwbc_funcs {
 		struct dwbc *dwbc,
 		struct dwb_warmup_params *warmup_params);
 
-
+	bool (*dwb_get_mcifbuf_line)(
+		struct dwbc *dwbc, unsigned int *buf_idx,
+		unsigned int *cur_line,
+		unsigned int *over_run);
 #if defined(CONFIG_DRM_AMD_DC_FP)
-
 	void (*dwb_program_output_csc)(
 		struct dwbc *dwbc,
 		enum dc_color_space color_space,
@@ -216,17 +224,17 @@ struct dwbc_funcs {
 	bool (*dwb_ogam_set_output_transfer_func)(
 		struct dwbc *dwbc,
 		const struct dc_transfer_func *in_transfer_func_dwb_ogam);
-
+#endif
 	//TODO: merge with output_transfer_func?
 	bool (*dwb_ogam_set_input_transfer_func)(
 		struct dwbc *dwbc,
 		const struct dc_transfer_func *in_transfer_func_dwb_ogam);
-#endif
+
+	void (*get_drr_time_stamp)(
+		struct dwbc *dwbc, uint32_t *time_stamp);
+
 	bool (*get_dwb_status)(
 		struct dwbc *dwbc);
-	void (*dwb_set_scaler)(
-		struct dwbc *dwbc,
-		struct dc_dwb_params *params);
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
index dcae23faeee3..c80ebb407add 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
@@ -44,10 +44,11 @@
  */
 #define MAX_PIPES 6
 #define MAX_PHANTOM_PIPES (MAX_PIPES / 2)
+#define MAX_LINKS (MAX_PIPES * 2)
 #define MAX_DIG_LINK_ENCODERS 7
 #define MAX_DWB_PIPES	1
 #define MAX_HPO_DP2_ENCODERS	4
-#define MAX_HPO_DP2_LINK_ENCODERS	2
+#define MAX_HPO_DP2_LINK_ENCODERS	4
 
 struct gamma_curve {
 	uint32_t offset;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
index dbe7afa9d3a2..af9183f5d69b 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
@@ -163,12 +163,11 @@ struct link_encoder_funcs {
 
 	enum signal_type (*get_dig_mode)(
 		struct link_encoder *enc);
+
 	void (*set_dio_phy_mux)(
 		struct link_encoder *enc,
 		enum encoder_type_select sel,
 		uint32_t hpo_inst);
-	void (*set_dig_output_mode)(
-			struct link_encoder *enc, uint8_t pix_per_container);
 };
 
 /*
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h b/drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h
index b72fb314d804..86c12cd6f47d 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/mem_input.h
@@ -50,11 +50,13 @@ struct dcn_watermarks {
 	uint32_t usr_retraining_ns;
 };
 
-struct dcn_watermark_set {
-	struct dcn_watermarks a;
-	struct dcn_watermarks b;
-	struct dcn_watermarks c;
-	struct dcn_watermarks d;
+union dcn_watermark_set {
+	struct {
+		struct dcn_watermarks a;
+		struct dcn_watermarks b;
+		struct dcn_watermarks c;
+		struct dcn_watermarks d;
+	}; // legacy
 };
 
 struct dce_watermarks {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/optc.h b/drivers/gpu/drm/amd/display/dc/inc/hw/optc.h
index 9a8bf6ec70ea..8d32e525f05a 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/optc.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/optc.h
@@ -93,6 +93,8 @@ struct dcn_otg_state {
 	uint32_t vertical_interrupt1_line;
 	uint32_t vertical_interrupt2_en;
 	uint32_t vertical_interrupt2_line;
+	uint32_t otg_master_update_lock;
+	uint32_t otg_double_buffer_control;
 };
 
 void optc1_read_otg_state(struct optc *optc1, struct dcn_otg_state *s);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
index a15efadb9183..75b9ec21f297 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
@@ -178,10 +178,6 @@ struct stream_encoder_funcs {
 	void (*stop_dp_info_packets)(
 		struct stream_encoder *enc);
 
-	void (*reset_fifo)(
-		struct stream_encoder *enc
-	);
-
 	void (*dp_blank)(
 		struct dc_link *link,
 		struct stream_encoder *enc);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
index ffad8fe16c54..cd68ecc242c1 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
@@ -64,6 +64,12 @@ struct drr_params {
 	bool immediate_flip;
 };
 
+struct long_vtotal_params {
+	uint32_t vertical_total_min;
+	uint32_t vertical_total_max;
+	uint32_t vertical_blank_start;
+};
+
 #define LEFT_EYE_3D_PRIMARY_SURFACE 1
 #define RIGHT_EYE_3D_PRIMARY_SURFACE 0
 
@@ -331,6 +337,7 @@ struct timing_generator_funcs {
 
 	void (*init_odm)(struct timing_generator *tg);
 	void (*wait_drr_doublebuffer_pending_clear)(struct timing_generator *tg);
+	void (*set_long_vtotal)(struct timing_generator *optc, const struct long_vtotal_params *params);
 	void (*wait_odm_doublebuffer_pending_clear)(struct timing_generator *tg);
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/vpg.h b/drivers/gpu/drm/amd/display/dc/inc/hw/vpg.h
new file mode 100644
index 000000000000..51da368f5c3e
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/vpg.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2024 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ */
+
+#ifndef __DC_VPG_H__
+#define __DC_VPG_H__
+
+struct dc_context;
+struct dc_info_packet;
+
+struct vpg;
+
+struct vpg_funcs {
+	void (*update_generic_info_packet)(
+		struct vpg *vpg,
+		uint32_t packet_index,
+		const struct dc_info_packet *info_packet,
+		bool immediate_update);
+
+	void (*vpg_poweron)(
+		struct vpg *vpg);
+
+	void (*vpg_powerdown)(
+		struct vpg *vpg);
+};
+
+struct vpg {
+	const struct vpg_funcs *funcs;
+	struct dc_context *ctx;
+	int inst;
+};
+
+#endif /* DC_INC_VPG_H_ */
\ No newline at end of file
diff --git a/drivers/gpu/drm/amd/display/dc/inc/link.h b/drivers/gpu/drm/amd/display/dc/inc/link.h
index bf29fc58ea6a..7ab8ba5e23ed 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/link.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/link.h
@@ -288,7 +288,7 @@ struct link_service {
 			struct dc_link *link, uint32_t coasting_vtotal);
 	bool (*edp_replay_residency)(const struct dc_link *link,
 			unsigned int *residency, const bool is_start,
-			const bool is_alpm);
+			const enum pr_residency_mode mode);
 	bool (*edp_set_replay_power_opt_and_coasting_vtotal)(struct dc_link *link,
 			const unsigned int *power_opts, uint32_t coasting_vtotal);
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/resource.h b/drivers/gpu/drm/amd/display/dc/inc/resource.h
index 77a60aa9f27b..361ad6b16b96 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/resource.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/resource.h
@@ -510,6 +510,17 @@ int recource_find_free_pipe_used_as_otg_master_in_cur_res_ctx(
 
 /*
  * Look for a free pipe in new resource context that is used as a secondary DPP
+ * pipe in current resource context.
+ * return - FREE_PIPE_INDEX_NOT_FOUND if free pipe is not found, otherwise
+ * pipe idx of the free pipe
+ */
+int resource_find_free_pipe_used_as_cur_sec_dpp(
+		const struct resource_context *cur_res_ctx,
+		struct resource_context *new_res_ctx,
+		const struct resource_pool *pool);
+
+/*
+ * Look for a free pipe in new resource context that is used as a secondary DPP
  * pipe in any MPCC combine in current resource context.
  * return - FREE_PIPE_INDEX_NOT_FOUND if free pipe is not found, otherwise
  * pipe idx of the free pipe
@@ -573,13 +584,6 @@ bool get_temp_dp_link_res(struct dc_link *link,
 		struct link_resource *link_res,
 		struct dc_link_settings *link_settings);
 
-#if defined(CONFIG_DRM_AMD_DC_FP)
-struct hpo_dp_link_encoder *resource_get_hpo_dp_link_enc_for_det_lt(
-		const struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		const struct dc_link *link);
-#endif
-
 void reset_syncd_pipes_from_disabled_pipes(struct dc *dc,
 	struct dc_state *context);
 
@@ -615,4 +619,10 @@ enum dc_status update_dp_encoder_resources_for_test_harness(const struct dc *dc,
 		struct pipe_ctx *pipe_ctx);
 
 bool check_subvp_sw_cursor_fallback_req(const struct dc *dc, struct dc_stream_state *stream);
+
+/* Setup dc callbacks for dml2
+ * @dc: the display core structure
+ * @dml2_options: struct to hold callbacks
+ */
+void resource_init_common_dml2_callbacks(struct dc *dc, struct dml2_configuration_options *dml2_options);
 #endif /* DRIVERS_GPU_DRM_AMD_DC_DEV_DC_INC_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
index 1c0d89e675da..bb576a9c5fdb 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
@@ -211,8 +211,12 @@ bool dce110_vblank_set(struct irq_service *irq_service,
 						   info->ext_id);
 	uint8_t pipe_offset = dal_irq_src - IRQ_TYPE_VBLANK;
 
-	struct timing_generator *tg =
-			dc->current_state->res_ctx.pipe_ctx[pipe_offset].stream_res.tg;
+	struct timing_generator *tg;
+
+	if (pipe_offset >= MAX_PIPES)
+		return false;
+
+	tg = dc->current_state->res_ctx.pipe_ctx[pipe_offset].stream_res.tg;
 
 	if (enable) {
 		if (!tg || !tg->funcs->arm_vert_intr(tg, 2)) {
diff --git a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c
index 22b24749c9d2..8d1a1cc94a8b 100644
--- a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c
+++ b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c
@@ -884,7 +884,7 @@ void dp_set_preferred_link_settings(struct dc *dc,
 {
 	int i;
 	struct pipe_ctx *pipe;
-	struct dc_stream_state *link_stream;
+	struct dc_stream_state *link_stream = 0;
 	struct dc_link_settings store_settings = *link_setting;
 
 	link->preferred_link_setting = store_settings;
diff --git a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_trace.c b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_trace.c
index fbcd8fb58ea8..c8c55f196f8d 100644
--- a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_trace.c
+++ b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_trace.c
@@ -24,7 +24,6 @@
  */
 #include "link_dp_trace.h"
 #include "link/protocols/link_dpcd.h"
-#include "link.h"
 
 void dp_trace_init(struct dc_link *link)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
index b8c4a04dd175..0d523dc43d02 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
@@ -516,8 +516,8 @@ static void query_hdcp_capability(enum signal_type signal, struct dc_link *link)
 static void read_current_link_settings_on_detect(struct dc_link *link)
 {
 	union lane_count_set lane_count_set = {0};
-	uint8_t link_bw_set;
-	uint8_t link_rate_set;
+	uint8_t link_bw_set = 0;
+	uint8_t link_rate_set = 0;
 	uint32_t read_dpcd_retry_cnt = 10;
 	enum dc_status status = DC_ERROR_UNEXPECTED;
 	int i;
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
index a72de44a5747..b53ad18dbfbc 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
@@ -55,6 +55,8 @@
 #include "dccg.h"
 #include "clk_mgr.h"
 #include "atomfirmware.h"
+#include "vpg.h"
+
 #define DC_LOGGER \
 	dc_logger
 #define DC_LOGGER_INIT(logger) \
@@ -67,7 +69,6 @@
 #define RETIMER_REDRIVER_INFO(...) \
 	DC_LOG_RETIMER_REDRIVER(  \
 		__VA_ARGS__)
-#include "dc/dcn30/dcn30_vpg.h"
 
 #define MAX_MTP_SLOT_COUNT 64
 #define LINK_TRAINING_ATTEMPTS 4
@@ -127,7 +128,7 @@ void link_blank_dp_stream(struct dc_link *link, bool hw_init)
 		if (link->ep_type == DISPLAY_ENDPOINT_PHY &&
 			link->link_enc->funcs->get_dig_frontend &&
 			link->link_enc->funcs->is_dig_enabled(link->link_enc)) {
-			unsigned int fe = link->link_enc->funcs->get_dig_frontend(link->link_enc);
+			int fe = link->link_enc->funcs->get_dig_frontend(link->link_enc);
 
 			if (fe != ENGINE_ID_UNKNOWN)
 				for (j = 0; j < dc->res_pool->stream_enc_count; j++) {
@@ -725,7 +726,7 @@ static void set_avmute(struct pipe_ctx *pipe_ctx, bool enable)
 
 static void enable_mst_on_sink(struct dc_link *link, bool enable)
 {
-	unsigned char mstmCntl;
+	unsigned char mstmCntl = 0;
 
 	core_link_read_dpcd(link, DP_MSTM_CTRL, &mstmCntl, 1);
 	if (enable)
@@ -803,7 +804,7 @@ void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 
 	if (enable) {
 		struct dsc_config dsc_cfg;
-		struct dsc_optc_config dsc_optc_cfg;
+		struct dsc_optc_config dsc_optc_cfg = {0};
 		enum optc_dsc_mode optc_dsc_mode;
 
 		/* Enable DSC hw block */
@@ -1575,7 +1576,7 @@ static bool write_128b_132b_sst_payload_allocation_table(
 				break;
 			}
 		} else {
-			union dpcd_rev dpcdRev;
+			union dpcd_rev dpcdRev = {0};
 
 			if (core_link_read_dpcd(
 					link,
@@ -2119,7 +2120,7 @@ static enum dc_status enable_link_dp_mst(
 		struct pipe_ctx *pipe_ctx)
 {
 	struct dc_link *link = pipe_ctx->stream->link;
-	unsigned char mstm_cntl;
+	unsigned char mstm_cntl = 0;
 
 	/* sink signal type after MST branch is MST. Multiple MST sinks
 	 * share one link. Link DP PHY is enable or training only once.
@@ -2285,6 +2286,7 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 	struct dc_stream_state *stream = pipe_ctx->stream;
 	struct dc_link *link = stream->sink->link;
 	struct vpg *vpg = pipe_ctx->stream_res.stream_enc->vpg;
+	enum dp_panel_mode panel_mode_dp = dp_get_panel_mode(link);
 
 	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
 
@@ -2311,6 +2313,8 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 
 	dc->hwss.disable_audio_stream(pipe_ctx);
 
+	edp_set_panel_assr(link, pipe_ctx, &panel_mode_dp, false);
+
 	update_psp_stream_config(pipe_ctx, true);
 	dc->hwss.blank_stream(pipe_ctx);
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
index 289f5d133342..a01d0842bf8e 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
@@ -992,7 +992,7 @@ enum dp_link_encoding mst_decide_link_encoding_format(const struct dc_link *link
 
 static void read_dp_device_vendor_id(struct dc_link *link)
 {
-	struct dp_device_vendor_id dp_id;
+	struct dp_device_vendor_id dp_id = {0};
 
 	/* read IEEE branch device id */
 	core_link_read_dpcd(
@@ -1087,7 +1087,7 @@ static void get_active_converter_info(
 	}
 
 	if (link->dpcd_caps.dpcd_rev.raw >= DPCD_REV_11) {
-		uint8_t det_caps[16]; /* CTS 4.2.2.7 expects source to read Detailed Capabilities Info : 00080h-0008F.*/
+		uint8_t det_caps[16] = {0}; /* CTS 4.2.2.7 expects source to read Detailed Capabilities Info : 00080h-0008F.*/
 		union dwnstream_port_caps_byte0 *port_caps =
 			(union dwnstream_port_caps_byte0 *)det_caps;
 		if (core_link_read_dpcd(link, DP_DOWNSTREAM_PORT_0,
@@ -1172,7 +1172,7 @@ static void get_active_converter_info(
 	set_dongle_type(link->ddc, link->dpcd_caps.dongle_type);
 
 	{
-		struct dp_sink_hw_fw_revision dp_hw_fw_revision;
+		struct dp_sink_hw_fw_revision dp_hw_fw_revision = {0};
 
 		core_link_read_dpcd(
 			link,
@@ -1242,7 +1242,7 @@ static void apply_usbc_combo_phy_reset_wa(struct dc_link *link,
 
 bool dp_overwrite_extended_receiver_cap(struct dc_link *link)
 {
-	uint8_t dpcd_data[16];
+	uint8_t dpcd_data[16] = {0};
 	uint32_t read_dpcd_retry_cnt = 3;
 	enum dc_status status = DC_ERROR_UNEXPECTED;
 	union dp_downstream_port_present ds_port = { 0 };
@@ -1408,7 +1408,7 @@ static bool get_usbc_cable_id(struct dc_link *link, union dp_cable_id *cable_id)
 
 static void retrieve_cable_id(struct dc_link *link)
 {
-	union dp_cable_id usbc_cable_id;
+	union dp_cable_id usbc_cable_id = {0};
 
 	link->dpcd_caps.cable_id.raw = 0;
 	core_link_read_dpcd(link, DP_CABLE_ATTRIBUTES_UPDATED_BY_DPRX,
@@ -1475,7 +1475,7 @@ static bool dpcd_read_sink_ext_caps(struct dc_link *link)
 
 enum dc_status dp_retrieve_lttpr_cap(struct dc_link *link)
 {
-	uint8_t lttpr_dpcd_data[8];
+	uint8_t lttpr_dpcd_data[8] = {0};
 	enum dc_status status;
 	bool is_lttpr_present;
 
@@ -1931,8 +1931,8 @@ void detect_edp_sink_caps(struct dc_link *link)
 	uint32_t entry;
 	uint32_t link_rate_in_khz;
 	enum dc_link_rate link_rate = LINK_RATE_UNKNOWN;
-	uint8_t backlight_adj_cap;
-	uint8_t general_edp_cap;
+	uint8_t backlight_adj_cap = 0;
+	uint8_t general_edp_cap = 0;
 
 	retrieve_link_cap(link);
 	link->dpcd_caps.edp_supported_link_rates_count = 0;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c
index 5491b707cec8..0f1c411523a2 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c
@@ -166,7 +166,7 @@ static uint8_t get_lowest_dpia_index(struct dc_link *link)
 	uint8_t idx = 0xFF;
 	int i;
 
-	for (i = 0; i < MAX_PIPES * 2; ++i) {
+	for (i = 0; i < MAX_LINKS; ++i) {
 
 		if (!dc_struct->links[i] ||
 				dc_struct->links[i]->ep_type != DISPLAY_ENDPOINT_USB4_DPIA)
@@ -196,7 +196,7 @@ static int get_host_router_total_dp_tunnel_bw(const struct dc *dc, uint8_t hr_in
 	struct dc_link *link_dpia_primary, *link_dpia_secondary;
 	int total_bw = 0;
 
-	for (uint8_t i = 0; i < (MAX_PIPES * 2) - 1; ++i) {
+	for (uint8_t i = 0; i < MAX_LINKS - 1; ++i) {
 
 		if (!dc->links[i] || dc->links[i]->ep_type != DISPLAY_ENDPOINT_USB4_DPIA)
 			continue;
@@ -270,7 +270,7 @@ static void set_usb4_req_bw_req(struct dc_link *link, int req_bw)
 
 	/* Error check whether requested and allocated are equal */
 	req_bw = requested_bw * (Kbps_TO_Gbps / link->dpia_bw_alloc_config.bw_granularity);
-	if (req_bw == link->dpia_bw_alloc_config.allocated_bw) {
+	if (req_bw && (req_bw == link->dpia_bw_alloc_config.allocated_bw)) {
 		DC_LOG_ERROR("%s: Request bw equals to allocated bw for link(%d)\n",
 			__func__, link->link_index);
 	}
@@ -341,6 +341,14 @@ bool link_dp_dpia_set_dptx_usb4_bw_alloc_support(struct dc_link *link)
 			ret = true;
 			init_usb4_bw_struct(link);
 			link->dpia_bw_alloc_config.bw_alloc_enabled = true;
+
+			/*
+			 * During DP tunnel creation, CM preallocates BW and reduces estimated BW of other
+			 * DPIA. CM release preallocation only when allocation is complete. Do zero alloc
+			 * to make the CM to release preallocation and update estimated BW correctly for
+			 * all DPIAs per host router
+			 */
+			link_dp_dpia_allocate_usb4_bandwidth_for_stream(link, 0);
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
index ba69874be5a4..0fcf0b8530ac 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
@@ -120,7 +120,7 @@ bool dp_parse_link_loss_status(
 
 static bool handle_hpd_irq_psr_sink(struct dc_link *link)
 {
-	union dpcd_psr_configuration psr_configuration;
+	union dpcd_psr_configuration psr_configuration = {0};
 
 	if (!link->psr_settings.psr_feature_enabled)
 		return false;
@@ -186,9 +186,9 @@ static bool handle_hpd_irq_psr_sink(struct dc_link *link)
 
 static void handle_hpd_irq_replay_sink(struct dc_link *link)
 {
-	union dpcd_replay_configuration replay_configuration;
+	union dpcd_replay_configuration replay_configuration = {0};
 	/*AMD Replay version reuse DP_PSR_ERROR_STATUS for REPLAY_ERROR status.*/
-	union psr_error_status replay_error_status;
+	union psr_error_status replay_error_status = {0};
 
 	if (!link->replay_settings.replay_feature_enabled)
 		return;
@@ -280,7 +280,7 @@ void dp_handle_link_loss(struct dc_link *link)
 static void read_dpcd204h_on_irq_hpd(struct dc_link *link, union hpd_irq_data *irq_data)
 {
 	enum dc_status retval;
-	union lane_align_status_updated dpcd_lane_status_updated;
+	union lane_align_status_updated dpcd_lane_status_updated = {0};
 
 	retval = core_link_read_dpcd(
 			link,
@@ -320,7 +320,7 @@ enum dc_status dp_read_hpd_rx_irq_data(
 		/* Read 14 bytes in a single read and then copy only the required fields.
 		 * This is more efficient than doing it in two separate AUX reads. */
 
-		uint8_t tmp[DP_SINK_STATUS_ESI - DP_SINK_COUNT_ESI + 1];
+		uint8_t tmp[DP_SINK_STATUS_ESI - DP_SINK_COUNT_ESI + 1] = {0};
 
 		retval = core_link_read_dpcd(
 			link,
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c
index e538c67d3ed9..1818970b8eaf 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c
@@ -1071,7 +1071,7 @@ enum dc_status dpcd_set_link_settings(
 		 * MUX chip gets link rate set back before link training.
 		 */
 		if (link->connector_signal == SIGNAL_TYPE_EDP) {
-			uint8_t supported_link_rates[16];
+			uint8_t supported_link_rates[16] = {0};
 
 			core_link_read_dpcd(link, DP_SUPPORTED_LINK_RATES,
 					supported_link_rates, sizeof(supported_link_rates));
@@ -1587,21 +1587,7 @@ bool perform_link_training_with_retries(
 			msleep(delay_dp_power_up_in_ms);
 		}
 
-		if (panel_mode == DP_PANEL_MODE_EDP) {
-			struct cp_psp *cp_psp = &stream->ctx->cp_psp;
-
-			if (cp_psp && cp_psp->funcs.enable_assr) {
-				/* ASSR is bound to fail with unsigned PSP
-				 * verstage used during devlopment phase.
-				 * Report and continue with eDP panel mode to
-				 * perform eDP link training with right settings
-				 */
-				bool result;
-				result = cp_psp->funcs.enable_assr(cp_psp->handle, link);
-				if (!result && link->panel_mode != DP_PANEL_MODE_EDP)
-					panel_mode = DP_PANEL_MODE_DEFAULT;
-			}
-		}
+		edp_set_panel_assr(link, pipe_ctx, &panel_mode, true);
 
 		dp_set_panel_mode(link, panel_mode);
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_dpia.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_dpia.c
index 5d36bab0029c..edb21d21952a 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_dpia.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_dpia.c
@@ -291,7 +291,7 @@ static enum link_training_result dpia_training_cr_non_transparent(
 {
 	enum link_training_result result = LINK_TRAINING_CR_FAIL_LANE0;
 	uint8_t repeater_cnt = 0; /* Number of hops/repeaters in display path. */
-	enum dc_status status;
+	enum dc_status status = DC_ERROR_UNEXPECTED;
 	uint32_t retries_cr = 0; /* Number of consecutive attempts with same VS or PE. */
 	uint32_t retry_count = 0;
 	uint32_t wait_time_microsec = TRAINING_AUX_RD_INTERVAL; /* From DP spec, CR read interval is always 100us. */
@@ -617,7 +617,7 @@ static enum link_training_result dpia_training_eq_non_transparent(
 	enum link_training_result result = LINK_TRAINING_EQ_FAIL_EQ;
 	uint8_t repeater_cnt = 0; /* Number of hops/repeaters in display path. */
 	uint32_t retries_eq = 0;
-	enum dc_status status;
+	enum dc_status status = DC_ERROR_UNEXPECTED;
 	enum dc_dp_training_pattern tr_pattern;
 	uint32_t wait_time_microsec = 0;
 	enum dc_lane_count lane_count = lt_settings->link_settings.lane_count;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dpcd.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dpcd.c
index c5de6ed5bf58..a72c898b64fa 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dpcd.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dpcd.c
@@ -130,7 +130,7 @@ static uint32_t dpcd_get_next_partition_size(const uint32_t address, const uint3
  * XXX: Do not allow any two address ranges in this array to overlap
  */
 static const struct dpcd_address_range mandatory_dpcd_blocks[] = {
-	{ DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV, DP_PHY_REPEATER_EXTENDED_WAIT_TIMEOUT }};
+	{ DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV, DP_PHY_REPEATER_128B132B_RATES }};
 
 /*
  * extend addresses to read all mandatory blocks together
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
index 3baa2bdd6dd6..ad9aca790dd7 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
@@ -38,6 +38,7 @@
 #include "dc/dc_dmub_srv.h"
 #include "dce/dmub_replay.h"
 #include "abm.h"
+#include "resource.h"
 #define DC_LOGGER \
 	link->ctx->logger
 #define DC_LOGGER_INIT(logger)
@@ -320,8 +321,8 @@ bool edp_is_ilr_optimization_required(struct dc_link *link,
 		struct dc_crtc_timing *crtc_timing)
 {
 	struct dc_link_settings link_setting;
-	uint8_t link_bw_set;
-	uint8_t link_rate_set;
+	uint8_t link_bw_set = 0;
+	uint8_t link_rate_set = 0;
 	uint32_t req_bw;
 	union lane_count_set lane_count_set = {0};
 
@@ -1055,7 +1056,7 @@ bool edp_set_coasting_vtotal(struct dc_link *link, uint32_t coasting_vtotal)
 }
 
 bool edp_replay_residency(const struct dc_link *link,
-	unsigned int *residency, const bool is_start, const bool is_alpm)
+	unsigned int *residency, const bool is_start, const enum pr_residency_mode mode)
 {
 	struct dc  *dc = link->ctx->dc;
 	struct dmub_replay *replay = dc->res_pool->replay;
@@ -1064,8 +1065,11 @@ bool edp_replay_residency(const struct dc_link *link,
 	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
 		return false;
 
+	if (!residency)
+		return false;
+
 	if (replay != NULL && link->replay_settings.replay_feature_enabled)
-		replay->funcs->replay_residency(replay, panel_inst, residency, is_start, is_alpm);
+		replay->funcs->replay_residency(replay, panel_inst, residency, is_start, mode);
 	else
 		*residency = 0;
 
@@ -1145,3 +1149,66 @@ int edp_get_target_backlight_pwm(const struct dc_link *link)
 
 	return (int) abm->funcs->get_target_backlight(abm);
 }
+
+static void edp_set_assr_enable(const struct dc *pDC, struct dc_link *link,
+		struct link_resource *link_res, bool enable)
+{
+	union dmub_rb_cmd cmd;
+	bool use_hpo_dp_link_enc = false;
+	uint8_t link_enc_index = 0;
+	uint8_t phy_type = 0;
+	uint8_t phy_id = 0;
+
+	if (!pDC->config.use_assr_psp_message)
+		return;
+
+	memset(&cmd, 0, sizeof(cmd));
+
+	link_enc_index = link->link_enc->transmitter - TRANSMITTER_UNIPHY_A;
+
+	if (link_res->hpo_dp_link_enc) {
+		link_enc_index = link_res->hpo_dp_link_enc->inst;
+		use_hpo_dp_link_enc = true;
+	}
+
+	if (enable)
+		phy_type = ((dp_get_panel_mode(link) == DP_PANEL_MODE_EDP) ? 1 : 0);
+
+	phy_id = resource_transmitter_to_phy_idx(pDC, link->link_enc->transmitter);
+
+	cmd.assr_enable.header.type = DMUB_CMD__PSP;
+	cmd.assr_enable.header.sub_type = DMUB_CMD__PSP_ASSR_ENABLE;
+	cmd.assr_enable.assr_data.enable = enable;
+	cmd.assr_enable.assr_data.phy_port_type = phy_type;
+	cmd.assr_enable.assr_data.phy_port_id = phy_id;
+	cmd.assr_enable.assr_data.link_enc_index = link_enc_index;
+	cmd.assr_enable.assr_data.hpo_mode = use_hpo_dp_link_enc;
+
+	dc_wake_and_execute_dmub_cmd(pDC->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+}
+
+void edp_set_panel_assr(struct dc_link *link, struct pipe_ctx *pipe_ctx,
+		enum dp_panel_mode *panel_mode, bool enable)
+{
+	struct link_resource *link_res = &pipe_ctx->link_res;
+	struct cp_psp *cp_psp = &pipe_ctx->stream->ctx->cp_psp;
+
+	if (*panel_mode != DP_PANEL_MODE_EDP)
+		return;
+
+	if (link->dc->config.use_assr_psp_message) {
+		edp_set_assr_enable(link->dc, link, link_res, enable);
+	} else if (cp_psp && cp_psp->funcs.enable_assr && enable) {
+		/* ASSR is bound to fail with unsigned PSP
+		 * verstage used during devlopment phase.
+		 * Report and continue with eDP panel mode to
+		 * perform eDP link training with right settings
+		 */
+		bool result;
+
+		result = cp_psp->funcs.enable_assr(cp_psp->handle, link);
+
+		if (!result && link->panel_mode != DP_PANEL_MODE_EDP)
+			*panel_mode = DP_PANEL_MODE_DEFAULT;
+	}
+}
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h
index a158c6234d42..cb6d95cc36e4 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h
@@ -61,7 +61,7 @@ bool edp_send_replay_cmd(struct dc_link *link,
 			union dmub_replay_cmd_set *cmd_data);
 bool edp_set_coasting_vtotal(struct dc_link *link, uint32_t coasting_vtotal);
 bool edp_replay_residency(const struct dc_link *link,
-	unsigned int *residency, const bool is_start, const bool is_alpm);
+	unsigned int *residency, const bool is_start, const enum pr_residency_mode mode);
 bool edp_get_replay_state(const struct dc_link *link, uint64_t *state);
 bool edp_set_replay_power_opt_and_coasting_vtotal(struct dc_link *link,
 	const unsigned int *power_opts, uint32_t coasting_vtotal);
@@ -76,4 +76,6 @@ bool edp_receiver_ready_T9(struct dc_link *link);
 bool edp_receiver_ready_T7(struct dc_link *link);
 bool edp_power_alpm_dpcd_enable(struct dc_link *link, bool enable);
 void edp_set_panel_power(struct dc_link *link, bool powerOn);
+void edp_set_panel_assr(struct dc_link *link, struct pipe_ctx *pipe_ctx,
+		enum dp_panel_mode *panel_mode, bool enable);
 #endif /* __DC_LINK_EDP_POWER_CONTROL_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.c
index e3d729ab5b9f..caa617883f62 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.c
@@ -35,7 +35,7 @@
 
 bool link_get_hpd_state(struct dc_link *link)
 {
-	uint32_t state;
+	uint32_t state = 0;
 
 	dal_gpio_lock_pin(link->hpd_gpio);
 	dal_gpio_get_value(link->hpd_gpio, &state);
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c
index 0e8f4f36c87c..5574bc628053 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c
@@ -945,10 +945,19 @@ void optc1_set_drr(
 				OTG_FORCE_LOCK_ON_EVENT, 0,
 				OTG_SET_V_TOTAL_MIN_MASK_EN, 0,
 				OTG_SET_V_TOTAL_MIN_MASK, 0);
-	}
 
-	// Setup manual flow control for EOF via TRIG_A
-	optc->funcs->setup_manual_trigger(optc);
+		// Setup manual flow control for EOF via TRIG_A
+		optc->funcs->setup_manual_trigger(optc);
+
+	} else {
+		REG_UPDATE_4(OTG_V_TOTAL_CONTROL,
+				OTG_SET_V_TOTAL_MIN_MASK, 0,
+				OTG_V_TOTAL_MIN_SEL, 0,
+				OTG_V_TOTAL_MAX_SEL, 0,
+				OTG_FORCE_LOCK_ON_EVENT, 0);
+
+		optc->funcs->set_vtotal_min_max(optc, 0, 0);
+	}
 }
 
 void optc1_set_vtotal_min_max(struct timing_generator *optc, int vtotal_min, int vtotal_max)
@@ -1383,6 +1392,9 @@ void optc1_read_otg_state(struct optc *optc1,
 
 	REG_GET(OTG_VERTICAL_INTERRUPT2_POSITION,
 			OTG_VERTICAL_INTERRUPT2_LINE_START, &s->vertical_interrupt2_line);
+
+	s->otg_master_update_lock = REG_READ(OTG_MASTER_UPDATE_LOCK);
+	s->otg_double_buffer_control = REG_READ(OTG_DOUBLE_BUFFER_CONTROL);
 }
 
 bool optc1_get_otg_active_size(struct timing_generator *optc,
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.h b/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.h
index 6c2e84d3967f..2f3bd7648ba7 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.h
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.h
@@ -129,6 +129,8 @@ struct dcn_optc_registers {
 	uint32_t OTG_V_TOTAL_MID;
 	uint32_t OTG_V_TOTAL_MIN;
 	uint32_t OTG_V_TOTAL_CONTROL;
+	uint32_t OTG_V_COUNT_STOP_CONTROL;
+	uint32_t OTG_V_COUNT_STOP_CONTROL2;
 	uint32_t OTG_TRIGA_CNTL;
 	uint32_t OTG_TRIGA_MANUAL_TRIG;
 	uint32_t OTG_MANUAL_FLOW_CONTROL;
@@ -515,12 +517,15 @@ struct dcn_optc_registers {
 	type MANUAL_FLOW_CONTROL;\
 	type MANUAL_FLOW_CONTROL_SEL;
 
+#define V_TOTAL_REGS(type)
+
 #define TG_REG_FIELD_LIST(type) \
 	TG_REG_FIELD_LIST_DCN1_0(type)\
 	type OTG_V_SYNC_MODE;\
 	type OTG_DRR_TRIGGER_WINDOW_START_X;\
 	type OTG_DRR_TRIGGER_WINDOW_END_X;\
 	type OTG_DRR_V_TOTAL_CHANGE_LIMIT;\
+	V_TOTAL_REGS(type)\
 	type OTG_OUT_MUX;\
 	type OTG_M_CONST_DTO_PHASE;\
 	type OTG_M_CONST_DTO_MODULO;\
@@ -581,7 +586,9 @@ struct dcn_optc_registers {
 	type OTG_CRC1_WINDOWB_X_END_READBACK;\
 	type OTG_CRC1_WINDOWB_Y_START_READBACK;\
 	type OTG_CRC1_WINDOWB_Y_END_READBACK;\
-	type OPTC_FGCG_REP_DIS;
+	type OPTC_FGCG_REP_DIS;\
+	type OTG_V_COUNT_STOP;\
+	type OTG_V_COUNT_STOP_TIMER;
 
 struct dcn_optc_shift {
 	TG_REG_FIELD_LIST(uint8_t)
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c
index 58bdbd859bf9..d6f095b4555d 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c
@@ -462,16 +462,6 @@ void optc2_setup_manual_trigger(struct timing_generator *optc)
 {
 	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
-	/* Set the min/max selectors unconditionally so that
-	 * DMCUB fw may change OTG timings when necessary
-	 * TODO: Remove the w/a after fixing the issue in DMCUB firmware
-	 */
-	REG_UPDATE_4(OTG_V_TOTAL_CONTROL,
-				 OTG_V_TOTAL_MIN_SEL, 1,
-				 OTG_V_TOTAL_MAX_SEL, 1,
-				 OTG_FORCE_LOCK_ON_EVENT, 0,
-				 OTG_SET_V_TOTAL_MIN_MASK, (1 << 1)); /* TRIGA */
-
 	REG_SET_8(OTG_TRIGA_CNTL, 0,
 			OTG_TRIGA_SOURCE_SELECT, 21,
 			OTG_TRIGA_SOURCE_PIPE_SELECT, optc->inst,
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c
index 5b1547508850..d393be30dff8 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c
@@ -32,6 +32,7 @@
 #include "reg_helper.h"
 #include "dc.h"
 #include "dcn_calc_math.h"
+#include "dc_dmub_srv.h"
 
 #define REG(reg)\
 	optc1->tg_regs->reg
@@ -213,6 +214,167 @@ static bool optc35_configure_crc(struct timing_generator *optc,
 	return true;
 }
 
+static void optc35_setup_manual_trigger(struct timing_generator *optc)
+{
+	if (!optc || !optc->ctx)
+		return;
+
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+	struct dc *dc = optc->ctx->dc;
+
+	if (dc->caps.dmub_caps.mclk_sw && !dc->debug.disable_fams)
+		dc_dmub_srv_set_drr_manual_trigger_cmd(dc, optc->inst);
+	else {
+		/*
+		 * MIN_MASK_EN is gone and MASK is now always enabled.
+		 *
+		 * To get it to it work with manual trigger we need to make sure
+		 * we program the correct bit.
+		 */
+		REG_UPDATE_4(OTG_V_TOTAL_CONTROL,
+				OTG_V_TOTAL_MIN_SEL, 1,
+				OTG_V_TOTAL_MAX_SEL, 1,
+				OTG_FORCE_LOCK_ON_EVENT, 0,
+				OTG_SET_V_TOTAL_MIN_MASK, (1 << 1)); /* TRIGA */
+
+		// Setup manual flow control for EOF via TRIG_A
+		if (optc->funcs && optc->funcs->setup_manual_trigger)
+			optc->funcs->setup_manual_trigger(optc);
+	}
+}
+
+void optc35_set_drr(
+	struct timing_generator *optc,
+	const struct drr_params *params)
+{
+	if (!optc || !params)
+		return;
+
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+	uint32_t max_otg_v_total = optc1->max_v_total - 1;
+
+	if (params != NULL &&
+		params->vertical_total_max > 0 &&
+		params->vertical_total_min > 0) {
+
+		if (params->vertical_total_mid != 0) {
+
+			REG_SET(OTG_V_TOTAL_MID, 0,
+				OTG_V_TOTAL_MID, params->vertical_total_mid - 1);
+
+			REG_UPDATE_2(OTG_V_TOTAL_CONTROL,
+					OTG_VTOTAL_MID_REPLACING_MAX_EN, 1,
+					OTG_VTOTAL_MID_FRAME_NUM,
+					(uint8_t)params->vertical_total_mid_frame_num);
+
+		}
+
+		if (optc->funcs && optc->funcs->set_vtotal_min_max)
+			optc->funcs->set_vtotal_min_max(optc,
+				params->vertical_total_min - 1, params->vertical_total_max - 1);
+		optc35_setup_manual_trigger(optc);
+	} else {
+		REG_UPDATE_4(OTG_V_TOTAL_CONTROL,
+				OTG_SET_V_TOTAL_MIN_MASK, 0,
+				OTG_V_TOTAL_MIN_SEL, 0,
+				OTG_V_TOTAL_MAX_SEL, 0,
+				OTG_FORCE_LOCK_ON_EVENT, 0);
+
+		if (optc->funcs && optc->funcs->set_vtotal_min_max)
+			optc->funcs->set_vtotal_min_max(optc, 0, 0);
+	}
+
+	REG_WRITE(OTG_V_COUNT_STOP_CONTROL, max_otg_v_total);
+	REG_WRITE(OTG_V_COUNT_STOP_CONTROL2, 0);
+}
+
+static void optc35_set_long_vtotal(
+	struct timing_generator *optc,
+	const struct long_vtotal_params *params)
+{
+	if (!optc || !params)
+		return;
+
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+	uint32_t vcount_stop_timer = 0, vcount_stop = 0;
+	uint32_t max_otg_v_total = optc1->max_v_total - 1;
+
+	if (params->vertical_total_min <= max_otg_v_total && params->vertical_total_max <= max_otg_v_total)
+		return;
+
+	if (params->vertical_total_max == 0 || params->vertical_total_min == 0) {
+		REG_UPDATE_4(OTG_V_TOTAL_CONTROL,
+						OTG_SET_V_TOTAL_MIN_MASK, 0,
+						OTG_V_TOTAL_MIN_SEL, 0,
+						OTG_V_TOTAL_MAX_SEL, 0,
+						OTG_FORCE_LOCK_ON_EVENT, 0);
+
+		if (optc->funcs && optc->funcs->set_vtotal_min_max)
+			optc->funcs->set_vtotal_min_max(optc, 0, 0);
+	} else if (params->vertical_total_max == params->vertical_total_min) {
+		vcount_stop = params->vertical_blank_start;
+		vcount_stop_timer = params->vertical_total_max - max_otg_v_total;
+
+		REG_UPDATE_4(OTG_V_TOTAL_CONTROL,
+				OTG_V_TOTAL_MIN_SEL, 1,
+				OTG_V_TOTAL_MAX_SEL, 1,
+				OTG_FORCE_LOCK_ON_EVENT, 0,
+				OTG_SET_V_TOTAL_MIN_MASK, 0);
+
+		if (optc->funcs && optc->funcs->set_vtotal_min_max)
+			optc->funcs->set_vtotal_min_max(optc, max_otg_v_total, max_otg_v_total);
+
+		REG_WRITE(OTG_V_COUNT_STOP_CONTROL, vcount_stop);
+		REG_WRITE(OTG_V_COUNT_STOP_CONTROL2, vcount_stop_timer);
+	} else {
+		// Variable rate, keep DRR trigger mask
+		if (params->vertical_total_min > max_otg_v_total) {
+			// cannot be supported
+			// If MAX_OTG_V_COUNT < DRR trigger < v_total_min < v_total_max,
+			// DRR trigger will drop the vtotal counting directly to a new frame.
+			// But it should trigger between v_total_min and v_total_max.
+			ASSERT(0);
+
+			REG_UPDATE_4(OTG_V_TOTAL_CONTROL,
+				OTG_SET_V_TOTAL_MIN_MASK, 0,
+				OTG_V_TOTAL_MIN_SEL, 0,
+				OTG_V_TOTAL_MAX_SEL, 0,
+				OTG_FORCE_LOCK_ON_EVENT, 0);
+
+			if (optc->funcs && optc->funcs->set_vtotal_min_max)
+				optc->funcs->set_vtotal_min_max(optc, 0, 0);
+
+			REG_WRITE(OTG_V_COUNT_STOP_CONTROL, max_otg_v_total);
+			REG_WRITE(OTG_V_COUNT_STOP_CONTROL2, 0);
+		} else {
+			// For total_min <= MAX_OTG_V_COUNT and total_max > MAX_OTG_V_COUNT
+			vcount_stop = params->vertical_total_min;
+			vcount_stop_timer = params->vertical_total_max - max_otg_v_total;
+
+			// Example:
+			// params->vertical_total_min 1000
+			// params->vertical_total_max 2000
+			// MAX_OTG_V_COUNT_STOP = 1500
+			//
+			// If DRR event not happened,
+			//     time     0,1,2,3,4,...1000,1001,........,1500,1501,1502,     ...1999
+			//     vcount   0,1,2,3,4....1000...................,1001,1002,1003,...1399
+			//     vcount2                       0,1,2,3,4,..499,
+			// else (DRR event happened, ex : at line 1004)
+			//     time    0,1,2,3,4,...1000,1001.....1004, 0
+			//     vcount  0,1,2,3,4....1000,.............. 0 (new frame)
+			//     vcount2                      0,1,2,   3, -
+			if (optc->funcs && optc->funcs->set_vtotal_min_max)
+				optc->funcs->set_vtotal_min_max(optc,
+					params->vertical_total_min - 1, max_otg_v_total);
+			optc35_setup_manual_trigger(optc);
+
+			REG_WRITE(OTG_V_COUNT_STOP_CONTROL, vcount_stop);
+			REG_WRITE(OTG_V_COUNT_STOP_CONTROL2, vcount_stop_timer);
+		}
+	}
+}
+
 static struct timing_generator_funcs dcn35_tg_funcs = {
 		.validate_timing = optc1_validate_timing,
 		.program_timing = optc1_program_timing,
@@ -245,7 +407,7 @@ static struct timing_generator_funcs dcn35_tg_funcs = {
 		.lock_doublebuffer_enable = optc3_lock_doublebuffer_enable,
 		.lock_doublebuffer_disable = optc3_lock_doublebuffer_disable,
 		.enable_optc_clock = optc1_enable_optc_clock,
-		.set_drr = optc31_set_drr,
+		.set_drr = optc35_set_drr,
 		.get_last_used_drr_vtotal = optc2_get_last_used_drr_vtotal,
 		.set_vtotal_min_max = optc1_set_vtotal_min_max,
 		.set_static_screen_control = optc1_set_static_screen_control,
@@ -275,6 +437,7 @@ static struct timing_generator_funcs dcn35_tg_funcs = {
 		.setup_manual_trigger = optc2_setup_manual_trigger,
 		.get_hw_timing = optc1_get_hw_timing,
 		.init_odm = optc3_init_odm,
+		.set_long_vtotal = optc35_set_long_vtotal,
 };
 
 void dcn35_timing_generator_init(struct optc *optc1)
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.h b/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.h
index 1f422e4c468f..d077e2392379 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.h
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.h
@@ -65,10 +65,14 @@
 	SF(OTG0_OTG_CRC1_WINDOWB_X_CONTROL_READBACK, OTG_CRC1_WINDOWB_X_END_READBACK, mask_sh),\
 	SF(OTG0_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK, OTG_CRC1_WINDOWB_Y_START_READBACK, mask_sh),\
 	SF(OTG0_OTG_CRC1_WINDOWB_Y_CONTROL_READBACK, OTG_CRC1_WINDOWB_Y_END_READBACK, mask_sh),\
-	SF(OPTC_CLOCK_CONTROL, OPTC_FGCG_REP_DIS, mask_sh)
+	SF(OPTC_CLOCK_CONTROL, OPTC_FGCG_REP_DIS, mask_sh),\
+	SF(OTG0_OTG_V_COUNT_STOP_CONTROL, OTG_V_COUNT_STOP, mask_sh),\
+	SF(OTG0_OTG_V_COUNT_STOP_CONTROL2, OTG_V_COUNT_STOP_TIMER, mask_sh)
 
 void dcn35_timing_generator_init(struct optc *optc1);
 
 void dcn35_timing_generator_set_fgcg(struct optc *optc1, bool enable);
 
+void optc35_set_drr(struct timing_generator *optc, const struct drr_params *params);
+
 #endif /* __DC_OPTC_DCN35_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/Makefile b/drivers/gpu/drm/amd/display/dc/resource/Makefile
index 184b1f23aa77..db9048974d74 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/resource/Makefile
@@ -102,10 +102,6 @@ AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN21)
 
 ###############################################################################
 
-###############################################################################
-
-###############################################################################
-
 RESOURCE_DCN30 = dcn30_resource.o
 
 AMD_DAL_RESOURCE_DCN30 = $(addprefix $(AMDDALPATH)/dc/resource/dcn30/,$(RESOURCE_DCN30))
@@ -202,6 +198,4 @@ AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN351)
 
 ###############################################################################
 
-###############################################################################
-
 endif
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c
index d1edac46c9a0..88afb2a30eef 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c
@@ -864,8 +864,6 @@ static struct clock_source *find_matching_pll(
 	default:
 		return NULL;
 	}
-
-	return NULL;
 }
 
 static enum dc_status build_mapped_resource(
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c
index 20662edd0ae4..621825a51f46 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c
@@ -1060,7 +1060,7 @@ static bool dce120_resource_construct(
 	struct irq_service_init_data irq_init_data;
 	static const struct resource_create_funcs *res_funcs;
 	bool is_vg20 = ASICREV_IS_VEGA20_P(ctx->asic_id.hw_internal_rev);
-	uint32_t pipe_fuses;
+	uint32_t pipe_fuses = 0;
 
 	ctx->dc_bios->regs = &bios_regs;
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c
index 35a2cce0c2b8..56ee45e12b46 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c
@@ -56,7 +56,6 @@
 #include "dce/dce_aux.h"
 #include "dce/dce_abm.h"
 #include "dce/dce_i2c.h"
-/* TODO remove this include */
 
 #ifndef mmMC_HUB_RDREQ_DMIF_LIMIT
 #include "gmc/gmc_7_1_d.h"
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c
index d08d10969251..563c5eec83ff 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c
@@ -513,7 +513,7 @@ static const struct dc_plane_cap plane_cap = {
 			.argb8888 = true,
 			.nv12 = true,
 			.fp16 = true,
-			.p010 = true
+			.p010 = false
 	},
 
 	.max_upscale_factor = {
@@ -569,6 +569,7 @@ static const struct dc_debug_options debug_defaults_diags = {
 		.disable_pplib_clock_request = true,
 		.disable_pplib_wm_range = true,
 		.underflow_assert_delay_us = 0xFFFFFFFF,
+		.enable_legacy_fast_update = true,
 };
 
 static void dcn10_dpp_destroy(struct dpp **dpp)
@@ -1631,6 +1632,7 @@ static bool dcn10_resource_construct(
 	/* valid pipe num */
 	pool->base.pipe_count = j;
 	pool->base.timing_generator_count = j;
+	pool->base.mpcc_count = j;
 
 	/* within dml lib, it is hard code to 4. If ASIC pipe is fused,
 	 * the value may be changed
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
index a2387cea1af9..0a939437e19f 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
@@ -62,6 +62,9 @@
 #include "dcn20/dcn20_vmid.h"
 #include "dce/dce_panel_cntl.h"
 
+#include "dcn20/dcn20_dwb.h"
+#include "dcn20/dcn20_mmhubbub.h"
+
 #include "navi10_ip_offset.h"
 
 #include "dcn/dcn_2_0_0_offset.h"
@@ -71,9 +74,6 @@
 
 #include "nbio/nbio_2_3_offset.h"
 
-#include "dcn20/dcn20_dwb.h"
-#include "dcn20/dcn20_mmhubbub.h"
-
 #include "mmhub/mmhub_2_0_0_offset.h"
 #include "mmhub/mmhub_2_0_0_sh_mask.h"
 
@@ -83,11 +83,10 @@
 #include "dce/dce_aux.h"
 #include "dce/dce_i2c.h"
 #include "vm_helper.h"
-#include "link_enc_cfg.h"
-
-#include "amdgpu_socbb.h"
 
+#include "link_enc_cfg.h"
 #include "link.h"
+
 #define DC_LOGGER_INIT(logger)
 
 #ifndef mmDP0_DP_DPHY_INTERNAL_CTRL
@@ -1282,8 +1281,13 @@ void dcn20_build_pipe_pix_clk_params(struct pipe_ctx *pipe_ctx)
 
 static enum dc_status build_pipe_hw_param(struct pipe_ctx *pipe_ctx)
 {
+	struct resource_pool *pool = pipe_ctx->stream->ctx->dc->res_pool;
 
-	dcn20_build_pipe_pix_clk_params(pipe_ctx);
+	if (pool->funcs->build_pipe_pix_clk_params) {
+		pool->funcs->build_pipe_pix_clk_params(pipe_ctx);
+	} else {
+		dcn20_build_pipe_pix_clk_params(pipe_ctx);
+	}
 
 	pipe_ctx->stream->clamping.pixel_encoding = pipe_ctx->stream->timing.pixel_encoding;
 
@@ -2449,6 +2453,7 @@ static bool dcn20_resource_construct(
 	dc->caps.post_blend_color_processing = true;
 	dc->caps.force_dp_tps4_for_cp2520 = true;
 	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
 
 	/* Color pipeline capabilities */
 	dc->caps.color.dpp.dcn_arch = 1;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c
index 914b234d7f6b..070a4efb308b 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c
@@ -55,7 +55,6 @@
 #include "dce110/dce110_resource.h"
 #include "dce/dce_aux.h"
 #include "dce/dce_i2c.h"
-#include "dcn201/dcn201_hubbub.h"
 #include "dcn10/dcn10_resource.h"
 
 #include "cyan_skillfish_ip_offset.h"
@@ -182,6 +181,7 @@ static struct _vcs_dpi_soc_bounding_box_st dcn201_soc = {
 				.socclk_mhz = 1254.0,
 				.dram_speed_mts = 14000.0,
 			},
+			/* state4 is not an actual state, just defines unsupported for dml*/
 			{
 				.state = 4,
 				.dscclk_mhz = 400.0,
@@ -566,6 +566,8 @@ static const struct resource_caps res_cap_dnc201 = {
 		.num_audio = 2,
 		.num_stream_encoder = 2,
 		.num_pll = 2,
+		.num_dwb = 0,
+		.num_dsc = 0,
 		.num_ddc = 2,
 };
 
@@ -612,7 +614,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 		.scl_reset_length10 = true,
 		.sanity_checks = false,
 		.underflow_assert_delay_us = 0xFFFFFFFF,
-		.enable_tri_buf = false,
+		.enable_tri_buf = true,
 		.enable_legacy_fast_update = true,
 		.using_dml2 = false,
 };
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c
index 65d337731f56..8663cbc3d1cf 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c
@@ -581,32 +581,6 @@ static const struct resource_caps res_cap_rn = {
 		.num_dsc = 3,
 };
 
-#ifdef DIAGS_BUILD
-static const struct resource_caps res_cap_rn_FPGA_4pipe = {
-		.num_timing_generator = 4,
-		.num_opp = 4,
-		.num_video_plane = 4,
-		.num_audio = 7,
-		.num_stream_encoder = 4,
-		.num_pll = 4,
-		.num_dwb = 1,
-		.num_ddc = 4,
-		.num_dsc = 0,
-};
-
-static const struct resource_caps res_cap_rn_FPGA_2pipe_dsc = {
-		.num_timing_generator = 2,
-		.num_opp = 2,
-		.num_video_plane = 2,
-		.num_audio = 7,
-		.num_stream_encoder = 2,
-		.num_pll = 4,
-		.num_dwb = 1,
-		.num_ddc = 4,
-		.num_dsc = 2,
-};
-#endif
-
 static const struct dc_plane_cap plane_cap = {
 	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
 	.per_pixel_alpha = true,
@@ -1415,16 +1389,11 @@ static bool dcn21_resource_construct(
 	struct dc_context *ctx = dc->ctx;
 	struct irq_service_init_data init_data;
 	uint32_t pipe_fuses = read_pipe_fuses(ctx);
-	uint32_t num_pipes;
+	uint32_t num_pipes = 0;
 
 	ctx->dc_bios->regs = &bios_regs;
 
 	pool->base.res_cap = &res_cap_rn;
-#ifdef DIAGS_BUILD
-	if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment))
-		//pool->base.res_cap = &res_cap_nv10_FPGA_2pipe_dsc;
-		pool->base.res_cap = &res_cap_rn_FPGA_4pipe;
-#endif
 
 	pool->base.funcs = &dcn21_res_pool_funcs;
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
index ecc477ef8e3b..f35cc307830b 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
@@ -1639,7 +1639,7 @@ noinline bool dcn30_internal_validate_bw(
 	int split[MAX_PIPES] = { 0 };
 	bool merge[MAX_PIPES] = { false };
 	bool newly_split[MAX_PIPES] = { false };
-	int pipe_cnt, i, pipe_idx, vlevel;
+	int pipe_cnt, i, pipe_idx, vlevel = 0;
 	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
 
 	ASSERT(pipes);
@@ -2050,6 +2050,9 @@ bool dcn30_validate_bandwidth(struct dc *dc,
 
 	BW_VAL_TRACE_COUNT();
 
+	if (!pipes)
+		goto validate_fail;
+
 	DC_FP_START();
 	out = dcn30_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate, true);
 	DC_FP_END();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
index 25cd6236b054..8bc1bcaeaa47 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
@@ -1143,7 +1143,7 @@ static bool dcn303_resource_construct(
 	int i;
 	struct dc_context *ctx = dc->ctx;
 	struct irq_service_init_data init_data;
-	struct ddc_service_init_data ddc_init_data;
+	struct ddc_service_init_data ddc_init_data = {0};
 
 	ctx->dc_bios->regs = &bios_regs;
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
index 04d142f97474..d4c3e2754f51 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
@@ -75,7 +75,6 @@
 #include "dcn30/dcn30_dwb.h"
 #include "dcn30/dcn30_mmhubbub.h"
 
-// TODO: change include headers /amd/include/asic_reg after upstream
 #include "yellow_carp_offset.h"
 #include "dcn/dcn_3_1_2_offset.h"
 #include "dcn/dcn_3_1_2_sh_mask.h"
@@ -892,7 +891,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.disable_z10 = true,
 	.enable_legacy_fast_update = true,
 	.enable_z9_disable_interface = true, /* Allow support for the PMFW interface for disable Z9*/
-	.dml_hostvm_override = DML_HOSTVM_OVERRIDE_FALSE,
+	.dml_hostvm_override = DML_HOSTVM_NO_OVERRIDE,
 	.using_dml2 = false,
 };
 
@@ -1311,6 +1310,8 @@ static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
 
 	/* allocate HPO link encoder */
 	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+	if (!hpo_dp_enc31)
+		return NULL; /* out of memory */
 
 	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
 					&hpo_dp_link_enc_regs[inst],
@@ -1645,7 +1646,7 @@ int dcn31_populate_dml_pipes_from_context(
 {
 	int i, pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
-	struct pipe_ctx *pipe;
+	struct pipe_ctx *pipe = 0;
 	bool upscaled = false;
 
 	DC_FP_START();
@@ -1767,11 +1768,14 @@ bool dcn31_validate_bandwidth(struct dc *dc,
 
 	BW_VAL_TRACE_COUNT();
 
+	if (!pipes)
+		goto validate_fail;
+
 	DC_FP_START();
 	out = dcn30_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate, true);
 	DC_FP_END();
 
-	// Disable fast_validate to set min dcfclk in alculate_wm_and_dlg
+	// Disable fast_validate to set min dcfclk in calculate_wm_and_dlg
 	if (pipe_cnt == 0)
 		fast_validate = false;
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
index c97391edb5ff..ff50f43e4c00 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
@@ -925,27 +925,10 @@ static const struct dc_debug_options debug_defaults_drv = {
 	},
 
 	.seamless_boot_odm_combine = true,
+	.enable_legacy_fast_update = true,
 	.using_dml2 = false,
 };
 
-static const struct dc_debug_options debug_defaults_diags = {
-	.disable_dmcu = true,
-	.force_abm_enable = false,
-	.timing_trace = true,
-	.clock_trace = true,
-	.disable_dpp_power_gate = true,
-	.disable_hubp_power_gate = true,
-	.disable_clock_gate = true,
-	.disable_pplib_clock_request = true,
-	.disable_pplib_wm_range = true,
-	.disable_stutter = false,
-	.scl_reset_length10 = true,
-	.dwb_fi_phase = -1, // -1 = disable
-	.dmub_command_table = true,
-	.enable_tri_buf = true,
-	.use_max_lb = true
-};
-
 static const struct dc_panel_config panel_config_defaults = {
 	.psr = {
 		.disable_psr = false,
@@ -1384,6 +1367,8 @@ static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
 
 	/* allocate HPO link encoder */
 	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+	if (!hpo_dp_enc31)
+		return NULL; /* out of memory */
 
 	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
 					&hpo_dp_link_enc_regs[inst],
@@ -1744,6 +1729,9 @@ bool dcn314_validate_bandwidth(struct dc *dc,
 
 	BW_VAL_TRACE_COUNT();
 
+	if (!pipes)
+		goto validate_fail;
+
 	if (filter_modes_for_single_channel_workaround(dc, context))
 		goto validate_fail;
 
@@ -1938,8 +1926,6 @@ static bool dcn314_resource_construct(
 
 	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
 		dc->debug = debug_defaults_drv;
-	else
-		dc->debug = debug_defaults_diags;
 
 	/* Disable pipe power gating */
 	dc->debug.disable_dpp_power_gate = true;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
index 515ba435f759..4ce0f4bf1d9b 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
@@ -1309,6 +1309,8 @@ static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
 
 	/* allocate HPO link encoder */
 	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+	if (!hpo_dp_enc31)
+		return NULL; /* out of memory */
 
 	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
 					&hpo_dp_link_enc_regs[inst],
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
index b9753d4606f8..5fd52c5fcee4 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
@@ -125,7 +125,6 @@
 #include "link_enc_cfg.h"
 
 #define DCN3_16_MAX_DET_SIZE 384
-#define DCN3_16_MIN_COMPBUF_SIZE_KB 128
 #define DCN3_16_CRB_SEGMENT_SIZE_KB 64
 
 enum dcn31_clk_src_array_id {
@@ -1306,6 +1305,8 @@ static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
 
 	/* allocate HPO link encoder */
 	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+	if (!hpo_dp_enc31)
+		return NULL; /* out of memory */
 
 	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
 					&hpo_dp_link_enc_regs[inst],
@@ -1614,7 +1615,7 @@ static int dcn316_populate_dml_pipes_from_context(
 {
 	int i, pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
-	struct pipe_ctx *pipe;
+	struct pipe_ctx *pipe = 0;
 	const int max_usable_det = context->bw_ctx.dml.ip.config_return_buffer_size_in_kbytes - DCN3_16_MIN_COMPBUF_SIZE_KB;
 
 	DC_FP_START();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
index ce1754cc1f46..abd76345d1e4 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
@@ -1304,6 +1304,8 @@ static struct hpo_dp_link_encoder *dcn32_hpo_dp_link_encoder_create(
 
 	/* allocate HPO link encoder */
 	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+	if (!hpo_dp_enc31)
+		return NULL; /* out of memory */
 
 #undef REG_STRUCT
 #define REG_STRUCT hpo_dp_link_enc_regs
@@ -1751,6 +1753,9 @@ static bool dml1_validate(struct dc *dc, struct dc_state *context, bool fast_val
 
 	BW_VAL_TRACE_COUNT();
 
+	if (!pipes)
+		goto validate_fail;
+
 	DC_FP_START();
 	out = dcn32_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate);
 	DC_FP_END();
@@ -1799,7 +1804,9 @@ bool dcn32_validate_bandwidth(struct dc *dc,
 	bool out = false;
 
 	if (dc->debug.using_dml2)
-		out = dml2_validate(dc, context, fast_validate);
+		out = dml2_validate(dc, context,
+				context->power_source == DC_POWER_SOURCE_DC ? context->bw_ctx.dml2_dc_power_source : context->bw_ctx.dml2,
+				fast_validate);
 	else
 		out = dml1_validate(dc, context, fast_validate);
 	return out;
@@ -1815,9 +1822,48 @@ int dcn32_populate_dml_pipes_from_context(
 	struct pipe_ctx *pipe = NULL;
 	bool subvp_in_use = false;
 	struct dc_crtc_timing *timing;
+	int subvp_main_pipe_index = -1;
+	enum mall_stream_type mall_type;
+	bool single_display_subvp = false;
+	struct dc_stream_state *stream = NULL;
+	int num_subvp_main = 0;
+	int num_subvp_phantom = 0;
+	int num_subvp_none = 0;
+	int odm_slice_count;
 
 	dcn20_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
 
+	/* For single display subvp, look for subvp main so if we have phantom
+	 *  pipe, we can set odm policy to match main pipe
+	 */
+	for (i = 0; i < context->stream_count; i++) {
+		stream = context->streams[i];
+		mall_type = dc_state_get_stream_subvp_type(context, stream);
+		if (mall_type == SUBVP_MAIN)
+			num_subvp_main++;
+		else if (mall_type == SUBVP_PHANTOM)
+			num_subvp_phantom++;
+		else
+			num_subvp_none++;
+	}
+	if (num_subvp_main == 1 && num_subvp_phantom == 1 && num_subvp_none == 0)
+		single_display_subvp = true;
+
+	if (single_display_subvp) {
+		for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
+			pipe = &res_ctx->pipe_ctx[i];
+			if (!res_ctx->pipe_ctx[i].stream)
+				continue;
+
+			mall_type = dc_state_get_pipe_subvp_type(context, pipe);
+			if (mall_type == SUBVP_MAIN) {
+				if (resource_is_pipe_type(pipe, OTG_MASTER))
+					subvp_main_pipe_index = i;
+			}
+			pipe_cnt++;
+		}
+	}
+
 	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
 
 		if (!res_ctx->pipe_ctx[i].stream)
@@ -1832,7 +1878,21 @@ int dcn32_populate_dml_pipes_from_context(
 		pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch;
 		if (dc->config.enable_windowed_mpo_odm &&
 				dc->debug.enable_single_display_2to1_odm_policy) {
-			switch (resource_get_odm_slice_count(pipe)) {
+			/* For single display subvp, if pipe is phantom pipe,
+			 *  then copy odm policy from subvp main pipe
+			 */
+			mall_type = dc_state_get_pipe_subvp_type(context, pipe);
+			if (single_display_subvp && (mall_type == SUBVP_PHANTOM)) {
+				if (subvp_main_pipe_index < 0) {
+					odm_slice_count = -1;
+					ASSERT(0);
+				} else {
+					odm_slice_count = resource_get_odm_slice_count(&res_ctx->pipe_ctx[subvp_main_pipe_index]);
+				}
+			} else {
+				odm_slice_count = resource_get_odm_slice_count(pipe);
+			}
+			switch (odm_slice_count) {
 			case 2:
 				pipes[pipe_cnt].pipe.dest.odm_combine_policy = dm_odm_combine_policy_2to1;
 				break;
@@ -1845,6 +1905,7 @@ int dcn32_populate_dml_pipes_from_context(
 		} else {
 			pipes[pipe_cnt].pipe.dest.odm_combine_policy = dm_odm_combine_policy_dal;
 		}
+
 		pipes[pipe_cnt].pipe.src.gpuvm_min_page_size_kbytes = 256; // according to spreadsheet
 		pipes[pipe_cnt].pipe.src.unbounded_req_mode = false;
 		pipes[pipe_cnt].pipe.scale_ratio_depth.lb_depth = dm_lb_19;
@@ -1912,6 +1973,22 @@ int dcn32_populate_dml_pipes_from_context(
 	return pipe_cnt;
 }
 
+unsigned int dcn32_calculate_mall_ways_from_bytes(const struct dc *dc, unsigned int total_size_in_mall_bytes)
+{
+	uint32_t cache_lines_used, lines_per_way, total_cache_lines, num_ways;
+
+	/* add 2 lines for worst case alignment */
+	cache_lines_used = total_size_in_mall_bytes / dc->caps.cache_line_size + 2;
+
+	total_cache_lines = dc->caps.max_cab_allocation_bytes / dc->caps.cache_line_size;
+	lines_per_way = total_cache_lines / dc->caps.cache_num_ways;
+	num_ways = cache_lines_used / lines_per_way;
+	if (cache_lines_used % lines_per_way > 0)
+		num_ways++;
+
+	return num_ways;
+}
+
 static struct dc_cap_funcs cap_funcs = {
 	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap,
 	.get_subvp_en = dcn32_subvp_in_use,
@@ -1929,10 +2006,20 @@ void dcn32_calculate_wm_and_dlg(struct dc *dc, struct dc_state *context,
 
 static void dcn32_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
 {
+	struct dml2_configuration_options dml2_opt = dc->dml2_options;
+
 	DC_FP_START();
+
 	dcn32_update_bw_bounding_box_fpu(dc, bw_params);
+
+	dml2_opt.use_clock_dc_limits = false;
 	if (dc->debug.using_dml2 && dc->current_state && dc->current_state->bw_ctx.dml2)
-		dml2_reinit(dc, &dc->dml2_options, &dc->current_state->bw_ctx.dml2);
+		dml2_reinit(dc, &dml2_opt, &dc->current_state->bw_ctx.dml2);
+
+	dml2_opt.use_clock_dc_limits = true;
+	if (dc->debug.using_dml2 && dc->current_state && dc->current_state->bw_ctx.dml2_dc_power_source)
+		dml2_reinit(dc, &dml2_opt, &dc->current_state->bw_ctx.dml2_dc_power_source);
+
 	DC_FP_END();
 }
 
@@ -1960,6 +2047,7 @@ static struct resource_funcs dcn32_res_pool_funcs = {
 	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
 	.add_phantom_pipes = dcn32_add_phantom_pipes,
 	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params,
+	.calculate_mall_ways_from_bytes = dcn32_calculate_mall_ways_from_bytes,
 };
 
 static uint32_t read_pipe_fuses(struct dc_context *ctx)
@@ -2048,7 +2136,8 @@ static bool dcn32_resource_construct(
 	dc->caps.min_horizontal_blanking_period = 80;
 	dc->caps.dmdata_alloc_size = 2048;
 	dc->caps.mall_size_per_mem_channel = 4;
-	dc->caps.mall_size_total = 0;
+	/* total size = mall per channel * num channels * 1024 * 1024 */
+	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel * dc->ctx->dc_bios->vram_info.num_chans * 1048576;
 	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
 
 	dc->caps.cache_line_size = 64;
@@ -2362,30 +2451,10 @@ static bool dcn32_resource_construct(
 	dc->dml2_options.use_native_soc_bb_construction = true;
 	dc->dml2_options.minimize_dispclk_using_odm = true;
 
-	dc->dml2_options.callbacks.dc = dc;
-	dc->dml2_options.callbacks.build_scaling_params = &resource_build_scaling_params;
+	resource_init_common_dml2_callbacks(dc, &dc->dml2_options);
 	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch = &dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
-	dc->dml2_options.callbacks.acquire_secondary_pipe_for_mpc_odm = &dc_resource_acquire_secondary_pipe_for_mpc_odm_legacy;
-	dc->dml2_options.callbacks.update_pipes_for_stream_with_slice_count = &resource_update_pipes_for_stream_with_slice_count;
-	dc->dml2_options.callbacks.update_pipes_for_plane_with_slice_count = &resource_update_pipes_for_plane_with_slice_count;
-	dc->dml2_options.callbacks.get_mpc_slice_index = &resource_get_mpc_slice_index;
-	dc->dml2_options.callbacks.get_odm_slice_index = &resource_get_odm_slice_index;
-	dc->dml2_options.callbacks.get_opp_head = &resource_get_opp_head;
-
-	dc->dml2_options.svp_pstate.callbacks.dc = dc;
-	dc->dml2_options.svp_pstate.callbacks.add_phantom_plane = &dc_state_add_phantom_plane;
-	dc->dml2_options.svp_pstate.callbacks.add_phantom_stream = &dc_state_add_phantom_stream;
-	dc->dml2_options.svp_pstate.callbacks.build_scaling_params = &resource_build_scaling_params;
-	dc->dml2_options.svp_pstate.callbacks.create_phantom_plane = &dc_state_create_phantom_plane;
-	dc->dml2_options.svp_pstate.callbacks.remove_phantom_plane = &dc_state_remove_phantom_plane;
-	dc->dml2_options.svp_pstate.callbacks.remove_phantom_stream = &dc_state_remove_phantom_stream;
-	dc->dml2_options.svp_pstate.callbacks.create_phantom_stream = &dc_state_create_phantom_stream;
-	dc->dml2_options.svp_pstate.callbacks.release_phantom_plane = &dc_state_release_phantom_plane;
-	dc->dml2_options.svp_pstate.callbacks.release_phantom_stream = &dc_state_release_phantom_stream;
 	dc->dml2_options.svp_pstate.callbacks.release_dsc = &dcn20_release_dsc;
-	dc->dml2_options.svp_pstate.callbacks.get_pipe_subvp_type = &dc_state_get_pipe_subvp_type;
-	dc->dml2_options.svp_pstate.callbacks.get_stream_subvp_type = &dc_state_get_stream_subvp_type;
-	dc->dml2_options.svp_pstate.callbacks.get_paired_subvp_stream = &dc_state_get_paired_subvp_stream;
+	dc->dml2_options.svp_pstate.callbacks.calculate_mall_ways_from_bytes = pool->base.funcs->calculate_mall_ways_from_bytes;
 
 	dc->dml2_options.svp_pstate.subvp_fw_processing_delay_us = dc->caps.subvp_fw_processing_delay_us;
 	dc->dml2_options.svp_pstate.subvp_prefetch_end_to_mall_start_us = dc->caps.subvp_prefetch_end_to_mall_start_us;
@@ -2483,7 +2552,7 @@ struct resource_pool *dcn32_create_resource_pool(
  * full update which delays the flip for 1 frame. If we use the original pipe
  * we don't have to toggle its power. So we can flip faster.
  */
-static int find_optimal_free_pipe_as_secondary_dpp_pipe(
+int dcn32_find_optimal_free_pipe_as_secondary_dpp_pipe(
 		const struct resource_context *cur_res_ctx,
 		struct resource_context *new_res_ctx,
 		const struct resource_pool *pool,
@@ -2666,7 +2735,7 @@ struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_dpp_pipe(
 		return dcn32_acquire_idle_pipe_for_head_pipe_in_layer(
 				new_ctx, pool, opp_head_pipe->stream, opp_head_pipe);
 
-	free_pipe_idx = find_optimal_free_pipe_as_secondary_dpp_pipe(
+	free_pipe_idx = dcn32_find_optimal_free_pipe_as_secondary_dpp_pipe(
 					&cur_ctx->res_ctx, &new_ctx->res_ctx,
 					pool, opp_head_pipe);
 	if (free_pipe_idx >= 0) {
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.h
index 2258c5c7212d..fee67fbab8e2 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.h
@@ -113,10 +113,6 @@ void dcn32_calculate_wm_and_dlg(
 		int pipe_cnt,
 		int vlevel);
 
-uint32_t dcn32_helper_mall_bytes_to_ways(
-		struct dc *dc,
-		uint32_t total_size_in_mall_bytes);
-
 uint32_t dcn32_helper_calculate_mall_bytes_for_cursor(
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
@@ -141,6 +137,12 @@ bool dcn32_any_surfaces_rotated(struct dc *dc, struct dc_state *context);
 bool dcn32_is_center_timing(struct pipe_ctx *pipe);
 bool dcn32_is_psr_capable(struct pipe_ctx *pipe);
 
+int dcn32_find_optimal_free_pipe_as_secondary_dpp_pipe(
+		const struct resource_context *cur_res_ctx,
+		struct resource_context *new_res_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *new_opp_head);
+
 struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_dpp_pipe(
 		const struct dc_state *cur_ctx,
 		struct dc_state *new_ctx,
@@ -184,6 +186,8 @@ void dcn32_update_dml_pipes_odm_policy_based_on_context(struct dc *dc, struct dc
 
 void dcn32_override_min_req_dcfclk(struct dc *dc, struct dc_state *context);
 
+unsigned int dcn32_calculate_mall_ways_from_bytes(const struct dc *dc, unsigned int total_size_in_mall_bytes);
+
 /* definitions for run time init of reg offsets */
 
 /* CLK SRC */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
index 296a0a8e7145..e4b360d89b3b 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
@@ -1288,6 +1288,8 @@ static struct hpo_dp_link_encoder *dcn321_hpo_dp_link_encoder_create(
 
 	/* allocate HPO link encoder */
 	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+	if (!hpo_dp_enc31)
+		return NULL; /* out of memory */
 
 #undef REG_STRUCT
 #define REG_STRUCT hpo_dp_link_enc_regs
@@ -1579,10 +1581,20 @@ static struct dc_cap_funcs cap_funcs = {
 
 static void dcn321_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
 {
+	struct dml2_configuration_options dml2_opt = dc->dml2_options;
+
 	DC_FP_START();
+
 	dcn321_update_bw_bounding_box_fpu(dc, bw_params);
+
+	dml2_opt.use_clock_dc_limits = false;
 	if (dc->debug.using_dml2 && dc->current_state && dc->current_state->bw_ctx.dml2)
-		dml2_reinit(dc, &dc->dml2_options, &dc->current_state->bw_ctx.dml2);
+		dml2_reinit(dc, &dml2_opt, &dc->current_state->bw_ctx.dml2);
+
+	dml2_opt.use_clock_dc_limits = true;
+	if (dc->debug.using_dml2 && dc->current_state && dc->current_state->bw_ctx.dml2_dc_power_source)
+		dml2_reinit(dc, &dml2_opt, &dc->current_state->bw_ctx.dml2_dc_power_source);
+
 	DC_FP_END();
 }
 
@@ -1610,6 +1622,7 @@ static struct resource_funcs dcn321_res_pool_funcs = {
 	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
 	.add_phantom_pipes = dcn32_add_phantom_pipes,
 	.build_pipe_pix_clk_params = dcn20_build_pipe_pix_clk_params,
+	.calculate_mall_ways_from_bytes = dcn32_calculate_mall_ways_from_bytes,
 };
 
 static uint32_t read_pipe_fuses(struct dc_context *ctx)
@@ -1697,7 +1710,9 @@ static bool dcn321_resource_construct(
 	dc->caps.min_horizontal_blanking_period = 80;
 	dc->caps.dmdata_alloc_size = 2048;
 	dc->caps.mall_size_per_mem_channel = 4;
-	dc->caps.mall_size_total = 0;
+	/* total size = mall per channel * num channels * 1024 * 1024 */
+	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel * dc->ctx->dc_bios->vram_info.num_chans * 1048576;
+
 	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
 	dc->caps.cache_line_size = 64;
 	dc->caps.cache_num_ways = 16;
@@ -1998,30 +2013,10 @@ static bool dcn321_resource_construct(
 	dc->dml2_options.use_native_soc_bb_construction = true;
 	dc->dml2_options.minimize_dispclk_using_odm = true;
 
-	dc->dml2_options.callbacks.dc = dc;
-	dc->dml2_options.callbacks.build_scaling_params = &resource_build_scaling_params;
+	resource_init_common_dml2_callbacks(dc, &dc->dml2_options);
 	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch = &dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
-	dc->dml2_options.callbacks.acquire_secondary_pipe_for_mpc_odm = &dc_resource_acquire_secondary_pipe_for_mpc_odm_legacy;
-	dc->dml2_options.callbacks.update_pipes_for_stream_with_slice_count = &resource_update_pipes_for_stream_with_slice_count;
-	dc->dml2_options.callbacks.update_pipes_for_plane_with_slice_count = &resource_update_pipes_for_plane_with_slice_count;
-	dc->dml2_options.callbacks.get_mpc_slice_index = &resource_get_mpc_slice_index;
-	dc->dml2_options.callbacks.get_odm_slice_index = &resource_get_odm_slice_index;
-	dc->dml2_options.callbacks.get_opp_head = &resource_get_opp_head;
-
-	dc->dml2_options.svp_pstate.callbacks.dc = dc;
-	dc->dml2_options.svp_pstate.callbacks.add_phantom_plane = &dc_state_add_phantom_plane;
-	dc->dml2_options.svp_pstate.callbacks.add_phantom_stream = &dc_state_add_phantom_stream;
-	dc->dml2_options.svp_pstate.callbacks.build_scaling_params = &resource_build_scaling_params;
-	dc->dml2_options.svp_pstate.callbacks.create_phantom_plane = &dc_state_create_phantom_plane;
-	dc->dml2_options.svp_pstate.callbacks.remove_phantom_plane = &dc_state_remove_phantom_plane;
-	dc->dml2_options.svp_pstate.callbacks.remove_phantom_stream = &dc_state_remove_phantom_stream;
-	dc->dml2_options.svp_pstate.callbacks.create_phantom_stream = &dc_state_create_phantom_stream;
-	dc->dml2_options.svp_pstate.callbacks.release_phantom_plane = &dc_state_release_phantom_plane;
-	dc->dml2_options.svp_pstate.callbacks.release_phantom_stream = &dc_state_release_phantom_stream;
 	dc->dml2_options.svp_pstate.callbacks.release_dsc = &dcn20_release_dsc;
-	dc->dml2_options.svp_pstate.callbacks.get_pipe_subvp_type = &dc_state_get_pipe_subvp_type;
-	dc->dml2_options.svp_pstate.callbacks.get_stream_subvp_type = &dc_state_get_stream_subvp_type;
-	dc->dml2_options.svp_pstate.callbacks.get_paired_subvp_stream = &dc_state_get_paired_subvp_stream;
+	dc->dml2_options.svp_pstate.callbacks.calculate_mall_ways_from_bytes = pool->base.funcs->calculate_mall_ways_from_bytes;
 
 	dc->dml2_options.svp_pstate.subvp_fw_processing_delay_us = dc->caps.subvp_fw_processing_delay_us;
 	dc->dml2_options.svp_pstate.subvp_prefetch_end_to_mall_start_us = dc->caps.subvp_prefetch_end_to_mall_start_us;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
index 5d52853cac96..2df8a742516c 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
@@ -721,7 +721,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.disable_dpp_power_gate = true,
 	.disable_hubp_power_gate = true,
 	.disable_optc_power_gate = true, /*should the same as above two*/
-	.disable_hpo_power_gate = true, /*dmubfw force domain25 on*/
+	.disable_hpo_power_gate = false, /*dmubfw force domain25 on*/
 	.disable_clock_gate = false,
 	.disable_dsc_power_gate = true,
 	.vsr_support = true,
@@ -764,12 +764,12 @@ static const struct dc_debug_options debug_defaults_drv = {
 	},
 	.seamless_boot_odm_combine = DML_FAIL_SOURCE_PIXEL_FORMAT,
 	.enable_z9_disable_interface = true, /* Allow support for the PMFW interface for disable Z9*/
-	.minimum_z8_residency_time = 2100,
+	.minimum_z8_residency_time = 1, /* Always allow when other conditions are met */
 	.using_dml2 = true,
 	.support_eDP1_5 = true,
 	.enable_hpo_pg_support = false,
 	.enable_legacy_fast_update = true,
-	.enable_single_display_2to1_odm_policy = false,
+	.enable_single_display_2to1_odm_policy = true,
 	.disable_idle_power_optimizations = false,
 	.dmcub_emulation = false,
 	.disable_boot_optimizations = false,
@@ -783,7 +783,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.psp_disabled_wa = true,
 	.ips2_eval_delay_us = 2000,
 	.ips2_entry_delay_us = 800,
-	.disable_dmub_reallow_idle = true,
+	.disable_dmub_reallow_idle = false,
 	.static_screen_wait_frames = 2,
 };
 
@@ -1368,6 +1368,8 @@ static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
 
 	/* allocate HPO link encoder */
 	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+	if (!hpo_dp_enc31)
+		return NULL; /* out of memory */
 
 #undef REG_STRUCT
 #define REG_STRUCT hpo_dp_link_enc_regs
@@ -1734,7 +1736,9 @@ static bool dcn35_validate_bandwidth(struct dc *dc,
 {
 	bool out = false;
 
-	out = dml2_validate(dc, context, fast_validate);
+	out = dml2_validate(dc, context,
+			context->power_source == DC_POWER_SOURCE_DC ? context->bw_ctx.dml2_dc_power_source : context->bw_ctx.dml2,
+			fast_validate);
 
 	if (fast_validate)
 		return out;
@@ -2138,15 +2142,9 @@ static bool dcn35_resource_construct(
 		dc->dml2_options.minimize_dispclk_using_odm = true;
 	dc->dml2_options.enable_windowed_mpo_odm = dc->config.enable_windowed_mpo_odm;
 
-	dc->dml2_options.callbacks.dc = dc;
-	dc->dml2_options.callbacks.build_scaling_params = &resource_build_scaling_params;
+	resource_init_common_dml2_callbacks(dc, &dc->dml2_options);
 	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch = &dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
-	dc->dml2_options.callbacks.acquire_secondary_pipe_for_mpc_odm = &dc_resource_acquire_secondary_pipe_for_mpc_odm_legacy;
-	dc->dml2_options.callbacks.update_pipes_for_stream_with_slice_count = &resource_update_pipes_for_stream_with_slice_count;
-	dc->dml2_options.callbacks.update_pipes_for_plane_with_slice_count = &resource_update_pipes_for_plane_with_slice_count;
-	dc->dml2_options.callbacks.get_mpc_slice_index = &resource_get_mpc_slice_index;
-	dc->dml2_options.callbacks.get_odm_slice_index = &resource_get_odm_slice_index;
-	dc->dml2_options.callbacks.get_opp_head = &resource_get_opp_head;
+
 	dc->dml2_options.max_segments_per_hubp = 24;
 
 	dc->dml2_options.det_segment_size = DCN3_2_DET_SEG_SIZE;/*todo*/
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.h
index a51c4a9eaafe..f97bb4cb3761 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.h
@@ -240,6 +240,8 @@ struct resource_pool *dcn35_create_resource_pool(
 	SRI_ARR(OTG_V_TOTAL_MAX, OTG, inst),\
 	SRI_ARR(OTG_V_TOTAL_MIN, OTG, inst),\
 	SRI_ARR(OTG_V_TOTAL_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_V_COUNT_STOP_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_V_COUNT_STOP_CONTROL2, OTG, inst),\
 	SRI_ARR(OTG_TRIGA_CNTL, OTG, inst),\
 	SRI_ARR(OTG_FORCE_COUNT_NOW_CNTL, OTG, inst),\
 	SRI_ARR(OTG_STATIC_SCREEN_CONTROL, OTG, inst),\
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
index 909e14261f9b..ddf9560ab772 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
@@ -758,7 +758,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 	//must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions
 	.enable_double_buffered_dsc_pg_support = true,
 	.enable_dp_dig_pixel_rate_div_policy = 1,
-	.disable_z10 = true,
+	.disable_z10 = false,
 	.ignore_pg = true,
 	.psp_disabled_wa = true,
 	.ips2_eval_delay_us = 2000,
@@ -1348,6 +1348,8 @@ static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
 
 	/* allocate HPO link encoder */
 	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+	if (!hpo_dp_enc31)
+		return NULL; /* out of memory */
 
 #undef REG_STRUCT
 #define REG_STRUCT hpo_dp_link_enc_regs
@@ -1714,19 +1716,20 @@ static bool dcn351_validate_bandwidth(struct dc *dc,
 {
 	bool out = false;
 
-	out = dml2_validate(dc, context, fast_validate);
+	out = dml2_validate(dc, context,
+			context->power_source == DC_POWER_SOURCE_DC ? context->bw_ctx.dml2_dc_power_source : context->bw_ctx.dml2,
+			fast_validate);
 
 	if (fast_validate)
 		return out;
 
 	DC_FP_START();
-	dcn351_decide_zstate_support(dc, context);
+	dcn35_decide_zstate_support(dc, context);
 	DC_FP_END();
 
 	return out;
 }
 
-
 static struct resource_funcs dcn351_res_pool_funcs = {
 	.destroy = dcn351_destroy_resource_pool,
 	.link_enc_create = dcn35_link_encoder_create,
@@ -1869,6 +1872,9 @@ static bool dcn351_resource_construct(
 	/* Use pipe context based otg sync logic */
 	dc->config.use_pipe_ctx_sync_logic = true;
 
+	/* Use psp mailbox to enable assr */
+	dc->config.use_assr_psp_message = true;
+
 	/* read VBIOS LTTPR caps */
 	{
 		if (ctx->dc_bios->funcs->get_lttpr_caps) {
@@ -1888,6 +1894,8 @@ static bool dcn351_resource_construct(
 	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
 		dc->debug = debug_defaults_drv;
 
+	/*HW default is to have all the FGCG enabled, SW no need to program them*/
+	dc->debug.enable_fine_grain_clock_gating.u32All = 0xFFFF;
 	// Init the vm_helper
 	if (dc->vm_helper)
 		vm_helper_init(dc->vm_helper, 16);
@@ -2118,15 +2126,9 @@ static bool dcn351_resource_construct(
 		dc->dml2_options.minimize_dispclk_using_odm = true;
 	dc->dml2_options.enable_windowed_mpo_odm = dc->config.enable_windowed_mpo_odm;
 
-	dc->dml2_options.callbacks.dc = dc;
-	dc->dml2_options.callbacks.build_scaling_params = &resource_build_scaling_params;
+	resource_init_common_dml2_callbacks(dc, &dc->dml2_options);
 	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch = &dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
-	dc->dml2_options.callbacks.acquire_secondary_pipe_for_mpc_odm = &dc_resource_acquire_secondary_pipe_for_mpc_odm_legacy;
-	dc->dml2_options.callbacks.update_pipes_for_stream_with_slice_count = &resource_update_pipes_for_stream_with_slice_count;
-	dc->dml2_options.callbacks.update_pipes_for_plane_with_slice_count = &resource_update_pipes_for_plane_with_slice_count;
-	dc->dml2_options.callbacks.get_mpc_slice_index = &resource_get_mpc_slice_index;
-	dc->dml2_options.callbacks.get_odm_slice_index = &resource_get_odm_slice_index;
-	dc->dml2_options.callbacks.get_opp_head = &resource_get_opp_head;
+
 	dc->dml2_options.max_segments_per_hubp = 24;
 	dc->dml2_options.det_segment_size = DCN3_2_DET_SEG_SIZE;/*todo*/
 
diff --git a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
index 7785908a6676..2fde1f043d50 100644
--- a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
@@ -71,6 +71,8 @@
 extern "C" {
 #endif
 
+#define DMUB_PC_SNAPSHOT_COUNT 10
+
 /* Forward declarations */
 struct dmub_srv;
 struct dmub_srv_common_regs;
@@ -295,18 +297,30 @@ struct dmub_srv_hw_params {
 	bool dpia_hpd_int_enable_supported;
 	bool disable_clock_gate;
 	bool disallow_dispclk_dppclk_ds;
+	bool ips_sequential_ono;
 	enum dmub_memory_access_type mem_access_type;
 	enum dmub_ips_disable_type disable_ips;
 };
 
 /**
+ * struct dmub_srv_debug - Debug info for dmub_srv
+ * @timeout_occured: Indicates a timeout occured on any message from driver to dmub
+ * @timeout_cmd: first cmd sent from driver that timed out - subsequent timeouts are not stored
+ */
+struct dmub_srv_debug {
+	bool timeout_occured;
+	union dmub_rb_cmd timeout_cmd;
+	unsigned long long timestamp;
+};
+
+/**
  * struct dmub_diagnostic_data - Diagnostic data retrieved from DMCUB for
  * debugging purposes, including logging, crash analysis, etc.
  */
 struct dmub_diagnostic_data {
 	uint32_t dmcub_version;
 	uint32_t scratch[17];
-	uint32_t pc;
+	uint32_t pc[DMUB_PC_SNAPSHOT_COUNT];
 	uint32_t undefined_address_fault_addr;
 	uint32_t inst_fetch_fault_addr;
 	uint32_t data_write_fault_addr;
@@ -317,6 +331,7 @@ struct dmub_diagnostic_data {
 	uint32_t inbox0_wptr;
 	uint32_t inbox0_size;
 	uint32_t gpint_datain0;
+	struct dmub_srv_debug timeout_info;
 	uint8_t is_dmcub_enabled : 1;
 	uint8_t is_dmcub_soft_reset : 1;
 	uint8_t is_dmcub_secure_reset : 1;
@@ -506,6 +521,7 @@ struct dmub_srv {
 	struct dmub_visual_confirm_color visual_confirm_color;
 
 	enum dmub_srv_power_state_type power_state;
+	struct dmub_srv_debug debug;
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
index af3fe8bb0728..e85fd3ac52c7 100644
--- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
+++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
@@ -97,6 +97,9 @@
 /* Maximum number of planes on any ASIC. */
 #define DMUB_MAX_PLANES 6
 
+/* Maximum number of phantom planes on any ASIC */
+#define DMUB_MAX_PHANTOM_PLANES ((DMUB_MAX_PLANES) / 2)
+
 /* Trace buffer offset for entry */
 #define TRACE_BUFFER_ENTRY_OFFSET  16
 
@@ -194,6 +197,11 @@ union abm_flags {
 		 * of user backlight level.
 		 */
 		unsigned int abm_gradual_bl_change : 1;
+
+		/**
+		 * @abm_new_frame: Indicates if a new frame update needed for ABM to ramp up into steady
+		 */
+		unsigned int abm_new_frame : 1;
 	} bitfields;
 
 	unsigned int u32All;
@@ -461,7 +469,7 @@ struct dmub_feature_caps {
 	 * Max PSR version supported by FW.
 	 */
 	uint8_t psr;
-	uint8_t fw_assisted_mclk_switch;
+	uint8_t fw_assisted_mclk_switch_ver;
 	uint8_t reserved[4];
 	uint8_t subvp_psr_support;
 	uint8_t gecc_enable;
@@ -619,6 +627,7 @@ enum dmub_ips_disable_type {
 	DMUB_IPS_DISABLE_IPS2 = 3,
 	DMUB_IPS_DISABLE_IPS2_Z10 = 4,
 	DMUB_IPS_DISABLE_DYNAMIC = 5,
+	DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF = 6,
 };
 
 #define DMUB_IPS1_ALLOW_MASK 0x00000001
@@ -653,6 +662,7 @@ union dmub_fw_boot_options {
 		uint32_t disable_timeout_recovery : 1; /* 1 if timeout recovery should be disabled */
 		uint32_t ips_pg_disable: 1; /* 1 to disable ONO domains power gating*/
 		uint32_t ips_disable: 3; /* options to disable ips support*/
+		uint32_t ips_sequential_ono: 1; /**< 1 to enable sequential ONO IPS sequence */
 		uint32_t reserved : 9; /**< reserved */
 	} bits; /**< boot bits */
 	uint32_t all; /**< 32-bit access to bits */
@@ -695,7 +705,8 @@ union dmub_shared_state_ips_fw_signals {
 	struct {
 		uint32_t ips1_commit : 1;  /**< 1 if in IPS1 */
 		uint32_t ips2_commit : 1; /**< 1 if in IPS2 */
-		uint32_t reserved_bits : 30; /**< Reversed */
+		uint32_t in_idle : 1; /**< 1 if DMCUB is in idle */
+		uint32_t reserved_bits : 29; /**< Reversed */
 	} bits;
 	uint32_t all;
 };
@@ -724,7 +735,13 @@ union dmub_shared_state_ips_driver_signals {
  */
 struct dmub_shared_state_ips_fw {
 	union dmub_shared_state_ips_fw_signals signals; /**< 4 bytes, IPS signal bits */
-	uint32_t reserved[61]; /**< Reversed, to be updated when adding new fields. */
+	uint32_t rcg_entry_count; /**< Entry counter for RCG */
+	uint32_t rcg_exit_count; /**< Exit counter for RCG */
+	uint32_t ips1_entry_count; /**< Entry counter for IPS1 */
+	uint32_t ips1_exit_count; /**< Exit counter for IPS1 */
+	uint32_t ips2_entry_count; /**< Entry counter for IPS2 */
+	uint32_t ips2_exit_count; /**< Exit counter for IPS2 */
+	uint32_t reserved[55]; /**< Reversed, to be updated when adding new fields. */
 }; /* 248-bytes, fixed */
 
 /**
@@ -812,6 +829,10 @@ enum dmub_cmd_vbios_type {
 	 */
 	DMUB_CMD__VBIOS_TRANSMITTER_QUERY_DP_ALT  = 26,
 	/**
+	 * Control PHY FSM
+	 */
+	DMUB_CMD__VBIOS_TRANSMITTER_SET_PHY_FSM  = 29,
+	/**
 	 * Controls domain power gating
 	 */
 	DMUB_CMD__VBIOS_DOMAIN_CONTROL = 28,
@@ -1186,6 +1207,11 @@ enum dmub_cmd_type {
 	 */
 	DMUB_CMD__DPIA_HPD_INT_ENABLE = 86,
 
+	/**
+	 * Command type used for all PSP commands.
+	 */
+	DMUB_CMD__PSP = 88,
+
 	DMUB_CMD__VBIOS = 128,
 };
 
@@ -1588,7 +1614,7 @@ struct dmub_rb_cmd_idle_opt_dcn_restore {
  */
 struct dmub_dcn_notify_idle_cntl_data {
 	uint8_t driver_idle;
-	uint8_t pad[1];
+	uint8_t reserved[59];
 };
 
 /**
@@ -2309,6 +2335,11 @@ enum phy_link_rate {
 	 * UHBR10 - 20.0 Gbps/Lane
 	 */
 	PHY_RATE_2000 = 11,
+
+	PHY_RATE_675 = 12,
+	/**
+	 * Rate 12 - 6.75 Gbps/Lane
+	 */
 };
 
 /**
@@ -2327,6 +2358,7 @@ enum dmub_phy_fsm_state {
 	DMUB_PHY_FSM_POWER_DOWN,
 	DMUB_PHY_FSM_PLL_EN,
 	DMUB_PHY_FSM_TX_EN,
+	DMUB_PHY_FSM_TX_EN_TEST_MODE,
 	DMUB_PHY_FSM_FAST_LP,
 	DMUB_PHY_FSM_P2_PLL_OFF_CPM,
 	DMUB_PHY_FSM_P2_PLL_OFF_PG,
@@ -2931,18 +2963,49 @@ struct dmub_rb_cmd_psr_set_power_opt {
 	struct dmub_cmd_psr_set_power_opt_data psr_set_power_opt_data;
 };
 
+/**
+ * Definition of Replay Residency GPINT command.
+ * Bit[0] - Residency mode for Revision 0
+ * Bit[1] - Enable/Disable state
+ * Bit[2-3] - Revision number
+ * Bit[4-7] - Residency mode for Revision 1
+ * Bit[8] - Panel instance
+ * Bit[9-15] - Reserved
+ */
+
+enum pr_residency_mode {
+	PR_RESIDENCY_MODE_PHY = 0x0,
+	PR_RESIDENCY_MODE_ALPM,
+	PR_RESIDENCY_MODE_IPS2,
+	PR_RESIDENCY_MODE_FRAME_CNT,
+	PR_RESIDENCY_MODE_ENABLEMENT_PERIOD,
+};
+
 #define REPLAY_RESIDENCY_MODE_SHIFT            (0)
 #define REPLAY_RESIDENCY_ENABLE_SHIFT          (1)
+#define REPLAY_RESIDENCY_REVISION_SHIFT        (2)
+#define REPLAY_RESIDENCY_MODE2_SHIFT           (4)
 
 #define REPLAY_RESIDENCY_MODE_MASK             (0x1 << REPLAY_RESIDENCY_MODE_SHIFT)
-# define REPLAY_RESIDENCY_MODE_PHY             (0x0 << REPLAY_RESIDENCY_MODE_SHIFT)
-# define REPLAY_RESIDENCY_MODE_ALPM            (0x1 << REPLAY_RESIDENCY_MODE_SHIFT)
-# define REPLAY_RESIDENCY_MODE_IPS             0x10
+# define REPLAY_RESIDENCY_FIELD_MODE_PHY       (0x0 << REPLAY_RESIDENCY_MODE_SHIFT)
+# define REPLAY_RESIDENCY_FIELD_MODE_ALPM      (0x1 << REPLAY_RESIDENCY_MODE_SHIFT)
+
+#define REPLAY_RESIDENCY_MODE2_MASK            (0xF << REPLAY_RESIDENCY_MODE2_SHIFT)
+# define REPLAY_RESIDENCY_FIELD_MODE2_IPS      (0x1 << REPLAY_RESIDENCY_MODE2_SHIFT)
+# define REPLAY_RESIDENCY_FIELD_MODE2_FRAME_CNT    (0x2 << REPLAY_RESIDENCY_MODE2_SHIFT)
+# define REPLAY_RESIDENCY_FIELD_MODE2_EN_PERIOD	(0x3 << REPLAY_RESIDENCY_MODE2_SHIFT)
 
 #define REPLAY_RESIDENCY_ENABLE_MASK           (0x1 << REPLAY_RESIDENCY_ENABLE_SHIFT)
 # define REPLAY_RESIDENCY_DISABLE              (0x0 << REPLAY_RESIDENCY_ENABLE_SHIFT)
 # define REPLAY_RESIDENCY_ENABLE               (0x1 << REPLAY_RESIDENCY_ENABLE_SHIFT)
 
+#define REPLAY_RESIDENCY_REVISION_MASK         (0x3 << REPLAY_RESIDENCY_REVISION_SHIFT)
+# define REPLAY_RESIDENCY_REVISION_0           (0x0 << REPLAY_RESIDENCY_REVISION_SHIFT)
+# define REPLAY_RESIDENCY_REVISION_1           (0x1 << REPLAY_RESIDENCY_REVISION_SHIFT)
+
+/**
+ * Definition of a replay_state.
+ */
 enum replay_state {
 	REPLAY_STATE_0			= 0x0,
 	REPLAY_STATE_1			= 0x10,
@@ -3004,6 +3067,11 @@ enum dmub_cmd_replay_type {
 	 * Set pseudo vtotal
 	 */
 	DMUB_CMD__REPLAY_SET_PSEUDO_VTOTAL = 7,
+	/**
+	 * Set adaptive sync sdp enabled
+	 */
+	DMUB_CMD__REPLAY_DISABLED_ADAPTIVE_SYNC_SDP = 8,
+
 };
 
 /**
@@ -3205,6 +3273,20 @@ struct dmub_cmd_replay_set_pseudo_vtotal {
 	 */
 	uint8_t pad;
 };
+struct dmub_cmd_replay_disabled_adaptive_sync_sdp_data {
+	/**
+	 * Panel Instance.
+	 * Panel isntance to identify which replay_state to use
+	 * Currently the support is only for 0 or 1
+	 */
+	uint8_t panel_inst;
+	/**
+	 * enabled: set adaptive sync sdp enabled
+	 */
+	uint8_t force_disabled;
+
+	uint8_t pad[2];
+};
 
 /**
  * Definition of a DMUB_CMD__SET_REPLAY_POWER_OPT command.
@@ -3309,6 +3391,20 @@ struct dmub_rb_cmd_replay_set_pseudo_vtotal {
 };
 
 /**
+ * Definition of a DMUB_CMD__REPLAY_DISABLED_ADAPTIVE_SYNC_SDP command.
+ */
+struct dmub_rb_cmd_replay_disabled_adaptive_sync_sdp {
+	/**
+	 * Command header.
+	 */
+	struct dmub_cmd_header header;
+	/**
+	 * Definition of DMUB_CMD__REPLAY_DISABLED_ADAPTIVE_SYNC_SDP command.
+	 */
+	struct dmub_cmd_replay_disabled_adaptive_sync_sdp_data data;
+};
+
+/**
  * Data passed from driver to FW in  DMUB_CMD__REPLAY_SET_RESIDENCY_FRAMEUPDATE_TIMER command.
  */
 struct dmub_cmd_replay_frameupdate_timer_data {
@@ -3363,6 +3459,11 @@ union dmub_replay_cmd_set {
 	 * Definition of DMUB_CMD__REPLAY_SET_PSEUDO_VTOTAL command data.
 	 */
 	struct dmub_cmd_replay_set_pseudo_vtotal pseudo_vtotal_data;
+	/**
+	 * Definition of DMUB_CMD__REPLAY_DISABLED_ADAPTIVE_SYNC_SDP command data.
+	 */
+	struct dmub_cmd_replay_disabled_adaptive_sync_sdp_data disabled_adaptive_sync_sdp_data;
+
 };
 
 /**
@@ -3445,7 +3546,7 @@ enum hw_lock_client {
 	/**
 	 * Replay is the client of HW Lock Manager.
 	 */
-	HW_LOCK_CLIENT_REPLAY           = 4,
+	HW_LOCK_CLIENT_REPLAY		= 4,
 	/**
 	 * Invalid client.
 	 */
@@ -4038,6 +4139,10 @@ enum dmub_cmd_panel_cntl_type {
 	 * Queries backlight info for the embedded panel.
 	 */
 	DMUB_CMD__PANEL_CNTL_QUERY_BACKLIGHT_INFO = 1,
+	/**
+	 * Sets the PWM Freq as per user's requirement.
+	 */
+	DMUB_CMD__PANEL_DEBUG_PWM_FREQ = 2,
 };
 
 /**
@@ -4139,6 +4244,34 @@ struct dmub_rb_cmd_transmitter_query_dp_alt {
 	struct dmub_rb_cmd_transmitter_query_dp_alt_data data; /**< payload */
 };
 
+struct phy_test_mode {
+	uint8_t mode;
+	uint8_t pat0;
+	uint8_t pad[2];
+};
+
+/**
+ * Data passed in/out in a DMUB_CMD__VBIOS_TRANSMITTER_SET_PHY_FSM command.
+ */
+struct dmub_rb_cmd_transmitter_set_phy_fsm_data {
+	uint8_t phy_id; /**< 0=UNIPHYA, 1=UNIPHYB, 2=UNIPHYC, 3=UNIPHYD, 4=UNIPHYE, 5=UNIPHYF */
+	uint8_t mode; /**< HDMI/DP/DP2 etc */
+	uint8_t lane_num; /**< Number of lanes */
+	uint32_t symclk_100Hz; /**< PLL symclock in 100hz */
+	struct phy_test_mode test_mode;
+	enum dmub_phy_fsm_state state;
+	uint32_t status;
+	uint8_t pad;
+};
+
+/**
+ * Definition of a DMUB_CMD__VBIOS_TRANSMITTER_SET_PHY_FSM command.
+ */
+struct dmub_rb_cmd_transmitter_set_phy_fsm {
+	struct dmub_cmd_header header; /**< header */
+	struct dmub_rb_cmd_transmitter_set_phy_fsm_data data; /**< payload */
+};
+
 /**
  * Maximum number of bytes a chunk sent to DMUB for parsing
  */
@@ -4261,6 +4394,65 @@ struct dmub_rb_cmd_secure_display {
 };
 
 /**
+ * Command type of a DMUB_CMD__PSP command
+ */
+enum dmub_cmd_psp_type {
+	DMUB_CMD__PSP_ASSR_ENABLE = 0
+};
+
+/**
+ * Data passed from driver to FW in a DMUB_CMD__PSP_ASSR_ENABLE command.
+ */
+struct dmub_cmd_assr_enable_data {
+	/**
+	 * ASSR enable or disable.
+	 */
+	uint8_t enable;
+	/**
+	 * PHY port type.
+	 * Indicates eDP / non-eDP port type
+	 */
+	uint8_t phy_port_type;
+	/**
+	 * PHY port ID.
+	 */
+	uint8_t phy_port_id;
+	/**
+	 * Link encoder index.
+	 */
+	uint8_t link_enc_index;
+	/**
+	 * HPO mode.
+	 */
+	uint8_t hpo_mode;
+
+	/**
+	 * Reserved field.
+	 */
+	uint8_t reserved[7];
+};
+
+/**
+ * Definition of a DMUB_CMD__PSP_ASSR_ENABLE command.
+ */
+struct dmub_rb_cmd_assr_enable {
+	/**
+	 * Command header.
+	 */
+	struct dmub_cmd_header header;
+
+	/**
+	 * Assr data.
+	 */
+	struct dmub_cmd_assr_enable_data assr_data;
+
+	/**
+	 * Reserved field.
+	 */
+	uint32_t reserved[3];
+};
+
+/**
  * union dmub_rb_cmd - DMUB inbox command.
  */
 union dmub_rb_cmd {
@@ -4451,6 +4643,10 @@ union dmub_rb_cmd {
 	 */
 	struct dmub_rb_cmd_transmitter_query_dp_alt query_dp_alt;
 	/**
+	 * Definition of a DMUB_CMD__VBIOS_TRANSMITTER_SET_PHY_FSM command.
+	 */
+	struct dmub_rb_cmd_transmitter_set_phy_fsm set_phy_fsm;
+	/**
 	 * Definition of a DMUB_CMD__DPIA_DIG1_CONTROL command.
 	 */
 	struct dmub_rb_cmd_dig1_dpia_control dig1_dpia_control;
@@ -4518,6 +4714,15 @@ union dmub_rb_cmd {
 	 * Definition of a DMUB_CMD__REPLAY_SET_PSEUDO_VTOTAL command.
 	 */
 	struct dmub_rb_cmd_replay_set_pseudo_vtotal replay_set_pseudo_vtotal;
+	/**
+	 * Definition of a DMUB_CMD__REPLAY_DISABLED_ADAPTIVE_SYNC_SDP command.
+	 */
+	struct dmub_rb_cmd_replay_disabled_adaptive_sync_sdp replay_disabled_adaptive_sync_sdp;
+	/**
+	 * Definition of a DMUB_CMD__PSP_ASSR_ENABLE command.
+	 */
+	struct dmub_rb_cmd_assr_enable assr_enable;
+
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c
index cae96fba6349..e500ca9ae09c 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c
@@ -472,4 +472,5 @@ void dmub_dcn20_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_diagnosti
 
 	REG_GET(DMCUB_REGION3_CW6_TOP_ADDRESS, DMCUB_REGION3_CW6_ENABLE, &is_cw6_enabled);
 	diag_data->is_cw6_enabled = is_cw6_enabled;
+	diag_data->timeout_info = dmub->debug;
 }
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c
index 2bcf5fb87dd9..662c34e9495c 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c
@@ -466,6 +466,7 @@ void dmub_dcn31_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_diagnosti
 
 	REG_GET(DMCUB_REGION3_CW6_TOP_ADDRESS, DMCUB_REGION3_CW6_ENABLE, &is_cw6_enabled);
 	diag_data->is_cw6_enabled = is_cw6_enabled;
+	diag_data->timeout_info = dmub->debug;
 }
 
 bool dmub_dcn31_should_detect(struct dmub_srv *dmub)
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.c
index 0d521eeda050..e1da270502cc 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.c
@@ -478,6 +478,8 @@ void dmub_dcn32_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_diagnosti
 	diag_data->is_cw6_enabled = is_cw6_enabled;
 
 	diag_data->gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
+
+	diag_data->timeout_info = dmub->debug;
 }
 void dmub_dcn32_configure_dmub_in_system_memory(struct dmub_srv *dmub)
 {
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.c
index 53f359f3fae2..70e63aeb8f89 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.c
@@ -420,6 +420,7 @@ void dmub_dcn35_enable_dmub_boot_options(struct dmub_srv *dmub, const struct dmu
 	boot_options.bits.disable_clk_ds = params->disallow_dispclk_dppclk_ds;
 	boot_options.bits.disable_clk_gate = params->disable_clock_gate;
 	boot_options.bits.ips_disable = params->disable_ips;
+	boot_options.bits.ips_sequential_ono = params->ips_sequential_ono;
 
 	REG_WRITE(DMCUB_SCRATCH14, boot_options.all);
 }
@@ -516,6 +517,7 @@ void dmub_dcn35_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_diagnosti
 	diag_data->is_cw6_enabled = is_cw6_enabled;
 
 	diag_data->gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
+	diag_data->timeout_info = dmub->debug;
 }
 void dmub_dcn35_configure_dmub_in_system_memory(struct dmub_srv *dmub)
 {
diff --git a/drivers/gpu/drm/amd/display/include/dal_types.h b/drivers/gpu/drm/amd/display/include/dal_types.h
index 1c6f24cb1d2f..447768dec887 100644
--- a/drivers/gpu/drm/amd/display/include/dal_types.h
+++ b/drivers/gpu/drm/amd/display/include/dal_types.h
@@ -27,7 +27,6 @@
 #define __DAL_TYPES_H__
 
 #include "signal_types.h"
-#include "dc_types.h"
 
 struct dal_logger;
 struct dc_bios;
diff --git a/drivers/gpu/drm/amd/display/include/grph_object_id.h b/drivers/gpu/drm/amd/display/include/grph_object_id.h
index c6bbd262f1ac..08ee0350b31f 100644
--- a/drivers/gpu/drm/amd/display/include/grph_object_id.h
+++ b/drivers/gpu/drm/amd/display/include/grph_object_id.h
@@ -226,8 +226,8 @@ enum dp_alt_mode {
 
 struct graphics_object_id {
 	uint32_t  id:8;
-	uint32_t  enum_id:4;
-	uint32_t  type:4;
+	enum object_enum_id  enum_id;
+	enum object_type  type;
 	uint32_t  reserved:16; /* for padding. total size should be u32 */
 };
 
diff --git a/drivers/gpu/drm/amd/display/include/link_service_types.h b/drivers/gpu/drm/amd/display/include/link_service_types.h
index 92dbff22a7c6..1867aac57cf2 100644
--- a/drivers/gpu/drm/amd/display/include/link_service_types.h
+++ b/drivers/gpu/drm/amd/display/include/link_service_types.h
@@ -73,7 +73,6 @@ struct link_training_settings {
 	enum dc_pre_emphasis *pre_emphasis;
 	enum dc_post_cursor2 *post_cursor2;
 	bool should_set_fec_ready;
-	/* TODO - factor lane_settings out because it changes during LT */
 	union dc_dp_ffe_preset *ffe_preset;
 
 	uint16_t cr_pattern_time;
diff --git a/drivers/gpu/drm/amd/display/include/logger_types.h b/drivers/gpu/drm/amd/display/include/logger_types.h
index f39e2785e618..83479951732a 100644
--- a/drivers/gpu/drm/amd/display/include/logger_types.h
+++ b/drivers/gpu/drm/amd/display/include/logger_types.h
@@ -64,6 +64,7 @@
 #define DC_LOG_DWB(...) drm_dbg((DC_LOGGER)->dev, __VA_ARGS__)
 #define DC_LOG_DP2(...) drm_dbg_dp((DC_LOGGER)->dev, __VA_ARGS__)
 #define DC_LOG_AUTO_DPM_TEST(...) pr_debug("[AutoDPMTest]: "__VA_ARGS__)
+#define DC_LOG_IPS(...) pr_debug("[IPS]: "__VA_ARGS__)
 
 struct dc_log_buffer_ctx {
 	char *buf;
diff --git a/drivers/gpu/drm/amd/display/include/signal_types.h b/drivers/gpu/drm/amd/display/include/signal_types.h
index 1b14b17a79c7..a10d6b988aab 100644
--- a/drivers/gpu/drm/amd/display/include/signal_types.h
+++ b/drivers/gpu/drm/amd/display/include/signal_types.h
@@ -118,6 +118,19 @@ static inline bool dc_is_dvi_signal(enum signal_type signal)
 	}
 }
 
+static inline bool dc_is_tmds_signal(enum signal_type signal)
+{
+	switch (signal) {
+	case SIGNAL_TYPE_DVI_SINGLE_LINK:
+	case SIGNAL_TYPE_DVI_DUAL_LINK:
+	case SIGNAL_TYPE_HDMI_TYPE_A:
+		return true;
+	break;
+	default:
+		return false;
+	}
+}
+
 static inline bool dc_is_dvi_single_link_signal(enum signal_type signal)
 {
 	return (signal == SIGNAL_TYPE_DVI_SINGLE_LINK);
diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
index 8b5c27857671..3699e633801d 100644
--- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
+++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
@@ -1059,7 +1059,7 @@ static bool build_freesync_hdr(struct pwl_float_data_ex *rgb_regamma,
 	struct fixed31_32 min_display;
 	struct fixed31_32 max_content;
 	struct fixed31_32 clip = dc_fixpt_one;
-	struct fixed31_32 output;
+	struct fixed31_32 output = dc_fixpt_zero;
 	bool use_eetf = false;
 	bool is_clipped = false;
 	struct fixed31_32 sdr_white_level;
diff --git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
index 3955b7e4b2e2..d09627c15b9c 100644
--- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
+++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
@@ -158,13 +158,13 @@ static unsigned int calc_v_total_from_duration(
 	if (duration_in_us > vrr->max_duration_in_us)
 		duration_in_us = vrr->max_duration_in_us;
 
-	if (dc_is_hdmi_signal(stream->signal)) {
+	if (dc_is_hdmi_signal(stream->signal)) { // change for HDMI to comply with spec
 		uint32_t h_total_up_scaled;
 
 		h_total_up_scaled = stream->timing.h_total * 10000;
 		v_total = div_u64((unsigned long long)duration_in_us
 					* stream->timing.pix_clk_100hz + (h_total_up_scaled - 1),
-					h_total_up_scaled);
+					h_total_up_scaled); //ceiling for MMax and MMin for MVRR
 	} else {
 		v_total = div64_u64(div64_u64(((unsigned long long)(
 					duration_in_us) * (stream->timing.pix_clk_100hz / 10)),
@@ -1057,7 +1057,7 @@ void mod_freesync_build_vrr_params(struct mod_freesync *mod_freesync,
 			in_out_vrr->fixed_refresh_in_uhz = 0;
 
 		refresh_range = div_u64(in_out_vrr->max_refresh_in_uhz + 500000, 1000000) -
-+				div_u64(in_out_vrr->min_refresh_in_uhz + 500000, 1000000);
+				div_u64(in_out_vrr->min_refresh_in_uhz + 500000, 1000000);
 
 		in_out_vrr->supported = true;
 	}
@@ -1126,6 +1126,8 @@ void mod_freesync_build_vrr_params(struct mod_freesync *mod_freesync,
 		in_out_vrr->adjust.v_total_min = stream->timing.v_total;
 		in_out_vrr->adjust.v_total_max = stream->timing.v_total;
 	}
+
+	in_out_vrr->adjust.allow_otg_v_count_halt = (in_config->state == VRR_STATE_ACTIVE_FIXED) ? true : false;
 }
 
 void mod_freesync_handle_preflip(struct mod_freesync *mod_freesync,
diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_execution.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_execution.c
index 733f22bed021..c996365e84b0 100644
--- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_execution.c
+++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_execution.c
@@ -151,7 +151,7 @@ out:
 
 static enum mod_hdcp_status poll_l_prime_available(struct mod_hdcp *hdcp)
 {
-	enum mod_hdcp_status status;
+	enum mod_hdcp_status status = MOD_HDCP_STATUS_FAILURE;
 	uint8_t size;
 	uint16_t max_wait = 20; // units of ms
 	uint16_t num_polls = 5;
diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_ddc.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_ddc.c
index f7b5583ee609..8e9caae7c955 100644
--- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_ddc.c
+++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_ddc.c
@@ -156,6 +156,10 @@ static enum mod_hdcp_status read(struct mod_hdcp *hdcp,
 	uint32_t cur_size = 0;
 	uint32_t data_offset = 0;
 
+	if (msg_id == MOD_HDCP_MESSAGE_ID_INVALID) {
+		return MOD_HDCP_STATUS_DDC_FAILURE;
+	}
+
 	if (is_dp_hdcp(hdcp)) {
 		while (buf_len > 0) {
 			cur_size = MIN(buf_len, HDCP_MAX_AUX_TRANSACTION_SIZE);
@@ -215,6 +219,10 @@ static enum mod_hdcp_status write(struct mod_hdcp *hdcp,
 	uint32_t cur_size = 0;
 	uint32_t data_offset = 0;
 
+	if (msg_id == MOD_HDCP_MESSAGE_ID_INVALID) {
+		return MOD_HDCP_STATUS_DDC_FAILURE;
+	}
+
 	if (is_dp_hdcp(hdcp)) {
 		while (buf_len > 0) {
 			cur_size = MIN(buf_len, HDCP_MAX_AUX_TRANSACTION_SIZE);
diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
index 84f9b412a4f1..a344e2e49b0e 100644
--- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
+++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
@@ -536,8 +536,6 @@ void mod_build_adaptive_sync_infopacket(const struct dc_stream_state *stream,
 			mod_build_adaptive_sync_infopacket_v2(stream, param, info_packet);
 		break;
 	case FREESYNC_TYPE_PCON_IN_WHITELIST:
-		mod_build_adaptive_sync_infopacket_v1(info_packet);
-		break;
 	case ADAPTIVE_SYNC_TYPE_EDP:
 		mod_build_adaptive_sync_infopacket_v1(info_packet);
 		break;
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index b0a6256e89f4..7536c173a546 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -24,6 +24,7 @@
 #define __AMD_SHARED_H__
 
 #include <drm/amd_asic_type.h>
+#include <drm/drm_print.h>
 
 
 #define AMD_MAX_USEC_TIMEOUT		1000000  /* 1000 ms */
@@ -321,6 +322,8 @@ struct amd_ip_funcs {
 	int (*set_powergating_state)(void *handle,
 				     enum amd_powergating_state state);
 	void (*get_clockgating_state)(void *handle, u64 *flags);
+	void (*dump_ip_state)(void *handle);
+	void (*print_ip_state)(void *handle, struct drm_printer *p);
 };
 
 
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_offset.h
index f2f8f9b39c6b..fc72c2267060 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_offset.h
@@ -311,6 +311,10 @@
 #define mmPHYESYMCLK_CLOCK_CNTL_BASE_IDX                                                               2
 #define mmPHYFSYMCLK_CLOCK_CNTL                                                                        0x0057
 #define mmPHYFSYMCLK_CLOCK_CNTL_BASE_IDX                                                               2
+#define regHDMICHARCLK0_CLOCK_CNTL                                                                      0x004a
+#define regHDMICHARCLK0_CLOCK_CNTL_BASE_IDX                                                             2
+#define mmHDMICHARCLK0_CLOCK_CNTL                                                                      0x004a
+#define mmHDMICHARCLK0_CLOCK_CNTL_BASE_IDX                                                             2
 
 
 // addressBlock: dce_dc_dccg_dccg_dfs_dispdec
@@ -4513,6 +4517,10 @@
 #define mmCM0_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM0_CM_3DLUT_OUT_OFFSET_B                                                                    0x0e18
 #define mmCM0_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM0_CM_TEST_DEBUG_INDEX                                                                      0x0e19
+#define mmCM0_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM0_CM_TEST_DEBUG_DATA                                                                       0x0e1a
+#define mmCM0_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp0_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
@@ -5201,6 +5209,10 @@
 #define mmCM1_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM1_CM_3DLUT_OUT_OFFSET_B                                                                    0x0f83
 #define mmCM1_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM1_CM_TEST_DEBUG_INDEX                                                                      0x0f84
+#define mmCM1_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM1_CM_TEST_DEBUG_DATA                                                                       0x0f85
+#define mmCM1_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp1_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
@@ -5888,6 +5900,10 @@
 #define mmCM2_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM2_CM_3DLUT_OUT_OFFSET_B                                                                    0x10ee
 #define mmCM2_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM2_CM_TEST_DEBUG_INDEX                                                                      0x10ef
+#define mmCM2_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM2_CM_TEST_DEBUG_DATA                                                                       0x10f0
+#define mmCM2_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp2_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
@@ -6576,6 +6592,10 @@
 #define mmCM3_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM3_CM_3DLUT_OUT_OFFSET_B                                                                    0x1259
 #define mmCM3_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM3_CM_TEST_DEBUG_INDEX                                                                      0x125a
+#define mmCM3_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM3_CM_TEST_DEBUG_DATA                                                                       0x125b
+#define mmCM3_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp3_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
@@ -7264,6 +7284,10 @@
 #define mmCM4_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM4_CM_3DLUT_OUT_OFFSET_B                                                                    0x13c4
 #define mmCM4_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM4_CM_TEST_DEBUG_INDEX                                                                      0x13c5
+#define mmCM4_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM4_CM_TEST_DEBUG_DATA                                                                       0x13c6
+#define mmCM4_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp4_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
@@ -7952,6 +7976,10 @@
 #define mmCM5_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM5_CM_3DLUT_OUT_OFFSET_B                                                                    0x152f
 #define mmCM5_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM5_CM_TEST_DEBUG_INDEX                                                                      0x1530
+#define mmCM5_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM5_CM_TEST_DEBUG_DATA                                                                       0x1531
+#define mmCM5_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp5_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h
index e0a447351623..daf71e82f0ba 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h
@@ -1189,6 +1189,11 @@
 #define PHYFSYMCLK_CLOCK_CNTL__PHYFSYMCLK_FORCE_SRC_SEL__SHIFT                                                0x4
 #define PHYFSYMCLK_CLOCK_CNTL__PHYFSYMCLK_FORCE_EN_MASK                                                       0x00000001L
 #define PHYFSYMCLK_CLOCK_CNTL__PHYFSYMCLK_FORCE_SRC_SEL_MASK                                                  0x00000010L
+//HDMICHARCLK0_CLOCK_CNTL
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_EN__SHIFT                                                       0x0
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_SRC_SEL__SHIFT                                                  0x4
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_EN_MASK                                                         0x00000001L
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_SRC_SEL_MASK                                                    0x00000070L
 
 
 // addressBlock: dce_dc_dccg_dccg_dfs_dispdec
@@ -16739,6 +16744,15 @@
 #define CM0_CM_3DLUT_OUT_OFFSET_B__CM_3DLUT_OUT_SCALE_B__SHIFT                                                0x10
 #define CM0_CM_3DLUT_OUT_OFFSET_B__CM_3DLUT_OUT_OFFSET_B_MASK                                                 0x0000FFFFL
 #define CM0_CM_3DLUT_OUT_OFFSET_B__CM_3DLUT_OUT_SCALE_B_MASK                                                  0xFFFF0000L
+//CM0_CM_TEST_DEBUG_INDEX
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX__SHIFT                                                   0x0
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN__SHIFT                                                0x8
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX_MASK                                                     0x000000FFL
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN_MASK                                                  0x00000100L
+//CM0_CM_TEST_DEBUG_DATA
+#define CM0_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA__SHIFT                                                     0x0
+#define CM0_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA_MASK                                                       0xFFFFFFFFL
+
 
 // addressBlock: dce_dc_dpp0_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
 //DC_PERFMON12_PERFCOUNTER_CNTL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_offset.h
index b45a35aae241..bf84f97d9162 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_offset.h
@@ -4466,6 +4466,10 @@
 #define mmCM0_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM0_CM_3DLUT_OUT_OFFSET_B                                                                    0x0e18
 #define mmCM0_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM0_CM_TEST_DEBUG_INDEX                                                                      0x0e19
+#define mmCM0_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM0_CM_TEST_DEBUG_DATA                                                                       0x0e1a
+#define mmCM0_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp0_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
@@ -5154,6 +5158,10 @@
 #define mmCM1_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM1_CM_3DLUT_OUT_OFFSET_B                                                                    0x0f83
 #define mmCM1_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM1_CM_TEST_DEBUG_INDEX                                                                      0x0f84
+#define mmCM1_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM1_CM_TEST_DEBUG_DATA                                                                       0x0f85
+#define mmCM1_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp1_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
@@ -5841,6 +5849,10 @@
 #define mmCM2_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM2_CM_3DLUT_OUT_OFFSET_B                                                                    0x10ee
 #define mmCM2_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM2_CM_TEST_DEBUG_INDEX                                                                      0x10ef
+#define mmCM2_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM2_CM_TEST_DEBUG_DATA                                                                       0x10f0
+#define mmCM2_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp2_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
@@ -6529,6 +6541,10 @@
 #define mmCM3_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM3_CM_3DLUT_OUT_OFFSET_B                                                                    0x1259
 #define mmCM3_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM3_CM_TEST_DEBUG_INDEX                                                                      0x125a
+#define mmCM3_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM3_CM_TEST_DEBUG_DATA                                                                       0x125b
+#define mmCM3_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp3_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
@@ -7217,6 +7233,10 @@
 #define mmCM4_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM4_CM_3DLUT_OUT_OFFSET_B                                                                    0x13c4
 #define mmCM4_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM4_CM_TEST_DEBUG_INDEX                                                                      0x13c5
+#define mmCM4_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM4_CM_TEST_DEBUG_DATA                                                                       0x13c6
+#define mmCM4_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp4_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_sh_mask.h
index 3dae29f9581e..56cdb219874a 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_sh_mask.h
@@ -15676,6 +15676,14 @@
 #define CM0_CM_3DLUT_OUT_OFFSET_B__CM_3DLUT_OUT_SCALE_B__SHIFT                                                0x10
 #define CM0_CM_3DLUT_OUT_OFFSET_B__CM_3DLUT_OUT_OFFSET_B_MASK                                                 0x0000FFFFL
 #define CM0_CM_3DLUT_OUT_OFFSET_B__CM_3DLUT_OUT_SCALE_B_MASK                                                  0xFFFF0000L
+//CM0_CM_TEST_DEBUG_INDEX
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX__SHIFT                                                   0x0
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN__SHIFT                                                0x8
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX_MASK                                                     0x000000FFL
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN_MASK                                                  0x00000100L
+//CM0_CM_TEST_DEBUG_DATA
+#define CM0_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA__SHIFT                                                     0x0
+#define CM0_CM_TEST_DEBUG_DATA__CM_TEST_DEBUG_DATA_MASK                                                       0xFFFFFFFFL
 
 
 // addressBlock: dce_dc_dpp0_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_offset.h
index daa8130636f0..8b0d2638a6b0 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_offset.h
@@ -3110,6 +3110,10 @@
 #define mmCM0_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM0_CM_3DLUT_OUT_OFFSET_B                                                                    0x0e18
 #define mmCM0_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM0_CM_TEST_DEBUG_INDEX                                                                      0x0e19
+#define mmCM0_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM0_CM_TEST_DEBUG_DATA                                                                       0x0e1a
+#define mmCM0_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp0_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
@@ -3798,6 +3802,10 @@
 #define mmCM1_CM_3DLUT_OUT_OFFSET_G_BASE_IDX                                                           2
 #define mmCM1_CM_3DLUT_OUT_OFFSET_B                                                                    0x0f83
 #define mmCM1_CM_3DLUT_OUT_OFFSET_B_BASE_IDX                                                           2
+#define mmCM1_CM_TEST_DEBUG_INDEX                                                                      0x0f84
+#define mmCM1_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define mmCM1_CM_TEST_DEBUG_DATA                                                                       0x0f85
+#define mmCM1_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp1_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
@@ -5687,6 +5695,16 @@
 #define mmDSCC0_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
 #define mmDSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL                                           0x3035
 #define mmDSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
+#define mmDSCC0_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x303a
+#define mmDSCC0_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define mmDSCC0_DSCC_TEST_DEBUG_DATA0                                                                  0x303b
+#define mmDSCC0_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
+#define mmDSCC0_DSCC_TEST_DEBUG_DATA1                                                                  0x303c
+#define mmDSCC0_DSCC_TEST_DEBUG_DATA1_BASE_IDX                                                         2
+#define mmDSCC0_DSCC_TEST_DEBUG_DATA2                                                                  0x303d
+#define mmDSCC0_DSCC_TEST_DEBUG_DATA2_BASE_IDX                                                         2
+#define mmDSCC0_DSCC_TEST_DEBUG_DATA3                                                                  0x303e
+#define mmDSCC0_DSCC_TEST_DEBUG_DATA3_BASE_IDX                                                         2
 
 
 // addressBlock: dce_dc_dsc0_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
@@ -5817,6 +5835,16 @@
 #define mmDSCC1_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
 #define mmDSCC1_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL                                           0x3091
 #define mmDSCC1_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
+#define mmDSCC1_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x3096
+#define mmDSCC1_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define mmDSCC1_DSCC_TEST_DEBUG_DATA0                                                                  0x3097
+#define mmDSCC1_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
+#define mmDSCC1_DSCC_TEST_DEBUG_DATA1                                                                  0x3098
+#define mmDSCC1_DSCC_TEST_DEBUG_DATA1_BASE_IDX                                                         2
+#define mmDSCC1_DSCC_TEST_DEBUG_DATA2                                                                  0x3099
+#define mmDSCC1_DSCC_TEST_DEBUG_DATA2_BASE_IDX                                                         2
+#define mmDSCC1_DSCC_TEST_DEBUG_DATA3                                                                  0x309a
+#define mmDSCC1_DSCC_TEST_DEBUG_DATA3_BASE_IDX                                                         2
 
 
 // addressBlock: dce_dc_dsc1_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_sh_mask.h
index 5c469cf635e5..53f1705f8d99 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_sh_mask.h
@@ -10701,6 +10701,13 @@
 #define CM0_CM_COEF_FORMAT__CM_BIAS_FORMAT_MASK                                                               0x00000001L
 #define CM0_CM_COEF_FORMAT__CM_POST_CSC_COEF_FORMAT_MASK                                                      0x00000010L
 #define CM0_CM_COEF_FORMAT__CM_GAMUT_REMAP_COEF_FORMAT_MASK                                                   0x00000100L
+
+//CM0_CM_TEST_DEBUG_INDEX
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX__SHIFT                                                   0x0
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN__SHIFT                                                0x8
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX_MASK                                                     0x000000FFL
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN_MASK                                                  0x00000100L
+
 //CM0_CM_SHAPER_CONTROL
 #define CM0_CM_SHAPER_CONTROL__CM_SHAPER_LUT_MODE__SHIFT                                                      0x0
 #define CM0_CM_SHAPER_CONTROL__CM_SHAPER_MODE_CURRENT__SHIFT                                                  0x2
@@ -22258,7 +22265,9 @@
 #define DSC_TOP0_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK                                                  0x00000100L
 //DSC_TOP0_DSC_DEBUG_CONTROL
 #define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT                                                         0x0
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL__SHIFT                                             0x4
 #define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK                                                           0x00000001L
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL_MASK                                               0x00000070L
 
 
 // addressBlock: dce_dc_dsc0_dispdec_dsccif_dispdec
@@ -22631,6 +22640,15 @@
 //DSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL
 #define DSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL__DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL__SHIFT  0x0
 #define DSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL__DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_MASK  0x0003FFFFL
+//DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE__SHIFT                                  0x0
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE__SHIFT                                  0x8
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE__SHIFT                                  0x10
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE__SHIFT                                  0x18
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE_MASK                                    0x0000001FL
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE_MASK                                    0x00001F00L
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE_MASK                                    0x001F0000L
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE_MASK                                    0x1F000000L
 
 
 // addressBlock: dce_dc_dsc0_dispdec_dsc_dcperfmon_dc_perfmon_dispdec
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_offset.h
index f268d33c4744..7fd906f10803 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_offset.h
@@ -424,6 +424,8 @@
 #define regDTBCLK_DTO2_MODULO_BASE_IDX                                                                  2
 #define regDTBCLK_DTO3_MODULO                                                                           0x0022
 #define regDTBCLK_DTO3_MODULO_BASE_IDX                                                                  2
+#define regHDMICHARCLK0_CLOCK_CNTL                                                                      0x004a
+#define regHDMICHARCLK0_CLOCK_CNTL_BASE_IDX                                                             2
 #define regPHYASYMCLK_CLOCK_CNTL                                                                        0x0052
 #define regPHYASYMCLK_CLOCK_CNTL_BASE_IDX                                                               2
 #define regPHYBSYMCLK_CLOCK_CNTL                                                                        0x0053
@@ -434,6 +436,8 @@
 #define regPHYDSYMCLK_CLOCK_CNTL_BASE_IDX                                                               2
 #define regPHYESYMCLK_CLOCK_CNTL                                                                        0x0056
 #define regPHYESYMCLK_CLOCK_CNTL_BASE_IDX                                                               2
+#define regHDMISTREAMCLK_CNTL                                                                           0x0059
+#define regHDMISTREAMCLK_CNTL_BASE_IDX                                                                  2
 #define regDCCG_GATE_DISABLE_CNTL3                                                                      0x005a
 #define regDCCG_GATE_DISABLE_CNTL3_BASE_IDX                                                             2
 #define regHDMISTREAMCLK0_DTO_PARAM                                                                     0x005b
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_sh_mask.h
index cf3398f15666..07fbfafe6056 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_sh_mask.h
@@ -1372,6 +1372,11 @@
 //DTBCLK_DTO3_MODULO
 #define DTBCLK_DTO3_MODULO__DTBCLK_DTO3_MODULO__SHIFT                                                         0x0
 #define DTBCLK_DTO3_MODULO__DTBCLK_DTO3_MODULO_MASK                                                           0xFFFFFFFFL
+//HDMICHARCLK0_CLOCK_CNTL
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_EN__SHIFT                                                       0x0
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_SRC_SEL__SHIFT                                                  0x4
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_EN_MASK                                                         0x00000001L
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_SRC_SEL_MASK                                                    0x00000070L
 //PHYASYMCLK_CLOCK_CNTL
 #define PHYASYMCLK_CLOCK_CNTL__PHYASYMCLK_FORCE_EN__SHIFT                                                     0x0
 #define PHYASYMCLK_CLOCK_CNTL__PHYASYMCLK_FORCE_SRC_SEL__SHIFT                                                0x4
@@ -1397,6 +1402,13 @@
 #define PHYESYMCLK_CLOCK_CNTL__PHYESYMCLK_FORCE_SRC_SEL__SHIFT                                                0x4
 #define PHYESYMCLK_CLOCK_CNTL__PHYESYMCLK_FORCE_EN_MASK                                                       0x00000001L
 #define PHYESYMCLK_CLOCK_CNTL__PHYESYMCLK_FORCE_SRC_SEL_MASK                                                  0x00000030L
+//HDMISTREAMCLK_CNTL
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_SRC_SEL__SHIFT                                                     0x0
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_EN__SHIFT                                                          0x3
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_DTO_FORCE_DIS__SHIFT                                               0x4
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_SRC_SEL_MASK                                                       0x00000007L
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_EN_MASK                                                            0x00000008L
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_DTO_FORCE_DIS_MASK                                                 0x00000010L
 //DCCG_GATE_DISABLE_CNTL3
 #define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK0_GATE_DISABLE__SHIFT                                           0x0
 #define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK1_GATE_DISABLE__SHIFT                                           0x1
@@ -46978,6 +46990,13 @@
 #define DSC_TOP0_DSC_TOP_CONTROL__DSC_CLOCK_EN_MASK                                                           0x00000001L
 #define DSC_TOP0_DSC_TOP_CONTROL__DSC_DISPCLK_R_GATE_DIS_MASK                                                 0x00000010L
 #define DSC_TOP0_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK                                                  0x00000100L
+
+
+//DSC_TOP0_DSC_DEBUG_CONTROL
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT                                                         0x0
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL__SHIFT                                             0x4
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK                                                           0x00000001L
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL_MASK                                               0x00000070L
 //DSC_TOP0_DSC_DEBUG_CONTROL
 #define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT                                                         0x0
 #define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL__SHIFT                                             0x4
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_offset.h
index 50c34d88c17c..16a69d17bb1e 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_offset.h
@@ -213,6 +213,8 @@
 #define regDTBCLK_DTO2_MODULO_BASE_IDX                                                                  2
 #define regDTBCLK_DTO3_MODULO                                                                           0x0022
 #define regDTBCLK_DTO3_MODULO_BASE_IDX                                                                  2
+#define regHDMICHARCLK0_CLOCK_CNTL                                                                      0x004a
+#define regHDMICHARCLK0_CLOCK_CNTL_BASE_IDX                                                             2
 #define regPHYASYMCLK_CLOCK_CNTL                                                                        0x0052
 #define regPHYASYMCLK_CLOCK_CNTL_BASE_IDX                                                               2
 #define regPHYBSYMCLK_CLOCK_CNTL                                                                        0x0053
@@ -233,6 +235,8 @@
 #define regDCCG_AUDIO_DTBCLK_DTO_MODULO_BASE_IDX                                                        2
 #define regDTBCLK_DTO_DBUF_EN                                                                           0x0063
 #define regDTBCLK_DTO_DBUF_EN_BASE_IDX                                                                  2
+#define regHDMISTREAMCLK_CNTL                                                                           0x0059
+#define regHDMISTREAMCLK_CNTL_BASE_IDX                                                                  2
 
 
 // addressBlock: dce_dc_dccg_dccg_dcperfmon0_dc_perfmon_dispdec
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_sh_mask.h
index 295e0dac9ffa..6473362e39a8 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_sh_mask.h
@@ -886,6 +886,11 @@
 //DTBCLK_DTO3_MODULO
 #define DTBCLK_DTO3_MODULO__DTBCLK_DTO3_MODULO__SHIFT                                                         0x0
 #define DTBCLK_DTO3_MODULO__DTBCLK_DTO3_MODULO_MASK                                                           0xFFFFFFFFL
+//HDMICHARCLK0_CLOCK_CNTL
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_EN__SHIFT                                                       0x0
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_SRC_SEL__SHIFT                                                  0x4
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_EN_MASK                                                         0x00000001L
+#define HDMICHARCLK0_CLOCK_CNTL__HDMICHARCLK0_SRC_SEL_MASK                                                    0x00000070L
 //PHYASYMCLK_CLOCK_CNTL
 #define PHYASYMCLK_CLOCK_CNTL__PHYASYMCLK_FORCE_EN__SHIFT                                                     0x0
 #define PHYASYMCLK_CLOCK_CNTL__PHYASYMCLK_FORCE_SRC_SEL__SHIFT                                                0x4
@@ -911,6 +916,11 @@
 #define PHYESYMCLK_CLOCK_CNTL__PHYESYMCLK_FORCE_SRC_SEL__SHIFT                                                0x4
 #define PHYESYMCLK_CLOCK_CNTL__PHYESYMCLK_FORCE_EN_MASK                                                       0x00000001L
 #define PHYESYMCLK_CLOCK_CNTL__PHYESYMCLK_FORCE_SRC_SEL_MASK                                                  0x00000030L
+//HDMISTREAMCLK_CNTL
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_SRC_SEL__SHIFT                                                     0x0
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_DTO_FORCE_DIS__SHIFT                                               0x10
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_SRC_SEL_MASK                                                       0x00000003L
+#define HDMISTREAMCLK_CNTL__HDMISTREAMCLK0_DTO_FORCE_DIS_MASK                                                 0x00010000L
 //DCCG_GATE_DISABLE_CNTL3
 #define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK0_GATE_DISABLE__SHIFT                                           0x0
 #define DCCG_GATE_DISABLE_CNTL3__HDMISTREAMCLK1_GATE_DISABLE__SHIFT                                           0x1
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_offset.h
index 14c29ce4c7b3..78cb61d5800a 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_offset.h
@@ -1719,6 +1719,10 @@
 #define regDCHUBBUB_TIMEOUT_INTERRUPT_STATUS_BASE_IDX                                                   2
 #define regFMON_CTRL                                                                                    0x0541
 #define regFMON_CTRL_BASE_IDX                                                                           2
+#define regDCHUBBUB_TEST_DEBUG_INDEX                                                                    0x0542
+#define regDCHUBBUB_TEST_DEBUG_INDEX_BASE_IDX                                                           2
+#define regDCHUBBUB_TEST_DEBUG_DATA                                                                     0x0543
+#define regDCHUBBUB_TEST_DEBUG_DATA_BASE_IDX                                                            2
 
 
 // addressBlock: dcn_dc_dchubbubl_hubbub_sdpif_dispdec
@@ -3574,6 +3578,10 @@
 #define regCM0_CM_DEALPHA_BASE_IDX                                                                      2
 #define regCM0_CM_COEF_FORMAT                                                                           0x0d8c
 #define regCM0_CM_COEF_FORMAT_BASE_IDX                                                                  2
+#define regCM0_CM_TEST_DEBUG_INDEX                                                                      0x0d8d
+#define regCM0_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define regCM0_CM_TEST_DEBUG_DATA                                                                       0x0d8e
+#define regCM0_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dcn_dc_dpp0_dispdec_dpp_top_dispdec
@@ -3960,6 +3968,10 @@
 #define regCM1_CM_DEALPHA_BASE_IDX                                                                      2
 #define regCM1_CM_COEF_FORMAT                                                                           0x0ef7
 #define regCM1_CM_COEF_FORMAT_BASE_IDX                                                                  2
+#define regCM1_CM_TEST_DEBUG_INDEX                                                                      0x0ef8
+#define regCM1_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define regCM1_CM_TEST_DEBUG_DATA                                                                       0x0ef9
+#define regCM1_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dcn_dc_dpp1_dispdec_dpp_top_dispdec
@@ -4346,6 +4358,10 @@
 #define regCM2_CM_DEALPHA_BASE_IDX                                                                      2
 #define regCM2_CM_COEF_FORMAT                                                                           0x1062
 #define regCM2_CM_COEF_FORMAT_BASE_IDX                                                                  2
+#define regCM2_CM_TEST_DEBUG_INDEX                                                                      0x1063
+#define regCM2_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define regCM2_CM_TEST_DEBUG_DATA                                                                       0x1064
+#define regCM2_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dcn_dc_dpp2_dispdec_dpp_top_dispdec
@@ -4732,6 +4748,10 @@
 #define regCM3_CM_DEALPHA_BASE_IDX                                                                      2
 #define regCM3_CM_COEF_FORMAT                                                                           0x11cd
 #define regCM3_CM_COEF_FORMAT_BASE_IDX                                                                  2
+#define regCM3_CM_TEST_DEBUG_INDEX                                                                      0x11ce
+#define regCM3_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define regCM3_CM_TEST_DEBUG_DATA                                                                       0x11cf
+#define regCM3_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dcn_dc_dpp3_dispdec_dpp_top_dispdec
@@ -11780,6 +11800,16 @@
 #define regDSCC0_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
 #define regDSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL                                           0x3035
 #define regDSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
+#define regDSCC0_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x303a
+#define regDSCC0_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define regDSCC0_DSCC_TEST_DEBUG_DATA0                                                                  0x303b
+#define regDSCC0_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
+#define regDSCC0_DSCC_TEST_DEBUG_DATA1                                                                  0x303c
+#define regDSCC0_DSCC_TEST_DEBUG_DATA1_BASE_IDX                                                         2
+#define regDSCC0_DSCC_TEST_DEBUG_DATA2                                                                  0x303d
+#define regDSCC0_DSCC_TEST_DEBUG_DATA2_BASE_IDX                                                         2
+#define regDSCC0_DSCC_TEST_DEBUG_DATA3                                                                  0x303e
+#define regDSCC0_DSCC_TEST_DEBUG_DATA3_BASE_IDX                                                         2
 
 
 // addressBlock: dcn_dc_dsc0_dispdec_dsccif_dispdec
@@ -11888,6 +11918,16 @@
 #define regDSCC1_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
 #define regDSCC1_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL                                           0x3091
 #define regDSCC1_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
+#define regDSCC1_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x3096
+#define regDSCC1_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define regDSCC1_DSCC_TEST_DEBUG_DATA0                                                                  0x3097
+#define regDSCC1_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
+#define regDSCC1_DSCC_TEST_DEBUG_DATA1                                                                  0x3098
+#define regDSCC1_DSCC_TEST_DEBUG_DATA1_BASE_IDX                                                         2
+#define regDSCC1_DSCC_TEST_DEBUG_DATA2                                                                  0x3099
+#define regDSCC1_DSCC_TEST_DEBUG_DATA2_BASE_IDX                                                         2
+#define regDSCC1_DSCC_TEST_DEBUG_DATA3                                                                  0x309a
+#define regDSCC1_DSCC_TEST_DEBUG_DATA3_BASE_IDX                                                         2
 
 
 // addressBlock: dcn_dc_dsc1_dispdec_dsccif_dispdec
@@ -11996,6 +12036,16 @@
 #define regDSCC2_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
 #define regDSCC2_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL                                           0x30ed
 #define regDSCC2_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
+#define regDSCC2_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x30f2
+#define regDSCC2_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define regDSCC2_DSCC_TEST_DEBUG_DATA0                                                                  0x30f3
+#define regDSCC2_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
+#define regDSCC2_DSCC_TEST_DEBUG_DATA1                                                                  0x30f4
+#define regDSCC2_DSCC_TEST_DEBUG_DATA1_BASE_IDX                                                         2
+#define regDSCC2_DSCC_TEST_DEBUG_DATA2                                                                  0x30f5
+#define regDSCC2_DSCC_TEST_DEBUG_DATA2_BASE_IDX                                                         2
+#define regDSCC2_DSCC_TEST_DEBUG_DATA3                                                                  0x30f6
+#define regDSCC2_DSCC_TEST_DEBUG_DATA3_BASE_IDX                                                         2
 
 
 // addressBlock: dcn_dc_dsc2_dispdec_dsccif_dispdec
@@ -12104,6 +12154,16 @@
 #define regDSCC3_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
 #define regDSCC3_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL                                           0x3149
 #define regDSCC3_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
+#define regDSCC3_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x314e
+#define regDSCC3_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define regDSCC3_DSCC_TEST_DEBUG_DATA0                                                                  0x314f
+#define regDSCC3_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
+#define regDSCC3_DSCC_TEST_DEBUG_DATA1                                                                  0x3150
+#define regDSCC3_DSCC_TEST_DEBUG_DATA1_BASE_IDX                                                         2
+#define regDSCC3_DSCC_TEST_DEBUG_DATA2                                                                  0x3151
+#define regDSCC3_DSCC_TEST_DEBUG_DATA2_BASE_IDX                                                         2
+#define regDSCC3_DSCC_TEST_DEBUG_DATA3                                                                  0x3152
+#define regDSCC3_DSCC_TEST_DEBUG_DATA3_BASE_IDX                                                         2
 
 
 // addressBlock: dcn_dc_dsc3_dispdec_dsccif_dispdec
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_sh_mask.h
index 0691e328d0f0..1093105ca35b 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_sh_mask.h
@@ -11544,6 +11544,11 @@
 #define CM0_CM_COEF_FORMAT__CM_BIAS_FORMAT_MASK                                                               0x00000001L
 #define CM0_CM_COEF_FORMAT__CM_POST_CSC_COEF_FORMAT_MASK                                                      0x00000010L
 #define CM0_CM_COEF_FORMAT__CM_GAMUT_REMAP_COEF_FORMAT_MASK                                                   0x00000100L
+//CM0_CM_TEST_DEBUG_INDEX
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX__SHIFT                                                   0x0
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN__SHIFT                                                0x8
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX_MASK                                                     0x000000FFL
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN_MASK                                                  0x00000100L
 
 
 // addressBlock: dcn_dc_dpp0_dispdec_dpp_top_dispdec
@@ -42267,6 +42272,18 @@
 //DSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL
 #define DSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL__DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL__SHIFT  0x0
 #define DSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL__DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_MASK  0x0003FFFFL
+//DSCC0_DSCC_TEST_DEBUG_INDEX2
+#define DSCC0_DSCC_TEST_DEBUG_INDEX2__DSCC_TEST_DEBUG_INDEX2__SHIFT                                           0x0
+#define DSCC0_DSCC_TEST_DEBUG_INDEX2__DSCC_TEST_DEBUG_INDEX2_MASK                                             0x000000FFL
+//DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE__SHIFT                                  0x0
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE__SHIFT                                  0x8
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE__SHIFT                                  0x10
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE__SHIFT                                  0x18
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE_MASK                                    0x0000001FL
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE_MASK                                    0x00001F00L
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE_MASK                                    0x001F0000L
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE_MASK                                    0x1F000000L
 
 
 // addressBlock: dcn_dc_dsc0_dispdec_dsccif_dispdec
@@ -42300,6 +42317,16 @@
 #define DSC_TOP0_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK                                                  0x00000100L
 //DSC_TOP0_DSC_DEBUG_CONTROL
 #define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT                                                         0x0
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL__SHIFT                                             0x4
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK                                                           0x00000001L
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL_MASK                                               0x00000070L
+
+
+//DSC_TOP0_DSC_DEBUG_CONTROL
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT                                                         0x0
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL__SHIFT                                             0x4
+//DSC_TOP0_DSC_DEBUG_CONTROL
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT                                                         0x0
 #define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK                                                           0x00000001L
 
 
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_offset.h
index 3bd8792fd7b3..a04b8c32c564 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_offset.h
@@ -1719,6 +1719,10 @@
 #define regDCHUBBUB_TIMEOUT_INTERRUPT_STATUS_BASE_IDX                                                   2
 #define regFMON_CTRL                                                                                    0x0541
 #define regFMON_CTRL_BASE_IDX                                                                           2
+#define regDCHUBBUB_TEST_DEBUG_INDEX                                                                    0x0542
+#define regDCHUBBUB_TEST_DEBUG_INDEX_BASE_IDX                                                           2
+#define regDCHUBBUB_TEST_DEBUG_DATA                                                                     0x0543
+#define regDCHUBBUB_TEST_DEBUG_DATA_BASE_IDX                                                            2
 
 
 // addressBlock: dce_dc_dchubbubl_hubbub_sdpif_dispdec
@@ -3573,6 +3577,10 @@
 #define regCM0_CM_DEALPHA_BASE_IDX                                                                      2
 #define regCM0_CM_COEF_FORMAT                                                                           0x0d8c
 #define regCM0_CM_COEF_FORMAT_BASE_IDX                                                                  2
+#define regCM0_CM_TEST_DEBUG_INDEX                                                                      0x0d8d
+#define regCM0_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define regCM0_CM_TEST_DEBUG_DATA                                                                       0x0d8e
+#define regCM0_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp0_dispdec_dpp_top_dispdec
@@ -3959,6 +3967,10 @@
 #define regCM1_CM_DEALPHA_BASE_IDX                                                                      2
 #define regCM1_CM_COEF_FORMAT                                                                           0x0ef7
 #define regCM1_CM_COEF_FORMAT_BASE_IDX                                                                  2
+#define regCM1_CM_TEST_DEBUG_INDEX                                                                      0x0ef8
+#define regCM1_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define regCM1_CM_TEST_DEBUG_DATA                                                                       0x0ef9
+#define regCM1_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp1_dispdec_dpp_top_dispdec
@@ -4345,6 +4357,10 @@
 #define regCM2_CM_DEALPHA_BASE_IDX                                                                      2
 #define regCM2_CM_COEF_FORMAT                                                                           0x1062
 #define regCM2_CM_COEF_FORMAT_BASE_IDX                                                                  2
+#define regCM2_CM_TEST_DEBUG_INDEX                                                                      0x1063
+#define regCM2_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define regCM2_CM_TEST_DEBUG_DATA                                                                       0x1064
+#define regCM2_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp2_dispdec_dpp_top_dispdec
@@ -4731,6 +4747,10 @@
 #define regCM3_CM_DEALPHA_BASE_IDX                                                                      2
 #define regCM3_CM_COEF_FORMAT                                                                           0x11cd
 #define regCM3_CM_COEF_FORMAT_BASE_IDX                                                                  2
+#define regCM3_CM_TEST_DEBUG_INDEX                                                                      0x11ce
+#define regCM3_CM_TEST_DEBUG_INDEX_BASE_IDX                                                             2
+#define regCM3_CM_TEST_DEBUG_DATA                                                                       0x11cf
+#define regCM3_CM_TEST_DEBUG_DATA_BASE_IDX                                                              2
 
 
 // addressBlock: dce_dc_dpp3_dispdec_dpp_top_dispdec
@@ -11789,6 +11809,10 @@
 #define regDSCC0_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
 #define regDSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL                                           0x3035
 #define regDSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
+#define regDSCC0_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x303a
+#define regDSCC0_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define regDSCC0_DSCC_TEST_DEBUG_DATA0                                                                  0x303b
+#define regDSCC0_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
 
 
 // addressBlock: dce_dc_dsc0_dispdec_dsccif_dispdec
@@ -11897,6 +11921,10 @@
 #define regDSCC1_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
 #define regDSCC1_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL                                           0x3091
 #define regDSCC1_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
+#define regDSCC1_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x3096
+#define regDSCC1_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define regDSCC1_DSCC_TEST_DEBUG_DATA0                                                                  0x3097
+#define regDSCC1_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
 
 
 // addressBlock: dce_dc_dsc1_dispdec_dsccif_dispdec
@@ -12005,7 +12033,10 @@
 #define regDSCC2_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
 #define regDSCC2_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL                                           0x30ed
 #define regDSCC2_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
-
+#define regDSCC2_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x30f2
+#define regDSCC2_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define regDSCC2_DSCC_TEST_DEBUG_DATA0                                                                  0x30f3
+#define regDSCC2_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
 
 // addressBlock: dce_dc_dsc2_dispdec_dsccif_dispdec
 // base address: 0x2e0
@@ -12113,6 +12144,10 @@
 #define regDSCC3_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
 #define regDSCC3_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL                                           0x3149
 #define regDSCC3_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_BASE_IDX                                  2
+#define regDSCC3_DSCC_TEST_DEBUG_BUS_ROTATE                                                             0x314e
+#define regDSCC3_DSCC_TEST_DEBUG_BUS_ROTATE_BASE_IDX                                                    2
+#define regDSCC3_DSCC_TEST_DEBUG_DATA0                                                                  0x314f
+#define regDSCC3_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
 
 
 // addressBlock: dce_dc_dsc3_dispdec_dsccif_dispdec
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_sh_mask.h
index e82dffc2b9b0..ce773fca621f 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_sh_mask.h
@@ -11547,6 +11547,11 @@
 #define CM0_CM_COEF_FORMAT__CM_BIAS_FORMAT_MASK                                                               0x00000001L
 #define CM0_CM_COEF_FORMAT__CM_POST_CSC_COEF_FORMAT_MASK                                                      0x00000010L
 #define CM0_CM_COEF_FORMAT__CM_GAMUT_REMAP_COEF_FORMAT_MASK                                                   0x00000100L
+//CM0_CM_TEST_DEBUG_INDEX
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX__SHIFT                                                   0x0
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN__SHIFT                                                0x8
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_INDEX_MASK                                                     0x000000FFL
+#define CM0_CM_TEST_DEBUG_INDEX__CM_TEST_DEBUG_WRITE_EN_MASK                                                  0x00000100L
 
 
 // addressBlock: dce_dc_dpp0_dispdec_dpp_top_dispdec
@@ -42315,6 +42320,15 @@
 //DSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL
 #define DSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL__DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL__SHIFT  0x0
 #define DSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL__DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL_MASK  0x0003FFFFL
+//DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE__SHIFT                                  0x0
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE__SHIFT                                  0x8
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE__SHIFT                                  0x10
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE__SHIFT                                  0x18
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS0_ROTATE_MASK                                    0x0000001FL
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS1_ROTATE_MASK                                    0x00001F00L
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS2_ROTATE_MASK                                    0x001F0000L
+#define DSCC0_DSCC_TEST_DEBUG_BUS_ROTATE__DSCC_TEST_DEBUG_BUS3_ROTATE_MASK                                    0x1F000000L
 
 
 // addressBlock: dce_dc_dsc0_dispdec_dsccif_dispdec
@@ -42348,7 +42362,9 @@
 #define DSC_TOP0_DSC_TOP_CONTROL__DSC_DSCCLK_R_GATE_DIS_MASK                                                  0x00000100L
 //DSC_TOP0_DSC_DEBUG_CONTROL
 #define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN__SHIFT                                                         0x0
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL__SHIFT                                             0x4
 #define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_DBG_EN_MASK                                                           0x00000001L
+#define DSC_TOP0_DSC_DEBUG_CONTROL__DSC_TEST_CLOCK_MUX_SEL_MASK                                               0x00000070L
 
 
 // addressBlock: dce_dc_dsc1_dispdec_dscc_dispdec
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_3_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_3_0_0_offset.h
index 0bb47e06eee8..081e726afbf0 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_3_0_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_3_0_0_offset.h
@@ -24,6 +24,8 @@
 #define mmDPCSTX0_DPCSTX_PLL_UPDATE_ADDR_BASE_IDX                                                      2
 #define mmDPCSTX0_DPCSTX_PLL_UPDATE_DATA                                                               0x292d
 #define mmDPCSTX0_DPCSTX_PLL_UPDATE_DATA_BASE_IDX                                                      2
+#define mmDPCSTX0_DPCSTX_DEBUG_CONFIG                                                                  0x292e
+#define mmDPCSTX0_DPCSTX_DEBUG_CONFIG_BASE_IDX                                                         2
 
 
 // addressBlock: dpcssys_dpcs0_rdpcstx0_dispdec
@@ -50,6 +52,8 @@
 #define mmRDPCSTX0_RDPCSTX_CNTL2_BASE_IDX                                                              2
 #define mmRDPCSTX0_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG                                                    0x293c
 #define mmRDPCSTX0_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG_BASE_IDX                                           2
+#define mmRDPCSTX0_RDPCSTX_DEBUG_CONFIG                                                                0x293d
+#define mmRDPCSTX0_RDPCSTX_DEBUG_CONFIG_BASE_IDX                                                       2
 #define mmRDPCSTX0_RDPCSTX_PHY_CNTL0                                                                   0x2940
 #define mmRDPCSTX0_RDPCSTX_PHY_CNTL0_BASE_IDX                                                          2
 #define mmRDPCSTX0_RDPCSTX_PHY_CNTL1                                                                   0x2941
@@ -120,6 +124,8 @@
 #define mmDPCSTX1_DPCSTX_PLL_UPDATE_ADDR_BASE_IDX                                                      2
 #define mmDPCSTX1_DPCSTX_PLL_UPDATE_DATA                                                               0x2a05
 #define mmDPCSTX1_DPCSTX_PLL_UPDATE_DATA_BASE_IDX                                                      2
+#define mmDPCSTX1_DPCSTX_DEBUG_CONFIG                                                                  0x2a06
+#define mmDPCSTX1_DPCSTX_DEBUG_CONFIG_BASE_IDX                                                         2
 
 
 // addressBlock: dpcssys_dpcs0_rdpcstx1_dispdec
@@ -146,6 +152,8 @@
 #define mmRDPCSTX1_RDPCSTX_CNTL2_BASE_IDX                                                              2
 #define mmRDPCSTX1_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG                                                    0x2a14
 #define mmRDPCSTX1_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG_BASE_IDX                                           2
+#define mmRDPCSTX1_RDPCSTX_DEBUG_CONFIG                                                                0x2a15
+#define mmRDPCSTX1_RDPCSTX_DEBUG_CONFIG_BASE_IDX                                                       2
 #define mmRDPCSTX1_RDPCSTX_PHY_CNTL0                                                                   0x2a18
 #define mmRDPCSTX1_RDPCSTX_PHY_CNTL0_BASE_IDX                                                          2
 #define mmRDPCSTX1_RDPCSTX_PHY_CNTL1                                                                   0x2a19
@@ -216,6 +224,8 @@
 #define mmDPCSTX2_DPCSTX_PLL_UPDATE_ADDR_BASE_IDX                                                      2
 #define mmDPCSTX2_DPCSTX_PLL_UPDATE_DATA                                                               0x2add
 #define mmDPCSTX2_DPCSTX_PLL_UPDATE_DATA_BASE_IDX                                                      2
+#define mmDPCSTX2_DPCSTX_DEBUG_CONFIG                                                                  0x2ade
+#define mmDPCSTX2_DPCSTX_DEBUG_CONFIG_BASE_IDX                                                         2
 
 
 // addressBlock: dpcssys_dpcs0_rdpcstx2_dispdec
@@ -242,6 +252,8 @@
 #define mmRDPCSTX2_RDPCSTX_CNTL2_BASE_IDX                                                              2
 #define mmRDPCSTX2_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG                                                    0x2aec
 #define mmRDPCSTX2_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG_BASE_IDX                                           2
+#define mmRDPCSTX2_RDPCSTX_DEBUG_CONFIG                                                                0x2aed
+#define mmRDPCSTX2_RDPCSTX_DEBUG_CONFIG_BASE_IDX                                                       2
 #define mmRDPCSTX2_RDPCSTX_PHY_CNTL0                                                                   0x2af0
 #define mmRDPCSTX2_RDPCSTX_PHY_CNTL0_BASE_IDX                                                          2
 #define mmRDPCSTX2_RDPCSTX_PHY_CNTL1                                                                   0x2af1
@@ -312,6 +324,8 @@
 #define mmDPCSTX3_DPCSTX_PLL_UPDATE_ADDR_BASE_IDX                                                      2
 #define mmDPCSTX3_DPCSTX_PLL_UPDATE_DATA                                                               0x2bb5
 #define mmDPCSTX3_DPCSTX_PLL_UPDATE_DATA_BASE_IDX                                                      2
+#define mmDPCSTX3_DPCSTX_DEBUG_CONFIG                                                                  0x2bb6
+#define mmDPCSTX3_DPCSTX_DEBUG_CONFIG_BASE_IDX                                                         2
 
 
 // addressBlock: dpcssys_dpcs0_rdpcstx3_dispdec
@@ -338,6 +352,8 @@
 #define mmRDPCSTX3_RDPCSTX_CNTL2_BASE_IDX                                                              2
 #define mmRDPCSTX3_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG                                                    0x2bc4
 #define mmRDPCSTX3_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG_BASE_IDX                                           2
+#define mmRDPCSTX3_RDPCSTX_DEBUG_CONFIG                                                                0x2bc5
+#define mmRDPCSTX3_RDPCSTX_DEBUG_CONFIG_BASE_IDX                                                       2
 #define mmRDPCSTX3_RDPCSTX_PHY_CNTL0                                                                   0x2bc8
 #define mmRDPCSTX3_RDPCSTX_PHY_CNTL0_BASE_IDX                                                          2
 #define mmRDPCSTX3_RDPCSTX_PHY_CNTL1                                                                   0x2bc9
@@ -408,6 +424,8 @@
 #define mmDPCSTX4_DPCSTX_PLL_UPDATE_ADDR_BASE_IDX                                                      2
 #define mmDPCSTX4_DPCSTX_PLL_UPDATE_DATA                                                               0x2c8d
 #define mmDPCSTX4_DPCSTX_PLL_UPDATE_DATA_BASE_IDX                                                      2
+#define mmDPCSTX4_DPCSTX_DEBUG_CONFIG                                                                  0x2c8e
+#define mmDPCSTX4_DPCSTX_DEBUG_CONFIG_BASE_IDX                                                         2
 
 
 // addressBlock: dpcssys_dpcs0_rdpcstx4_dispdec
@@ -434,6 +452,8 @@
 #define mmRDPCSTX4_RDPCSTX_CNTL2_BASE_IDX                                                              2
 #define mmRDPCSTX4_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG                                                    0x2c9c
 #define mmRDPCSTX4_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG_BASE_IDX                                           2
+#define mmRDPCSTX4_RDPCSTX_DEBUG_CONFIG                                                                0x2c9d
+#define mmRDPCSTX4_RDPCSTX_DEBUG_CONFIG_BASE_IDX                                                       2
 #define mmRDPCSTX4_RDPCSTX_PHY_CNTL0                                                                   0x2ca0
 #define mmRDPCSTX4_RDPCSTX_PHY_CNTL0_BASE_IDX                                                          2
 #define mmRDPCSTX4_RDPCSTX_PHY_CNTL1                                                                   0x2ca1
@@ -504,6 +524,8 @@
 #define mmDPCSTX5_DPCSTX_PLL_UPDATE_ADDR_BASE_IDX                                                      2
 #define mmDPCSTX5_DPCSTX_PLL_UPDATE_DATA                                                               0x2d65
 #define mmDPCSTX5_DPCSTX_PLL_UPDATE_DATA_BASE_IDX                                                      2
+#define mmDPCSTX5_DPCSTX_DEBUG_CONFIG                                                                  0x2d66
+#define mmDPCSTX5_DPCSTX_DEBUG_CONFIG_BASE_IDX                                                         2
 
 
 // addressBlock: dpcssys_dpcs0_rdpcstx5_dispdec
@@ -530,6 +552,8 @@
 #define mmRDPCSTX5_RDPCSTX_CNTL2_BASE_IDX                                                              2
 #define mmRDPCSTX5_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG                                                    0x2d74
 #define mmRDPCSTX5_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG_BASE_IDX                                           2
+#define mmRDPCSTX5_RDPCSTX_DEBUG_CONFIG                                                                0x2d75
+#define mmRDPCSTX5_RDPCSTX_DEBUG_CONFIG_BASE_IDX                                                       2
 #define mmRDPCSTX5_RDPCSTX_PHY_CNTL0                                                                   0x2d78
 #define mmRDPCSTX5_RDPCSTX_PHY_CNTL0_BASE_IDX                                                          2
 #define mmRDPCSTX5_RDPCSTX_PHY_CNTL1                                                                   0x2d79
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_3_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_3_0_0_sh_mask.h
index 23fa1121a967..1f846fa6c1a2 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_3_0_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_3_0_0_sh_mask.h
@@ -70,7 +70,9 @@
 //DPCSTX0_DPCSTX_PLL_UPDATE_DATA
 #define DPCSTX0_DPCSTX_PLL_UPDATE_DATA__DPCS_PLL_UPDATE_DATA__SHIFT                                           0x0
 #define DPCSTX0_DPCSTX_PLL_UPDATE_DATA__DPCS_PLL_UPDATE_DATA_MASK                                             0xFFFFFFFFL
-
+//DPCSTX0_DPCSTX_DEBUG_CONFIG
+#define DPCSTX0_DPCSTX_DEBUG_CONFIG__DPCS_DBG_CBUS_DIS__SHIFT                                                 0xe
+#define DPCSTX0_DPCSTX_DEBUG_CONFIG__DPCS_DBG_CBUS_DIS_MASK                                                   0x00004000L
 
 // addressBlock: dpcssys_dpcs0_rdpcstx0_dispdec
 //RDPCSTX0_RDPCSTX_CNTL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_3_0_3_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_3_0_3_sh_mask.h
index 55743d06f728..e55ff0e8d74c 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_3_0_3_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_3_0_3_sh_mask.h
@@ -70,7 +70,9 @@
 //DPCSTX0_DPCSTX_PLL_UPDATE_DATA
 #define DPCSTX0_DPCSTX_PLL_UPDATE_DATA__DPCS_PLL_UPDATE_DATA__SHIFT                                           0x0
 #define DPCSTX0_DPCSTX_PLL_UPDATE_DATA__DPCS_PLL_UPDATE_DATA_MASK                                             0xFFFFFFFFL
-
+//DPCSTX0_DPCSTX_DEBUG_CONFIG
+#define DPCSTX0_DPCSTX_DEBUG_CONFIG__DPCS_DBG_CBUS_DIS__SHIFT                                                 0xe
+#define DPCSTX0_DPCSTX_DEBUG_CONFIG__DPCS_DBG_CBUS_DIS_MASK                                                   0x00004000L
 
 // addressBlock: dpcssys_dpcs0_rdpcstx0_dispdec
 //RDPCSTX0_RDPCSTX_CNTL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_4_2_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_4_2_0_offset.h
index 01a56556cde1..5b4fdeda1040 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_4_2_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dpcs/dpcs_4_2_0_offset.h
@@ -155,6 +155,8 @@
 #define regRDPCSTX0_RDPCSTX_CNTL2_BASE_IDX                                                              2
 #define regRDPCSTX0_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG                                                    0x293c
 #define regRDPCSTX0_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG_BASE_IDX                                           2
+#define regRDPCSTX0_RDPCSTX_DEBUG_CONFIG                                                                0x293d
+#define regRDPCSTX0_RDPCSTX_DEBUG_CONFIG_BASE_IDX                                                       2
 #define regRDPCSTX0_RDPCSTX_PHY_CNTL0                                                                   0x2940
 #define regRDPCSTX0_RDPCSTX_PHY_CNTL0_BASE_IDX                                                          2
 #define regRDPCSTX0_RDPCSTX_PHY_CNTL1                                                                   0x2941
@@ -239,6 +241,8 @@
 #define regRDPCSTX1_RDPCSTX_CNTL2_BASE_IDX                                                              2
 #define regRDPCSTX1_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG                                                    0x2a14
 #define regRDPCSTX1_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG_BASE_IDX                                           2
+#define regRDPCSTX1_RDPCSTX_DEBUG_CONFIG                                                                0x2a15
+#define regRDPCSTX1_RDPCSTX_DEBUG_CONFIG_BASE_IDX                                                       2
 #define regRDPCSTX1_RDPCSTX_PHY_CNTL0                                                                   0x2a18
 #define regRDPCSTX1_RDPCSTX_PHY_CNTL0_BASE_IDX                                                          2
 #define regRDPCSTX1_RDPCSTX_PHY_CNTL1                                                                   0x2a19
@@ -323,6 +327,8 @@
 #define regRDPCSTX2_RDPCSTX_CNTL2_BASE_IDX                                                              2
 #define regRDPCSTX2_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG                                                    0x2aec
 #define regRDPCSTX2_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG_BASE_IDX                                           2
+#define regRDPCSTX2_RDPCSTX_DEBUG_CONFIG                                                                0x2aed
+#define regRDPCSTX2_RDPCSTX_DEBUG_CONFIG_BASE_IDX                                                       2
 #define regRDPCSTX2_RDPCSTX_PHY_CNTL0                                                                   0x2af0
 #define regRDPCSTX2_RDPCSTX_PHY_CNTL0_BASE_IDX                                                          2
 #define regRDPCSTX2_RDPCSTX_PHY_CNTL1                                                                   0x2af1
@@ -407,6 +413,8 @@
 #define regRDPCSTX3_RDPCSTX_CNTL2_BASE_IDX                                                              2
 #define regRDPCSTX3_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG                                                    0x2bc4
 #define regRDPCSTX3_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG_BASE_IDX                                           2
+#define regRDPCSTX3_RDPCSTX_DEBUG_CONFIG                                                                0x2bc5
+#define regRDPCSTX3_RDPCSTX_DEBUG_CONFIG_BASE_IDX                                                       2
 #define regRDPCSTX3_RDPCSTX_PHY_CNTL0                                                                   0x2bc8
 #define regRDPCSTX3_RDPCSTX_PHY_CNTL0_BASE_IDX                                                          2
 #define regRDPCSTX3_RDPCSTX_PHY_CNTL1                                                                   0x2bc9
@@ -491,6 +499,8 @@
 #define regRDPCSTX4_RDPCSTX_CNTL2_BASE_IDX                                                              2
 #define regRDPCSTX4_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG                                                    0x2c9c
 #define regRDPCSTX4_RDPCSTX_DMCU_DPALT_DIS_BLOCK_REG_BASE_IDX                                           2
+#define regRDPCSTX4_RDPCSTX_DEBUG_CONFIG                                                                0x2c9d
+#define regRDPCSTX4_RDPCSTX_DEBUG_CONFIG_BASE_IDX                                                       2
 #define regRDPCSTX4_RDPCSTX_PHY_CNTL0                                                                   0x2ca0
 #define regRDPCSTX4_RDPCSTX_PHY_CNTL0_BASE_IDX                                                          2
 #define regRDPCSTX4_RDPCSTX_PHY_CNTL1                                                                   0x2ca1
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_offset.h
index 4908044f7409..4c8e7fdb6976 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_1_0_offset.h
@@ -4830,6 +4830,8 @@
 #define mmCP_ECC_FIRSTOCCURRENCE_RING2_BASE_IDX                                                        0
 #define mmGB_EDC_MODE                                                                                  0x1e1e
 #define mmGB_EDC_MODE_BASE_IDX                                                                         0
+#define mmCP_DEBUG                                                                                     0x1e1f
+#define mmCP_DEBUG_BASE_IDX                                                                            0
 #define mmCP_FETCHER_SOURCE                                                                            0x1e22
 #define mmCP_FETCHER_SOURCE_BASE_IDX                                                                   0
 #define mmCP_PQ_WPTR_POLL_CNTL                                                                         0x1e23
@@ -7778,6 +7780,8 @@
 #define mmCP_MES_DOORBELL_CONTROL5_BASE_IDX                                                            1
 #define mmCP_MES_DOORBELL_CONTROL6                                                                     0x2841
 #define mmCP_MES_DOORBELL_CONTROL6_BASE_IDX                                                            1
+#define mmCP_MES_DEBUG_INTERRUPT_INSTR_PNTR                                                            0x2842
+#define mmCP_MES_DEBUG_INTERRUPT_INSTR_PNTR_BASE_IDX                                                   1
 #define mmCP_MES_GP0_LO                                                                                0x2843
 #define mmCP_MES_GP0_LO_BASE_IDX                                                                       1
 #define mmCP_MES_GP0_HI                                                                                0x2844
@@ -9332,10 +9336,16 @@
 #define mmRLC_LB_CNTR_INIT_1_BASE_IDX                                                                  1
 #define mmRLC_LB_CNTR_1                                                                                0x4c1c
 #define mmRLC_LB_CNTR_1_BASE_IDX                                                                       1
+#define mmRLC_GPM_DEBUG_INST_ADDR                                                                      0x4c1d
+#define mmRLC_GPM_DEBUG_INST_ADDR_BASE_IDX                                                             1
 #define mmRLC_JUMP_TABLE_RESTORE                                                                       0x4c1e
 #define mmRLC_JUMP_TABLE_RESTORE_BASE_IDX                                                              1
 #define mmRLC_PG_DELAY_2                                                                               0x4c1f
 #define mmRLC_PG_DELAY_2_BASE_IDX                                                                      1
+#define mmRLC_GPM_DEBUG_INST_A                                                                         0x4c22
+#define mmRLC_GPM_DEBUG_INST_A_BASE_IDX                                                                1
+#define mmRLC_GPM_DEBUG_INST_B                                                                         0x4c23
+#define mmRLC_GPM_DEBUG_INST_B_BASE_IDX                                                                1
 #define mmRLC_GPU_CLOCK_COUNT_LSB                                                                      0x4c24
 #define mmRLC_GPU_CLOCK_COUNT_LSB_BASE_IDX                                                             1
 #define mmRLC_GPU_CLOCK_COUNT_MSB                                                                      0x4c25
@@ -9720,6 +9730,8 @@
 #define mmRLC_SPM_THREAD_TRACE_CTRL_BASE_IDX                                                           1
 #define mmRLC_LB_CNTR_2                                                                                0x4de7
 #define mmRLC_LB_CNTR_2_BASE_IDX                                                                       1
+#define mmRLC_LX6_CORE_PDEBUG_INST                                                                     0x4deb
+#define mmRLC_LX6_CORE_PDEBUG_INST_BASE_IDX                                                            1
 #define mmRLC_CPAXI_DOORBELL_MON_CTRL                                                                  0x4df1
 #define mmRLC_CPAXI_DOORBELL_MON_CTRL_BASE_IDX                                                         1
 #define mmRLC_CPAXI_DOORBELL_MON_STAT                                                                  0x4df2
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_sh_mask.h
index efc16ddf274a..2dfa0e5b1aa3 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_sh_mask.h
@@ -6822,6 +6822,8 @@
 #define VM_L2_PROTECTION_FAULT_STATUS__VMID__SHIFT                                                            0x14
 #define VM_L2_PROTECTION_FAULT_STATUS__VF__SHIFT                                                              0x18
 #define VM_L2_PROTECTION_FAULT_STATUS__VFID__SHIFT                                                            0x19
+#define VM_L2_PROTECTION_FAULT_STATUS__UCE__SHIFT                                                             0x1d
+#define VM_L2_PROTECTION_FAULT_STATUS__FED__SHIFT                                                             0x1e
 #define VM_L2_PROTECTION_FAULT_STATUS__MORE_FAULTS_MASK                                                       0x00000001L
 #define VM_L2_PROTECTION_FAULT_STATUS__WALKER_ERROR_MASK                                                      0x0000000EL
 #define VM_L2_PROTECTION_FAULT_STATUS__PERMISSION_FAULTS_MASK                                                 0x000000F0L
@@ -6832,6 +6834,8 @@
 #define VM_L2_PROTECTION_FAULT_STATUS__VMID_MASK                                                              0x00F00000L
 #define VM_L2_PROTECTION_FAULT_STATUS__VF_MASK                                                                0x01000000L
 #define VM_L2_PROTECTION_FAULT_STATUS__VFID_MASK                                                              0x1E000000L
+#define VM_L2_PROTECTION_FAULT_STATUS__UCE_MASK                                                               0x20000000L
+#define VM_L2_PROTECTION_FAULT_STATUS__FED_MASK                                                               0x40000000L
 //VM_L2_PROTECTION_FAULT_ADDR_LO32
 #define VM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32__SHIFT                                       0x0
 #define VM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32_MASK                                         0xFFFFFFFFL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_6_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_6_0_0_offset.h
index 8b931bbabe70..969e006b859b 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_6_0_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_6_0_0_offset.h
@@ -237,6 +237,10 @@
 #define regSEM_REGISTER_LAST_PART2_BASE_IDX                                                             0
 #define regIH_CLIENT_CFG                                                                                0x0184
 #define regIH_CLIENT_CFG_BASE_IDX                                                                       0
+#define regIH_RING1_CLIENT_CFG_INDEX                                                                    0x0185
+#define regIH_RING1_CLIENT_CFG_INDEX_BASE_IDX                                                           0
+#define regIH_RING1_CLIENT_CFG_DATA                                                                     0x0186
+#define regIH_RING1_CLIENT_CFG_DATA_BASE_IDX                                                            0
 #define regIH_CLIENT_CFG_INDEX                                                                          0x0188
 #define regIH_CLIENT_CFG_INDEX_BASE_IDX                                                                 0
 #define regIH_CLIENT_CFG_DATA                                                                           0x0189
diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_6_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_6_0_0_sh_mask.h
index f262f44fa68c..a672a91e58f0 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_6_0_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_6_0_0_sh_mask.h
@@ -888,6 +888,16 @@
 //IH_CLIENT_CFG
 #define IH_CLIENT_CFG__TOTAL_CLIENT_NUM__SHIFT                                                                0x0
 #define IH_CLIENT_CFG__TOTAL_CLIENT_NUM_MASK                                                                  0x0000003FL
+//IH_RING1_CLIENT_CFG_INDEX
+#define IH_RING1_CLIENT_CFG_INDEX__INDEX__SHIFT                                                               0x0
+#define IH_RING1_CLIENT_CFG_INDEX__INDEX_MASK                                                                 0x00000007L
+//IH_RING1_CLIENT_CFG_DATA
+#define IH_RING1_CLIENT_CFG_DATA__CLIENT_ID__SHIFT                                                            0x0
+#define IH_RING1_CLIENT_CFG_DATA__SOURCE_ID__SHIFT                                                            0x8
+#define IH_RING1_CLIENT_CFG_DATA__SOURCE_ID_MATCH_ENABLE__SHIFT                                               0x10
+#define IH_RING1_CLIENT_CFG_DATA__CLIENT_ID_MASK                                                              0x000000FFL
+#define IH_RING1_CLIENT_CFG_DATA__SOURCE_ID_MASK                                                              0x0000FF00L
+#define IH_RING1_CLIENT_CFG_DATA__SOURCE_ID_MATCH_ENABLE_MASK                                                 0x00010000L
 //IH_CLIENT_CFG_INDEX
 #define IH_CLIENT_CFG_INDEX__INDEX__SHIFT                                                                     0x0
 #define IH_CLIENT_CFG_INDEX__INDEX_MASK                                                                       0x0000001FL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_14_0_2_offset.h b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_14_0_2_offset.h
new file mode 100644
index 000000000000..da7e31fedd58
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_14_0_2_offset.h
@@ -0,0 +1,511 @@
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _smuio_14_0_2_OFFSET_HEADER
+#define _smuio_14_0_2_OFFSET_HEADER
+
+
+
+// addressBlock: smuio_smuio_tsc_SmuSmuioDec
+// base address: 0x5a8a0
+#define regPWROK_REFCLK_GAP_CYCLES                                                                      0x0028
+#define regPWROK_REFCLK_GAP_CYCLES_BASE_IDX                                                             1
+#define regGOLDEN_TSC_INCREMENT_UPPER                                                                   0x002b
+#define regGOLDEN_TSC_INCREMENT_UPPER_BASE_IDX                                                          1
+#define regGOLDEN_TSC_INCREMENT_LOWER                                                                   0x002c
+#define regGOLDEN_TSC_INCREMENT_LOWER_BASE_IDX                                                          1
+#define regGOLDEN_TSC_COUNT_UPPER                                                                       0x002d
+#define regGOLDEN_TSC_COUNT_UPPER_BASE_IDX                                                              1
+#define regGOLDEN_TSC_COUNT_LOWER                                                                       0x002e
+#define regGOLDEN_TSC_COUNT_LOWER_BASE_IDX                                                              1
+#define regSOC_GOLDEN_TSC_SHADOW_UPPER                                                                  0x002f
+#define regSOC_GOLDEN_TSC_SHADOW_UPPER_BASE_IDX                                                         1
+#define regSOC_GOLDEN_TSC_SHADOW_LOWER                                                                  0x0030
+#define regSOC_GOLDEN_TSC_SHADOW_LOWER_BASE_IDX                                                         1
+#define regSOC_GAP_PWROK                                                                                0x0031
+#define regSOC_GAP_PWROK_BASE_IDX                                                                       1
+
+
+// addressBlock: smuio_smuio_swtimer_SmuSmuioDec
+// base address: 0x5aca8
+#define regPWR_VIRT_RESET_REQ                                                                           0x012a
+#define regPWR_VIRT_RESET_REQ_BASE_IDX                                                                  1
+#define regPWR_DISP_TIMER_CONTROL                                                                       0x012b
+#define regPWR_DISP_TIMER_CONTROL_BASE_IDX                                                              1
+#define regPWR_DISP_TIMER_DEBUG                                                                         0x012c
+#define regPWR_DISP_TIMER_DEBUG_BASE_IDX                                                                1
+#define regPWR_DISP_TIMER2_CONTROL                                                                      0x012d
+#define regPWR_DISP_TIMER2_CONTROL_BASE_IDX                                                             1
+#define regPWR_DISP_TIMER2_DEBUG                                                                        0x012e
+#define regPWR_DISP_TIMER2_DEBUG_BASE_IDX                                                               1
+#define regPWR_DISP_TIMER_GLOBAL_CONTROL                                                                0x012f
+#define regPWR_DISP_TIMER_GLOBAL_CONTROL_BASE_IDX                                                       1
+#define regPWR_IH_CONTROL                                                                               0x0130
+#define regPWR_IH_CONTROL_BASE_IDX                                                                      1
+
+
+// addressBlock: smuio_smuio_misc_SmuSmuioDec
+// base address: 0x5a000
+#define regSMUIO_MCM_CONFIG                                                                             0x0023
+#define regSMUIO_MCM_CONFIG_BASE_IDX                                                                    0
+#define regIP_DISCOVERY_VERSION                                                                         0x0000
+#define regIP_DISCOVERY_VERSION_BASE_IDX                                                                1
+#define regSCRATCH_REGISTER0                                                                            0x01bd
+#define regSCRATCH_REGISTER0_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER1                                                                            0x01be
+#define regSCRATCH_REGISTER1_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER2                                                                            0x01bf
+#define regSCRATCH_REGISTER2_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER3                                                                            0x01c0
+#define regSCRATCH_REGISTER3_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER4                                                                            0x01c1
+#define regSCRATCH_REGISTER4_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER5                                                                            0x01c2
+#define regSCRATCH_REGISTER5_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER6                                                                            0x01c3
+#define regSCRATCH_REGISTER6_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER7                                                                            0x01c4
+#define regSCRATCH_REGISTER7_BASE_IDX                                                                   1
+
+
+// addressBlock: smuio_smuio_i2c_SmuSmuioDec
+// base address: 0x5a100
+#define regCKSVII2C_IC_CON                                                                              0x0040
+#define regCKSVII2C_IC_CON_BASE_IDX                                                                     0
+#define regCKSVII2C_IC_TAR                                                                              0x0041
+#define regCKSVII2C_IC_TAR_BASE_IDX                                                                     0
+#define regCKSVII2C_IC_SAR                                                                              0x0042
+#define regCKSVII2C_IC_SAR_BASE_IDX                                                                     0
+#define regCKSVII2C_IC_HS_MADDR                                                                         0x0043
+#define regCKSVII2C_IC_HS_MADDR_BASE_IDX                                                                0
+#define regCKSVII2C_IC_DATA_CMD                                                                         0x0044
+#define regCKSVII2C_IC_DATA_CMD_BASE_IDX                                                                0
+#define regCKSVII2C_IC_SS_SCL_HCNT                                                                      0x0045
+#define regCKSVII2C_IC_SS_SCL_HCNT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_SS_SCL_LCNT                                                                      0x0046
+#define regCKSVII2C_IC_SS_SCL_LCNT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_FS_SCL_HCNT                                                                      0x0047
+#define regCKSVII2C_IC_FS_SCL_HCNT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_FS_SCL_LCNT                                                                      0x0048
+#define regCKSVII2C_IC_FS_SCL_LCNT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_HS_SCL_HCNT                                                                      0x0049
+#define regCKSVII2C_IC_HS_SCL_HCNT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_HS_SCL_LCNT                                                                      0x004a
+#define regCKSVII2C_IC_HS_SCL_LCNT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_INTR_STAT                                                                        0x004b
+#define regCKSVII2C_IC_INTR_STAT_BASE_IDX                                                               0
+#define regCKSVII2C_IC_INTR_MASK                                                                        0x004c
+#define regCKSVII2C_IC_INTR_MASK_BASE_IDX                                                               0
+#define regCKSVII2C_IC_RAW_INTR_STAT                                                                    0x004d
+#define regCKSVII2C_IC_RAW_INTR_STAT_BASE_IDX                                                           0
+#define regCKSVII2C_IC_RX_TL                                                                            0x004e
+#define regCKSVII2C_IC_RX_TL_BASE_IDX                                                                   0
+#define regCKSVII2C_IC_TX_TL                                                                            0x004f
+#define regCKSVII2C_IC_TX_TL_BASE_IDX                                                                   0
+#define regCKSVII2C_IC_CLR_INTR                                                                         0x0050
+#define regCKSVII2C_IC_CLR_INTR_BASE_IDX                                                                0
+#define regCKSVII2C_IC_CLR_RX_UNDER                                                                     0x0051
+#define regCKSVII2C_IC_CLR_RX_UNDER_BASE_IDX                                                            0
+#define regCKSVII2C_IC_CLR_RX_OVER                                                                      0x0052
+#define regCKSVII2C_IC_CLR_RX_OVER_BASE_IDX                                                             0
+#define regCKSVII2C_IC_CLR_TX_OVER                                                                      0x0053
+#define regCKSVII2C_IC_CLR_TX_OVER_BASE_IDX                                                             0
+#define regCKSVII2C_IC_CLR_RD_REQ                                                                       0x0054
+#define regCKSVII2C_IC_CLR_RD_REQ_BASE_IDX                                                              0
+#define regCKSVII2C_IC_CLR_TX_ABRT                                                                      0x0055
+#define regCKSVII2C_IC_CLR_TX_ABRT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_CLR_RX_DONE                                                                      0x0056
+#define regCKSVII2C_IC_CLR_RX_DONE_BASE_IDX                                                             0
+#define regCKSVII2C_IC_CLR_ACTIVITY                                                                     0x0057
+#define regCKSVII2C_IC_CLR_ACTIVITY_BASE_IDX                                                            0
+#define regCKSVII2C_IC_CLR_STOP_DET                                                                     0x0058
+#define regCKSVII2C_IC_CLR_STOP_DET_BASE_IDX                                                            0
+#define regCKSVII2C_IC_CLR_START_DET                                                                    0x0059
+#define regCKSVII2C_IC_CLR_START_DET_BASE_IDX                                                           0
+#define regCKSVII2C_IC_CLR_GEN_CALL                                                                     0x005a
+#define regCKSVII2C_IC_CLR_GEN_CALL_BASE_IDX                                                            0
+#define regCKSVII2C_IC_ENABLE                                                                           0x005b
+#define regCKSVII2C_IC_ENABLE_BASE_IDX                                                                  0
+#define regCKSVII2C_IC_STATUS                                                                           0x005c
+#define regCKSVII2C_IC_STATUS_BASE_IDX                                                                  0
+#define regCKSVII2C_IC_TXFLR                                                                            0x005d
+#define regCKSVII2C_IC_TXFLR_BASE_IDX                                                                   0
+#define regCKSVII2C_IC_RXFLR                                                                            0x005e
+#define regCKSVII2C_IC_RXFLR_BASE_IDX                                                                   0
+#define regCKSVII2C_IC_SDA_HOLD                                                                         0x005f
+#define regCKSVII2C_IC_SDA_HOLD_BASE_IDX                                                                0
+#define regCKSVII2C_IC_TX_ABRT_SOURCE                                                                   0x0060
+#define regCKSVII2C_IC_TX_ABRT_SOURCE_BASE_IDX                                                          0
+#define regCKSVII2C_IC_SLV_DATA_NACK_ONLY                                                               0x0061
+#define regCKSVII2C_IC_SLV_DATA_NACK_ONLY_BASE_IDX                                                      0
+#define regCKSVII2C_IC_DMA_CR                                                                           0x0062
+#define regCKSVII2C_IC_DMA_CR_BASE_IDX                                                                  0
+#define regCKSVII2C_IC_DMA_TDLR                                                                         0x0063
+#define regCKSVII2C_IC_DMA_TDLR_BASE_IDX                                                                0
+#define regCKSVII2C_IC_DMA_RDLR                                                                         0x0064
+#define regCKSVII2C_IC_DMA_RDLR_BASE_IDX                                                                0
+#define regCKSVII2C_IC_SDA_SETUP                                                                        0x0065
+#define regCKSVII2C_IC_SDA_SETUP_BASE_IDX                                                               0
+#define regCKSVII2C_IC_ACK_GENERAL_CALL                                                                 0x0066
+#define regCKSVII2C_IC_ACK_GENERAL_CALL_BASE_IDX                                                        0
+#define regCKSVII2C_IC_ENABLE_STATUS                                                                    0x0067
+#define regCKSVII2C_IC_ENABLE_STATUS_BASE_IDX                                                           0
+#define regCKSVII2C_IC_FS_SPKLEN                                                                        0x0068
+#define regCKSVII2C_IC_FS_SPKLEN_BASE_IDX                                                               0
+#define regCKSVII2C_IC_HS_SPKLEN                                                                        0x0069
+#define regCKSVII2C_IC_HS_SPKLEN_BASE_IDX                                                               0
+#define regCKSVII2C_IC_CLR_RESTART_DET                                                                  0x006a
+#define regCKSVII2C_IC_CLR_RESTART_DET_BASE_IDX                                                         0
+#define regCKSVII2C_IC_COMP_PARAM_1                                                                     0x006d
+#define regCKSVII2C_IC_COMP_PARAM_1_BASE_IDX                                                            0
+#define regCKSVII2C_IC_COMP_VERSION                                                                     0x006e
+#define regCKSVII2C_IC_COMP_VERSION_BASE_IDX                                                            0
+#define regCKSVII2C_IC_COMP_TYPE                                                                        0x006f
+#define regCKSVII2C_IC_COMP_TYPE_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_CON                                                                             0x0080
+#define regCKSVII2C1_IC_CON_BASE_IDX                                                                    0
+#define regCKSVII2C1_IC_TAR                                                                             0x0081
+#define regCKSVII2C1_IC_TAR_BASE_IDX                                                                    0
+#define regCKSVII2C1_IC_SAR                                                                             0x0082
+#define regCKSVII2C1_IC_SAR_BASE_IDX                                                                    0
+#define regCKSVII2C1_IC_HS_MADDR                                                                        0x0083
+#define regCKSVII2C1_IC_HS_MADDR_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_DATA_CMD                                                                        0x0084
+#define regCKSVII2C1_IC_DATA_CMD_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_SS_SCL_HCNT                                                                     0x0085
+#define regCKSVII2C1_IC_SS_SCL_HCNT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_SS_SCL_LCNT                                                                     0x0086
+#define regCKSVII2C1_IC_SS_SCL_LCNT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_FS_SCL_HCNT                                                                     0x0087
+#define regCKSVII2C1_IC_FS_SCL_HCNT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_FS_SCL_LCNT                                                                     0x0088
+#define regCKSVII2C1_IC_FS_SCL_LCNT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_HS_SCL_HCNT                                                                     0x0089
+#define regCKSVII2C1_IC_HS_SCL_HCNT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_HS_SCL_LCNT                                                                     0x008a
+#define regCKSVII2C1_IC_HS_SCL_LCNT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_INTR_STAT                                                                       0x008b
+#define regCKSVII2C1_IC_INTR_STAT_BASE_IDX                                                              0
+#define regCKSVII2C1_IC_INTR_MASK                                                                       0x008c
+#define regCKSVII2C1_IC_INTR_MASK_BASE_IDX                                                              0
+#define regCKSVII2C1_IC_RAW_INTR_STAT                                                                   0x008d
+#define regCKSVII2C1_IC_RAW_INTR_STAT_BASE_IDX                                                          0
+#define regCKSVII2C1_IC_RX_TL                                                                           0x008e
+#define regCKSVII2C1_IC_RX_TL_BASE_IDX                                                                  0
+#define regCKSVII2C1_IC_TX_TL                                                                           0x008f
+#define regCKSVII2C1_IC_TX_TL_BASE_IDX                                                                  0
+#define regCKSVII2C1_IC_CLR_INTR                                                                        0x0090
+#define regCKSVII2C1_IC_CLR_INTR_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_CLR_RX_UNDER                                                                    0x0091
+#define regCKSVII2C1_IC_CLR_RX_UNDER_BASE_IDX                                                           0
+#define regCKSVII2C1_IC_CLR_RX_OVER                                                                     0x0092
+#define regCKSVII2C1_IC_CLR_RX_OVER_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_CLR_TX_OVER                                                                     0x0093
+#define regCKSVII2C1_IC_CLR_TX_OVER_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_CLR_RD_REQ                                                                      0x0094
+#define regCKSVII2C1_IC_CLR_RD_REQ_BASE_IDX                                                             0
+#define regCKSVII2C1_IC_CLR_TX_ABRT                                                                     0x0095
+#define regCKSVII2C1_IC_CLR_TX_ABRT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_CLR_RX_DONE                                                                     0x0096
+#define regCKSVII2C1_IC_CLR_RX_DONE_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_CLR_ACTIVITY                                                                    0x0097
+#define regCKSVII2C1_IC_CLR_ACTIVITY_BASE_IDX                                                           0
+#define regCKSVII2C1_IC_CLR_STOP_DET                                                                    0x0098
+#define regCKSVII2C1_IC_CLR_STOP_DET_BASE_IDX                                                           0
+#define regCKSVII2C1_IC_CLR_START_DET                                                                   0x0099
+#define regCKSVII2C1_IC_CLR_START_DET_BASE_IDX                                                          0
+#define regCKSVII2C1_IC_CLR_GEN_CALL                                                                    0x009a
+#define regCKSVII2C1_IC_CLR_GEN_CALL_BASE_IDX                                                           0
+#define regCKSVII2C1_IC_ENABLE                                                                          0x009b
+#define regCKSVII2C1_IC_ENABLE_BASE_IDX                                                                 0
+#define regCKSVII2C1_IC_STATUS                                                                          0x009c
+#define regCKSVII2C1_IC_STATUS_BASE_IDX                                                                 0
+#define regCKSVII2C1_IC_TXFLR                                                                           0x009d
+#define regCKSVII2C1_IC_TXFLR_BASE_IDX                                                                  0
+#define regCKSVII2C1_IC_RXFLR                                                                           0x009e
+#define regCKSVII2C1_IC_RXFLR_BASE_IDX                                                                  0
+#define regCKSVII2C1_IC_SDA_HOLD                                                                        0x009f
+#define regCKSVII2C1_IC_SDA_HOLD_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_TX_ABRT_SOURCE                                                                  0x00a0
+#define regCKSVII2C1_IC_TX_ABRT_SOURCE_BASE_IDX                                                         0
+#define regCKSVII2C1_IC_SLV_DATA_NACK_ONLY                                                              0x00a1
+#define regCKSVII2C1_IC_SLV_DATA_NACK_ONLY_BASE_IDX                                                     0
+#define regCKSVII2C1_IC_DMA_CR                                                                          0x00a2
+#define regCKSVII2C1_IC_DMA_CR_BASE_IDX                                                                 0
+#define regCKSVII2C1_IC_DMA_TDLR                                                                        0x00a3
+#define regCKSVII2C1_IC_DMA_TDLR_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_DMA_RDLR                                                                        0x00a4
+#define regCKSVII2C1_IC_DMA_RDLR_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_SDA_SETUP                                                                       0x00a5
+#define regCKSVII2C1_IC_SDA_SETUP_BASE_IDX                                                              0
+#define regCKSVII2C1_IC_ACK_GENERAL_CALL                                                                0x00a6
+#define regCKSVII2C1_IC_ACK_GENERAL_CALL_BASE_IDX                                                       0
+#define regCKSVII2C1_IC_ENABLE_STATUS                                                                   0x00a7
+#define regCKSVII2C1_IC_ENABLE_STATUS_BASE_IDX                                                          0
+#define regCKSVII2C1_IC_FS_SPKLEN                                                                       0x00a8
+#define regCKSVII2C1_IC_FS_SPKLEN_BASE_IDX                                                              0
+#define regCKSVII2C1_IC_HS_SPKLEN                                                                       0x00a9
+#define regCKSVII2C1_IC_HS_SPKLEN_BASE_IDX                                                              0
+#define regCKSVII2C1_IC_CLR_RESTART_DET                                                                 0x00aa
+#define regCKSVII2C1_IC_CLR_RESTART_DET_BASE_IDX                                                        0
+#define regCKSVII2C1_IC_COMP_PARAM_1                                                                    0x00ad
+#define regCKSVII2C1_IC_COMP_PARAM_1_BASE_IDX                                                           0
+#define regCKSVII2C1_IC_COMP_VERSION                                                                    0x00ae
+#define regCKSVII2C1_IC_COMP_VERSION_BASE_IDX                                                           0
+#define regCKSVII2C1_IC_COMP_TYPE                                                                       0x00af
+#define regCKSVII2C1_IC_COMP_TYPE_BASE_IDX                                                              0
+#define regSMUIO_PWRMGT                                                                                 0x018c
+#define regSMUIO_PWRMGT_BASE_IDX                                                                        0
+
+
+// addressBlock: smuio_smuio_rom_SmuSmuioDec
+// base address: 0x5a380
+#define regROM_CNTL                                                                                     0x00e0
+#define regROM_CNTL_BASE_IDX                                                                            0
+#define regPAGE_MIRROR_CNTL                                                                             0x00e1
+#define regPAGE_MIRROR_CNTL_BASE_IDX                                                                    0
+#define regROM_STATUS                                                                                   0x00e2
+#define regROM_STATUS_BASE_IDX                                                                          0
+#define regCGTT_ROM_CLK_CTRL0                                                                           0x00e3
+#define regCGTT_ROM_CLK_CTRL0_BASE_IDX                                                                  0
+#define regROM_INDEX                                                                                    0x00e4
+#define regROM_INDEX_BASE_IDX                                                                           0
+#define regROM_DATA                                                                                     0x00e5
+#define regROM_DATA_BASE_IDX                                                                            0
+#define regROM_START                                                                                    0x00e6
+#define regROM_START_BASE_IDX                                                                           0
+#define regROM_SW_CNTL                                                                                  0x00e8
+#define regROM_SW_CNTL_BASE_IDX                                                                         0
+#define regROM_SW_STATUS                                                                                0x00e9
+#define regROM_SW_STATUS_BASE_IDX                                                                       0
+#define regROM_SW_COMMAND                                                                               0x00ea
+#define regROM_SW_COMMAND_BASE_IDX                                                                      0
+#define regROM_SW_DATA_1                                                                                0x00ec
+#define regROM_SW_DATA_1_BASE_IDX                                                                       0
+#define regROM_SW_DATA_2                                                                                0x00ed
+#define regROM_SW_DATA_2_BASE_IDX                                                                       0
+#define regROM_SW_DATA_3                                                                                0x00ee
+#define regROM_SW_DATA_3_BASE_IDX                                                                       0
+#define regROM_SW_DATA_4                                                                                0x00ef
+#define regROM_SW_DATA_4_BASE_IDX                                                                       0
+#define regROM_SW_DATA_5                                                                                0x00f0
+#define regROM_SW_DATA_5_BASE_IDX                                                                       0
+#define regROM_SW_DATA_6                                                                                0x00f1
+#define regROM_SW_DATA_6_BASE_IDX                                                                       0
+#define regROM_SW_DATA_7                                                                                0x00f2
+#define regROM_SW_DATA_7_BASE_IDX                                                                       0
+#define regROM_SW_DATA_8                                                                                0x00f3
+#define regROM_SW_DATA_8_BASE_IDX                                                                       0
+#define regROM_SW_DATA_9                                                                                0x00f4
+#define regROM_SW_DATA_9_BASE_IDX                                                                       0
+#define regROM_SW_DATA_10                                                                               0x00f5
+#define regROM_SW_DATA_10_BASE_IDX                                                                      0
+#define regROM_SW_DATA_11                                                                               0x00f6
+#define regROM_SW_DATA_11_BASE_IDX                                                                      0
+#define regROM_SW_DATA_12                                                                               0x00f7
+#define regROM_SW_DATA_12_BASE_IDX                                                                      0
+#define regROM_SW_DATA_13                                                                               0x00f8
+#define regROM_SW_DATA_13_BASE_IDX                                                                      0
+#define regROM_SW_DATA_14                                                                               0x00f9
+#define regROM_SW_DATA_14_BASE_IDX                                                                      0
+#define regROM_SW_DATA_15                                                                               0x00fa
+#define regROM_SW_DATA_15_BASE_IDX                                                                      0
+#define regROM_SW_DATA_16                                                                               0x00fb
+#define regROM_SW_DATA_16_BASE_IDX                                                                      0
+#define regROM_SW_DATA_17                                                                               0x00fc
+#define regROM_SW_DATA_17_BASE_IDX                                                                      0
+#define regROM_SW_DATA_18                                                                               0x00fd
+#define regROM_SW_DATA_18_BASE_IDX                                                                      0
+#define regROM_SW_DATA_19                                                                               0x00fe
+#define regROM_SW_DATA_19_BASE_IDX                                                                      0
+#define regROM_SW_DATA_20                                                                               0x00ff
+#define regROM_SW_DATA_20_BASE_IDX                                                                      0
+#define regROM_SW_DATA_21                                                                               0x0100
+#define regROM_SW_DATA_21_BASE_IDX                                                                      0
+#define regROM_SW_DATA_22                                                                               0x0101
+#define regROM_SW_DATA_22_BASE_IDX                                                                      0
+#define regROM_SW_DATA_23                                                                               0x0102
+#define regROM_SW_DATA_23_BASE_IDX                                                                      0
+#define regROM_SW_DATA_24                                                                               0x0103
+#define regROM_SW_DATA_24_BASE_IDX                                                                      0
+#define regROM_SW_DATA_25                                                                               0x0104
+#define regROM_SW_DATA_25_BASE_IDX                                                                      0
+#define regROM_SW_DATA_26                                                                               0x0105
+#define regROM_SW_DATA_26_BASE_IDX                                                                      0
+#define regROM_SW_DATA_27                                                                               0x0106
+#define regROM_SW_DATA_27_BASE_IDX                                                                      0
+#define regROM_SW_DATA_28                                                                               0x0107
+#define regROM_SW_DATA_28_BASE_IDX                                                                      0
+#define regROM_SW_DATA_29                                                                               0x0108
+#define regROM_SW_DATA_29_BASE_IDX                                                                      0
+#define regROM_SW_DATA_30                                                                               0x0109
+#define regROM_SW_DATA_30_BASE_IDX                                                                      0
+#define regROM_SW_DATA_31                                                                               0x010a
+#define regROM_SW_DATA_31_BASE_IDX                                                                      0
+#define regROM_SW_DATA_32                                                                               0x010b
+#define regROM_SW_DATA_32_BASE_IDX                                                                      0
+#define regROM_SW_DATA_33                                                                               0x010c
+#define regROM_SW_DATA_33_BASE_IDX                                                                      0
+#define regROM_SW_DATA_34                                                                               0x010d
+#define regROM_SW_DATA_34_BASE_IDX                                                                      0
+#define regROM_SW_DATA_35                                                                               0x010e
+#define regROM_SW_DATA_35_BASE_IDX                                                                      0
+#define regROM_SW_DATA_36                                                                               0x010f
+#define regROM_SW_DATA_36_BASE_IDX                                                                      0
+#define regROM_SW_DATA_37                                                                               0x0110
+#define regROM_SW_DATA_37_BASE_IDX                                                                      0
+#define regROM_SW_DATA_38                                                                               0x0111
+#define regROM_SW_DATA_38_BASE_IDX                                                                      0
+#define regROM_SW_DATA_39                                                                               0x0112
+#define regROM_SW_DATA_39_BASE_IDX                                                                      0
+#define regROM_SW_DATA_40                                                                               0x0113
+#define regROM_SW_DATA_40_BASE_IDX                                                                      0
+#define regROM_SW_DATA_41                                                                               0x0114
+#define regROM_SW_DATA_41_BASE_IDX                                                                      0
+#define regROM_SW_DATA_42                                                                               0x0115
+#define regROM_SW_DATA_42_BASE_IDX                                                                      0
+#define regROM_SW_DATA_43                                                                               0x0116
+#define regROM_SW_DATA_43_BASE_IDX                                                                      0
+#define regROM_SW_DATA_44                                                                               0x0117
+#define regROM_SW_DATA_44_BASE_IDX                                                                      0
+#define regROM_SW_DATA_45                                                                               0x0118
+#define regROM_SW_DATA_45_BASE_IDX                                                                      0
+#define regROM_SW_DATA_46                                                                               0x0119
+#define regROM_SW_DATA_46_BASE_IDX                                                                      0
+#define regROM_SW_DATA_47                                                                               0x011a
+#define regROM_SW_DATA_47_BASE_IDX                                                                      0
+#define regROM_SW_DATA_48                                                                               0x011b
+#define regROM_SW_DATA_48_BASE_IDX                                                                      0
+#define regROM_SW_DATA_49                                                                               0x011c
+#define regROM_SW_DATA_49_BASE_IDX                                                                      0
+#define regROM_SW_DATA_50                                                                               0x011d
+#define regROM_SW_DATA_50_BASE_IDX                                                                      0
+#define regROM_SW_DATA_51                                                                               0x011e
+#define regROM_SW_DATA_51_BASE_IDX                                                                      0
+#define regROM_SW_DATA_52                                                                               0x011f
+#define regROM_SW_DATA_52_BASE_IDX                                                                      0
+#define regROM_SW_DATA_53                                                                               0x0120
+#define regROM_SW_DATA_53_BASE_IDX                                                                      0
+#define regROM_SW_DATA_54                                                                               0x0121
+#define regROM_SW_DATA_54_BASE_IDX                                                                      0
+#define regROM_SW_DATA_55                                                                               0x0122
+#define regROM_SW_DATA_55_BASE_IDX                                                                      0
+#define regROM_SW_DATA_56                                                                               0x0123
+#define regROM_SW_DATA_56_BASE_IDX                                                                      0
+#define regROM_SW_DATA_57                                                                               0x0124
+#define regROM_SW_DATA_57_BASE_IDX                                                                      0
+#define regROM_SW_DATA_58                                                                               0x0125
+#define regROM_SW_DATA_58_BASE_IDX                                                                      0
+#define regROM_SW_DATA_59                                                                               0x0126
+#define regROM_SW_DATA_59_BASE_IDX                                                                      0
+#define regROM_SW_DATA_60                                                                               0x0127
+#define regROM_SW_DATA_60_BASE_IDX                                                                      0
+#define regROM_SW_DATA_61                                                                               0x0128
+#define regROM_SW_DATA_61_BASE_IDX                                                                      0
+#define regROM_SW_DATA_62                                                                               0x0129
+#define regROM_SW_DATA_62_BASE_IDX                                                                      0
+#define regROM_SW_DATA_63                                                                               0x012a
+#define regROM_SW_DATA_63_BASE_IDX                                                                      0
+#define regROM_SW_DATA_64                                                                               0x012b
+#define regROM_SW_DATA_64_BASE_IDX                                                                      0
+
+
+// addressBlock: smuio_smuio_gpio_SmuSmuioDec
+// base address: 0x5a500
+#define regSMU_GPIOPAD_SW_INT_STAT                                                                      0x0140
+#define regSMU_GPIOPAD_SW_INT_STAT_BASE_IDX                                                             0
+#define regSMU_GPIOPAD_MASK                                                                             0x0141
+#define regSMU_GPIOPAD_MASK_BASE_IDX                                                                    0
+#define regSMU_GPIOPAD_A                                                                                0x0142
+#define regSMU_GPIOPAD_A_BASE_IDX                                                                       0
+#define regSMU_GPIOPAD_TXIMPSEL                                                                         0x0143
+#define regSMU_GPIOPAD_TXIMPSEL_BASE_IDX                                                                0
+#define regSMU_GPIOPAD_EN                                                                               0x0144
+#define regSMU_GPIOPAD_EN_BASE_IDX                                                                      0
+#define regSMU_GPIOPAD_Y                                                                                0x0145
+#define regSMU_GPIOPAD_Y_BASE_IDX                                                                       0
+#define regSMU_GPIOPAD_RXEN                                                                             0x0146
+#define regSMU_GPIOPAD_RXEN_BASE_IDX                                                                    0
+#define regSMU_GPIOPAD_RCVR_SEL0                                                                        0x0147
+#define regSMU_GPIOPAD_RCVR_SEL0_BASE_IDX                                                               0
+#define regSMU_GPIOPAD_RCVR_SEL1                                                                        0x0148
+#define regSMU_GPIOPAD_RCVR_SEL1_BASE_IDX                                                               0
+#define regSMU_GPIOPAD_PU_EN                                                                            0x0149
+#define regSMU_GPIOPAD_PU_EN_BASE_IDX                                                                   0
+#define regSMU_GPIOPAD_PD_EN                                                                            0x014a
+#define regSMU_GPIOPAD_PD_EN_BASE_IDX                                                                   0
+#define regSMU_GPIOPAD_PINSTRAPS                                                                        0x014b
+#define regSMU_GPIOPAD_PINSTRAPS_BASE_IDX                                                               0
+#define regDFT_PINSTRAPS                                                                                0x014c
+#define regDFT_PINSTRAPS_BASE_IDX                                                                       0
+#define regSMU_GPIOPAD_INT_STAT_EN                                                                      0x014d
+#define regSMU_GPIOPAD_INT_STAT_EN_BASE_IDX                                                             0
+#define regSMU_GPIOPAD_INT_STAT                                                                         0x014e
+#define regSMU_GPIOPAD_INT_STAT_BASE_IDX                                                                0
+#define regSMU_GPIOPAD_INT_STAT_AK                                                                      0x014f
+#define regSMU_GPIOPAD_INT_STAT_AK_BASE_IDX                                                             0
+#define regSMU_GPIOPAD_INT_EN                                                                           0x0150
+#define regSMU_GPIOPAD_INT_EN_BASE_IDX                                                                  0
+#define regSMU_GPIOPAD_INT_TYPE                                                                         0x0151
+#define regSMU_GPIOPAD_INT_TYPE_BASE_IDX                                                                0
+#define regSMU_GPIOPAD_INT_POLARITY                                                                     0x0152
+#define regSMU_GPIOPAD_INT_POLARITY_BASE_IDX                                                            0
+#define regSMUIO_PCC_GPIO_SELECT                                                                        0x0155
+#define regSMUIO_PCC_GPIO_SELECT_BASE_IDX                                                               0
+#define regSMU_GPIOPAD_S0                                                                               0x0156
+#define regSMU_GPIOPAD_S0_BASE_IDX                                                                      0
+#define regSMU_GPIOPAD_S1                                                                               0x0157
+#define regSMU_GPIOPAD_S1_BASE_IDX                                                                      0
+#define regSMU_GPIOPAD_SCHMEN                                                                           0x0158
+#define regSMU_GPIOPAD_SCHMEN_BASE_IDX                                                                  0
+#define regSMU_GPIOPAD_SCL_EN                                                                           0x0159
+#define regSMU_GPIOPAD_SCL_EN_BASE_IDX                                                                  0
+#define regSMU_GPIOPAD_SDA_EN                                                                           0x015a
+#define regSMU_GPIOPAD_SDA_EN_BASE_IDX                                                                  0
+#define regSMUIO_GPIO_INT0_SELECT                                                                       0x015b
+#define regSMUIO_GPIO_INT0_SELECT_BASE_IDX                                                              0
+#define regSMUIO_GPIO_INT1_SELECT                                                                       0x015c
+#define regSMUIO_GPIO_INT1_SELECT_BASE_IDX                                                              0
+#define regSMUIO_GPIO_INT2_SELECT                                                                       0x015d
+#define regSMUIO_GPIO_INT2_SELECT_BASE_IDX                                                              0
+#define regSMUIO_GPIO_INT3_SELECT                                                                       0x015e
+#define regSMUIO_GPIO_INT3_SELECT_BASE_IDX                                                              0
+#define regSMU_GPIOPAD_MP_INT0_STAT                                                                     0x015f
+#define regSMU_GPIOPAD_MP_INT0_STAT_BASE_IDX                                                            0
+#define regSMU_GPIOPAD_MP_INT1_STAT                                                                     0x0160
+#define regSMU_GPIOPAD_MP_INT1_STAT_BASE_IDX                                                            0
+#define regSMU_GPIOPAD_MP_INT2_STAT                                                                     0x0161
+#define regSMU_GPIOPAD_MP_INT2_STAT_BASE_IDX                                                            0
+#define regSMU_GPIOPAD_MP_INT3_STAT                                                                     0x0162
+#define regSMU_GPIOPAD_MP_INT3_STAT_BASE_IDX                                                            0
+#define regSMIO_INDEX                                                                                   0x0163
+#define regSMIO_INDEX_BASE_IDX                                                                          0
+#define regS0_VID_SMIO_CNTL                                                                             0x0164
+#define regS0_VID_SMIO_CNTL_BASE_IDX                                                                    0
+#define regS1_VID_SMIO_CNTL                                                                             0x0165
+#define regS1_VID_SMIO_CNTL_BASE_IDX                                                                    0
+#define regOPEN_DRAIN_SELECT                                                                            0x0166
+#define regOPEN_DRAIN_SELECT_BASE_IDX                                                                   0
+#define regSMIO_ENABLE                                                                                  0x0167
+#define regSMIO_ENABLE_BASE_IDX                                                                         0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_14_0_2_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_14_0_2_sh_mask.h
new file mode 100644
index 000000000000..6204505e553b
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_14_0_2_sh_mask.h
@@ -0,0 +1,1106 @@
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _smuio_14_0_2_SH_MASK_HEADER
+#define _smuio_14_0_2_SH_MASK_HEADER
+
+
+// addressBlock: smuio_smuio_tsc_SmuSmuioDec
+//PWROK_REFCLK_GAP_CYCLES
+#define PWROK_REFCLK_GAP_CYCLES__Pwrok_PreAssertion_clkgap_cycles__SHIFT                                      0x0
+#define PWROK_REFCLK_GAP_CYCLES__Pwrok_PostAssertion_clkgap_cycles__SHIFT                                     0x8
+#define PWROK_REFCLK_GAP_CYCLES__Pwrok_PreAssertion_clkgap_cycles_MASK                                        0x000000FFL
+#define PWROK_REFCLK_GAP_CYCLES__Pwrok_PostAssertion_clkgap_cycles_MASK                                       0x0000FF00L
+//GOLDEN_TSC_INCREMENT_UPPER
+#define GOLDEN_TSC_INCREMENT_UPPER__GoldenTscIncrementUpper__SHIFT                                            0x0
+#define GOLDEN_TSC_INCREMENT_UPPER__GoldenTscIncrementUpper_MASK                                              0x00FFFFFFL
+//GOLDEN_TSC_INCREMENT_LOWER
+#define GOLDEN_TSC_INCREMENT_LOWER__GoldenTscIncrementLower__SHIFT                                            0x0
+#define GOLDEN_TSC_INCREMENT_LOWER__GoldenTscIncrementLower_MASK                                              0xFFFFFFFFL
+//GOLDEN_TSC_COUNT_UPPER
+#define GOLDEN_TSC_COUNT_UPPER__GoldenTscCountUpper__SHIFT                                                    0x0
+#define GOLDEN_TSC_COUNT_UPPER__GoldenTscCountUpper_MASK                                                      0x00FFFFFFL
+//GOLDEN_TSC_COUNT_LOWER
+#define GOLDEN_TSC_COUNT_LOWER__GoldenTscCountLower__SHIFT                                                    0x0
+#define GOLDEN_TSC_COUNT_LOWER__GoldenTscCountLower_MASK                                                      0xFFFFFFFFL
+//SOC_GOLDEN_TSC_SHADOW_UPPER
+#define SOC_GOLDEN_TSC_SHADOW_UPPER__SocGoldenTscShadowUpper__SHIFT                                           0x0
+#define SOC_GOLDEN_TSC_SHADOW_UPPER__SocGoldenTscShadowUpper_MASK                                             0x00FFFFFFL
+//SOC_GOLDEN_TSC_SHADOW_LOWER
+#define SOC_GOLDEN_TSC_SHADOW_LOWER__SocGoldenTscShadowLower__SHIFT                                           0x0
+#define SOC_GOLDEN_TSC_SHADOW_LOWER__SocGoldenTscShadowLower_MASK                                             0xFFFFFFFFL
+//SOC_GAP_PWROK
+#define SOC_GAP_PWROK__soc_gap_pwrok__SHIFT                                                                   0x0
+#define SOC_GAP_PWROK__soc_gap_pwrok_MASK                                                                     0x00000001L
+
+
+// addressBlock: smuio_smuio_swtimer_SmuSmuioDec
+//PWR_VIRT_RESET_REQ
+#define PWR_VIRT_RESET_REQ__VF_FLR__SHIFT                                                                     0x0
+#define PWR_VIRT_RESET_REQ__PF_FLR__SHIFT                                                                     0x1f
+#define PWR_VIRT_RESET_REQ__VF_FLR_MASK                                                                       0x7FFFFFFFL
+#define PWR_VIRT_RESET_REQ__PF_FLR_MASK                                                                       0x80000000L
+//PWR_DISP_TIMER_CONTROL
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_COUNT__SHIFT                                                   0x0
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_ENABLE__SHIFT                                                  0x19
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_DISABLE__SHIFT                                                 0x1a
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_MASK__SHIFT                                                    0x1b
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_STAT_AK__SHIFT                                                 0x1c
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_TYPE__SHIFT                                                    0x1d
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_MODE__SHIFT                                                    0x1e
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_COUNT_MASK                                                     0x01FFFFFFL
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_ENABLE_MASK                                                    0x02000000L
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_DISABLE_MASK                                                   0x04000000L
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_MASK_MASK                                                      0x08000000L
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_STAT_AK_MASK                                                   0x10000000L
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_TYPE_MASK                                                      0x20000000L
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_MODE_MASK                                                      0x40000000L
+//PWR_DISP_TIMER_DEBUG
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_RUNNING__SHIFT                                                   0x0
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_STAT__SHIFT                                                      0x1
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT__SHIFT                                                           0x2
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_RUN_VAL__SHIFT                                                       0x7
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_RUNNING_MASK                                                     0x00000001L
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_STAT_MASK                                                        0x00000002L
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_MASK                                                             0x00000004L
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_RUN_VAL_MASK                                                         0xFFFFFF80L
+//PWR_DISP_TIMER2_CONTROL
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_COUNT__SHIFT                                                  0x0
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_ENABLE__SHIFT                                                 0x19
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_DISABLE__SHIFT                                                0x1a
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_MASK__SHIFT                                                   0x1b
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_STAT_AK__SHIFT                                                0x1c
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_TYPE__SHIFT                                                   0x1d
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_MODE__SHIFT                                                   0x1e
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_COUNT_MASK                                                    0x01FFFFFFL
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_ENABLE_MASK                                                   0x02000000L
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_DISABLE_MASK                                                  0x04000000L
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_MASK_MASK                                                     0x08000000L
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_STAT_AK_MASK                                                  0x10000000L
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_TYPE_MASK                                                     0x20000000L
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_MODE_MASK                                                     0x40000000L
+//PWR_DISP_TIMER2_DEBUG
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_RUNNING__SHIFT                                                  0x0
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_STAT__SHIFT                                                     0x1
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT__SHIFT                                                          0x2
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_RUN_VAL__SHIFT                                                      0x7
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_RUNNING_MASK                                                    0x00000001L
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_STAT_MASK                                                       0x00000002L
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_MASK                                                            0x00000004L
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_RUN_VAL_MASK                                                        0xFFFFFF80L
+//PWR_DISP_TIMER_GLOBAL_CONTROL
+#define PWR_DISP_TIMER_GLOBAL_CONTROL__DISP_TIMER_PULSE_WIDTH__SHIFT                                          0x0
+#define PWR_DISP_TIMER_GLOBAL_CONTROL__DISP_TIMER_PULSE_EN__SHIFT                                             0xa
+#define PWR_DISP_TIMER_GLOBAL_CONTROL__DISP_TIMER_PULSE_WIDTH_MASK                                            0x000003FFL
+#define PWR_DISP_TIMER_GLOBAL_CONTROL__DISP_TIMER_PULSE_EN_MASK                                               0x00000400L
+//PWR_IH_CONTROL
+#define PWR_IH_CONTROL__MAX_CREDIT__SHIFT                                                                     0x0
+#define PWR_IH_CONTROL__DISP_TIMER_TRIGGER_MASK__SHIFT                                                        0x5
+#define PWR_IH_CONTROL__DISP_TIMER2_TRIGGER_MASK__SHIFT                                                       0x6
+#define PWR_IH_CONTROL__PWR_IH_CLK_GATE_EN__SHIFT                                                             0x1f
+#define PWR_IH_CONTROL__MAX_CREDIT_MASK                                                                       0x0000001FL
+#define PWR_IH_CONTROL__DISP_TIMER_TRIGGER_MASK_MASK                                                          0x00000020L
+#define PWR_IH_CONTROL__DISP_TIMER2_TRIGGER_MASK_MASK                                                         0x00000040L
+#define PWR_IH_CONTROL__PWR_IH_CLK_GATE_EN_MASK                                                               0x80000000L
+
+
+// addressBlock: smuio_smuio_misc_SmuSmuioDec
+//SMUIO_MCM_CONFIG
+#define SMUIO_MCM_CONFIG__DIE_ID__SHIFT                                                                       0x0
+#define SMUIO_MCM_CONFIG__PKG_TYPE__SHIFT                                                                     0x2
+#define SMUIO_MCM_CONFIG__SOCKET_ID__SHIFT                                                                    0x8
+#define SMUIO_MCM_CONFIG__PKG_SUBTYPE__SHIFT                                                                  0xc
+#define SMUIO_MCM_CONFIG__DIE_CONFIG__SHIFT                                                                   0xd
+#define SMUIO_MCM_CONFIG__CONSOLE_K__SHIFT                                                                    0x10
+#define SMUIO_MCM_CONFIG__CONSOLE_A__SHIFT                                                                    0x11
+#define SMUIO_MCM_CONFIG__DIE_ID_MASK                                                                         0x00000003L
+#define SMUIO_MCM_CONFIG__PKG_TYPE_MASK                                                                       0x0000001CL
+#define SMUIO_MCM_CONFIG__SOCKET_ID_MASK                                                                      0x00000300L
+#define SMUIO_MCM_CONFIG__PKG_SUBTYPE_MASK                                                                    0x00001000L
+#define SMUIO_MCM_CONFIG__CONSOLE_K_MASK                                                                      0x00010000L
+#define SMUIO_MCM_CONFIG__CONSOLE_A_MASK                                                                      0x00020000L
+//IP_DISCOVERY_VERSION
+#define IP_DISCOVERY_VERSION__IP_DISCOVERY_VERSION__SHIFT                                                     0x0
+#define IP_DISCOVERY_VERSION__IP_DISCOVERY_VERSION_MASK                                                       0xFFFFFFFFL
+//SCRATCH_REGISTER0
+#define SCRATCH_REGISTER0__ScratchPad0__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER0__ScratchPad0_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER1
+#define SCRATCH_REGISTER1__ScratchPad1__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER1__ScratchPad1_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER2
+#define SCRATCH_REGISTER2__ScratchPad2__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER2__ScratchPad2_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER3
+#define SCRATCH_REGISTER3__ScratchPad3__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER3__ScratchPad3_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER4
+#define SCRATCH_REGISTER4__ScratchPad4__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER4__ScratchPad4_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER5
+#define SCRATCH_REGISTER5__ScratchPad5__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER5__ScratchPad5_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER6
+#define SCRATCH_REGISTER6__ScratchPad6__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER6__ScratchPad6_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER7
+#define SCRATCH_REGISTER7__ScratchPad7__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER7__ScratchPad7_MASK                                                                   0xFFFFFFFFL
+
+
+// addressBlock: smuio_smuio_i2c_SmuSmuioDec
+//CKSVII2C_IC_CON
+#define CKSVII2C_IC_CON__IC_MASTER_MODE__SHIFT                                                                0x0
+#define CKSVII2C_IC_CON__IC_MAX_SPEED_MODE__SHIFT                                                             0x1
+#define CKSVII2C_IC_CON__IC_10BITADDR_SLAVE__SHIFT                                                            0x3
+#define CKSVII2C_IC_CON__IC_10BITADDR_MASTER__SHIFT                                                           0x4
+#define CKSVII2C_IC_CON__IC_RESTART_EN__SHIFT                                                                 0x5
+#define CKSVII2C_IC_CON__IC_SLAVE_DISABLE__SHIFT                                                              0x6
+#define CKSVII2C_IC_CON__STOP_DET_IFADDRESSED__SHIFT                                                          0x7
+#define CKSVII2C_IC_CON__TX_EMPTY_CTRL__SHIFT                                                                 0x8
+#define CKSVII2C_IC_CON__RX_FIFO_FULL_HLD_CTRL__SHIFT                                                         0x9
+#define CKSVII2C_IC_CON__BUS_CLEAR_FEATURE_CTRL__SHIFT                                                        0xb
+#define CKSVII2C_IC_CON__IC_MASTER_MODE_MASK                                                                  0x00000001L
+#define CKSVII2C_IC_CON__IC_MAX_SPEED_MODE_MASK                                                               0x00000006L
+#define CKSVII2C_IC_CON__IC_10BITADDR_SLAVE_MASK                                                              0x00000008L
+#define CKSVII2C_IC_CON__IC_10BITADDR_MASTER_MASK                                                             0x00000010L
+#define CKSVII2C_IC_CON__IC_RESTART_EN_MASK                                                                   0x00000020L
+#define CKSVII2C_IC_CON__IC_SLAVE_DISABLE_MASK                                                                0x00000040L
+#define CKSVII2C_IC_CON__STOP_DET_IFADDRESSED_MASK                                                            0x00000080L
+#define CKSVII2C_IC_CON__TX_EMPTY_CTRL_MASK                                                                   0x00000100L
+#define CKSVII2C_IC_CON__RX_FIFO_FULL_HLD_CTRL_MASK                                                           0x00000200L
+//CKSVII2C_IC_TAR
+#define CKSVII2C_IC_TAR__IC_TAR__SHIFT                                                                        0x0
+#define CKSVII2C_IC_TAR__GC_OR_START__SHIFT                                                                   0xa
+#define CKSVII2C_IC_TAR__SPECIAL__SHIFT                                                                       0xb
+#define CKSVII2C_IC_TAR__IC_10BITADDR_MASTER__SHIFT                                                           0xc
+#define CKSVII2C_IC_TAR__IC_TAR_MASK                                                                          0x000003FFL
+#define CKSVII2C_IC_TAR__GC_OR_START_MASK                                                                     0x00000400L
+#define CKSVII2C_IC_TAR__SPECIAL_MASK                                                                         0x00000800L
+#define CKSVII2C_IC_TAR__IC_10BITADDR_MASTER_MASK                                                             0x00001000L
+//CKSVII2C_IC_SAR
+#define CKSVII2C_IC_SAR__IC_SAR__SHIFT                                                                        0x0
+#define CKSVII2C_IC_SAR__IC_SAR_MASK                                                                          0x000003FFL
+//CKSVII2C_IC_HS_MADDR
+#define CKSVII2C_IC_HS_MADDR__IC_HS_MADDR__SHIFT                                                              0x0
+#define CKSVII2C_IC_HS_MADDR__IC_HS_MADDR_MASK                                                                0x00000007L
+//CKSVII2C_IC_DATA_CMD
+#define CKSVII2C_IC_DATA_CMD__DAT__SHIFT                                                                      0x0
+#define CKSVII2C_IC_DATA_CMD__CMD__SHIFT                                                                      0x8
+#define CKSVII2C_IC_DATA_CMD__STOP__SHIFT                                                                     0x9
+#define CKSVII2C_IC_DATA_CMD__RESTART__SHIFT                                                                  0xa
+#define CKSVII2C_IC_DATA_CMD__FIRST_DATA_BYTE__SHIFT                                                          0xb
+#define CKSVII2C_IC_DATA_CMD__DAT_MASK                                                                        0x000000FFL
+#define CKSVII2C_IC_DATA_CMD__CMD_MASK                                                                        0x00000100L
+#define CKSVII2C_IC_DATA_CMD__STOP_MASK                                                                       0x00000200L
+#define CKSVII2C_IC_DATA_CMD__RESTART_MASK                                                                    0x00000400L
+//CKSVII2C_IC_SS_SCL_HCNT
+#define CKSVII2C_IC_SS_SCL_HCNT__IC_SS_SCL_HCNT__SHIFT                                                        0x0
+#define CKSVII2C_IC_SS_SCL_HCNT__IC_SS_SCL_HCNT_MASK                                                          0x0000FFFFL
+//CKSVII2C_IC_SS_SCL_LCNT
+#define CKSVII2C_IC_SS_SCL_LCNT__IC_SS_SCL_LCNT__SHIFT                                                        0x0
+#define CKSVII2C_IC_SS_SCL_LCNT__IC_SS_SCL_LCNT_MASK                                                          0x0000FFFFL
+//CKSVII2C_IC_FS_SCL_HCNT
+#define CKSVII2C_IC_FS_SCL_HCNT__IC_FS_SCL_HCNT__SHIFT                                                        0x0
+#define CKSVII2C_IC_FS_SCL_HCNT__IC_FS_SCL_HCNT_MASK                                                          0x0000FFFFL
+//CKSVII2C_IC_FS_SCL_LCNT
+#define CKSVII2C_IC_FS_SCL_LCNT__IC_FS_SCL_LCNT__SHIFT                                                        0x0
+#define CKSVII2C_IC_FS_SCL_LCNT__IC_FS_SCL_LCNT_MASK                                                          0x0000FFFFL
+//CKSVII2C_IC_HS_SCL_HCNT
+#define CKSVII2C_IC_HS_SCL_HCNT__IC_HS_SCL_HCNT__SHIFT                                                        0x0
+#define CKSVII2C_IC_HS_SCL_HCNT__IC_HS_SCL_HCNT_MASK                                                          0x0000FFFFL
+//CKSVII2C_IC_HS_SCL_LCNT
+#define CKSVII2C_IC_HS_SCL_LCNT__IC_HS_SCL_LCNT__SHIFT                                                        0x0
+#define CKSVII2C_IC_HS_SCL_LCNT__IC_HS_SCL_LCNT_MASK                                                          0x0000FFFFL
+//CKSVII2C_IC_INTR_STAT
+#define CKSVII2C_IC_INTR_STAT__R_RX_UNDER__SHIFT                                                              0x0
+#define CKSVII2C_IC_INTR_STAT__R_RX_OVER__SHIFT                                                               0x1
+#define CKSVII2C_IC_INTR_STAT__R_RX_FULL__SHIFT                                                               0x2
+#define CKSVII2C_IC_INTR_STAT__R_TX_OVER__SHIFT                                                               0x3
+#define CKSVII2C_IC_INTR_STAT__R_TX_EMPTY__SHIFT                                                              0x4
+#define CKSVII2C_IC_INTR_STAT__R_RD_REQ__SHIFT                                                                0x5
+#define CKSVII2C_IC_INTR_STAT__R_TX_ABRT__SHIFT                                                               0x6
+#define CKSVII2C_IC_INTR_STAT__R_RX_DONE__SHIFT                                                               0x7
+#define CKSVII2C_IC_INTR_STAT__R_ACTIVITY__SHIFT                                                              0x8
+#define CKSVII2C_IC_INTR_STAT__R_STOP_DET__SHIFT                                                              0x9
+#define CKSVII2C_IC_INTR_STAT__R_START_DET__SHIFT                                                             0xa
+#define CKSVII2C_IC_INTR_STAT__R_GEN_CALL__SHIFT                                                              0xb
+#define CKSVII2C_IC_INTR_STAT__R_RESTART_DET__SHIFT                                                           0xc
+#define CKSVII2C_IC_INTR_STAT__R_MST_ON_HOLD__SHIFT                                                           0xd
+#define CKSVII2C_IC_INTR_STAT__R_SCL_STUCK_AT_LOW__SHIFT                                                      0xe
+#define CKSVII2C_IC_INTR_STAT__R_RX_UNDER_MASK                                                                0x00000001L
+#define CKSVII2C_IC_INTR_STAT__R_RX_OVER_MASK                                                                 0x00000002L
+#define CKSVII2C_IC_INTR_STAT__R_RX_FULL_MASK                                                                 0x00000004L
+#define CKSVII2C_IC_INTR_STAT__R_TX_OVER_MASK                                                                 0x00000008L
+#define CKSVII2C_IC_INTR_STAT__R_TX_EMPTY_MASK                                                                0x00000010L
+#define CKSVII2C_IC_INTR_STAT__R_RD_REQ_MASK                                                                  0x00000020L
+#define CKSVII2C_IC_INTR_STAT__R_TX_ABRT_MASK                                                                 0x00000040L
+#define CKSVII2C_IC_INTR_STAT__R_RX_DONE_MASK                                                                 0x00000080L
+#define CKSVII2C_IC_INTR_STAT__R_ACTIVITY_MASK                                                                0x00000100L
+#define CKSVII2C_IC_INTR_STAT__R_STOP_DET_MASK                                                                0x00000200L
+#define CKSVII2C_IC_INTR_STAT__R_START_DET_MASK                                                               0x00000400L
+#define CKSVII2C_IC_INTR_STAT__R_GEN_CALL_MASK                                                                0x00000800L
+#define CKSVII2C_IC_INTR_STAT__R_RESTART_DET_MASK                                                             0x00001000L
+#define CKSVII2C_IC_INTR_STAT__R_MST_ON_HOLD_MASK                                                             0x00002000L
+//CKSVII2C_IC_INTR_MASK
+#define CKSVII2C_IC_INTR_MASK__M_RX_UNDER__SHIFT                                                              0x0
+#define CKSVII2C_IC_INTR_MASK__M_RX_OVER__SHIFT                                                               0x1
+#define CKSVII2C_IC_INTR_MASK__M_RX_FULL__SHIFT                                                               0x2
+#define CKSVII2C_IC_INTR_MASK__M_TX_OVER__SHIFT                                                               0x3
+#define CKSVII2C_IC_INTR_MASK__M_TX_EMPTY__SHIFT                                                              0x4
+#define CKSVII2C_IC_INTR_MASK__M_RD_REQ__SHIFT                                                                0x5
+#define CKSVII2C_IC_INTR_MASK__M_TX_ABRT__SHIFT                                                               0x6
+#define CKSVII2C_IC_INTR_MASK__M_RX_DONE__SHIFT                                                               0x7
+#define CKSVII2C_IC_INTR_MASK__M_ACTIVITY__SHIFT                                                              0x8
+#define CKSVII2C_IC_INTR_MASK__M_STOP_DET__SHIFT                                                              0x9
+#define CKSVII2C_IC_INTR_MASK__M_START_DET__SHIFT                                                             0xa
+#define CKSVII2C_IC_INTR_MASK__M_GEN_CALL__SHIFT                                                              0xb
+#define CKSVII2C_IC_INTR_MASK__M_RESTART_DET__SHIFT                                                           0xc
+#define CKSVII2C_IC_INTR_MASK__M_MST_ON_HOLD__SHIFT                                                           0xd
+#define CKSVII2C_IC_INTR_MASK__M_SCL_STUCK_AT_LOW__SHIFT                                                      0xe
+#define CKSVII2C_IC_INTR_MASK__M_RX_UNDER_MASK                                                                0x00000001L
+#define CKSVII2C_IC_INTR_MASK__M_RX_OVER_MASK                                                                 0x00000002L
+#define CKSVII2C_IC_INTR_MASK__M_RX_FULL_MASK                                                                 0x00000004L
+#define CKSVII2C_IC_INTR_MASK__M_TX_OVER_MASK                                                                 0x00000008L
+#define CKSVII2C_IC_INTR_MASK__M_TX_EMPTY_MASK                                                                0x00000010L
+#define CKSVII2C_IC_INTR_MASK__M_RD_REQ_MASK                                                                  0x00000020L
+#define CKSVII2C_IC_INTR_MASK__M_TX_ABRT_MASK                                                                 0x00000040L
+#define CKSVII2C_IC_INTR_MASK__M_RX_DONE_MASK                                                                 0x00000080L
+#define CKSVII2C_IC_INTR_MASK__M_ACTIVITY_MASK                                                                0x00000100L
+#define CKSVII2C_IC_INTR_MASK__M_STOP_DET_MASK                                                                0x00000200L
+#define CKSVII2C_IC_INTR_MASK__M_START_DET_MASK                                                               0x00000400L
+#define CKSVII2C_IC_INTR_MASK__M_GEN_CALL_MASK                                                                0x00000800L
+#define CKSVII2C_IC_INTR_MASK__M_RESTART_DET_MASK                                                             0x00001000L
+#define CKSVII2C_IC_INTR_MASK__M_MST_ON_HOLD_MASK                                                             0x00002000L
+//CKSVII2C_IC_RAW_INTR_STAT
+//CKSVII2C_IC_RX_TL
+#define CKSVII2C_IC_RX_TL__RX_TL__SHIFT                                                                       0x0
+//CKSVII2C_IC_TX_TL
+#define CKSVII2C_IC_TX_TL__TX_TL__SHIFT                                                                       0x0
+//CKSVII2C_IC_CLR_INTR
+//CKSVII2C_IC_CLR_RX_UNDER
+//CKSVII2C_IC_CLR_RX_OVER
+//CKSVII2C_IC_CLR_TX_OVER
+//CKSVII2C_IC_CLR_RD_REQ
+//CKSVII2C_IC_CLR_TX_ABRT
+//CKSVII2C_IC_CLR_RX_DONE
+//CKSVII2C_IC_CLR_ACTIVITY
+//CKSVII2C_IC_CLR_STOP_DET
+//CKSVII2C_IC_CLR_START_DET
+//CKSVII2C_IC_CLR_GEN_CALL
+//CKSVII2C_IC_ENABLE
+#define CKSVII2C_IC_ENABLE__ENABLE__SHIFT                                                                     0x0
+#define CKSVII2C_IC_ENABLE__ABORT__SHIFT                                                                      0x1
+#define CKSVII2C_IC_ENABLE__TX_CMD_BLOCK__SHIFT                                                               0x2
+#define CKSVII2C_IC_ENABLE__SDA_STUCK_RECOVERY_ENABLE__SHIFT                                                  0x3
+#define CKSVII2C_IC_ENABLE__ENABLE_MASK                                                                       0x00000001L
+#define CKSVII2C_IC_ENABLE__ABORT_MASK                                                                        0x00000002L
+//CKSVII2C_IC_STATUS
+#define CKSVII2C_IC_STATUS__ACTIVITY__SHIFT                                                                   0x0
+#define CKSVII2C_IC_STATUS__TFNF__SHIFT                                                                       0x1
+#define CKSVII2C_IC_STATUS__TFE__SHIFT                                                                        0x2
+#define CKSVII2C_IC_STATUS__RFNE__SHIFT                                                                       0x3
+#define CKSVII2C_IC_STATUS__RFF__SHIFT                                                                        0x4
+#define CKSVII2C_IC_STATUS__MST_ACTIVITY__SHIFT                                                               0x5
+#define CKSVII2C_IC_STATUS__SLV_ACTIVITY__SHIFT                                                               0x6
+#define CKSVII2C_IC_STATUS__MST_HOLD_TX_FIFO_EMPTY__SHIFT                                                     0x7
+#define CKSVII2C_IC_STATUS__MST_HOLD_RX_FIFO_FULL__SHIFT                                                      0x8
+#define CKSVII2C_IC_STATUS__SLV_HOLD_TX_FIFO_EMPTY__SHIFT                                                     0x9
+#define CKSVII2C_IC_STATUS__SLV_HOLD_RX_FIFO_FULL__SHIFT                                                      0xa
+#define CKSVII2C_IC_STATUS__SDA_STUCK_NOT_RECOVERED__SHIFT                                                    0xb
+#define CKSVII2C_IC_STATUS__ACTIVITY_MASK                                                                     0x00000001L
+#define CKSVII2C_IC_STATUS__TFNF_MASK                                                                         0x00000002L
+#define CKSVII2C_IC_STATUS__TFE_MASK                                                                          0x00000004L
+#define CKSVII2C_IC_STATUS__RFNE_MASK                                                                         0x00000008L
+#define CKSVII2C_IC_STATUS__RFF_MASK                                                                          0x00000010L
+#define CKSVII2C_IC_STATUS__MST_ACTIVITY_MASK                                                                 0x00000020L
+#define CKSVII2C_IC_STATUS__SLV_ACTIVITY_MASK                                                                 0x00000040L
+//CKSVII2C_IC_TXFLR
+#define CKSVII2C_IC_TXFLR__TXFLR__SHIFT                                                                       0x0
+//CKSVII2C_IC_RXFLR
+#define CKSVII2C_IC_RXFLR__RXFLR__SHIFT                                                                       0x0
+//CKSVII2C_IC_SDA_HOLD
+#define CKSVII2C_IC_SDA_HOLD__IC_SDA_TX_HOLD__SHIFT                                                           0x0
+#define CKSVII2C_IC_SDA_HOLD__IC_SDA_RX_HOLD__SHIFT                                                           0x10
+//CKSVII2C_IC_TX_ABRT_SOURCE
+//CKSVII2C_IC_SLV_DATA_NACK_ONLY
+//CKSVII2C_IC_DMA_CR
+//CKSVII2C_IC_DMA_TDLR
+//CKSVII2C_IC_DMA_RDLR
+//CKSVII2C_IC_SDA_SETUP
+#define CKSVII2C_IC_SDA_SETUP__SDA_SETUP__SHIFT                                                               0x0
+#define CKSVII2C_IC_SDA_SETUP__SDA_SETUP_MASK                                                                 0x000000FFL
+//CKSVII2C_IC_ACK_GENERAL_CALL
+#define CKSVII2C_IC_ACK_GENERAL_CALL__ACK_GENERAL_CALL__SHIFT                                                 0x0
+#define CKSVII2C_IC_ACK_GENERAL_CALL__ACK_GENERAL_CALL_MASK                                                   0x00000001L
+//CKSVII2C_IC_ENABLE_STATUS
+#define CKSVII2C_IC_ENABLE_STATUS__IC_EN__SHIFT                                                               0x0
+#define CKSVII2C_IC_ENABLE_STATUS__SLV_DISABLED_WHILE_BUSY__SHIFT                                             0x1
+#define CKSVII2C_IC_ENABLE_STATUS__SLV_RX_DATA_LOST__SHIFT                                                    0x2
+#define CKSVII2C_IC_ENABLE_STATUS__IC_EN_MASK                                                                 0x00000001L
+//CKSVII2C_IC_FS_SPKLEN
+#define CKSVII2C_IC_FS_SPKLEN__FS_SPKLEN__SHIFT                                                               0x0
+#define CKSVII2C_IC_FS_SPKLEN__FS_SPKLEN_MASK                                                                 0x000000FFL
+//CKSVII2C_IC_HS_SPKLEN
+#define CKSVII2C_IC_HS_SPKLEN__HS_SPKLEN__SHIFT                                                               0x0
+#define CKSVII2C_IC_HS_SPKLEN__HS_SPKLEN_MASK                                                                 0x000000FFL
+//CKSVII2C_IC_CLR_RESTART_DET
+//CKSVII2C_IC_COMP_PARAM_1
+#define CKSVII2C_IC_COMP_PARAM_1__APB_DATA_WIDTH__SHIFT                                                       0x0
+#define CKSVII2C_IC_COMP_PARAM_1__MAX_SPEED_MODE__SHIFT                                                       0x2
+#define CKSVII2C_IC_COMP_PARAM_1__HC_COUNT_VALUES__SHIFT                                                      0x4
+#define CKSVII2C_IC_COMP_PARAM_1__INTR_IO__SHIFT                                                              0x5
+#define CKSVII2C_IC_COMP_PARAM_1__HAS_DMA__SHIFT                                                              0x6
+#define CKSVII2C_IC_COMP_PARAM_1__ADD_ENCODED_PARAMS__SHIFT                                                   0x7
+#define CKSVII2C_IC_COMP_PARAM_1__RX_BUFFER_DEPTH__SHIFT                                                      0x8
+#define CKSVII2C_IC_COMP_PARAM_1__TX_BUFFER_DEPTH__SHIFT                                                      0x10
+//CKSVII2C_IC_COMP_VERSION
+#define CKSVII2C_IC_COMP_VERSION__COMP_VERSION__SHIFT                                                         0x0
+#define CKSVII2C_IC_COMP_VERSION__COMP_VERSION_MASK                                                           0xFFFFFFFFL
+//CKSVII2C_IC_COMP_TYPE
+#define CKSVII2C_IC_COMP_TYPE__COMP_TYPE__SHIFT                                                               0x0
+#define CKSVII2C_IC_COMP_TYPE__COMP_TYPE_MASK                                                                 0xFFFFFFFFL
+//CKSVII2C1_IC_CON
+#define CKSVII2C1_IC_CON__IC1_MASTER_MODE__SHIFT                                                              0x0
+#define CKSVII2C1_IC_CON__IC1_MAX_SPEED_MODE__SHIFT                                                           0x1
+#define CKSVII2C1_IC_CON__IC1_10BITADDR_SLAVE__SHIFT                                                          0x3
+#define CKSVII2C1_IC_CON__IC1_10BITADDR_MASTER__SHIFT                                                         0x4
+#define CKSVII2C1_IC_CON__IC1_RESTART_EN__SHIFT                                                               0x5
+#define CKSVII2C1_IC_CON__IC1_SLAVE_DISABLE__SHIFT                                                            0x6
+#define CKSVII2C1_IC_CON__STOP1_DET_IFADDRESSED__SHIFT                                                        0x7
+#define CKSVII2C1_IC_CON__TX1_EMPTY_CTRL__SHIFT                                                               0x8
+#define CKSVII2C1_IC_CON__RX1_FIFO_FULL_HLD_CTRL__SHIFT                                                       0x9
+#define CKSVII2C1_IC_CON__BUS_CLEAR_FEATURE_CTRL1__SHIFT                                                      0xb
+#define CKSVII2C1_IC_CON__IC1_MASTER_MODE_MASK                                                                0x00000001L
+#define CKSVII2C1_IC_CON__IC1_MAX_SPEED_MODE_MASK                                                             0x00000006L
+#define CKSVII2C1_IC_CON__IC1_10BITADDR_SLAVE_MASK                                                            0x00000008L
+#define CKSVII2C1_IC_CON__IC1_10BITADDR_MASTER_MASK                                                           0x00000010L
+#define CKSVII2C1_IC_CON__IC1_RESTART_EN_MASK                                                                 0x00000020L
+#define CKSVII2C1_IC_CON__IC1_SLAVE_DISABLE_MASK                                                              0x00000040L
+#define CKSVII2C1_IC_CON__STOP1_DET_IFADDRESSED_MASK                                                          0x00000080L
+#define CKSVII2C1_IC_CON__TX1_EMPTY_CTRL_MASK                                                                 0x00000100L
+#define CKSVII2C1_IC_CON__RX1_FIFO_FULL_HLD_CTRL_MASK                                                         0x00000200L
+//CKSVII2C1_IC_TAR
+#define CKSVII2C1_IC_TAR__IC1_TAR__SHIFT                                                                      0x0
+#define CKSVII2C1_IC_TAR__GC1_OR_START__SHIFT                                                                 0xa
+#define CKSVII2C1_IC_TAR__SPECIAL1__SHIFT                                                                     0xb
+#define CKSVII2C1_IC_TAR__IC1_10BITADDR_MASTER__SHIFT                                                         0xc
+#define CKSVII2C1_IC_TAR__IC1_TAR_MASK                                                                        0x000003FFL
+#define CKSVII2C1_IC_TAR__GC1_OR_START_MASK                                                                   0x00000400L
+#define CKSVII2C1_IC_TAR__SPECIAL1_MASK                                                                       0x00000800L
+#define CKSVII2C1_IC_TAR__IC1_10BITADDR_MASTER_MASK                                                           0x00001000L
+//CKSVII2C1_IC_SAR
+#define CKSVII2C1_IC_SAR__IC1_SAR__SHIFT                                                                      0x0
+#define CKSVII2C1_IC_SAR__IC1_SAR_MASK                                                                        0x000003FFL
+//CKSVII2C1_IC_HS_MADDR
+#define CKSVII2C1_IC_HS_MADDR__IC1_HS_MADDR__SHIFT                                                            0x0
+#define CKSVII2C1_IC_HS_MADDR__IC1_HS_MADDR_MASK                                                              0x00000007L
+//CKSVII2C1_IC_DATA_CMD
+#define CKSVII2C1_IC_DATA_CMD__DAT1__SHIFT                                                                    0x0
+#define CKSVII2C1_IC_DATA_CMD__CMD1__SHIFT                                                                    0x8
+#define CKSVII2C1_IC_DATA_CMD__STOP1__SHIFT                                                                   0x9
+#define CKSVII2C1_IC_DATA_CMD__RESTART1__SHIFT                                                                0xa
+#define CKSVII2C1_IC_DATA_CMD__FIRST1_DATA_BYTE__SHIFT                                                        0xb
+#define CKSVII2C1_IC_DATA_CMD__DAT1_MASK                                                                      0x000000FFL
+#define CKSVII2C1_IC_DATA_CMD__CMD1_MASK                                                                      0x00000100L
+#define CKSVII2C1_IC_DATA_CMD__STOP1_MASK                                                                     0x00000200L
+#define CKSVII2C1_IC_DATA_CMD__RESTART1_MASK                                                                  0x00000400L
+//CKSVII2C1_IC_SS_SCL_HCNT
+#define CKSVII2C1_IC_SS_SCL_HCNT__IC1_SS_SCL_HCNT__SHIFT                                                      0x0
+#define CKSVII2C1_IC_SS_SCL_HCNT__IC1_SS_SCL_HCNT_MASK                                                        0x0000FFFFL
+//CKSVII2C1_IC_SS_SCL_LCNT
+#define CKSVII2C1_IC_SS_SCL_LCNT__IC1_SS_SCL_LCNT__SHIFT                                                      0x0
+#define CKSVII2C1_IC_SS_SCL_LCNT__IC1_SS_SCL_LCNT_MASK                                                        0x0000FFFFL
+//CKSVII2C1_IC_FS_SCL_HCNT
+#define CKSVII2C1_IC_FS_SCL_HCNT__IC1_FS_SCL_HCNT__SHIFT                                                      0x0
+#define CKSVII2C1_IC_FS_SCL_HCNT__IC1_FS_SCL_HCNT_MASK                                                        0x0000FFFFL
+//CKSVII2C1_IC_FS_SCL_LCNT
+#define CKSVII2C1_IC_FS_SCL_LCNT__IC1_FS_SCL_LCNT__SHIFT                                                      0x0
+#define CKSVII2C1_IC_FS_SCL_LCNT__IC1_FS_SCL_LCNT_MASK                                                        0x0000FFFFL
+//CKSVII2C1_IC_HS_SCL_HCNT
+#define CKSVII2C1_IC_HS_SCL_HCNT__IC1_HS_SCL_HCNT__SHIFT                                                      0x0
+#define CKSVII2C1_IC_HS_SCL_HCNT__IC1_HS_SCL_HCNT_MASK                                                        0x0000FFFFL
+//CKSVII2C1_IC_HS_SCL_LCNT
+#define CKSVII2C1_IC_HS_SCL_LCNT__IC1_HS_SCL_LCNT__SHIFT                                                      0x0
+#define CKSVII2C1_IC_HS_SCL_LCNT__IC1_HS_SCL_LCNT_MASK                                                        0x0000FFFFL
+//CKSVII2C1_IC_INTR_STAT
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_UNDER__SHIFT                                                            0x0
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_OVER__SHIFT                                                             0x1
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_FULL__SHIFT                                                             0x2
+#define CKSVII2C1_IC_INTR_STAT__R1_TX_OVER__SHIFT                                                             0x3
+#define CKSVII2C1_IC_INTR_STAT__R1_TX_EMPTY__SHIFT                                                            0x4
+#define CKSVII2C1_IC_INTR_STAT__R1_RD_REQ__SHIFT                                                              0x5
+#define CKSVII2C1_IC_INTR_STAT__R1_TX_ABRT__SHIFT                                                             0x6
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_DONE__SHIFT                                                             0x7
+#define CKSVII2C1_IC_INTR_STAT__R1_ACTIVITY__SHIFT                                                            0x8
+#define CKSVII2C1_IC_INTR_STAT__R1_STOP_DET__SHIFT                                                            0x9
+#define CKSVII2C1_IC_INTR_STAT__R1_START_DET__SHIFT                                                           0xa
+#define CKSVII2C1_IC_INTR_STAT__R1_GEN_CALL__SHIFT                                                            0xb
+#define CKSVII2C1_IC_INTR_STAT__R1_RESTART_DET__SHIFT                                                         0xc
+#define CKSVII2C1_IC_INTR_STAT__R1_MST_ON_HOLD__SHIFT                                                         0xd
+#define CKSVII2C1_IC_INTR_STAT__R1_SCL_STUCK_AT_LOW__SHIFT                                                    0xe
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_UNDER_MASK                                                              0x00000001L
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_OVER_MASK                                                               0x00000002L
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_FULL_MASK                                                               0x00000004L
+#define CKSVII2C1_IC_INTR_STAT__R1_TX_OVER_MASK                                                               0x00000008L
+#define CKSVII2C1_IC_INTR_STAT__R1_TX_EMPTY_MASK                                                              0x00000010L
+#define CKSVII2C1_IC_INTR_STAT__R1_RD_REQ_MASK                                                                0x00000020L
+#define CKSVII2C1_IC_INTR_STAT__R1_TX_ABRT_MASK                                                               0x00000040L
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_DONE_MASK                                                               0x00000080L
+#define CKSVII2C1_IC_INTR_STAT__R1_ACTIVITY_MASK                                                              0x00000100L
+#define CKSVII2C1_IC_INTR_STAT__R1_STOP_DET_MASK                                                              0x00000200L
+#define CKSVII2C1_IC_INTR_STAT__R1_START_DET_MASK                                                             0x00000400L
+#define CKSVII2C1_IC_INTR_STAT__R1_GEN_CALL_MASK                                                              0x00000800L
+#define CKSVII2C1_IC_INTR_STAT__R1_RESTART_DET_MASK                                                           0x00001000L
+#define CKSVII2C1_IC_INTR_STAT__R1_MST_ON_HOLD_MASK                                                           0x00002000L
+//CKSVII2C1_IC_INTR_MASK
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_UNDER__SHIFT                                                            0x0
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_OVER__SHIFT                                                             0x1
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_FULL__SHIFT                                                             0x2
+#define CKSVII2C1_IC_INTR_MASK__M1_TX_OVER__SHIFT                                                             0x3
+#define CKSVII2C1_IC_INTR_MASK__M1_TX_EMPTY__SHIFT                                                            0x4
+#define CKSVII2C1_IC_INTR_MASK__M1_RD_REQ__SHIFT                                                              0x5
+#define CKSVII2C1_IC_INTR_MASK__M1_TX_ABRT__SHIFT                                                             0x6
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_DONE__SHIFT                                                             0x7
+#define CKSVII2C1_IC_INTR_MASK__M1_ACTIVITY__SHIFT                                                            0x8
+#define CKSVII2C1_IC_INTR_MASK__M1_STOP_DET__SHIFT                                                            0x9
+#define CKSVII2C1_IC_INTR_MASK__M1_START_DET__SHIFT                                                           0xa
+#define CKSVII2C1_IC_INTR_MASK__M1_GEN_CALL__SHIFT                                                            0xb
+#define CKSVII2C1_IC_INTR_MASK__M1_RESTART_DET__SHIFT                                                         0xc
+#define CKSVII2C1_IC_INTR_MASK__M1_MST_ON_HOLD__SHIFT                                                         0xd
+#define CKSVII2C1_IC_INTR_MASK__M1_SCL_STUCK_AT_LOW__SHIFT                                                    0xe
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_UNDER_MASK                                                              0x00000001L
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_OVER_MASK                                                               0x00000002L
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_FULL_MASK                                                               0x00000004L
+#define CKSVII2C1_IC_INTR_MASK__M1_TX_OVER_MASK                                                               0x00000008L
+#define CKSVII2C1_IC_INTR_MASK__M1_TX_EMPTY_MASK                                                              0x00000010L
+#define CKSVII2C1_IC_INTR_MASK__M1_RD_REQ_MASK                                                                0x00000020L
+#define CKSVII2C1_IC_INTR_MASK__M1_TX_ABRT_MASK                                                               0x00000040L
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_DONE_MASK                                                               0x00000080L
+#define CKSVII2C1_IC_INTR_MASK__M1_ACTIVITY_MASK                                                              0x00000100L
+#define CKSVII2C1_IC_INTR_MASK__M1_STOP_DET_MASK                                                              0x00000200L
+#define CKSVII2C1_IC_INTR_MASK__M1_START_DET_MASK                                                             0x00000400L
+#define CKSVII2C1_IC_INTR_MASK__M1_GEN_CALL_MASK                                                              0x00000800L
+#define CKSVII2C1_IC_INTR_MASK__M1_RESTART_DET_MASK                                                           0x00001000L
+#define CKSVII2C1_IC_INTR_MASK__M1_MST_ON_HOLD_MASK                                                           0x00002000L
+//CKSVII2C1_IC_RAW_INTR_STAT
+//CKSVII2C1_IC_RX_TL
+#define CKSVII2C1_IC_RX_TL__RX1_TL__SHIFT                                                                     0x0
+//CKSVII2C1_IC_TX_TL
+#define CKSVII2C1_IC_TX_TL__TX1_TL__SHIFT                                                                     0x0
+//CKSVII2C1_IC_CLR_INTR
+//CKSVII2C1_IC_CLR_RX_UNDER
+//CKSVII2C1_IC_CLR_RX_OVER
+//CKSVII2C1_IC_CLR_TX_OVER
+//CKSVII2C1_IC_CLR_RD_REQ
+//CKSVII2C1_IC_CLR_TX_ABRT
+//CKSVII2C1_IC_CLR_RX_DONE
+//CKSVII2C1_IC_CLR_ACTIVITY
+//CKSVII2C1_IC_CLR_STOP_DET
+//CKSVII2C1_IC_CLR_START_DET
+//CKSVII2C1_IC_CLR_GEN_CALL
+//CKSVII2C1_IC_ENABLE
+#define CKSVII2C1_IC_ENABLE__ENABLE1__SHIFT                                                                   0x0
+#define CKSVII2C1_IC_ENABLE__ABORT1__SHIFT                                                                    0x1
+#define CKSVII2C1_IC_ENABLE__TX1_CMD_BLOCK__SHIFT                                                             0x2
+#define CKSVII2C1_IC_ENABLE__SDA1_STUCK_RECOVERY_ENABLE__SHIFT                                                0x3
+#define CKSVII2C1_IC_ENABLE__ENABLE1_MASK                                                                     0x00000001L
+#define CKSVII2C1_IC_ENABLE__ABORT1_MASK                                                                      0x00000002L
+//CKSVII2C1_IC_STATUS
+#define CKSVII2C1_IC_STATUS__ACTIVITY1__SHIFT                                                                 0x0
+#define CKSVII2C1_IC_STATUS__TFNF1__SHIFT                                                                     0x1
+#define CKSVII2C1_IC_STATUS__TFE1__SHIFT                                                                      0x2
+#define CKSVII2C1_IC_STATUS__RFNE1__SHIFT                                                                     0x3
+#define CKSVII2C1_IC_STATUS__RFF1__SHIFT                                                                      0x4
+#define CKSVII2C1_IC_STATUS__MST1_ACTIVITY__SHIFT                                                             0x5
+#define CKSVII2C1_IC_STATUS__SLV1_ACTIVITY__SHIFT                                                             0x6
+#define CKSVII2C1_IC_STATUS__MST1_HOLD_TX_FIFO_EMPTY__SHIFT                                                   0x7
+#define CKSVII2C1_IC_STATUS__MST1_HOLD_RX_FIFO_FULL__SHIFT                                                    0x8
+#define CKSVII2C1_IC_STATUS__SLV1_HOLD_TX_FIFO_EMPTY__SHIFT                                                   0x9
+#define CKSVII2C1_IC_STATUS__SLV1_HOLD_RX_FIFO_FULL__SHIFT                                                    0xa
+#define CKSVII2C1_IC_STATUS__SDA1_STUCK_NOT_RECOVERED__SHIFT                                                  0xb
+#define CKSVII2C1_IC_STATUS__ACTIVITY1_MASK                                                                   0x00000001L
+#define CKSVII2C1_IC_STATUS__TFNF1_MASK                                                                       0x00000002L
+#define CKSVII2C1_IC_STATUS__TFE1_MASK                                                                        0x00000004L
+#define CKSVII2C1_IC_STATUS__RFNE1_MASK                                                                       0x00000008L
+#define CKSVII2C1_IC_STATUS__RFF1_MASK                                                                        0x00000010L
+#define CKSVII2C1_IC_STATUS__MST1_ACTIVITY_MASK                                                               0x00000020L
+#define CKSVII2C1_IC_STATUS__SLV1_ACTIVITY_MASK                                                               0x00000040L
+//CKSVII2C1_IC_TXFLR
+#define CKSVII2C1_IC_TXFLR__TXFLR1__SHIFT                                                                     0x0
+//CKSVII2C1_IC_RXFLR
+#define CKSVII2C1_IC_RXFLR__RXFLR1__SHIFT                                                                     0x0
+//CKSVII2C1_IC_SDA_HOLD
+#define CKSVII2C1_IC_SDA_HOLD__IC1_SDA_TX_HOLD__SHIFT                                                         0x0
+#define CKSVII2C1_IC_SDA_HOLD__IC1_SDA_RX_HOLD__SHIFT                                                         0x10
+//CKSVII2C1_IC_TX_ABRT_SOURCE
+//CKSVII2C1_IC_SLV_DATA_NACK_ONLY
+//CKSVII2C1_IC_DMA_CR
+//CKSVII2C1_IC_DMA_TDLR
+//CKSVII2C1_IC_DMA_RDLR
+//CKSVII2C1_IC_SDA_SETUP
+#define CKSVII2C1_IC_SDA_SETUP__SDA1_SETUP__SHIFT                                                             0x0
+#define CKSVII2C1_IC_SDA_SETUP__SDA1_SETUP_MASK                                                               0x000000FFL
+//CKSVII2C1_IC_ACK_GENERAL_CALL
+#define CKSVII2C1_IC_ACK_GENERAL_CALL__ACK1_GENERAL_CALL__SHIFT                                               0x0
+#define CKSVII2C1_IC_ACK_GENERAL_CALL__ACK1_GENERAL_CALL_MASK                                                 0x00000001L
+//CKSVII2C1_IC_ENABLE_STATUS
+#define CKSVII2C1_IC_ENABLE_STATUS__IC1_EN__SHIFT                                                             0x0
+#define CKSVII2C1_IC_ENABLE_STATUS__SLV1_DISABLED_WHILE_BUSY__SHIFT                                           0x1
+#define CKSVII2C1_IC_ENABLE_STATUS__SLV1_RX_DATA_LOST__SHIFT                                                  0x2
+#define CKSVII2C1_IC_ENABLE_STATUS__IC1_EN_MASK                                                               0x00000001L
+//CKSVII2C1_IC_FS_SPKLEN
+#define CKSVII2C1_IC_FS_SPKLEN__FS1_SPKLEN__SHIFT                                                             0x0
+//CKSVII2C1_IC_HS_SPKLEN
+#define CKSVII2C1_IC_HS_SPKLEN__HS1_SPKLEN__SHIFT                                                             0x0
+//CKSVII2C1_IC_CLR_RESTART_DET
+//CKSVII2C1_IC_COMP_PARAM_1
+#define CKSVII2C1_IC_COMP_PARAM_1__APB1_DATA_WIDTH__SHIFT                                                     0x0
+#define CKSVII2C1_IC_COMP_PARAM_1__MAX1_SPEED_MODE__SHIFT                                                     0x2
+#define CKSVII2C1_IC_COMP_PARAM_1__HC1_COUNT_VALUES__SHIFT                                                    0x4
+#define CKSVII2C1_IC_COMP_PARAM_1__INTR1_IO__SHIFT                                                            0x5
+#define CKSVII2C1_IC_COMP_PARAM_1__HAS1_DMA__SHIFT                                                            0x6
+#define CKSVII2C1_IC_COMP_PARAM_1__ADD1_ENCODED_PARAMS__SHIFT                                                 0x7
+#define CKSVII2C1_IC_COMP_PARAM_1__RX1_BUFFER_DEPTH__SHIFT                                                    0x8
+#define CKSVII2C1_IC_COMP_PARAM_1__TX1_BUFFER_DEPTH__SHIFT                                                    0x10
+//CKSVII2C1_IC_COMP_VERSION
+#define CKSVII2C1_IC_COMP_VERSION__COMP1_VERSION__SHIFT                                                       0x0
+//CKSVII2C1_IC_COMP_TYPE
+#define CKSVII2C1_IC_COMP_TYPE__COMP1_TYPE__SHIFT                                                             0x0
+//SMUIO_PWRMGT
+#define SMUIO_PWRMGT__i2c_clk_gate_en__SHIFT                                                                  0x0
+#define SMUIO_PWRMGT__i2c1_clk_gate_en__SHIFT                                                                 0x4
+#define SMUIO_PWRMGT__i2c_clk_gate_en_MASK                                                                    0x00000001L
+#define SMUIO_PWRMGT__i2c1_clk_gate_en_MASK                                                                   0x00000010L
+
+
+// addressBlock: smuio_smuio_rom_SmuSmuioDec
+//ROM_CNTL
+#define ROM_CNTL__CLOCK_GATING_EN__SHIFT                                                                      0x0
+#define ROM_CNTL__READ_MODE__SHIFT                                                                            0x1
+#define ROM_CNTL__READ_MODE_OVERRIDE__SHIFT                                                                   0x3
+#define ROM_CNTL__SPI_TIMING_RELAX_SCK__SHIFT                                                                 0x4
+#define ROM_CNTL__SPI_TIMING_RELAX_SCK_OVERRIDE__SHIFT                                                        0x5
+#define ROM_CNTL__FOUR_BYTE_ADDRESS_MODE__SHIFT                                                               0x6
+#define ROM_CNTL__DUMMY_CYCLE_NUM__SHIFT                                                                      0x8
+#define ROM_CNTL__SPI_TIMING_RELAX__SHIFT                                                                     0x13
+#define ROM_CNTL__SPI_TIMING_RELAX_OVERRIDE__SHIFT                                                            0x14
+#define ROM_CNTL__SPI_FAST_MODE__SHIFT                                                                        0x15
+#define ROM_CNTL__SPI_FAST_MODE_OVERRIDE__SHIFT                                                               0x16
+#define ROM_CNTL__SCK_PRESCALE_REFCLK__SHIFT                                                                  0x17
+#define ROM_CNTL__SCK_PRESCALE_REFCLK_OVERRIDE__SHIFT                                                         0x1c
+#define ROM_CNTL__ROM_INDEX_ADDRESS_AUTO_INCREASE__SHIFT                                                      0x1d
+#define ROM_CNTL__PAD_SAMPLE_MODE__SHIFT                                                                      0x1e
+#define ROM_CNTL__PAD_SAMPLE_MODE_OVERRIDE__SHIFT                                                             0x1f
+#define ROM_CNTL__CLOCK_GATING_EN_MASK                                                                        0x00000001L
+#define ROM_CNTL__SPI_TIMING_RELAX_MASK                                                                       0x00080000L
+#define ROM_CNTL__SPI_TIMING_RELAX_OVERRIDE_MASK                                                              0x00100000L
+#define ROM_CNTL__SPI_FAST_MODE_MASK                                                                          0x00200000L
+#define ROM_CNTL__SPI_FAST_MODE_OVERRIDE_MASK                                                                 0x00400000L
+#define ROM_CNTL__SCK_PRESCALE_REFCLK_MASK                                                                    0x0F800000L
+#define ROM_CNTL__SCK_PRESCALE_REFCLK_OVERRIDE_MASK                                                           0x10000000L
+//PAGE_MIRROR_CNTL
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_BASE_ADDR__SHIFT                                                        0x0
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_ENABLE__SHIFT                                                           0x19
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_USAGE__SHIFT                                                            0x1a
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_INVALIDATE__SHIFT                                                       0x1c
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_BASE_ADDR_MASK                                                          0x01FFFFFFL
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_ENABLE_MASK                                                             0x02000000L
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_USAGE_MASK                                                              0x0C000000L
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_INVALIDATE_MASK                                                         0x10000000L
+//ROM_STATUS
+#define ROM_STATUS__ROM_BUSY__SHIFT                                                                           0x0
+#define ROM_STATUS__ROM_BUSY_MASK                                                                             0x00000001L
+//CGTT_ROM_CLK_CTRL0
+#define CGTT_ROM_CLK_CTRL0__ON_DELAY__SHIFT                                                                   0x0
+#define CGTT_ROM_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT                                                             0x1e
+#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT                                                             0x1f
+#define CGTT_ROM_CLK_CTRL0__ON_DELAY_MASK                                                                     0x0000000FL
+#define CGTT_ROM_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK                                                               0x40000000L
+#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK                                                               0x80000000L
+//ROM_INDEX
+#define ROM_INDEX__ROM_INDEX__SHIFT                                                                           0x0
+#define ROM_INDEX__ROM_INDEX_MASK                                                                             0x01FFFFFFL
+//ROM_DATA
+#define ROM_DATA__ROM_DATA__SHIFT                                                                             0x0
+#define ROM_DATA__ROM_DATA_MASK                                                                               0xFFFFFFFFL
+//ROM_START
+#define ROM_START__ROM_START__SHIFT                                                                           0x0
+#define ROM_START__ROM_START_MASK                                                                             0x01FFFFFFL
+//ROM_SW_CNTL
+#define ROM_SW_CNTL__DATA_SIZE__SHIFT                                                                         0x0
+#define ROM_SW_CNTL__COMMAND_SIZE__SHIFT                                                                      0x10
+#define ROM_SW_CNTL__ROM_SW_RETURN_DATA_ENABLE__SHIFT                                                         0x13
+#define ROM_SW_CNTL__DATA_SIZE_MASK                                                                           0x0000FFFFL
+#define ROM_SW_CNTL__COMMAND_SIZE_MASK                                                                        0x00070000L
+#define ROM_SW_CNTL__ROM_SW_RETURN_DATA_ENABLE_MASK                                                           0x00080000L
+//ROM_SW_STATUS
+#define ROM_SW_STATUS__ROM_SW_DONE__SHIFT                                                                     0x0
+#define ROM_SW_STATUS__ROM_SW_DONE_MASK                                                                       0x00000001L
+//ROM_SW_COMMAND
+#define ROM_SW_COMMAND__ROM_SW_INSTRUCTION__SHIFT                                                             0x0
+#define ROM_SW_COMMAND__ROM_SW_ADDRESS__SHIFT                                                                 0x8
+#define ROM_SW_COMMAND__ROM_SW_INSTRUCTION_MASK                                                               0x000000FFL
+#define ROM_SW_COMMAND__ROM_SW_ADDRESS_MASK                                                                   0xFFFFFF00L
+//ROM_SW_DATA_1
+#define ROM_SW_DATA_1__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_1__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_2
+#define ROM_SW_DATA_2__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_2__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_3
+#define ROM_SW_DATA_3__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_3__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_4
+#define ROM_SW_DATA_4__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_4__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_5
+#define ROM_SW_DATA_5__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_5__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_6
+#define ROM_SW_DATA_6__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_6__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_7
+#define ROM_SW_DATA_7__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_7__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_8
+#define ROM_SW_DATA_8__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_8__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_9
+#define ROM_SW_DATA_9__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_9__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_10
+#define ROM_SW_DATA_10__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_10__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_11
+#define ROM_SW_DATA_11__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_11__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_12
+#define ROM_SW_DATA_12__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_12__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_13
+#define ROM_SW_DATA_13__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_13__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_14
+#define ROM_SW_DATA_14__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_14__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_15
+#define ROM_SW_DATA_15__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_15__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_16
+#define ROM_SW_DATA_16__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_16__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_17
+#define ROM_SW_DATA_17__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_17__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_18
+#define ROM_SW_DATA_18__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_18__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_19
+#define ROM_SW_DATA_19__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_19__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_20
+#define ROM_SW_DATA_20__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_20__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_21
+#define ROM_SW_DATA_21__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_21__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_22
+#define ROM_SW_DATA_22__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_22__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_23
+#define ROM_SW_DATA_23__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_23__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_24
+#define ROM_SW_DATA_24__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_24__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_25
+#define ROM_SW_DATA_25__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_25__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_26
+#define ROM_SW_DATA_26__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_26__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_27
+#define ROM_SW_DATA_27__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_27__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_28
+#define ROM_SW_DATA_28__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_28__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_29
+#define ROM_SW_DATA_29__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_29__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_30
+#define ROM_SW_DATA_30__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_30__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_31
+#define ROM_SW_DATA_31__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_31__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_32
+#define ROM_SW_DATA_32__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_32__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_33
+#define ROM_SW_DATA_33__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_33__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_34
+#define ROM_SW_DATA_34__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_34__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_35
+#define ROM_SW_DATA_35__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_35__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_36
+#define ROM_SW_DATA_36__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_36__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_37
+#define ROM_SW_DATA_37__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_37__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_38
+#define ROM_SW_DATA_38__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_38__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_39
+#define ROM_SW_DATA_39__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_39__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_40
+#define ROM_SW_DATA_40__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_40__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_41
+#define ROM_SW_DATA_41__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_41__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_42
+#define ROM_SW_DATA_42__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_42__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_43
+#define ROM_SW_DATA_43__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_43__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_44
+#define ROM_SW_DATA_44__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_44__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_45
+#define ROM_SW_DATA_45__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_45__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_46
+#define ROM_SW_DATA_46__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_46__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_47
+#define ROM_SW_DATA_47__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_47__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_48
+#define ROM_SW_DATA_48__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_48__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_49
+#define ROM_SW_DATA_49__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_49__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_50
+#define ROM_SW_DATA_50__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_50__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_51
+#define ROM_SW_DATA_51__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_51__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_52
+#define ROM_SW_DATA_52__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_52__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_53
+#define ROM_SW_DATA_53__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_53__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_54
+#define ROM_SW_DATA_54__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_54__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_55
+#define ROM_SW_DATA_55__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_55__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_56
+#define ROM_SW_DATA_56__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_56__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_57
+#define ROM_SW_DATA_57__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_57__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_58
+#define ROM_SW_DATA_58__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_58__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_59
+#define ROM_SW_DATA_59__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_59__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_60
+#define ROM_SW_DATA_60__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_60__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_61
+#define ROM_SW_DATA_61__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_61__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_62
+#define ROM_SW_DATA_62__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_62__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_63
+#define ROM_SW_DATA_63__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_63__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_64
+#define ROM_SW_DATA_64__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_64__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+
+
+// addressBlock: smuio_smuio_gpio_SmuSmuioDec
+//SMU_GPIOPAD_SW_INT_STAT
+#define SMU_GPIOPAD_SW_INT_STAT__SW_INT_STAT__SHIFT                                                           0x0
+#define SMU_GPIOPAD_SW_INT_STAT__SW_INT_STAT_MASK                                                             0x00000001L
+//SMU_GPIOPAD_MASK
+#define SMU_GPIOPAD_MASK__GPIO_MASK__SHIFT                                                                    0x0
+#define SMU_GPIOPAD_MASK__GPIO_MASK_MASK                                                                      0x7FFFFFFFL
+//SMU_GPIOPAD_A
+#define SMU_GPIOPAD_A__GPIO_A__SHIFT                                                                          0x0
+#define SMU_GPIOPAD_A__GPIO_A_MASK                                                                            0x7FFFFFFFL
+//SMU_GPIOPAD_TXIMPSEL
+#define SMU_GPIOPAD_TXIMPSEL__GPIO_TXIMPSEL__SHIFT                                                            0x0
+#define SMU_GPIOPAD_TXIMPSEL__GPIO_TXIMPSEL_MASK                                                              0x7FFFFFFFL
+//SMU_GPIOPAD_EN
+#define SMU_GPIOPAD_EN__GPIO_EN__SHIFT                                                                        0x0
+#define SMU_GPIOPAD_EN__GPIO_EN_MASK                                                                          0x7FFFFFFFL
+//SMU_GPIOPAD_Y
+#define SMU_GPIOPAD_Y__GPIO_Y__SHIFT                                                                          0x0
+#define SMU_GPIOPAD_Y__GPIO_Y_MASK                                                                            0x7FFFFFFFL
+//SMU_GPIOPAD_RXEN
+#define SMU_GPIOPAD_RXEN__GPIO_RXEN__SHIFT                                                                    0x0
+#define SMU_GPIOPAD_RXEN__GPIO_RXEN_MASK                                                                      0x7FFFFFFFL
+//SMU_GPIOPAD_RCVR_SEL0
+#define SMU_GPIOPAD_RCVR_SEL0__GPIO_RCVR_SEL0__SHIFT                                                          0x0
+#define SMU_GPIOPAD_RCVR_SEL0__GPIO_RCVR_SEL0_MASK                                                            0x7FFFFFFFL
+//SMU_GPIOPAD_RCVR_SEL1
+#define SMU_GPIOPAD_RCVR_SEL1__GPIO_RCVR_SEL1__SHIFT                                                          0x0
+#define SMU_GPIOPAD_RCVR_SEL1__GPIO_RCVR_SEL1_MASK                                                            0x7FFFFFFFL
+//SMU_GPIOPAD_PU_EN
+#define SMU_GPIOPAD_PU_EN__GPIO_PU_EN__SHIFT                                                                  0x0
+#define SMU_GPIOPAD_PU_EN__GPIO_PU_EN_MASK                                                                    0x7FFFFFFFL
+//SMU_GPIOPAD_PD_EN
+#define SMU_GPIOPAD_PD_EN__GPIO_PD_EN__SHIFT                                                                  0x0
+#define SMU_GPIOPAD_PD_EN__GPIO_PD_EN_MASK                                                                    0x7FFFFFFFL
+//SMU_GPIOPAD_PINSTRAPS
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_0__SHIFT                                                         0x0
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_1__SHIFT                                                         0x1
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_2__SHIFT                                                         0x2
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_3__SHIFT                                                         0x3
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_4__SHIFT                                                         0x4
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_5__SHIFT                                                         0x5
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_6__SHIFT                                                         0x6
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_7__SHIFT                                                         0x7
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_8__SHIFT                                                         0x8
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_9__SHIFT                                                         0x9
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_10__SHIFT                                                        0xa
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_11__SHIFT                                                        0xb
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_12__SHIFT                                                        0xc
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_13__SHIFT                                                        0xd
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_14__SHIFT                                                        0xe
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_15__SHIFT                                                        0xf
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_16__SHIFT                                                        0x10
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_17__SHIFT                                                        0x11
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_18__SHIFT                                                        0x12
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_19__SHIFT                                                        0x13
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_20__SHIFT                                                        0x14
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_21__SHIFT                                                        0x15
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_22__SHIFT                                                        0x16
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_23__SHIFT                                                        0x17
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_24__SHIFT                                                        0x18
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_25__SHIFT                                                        0x19
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_26__SHIFT                                                        0x1a
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_27__SHIFT                                                        0x1b
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_28__SHIFT                                                        0x1c
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_29__SHIFT                                                        0x1d
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_30__SHIFT                                                        0x1e
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_0_MASK                                                           0x00000001L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_1_MASK                                                           0x00000002L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_2_MASK                                                           0x00000004L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_3_MASK                                                           0x00000008L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_4_MASK                                                           0x00000010L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_5_MASK                                                           0x00000020L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_6_MASK                                                           0x00000040L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_7_MASK                                                           0x00000080L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_8_MASK                                                           0x00000100L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_9_MASK                                                           0x00000200L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_10_MASK                                                          0x00000400L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_11_MASK                                                          0x00000800L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_12_MASK                                                          0x00001000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_13_MASK                                                          0x00002000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_14_MASK                                                          0x00004000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_15_MASK                                                          0x00008000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_16_MASK                                                          0x00010000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_17_MASK                                                          0x00020000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_18_MASK                                                          0x00040000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_19_MASK                                                          0x00080000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_20_MASK                                                          0x00100000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_21_MASK                                                          0x00200000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_22_MASK                                                          0x00400000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_23_MASK                                                          0x00800000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_24_MASK                                                          0x01000000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_25_MASK                                                          0x02000000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_26_MASK                                                          0x04000000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_27_MASK                                                          0x08000000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_28_MASK                                                          0x10000000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_29_MASK                                                          0x20000000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_30_MASK                                                          0x40000000L
+//DFT_PINSTRAPS
+#define DFT_PINSTRAPS__DFT_PINSTRAPS__SHIFT                                                                   0x0
+#define DFT_PINSTRAPS__DFT_PINSTRAPS_MASK                                                                     0x000000FFL
+//SMU_GPIOPAD_INT_STAT_EN
+#define SMU_GPIOPAD_INT_STAT_EN__GPIO_INT_STAT_EN__SHIFT                                                      0x0
+#define SMU_GPIOPAD_INT_STAT_EN__SW_INITIATED_INT_STAT_EN__SHIFT                                              0x1f
+#define SMU_GPIOPAD_INT_STAT_EN__GPIO_INT_STAT_EN_MASK                                                        0x1FFFFFFFL
+#define SMU_GPIOPAD_INT_STAT_EN__SW_INITIATED_INT_STAT_EN_MASK                                                0x80000000L
+//SMU_GPIOPAD_INT_STAT
+#define SMU_GPIOPAD_INT_STAT__GPIO_INT_STAT__SHIFT                                                            0x0
+#define SMU_GPIOPAD_INT_STAT__SW_INITIATED_INT_STAT__SHIFT                                                    0x1f
+#define SMU_GPIOPAD_INT_STAT__GPIO_INT_STAT_MASK                                                              0x1FFFFFFFL
+#define SMU_GPIOPAD_INT_STAT__SW_INITIATED_INT_STAT_MASK                                                      0x80000000L
+//SMU_GPIOPAD_INT_STAT_AK
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_0__SHIFT                                                    0x0
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_1__SHIFT                                                    0x1
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_2__SHIFT                                                    0x2
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_3__SHIFT                                                    0x3
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_4__SHIFT                                                    0x4
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_5__SHIFT                                                    0x5
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_6__SHIFT                                                    0x6
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_7__SHIFT                                                    0x7
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_8__SHIFT                                                    0x8
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_9__SHIFT                                                    0x9
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_10__SHIFT                                                   0xa
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_11__SHIFT                                                   0xb
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_12__SHIFT                                                   0xc
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_13__SHIFT                                                   0xd
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_14__SHIFT                                                   0xe
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_15__SHIFT                                                   0xf
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_16__SHIFT                                                   0x10
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_17__SHIFT                                                   0x11
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_18__SHIFT                                                   0x12
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_19__SHIFT                                                   0x13
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_20__SHIFT                                                   0x14
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_21__SHIFT                                                   0x15
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_22__SHIFT                                                   0x16
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_23__SHIFT                                                   0x17
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_24__SHIFT                                                   0x18
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_25__SHIFT                                                   0x19
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_26__SHIFT                                                   0x1a
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_27__SHIFT                                                   0x1b
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_28__SHIFT                                                   0x1c
+#define SMU_GPIOPAD_INT_STAT_AK__SW_INITIATED_INT_STAT_AK__SHIFT                                              0x1f
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_0_MASK                                                      0x00000001L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_1_MASK                                                      0x00000002L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_2_MASK                                                      0x00000004L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_3_MASK                                                      0x00000008L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_4_MASK                                                      0x00000010L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_5_MASK                                                      0x00000020L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_6_MASK                                                      0x00000040L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_7_MASK                                                      0x00000080L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_8_MASK                                                      0x00000100L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_9_MASK                                                      0x00000200L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_10_MASK                                                     0x00000400L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_11_MASK                                                     0x00000800L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_12_MASK                                                     0x00001000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_13_MASK                                                     0x00002000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_14_MASK                                                     0x00004000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_15_MASK                                                     0x00008000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_16_MASK                                                     0x00010000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_17_MASK                                                     0x00020000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_18_MASK                                                     0x00040000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_19_MASK                                                     0x00080000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_20_MASK                                                     0x00100000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_21_MASK                                                     0x00200000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_22_MASK                                                     0x00400000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_23_MASK                                                     0x00800000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_24_MASK                                                     0x01000000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_25_MASK                                                     0x02000000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_26_MASK                                                     0x04000000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_27_MASK                                                     0x08000000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_28_MASK                                                     0x10000000L
+#define SMU_GPIOPAD_INT_STAT_AK__SW_INITIATED_INT_STAT_AK_MASK                                                0x80000000L
+//SMU_GPIOPAD_INT_EN
+#define SMU_GPIOPAD_INT_EN__GPIO_INT_EN__SHIFT                                                                0x0
+#define SMU_GPIOPAD_INT_EN__SW_INITIATED_INT_EN__SHIFT                                                        0x1f
+#define SMU_GPIOPAD_INT_EN__GPIO_INT_EN_MASK                                                                  0x1FFFFFFFL
+#define SMU_GPIOPAD_INT_EN__SW_INITIATED_INT_EN_MASK                                                          0x80000000L
+//SMU_GPIOPAD_INT_TYPE
+#define SMU_GPIOPAD_INT_TYPE__GPIO_INT_TYPE__SHIFT                                                            0x0
+#define SMU_GPIOPAD_INT_TYPE__SW_INITIATED_INT_TYPE__SHIFT                                                    0x1f
+#define SMU_GPIOPAD_INT_TYPE__GPIO_INT_TYPE_MASK                                                              0x1FFFFFFFL
+#define SMU_GPIOPAD_INT_TYPE__SW_INITIATED_INT_TYPE_MASK                                                      0x80000000L
+//SMU_GPIOPAD_INT_POLARITY
+#define SMU_GPIOPAD_INT_POLARITY__GPIO_INT_POLARITY__SHIFT                                                    0x0
+#define SMU_GPIOPAD_INT_POLARITY__SW_INITIATED_INT_POLARITY__SHIFT                                            0x1f
+#define SMU_GPIOPAD_INT_POLARITY__GPIO_INT_POLARITY_MASK                                                      0x1FFFFFFFL
+#define SMU_GPIOPAD_INT_POLARITY__SW_INITIATED_INT_POLARITY_MASK                                              0x80000000L
+//SMUIO_PCC_GPIO_SELECT
+#define SMUIO_PCC_GPIO_SELECT__GPIO__SHIFT                                                                    0x0
+#define SMUIO_PCC_GPIO_SELECT__GPIO_MASK                                                                      0xFFFFFFFFL
+//SMU_GPIOPAD_S0
+#define SMU_GPIOPAD_S0__GPIO_S0__SHIFT                                                                        0x0
+#define SMU_GPIOPAD_S0__GPIO_S0_MASK                                                                          0x7FFFFFFFL
+//SMU_GPIOPAD_S1
+#define SMU_GPIOPAD_S1__GPIO_S1__SHIFT                                                                        0x0
+#define SMU_GPIOPAD_S1__GPIO_S1_MASK                                                                          0x7FFFFFFFL
+//SMU_GPIOPAD_SCHMEN
+#define SMU_GPIOPAD_SCHMEN__GPIO_SCHMEN__SHIFT                                                                0x0
+#define SMU_GPIOPAD_SCHMEN__GPIO_SCHMEN_MASK                                                                  0x7FFFFFFFL
+//SMU_GPIOPAD_SCL_EN
+#define SMU_GPIOPAD_SCL_EN__GPIO_SCL_EN__SHIFT                                                                0x0
+#define SMU_GPIOPAD_SCL_EN__GPIO_SCL_EN_MASK                                                                  0x7FFFFFFFL
+//SMU_GPIOPAD_SDA_EN
+#define SMU_GPIOPAD_SDA_EN__GPIO_SDA_EN__SHIFT                                                                0x0
+#define SMU_GPIOPAD_SDA_EN__GPIO_SDA_EN_MASK                                                                  0x7FFFFFFFL
+//SMUIO_GPIO_INT0_SELECT
+#define SMUIO_GPIO_INT0_SELECT__GPIO_INT0_SELECT__SHIFT                                                       0x0
+#define SMUIO_GPIO_INT0_SELECT__GPIO_INT0_SELECT_MASK                                                         0xFFFFFFFFL
+//SMUIO_GPIO_INT1_SELECT
+#define SMUIO_GPIO_INT1_SELECT__GPIO_INT1_SELECT__SHIFT                                                       0x0
+#define SMUIO_GPIO_INT1_SELECT__GPIO_INT1_SELECT_MASK                                                         0xFFFFFFFFL
+//SMUIO_GPIO_INT2_SELECT
+#define SMUIO_GPIO_INT2_SELECT__GPIO_INT2_SELECT__SHIFT                                                       0x0
+#define SMUIO_GPIO_INT2_SELECT__GPIO_INT2_SELECT_MASK                                                         0xFFFFFFFFL
+//SMUIO_GPIO_INT3_SELECT
+#define SMUIO_GPIO_INT3_SELECT__GPIO_INT3_SELECT__SHIFT                                                       0x0
+#define SMUIO_GPIO_INT3_SELECT__GPIO_INT3_SELECT_MASK                                                         0xFFFFFFFFL
+//SMU_GPIOPAD_MP_INT0_STAT
+#define SMU_GPIOPAD_MP_INT0_STAT__GPIO_MP_INT0_STAT__SHIFT                                                    0x0
+#define SMU_GPIOPAD_MP_INT0_STAT__GPIO_MP_INT0_STAT_MASK                                                      0x1FFFFFFFL
+//SMU_GPIOPAD_MP_INT1_STAT
+#define SMU_GPIOPAD_MP_INT1_STAT__GPIO_MP_INT1_STAT__SHIFT                                                    0x0
+#define SMU_GPIOPAD_MP_INT1_STAT__GPIO_MP_INT1_STAT_MASK                                                      0x1FFFFFFFL
+//SMU_GPIOPAD_MP_INT2_STAT
+#define SMU_GPIOPAD_MP_INT2_STAT__GPIO_MP_INT2_STAT__SHIFT                                                    0x0
+#define SMU_GPIOPAD_MP_INT2_STAT__GPIO_MP_INT2_STAT_MASK                                                      0x1FFFFFFFL
+//SMU_GPIOPAD_MP_INT3_STAT
+#define SMU_GPIOPAD_MP_INT3_STAT__GPIO_MP_INT3_STAT__SHIFT                                                    0x0
+#define SMU_GPIOPAD_MP_INT3_STAT__GPIO_MP_INT3_STAT_MASK                                                      0x1FFFFFFFL
+//SMIO_INDEX
+#define SMIO_INDEX__SW_SMIO_INDEX__SHIFT                                                                      0x0
+#define SMIO_INDEX__SW_SMIO_INDEX_MASK                                                                        0x00000001L
+//S0_VID_SMIO_CNTL
+#define S0_VID_SMIO_CNTL__S0_SMIO_VALUES__SHIFT                                                               0x0
+#define S0_VID_SMIO_CNTL__S0_SMIO_VALUES_MASK                                                                 0xFFFFFFFFL
+//S1_VID_SMIO_CNTL
+#define S1_VID_SMIO_CNTL__S1_SMIO_VALUES__SHIFT                                                               0x0
+#define S1_VID_SMIO_CNTL__S1_SMIO_VALUES_MASK                                                                 0xFFFFFFFFL
+//OPEN_DRAIN_SELECT
+#define OPEN_DRAIN_SELECT__OPEN_DRAIN_SELECT__SHIFT                                                           0x0
+#define OPEN_DRAIN_SELECT__RESERVED__SHIFT                                                                    0x1f
+#define OPEN_DRAIN_SELECT__OPEN_DRAIN_SELECT_MASK                                                             0x7FFFFFFFL
+#define OPEN_DRAIN_SELECT__RESERVED_MASK                                                                      0x80000000L
+//SMIO_ENABLE
+#define SMIO_ENABLE__SMIO_ENABLE__SHIFT                                                                       0x0
+#define SMIO_ENABLE__SMIO_ENABLE_MASK                                                                         0xFFFFFFFFL
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index af3eebb4c9bc..571691837200 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -1657,6 +1657,49 @@ struct atom_integrated_system_info_v2_2
 	uint32_t  reserved4[189];
 };
 
+struct uma_carveout_option {
+  char       optionName[29];        //max length of string is 28chars + '\0'. Current design is for "minimum", "Medium", "High". This makes entire struct size 64bits
+  uint8_t    memoryCarvedGb;        //memory carved out with setting
+  uint8_t    memoryRemainingGb;     //memory remaining on system
+  union {
+    struct _flags {
+      uint8_t Auto     : 1;
+      uint8_t Custom   : 1;
+      uint8_t Reserved : 6;
+    } flags;
+    uint8_t all8;
+  } uma_carveout_option_flags;
+};
+
+struct atom_integrated_system_info_v2_3 {
+  struct  atom_common_table_header table_header;
+  uint32_t  vbios_misc; // enum of atom_system_vbiosmisc_def
+  uint32_t  gpucapinfo; // enum of atom_system_gpucapinf_def
+  uint32_t  system_config;
+  uint32_t  cpucapinfo;
+  uint16_t  gpuclk_ss_percentage; // unit of 0.001%,   1000 mean 1%
+  uint16_t  gpuclk_ss_type;
+  uint16_t  dpphy_override;  // bit vector, enum of atom_sysinfo_dpphy_override_def
+  uint8_t memorytype;       // enum of atom_dmi_t17_mem_type_def, APU memory type indication.
+  uint8_t umachannelnumber; // number of memory channels
+  uint8_t htc_hyst_limit;
+  uint8_t htc_tmp_limit;
+  uint8_t reserved1; // dp_ss_control
+  uint8_t gpu_package_id;
+  struct  edp_info_table  edp1_info;
+  struct  edp_info_table  edp2_info;
+  uint32_t  reserved2[8];
+  struct  atom_external_display_connection_info extdispconninfo;
+  uint8_t UMACarveoutVersion;
+  uint8_t UMACarveoutIndexMax;
+  uint8_t UMACarveoutTypeDefault;
+  uint8_t UMACarveoutIndexDefault;
+  uint8_t UMACarveoutType;           //Auto or Custom
+  uint8_t UMACarveoutIndex;
+  struct  uma_carveout_option UMASizeControlOption[20];
+  uint8_t reserved3[110];
+};
+
 // system_config
 enum atom_system_vbiosmisc_def{
   INTEGRATED_SYSTEM_INFO__GET_EDID_CALLBACK_FUNC_SUPPORT = 0x01,
@@ -3540,7 +3583,7 @@ struct atom_gpio_voltage_object_v4
    uint8_t  phase_delay_us;                      // phase delay in unit of micro second
    uint8_t  reserved;   
    uint32_t gpio_mask_val;                         // GPIO Mask value
-   struct atom_voltage_gpio_map_lut voltage_gpio_lut[1];
+   struct atom_voltage_gpio_map_lut voltage_gpio_lut[] __counted_by(gpio_entry_num);
 };
 
 struct  atom_svid2_voltage_object_v4
diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
index 32054ecf0b87..805c9d37a2b4 100644
--- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
@@ -150,6 +150,7 @@ enum amd_pp_sensors {
 	AMDGPU_PP_SENSOR_VCN_POWER_STATE,
 	AMDGPU_PP_SENSOR_PEAK_PSTATE_SCLK,
 	AMDGPU_PP_SENSOR_PEAK_PSTATE_MCLK,
+	AMDGPU_PP_SENSOR_VCN_LOAD,
 };
 
 enum amd_pp_task {
@@ -420,7 +421,7 @@ struct amd_pm_funcs {
 	int (*set_hard_min_dcefclk_by_freq)(void *handle, uint32_t clock);
 	int (*set_hard_min_fclk_by_freq)(void *handle, uint32_t clock);
 	int (*set_min_deep_sleep_dcefclk)(void *handle, uint32_t clock);
-	bool (*get_asic_baco_capability)(void *handle);
+	int (*get_asic_baco_capability)(void *handle);
 	int (*get_asic_baco_state)(void *handle, int *state);
 	int (*set_asic_baco_state)(void *handle, int state);
 	int (*get_ppfeature_status)(void *handle, char *buf);
diff --git a/drivers/gpu/drm/amd/include/mes_v11_api_def.h b/drivers/gpu/drm/amd/include/mes_v11_api_def.h
index ec5b9ab67c5e..b72d5d362251 100644
--- a/drivers/gpu/drm/amd/include/mes_v11_api_def.h
+++ b/drivers/gpu/drm/amd/include/mes_v11_api_def.h
@@ -61,6 +61,7 @@ enum MES_SCH_API_OPCODE {
 	MES_SCH_API_MISC			= 14,
 	MES_SCH_API_UPDATE_ROOT_PAGE_TABLE      = 15,
 	MES_SCH_API_AMD_LOG                     = 16,
+	MES_SCH_API_SET_HW_RSRC_1               = 19,
 	MES_SCH_API_MAX				= 0xFF
 };
 
@@ -238,6 +239,26 @@ union MESAPI_SET_HW_RESOURCES {
 	uint32_t	max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS];
 };
 
+union MESAPI_SET_HW_RESOURCES_1 {
+	struct {
+		union MES_API_HEADER				header;
+		struct MES_API_STATUS			   api_status;
+		uint64_t							timestamp;
+		union {
+			struct {
+				uint32_t enable_mes_info_ctx : 1;
+				uint32_t reserved : 31;
+			};
+			uint32_t uint32_all;
+		};
+		uint64_t							mes_info_ctx_mc_addr;
+		uint32_t							mes_info_ctx_size;
+		uint32_t							mes_kiq_unmap_timeout; // unit is 100ms
+	};
+
+	uint32_t max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS];
+};
+
 union MESAPI__ADD_QUEUE {
 	struct {
 		union MES_API_HEADER		header;
@@ -278,10 +299,21 @@ union MESAPI__ADD_QUEUE {
 			uint32_t skip_process_ctx_clear : 1;
 			uint32_t map_legacy_kq		: 1;
 			uint32_t exclusively_scheduled	: 1;
-			uint32_t reserved		: 17;
+			uint32_t is_long_running        : 1;
+			uint32_t is_dwm_queue           : 1;
+			uint32_t is_video_blit_queue    : 1;
+			uint32_t reserved               : 14;
 		};
-		struct MES_API_STATUS		api_status;
-		uint64_t                        tma_addr;
+		struct MES_API_STATUS       api_status;
+		uint64_t                    tma_addr;
+		uint32_t                    sch_id;
+		uint64_t                    timestamp;
+		uint32_t                    process_context_array_index;
+		uint32_t                    gang_context_array_index;
+		uint32_t                    pipe_id;
+		uint32_t                    queue_id;
+		uint32_t                    alignment_mode_setting;
+		uint64_t                    unmap_flag_addr;
 	};
 
 	uint32_t	max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS];
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
index f84bfed50681..eee919577b44 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
@@ -199,14 +199,14 @@ int amdgpu_dpm_notify_rlc_state(struct amdgpu_device *adev, bool en)
 	return ret;
 }
 
-bool amdgpu_dpm_is_baco_supported(struct amdgpu_device *adev)
+int amdgpu_dpm_is_baco_supported(struct amdgpu_device *adev)
 {
 	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
 	void *pp_handle = adev->powerplay.pp_handle;
-	bool ret;
+	int ret;
 
 	if (!pp_funcs || !pp_funcs->get_asic_baco_capability)
-		return false;
+		return 0;
 	/* Don't use baco for reset in S3.
 	 * This is a workaround for some platforms
 	 * where entering BACO during suspend
@@ -217,7 +217,7 @@ bool amdgpu_dpm_is_baco_supported(struct amdgpu_device *adev)
 	 * devices.  Needs more investigation.
 	 */
 	if (adev->in_s3)
-		return false;
+		return 0;
 
 	mutex_lock(&adev->pm.mutex);
 
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
index f09b9d49297e..c11952a4389b 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
@@ -38,6 +38,8 @@
 #define MAX_NUM_OF_FEATURES_PER_SUBSET		8
 #define MAX_NUM_OF_SUBSETS			8
 
+#define DEVICE_ATTR_IS(_name)		(attr_id == device_attr_id__##_name)
+
 struct od_attribute {
 	struct kobj_attribute	attribute;
 	struct list_head	entry;
@@ -1582,6 +1584,30 @@ static ssize_t amdgpu_get_mem_busy_percent(struct device *dev,
 }
 
 /**
+ * DOC: vcn_busy_percent
+ *
+ * The amdgpu driver provides a sysfs API for reading how busy the VCN
+ * is as a percentage.  The file vcn_busy_percent is used for this.
+ * The SMU firmware computes a percentage of load based on the
+ * aggregate activity level in the IP cores.
+ */
+static ssize_t amdgpu_get_vcn_busy_percent(struct device *dev,
+						  struct device_attribute *attr,
+						  char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	unsigned int value;
+	int r;
+
+	r = amdgpu_hwmon_get_sensor_generic(adev, AMDGPU_PP_SENSOR_VCN_LOAD, &value);
+	if (r)
+		return r;
+
+	return sysfs_emit(buf, "%d\n", value);
+}
+
+/**
  * DOC: pcie_bw
  *
  * The amdgpu driver provides a sysfs API for estimating how much data
@@ -2091,6 +2117,99 @@ static int pp_dpm_dcefclk_attr_update(struct amdgpu_device *adev, struct amdgpu_
 	return 0;
 }
 
+static int pp_dpm_clk_default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_attr *attr,
+					  uint32_t mask, enum amdgpu_device_attr_states *states)
+{
+	struct device_attribute *dev_attr = &attr->dev_attr;
+	enum amdgpu_device_attr_id attr_id = attr->attr_id;
+	uint32_t mp1_ver = amdgpu_ip_version(adev, MP1_HWIP, 0);
+	uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
+
+	*states = ATTR_STATE_SUPPORTED;
+
+	if (!(attr->flags & mask)) {
+		*states = ATTR_STATE_UNSUPPORTED;
+		return 0;
+	}
+
+	if (DEVICE_ATTR_IS(pp_dpm_socclk)) {
+		if (gc_ver < IP_VERSION(9, 0, 0))
+			*states = ATTR_STATE_UNSUPPORTED;
+	} else if (DEVICE_ATTR_IS(pp_dpm_fclk)) {
+		if (mp1_ver < IP_VERSION(10, 0, 0))
+			*states = ATTR_STATE_UNSUPPORTED;
+	} else if (DEVICE_ATTR_IS(pp_dpm_vclk)) {
+		if (!(gc_ver == IP_VERSION(10, 3, 1) ||
+		      gc_ver == IP_VERSION(10, 3, 3) ||
+		      gc_ver == IP_VERSION(10, 3, 6) ||
+		      gc_ver == IP_VERSION(10, 3, 7) ||
+		      gc_ver == IP_VERSION(10, 3, 0) ||
+		      gc_ver == IP_VERSION(10, 1, 2) ||
+		      gc_ver == IP_VERSION(11, 0, 0) ||
+		      gc_ver == IP_VERSION(11, 0, 1) ||
+		      gc_ver == IP_VERSION(11, 0, 4) ||
+		      gc_ver == IP_VERSION(11, 5, 0) ||
+		      gc_ver == IP_VERSION(11, 0, 2) ||
+		      gc_ver == IP_VERSION(11, 0, 3) ||
+		      gc_ver == IP_VERSION(9, 4, 3)))
+			*states = ATTR_STATE_UNSUPPORTED;
+	} else if (DEVICE_ATTR_IS(pp_dpm_vclk1)) {
+		if (!((gc_ver == IP_VERSION(10, 3, 1) ||
+		       gc_ver == IP_VERSION(10, 3, 0) ||
+		       gc_ver == IP_VERSION(11, 0, 2) ||
+		       gc_ver == IP_VERSION(11, 0, 3)) && adev->vcn.num_vcn_inst >= 2))
+			*states = ATTR_STATE_UNSUPPORTED;
+	} else if (DEVICE_ATTR_IS(pp_dpm_dclk)) {
+		if (!(gc_ver == IP_VERSION(10, 3, 1) ||
+		      gc_ver == IP_VERSION(10, 3, 3) ||
+		      gc_ver == IP_VERSION(10, 3, 6) ||
+		      gc_ver == IP_VERSION(10, 3, 7) ||
+		      gc_ver == IP_VERSION(10, 3, 0) ||
+		      gc_ver == IP_VERSION(10, 1, 2) ||
+		      gc_ver == IP_VERSION(11, 0, 0) ||
+		      gc_ver == IP_VERSION(11, 0, 1) ||
+		      gc_ver == IP_VERSION(11, 0, 4) ||
+		      gc_ver == IP_VERSION(11, 5, 0) ||
+		      gc_ver == IP_VERSION(11, 0, 2) ||
+		      gc_ver == IP_VERSION(11, 0, 3) ||
+		      gc_ver == IP_VERSION(9, 4, 3)))
+			*states = ATTR_STATE_UNSUPPORTED;
+	} else if (DEVICE_ATTR_IS(pp_dpm_dclk1)) {
+		if (!((gc_ver == IP_VERSION(10, 3, 1) ||
+		       gc_ver == IP_VERSION(10, 3, 0) ||
+		       gc_ver == IP_VERSION(11, 0, 2) ||
+		       gc_ver == IP_VERSION(11, 0, 3)) && adev->vcn.num_vcn_inst >= 2))
+			*states = ATTR_STATE_UNSUPPORTED;
+	} else if (DEVICE_ATTR_IS(pp_dpm_pcie)) {
+		if (gc_ver == IP_VERSION(9, 4, 2) ||
+		    gc_ver == IP_VERSION(9, 4, 3))
+			*states = ATTR_STATE_UNSUPPORTED;
+	}
+
+	switch (gc_ver) {
+	case IP_VERSION(9, 4, 1):
+	case IP_VERSION(9, 4, 2):
+		/* the Mi series card does not support standalone mclk/socclk/fclk level setting */
+		if (DEVICE_ATTR_IS(pp_dpm_mclk) ||
+		    DEVICE_ATTR_IS(pp_dpm_socclk) ||
+		    DEVICE_ATTR_IS(pp_dpm_fclk)) {
+			dev_attr->attr.mode &= ~S_IWUGO;
+			dev_attr->store = NULL;
+		}
+		break;
+	default:
+		break;
+	}
+
+	/* setting should not be allowed from VF if not in one VF mode */
+	if (amdgpu_sriov_vf(adev) && amdgpu_sriov_is_pp_one_vf(adev)) {
+		dev_attr->attr.mode &= ~S_IWUGO;
+		dev_attr->store = NULL;
+	}
+
+	return 0;
+}
+
 /* Following items will be read out to indicate current plpd policy:
  *  - -1: none
  *  - 0: disallow
@@ -2162,17 +2281,26 @@ static struct amdgpu_device_attr amdgpu_device_attrs[] = {
 	AMDGPU_DEVICE_ATTR_RO(pp_cur_state,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RW(pp_force_state,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RW(pp_table,					ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
-	AMDGPU_DEVICE_ATTR_RW(pp_dpm_sclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
-	AMDGPU_DEVICE_ATTR_RW(pp_dpm_mclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
-	AMDGPU_DEVICE_ATTR_RW(pp_dpm_socclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
-	AMDGPU_DEVICE_ATTR_RW(pp_dpm_fclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
-	AMDGPU_DEVICE_ATTR_RW(pp_dpm_vclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
-	AMDGPU_DEVICE_ATTR_RW(pp_dpm_vclk1,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
-	AMDGPU_DEVICE_ATTR_RW(pp_dpm_dclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
-	AMDGPU_DEVICE_ATTR_RW(pp_dpm_dclk1,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
+	AMDGPU_DEVICE_ATTR_RW(pp_dpm_sclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF,
+			      .attr_update = pp_dpm_clk_default_attr_update),
+	AMDGPU_DEVICE_ATTR_RW(pp_dpm_mclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF,
+			      .attr_update = pp_dpm_clk_default_attr_update),
+	AMDGPU_DEVICE_ATTR_RW(pp_dpm_socclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF,
+			      .attr_update = pp_dpm_clk_default_attr_update),
+	AMDGPU_DEVICE_ATTR_RW(pp_dpm_fclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF,
+			      .attr_update = pp_dpm_clk_default_attr_update),
+	AMDGPU_DEVICE_ATTR_RW(pp_dpm_vclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF,
+			      .attr_update = pp_dpm_clk_default_attr_update),
+	AMDGPU_DEVICE_ATTR_RW(pp_dpm_vclk1,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF,
+			      .attr_update = pp_dpm_clk_default_attr_update),
+	AMDGPU_DEVICE_ATTR_RW(pp_dpm_dclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF,
+			      .attr_update = pp_dpm_clk_default_attr_update),
+	AMDGPU_DEVICE_ATTR_RW(pp_dpm_dclk1,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF,
+			      .attr_update = pp_dpm_clk_default_attr_update),
 	AMDGPU_DEVICE_ATTR_RW(pp_dpm_dcefclk,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF,
 			      .attr_update = pp_dpm_dcefclk_attr_update),
-	AMDGPU_DEVICE_ATTR_RW(pp_dpm_pcie,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
+	AMDGPU_DEVICE_ATTR_RW(pp_dpm_pcie,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF,
+			      .attr_update = pp_dpm_clk_default_attr_update),
 	AMDGPU_DEVICE_ATTR_RW(pp_sclk_od,				ATTR_FLAG_BASIC),
 	AMDGPU_DEVICE_ATTR_RW(pp_mclk_od,				ATTR_FLAG_BASIC),
 	AMDGPU_DEVICE_ATTR_RW(pp_power_profile_mode,			ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
@@ -2180,6 +2308,7 @@ static struct amdgpu_device_attr amdgpu_device_attrs[] = {
 			      .attr_update = pp_od_clk_voltage_attr_update),
 	AMDGPU_DEVICE_ATTR_RO(gpu_busy_percent,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RO(mem_busy_percent,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
+	AMDGPU_DEVICE_ATTR_RO(vcn_busy_percent,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RO(pcie_bw,					ATTR_FLAG_BASIC),
 	AMDGPU_DEVICE_ATTR_RW(pp_features,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
 	AMDGPU_DEVICE_ATTR_RO(unique_id,				ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF),
@@ -2201,28 +2330,28 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_
 			       uint32_t mask, enum amdgpu_device_attr_states *states)
 {
 	struct device_attribute *dev_attr = &attr->dev_attr;
-	uint32_t mp1_ver = amdgpu_ip_version(adev, MP1_HWIP, 0);
+	enum amdgpu_device_attr_id attr_id = attr->attr_id;
 	uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
-	const char *attr_name = dev_attr->attr.name;
 
 	if (!(attr->flags & mask)) {
 		*states = ATTR_STATE_UNSUPPORTED;
 		return 0;
 	}
 
-#define DEVICE_ATTR_IS(_name)	(!strcmp(attr_name, #_name))
-
-	if (DEVICE_ATTR_IS(pp_dpm_socclk)) {
-		if (gc_ver < IP_VERSION(9, 0, 0))
-			*states = ATTR_STATE_UNSUPPORTED;
-	} else if (DEVICE_ATTR_IS(pp_dpm_fclk)) {
-		if (mp1_ver < IP_VERSION(10, 0, 0))
-			*states = ATTR_STATE_UNSUPPORTED;
-	} else if (DEVICE_ATTR_IS(mem_busy_percent)) {
+	if (DEVICE_ATTR_IS(mem_busy_percent)) {
 		if ((adev->flags & AMD_IS_APU &&
 		     gc_ver != IP_VERSION(9, 4, 3)) ||
 		    gc_ver == IP_VERSION(9, 0, 1))
 			*states = ATTR_STATE_UNSUPPORTED;
+	} else if (DEVICE_ATTR_IS(vcn_busy_percent)) {
+		if (!(gc_ver == IP_VERSION(10, 3, 1) ||
+			  gc_ver == IP_VERSION(10, 3, 3) ||
+			  gc_ver == IP_VERSION(10, 3, 6) ||
+			  gc_ver == IP_VERSION(10, 3, 7) ||
+			  gc_ver == IP_VERSION(11, 0, 1) ||
+			  gc_ver == IP_VERSION(11, 0, 4) ||
+			  gc_ver == IP_VERSION(11, 5, 0)))
+			*states = ATTR_STATE_UNSUPPORTED;
 	} else if (DEVICE_ATTR_IS(pcie_bw)) {
 		/* PCIe Perf counters won't work on APU nodes */
 		if (adev->flags & AMD_IS_APU ||
@@ -2253,36 +2382,6 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_
 	} else if (DEVICE_ATTR_IS(gpu_metrics)) {
 		if (gc_ver < IP_VERSION(9, 1, 0))
 			*states = ATTR_STATE_UNSUPPORTED;
-	} else if (DEVICE_ATTR_IS(pp_dpm_vclk)) {
-		if (!(gc_ver == IP_VERSION(10, 3, 1) ||
-		      gc_ver == IP_VERSION(10, 3, 0) ||
-		      gc_ver == IP_VERSION(10, 1, 2) ||
-		      gc_ver == IP_VERSION(11, 0, 0) ||
-		      gc_ver == IP_VERSION(11, 0, 2) ||
-		      gc_ver == IP_VERSION(11, 0, 3) ||
-		      gc_ver == IP_VERSION(9, 4, 3)))
-			*states = ATTR_STATE_UNSUPPORTED;
-	} else if (DEVICE_ATTR_IS(pp_dpm_vclk1)) {
-		if (!((gc_ver == IP_VERSION(10, 3, 1) ||
-			   gc_ver == IP_VERSION(10, 3, 0) ||
-			   gc_ver == IP_VERSION(11, 0, 2) ||
-			   gc_ver == IP_VERSION(11, 0, 3)) && adev->vcn.num_vcn_inst >= 2))
-			*states = ATTR_STATE_UNSUPPORTED;
-	} else if (DEVICE_ATTR_IS(pp_dpm_dclk)) {
-		if (!(gc_ver == IP_VERSION(10, 3, 1) ||
-		      gc_ver == IP_VERSION(10, 3, 0) ||
-		      gc_ver == IP_VERSION(10, 1, 2) ||
-		      gc_ver == IP_VERSION(11, 0, 0) ||
-		      gc_ver == IP_VERSION(11, 0, 2) ||
-		      gc_ver == IP_VERSION(11, 0, 3) ||
-		      gc_ver == IP_VERSION(9, 4, 3)))
-			*states = ATTR_STATE_UNSUPPORTED;
-	} else if (DEVICE_ATTR_IS(pp_dpm_dclk1)) {
-		if (!((gc_ver == IP_VERSION(10, 3, 1) ||
-			   gc_ver == IP_VERSION(10, 3, 0) ||
-			   gc_ver == IP_VERSION(11, 0, 2) ||
-			   gc_ver == IP_VERSION(11, 0, 3)) && adev->vcn.num_vcn_inst >= 2))
-			*states = ATTR_STATE_UNSUPPORTED;
 	} else if (DEVICE_ATTR_IS(pp_power_profile_mode)) {
 		if (amdgpu_dpm_get_power_profile_mode(adev, NULL) == -EOPNOTSUPP)
 			*states = ATTR_STATE_UNSUPPORTED;
@@ -2304,23 +2403,9 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_
 		if (amdgpu_dpm_get_apu_thermal_limit(adev, &limit) ==
 		    -EOPNOTSUPP)
 			*states = ATTR_STATE_UNSUPPORTED;
-	} else if (DEVICE_ATTR_IS(pp_dpm_pcie)) {
-		if (gc_ver == IP_VERSION(9, 4, 2) ||
-		    gc_ver == IP_VERSION(9, 4, 3))
-			*states = ATTR_STATE_UNSUPPORTED;
 	}
 
 	switch (gc_ver) {
-	case IP_VERSION(9, 4, 1):
-	case IP_VERSION(9, 4, 2):
-		/* the Mi series card does not support standalone mclk/socclk/fclk level setting */
-		if (DEVICE_ATTR_IS(pp_dpm_mclk) ||
-		    DEVICE_ATTR_IS(pp_dpm_socclk) ||
-		    DEVICE_ATTR_IS(pp_dpm_fclk)) {
-			dev_attr->attr.mode &= ~S_IWUGO;
-			dev_attr->store = NULL;
-		}
-		break;
 	case IP_VERSION(10, 3, 0):
 		if (DEVICE_ATTR_IS(power_dpm_force_performance_level) &&
 		    amdgpu_sriov_vf(adev)) {
@@ -2332,14 +2417,6 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_
 		break;
 	}
 
-	/* setting should not be allowed from VF if not in one VF mode */
-	if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev)) {
-		dev_attr->attr.mode &= ~S_IWUGO;
-		dev_attr->store = NULL;
-	}
-
-#undef DEVICE_ATTR_IS
-
 	return 0;
 }
 
@@ -4261,6 +4338,13 @@ static int amdgpu_od_set_init(struct amdgpu_device *adev)
 		}
 	}
 
+	/*
+	 * If gpu_od is the only member in the list, that means gpu_od is an
+	 * empty directory, so remove it.
+	 */
+	if (list_is_singular(&adev->pm.od_kobj_list))
+		goto err_out;
+
 	return 0;
 
 err_out:
@@ -4322,6 +4406,8 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
 		ret = amdgpu_od_set_init(adev);
 		if (ret)
 			goto err_out1;
+	} else if (adev->pm.pp_feature & PP_OVERDRIVE_MASK) {
+		dev_info(adev->dev, "overdrive feature is not supported\n");
 	}
 
 	adev->pm.sysfs_initialized = true;
@@ -4429,6 +4515,9 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a
 	/* MEM Load */
 	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MEM_LOAD, (void *)&value, &size))
 		seq_printf(m, "MEM Load: %u %%\n", value);
+	/* VCN Load */
+	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VCN_LOAD, (void *)&value, &size))
+		seq_printf(m, "VCN Load: %u %%\n", value);
 
 	seq_printf(m, "\n");
 
diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
index 621200e0823f..501f8c726e8d 100644
--- a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
@@ -50,8 +50,12 @@ enum amdgpu_runpm_mode {
 	AMDGPU_RUNPM_PX,
 	AMDGPU_RUNPM_BOCO,
 	AMDGPU_RUNPM_BACO,
+	AMDGPU_RUNPM_BAMACO,
 };
 
+#define BACO_SUPPORT (1<<0)
+#define MACO_SUPPORT (1<<1)
+
 struct amdgpu_ps {
 	u32 caps; /* vbios flags */
 	u32 class; /* vbios flags */
@@ -407,7 +411,7 @@ int amdgpu_dpm_baco_reset(struct amdgpu_device *adev);
 int amdgpu_dpm_mode2_reset(struct amdgpu_device *adev);
 int amdgpu_dpm_enable_gfx_features(struct amdgpu_device *adev);
 
-bool amdgpu_dpm_is_baco_supported(struct amdgpu_device *adev);
+int amdgpu_dpm_is_baco_supported(struct amdgpu_device *adev);
 
 bool amdgpu_dpm_is_mode1_reset_supported(struct amdgpu_device *adev);
 int amdgpu_dpm_mode1_reset(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_pm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_pm.h
index eec816f0cbf9..448ba3a14584 100644
--- a/drivers/gpu/drm/amd/pm/inc/amdgpu_pm.h
+++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_pm.h
@@ -43,8 +43,48 @@ enum amdgpu_device_attr_states {
 	ATTR_STATE_SUPPORTED,
 };
 
+enum amdgpu_device_attr_id {
+	device_attr_id__unknown = -1,
+	device_attr_id__power_dpm_state = 0,
+	device_attr_id__power_dpm_force_performance_level,
+	device_attr_id__pp_num_states,
+	device_attr_id__pp_cur_state,
+	device_attr_id__pp_force_state,
+	device_attr_id__pp_table,
+	device_attr_id__pp_dpm_sclk,
+	device_attr_id__pp_dpm_mclk,
+	device_attr_id__pp_dpm_socclk,
+	device_attr_id__pp_dpm_fclk,
+	device_attr_id__pp_dpm_vclk,
+	device_attr_id__pp_dpm_vclk1,
+	device_attr_id__pp_dpm_dclk,
+	device_attr_id__pp_dpm_dclk1,
+	device_attr_id__pp_dpm_dcefclk,
+	device_attr_id__pp_dpm_pcie,
+	device_attr_id__pp_sclk_od,
+	device_attr_id__pp_mclk_od,
+	device_attr_id__pp_power_profile_mode,
+	device_attr_id__pp_od_clk_voltage,
+	device_attr_id__gpu_busy_percent,
+	device_attr_id__mem_busy_percent,
+	device_attr_id__vcn_busy_percent,
+	device_attr_id__pcie_bw,
+	device_attr_id__pp_features,
+	device_attr_id__unique_id,
+	device_attr_id__thermal_throttling_logging,
+	device_attr_id__apu_thermal_cap,
+	device_attr_id__gpu_metrics,
+	device_attr_id__smartshift_apu_power,
+	device_attr_id__smartshift_dgpu_power,
+	device_attr_id__smartshift_bias,
+	device_attr_id__xgmi_plpd_policy,
+	device_attr_id__pm_metrics,
+	device_attr_id__count,
+};
+
 struct amdgpu_device_attr {
 	struct device_attribute dev_attr;
+	enum amdgpu_device_attr_id attr_id;
 	enum amdgpu_device_attr_flags flags;
 	int (*attr_update)(struct amdgpu_device *adev, struct amdgpu_device_attr *attr,
 			   uint32_t mask, enum amdgpu_device_attr_states *states);
@@ -61,6 +101,7 @@ struct amdgpu_device_attr_entry {
 
 #define __AMDGPU_DEVICE_ATTR(_name, _mode, _show, _store, _flags, ...)	\
 	{ .dev_attr = __ATTR(_name, _mode, _show, _store),		\
+	  .attr_id = device_attr_id__##_name,				\
 	  .flags = _flags,						\
 	  ##__VA_ARGS__, }
 
diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c
index 5cb4725c773f..6bb42d04b247 100644
--- a/drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c
+++ b/drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c
@@ -3316,6 +3316,8 @@ static const struct amd_ip_funcs kv_dpm_ip_funcs = {
 	.soft_reset = kv_dpm_soft_reset,
 	.set_clockgating_state = kv_dpm_set_clockgating_state,
 	.set_powergating_state = kv_dpm_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version kv_smu_ip_block = {
diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
index eb4da3666e05..f245fc0bc6d3 100644
--- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
+++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
@@ -8060,6 +8060,8 @@ static const struct amd_ip_funcs si_dpm_ip_funcs = {
 	.soft_reset = si_dpm_soft_reset,
 	.set_clockgating_state = si_dpm_set_clockgating_state,
 	.set_powergating_state = si_dpm_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version si_smu_ip_block =
diff --git a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
index aed0e2cefbf9..5fb21a0508cd 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
@@ -302,6 +302,8 @@ static const struct amd_ip_funcs pp_ip_funcs = {
 	.soft_reset = pp_sw_reset,
 	.set_clockgating_state = pp_set_clockgating_state,
 	.set_powergating_state = pp_set_powergating_state,
+	.dump_ip_state = NULL,
+	.print_ip_state = NULL,
 };
 
 const struct amdgpu_ip_block_version pp_smu_ip_block =
@@ -1371,7 +1373,7 @@ static int pp_set_active_display_count(void *handle, uint32_t count)
 	return phm_set_active_display_count(hwmgr, count);
 }
 
-static bool pp_get_asic_baco_capability(void *handle)
+static int pp_get_asic_baco_capability(void *handle)
 {
 	struct pp_hwmgr *hwmgr = handle;
 
@@ -1379,10 +1381,10 @@ static bool pp_get_asic_baco_capability(void *handle)
 		return false;
 
 	if (!(hwmgr->not_vf && amdgpu_dpm) ||
-		!hwmgr->hwmgr_func->get_asic_baco_capability)
+		!hwmgr->hwmgr_func->get_bamaco_support)
 		return false;
 
-	return hwmgr->hwmgr_func->get_asic_baco_capability(hwmgr);
+	return hwmgr->hwmgr_func->get_bamaco_support(hwmgr);
 }
 
 static int pp_get_asic_baco_state(void *handle, int *state)
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.c
index e8a9471c1898..ad60918aaae1 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.c
@@ -33,7 +33,7 @@
 #include "smu/smu_7_1_2_d.h"
 #include "smu/smu_7_1_2_sh_mask.h"
 
-bool smu7_baco_get_capability(struct pp_hwmgr *hwmgr)
+int smu7_get_bamaco_support(struct pp_hwmgr *hwmgr)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev);
 	uint32_t reg;
@@ -44,9 +44,9 @@ bool smu7_baco_get_capability(struct pp_hwmgr *hwmgr)
 	reg = RREG32(mmCC_BIF_BX_FUSESTRAP0);
 
 	if (reg & CC_BIF_BX_FUSESTRAP0__STRAP_BIF_PX_CAPABLE_MASK)
-		return true;
+		return BACO_SUPPORT;
 
-	return false;
+	return 0;
 }
 
 int smu7_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state)
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.h
index 73a773f4ce2e..750082ea74d8 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.h
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.h
@@ -25,7 +25,7 @@
 #include "hwmgr.h"
 #include "common_baco.h"
 
-extern bool smu7_baco_get_capability(struct pp_hwmgr *hwmgr);
+extern int smu7_get_bamaco_support(struct pp_hwmgr *hwmgr);
 extern int smu7_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state);
 extern int smu7_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state);
 
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
index aa91730e4eaf..1fcd4451001f 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
@@ -5791,7 +5791,7 @@ static const struct pp_hwmgr_func smu7_hwmgr_funcs = {
 	.get_power_profile_mode = smu7_get_power_profile_mode,
 	.set_power_profile_mode = smu7_set_power_profile_mode,
 	.get_performance_level = smu7_get_performance_level,
-	.get_asic_baco_capability = smu7_baco_get_capability,
+	.get_bamaco_support = smu7_get_bamaco_support,
 	.get_asic_baco_state = smu7_baco_get_state,
 	.set_asic_baco_state = smu7_baco_set_state,
 	.power_off_asic = smu7_power_off_asic,
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.c
index c66ef9741535..c1ce1d7cae48 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.c
@@ -28,13 +28,13 @@
 #include "vega10_inc.h"
 #include "smu9_baco.h"
 
-bool smu9_baco_get_capability(struct pp_hwmgr *hwmgr)
+int smu9_get_bamaco_support(struct pp_hwmgr *hwmgr)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev);
 	uint32_t reg, data;
 
 	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_BACO))
-		return false;
+		return 0;
 
 	WREG32(0x12074, 0xFFF0003B);
 	data = RREG32(0x12075);
@@ -43,10 +43,10 @@ bool smu9_baco_get_capability(struct pp_hwmgr *hwmgr)
 		reg = RREG32_SOC15(NBIF, 0, mmRCC_BIF_STRAP0);
 
 		if (reg & RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK)
-			return true;
+			return BACO_SUPPORT;
 	}
 
-	return false;
+	return 0;
 }
 
 int smu9_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state)
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.h
index 9ff7c2ea1b58..2c100482084c 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.h
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.h
@@ -25,7 +25,7 @@
 #include "hwmgr.h"
 #include "common_baco.h"
 
-extern bool smu9_baco_get_capability(struct pp_hwmgr *hwmgr);
+extern int smu9_get_bamaco_support(struct pp_hwmgr *hwmgr);
 extern int smu9_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state);
 
 #endif
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c
index 6d6bc6a380b3..9f5bd998c6bf 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c
@@ -5756,7 +5756,7 @@ static const struct pp_hwmgr_func vega10_hwmgr_funcs = {
 	.set_power_limit = vega10_set_power_limit,
 	.odn_edit_dpm_table = vega10_odn_edit_dpm_table,
 	.get_performance_level = vega10_get_performance_level,
-	.get_asic_baco_capability = smu9_baco_get_capability,
+	.get_bamaco_support = smu9_get_bamaco_support,
 	.get_asic_baco_state = smu9_baco_get_state,
 	.set_asic_baco_state = vega10_baco_set_state,
 	.enable_mgpu_fan_boost = vega10_enable_mgpu_fan_boost,
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c
index 460067933de2..c223e3a6bfca 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c
@@ -2966,7 +2966,7 @@ static const struct pp_hwmgr_func vega12_hwmgr_funcs = {
 	.start_thermal_controller = vega12_start_thermal_controller,
 	.powergate_gfx = vega12_gfx_off_control,
 	.get_performance_level = vega12_get_performance_level,
-	.get_asic_baco_capability = smu9_baco_get_capability,
+	.get_bamaco_support = smu9_get_bamaco_support,
 	.get_asic_baco_state = smu9_baco_get_state,
 	.set_asic_baco_state = vega12_baco_set_state,
 	.get_ppfeature_status = vega12_get_ppfeature_status,
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c
index dad4c80aee58..424e4ec9e389 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c
@@ -36,22 +36,22 @@ static const struct soc15_baco_cmd_entry clean_baco_tbl[] = {
 	{CMD_WRITE, SOC15_REG_ENTRY(NBIF, 0, mmBIOS_SCRATCH_7), 0, 0, 0, 0},
 };
 
-bool vega20_baco_get_capability(struct pp_hwmgr *hwmgr)
+int vega20_get_bamaco_support(struct pp_hwmgr *hwmgr)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev);
 	uint32_t reg;
 
 	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_BACO))
-		return false;
+		return 0;
 
 	if (((RREG32(0x17569) & 0x20000000) >> 29) == 0x1) {
 		reg = RREG32_SOC15(NBIF, 0, mmRCC_BIF_STRAP0);
 
 		if (reg & RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK)
-			return true;
+			return BACO_SUPPORT;
 	}
 
-	return false;
+	return 0;
 }
 
 int vega20_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state)
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.h
index bdad9c915631..0f2dd8c008ba 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.h
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.h
@@ -25,7 +25,7 @@
 #include "hwmgr.h"
 #include "common_baco.h"
 
-extern bool vega20_baco_get_capability(struct pp_hwmgr *hwmgr);
+extern int vega20_get_bamaco_support(struct pp_hwmgr *hwmgr);
 extern int vega20_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state);
 extern int vega20_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state);
 extern int vega20_baco_apply_vdci_flush_workaround(struct pp_hwmgr *hwmgr);
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c
index 3b33af30eb0f..f9efb0bad807 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c
@@ -4422,7 +4422,7 @@ static const struct pp_hwmgr_func vega20_hwmgr_funcs = {
 	.notify_cac_buffer_info = vega20_notify_cac_buffer_info,
 	.enable_mgpu_fan_boost = vega20_enable_mgpu_fan_boost,
 	/* BACO related */
-	.get_asic_baco_capability = vega20_baco_get_capability,
+	.get_bamaco_support = vega20_get_bamaco_support,
 	.get_asic_baco_state = vega20_baco_get_state,
 	.set_asic_baco_state = vega20_baco_set_state,
 	.set_mp1_state = vega20_set_mp1_state,
diff --git a/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
index 6f536159df4d..69928a4a074b 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
+++ b/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
@@ -351,7 +351,7 @@ struct pp_hwmgr_func {
 	int (*set_hard_min_fclk_by_freq)(struct pp_hwmgr *hwmgr, uint32_t clock);
 	int (*set_hard_min_gfxclk_by_freq)(struct pp_hwmgr *hwmgr, uint32_t clock);
 	int (*set_soft_max_gfxclk_by_freq)(struct pp_hwmgr *hwmgr, uint32_t clock);
-	bool (*get_asic_baco_capability)(struct pp_hwmgr *hwmgr);
+	int (*get_bamaco_support)(struct pp_hwmgr *hwmgr);
 	int (*get_asic_baco_state)(struct pp_hwmgr *hwmgr, enum BACO_STATE *state);
 	int (*set_asic_baco_state)(struct pp_hwmgr *hwmgr, enum BACO_STATE state);
 	int (*get_ppfeature_status)(struct pp_hwmgr *hwmgr, char *buf);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
index 65333141b1c1..7789b313285c 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
@@ -45,6 +45,7 @@
 #include "smu_v13_0_6_ppt.h"
 #include "smu_v13_0_7_ppt.h"
 #include "smu_v14_0_0_ppt.h"
+#include "smu_v14_0_2_ppt.h"
 #include "amd_pcie.h"
 
 /*
@@ -715,6 +716,10 @@ static int smu_set_funcs(struct amdgpu_device *adev)
 	case IP_VERSION(14, 0, 1):
 		smu_v14_0_0_set_ppt_funcs(smu);
 		break;
+	case IP_VERSION(14, 0, 2):
+	case IP_VERSION(14, 0, 3):
+		smu_v14_0_2_set_ppt_funcs(smu);
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -737,6 +742,7 @@ static int smu_early_init(void *handle)
 	smu->is_apu = false;
 	smu->smu_baco.state = SMU_BACO_STATE_NONE;
 	smu->smu_baco.platform_support = false;
+	smu->smu_baco.maco_support = false;
 	smu->user_dpm_profile.fan_mode = -1;
 
 	mutex_init(&smu->message_lock);
@@ -3223,17 +3229,17 @@ static int smu_set_xgmi_pstate(void *handle,
 	return ret;
 }
 
-static bool smu_get_baco_capability(void *handle)
+static int smu_get_baco_capability(void *handle)
 {
 	struct smu_context *smu = handle;
 
 	if (!smu->pm_enabled)
 		return false;
 
-	if (!smu->ppt_funcs || !smu->ppt_funcs->baco_is_support)
+	if (!smu->ppt_funcs || !smu->ppt_funcs->get_bamaco_support)
 		return false;
 
-	return smu->ppt_funcs->baco_is_support(smu);
+	return smu->ppt_funcs->get_bamaco_support(smu);
 }
 
 static int smu_baco_set_state(void *handle, int state)
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
index 1fa81575788c..0917dec8efe3 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
@@ -459,7 +459,7 @@ struct smu_umd_pstate_table {
 struct cmn2asic_msg_mapping {
 	int	valid_mapping;
 	int	map_to;
-	int	valid_in_vf;
+	uint32_t flags;
 };
 
 struct cmn2asic_mapping {
@@ -539,6 +539,7 @@ struct smu_context {
 	uint32_t smc_driver_if_version;
 	uint32_t smc_fw_if_version;
 	uint32_t smc_fw_version;
+	uint32_t smc_fw_caps;
 
 	bool uploading_custom_pp_table;
 	bool dc_controlled_by_gpio;
@@ -1174,9 +1175,11 @@ struct pptable_funcs {
 	int (*get_max_sustainable_clocks_by_dc)(struct smu_context *smu, struct pp_smu_nv_clock_table *max_clocks);
 
 	/**
-	 * @baco_is_support: Check if GPU supports BACO (Bus Active, Chip Off).
+	 * @get_bamaco_support: Check if GPU supports BACO/MACO
+	 * BACO: Bus Active, Chip Off
+	 * MACO: Memory Active, Chip Off
 	 */
-	bool (*baco_is_support)(struct smu_context *smu);
+	int (*get_bamaco_support)(struct smu_context *smu);
 
 	/**
 	 * @baco_get_state: Get the current BACO state.
@@ -1483,8 +1486,8 @@ enum smu_baco_seq {
 	BACO_SEQ_COUNT,
 };
 
-#define MSG_MAP(msg, index, valid_in_vf) \
-	[SMU_MSG_##msg] = {1, (index), (valid_in_vf)}
+#define MSG_MAP(msg, index, flags) \
+	[SMU_MSG_##msg] = {1, (index), (flags)}
 
 #define CLK_MAP(clk, index) \
 	[SMU_##clk] = {1, (index)}
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu14_driver_if_v14_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu14_driver_if_v14_0.h
new file mode 100644
index 000000000000..97a29b80fb13
--- /dev/null
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu14_driver_if_v14_0.h
@@ -0,0 +1,1836 @@
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef SMU14_DRIVER_IF_V14_0_H
+#define SMU14_DRIVER_IF_V14_0_H
+
+//Increment this version if SkuTable_t or BoardTable_t change
+#define PPTABLE_VERSION 0x18
+
+#define NUM_GFXCLK_DPM_LEVELS    16
+#define NUM_SOCCLK_DPM_LEVELS    8
+#define NUM_MP0CLK_DPM_LEVELS    2
+#define NUM_DCLK_DPM_LEVELS      8
+#define NUM_VCLK_DPM_LEVELS      8
+#define NUM_DISPCLK_DPM_LEVELS   8
+#define NUM_DPPCLK_DPM_LEVELS    8
+#define NUM_DPREFCLK_DPM_LEVELS  8
+#define NUM_DCFCLK_DPM_LEVELS    8
+#define NUM_DTBCLK_DPM_LEVELS    8
+#define NUM_UCLK_DPM_LEVELS      6
+#define NUM_LINK_LEVELS          3
+#define NUM_FCLK_DPM_LEVELS      8
+#define NUM_OD_FAN_MAX_POINTS    6
+
+// Feature Control Defines
+#define FEATURE_FW_DATA_READ_BIT              0
+#define FEATURE_DPM_GFXCLK_BIT                1
+#define FEATURE_DPM_GFX_POWER_OPTIMIZER_BIT   2
+#define FEATURE_DPM_UCLK_BIT                  3
+#define FEATURE_DPM_FCLK_BIT                  4
+#define FEATURE_DPM_SOCCLK_BIT                5
+#define FEATURE_DPM_LINK_BIT                  6
+#define FEATURE_DPM_DCN_BIT                   7
+#define FEATURE_VMEMP_SCALING_BIT             8
+#define FEATURE_VDDIO_MEM_SCALING_BIT         9
+#define FEATURE_DS_GFXCLK_BIT                 10
+#define FEATURE_DS_SOCCLK_BIT                 11
+#define FEATURE_DS_FCLK_BIT                   12
+#define FEATURE_DS_LCLK_BIT                   13
+#define FEATURE_DS_DCFCLK_BIT                 14
+#define FEATURE_DS_UCLK_BIT                   15
+#define FEATURE_GFX_ULV_BIT                   16
+#define FEATURE_FW_DSTATE_BIT                 17
+#define FEATURE_GFXOFF_BIT                    18
+#define FEATURE_BACO_BIT                      19
+#define FEATURE_MM_DPM_BIT                    20
+#define FEATURE_SOC_MPCLK_DS_BIT              21
+#define FEATURE_BACO_MPCLK_DS_BIT             22
+#define FEATURE_THROTTLERS_BIT                23
+#define FEATURE_SMARTSHIFT_BIT                24
+#define FEATURE_GTHR_BIT                      25
+#define FEATURE_ACDC_BIT                      26
+#define FEATURE_VR0HOT_BIT                    27
+#define FEATURE_FW_CTF_BIT                    28
+#define FEATURE_FAN_CONTROL_BIT               29
+#define FEATURE_GFX_DCS_BIT                   30
+#define FEATURE_GFX_READ_MARGIN_BIT           31
+#define FEATURE_LED_DISPLAY_BIT               32
+#define FEATURE_GFXCLK_SPREAD_SPECTRUM_BIT    33
+#define FEATURE_OUT_OF_BAND_MONITOR_BIT       34
+#define FEATURE_OPTIMIZED_VMIN_BIT            35
+#define FEATURE_GFX_IMU_BIT                   36
+#define FEATURE_BOOT_TIME_CAL_BIT             37
+#define FEATURE_GFX_PCC_DFLL_BIT              38
+#define FEATURE_SOC_CG_BIT                    39
+#define FEATURE_DF_CSTATE_BIT                 40
+#define FEATURE_GFX_EDC_BIT                   41
+#define FEATURE_BOOT_POWER_OPT_BIT            42
+#define FEATURE_CLOCK_POWER_DOWN_BYPASS_BIT   43
+#define FEATURE_DS_VCN_BIT                    44
+#define FEATURE_BACO_CG_BIT                   45
+#define FEATURE_MEM_TEMP_READ_BIT             46
+#define FEATURE_ATHUB_MMHUB_PG_BIT            47
+#define FEATURE_SOC_PCC_BIT                   48
+#define FEATURE_EDC_PWRBRK_BIT                49
+#define FEATURE_SOC_EDC_XVMIN_BIT             50
+#define FEATURE_GFX_PSM_DIDT_BIT              51
+#define FEATURE_APT_ALL_ENABLE_BIT            52
+#define FEATURE_APT_SQ_THROTTLE_BIT           53
+#define FEATURE_APT_PF_DCS_BIT                54
+#define FEATURE_GFX_EDC_XVMIN_BIT             55
+#define FEATURE_GFX_DIDT_XVMIN_BIT            56
+#define FEATURE_FAN_ABNORMAL_BIT              57
+#define FEATURE_CLOCK_STRETCH_COMPENSATOR     58
+#define FEATURE_SPARE_59_BIT                  59
+#define FEATURE_SPARE_60_BIT                  60
+#define FEATURE_SPARE_61_BIT                  61
+#define FEATURE_SPARE_62_BIT                  62
+#define FEATURE_SPARE_63_BIT                  63
+#define NUM_FEATURES                          64
+
+#define ALLOWED_FEATURE_CTRL_DEFAULT 0xFFFFFFFFFFFFFFFFULL
+#define ALLOWED_FEATURE_CTRL_SCPM        (1 << FEATURE_DPM_GFXCLK_BIT) | \
+                                         (1 << FEATURE_DPM_GFX_POWER_OPTIMIZER_BIT) | \
+                                         (1 << FEATURE_DPM_UCLK_BIT) | \
+                                         (1 << FEATURE_DPM_FCLK_BIT) | \
+                                         (1 << FEATURE_DPM_SOCCLK_BIT) | \
+                                         (1 << FEATURE_DPM_LINK_BIT) | \
+                                         (1 << FEATURE_DPM_DCN_BIT) | \
+                                         (1 << FEATURE_DS_GFXCLK_BIT) | \
+                                         (1 << FEATURE_DS_SOCCLK_BIT) | \
+                                         (1 << FEATURE_DS_FCLK_BIT) | \
+                                         (1 << FEATURE_DS_LCLK_BIT) | \
+                                         (1 << FEATURE_DS_DCFCLK_BIT) | \
+                                         (1 << FEATURE_DS_UCLK_BIT) | \
+                                         (1ULL << FEATURE_DS_VCN_BIT)
+
+
+//For use with feature control messages
+typedef enum {
+  FEATURE_PWR_ALL,
+  FEATURE_PWR_S5,
+  FEATURE_PWR_BACO,
+  FEATURE_PWR_SOC,
+  FEATURE_PWR_GFX,
+  FEATURE_PWR_DOMAIN_COUNT,
+} FEATURE_PWR_DOMAIN_e;
+
+//For use with feature control + BTC save restore
+typedef enum {
+  FEATURE_BTC_NOP,
+  FEATURE_BTC_SAVE,
+  FEATURE_BTC_RESTORE,
+  FEATURE_BTC_COUNT,
+} FEATURE_BTC_e;
+
+// Debug Overrides Bitmask
+#define DEBUG_OVERRIDE_DISABLE_VOLT_LINK_VCN_FCLK      0x00000001
+#define DEBUG_OVERRIDE_DISABLE_VOLT_LINK_DCN_FCLK      0x00000002
+#define DEBUG_OVERRIDE_DISABLE_VOLT_LINK_MP0_FCLK      0x00000004
+#define DEBUG_OVERRIDE_DISABLE_VOLT_LINK_VCN_DCFCLK    0x00000008
+#define DEBUG_OVERRIDE_DISABLE_FAST_FCLK_TIMER         0x00000010
+#define DEBUG_OVERRIDE_DISABLE_VCN_PG                  0x00000020
+#define DEBUG_OVERRIDE_DISABLE_FMAX_VMAX               0x00000040
+#define DEBUG_OVERRIDE_DISABLE_IMU_FW_CHECKS           0x00000080
+#define DEBUG_OVERRIDE_DISABLE_D0i2_REENTRY_HSR_TIMER_CHECK 0x00000100
+#define DEBUG_OVERRIDE_DISABLE_DFLL                    0x00000200
+#define DEBUG_OVERRIDE_ENABLE_RLC_VF_BRINGUP_MODE      0x00000400
+#define DEBUG_OVERRIDE_DFLL_MASTER_MODE                0x00000800
+#define DEBUG_OVERRIDE_ENABLE_PROFILING_MODE           0x00001000
+#define DEBUG_OVERRIDE_ENABLE_SOC_VF_BRINGUP_MODE      0x00002000
+#define DEBUG_OVERRIDE_ENABLE_PER_WGP_RESIENCY         0x00004000
+#define DEBUG_OVERRIDE_DISABLE_MEMORY_VOLTAGE_SCALING  0x00008000
+
+// VR Mapping Bit Defines
+#define VR_MAPPING_VR_SELECT_MASK  0x01
+#define VR_MAPPING_VR_SELECT_SHIFT 0x00
+
+#define VR_MAPPING_PLANE_SELECT_MASK  0x02
+#define VR_MAPPING_PLANE_SELECT_SHIFT 0x01
+
+// PSI Bit Defines
+#define PSI_SEL_VR0_PLANE0_PSI0  0x01
+#define PSI_SEL_VR0_PLANE0_PSI1  0x02
+#define PSI_SEL_VR0_PLANE1_PSI0  0x04
+#define PSI_SEL_VR0_PLANE1_PSI1  0x08
+#define PSI_SEL_VR1_PLANE0_PSI0  0x10
+#define PSI_SEL_VR1_PLANE0_PSI1  0x20
+#define PSI_SEL_VR1_PLANE1_PSI0  0x40
+#define PSI_SEL_VR1_PLANE1_PSI1  0x80
+
+typedef enum {
+  SVI_PSI_0, // Full phase count (default)
+  SVI_PSI_1, // Phase count 1st level
+  SVI_PSI_2, // Phase count 2nd level
+  SVI_PSI_3, // Single phase operation + active diode emulation
+  SVI_PSI_4, // Single phase operation + passive diode emulation *optional*
+  SVI_PSI_5, // Reserved
+  SVI_PSI_6, // Power down to 0V (voltage regulation disabled)
+  SVI_PSI_7, // Automated phase shedding and diode emulation
+} SVI_PSI_e;
+
+// Throttler Control/Status Bits
+#define THROTTLER_TEMP_EDGE_BIT        0
+#define THROTTLER_TEMP_HOTSPOT_BIT     1
+#define THROTTLER_TEMP_HOTSPOT_GFX_BIT 2
+#define THROTTLER_TEMP_HOTSPOT_SOC_BIT 3
+#define THROTTLER_TEMP_MEM_BIT         4
+#define THROTTLER_TEMP_VR_GFX_BIT      5
+#define THROTTLER_TEMP_VR_SOC_BIT      6
+#define THROTTLER_TEMP_VR_MEM0_BIT     7
+#define THROTTLER_TEMP_VR_MEM1_BIT     8
+#define THROTTLER_TEMP_LIQUID0_BIT     9
+#define THROTTLER_TEMP_LIQUID1_BIT     10
+#define THROTTLER_TEMP_PLX_BIT         11
+#define THROTTLER_TDC_GFX_BIT          12
+#define THROTTLER_TDC_SOC_BIT          13
+#define THROTTLER_PPT0_BIT             14
+#define THROTTLER_PPT1_BIT             15
+#define THROTTLER_PPT2_BIT             16
+#define THROTTLER_PPT3_BIT             17
+#define THROTTLER_FIT_BIT              18
+#define THROTTLER_GFX_APCC_PLUS_BIT    19
+#define THROTTLER_GFX_DVO_BIT          20
+#define THROTTLER_COUNT                21
+
+// FW DState Features Control Bits
+#define FW_DSTATE_SOC_ULV_BIT               0
+#define FW_DSTATE_G6_HSR_BIT                1
+#define FW_DSTATE_G6_PHY_VMEMP_OFF_BIT      2
+#define FW_DSTATE_SMN_DS_BIT                3
+#define FW_DSTATE_MP1_WHISPER_MODE_BIT      4
+#define FW_DSTATE_SOC_LIV_MIN_BIT           5
+#define FW_DSTATE_SOC_PLL_PWRDN_BIT         6
+#define FW_DSTATE_MEM_PLL_PWRDN_BIT         7
+#define FW_DSTATE_MALL_ALLOC_BIT            8
+#define FW_DSTATE_MEM_PSI_BIT               9
+#define FW_DSTATE_HSR_NON_STROBE_BIT        10
+#define FW_DSTATE_MP0_ENTER_WFI_BIT         11
+#define FW_DSTATE_MALL_FLUSH_BIT            12
+#define FW_DSTATE_SOC_PSI_BIT               13
+#define FW_DSTATE_MMHUB_INTERLOCK_BIT       14
+#define FW_DSTATE_D0i3_2_QUIET_FW_BIT       15
+#define FW_DSTATE_CLDO_PRG_BIT              16
+#define FW_DSTATE_DF_PLL_PWRDN_BIT          17
+
+//LED Display Mask & Control Bits
+#define LED_DISPLAY_GFX_DPM_BIT            0
+#define LED_DISPLAY_PCIE_BIT               1
+#define LED_DISPLAY_ERROR_BIT              2
+
+
+#define MEM_TEMP_READ_OUT_OF_BAND_BIT          0
+#define MEM_TEMP_READ_IN_BAND_REFRESH_BIT      1
+#define MEM_TEMP_READ_IN_BAND_DUMMY_PSTATE_BIT 2
+
+typedef enum {
+  SMARTSHIFT_VERSION_1,
+  SMARTSHIFT_VERSION_2,
+  SMARTSHIFT_VERSION_3,
+} SMARTSHIFT_VERSION_e;
+
+typedef enum {
+  FOPT_CALC_AC_CALC_DC,
+  FOPT_PPTABLE_AC_CALC_DC,
+  FOPT_CALC_AC_PPTABLE_DC,
+  FOPT_PPTABLE_AC_PPTABLE_DC,
+} FOPT_CALC_e;
+
+typedef enum {
+  DRAM_BIT_WIDTH_DISABLED = 0,
+  DRAM_BIT_WIDTH_X_8 = 8,
+  DRAM_BIT_WIDTH_X_16 = 16,
+  DRAM_BIT_WIDTH_X_32 = 32,
+  DRAM_BIT_WIDTH_X_64 = 64,
+  DRAM_BIT_WIDTH_X_128 = 128,
+  DRAM_BIT_WIDTH_COUNT,
+} DRAM_BIT_WIDTH_TYPE_e;
+
+//I2C Interface
+#define NUM_I2C_CONTROLLERS                8
+
+#define I2C_CONTROLLER_ENABLED             1
+#define I2C_CONTROLLER_DISABLED            0
+
+#define MAX_SW_I2C_COMMANDS                24
+
+typedef enum {
+  I2C_CONTROLLER_PORT_0 = 0,  //CKSVII2C0
+  I2C_CONTROLLER_PORT_1 = 1,  //CKSVII2C1
+  I2C_CONTROLLER_PORT_COUNT,
+} I2cControllerPort_e;
+
+typedef enum {
+  I2C_CONTROLLER_NAME_VR_GFX = 0,
+  I2C_CONTROLLER_NAME_VR_SOC,
+  I2C_CONTROLLER_NAME_VR_VMEMP,
+  I2C_CONTROLLER_NAME_VR_VDDIO,
+  I2C_CONTROLLER_NAME_LIQUID0,
+  I2C_CONTROLLER_NAME_LIQUID1,
+  I2C_CONTROLLER_NAME_PLX,
+  I2C_CONTROLLER_NAME_FAN_INTAKE,
+  I2C_CONTROLLER_NAME_COUNT,
+} I2cControllerName_e;
+
+typedef enum {
+  I2C_CONTROLLER_THROTTLER_TYPE_NONE = 0,
+  I2C_CONTROLLER_THROTTLER_VR_GFX,
+  I2C_CONTROLLER_THROTTLER_VR_SOC,
+  I2C_CONTROLLER_THROTTLER_VR_VMEMP,
+  I2C_CONTROLLER_THROTTLER_VR_VDDIO,
+  I2C_CONTROLLER_THROTTLER_LIQUID0,
+  I2C_CONTROLLER_THROTTLER_LIQUID1,
+  I2C_CONTROLLER_THROTTLER_PLX,
+  I2C_CONTROLLER_THROTTLER_FAN_INTAKE,
+  I2C_CONTROLLER_THROTTLER_INA3221,
+  I2C_CONTROLLER_THROTTLER_COUNT,
+} I2cControllerThrottler_e;
+
+typedef enum {
+  I2C_CONTROLLER_PROTOCOL_VR_XPDE132G5,
+  I2C_CONTROLLER_PROTOCOL_VR_IR35217,
+  I2C_CONTROLLER_PROTOCOL_TMP_MAX31875,
+  I2C_CONTROLLER_PROTOCOL_INA3221,
+  I2C_CONTROLLER_PROTOCOL_TMP_MAX6604,
+  I2C_CONTROLLER_PROTOCOL_COUNT,
+} I2cControllerProtocol_e;
+
+typedef struct {
+  uint8_t   Enabled;
+  uint8_t   Speed;
+  uint8_t   SlaveAddress;
+  uint8_t   ControllerPort;
+  uint8_t   ControllerName;
+  uint8_t   ThermalThrotter;
+  uint8_t   I2cProtocol;
+  uint8_t   PaddingConfig;
+} I2cControllerConfig_t;
+
+typedef enum {
+  I2C_PORT_SVD_SCL = 0,
+  I2C_PORT_GPIO,
+} I2cPort_e;
+
+typedef enum {
+  I2C_SPEED_FAST_50K = 0,      //50  Kbits/s
+  I2C_SPEED_FAST_100K,         //100 Kbits/s
+  I2C_SPEED_FAST_400K,         //400 Kbits/s
+  I2C_SPEED_FAST_PLUS_1M,      //1   Mbits/s (in fast mode)
+  I2C_SPEED_HIGH_1M,           //1   Mbits/s (in high speed mode)
+  I2C_SPEED_HIGH_2M,           //2.3 Mbits/s
+  I2C_SPEED_COUNT,
+} I2cSpeed_e;
+
+typedef enum {
+  I2C_CMD_READ = 0,
+  I2C_CMD_WRITE,
+  I2C_CMD_COUNT,
+} I2cCmdType_e;
+
+#define CMDCONFIG_STOP_BIT             0
+#define CMDCONFIG_RESTART_BIT          1
+#define CMDCONFIG_READWRITE_BIT        2 //bit should be 0 for read, 1 for write
+
+#define CMDCONFIG_STOP_MASK           (1 << CMDCONFIG_STOP_BIT)
+#define CMDCONFIG_RESTART_MASK        (1 << CMDCONFIG_RESTART_BIT)
+#define CMDCONFIG_READWRITE_MASK      (1 << CMDCONFIG_READWRITE_BIT)
+
+typedef struct {
+  uint8_t ReadWriteData;  //Return data for read. Data to send for write
+  uint8_t CmdConfig; //Includes whether associated command should have a stop or restart command, and is a read or write
+} SwI2cCmd_t; //SW I2C Command Table
+
+typedef struct {
+  uint8_t     I2CcontrollerPort; //CKSVII2C0(0) or //CKSVII2C1(1)
+  uint8_t     I2CSpeed;          //Use I2cSpeed_e to indicate speed to select
+  uint8_t     SlaveAddress;      //Slave address of device
+  uint8_t     NumCmds;           //Number of commands
+
+  SwI2cCmd_t  SwI2cCmds[MAX_SW_I2C_COMMANDS];
+} SwI2cRequest_t; // SW I2C Request Table
+
+typedef struct {
+  SwI2cRequest_t SwI2cRequest;
+
+  uint32_t Spare[8];
+  uint32_t MmHubPadding[8]; // SMU internal use
+} SwI2cRequestExternal_t;
+
+typedef struct {
+  uint64_t mca_umc_status;
+  uint64_t mca_umc_addr;
+
+  uint16_t ce_count_lo_chip;
+  uint16_t ce_count_hi_chip;
+
+  uint32_t eccPadding;
+} EccInfo_t;
+
+typedef struct {
+  EccInfo_t  EccInfo[24];
+} EccInfoTable_t;
+
+//D3HOT sequences
+typedef enum {
+  BACO_SEQUENCE,
+  MSR_SEQUENCE,
+  BAMACO_SEQUENCE,
+  ULPS_SEQUENCE,
+  D3HOT_SEQUENCE_COUNT,
+} D3HOTSequence_e;
+
+//This is aligned with RSMU PGFSM Register Mapping
+typedef enum {
+  PG_DYNAMIC_MODE = 0,
+  PG_STATIC_MODE,
+} PowerGatingMode_e;
+
+//This is aligned with RSMU PGFSM Register Mapping
+typedef enum {
+  PG_POWER_DOWN = 0,
+  PG_POWER_UP,
+} PowerGatingSettings_e;
+
+typedef struct {
+  uint32_t a;  // store in IEEE float format in this variable
+  uint32_t b;  // store in IEEE float format in this variable
+  uint32_t c;  // store in IEEE float format in this variable
+} QuadraticInt_t;
+
+typedef struct {
+  uint32_t m;  // store in IEEE float format in this variable
+  uint32_t b;  // store in IEEE float format in this variable
+} LinearInt_t;
+
+typedef struct {
+  uint32_t a;  // store in IEEE float format in this variable
+  uint32_t b;  // store in IEEE float format in this variable
+  uint32_t c;  // store in IEEE float format in this variable
+} DroopInt_t;
+
+typedef enum {
+  DCS_ARCH_DISABLED,
+  DCS_ARCH_FADCS,
+  DCS_ARCH_ASYNC,
+} DCS_ARCH_e;
+
+//Only Clks that have DPM descriptors are listed here
+typedef enum {
+  PPCLK_GFXCLK = 0,
+  PPCLK_SOCCLK,
+  PPCLK_UCLK,
+  PPCLK_FCLK,
+  PPCLK_DCLK_0,
+  PPCLK_VCLK_0,
+  PPCLK_DISPCLK,
+  PPCLK_DPPCLK,
+  PPCLK_DPREFCLK,
+  PPCLK_DCFCLK,
+  PPCLK_DTBCLK,
+  PPCLK_COUNT,
+} PPCLK_e;
+
+typedef enum {
+  VOLTAGE_MODE_PPTABLE = 0,
+  VOLTAGE_MODE_FUSES,
+  VOLTAGE_MODE_COUNT,
+} VOLTAGE_MODE_e;
+
+typedef enum {
+  AVFS_VOLTAGE_GFX = 0,
+  AVFS_VOLTAGE_SOC,
+  AVFS_VOLTAGE_COUNT,
+} AVFS_VOLTAGE_TYPE_e;
+
+typedef enum {
+  AVFS_TEMP_COLD = 0,
+  AVFS_TEMP_HOT,
+  AVFS_TEMP_COUNT,
+} AVFS_TEMP_e;
+
+typedef enum {
+  AVFS_D_G,
+  AVFS_D_COUNT,
+} AVFS_D_e;
+
+
+typedef enum {
+  UCLK_DIV_BY_1 = 0,
+  UCLK_DIV_BY_2,
+  UCLK_DIV_BY_4,
+  UCLK_DIV_BY_8,
+} UCLK_DIV_e;
+
+typedef enum {
+  GPIO_INT_POLARITY_ACTIVE_LOW = 0,
+  GPIO_INT_POLARITY_ACTIVE_HIGH,
+} GpioIntPolarity_e;
+
+typedef enum {
+  PWR_CONFIG_TDP = 0,
+  PWR_CONFIG_TGP,
+  PWR_CONFIG_TCP_ESTIMATED,
+  PWR_CONFIG_TCP_MEASURED,
+  PWR_CONFIG_TBP_DESKTOP,
+  PWR_CONFIG_TBP_MOBILE,
+} PwrConfig_e;
+
+typedef struct {
+  uint8_t        Padding;
+  uint8_t        SnapToDiscrete;      // 0 - Fine grained DPM, 1 - Discrete DPM
+  uint8_t        NumDiscreteLevels;   // Set to 2 (Fmin, Fmax) when using fine grained DPM, otherwise set to # discrete levels used
+  uint8_t        CalculateFopt;       // Indication whether FW should calculate Fopt or use values below. Reference FOPT_CALC_e
+  LinearInt_t    ConversionToAvfsClk; // Transfer function to AVFS Clock (GHz->GHz)
+  uint32_t       Padding3[3];
+  uint16_t       Padding4;
+  uint16_t       FoptimalDc;          //Foptimal frequency in DC power mode.
+  uint16_t       FoptimalAc;          //Foptimal frequency in AC power mode.
+  uint16_t       Padding2;
+} DpmDescriptor_t;
+
+typedef enum  {
+  PPT_THROTTLER_PPT0,
+  PPT_THROTTLER_PPT1,
+  PPT_THROTTLER_PPT2,
+  PPT_THROTTLER_PPT3,
+  PPT_THROTTLER_COUNT
+} PPT_THROTTLER_e;
+
+typedef enum  {
+  TEMP_EDGE,
+  TEMP_HOTSPOT,
+  TEMP_HOTSPOT_GFX,
+  TEMP_HOTSPOT_SOC,
+  TEMP_MEM,
+  TEMP_VR_GFX,
+  TEMP_VR_SOC,
+  TEMP_VR_MEM0,
+  TEMP_VR_MEM1,
+  TEMP_LIQUID0,
+  TEMP_LIQUID1,
+  TEMP_PLX,
+  TEMP_COUNT,
+} TEMP_e;
+
+typedef enum {
+  TDC_THROTTLER_GFX,
+  TDC_THROTTLER_SOC,
+  TDC_THROTTLER_COUNT
+} TDC_THROTTLER_e;
+
+typedef enum {
+  SVI_PLANE_VDD_GFX,
+  SVI_PLANE_VDD_SOC,
+  SVI_PLANE_VDDCI_MEM,
+  SVI_PLANE_VDDIO_MEM,
+  SVI_PLANE_COUNT,
+} SVI_PLANE_e;
+
+typedef enum {
+  PMFW_VOLT_PLANE_GFX,
+  PMFW_VOLT_PLANE_SOC,
+  PMFW_VOLT_PLANE_COUNT
+} PMFW_VOLT_PLANE_e;
+
+typedef enum {
+  CUSTOMER_VARIANT_ROW,
+  CUSTOMER_VARIANT_FALCON,
+  CUSTOMER_VARIANT_COUNT,
+} CUSTOMER_VARIANT_e;
+
+typedef enum {
+  POWER_SOURCE_AC,
+  POWER_SOURCE_DC,
+  POWER_SOURCE_COUNT,
+} POWER_SOURCE_e;
+
+typedef enum {
+  MEM_VENDOR_PLACEHOLDER0,  // 0
+  MEM_VENDOR_SAMSUNG,       // 1
+  MEM_VENDOR_INFINEON,      // 2
+  MEM_VENDOR_ELPIDA,        // 3
+  MEM_VENDOR_ETRON,         // 4
+  MEM_VENDOR_NANYA,         // 5
+  MEM_VENDOR_HYNIX,         // 6
+  MEM_VENDOR_MOSEL,         // 7
+  MEM_VENDOR_WINBOND,       // 8
+  MEM_VENDOR_ESMT,          // 9
+  MEM_VENDOR_PLACEHOLDER1,  // 10
+  MEM_VENDOR_PLACEHOLDER2,  // 11
+  MEM_VENDOR_PLACEHOLDER3,  // 12
+  MEM_VENDOR_PLACEHOLDER4,  // 13
+  MEM_VENDOR_PLACEHOLDER5,  // 14
+  MEM_VENDOR_MICRON,        // 15
+  MEM_VENDOR_COUNT,
+} MEM_VENDOR_e;
+
+typedef enum {
+  PP_GRTAVFS_HW_CPO_CTL_ZONE0,
+  PP_GRTAVFS_HW_CPO_CTL_ZONE1,
+  PP_GRTAVFS_HW_CPO_CTL_ZONE2,
+  PP_GRTAVFS_HW_CPO_CTL_ZONE3,
+  PP_GRTAVFS_HW_CPO_CTL_ZONE4,
+  PP_GRTAVFS_HW_CPO_EN_0_31_ZONE0,
+  PP_GRTAVFS_HW_CPO_EN_32_63_ZONE0,
+  PP_GRTAVFS_HW_CPO_EN_0_31_ZONE1,
+  PP_GRTAVFS_HW_CPO_EN_32_63_ZONE1,
+  PP_GRTAVFS_HW_CPO_EN_0_31_ZONE2,
+  PP_GRTAVFS_HW_CPO_EN_32_63_ZONE2,
+  PP_GRTAVFS_HW_CPO_EN_0_31_ZONE3,
+  PP_GRTAVFS_HW_CPO_EN_32_63_ZONE3,
+  PP_GRTAVFS_HW_CPO_EN_0_31_ZONE4,
+  PP_GRTAVFS_HW_CPO_EN_32_63_ZONE4,
+  PP_GRTAVFS_HW_ZONE0_VF,
+  PP_GRTAVFS_HW_ZONE1_VF1,
+  PP_GRTAVFS_HW_ZONE2_VF2,
+  PP_GRTAVFS_HW_ZONE3_VF3,
+  PP_GRTAVFS_HW_VOLTAGE_GB,
+  PP_GRTAVFS_HW_CPOSCALINGCTRL_ZONE0,
+  PP_GRTAVFS_HW_CPOSCALINGCTRL_ZONE1,
+  PP_GRTAVFS_HW_CPOSCALINGCTRL_ZONE2,
+  PP_GRTAVFS_HW_CPOSCALINGCTRL_ZONE3,
+  PP_GRTAVFS_HW_CPOSCALINGCTRL_ZONE4,
+  PP_GRTAVFS_HW_RESERVED_0,
+  PP_GRTAVFS_HW_RESERVED_1,
+  PP_GRTAVFS_HW_RESERVED_2,
+  PP_GRTAVFS_HW_RESERVED_3,
+  PP_GRTAVFS_HW_RESERVED_4,
+  PP_GRTAVFS_HW_RESERVED_5,
+  PP_GRTAVFS_HW_RESERVED_6,
+  PP_GRTAVFS_HW_FUSE_COUNT,
+} PP_GRTAVFS_HW_FUSE_e;
+
+typedef enum {
+  PP_GRTAVFS_FW_COMMON_PPVMIN_Z1_HOT_T0,
+  PP_GRTAVFS_FW_COMMON_PPVMIN_Z1_COLD_T0,
+  PP_GRTAVFS_FW_COMMON_PPVMIN_Z2_HOT_T0,
+  PP_GRTAVFS_FW_COMMON_PPVMIN_Z2_COLD_T0,
+  PP_GRTAVFS_FW_COMMON_PPVMIN_Z3_HOT_T0,
+  PP_GRTAVFS_FW_COMMON_PPVMIN_Z3_COLD_T0,
+  PP_GRTAVFS_FW_COMMON_PPVMIN_Z4_HOT_T0,
+  PP_GRTAVFS_FW_COMMON_PPVMIN_Z4_COLD_T0,
+  PP_GRTAVFS_FW_COMMON_SRAM_RM_Z0,
+  PP_GRTAVFS_FW_COMMON_SRAM_RM_Z1,
+  PP_GRTAVFS_FW_COMMON_SRAM_RM_Z2,
+  PP_GRTAVFS_FW_COMMON_SRAM_RM_Z3,
+  PP_GRTAVFS_FW_COMMON_SRAM_RM_Z4,
+  PP_GRTAVFS_FW_COMMON_FUSE_COUNT,
+} PP_GRTAVFS_FW_COMMON_FUSE_e;
+
+typedef enum {
+  PP_GRTAVFS_FW_SEP_FUSE_GB1_PWL_VOLTAGE_NEG_1,
+  PP_GRTAVFS_FW_SEP_FUSE_GB1_PWL_VOLTAGE_0,
+  PP_GRTAVFS_FW_SEP_FUSE_GB1_PWL_VOLTAGE_1,
+  PP_GRTAVFS_FW_SEP_FUSE_GB1_PWL_VOLTAGE_2,
+  PP_GRTAVFS_FW_SEP_FUSE_GB1_PWL_VOLTAGE_3,
+  PP_GRTAVFS_FW_SEP_FUSE_GB1_PWL_VOLTAGE_4,
+  PP_GRTAVFS_FW_SEP_FUSE_GB2_PWL_VOLTAGE_NEG_1,
+  PP_GRTAVFS_FW_SEP_FUSE_GB2_PWL_VOLTAGE_0,
+  PP_GRTAVFS_FW_SEP_FUSE_GB2_PWL_VOLTAGE_1,
+  PP_GRTAVFS_FW_SEP_FUSE_GB2_PWL_VOLTAGE_2,
+  PP_GRTAVFS_FW_SEP_FUSE_GB2_PWL_VOLTAGE_3,
+  PP_GRTAVFS_FW_SEP_FUSE_GB2_PWL_VOLTAGE_4,
+  PP_GRTAVFS_FW_SEP_FUSE_VF_NEG_1_FREQUENCY,
+  PP_GRTAVFS_FW_SEP_FUSE_VF4_FREQUENCY,
+  PP_GRTAVFS_FW_SEP_FUSE_FREQUENCY_TO_COUNT_SCALER_0,
+  PP_GRTAVFS_FW_SEP_FUSE_FREQUENCY_TO_COUNT_SCALER_1,
+  PP_GRTAVFS_FW_SEP_FUSE_FREQUENCY_TO_COUNT_SCALER_2,
+  PP_GRTAVFS_FW_SEP_FUSE_FREQUENCY_TO_COUNT_SCALER_3,
+  PP_GRTAVFS_FW_SEP_FUSE_FREQUENCY_TO_COUNT_SCALER_4,
+  PP_GRTAVFS_FW_SEP_FUSE_COUNT,
+} PP_GRTAVFS_FW_SEP_FUSE_e;
+
+#define PP_NUM_RTAVFS_PWL_ZONES 5
+
+
+// VBIOS or PPLIB configures telemetry slope and offset. Only slope expected to be set for SVI3
+// Slope Q1.7, Offset Q1.2
+typedef struct {
+  int8_t   Offset; // in Amps
+  uint8_t  Padding;
+  uint16_t MaxCurrent; // in Amps
+} SviTelemetryScale_t;
+
+#define PP_NUM_OD_VF_CURVE_POINTS PP_NUM_RTAVFS_PWL_ZONES + 1
+
+#define PP_OD_FEATURE_GFX_VF_CURVE_BIT       0
+#define PP_OD_FEATURE_GFX_VMAX_BIT           1
+#define PP_OD_FEATURE_SOC_VMAX_BIT           2
+#define PP_OD_FEATURE_PPT_BIT                3
+#define PP_OD_FEATURE_FAN_CURVE_BIT          4
+#define PP_OD_FEATURE_FAN_LEGACY_BIT         5
+#define PP_OD_FEATURE_FULL_CTRL_BIT          6
+#define PP_OD_FEATURE_TDC_BIT                7
+#define PP_OD_FEATURE_GFXCLK_BIT             8
+#define PP_OD_FEATURE_UCLK_BIT               9
+#define PP_OD_FEATURE_FCLK_BIT               10
+#define PP_OD_FEATURE_ZERO_FAN_BIT           11
+#define PP_OD_FEATURE_TEMPERATURE_BIT        12
+#define PP_OD_FEATURE_EDC_BIT                13
+#define PP_OD_FEATURE_COUNT                  14
+
+typedef enum {
+  PP_OD_POWER_FEATURE_ALWAYS_ENABLED,
+  PP_OD_POWER_FEATURE_DISABLED_WHILE_GAMING,
+  PP_OD_POWER_FEATURE_ALWAYS_DISABLED,
+} PP_OD_POWER_FEATURE_e;
+
+typedef enum {
+  FAN_MODE_AUTO = 0,
+  FAN_MODE_MANUAL_LINEAR,
+} FanMode_e;
+
+typedef enum {
+  OD_NO_ERROR,
+  OD_REQUEST_ADVANCED_NOT_SUPPORTED,
+  OD_UNSUPPORTED_FEATURE,
+  OD_INVALID_FEATURE_COMBO_ERROR,
+  OD_GFXCLK_VF_CURVE_OFFSET_ERROR,
+  OD_VDD_GFX_VMAX_ERROR,
+  OD_VDD_SOC_VMAX_ERROR,
+  OD_PPT_ERROR,
+  OD_FAN_MIN_PWM_ERROR,
+  OD_FAN_ACOUSTIC_TARGET_ERROR,
+  OD_FAN_ACOUSTIC_LIMIT_ERROR,
+  OD_FAN_TARGET_TEMP_ERROR,
+  OD_FAN_ZERO_RPM_STOP_TEMP_ERROR,
+  OD_FAN_CURVE_PWM_ERROR,
+  OD_FAN_CURVE_TEMP_ERROR,
+  OD_FULL_CTRL_GFXCLK_ERROR,
+  OD_FULL_CTRL_UCLK_ERROR,
+  OD_FULL_CTRL_FCLK_ERROR,
+  OD_FULL_CTRL_VDD_GFX_ERROR,
+  OD_FULL_CTRL_VDD_SOC_ERROR,
+  OD_TDC_ERROR,
+  OD_GFXCLK_ERROR,
+  OD_UCLK_ERROR,
+  OD_FCLK_ERROR,
+  OD_OP_TEMP_ERROR,
+  OD_OP_GFX_EDC_ERROR,
+  OD_OP_GFX_PCC_ERROR,
+  OD_POWER_FEATURE_CTRL_ERROR,
+} OD_FAIL_e;
+
+typedef struct {
+  uint32_t               FeatureCtrlMask;
+
+  //Voltage control
+  int16_t                VoltageOffsetPerZoneBoundary[PP_NUM_OD_VF_CURVE_POINTS];
+
+  uint16_t               VddGfxVmax;         // in mV
+  uint16_t               VddSocVmax;
+
+  uint8_t                IdlePwrSavingFeaturesCtrl;
+  uint8_t                RuntimePwrSavingFeaturesCtrl;
+  uint16_t               Padding;
+
+  //Frequency changes
+  int16_t                GfxclkFmin;           // MHz
+  int16_t                GfxclkFmax;           // MHz
+  uint16_t               UclkFmin;             // MHz
+  uint16_t               UclkFmax;             // MHz
+  uint16_t               FclkFmin;
+  uint16_t               FclkFmax;
+
+  //PPT
+  int16_t                Ppt;         // %
+  int16_t                Tdc;
+
+  //Fan control
+  uint8_t                FanLinearPwmPoints[NUM_OD_FAN_MAX_POINTS];
+  uint8_t                FanLinearTempPoints[NUM_OD_FAN_MAX_POINTS];
+  uint16_t               FanMinimumPwm;
+  uint16_t               AcousticTargetRpmThreshold;
+  uint16_t               AcousticLimitRpmThreshold;
+  uint16_t               FanTargetTemperature; // Degree Celcius
+  uint8_t                FanZeroRpmEnable;
+  uint8_t                FanZeroRpmStopTemp;
+  uint8_t                FanMode;
+  uint8_t                MaxOpTemp;
+
+  uint8_t                AdvancedOdModeEnabled;
+  uint8_t                Padding1[3];
+
+  uint16_t               GfxVoltageFullCtrlMode;
+  uint16_t               SocVoltageFullCtrlMode;
+  uint16_t               GfxclkFullCtrlMode;
+  uint16_t               UclkFullCtrlMode;
+  uint16_t               FclkFullCtrlMode;
+  uint16_t               Padding2;
+
+  int16_t                GfxEdc;
+  int16_t                GfxPccLimitControl;
+
+  uint32_t               Spare[10];
+  uint32_t               MmHubPadding[8]; // SMU internal use. Adding here instead of external as a workaround
+} OverDriveTable_t;
+
+typedef struct {
+  OverDriveTable_t OverDriveTable;
+
+} OverDriveTableExternal_t;
+
+typedef struct {
+  uint32_t               FeatureCtrlMask;
+
+  //Gfx Vf Curve
+  int16_t                VoltageOffsetPerZoneBoundary[PP_NUM_OD_VF_CURVE_POINTS];
+  //gfx Vmax
+  uint16_t               VddGfxVmax;         // in mV
+  //soc Vmax
+  uint16_t               VddSocVmax;
+
+  //gfxclk
+  int16_t                GfxclkFmin;           // MHz
+  int16_t                GfxclkFmax;           // MHz
+  //uclk
+  uint16_t               UclkFmin;             // MHz
+  uint16_t               UclkFmax;             // MHz
+  //fclk
+  uint16_t               FclkFmin;
+  uint16_t               FclkFmax;
+
+  //PPT
+  int16_t                Ppt;         // %
+  //TDC
+  int16_t                Tdc;
+
+  //Fan Curve
+  uint8_t                FanLinearPwmPoints[NUM_OD_FAN_MAX_POINTS];
+  uint8_t                FanLinearTempPoints[NUM_OD_FAN_MAX_POINTS];
+  //Fan Legacy
+  uint16_t               FanMinimumPwm;
+  uint16_t               AcousticTargetRpmThreshold;
+  uint16_t               AcousticLimitRpmThreshold;
+  uint16_t               FanTargetTemperature; // Degree Celcius
+  //zero fan
+  uint8_t                FanZeroRpmEnable;
+  //temperature
+  uint8_t                MaxOpTemp;
+  uint8_t                Padding[2];
+
+  //Full Ctrl
+  uint16_t               GfxVoltageFullCtrlMode;
+  uint16_t               SocVoltageFullCtrlMode;
+  uint16_t               GfxclkFullCtrlMode;
+  uint16_t               UclkFullCtrlMode;
+  uint16_t               FclkFullCtrlMode;
+  //EDC
+  int16_t                GfxEdc;
+  int16_t                GfxPccLimitControl;
+  int16_t                Padding1;
+
+  uint32_t               Spare[5];
+} OverDriveLimits_t;
+
+typedef enum {
+  BOARD_GPIO_SMUIO_0,
+  BOARD_GPIO_SMUIO_1,
+  BOARD_GPIO_SMUIO_2,
+  BOARD_GPIO_SMUIO_3,
+  BOARD_GPIO_SMUIO_4,
+  BOARD_GPIO_SMUIO_5,
+  BOARD_GPIO_SMUIO_6,
+  BOARD_GPIO_SMUIO_7,
+  BOARD_GPIO_SMUIO_8,
+  BOARD_GPIO_SMUIO_9,
+  BOARD_GPIO_SMUIO_10,
+  BOARD_GPIO_SMUIO_11,
+  BOARD_GPIO_SMUIO_12,
+  BOARD_GPIO_SMUIO_13,
+  BOARD_GPIO_SMUIO_14,
+  BOARD_GPIO_SMUIO_15,
+  BOARD_GPIO_SMUIO_16,
+  BOARD_GPIO_SMUIO_17,
+  BOARD_GPIO_SMUIO_18,
+  BOARD_GPIO_SMUIO_19,
+  BOARD_GPIO_SMUIO_20,
+  BOARD_GPIO_SMUIO_21,
+  BOARD_GPIO_SMUIO_22,
+  BOARD_GPIO_SMUIO_23,
+  BOARD_GPIO_SMUIO_24,
+  BOARD_GPIO_SMUIO_25,
+  BOARD_GPIO_SMUIO_26,
+  BOARD_GPIO_SMUIO_27,
+  BOARD_GPIO_SMUIO_28,
+  BOARD_GPIO_SMUIO_29,
+  BOARD_GPIO_SMUIO_30,
+  BOARD_GPIO_SMUIO_31,
+  MAX_BOARD_GPIO_SMUIO_NUM,
+  BOARD_GPIO_DC_GEN_A,
+  BOARD_GPIO_DC_GEN_B,
+  BOARD_GPIO_DC_GEN_C,
+  BOARD_GPIO_DC_GEN_D,
+  BOARD_GPIO_DC_GEN_E,
+  BOARD_GPIO_DC_GEN_F,
+  BOARD_GPIO_DC_GEN_G,
+  BOARD_GPIO_DC_GENLK_CLK,
+  BOARD_GPIO_DC_GENLK_VSYNC,
+  BOARD_GPIO_DC_SWAPLOCK_A,
+  BOARD_GPIO_DC_SWAPLOCK_B,
+  MAX_BOARD_DC_GPIO_NUM,
+  BOARD_GPIO_LV_EN,
+} BOARD_GPIO_TYPE_e;
+
+#define INVALID_BOARD_GPIO 0xFF
+
+
+typedef struct {
+  //PLL 0
+  uint16_t InitImuClk;
+  uint16_t InitSocclk;
+  uint16_t InitMpioclk;
+  uint16_t InitSmnclk;
+  //PLL 1
+  uint16_t InitDispClk;
+  uint16_t InitDppClk;
+  uint16_t InitDprefclk;
+  uint16_t InitDcfclk;
+  uint16_t InitDtbclk;
+  uint16_t InitDbguSocClk;
+  //PLL 2
+  uint16_t InitGfxclk_bypass;
+  uint16_t InitMp1clk;
+  uint16_t InitLclk;
+  uint16_t InitDbguBacoClk;
+  uint16_t InitBaco400clk;
+  uint16_t InitBaco1200clk_bypass;
+  uint16_t InitBaco700clk_bypass;
+  uint16_t InitBaco500clk;
+  // PLL 3
+  uint16_t InitDclk0;
+  uint16_t InitVclk0;
+  // PLL 4
+  uint16_t InitFclk;
+  uint16_t Padding1;
+  // PLL 5
+  //UCLK clocks, assumed all UCLK instances will be the same.
+  uint8_t InitUclkLevel;    // =0,1,2,3,4,5 frequency from FreqTableUclk
+
+  uint8_t Padding[3];
+
+  uint32_t InitVcoFreqPll0; //smu_socclk_t
+  uint32_t InitVcoFreqPll1; //smu_displayclk_t
+  uint32_t InitVcoFreqPll2; //smu_nbioclk_t
+  uint32_t InitVcoFreqPll3; //smu_vcnclk_t
+  uint32_t InitVcoFreqPll4; //smu_fclk_t
+  uint32_t InitVcoFreqPll5; //smu_uclk_01_t
+  uint32_t InitVcoFreqPll6; //smu_uclk_23_t
+  uint32_t InitVcoFreqPll7; //smu_uclk_45_t
+  uint32_t InitVcoFreqPll8; //smu_uclk_67_t
+
+  //encoding will be SVI3
+  uint16_t InitGfx;       // In mV(Q2) ,  should be 0?
+  uint16_t InitSoc;       // In mV(Q2)
+  uint16_t InitVddIoMem;  // In mV(Q2) MemVdd
+  uint16_t InitVddCiMem;  // In mV(Q2) VMemP
+
+  //uint16_t Padding2;
+
+  uint32_t Spare[8];
+} BootValues_t;
+
+typedef struct {
+   uint16_t Power[PPT_THROTTLER_COUNT][POWER_SOURCE_COUNT]; // Watts
+  uint16_t Tdc[TDC_THROTTLER_COUNT];             // Amps
+
+  uint16_t Temperature[TEMP_COUNT]; // Celsius
+
+  uint8_t  PwmLimitMin;
+  uint8_t  PwmLimitMax;
+  uint8_t  FanTargetTemperature;
+  uint8_t  Spare1[1];
+
+  uint16_t AcousticTargetRpmThresholdMin;
+  uint16_t AcousticTargetRpmThresholdMax;
+
+  uint16_t AcousticLimitRpmThresholdMin;
+  uint16_t AcousticLimitRpmThresholdMax;
+
+  uint16_t  PccLimitMin;
+  uint16_t  PccLimitMax;
+
+  uint16_t  FanStopTempMin;
+  uint16_t  FanStopTempMax;
+  uint16_t  FanStartTempMin;
+  uint16_t  FanStartTempMax;
+
+  uint16_t  PowerMinPpt0[POWER_SOURCE_COUNT];
+  uint32_t  Spare[11];
+} MsgLimits_t;
+
+typedef struct {
+  uint16_t BaseClockAc;
+  uint16_t GameClockAc;
+  uint16_t BoostClockAc;
+  uint16_t BaseClockDc;
+  uint16_t GameClockDc;
+  uint16_t BoostClockDc;
+
+  uint32_t Reserved[4];
+} DriverReportedClocks_t;
+
+typedef struct {
+  uint8_t           DcBtcEnabled;
+  uint8_t           Padding[3];
+
+  uint16_t          DcTol;            // mV Q2
+  uint16_t          DcBtcGb;       // mV Q2
+
+  uint16_t          DcBtcMin;       // mV Q2
+  uint16_t          DcBtcMax;       // mV Q2
+
+  LinearInt_t       DcBtcGbScalar;
+} AvfsDcBtcParams_t;
+
+typedef struct {
+  uint16_t       AvfsTemp[AVFS_TEMP_COUNT]; //in degrees C
+  uint16_t      VftFMin;  // in MHz
+  uint16_t      VInversion; // in mV Q2
+  QuadraticInt_t qVft[AVFS_TEMP_COUNT];
+  QuadraticInt_t qAvfsGb;
+  QuadraticInt_t qAvfsGb2;
+} AvfsFuseOverride_t;
+
+//all settings maintained by PFE team
+typedef struct {
+  uint8_t      Version;
+  uint8_t      Spare8[3];
+  // SECTION: Feature Control
+  uint32_t     FeaturesToRun[NUM_FEATURES / 32]; // Features that PMFW will attempt to enable. Use FEATURE_*_BIT as mapping
+  // SECTION: FW DSTATE Settings
+  uint32_t     FwDStateMask;           // See FW_DSTATE_*_BIT for mapping
+  // SECTION: Advanced Options
+  uint32_t     DebugOverrides;
+
+  uint32_t     Spare[2];
+} PFE_Settings_t;
+
+typedef struct {
+  // SECTION: Version
+  uint32_t Version; // should be unique to each SKU(i.e if any value changes in below structure then this value must be different)
+
+  // SECTION: Miscellaneous Configuration
+  uint8_t      TotalPowerConfig;    // Determines how PMFW calculates the power. Use defines from PwrConfig_e
+  uint8_t      CustomerVariant; //To specify if this PPTable is intended for a particular customer. Use defines from CUSTOMER_VARIANT_e
+  uint8_t      MemoryTemperatureTypeMask; // Bit mapping indicating which methods of memory temperature reading are enabled. Use defines from MEM_TEMP_*BIT
+  uint8_t      SmartShiftVersion; // Determine what SmartShift feature version is supported Use defines from SMARTSHIFT_VERSION_e
+
+  // SECTION: Infrastructure Limits
+  uint8_t  SocketPowerLimitSpare[10];
+
+  //if set to 1, SocketPowerLimitAc and SocketPowerLimitDc will be interpreted as legacy programs(i.e absolute power). If 0, all except index 0 will be scalars
+  //relative index 0
+  uint8_t  EnableLegacyPptLimit;
+  uint8_t  UseInputTelemetry; //applicable to SVI3 only and only to be set if VRs support
+
+  uint8_t  SmartShiftMinReportedPptinDcs; //minimum possible active power consumption for this SKU. Used for SmartShift power reporting
+
+  uint8_t  PaddingPpt[7];
+
+  uint16_t HwCtfTempLimit; // In degrees Celsius. Temperature above which HW will trigger CTF. Consumed by VBIOS only
+
+  uint16_t PaddingInfra;
+
+  // Per year normalized Vmax state failure rates (sum of the two domains divided by life time in years)
+  uint32_t FitControllerFailureRateLimit; //in IEEE float
+  //Expected GFX Duty Cycle at Vmax.
+  uint32_t FitControllerGfxDutyCycle; // in IEEE float
+  //Expected SOC Duty Cycle at Vmax.
+  uint32_t FitControllerSocDutyCycle; // in IEEE float
+
+  //This offset will be deducted from the controller output to before it goes through the SOC Vset limiter block.
+  uint32_t FitControllerSocOffset;  //in IEEE float
+
+  uint32_t     GfxApccPlusResidencyLimit; // Percentage value. Used by APCC+ controller to control PCC residency to some value
+
+  // SECTION: Throttler settings
+  uint32_t ThrottlerControlMask;   // See THROTTLER_*_BIT for mapping
+
+
+  // SECTION: Voltage Control Parameters
+  uint16_t  UlvVoltageOffset[PMFW_VOLT_PLANE_COUNT]; // In mV(Q2). ULV offset used in either GFX_ULV or SOC_ULV(part of FW_DSTATE)
+
+  uint8_t      Padding[2];
+  uint16_t     DeepUlvVoltageOffsetSoc;        // In mV(Q2)  Long Idle Vmin (deep ULV), for VDD_SOC as part of FW_DSTATE
+
+  // Voltage Limits
+  uint16_t     DefaultMaxVoltage[PMFW_VOLT_PLANE_COUNT]; // In mV(Q2) Maximum voltage without FIT controller enabled
+  uint16_t     BoostMaxVoltage[PMFW_VOLT_PLANE_COUNT]; // In mV(Q2) Maximum voltage with FIT controller enabled
+
+  //Vmin Optimizations
+  int16_t         VminTempHystersis[PMFW_VOLT_PLANE_COUNT]; // Celsius Temperature hysteresis for switching between low/high temperature values for Vmin
+  int16_t         VminTempThreshold[PMFW_VOLT_PLANE_COUNT]; // Celsius Temperature threshold for switching between low/high temperature values for Vmin
+  uint16_t        Vmin_Hot_T0[PMFW_VOLT_PLANE_COUNT];            //In mV(Q2) Initial (pre-aging) Vset to be used at hot.
+  uint16_t        Vmin_Cold_T0[PMFW_VOLT_PLANE_COUNT];           //In mV(Q2) Initial (pre-aging) Vset to be used at cold.
+  uint16_t        Vmin_Hot_Eol[PMFW_VOLT_PLANE_COUNT];           //In mV(Q2) End-of-life Vset to be used at hot.
+  uint16_t        Vmin_Cold_Eol[PMFW_VOLT_PLANE_COUNT];          //In mV(Q2) End-of-life Vset to be used at cold.
+  uint16_t        Vmin_Aging_Offset[PMFW_VOLT_PLANE_COUNT];      //In mV(Q2) Worst-case aging margin
+  uint16_t        Spare_Vmin_Plat_Offset_Hot[PMFW_VOLT_PLANE_COUNT];   //In mV(Q2) Platform offset apply to T0 Hot
+  uint16_t        Spare_Vmin_Plat_Offset_Cold[PMFW_VOLT_PLANE_COUNT];  //In mV(Q2) Platform offset apply to T0 Cold
+
+  //This is a fixed/minimum VMIN aging degradation offset which is applied at T0. This reflects the minimum amount of aging already accounted for.
+  uint16_t        VcBtcFixedVminAgingOffset[PMFW_VOLT_PLANE_COUNT];
+  //Linear offset or GB term to account for mis-correlation between PSM and Vmin shift trends across parts.
+  uint16_t        VcBtcVmin2PsmDegrationGb[PMFW_VOLT_PLANE_COUNT];
+  //Scalar coefficient of the PSM aging degradation function
+  uint32_t        VcBtcPsmA[PMFW_VOLT_PLANE_COUNT];                   // A_PSM
+  //Exponential coefficient of the PSM aging degradation function
+  uint32_t        VcBtcPsmB[PMFW_VOLT_PLANE_COUNT];                   // B_PSM
+  //Scalar coefficient of the VMIN aging degradation function. Specified as worst case between hot and cold.
+  uint32_t        VcBtcVminA[PMFW_VOLT_PLANE_COUNT];                  // A_VMIN
+  //Exponential coefficient of the VMIN aging degradation function. Specified as worst case between hot and cold.
+  uint32_t        VcBtcVminB[PMFW_VOLT_PLANE_COUNT];                  // B_VMIN
+
+  uint8_t         PerPartVminEnabled[PMFW_VOLT_PLANE_COUNT];
+  uint8_t         VcBtcEnabled[PMFW_VOLT_PLANE_COUNT];
+
+  uint16_t        SocketPowerLimitAcTau[PPT_THROTTLER_COUNT]; // Time constant of LPF in ms
+  uint16_t        SocketPowerLimitDcTau[PPT_THROTTLER_COUNT]; // Time constant of LPF in ms
+
+  QuadraticInt_t  Gfx_Vmin_droop;
+  QuadraticInt_t  Soc_Vmin_droop;
+  uint32_t        SpareVmin[6];
+
+  //SECTION: DPM Configuration 1
+  DpmDescriptor_t DpmDescriptor[PPCLK_COUNT];
+
+  uint16_t      FreqTableGfx        [NUM_GFXCLK_DPM_LEVELS  ];     // In MHz
+  uint16_t      FreqTableVclk       [NUM_VCLK_DPM_LEVELS    ];     // In MHz
+  uint16_t      FreqTableDclk       [NUM_DCLK_DPM_LEVELS    ];     // In MHz
+  uint16_t      FreqTableSocclk     [NUM_SOCCLK_DPM_LEVELS  ];     // In MHz
+  uint16_t      FreqTableUclk       [NUM_UCLK_DPM_LEVELS    ];     // In MHz
+  uint16_t      FreqTableShadowUclk [NUM_UCLK_DPM_LEVELS    ];     // In MHz
+  uint16_t      FreqTableDispclk    [NUM_DISPCLK_DPM_LEVELS ];     // In MHz
+  uint16_t      FreqTableDppClk     [NUM_DPPCLK_DPM_LEVELS  ];     // In MHz
+  uint16_t      FreqTableDprefclk   [NUM_DPREFCLK_DPM_LEVELS];     // In MHz
+  uint16_t      FreqTableDcfclk     [NUM_DCFCLK_DPM_LEVELS  ];     // In MHz
+  uint16_t      FreqTableDtbclk     [NUM_DTBCLK_DPM_LEVELS  ];     // In MHz
+  uint16_t      FreqTableFclk       [NUM_FCLK_DPM_LEVELS    ];     // In MHz
+
+  uint32_t      DcModeMaxFreq     [PPCLK_COUNT            ];     // In MHz
+
+  uint16_t      GfxclkAibFmax;
+  uint16_t      GfxclkFreqCap;
+
+  //GFX Idle Power Settings
+  uint16_t      GfxclkFgfxoffEntry;   // Entry in RLC stage (PLL), in Mhz
+  uint16_t      GfxclkFgfxoffExitImu; // Exit/Entry in IMU stage (BYPASS), in Mhz
+  uint16_t      GfxclkFgfxoffExitRlc; // Exit in RLC stage (PLL), in Mhz
+  uint16_t      GfxclkThrottleClock;  //Used primarily in DCS
+  uint8_t       EnableGfxPowerStagesGpio; //Genlk_vsync GPIO flag used to control gfx power stages
+  uint8_t       GfxIdlePadding;
+
+  uint8_t       SmsRepairWRCKClkDivEn;
+  uint8_t       SmsRepairWRCKClkDivVal;
+  uint8_t       GfxOffEntryEarlyMGCGEn;
+  uint8_t       GfxOffEntryForceCGCGEn;
+  uint8_t       GfxOffEntryForceCGCGDelayEn;
+  uint8_t       GfxOffEntryForceCGCGDelayVal; // in microseconds
+
+  uint16_t      GfxclkFreqGfxUlv; // in MHz
+  uint8_t       GfxIdlePadding2[2];
+  uint32_t      GfxOffEntryHysteresis; //For RLC to count after it enters CGCG, and before triggers GFXOFF entry
+  uint32_t      GfxoffSpare[15];
+
+  // DFLL
+  uint16_t      DfllMstrOscConfigA; //Used for voltage sensitivity slope tuning: 0 = (en_leaker << 9) | (en_vint1_reduce << 8) | (gain_code << 6) | (bias_code << 3) | (vint1_code << 1) | en_bias
+  uint16_t      DfllSlvOscConfigA; //Used for voltage sensitivity slope tuning: 0 = (en_leaker << 9) | (en_vint1_reduce << 8) | (gain_code << 6) | (bias_code << 3) | (vint1_code << 1) | en_bias
+  uint32_t      DfllBtcMasterScalerM;
+  int32_t       DfllBtcMasterScalerB;
+  uint32_t      DfllBtcSlaveScalerM;
+  int32_t       DfllBtcSlaveScalerB;
+
+  uint32_t      DfllPccAsWaitCtrl; //GDFLL_AS_WAIT_CTRL_PCC register value to be passed to RLC msg
+  uint32_t      DfllPccAsStepCtrl; //GDFLL_AS_STEP_CTRL_PCC register value to be passed to RLC msg
+  uint32_t      GfxDfllSpare[9];
+
+  // DVO
+  uint32_t        DvoPsmDownThresholdVoltage; //Voltage float
+  uint32_t        DvoPsmUpThresholdVoltage; //Voltage float
+  uint32_t        DvoFmaxLowScaler; //Unitless float
+
+  // GFX DCS
+  uint16_t      DcsGfxOffVoltage;     //Voltage in mV(Q2) applied to VDDGFX when entering DCS GFXOFF phase
+  uint16_t      PaddingDcs;
+
+  uint16_t      DcsMinGfxOffTime;     //Minimum amount of time PMFW shuts GFX OFF as part of GFX DCS phase
+  uint16_t      DcsMaxGfxOffTime;      //Maximum amount of time PMFW can shut GFX OFF as part of GFX DCS phase at a stretch.
+
+  uint32_t      DcsMinCreditAccum;    //Min amount of positive credit accumulation before waking GFX up as part of DCS.
+
+  uint16_t      DcsExitHysteresis;    //The min amount of time power credit accumulator should have a value > 0 before SMU exits the DCS throttling phase.
+  uint16_t      DcsTimeout;           //This is the amount of time SMU FW waits for RLC to put GFX into GFXOFF before reverting to the fallback mechanism of throttling GFXCLK to Fmin.
+
+  uint32_t      DcsPfGfxFopt;         //Default to GFX FMIN
+  uint32_t      DcsPfUclkFopt;        //Default to UCLK FMIN
+
+  uint8_t       FoptEnabled;
+  uint8_t       DcsSpare2[3];
+  uint32_t      DcsFoptM;             //Tuning paramters to shift Fopt calculation, IEEE754 float
+  uint32_t      DcsFoptB;             //Tuning paramters to shift Fopt calculation, IEEE754 float
+  uint32_t      DcsSpare[9];
+
+  // UCLK section
+  uint8_t       UseStrobeModeOptimizations; //Set to indicate that FW should use strobe mode optimizations
+  uint8_t       PaddingMem[3];
+
+  uint8_t       UclkDpmPstates             [NUM_UCLK_DPM_LEVELS];     // 6 Primary SW DPM states (6 + 6 Shadow)
+  uint8_t       UclkDpmShadowPstates       [NUM_UCLK_DPM_LEVELS];      // 6 Shadow SW DPM states (6 + 6 Shadow)
+  uint8_t       FreqTableUclkDiv           [NUM_UCLK_DPM_LEVELS];     // 0:Div-1, 1:Div-1/2, 2:Div-1/4, 3:Div-1/8
+  uint8_t       FreqTableShadowUclkDiv     [NUM_UCLK_DPM_LEVELS];     // 0:Div-1, 1:Div-1/2, 2:Div-1/4, 3:Div-1/8
+  uint16_t      MemVmempVoltage            [NUM_UCLK_DPM_LEVELS];     // mV(Q2)
+  uint16_t      MemVddioVoltage            [NUM_UCLK_DPM_LEVELS];     // mV(Q2)
+  uint16_t      DalDcModeMaxUclkFreq;
+  uint8_t       PaddingsMem[2];
+  //FCLK Section
+  uint16_t      FclkDpmDisallowPstateFreq;  //Frequency which FW will target when indicated that display config cannot support P-state. Set to 0 use FW calculated value
+  uint16_t      PaddingFclk;
+
+  // Link DPM Settings
+  uint8_t       PcieGenSpeed[NUM_LINK_LEVELS];           ///< 0:PciE-gen1 1:PciE-gen2 2:PciE-gen3 3:PciE-gen4 4:PciE-gen5
+  uint8_t       PcieLaneCount[NUM_LINK_LEVELS];          ///< 1=x1, 2=x2, 3=x4, 4=x8, 5=x12, 6=x16
+  uint16_t      LclkFreq[NUM_LINK_LEVELS];
+
+  // SECTION: VDD_GFX AVFS
+  uint8_t       OverrideGfxAvfsFuses;
+  uint8_t       GfxAvfsPadding[3];
+
+  uint32_t      SocHwRtAvfsFuses[PP_GRTAVFS_HW_FUSE_COUNT];   //new added for Soc domain
+  uint32_t      GfxL2HwRtAvfsFuses[PP_GRTAVFS_HW_FUSE_COUNT]; //see fusedoc for encoding
+  //uint32_t      GfxSeHwRtAvfsFuses[PP_GRTAVFS_HW_FUSE_COUNT];
+  uint32_t      spare_HwRtAvfsFuses[PP_GRTAVFS_HW_FUSE_COUNT];
+
+  uint32_t      SocCommonRtAvfs[PP_GRTAVFS_FW_COMMON_FUSE_COUNT];
+  uint32_t      GfxCommonRtAvfs[PP_GRTAVFS_FW_COMMON_FUSE_COUNT];
+
+  uint32_t      SocFwRtAvfsFuses[PP_GRTAVFS_FW_SEP_FUSE_COUNT];
+  uint32_t      GfxL2FwRtAvfsFuses[PP_GRTAVFS_FW_SEP_FUSE_COUNT];
+  //uint32_t      GfxSeFwRtAvfsFuses[PP_GRTAVFS_FW_SEP_FUSE_COUNT];
+  uint32_t      spare_FwRtAvfsFuses[PP_GRTAVFS_FW_SEP_FUSE_COUNT];
+
+  uint32_t      Soc_Droop_PWL_F[PP_NUM_RTAVFS_PWL_ZONES];
+  uint32_t      Soc_Droop_PWL_a[PP_NUM_RTAVFS_PWL_ZONES];
+  uint32_t      Soc_Droop_PWL_b[PP_NUM_RTAVFS_PWL_ZONES];
+  uint32_t      Soc_Droop_PWL_c[PP_NUM_RTAVFS_PWL_ZONES];
+
+  uint32_t      Gfx_Droop_PWL_F[PP_NUM_RTAVFS_PWL_ZONES];
+  uint32_t      Gfx_Droop_PWL_a[PP_NUM_RTAVFS_PWL_ZONES];
+  uint32_t      Gfx_Droop_PWL_b[PP_NUM_RTAVFS_PWL_ZONES];
+  uint32_t      Gfx_Droop_PWL_c[PP_NUM_RTAVFS_PWL_ZONES];
+
+  uint32_t      Gfx_Static_PWL_Offset[PP_NUM_RTAVFS_PWL_ZONES];
+  uint32_t      Soc_Static_PWL_Offset[PP_NUM_RTAVFS_PWL_ZONES];
+
+  uint32_t      dGbV_dT_vmin;
+  uint32_t      dGbV_dT_vmax;
+
+  //Unused: PMFW-9370
+  uint32_t      V2F_vmin_range_low;
+  uint32_t      V2F_vmin_range_high;
+  uint32_t      V2F_vmax_range_low;
+  uint32_t      V2F_vmax_range_high;
+
+  AvfsDcBtcParams_t DcBtcGfxParams;
+  QuadraticInt_t    SSCurve_GFX;
+  uint32_t   GfxAvfsSpare[29];
+
+  //SECTION: VDD_SOC AVFS
+  uint8_t      OverrideSocAvfsFuses;
+  uint8_t      MinSocAvfsRevision;
+  uint8_t      SocAvfsPadding[2];
+
+  AvfsFuseOverride_t SocAvfsFuseOverride[AVFS_D_COUNT];
+
+  DroopInt_t        dBtcGbSoc[AVFS_D_COUNT];            // GHz->V BtcGb
+
+  LinearInt_t       qAgingGb[AVFS_D_COUNT];          // GHz->V
+
+  QuadraticInt_t    qStaticVoltageOffset[AVFS_D_COUNT]; // GHz->V
+
+  AvfsDcBtcParams_t DcBtcSocParams[AVFS_D_COUNT];
+
+  QuadraticInt_t    SSCurve_SOC;
+  uint32_t   SocAvfsSpare[29];
+
+  //SECTION: Boot clock and voltage values
+  BootValues_t BootValues;
+
+  //SECTION: Driver Reported Clocks
+  DriverReportedClocks_t DriverReportedClocks;
+
+  //SECTION: Message Limits
+  MsgLimits_t MsgLimits;
+
+  //SECTION: OverDrive Limits
+  OverDriveLimits_t OverDriveLimitsBasicMin;
+  OverDriveLimits_t OverDriveLimitsBasicMax;
+  OverDriveLimits_t OverDriveLimitsAdvancedMin;
+  OverDriveLimits_t OverDriveLimitsAdvancedMax;
+
+  // Section: Total Board Power idle vs active coefficients
+  uint8_t     TotalBoardPowerSupport;
+  uint8_t     TotalBoardPowerPadding[1];
+  uint16_t    TotalBoardPowerRoc;
+
+  //PMFW-11158
+  QuadraticInt_t qFeffCoeffGameClock[POWER_SOURCE_COUNT];
+  QuadraticInt_t qFeffCoeffBaseClock[POWER_SOURCE_COUNT];
+  QuadraticInt_t qFeffCoeffBoostClock[POWER_SOURCE_COUNT];
+
+  // APT GFX to UCLK mapping
+  int32_t     AptUclkGfxclkLookup[POWER_SOURCE_COUNT][6];
+  uint32_t    AptUclkGfxclkLookupHyst[POWER_SOURCE_COUNT][6];
+  uint32_t    AptPadding;
+
+  // Xvmin didt
+  QuadraticInt_t  GfxXvminDidtDroopThresh;
+  uint32_t        GfxXvminDidtResetDDWait;
+  uint32_t        GfxXvminDidtClkStopWait;
+  uint32_t        GfxXvminDidtFcsStepCtrl;
+  uint32_t        GfxXvminDidtFcsWaitCtrl;
+
+  // PSM based didt controller
+  uint32_t        PsmModeEnabled; //0: all disabled 1: static mode only 2: dynamic mode only 3:static + dynamic mode
+  uint32_t        P2v_a; // floating point in U32 format
+  uint32_t        P2v_b;
+  uint32_t        P2v_c;
+  uint32_t        T2p_a;
+  uint32_t        T2p_b;
+  uint32_t        T2p_c;
+  uint32_t        P2vTemp;
+  QuadraticInt_t  PsmDidtStaticSettings;
+  QuadraticInt_t  PsmDidtDynamicSettings;
+  uint8_t         PsmDidtAvgDiv;
+  uint8_t         PsmDidtForceStall;
+  uint16_t        PsmDidtReleaseTimer;
+  uint32_t        PsmDidtStallPattern; //Will be written to both pattern 1 and didt_static_level_prog
+  // CAC EDC
+  uint32_t        Leakage_C0; // in IEEE float
+  uint32_t        Leakage_C1; // in IEEE float
+  uint32_t        Leakage_C2; // in IEEE float
+  uint32_t        Leakage_C3; // in IEEE float
+  uint32_t        Leakage_C4; // in IEEE float
+  uint32_t        Leakage_C5; // in IEEE float
+  uint32_t        GFX_CLK_SCALAR; // in IEEE float
+  uint32_t        GFX_CLK_INTERCEPT; // in IEEE float
+  uint32_t        GFX_CAC_M; // in IEEE float
+  uint32_t        GFX_CAC_B; // in IEEE float
+  uint32_t        VDD_GFX_CurrentLimitGuardband; // in IEEE float
+  uint32_t        DynToTotalCacScalar; // in IEEE
+  // GFX EDC XVMIN
+  uint32_t        XVmin_Gfx_EdcThreshScalar;
+  uint32_t        XVmin_Gfx_EdcEnableFreq;
+  uint32_t        XVmin_Gfx_EdcPccAsStepCtrl;
+  uint32_t        XVmin_Gfx_EdcPccAsWaitCtrl;
+  uint16_t        XVmin_Gfx_EdcThreshold;
+  uint16_t        XVmin_Gfx_EdcFiltHysWaitCtrl;
+  // SOC EDC XVMIN
+  uint32_t        XVmin_Soc_EdcThreshScalar;
+  uint32_t        XVmin_Soc_EdcEnableFreq;
+  uint32_t        XVmin_Soc_EdcThreshold; // LPF: number of cycles Xvmin_trig_filt will react.
+  uint16_t        XVmin_Soc_EdcStepUpTime; // 10 bit, refclk count to step up throttle when PCC remains asserted.
+  uint16_t        XVmin_Soc_EdcStepDownTime;// 10 bit, refclk count to step down throttle when PCC remains asserted.
+  uint8_t         XVmin_Soc_EdcInitPccStep; // 3 bit, First Pcc Step number that will applied when PCC asserts.
+  uint8_t         PaddingSocEdc[3];
+
+  // Fuse Override for SOC and GFX XVMIN
+  uint8_t         GfxXvminFuseOverride;
+  uint8_t         SocXvminFuseOverride;
+  uint8_t         PaddingXvminFuseOverride[2];
+  uint8_t         GfxXvminFddTempLow;  // bit 7: sign, bit 0-6: ABS value
+  uint8_t         GfxXvminFddTempHigh; // bit 7: sign, bit 0-6: ABS value
+  uint8_t         SocXvminFddTempLow;  // bit 7: sign, bit 0-6: ABS value
+  uint8_t         SocXvminFddTempHigh; // bit 7: sign, bit 0-6: ABS value
+
+
+  uint16_t        GfxXvminFddVolt0;    // low voltage, in VID
+  uint16_t        GfxXvminFddVolt1;    // mid voltage, in VID
+  uint16_t        GfxXvminFddVolt2;    // high voltage, in VID
+  uint16_t        SocXvminFddVolt0;    // low voltage, in VID
+  uint16_t        SocXvminFddVolt1;    // mid voltage, in VID
+  uint16_t        SocXvminFddVolt2;    // high voltage, in VID
+  uint16_t        GfxXvminDsFddDsm[6]; // XVMIN DS, same organization with fuse
+  uint16_t        GfxXvminEdcFddDsm[6];// XVMIN GFX EDC, same organization with fuse
+  uint16_t        SocXvminEdcFddDsm[6];// XVMIN SOC EDC, same organization with fuse
+
+  // SECTION: Sku Reserved
+  uint32_t        Spare;
+
+  // Padding for MMHUB - do not modify this
+  uint32_t     MmHubPadding[8];
+} SkuTable_t;
+
+typedef struct {
+  uint8_t SlewRateConditions;
+  uint8_t LoadLineAdjust;
+  uint8_t VoutOffset;
+  uint8_t VidMax;
+  uint8_t VidMin;
+  uint8_t TenBitTelEn;
+  uint8_t SixteenBitTelEn;
+  uint8_t OcpThresh;
+  uint8_t OcpWarnThresh;
+  uint8_t OcpSettings;
+  uint8_t VrhotThresh;
+  uint8_t OtpThresh;
+  uint8_t UvpOvpDeltaRef;
+  uint8_t PhaseShed;
+  uint8_t Padding[10];
+  uint32_t SettingOverrideMask;
+} Svi3RegulatorSettings_t;
+
+typedef struct {
+  // SECTION: Version
+  uint32_t    Version; //should be unique to each board type
+
+  // SECTION: I2C Control
+  I2cControllerConfig_t  I2cControllers[NUM_I2C_CONTROLLERS];
+
+  //SECTION SVI3 Board Parameters
+  uint8_t      SlaveAddrMapping[SVI_PLANE_COUNT];
+  uint8_t      VrPsiSupport[SVI_PLANE_COUNT];
+
+  uint32_t     Svi3SvcSpeed;
+  uint8_t      EnablePsi6[SVI_PLANE_COUNT];       // only applicable in SVI3
+
+  // SECTION: Voltage Regulator Settings
+  Svi3RegulatorSettings_t  Svi3RegSettings[SVI_PLANE_COUNT];
+
+  // SECTION: GPIO Settings
+  uint8_t      LedOffGpio;
+  uint8_t      FanOffGpio;
+  uint8_t      GfxVrPowerStageOffGpio;
+
+  uint8_t      AcDcGpio;        // GPIO pin configured for AC/DC switching
+  uint8_t      AcDcPolarity;    // GPIO polarity for AC/DC switching
+  uint8_t      VR0HotGpio;      // GPIO pin configured for VR0 HOT event
+  uint8_t      VR0HotPolarity;  // GPIO polarity for VR0 HOT event
+
+  uint8_t      GthrGpio;        // GPIO pin configured for GTHR Event
+  uint8_t      GthrPolarity;    // replace GPIO polarity for GTHR
+
+  // LED Display Settings
+  uint8_t      LedPin0;         // GPIO number for LedPin[0]
+  uint8_t      LedPin1;         // GPIO number for LedPin[1]
+  uint8_t      LedPin2;         // GPIO number for LedPin[2]
+  uint8_t      LedEnableMask;
+
+  uint8_t      LedPcie;        // GPIO number for PCIE results
+  uint8_t      LedError;       // GPIO number for Error Cases
+  uint8_t      PaddingLed;
+
+  // SECTION: Clock Spread Spectrum
+
+  // UCLK Spread Spectrum
+  uint8_t      UclkTrainingModeSpreadPercent; // Q4.4
+  uint8_t      UclkSpreadPadding;
+  uint16_t     UclkSpreadFreq;      // kHz
+
+  // UCLK Spread Spectrum
+  uint8_t      UclkSpreadPercent[MEM_VENDOR_COUNT];
+
+  // DFLL Spread Spectrum
+  uint8_t      GfxclkSpreadEnable;
+
+  // FCLK Spread Spectrum
+  uint8_t      FclkSpreadPercent;   // Q4.4
+  uint16_t     FclkSpreadFreq;      // kHz
+
+  // Section: Memory Config
+  uint8_t      DramWidth; // Width of interface to the channel for each DRAM module. See DRAM_BIT_WIDTH_TYPE_e
+  uint8_t      PaddingMem1[7];
+
+  // SECTION: UMC feature flags
+  uint8_t      HsrEnabled;
+  uint8_t      VddqOffEnabled;
+  uint8_t      PaddingUmcFlags[2];
+
+  uint32_t    PostVoltageSetBacoDelay; // in microseconds. Amount of time FW will wait after power good is established or PSI0 command is issued
+  uint32_t    BacoEntryDelay; // in milliseconds. Amount of time FW will wait to trigger BACO entry after receiving entry notification from OS
+
+  uint8_t     FuseWritePowerMuxPresent;
+  uint8_t     FuseWritePadding[3];
+
+  // SECTION: EDC Params
+  uint32_t    LoadlineGfx;
+  uint32_t    LoadlineSoc;
+  uint32_t    GfxEdcLimit;
+  uint32_t    SocEdcLimit;
+
+  uint32_t    RestBoardPower;         //power consumed by board that is not captured by the SVI3 input telemetry
+  uint32_t    ConnectorsImpedance;   // impedance of the input ATX power connectors
+
+  uint8_t      EpcsSens0;       //GPIO number for External Power Connector Support Sense0
+  uint8_t      EpcsSens1;       //GPIO Number for External Power Connector Support Sense1
+  uint8_t      PaddingEpcs[2];
+
+  // SECTION: Board Reserved
+  uint32_t    BoardSpare[52];
+
+  // SECTION: Structure Padding
+
+  // Padding for MMHUB - do not modify this
+  uint32_t     MmHubPadding[8];
+} BoardTable_t;
+
+typedef struct {
+  // SECTION: Infrastructure Limits
+  uint16_t    SocketPowerLimitAc[PPT_THROTTLER_COUNT]; // In Watts. Power limit that PMFW attempts to control to in AC mode. Multiple limits supported
+
+  uint16_t    VrTdcLimit[TDC_THROTTLER_COUNT];         // In Amperes. Current limit associated with VR regulator maximum temperature
+
+  int16_t     TotalIdleBoardPowerM;
+  int16_t     TotalIdleBoardPowerB;
+  int16_t     TotalBoardPowerM;
+  int16_t     TotalBoardPowerB;
+
+  uint16_t    TemperatureLimit[TEMP_COUNT]; // In degrees Celsius. Temperature limit associated with each input
+
+  // SECTION: Fan Control
+  uint16_t    FanStopTemp[TEMP_COUNT];          //Celsius
+  uint16_t    FanStartTemp[TEMP_COUNT];         //Celsius
+
+  uint16_t    FanGain[TEMP_COUNT];
+
+  uint16_t    FanPwmMin;
+  uint16_t    AcousticTargetRpmThreshold;
+  uint16_t    AcousticLimitRpmThreshold;
+  uint16_t    FanMaximumRpm;
+  uint16_t    MGpuAcousticLimitRpmThreshold;
+  uint16_t    FanTargetGfxclk;
+  uint32_t    TempInputSelectMask;
+  uint8_t     FanZeroRpmEnable;
+  uint8_t     FanTachEdgePerRev;
+  uint16_t    FanPadding;
+  uint16_t    FanTargetTemperature[TEMP_COUNT];
+
+  // The following are AFC override parameters. Leave at 0 to use FW defaults.
+  int16_t     FuzzyFan_ErrorSetDelta;
+  int16_t     FuzzyFan_ErrorRateSetDelta;
+  int16_t     FuzzyFan_PwmSetDelta;
+  uint16_t    FuzzyFan_Reserved;
+
+  uint16_t    FwCtfLimit[TEMP_COUNT];
+
+  uint16_t    IntakeTempEnableRPM;
+  int16_t     IntakeTempOffsetTemp;
+  uint16_t    IntakeTempReleaseTemp;
+  uint16_t    IntakeTempHighIntakeAcousticLimit;
+
+  uint16_t    IntakeTempAcouticLimitReleaseRate;
+  int16_t     FanAbnormalTempLimitOffset;    // FanStalledTempLimitOffset
+  uint16_t    FanStalledTriggerRpm;          //
+  uint16_t    FanAbnormalTriggerRpmCoeff;    // FanAbnormalTriggerRpm
+
+  uint16_t    FanSpare[1];
+  uint8_t     FanIntakeSensorSupport;
+  uint8_t     FanIntakePadding;
+  uint32_t    FanAmbientPerfBoostThreshold;
+  uint32_t    FanSpare2[12];
+
+  uint16_t TemperatureLimit_Hynix; // In degrees Celsius. Memory temperature limit associated with Hynix
+  uint16_t TemperatureLimit_Micron; // In degrees Celsius. Memory temperature limit associated with Micron
+  uint16_t TemperatureFwCtfLimit_Hynix;
+  uint16_t TemperatureFwCtfLimit_Micron;
+
+  // SECTION: Board Reserved
+  uint16_t    PlatformTdcLimit[TDC_THROTTLER_COUNT];             // In Amperes. Current limit associated with platform maximum temperature per VR current rail
+  uint16_t    SocketPowerLimitDc[PPT_THROTTLER_COUNT];  // In Watts. Power limit that PMFW attempts to control to in DC mode. Multiple limits supported
+  uint16_t    SocketPowerLimitSmartShift2; // In Watts. Power limit used SmartShift
+  uint16_t    CustomSkuSpare16b;
+  uint32_t    CustomSkuSpare32b[10];
+
+  // SECTION: Structure Padding
+
+  // Padding for MMHUB - do not modify this
+  uint32_t    MmHubPadding[8];
+} CustomSkuTable_t;
+
+typedef struct {
+  PFE_Settings_t PFE_Settings;
+  SkuTable_t SkuTable;
+  CustomSkuTable_t CustomSkuTable;
+  BoardTable_t BoardTable;
+} PPTable_t;
+
+typedef struct {
+  // Time constant parameters for clock averages in ms
+  uint16_t     GfxclkAverageLpfTau;
+  uint16_t     FclkAverageLpfTau;
+  uint16_t     UclkAverageLpfTau;
+  uint16_t     GfxActivityLpfTau;
+  uint16_t     UclkActivityLpfTau;
+  uint16_t     UclkMaxActivityLpfTau;
+  uint16_t     SocketPowerLpfTau;
+  uint16_t     VcnClkAverageLpfTau;
+  uint16_t     VcnUsageAverageLpfTau;
+  uint16_t     PcieActivityLpTau;
+} DriverSmuConfig_t;
+
+typedef struct {
+  DriverSmuConfig_t DriverSmuConfig;
+
+  uint32_t     Spare[8];
+  // Padding - ignore
+  uint32_t     MmHubPadding[8]; // SMU internal use
+} DriverSmuConfigExternal_t;
+
+
+typedef struct {
+
+  uint16_t       FreqTableGfx      [NUM_GFXCLK_DPM_LEVELS  ];     // In MHz
+  uint16_t       FreqTableVclk     [NUM_VCLK_DPM_LEVELS    ];     // In MHz
+  uint16_t       FreqTableDclk     [NUM_DCLK_DPM_LEVELS    ];     // In MHz
+  uint16_t       FreqTableSocclk   [NUM_SOCCLK_DPM_LEVELS  ];     // In MHz
+  uint16_t       FreqTableUclk     [NUM_UCLK_DPM_LEVELS    ];     // In MHz
+  uint16_t       FreqTableDispclk  [NUM_DISPCLK_DPM_LEVELS ];     // In MHz
+  uint16_t       FreqTableDppClk   [NUM_DPPCLK_DPM_LEVELS  ];     // In MHz
+  uint16_t       FreqTableDprefclk [NUM_DPREFCLK_DPM_LEVELS];     // In MHz
+  uint16_t       FreqTableDcfclk   [NUM_DCFCLK_DPM_LEVELS  ];     // In MHz
+  uint16_t       FreqTableDtbclk   [NUM_DTBCLK_DPM_LEVELS  ];     // In MHz
+  uint16_t       FreqTableFclk     [NUM_FCLK_DPM_LEVELS    ];     // In MHz
+
+  uint16_t       DcModeMaxFreq     [PPCLK_COUNT            ];     // In MHz
+
+  uint16_t       Padding;
+
+  uint32_t Spare[32];
+
+  // Padding - ignore
+  uint32_t     MmHubPadding[8]; // SMU internal use
+
+} DriverInfoTable_t;
+
+typedef struct {
+  uint32_t CurrClock[PPCLK_COUNT];
+
+  uint16_t AverageGfxclkFrequencyTarget;
+  uint16_t AverageGfxclkFrequencyPreDs;
+  uint16_t AverageGfxclkFrequencyPostDs;
+  uint16_t AverageFclkFrequencyPreDs;
+  uint16_t AverageFclkFrequencyPostDs;
+  uint16_t AverageMemclkFrequencyPreDs  ; // this is scaled to actual memory clock
+  uint16_t AverageMemclkFrequencyPostDs  ; // this is scaled to actual memory clock
+  uint16_t AverageVclk0Frequency  ;
+  uint16_t AverageDclk0Frequency  ;
+  uint16_t AverageVclk1Frequency  ;
+  uint16_t AverageDclk1Frequency  ;
+  uint16_t PCIeBusy               ;
+  uint16_t dGPU_W_MAX             ;
+  uint16_t padding                ;
+
+  uint32_t MetricsCounter         ;
+
+  uint16_t AvgVoltage[SVI_PLANE_COUNT];
+  uint16_t AvgCurrent[SVI_PLANE_COUNT];
+
+  uint16_t AverageGfxActivity    ;
+  uint16_t AverageUclkActivity   ;
+  uint16_t Vcn0ActivityPercentage  ;
+  uint16_t Vcn1ActivityPercentage  ;
+
+  uint32_t EnergyAccumulator;
+  uint16_t AverageSocketPower;
+  uint16_t AverageTotalBoardPower;
+
+  uint16_t AvgTemperature[TEMP_COUNT];
+  uint16_t AvgTemperatureFanIntake;
+
+  uint8_t  PcieRate               ;
+  uint8_t  PcieWidth              ;
+
+  uint8_t  AvgFanPwm;
+  uint8_t  Padding[1];
+  uint16_t AvgFanRpm;
+
+
+  uint8_t  ThrottlingPercentage[THROTTLER_COUNT];
+  uint8_t  padding1[3];
+
+  //metrics for D3hot entry/exit and driver ARM msgs
+  uint32_t D3HotEntryCountPerMode[D3HOT_SEQUENCE_COUNT];
+  uint32_t D3HotExitCountPerMode[D3HOT_SEQUENCE_COUNT];
+  uint32_t ArmMsgReceivedCountPerMode[D3HOT_SEQUENCE_COUNT];
+
+  uint16_t ApuSTAPMSmartShiftLimit;
+  uint16_t ApuSTAPMLimit;
+  uint16_t AvgApuSocketPower;
+
+  uint16_t AverageUclkActivity_MAX;
+
+  uint32_t PublicSerialNumberLower;
+  uint32_t PublicSerialNumberUpper;
+
+} SmuMetrics_t;
+
+typedef struct {
+  SmuMetrics_t SmuMetrics;
+  uint32_t Spare[30];
+
+  // Padding - ignore
+  uint32_t     MmHubPadding[8]; // SMU internal use
+} SmuMetricsExternal_t;
+
+typedef struct {
+  uint8_t  WmSetting;
+  uint8_t  Flags;
+  uint8_t  Padding[2];
+
+} WatermarkRowGeneric_t;
+
+#define NUM_WM_RANGES 4
+
+typedef enum {
+  WATERMARKS_CLOCK_RANGE = 0,
+  WATERMARKS_DUMMY_PSTATE,
+  WATERMARKS_MALL,
+  WATERMARKS_COUNT,
+} WATERMARKS_FLAGS_e;
+
+typedef struct {
+  // Watermarks
+  WatermarkRowGeneric_t WatermarkRow[NUM_WM_RANGES];
+} Watermarks_t;
+
+typedef struct {
+  Watermarks_t Watermarks;
+  uint32_t  Spare[16];
+
+  uint32_t     MmHubPadding[8]; // SMU internal use
+} WatermarksExternal_t;
+
+typedef struct {
+  uint16_t avgPsmCount[76];
+  uint16_t minPsmCount[76];
+  uint16_t maxPsmCount[76];
+  float    avgPsmVoltage[76];
+  float    minPsmVoltage[76];
+  float    maxPsmVoltage[76];
+} AvfsDebugTable_t;
+
+typedef struct {
+  AvfsDebugTable_t AvfsDebugTable;
+
+  uint32_t     MmHubPadding[8]; // SMU internal use
+} AvfsDebugTableExternal_t;
+
+
+typedef struct {
+  uint8_t   Gfx_ActiveHystLimit;
+  uint8_t   Gfx_IdleHystLimit;
+  uint8_t   Gfx_FPS;
+  uint8_t   Gfx_MinActiveFreqType;
+  uint8_t   Gfx_BoosterFreqType;
+  uint8_t   PaddingGfx;
+  uint16_t  Gfx_MinActiveFreq;              // MHz
+  uint16_t  Gfx_BoosterFreq;                // MHz
+  uint16_t  Gfx_PD_Data_time_constant;      // Time constant of PD controller in ms
+  uint32_t  Gfx_PD_Data_limit_a;            // Q16
+  uint32_t  Gfx_PD_Data_limit_b;            // Q16
+  uint32_t  Gfx_PD_Data_limit_c;            // Q16
+  uint32_t  Gfx_PD_Data_error_coeff;        // Q16
+  uint32_t  Gfx_PD_Data_error_rate_coeff;   // Q16
+
+  uint8_t   Fclk_ActiveHystLimit;
+  uint8_t   Fclk_IdleHystLimit;
+  uint8_t   Fclk_FPS;
+  uint8_t   Fclk_MinActiveFreqType;
+  uint8_t   Fclk_BoosterFreqType;
+  uint8_t   PaddingFclk;
+  uint16_t  Fclk_MinActiveFreq;              // MHz
+  uint16_t  Fclk_BoosterFreq;                // MHz
+  uint16_t  Fclk_PD_Data_time_constant;      // Time constant of PD controller in ms
+  uint32_t  Fclk_PD_Data_limit_a;            // Q16
+  uint32_t  Fclk_PD_Data_limit_b;            // Q16
+  uint32_t  Fclk_PD_Data_limit_c;            // Q16
+  uint32_t  Fclk_PD_Data_error_coeff;        // Q16
+  uint32_t  Fclk_PD_Data_error_rate_coeff;   // Q16
+
+  uint32_t  Mem_UpThreshold_Limit[NUM_UCLK_DPM_LEVELS];          // Q16
+  uint8_t   Mem_UpHystLimit[NUM_UCLK_DPM_LEVELS];
+  uint16_t  Mem_DownHystLimit[NUM_UCLK_DPM_LEVELS];
+  uint16_t  Mem_Fps;
+
+} DpmActivityMonitorCoeffInt_t;
+
+
+typedef struct {
+  DpmActivityMonitorCoeffInt_t DpmActivityMonitorCoeffInt;
+  uint32_t     MmHubPadding[8]; // SMU internal use
+} DpmActivityMonitorCoeffIntExternal_t;
+
+
+
+// Workload bits
+#define WORKLOAD_PPLIB_DEFAULT_BIT        0
+#define WORKLOAD_PPLIB_FULL_SCREEN_3D_BIT 1
+#define WORKLOAD_PPLIB_POWER_SAVING_BIT   2
+#define WORKLOAD_PPLIB_VIDEO_BIT          3
+#define WORKLOAD_PPLIB_VR_BIT             4
+#define WORKLOAD_PPLIB_COMPUTE_BIT        5
+#define WORKLOAD_PPLIB_CUSTOM_BIT         6
+#define WORKLOAD_PPLIB_WINDOW_3D_BIT      7
+#define WORKLOAD_PPLIB_DIRECT_ML_BIT      8
+#define WORKLOAD_PPLIB_CGVDI_BIT          9
+#define WORKLOAD_PPLIB_COUNT              10
+
+
+// These defines are used with the following messages:
+// SMC_MSG_TransferTableDram2Smu
+// SMC_MSG_TransferTableSmu2Dram
+
+// Table transfer status
+#define TABLE_TRANSFER_OK         0x0
+#define TABLE_TRANSFER_FAILED     0xFF
+#define TABLE_TRANSFER_PENDING    0xAB
+
+// Table types
+#define TABLE_PPTABLE            0
+#define TABLE_COMBO_PPTABLE           1
+#define TABLE_WATERMARKS              2
+#define TABLE_AVFS_PSM_DEBUG          3
+#define TABLE_PMSTATUSLOG             4
+#define TABLE_SMU_METRICS             5
+#define TABLE_DRIVER_SMU_CONFIG       6
+#define TABLE_ACTIVITY_MONITOR_COEFF  7
+#define TABLE_OVERDRIVE               8
+#define TABLE_I2C_COMMANDS            9
+#define TABLE_DRIVER_INFO             10
+#define TABLE_ECCINFO                 11
+#define TABLE_CUSTOM_SKUTABLE         12
+#define TABLE_COUNT                   13
+
+//IH Interupt ID
+#define IH_INTERRUPT_ID_TO_DRIVER                   0xFE
+#define IH_INTERRUPT_CONTEXT_ID_BACO                0x2
+#define IH_INTERRUPT_CONTEXT_ID_AC                  0x3
+#define IH_INTERRUPT_CONTEXT_ID_DC                  0x4
+#define IH_INTERRUPT_CONTEXT_ID_AUDIO_D0            0x5
+#define IH_INTERRUPT_CONTEXT_ID_AUDIO_D3            0x6
+#define IH_INTERRUPT_CONTEXT_ID_THERMAL_THROTTLING  0x7
+#define IH_INTERRUPT_CONTEXT_ID_FAN_ABNORMAL        0x8
+#define IH_INTERRUPT_CONTEXT_ID_FAN_RECOVERY        0x9
+
+#endif
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_6_pmfw.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_6_pmfw.h
index 7b812b9994d7..0b3c2f54a343 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_6_pmfw.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_6_pmfw.h
@@ -123,7 +123,7 @@ typedef enum {
   VOLTAGE_GUARDBAND_COUNT
 } GFX_GUARDBAND_e;
 
-#define SMU_METRICS_TABLE_VERSION 0xB
+#define SMU_METRICS_TABLE_VERSION 0xC
 
 typedef struct __attribute__((packed, aligned(4))) {
   uint32_t AccumulationCounter;
@@ -223,6 +223,10 @@ typedef struct __attribute__((packed, aligned(4))) {
   // VCN/JPEG ACTIVITY
   uint32_t VcnBusy[4];
   uint32_t JpegBusy[32];
+
+  // PCIE LINK Speed and width
+  uint32_t PCIeLinkSpeed;
+  uint32_t PCIeLinkWidth;
 } MetricsTableX_t;
 
 typedef struct __attribute__((packed, aligned(4))) {
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v14_0_2_ppsmc.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v14_0_2_ppsmc.h
new file mode 100644
index 000000000000..de2e442281ff
--- /dev/null
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v14_0_2_ppsmc.h
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef SMU_V14_0_2_PPSMC_H
+#define SMU_V14_0_2_PPSMC_H
+
+#define PPSMC_VERSION 0x1
+
+// SMU Response Codes:
+#define PPSMC_Result_OK                    0x1
+#define PPSMC_Result_Failed                0xFF
+#define PPSMC_Result_UnknownCmd            0xFE
+#define PPSMC_Result_CmdRejectedPrereq     0xFD
+#define PPSMC_Result_CmdRejectedBusy       0xFC
+
+// Message Definitions:
+// BASIC
+#define PPSMC_MSG_TestMessage                    0x1
+#define PPSMC_MSG_GetSmuVersion                  0x2
+#define PPSMC_MSG_GetDriverIfVersion             0x3
+#define PPSMC_MSG_SetAllowedFeaturesMaskLow      0x4
+#define PPSMC_MSG_SetAllowedFeaturesMaskHigh     0x5
+#define PPSMC_MSG_EnableAllSmuFeatures           0x6
+#define PPSMC_MSG_DisableAllSmuFeatures          0x7
+#define PPSMC_MSG_EnableSmuFeaturesLow           0x8
+#define PPSMC_MSG_EnableSmuFeaturesHigh          0x9
+#define PPSMC_MSG_DisableSmuFeaturesLow          0xA
+#define PPSMC_MSG_DisableSmuFeaturesHigh         0xB
+#define PPSMC_MSG_GetRunningSmuFeaturesLow       0xC
+#define PPSMC_MSG_GetRunningSmuFeaturesHigh      0xD
+#define PPSMC_MSG_SetDriverDramAddrHigh          0xE
+#define PPSMC_MSG_SetDriverDramAddrLow           0xF
+#define PPSMC_MSG_SetToolsDramAddrHigh           0x10
+#define PPSMC_MSG_SetToolsDramAddrLow            0x11
+#define PPSMC_MSG_TransferTableSmu2Dram          0x12
+#define PPSMC_MSG_TransferTableDram2Smu          0x13
+#define PPSMC_MSG_UseDefaultPPTable              0x14
+
+//BACO/BAMACO/BOMACO
+#define PPSMC_MSG_EnterBaco                      0x15
+#define PPSMC_MSG_ExitBaco                       0x16
+#define PPSMC_MSG_ArmD3                          0x17
+#define PPSMC_MSG_BacoAudioD3PME                 0x18
+
+//DPM
+#define PPSMC_MSG_SetSoftMinByFreq               0x19
+#define PPSMC_MSG_SetSoftMaxByFreq               0x1A
+#define PPSMC_MSG_SetHardMinByFreq               0x1B
+#define PPSMC_MSG_SetHardMaxByFreq               0x1C
+#define PPSMC_MSG_GetMinDpmFreq                  0x1D
+#define PPSMC_MSG_GetMaxDpmFreq                  0x1E
+#define PPSMC_MSG_GetDpmFreqByIndex              0x1F
+#define PPSMC_MSG_OverridePcieParameters         0x20
+
+//DramLog Set DramAddr
+#define PPSMC_MSG_DramLogSetDramAddrHigh         0x21
+#define PPSMC_MSG_DramLogSetDramAddrLow          0x22
+#define PPSMC_MSG_DramLogSetDramSize             0x23
+#define PPSMC_MSG_SetWorkloadMask                0x24
+
+#define PPSMC_MSG_GetVoltageByDpm                0x25 // Can be removed
+#define PPSMC_MSG_SetVideoFps                    0x26 // Can be removed
+#define PPSMC_MSG_GetDcModeMaxDpmFreq            0x27
+
+//Power Gating
+#define PPSMC_MSG_AllowGfxOff                    0x28
+#define PPSMC_MSG_DisallowGfxOff                 0x29
+#define PPSMC_MSG_PowerUpVcn                     0x2A
+#define PPSMC_MSG_PowerDownVcn                   0x2B
+#define PPSMC_MSG_PowerUpJpeg                    0x2C
+#define PPSMC_MSG_PowerDownJpeg                  0x2D
+
+//Resets
+#define PPSMC_MSG_PrepareMp1ForUnload            0x2E
+#define PPSMC_MSG_Mode1Reset                     0x2F
+
+//Set SystemVirtual DramAddrHigh
+#define PPSMC_MSG_SetSystemVirtualDramAddrHigh   0x30
+#define PPSMC_MSG_SetSystemVirtualDramAddrLow    0x31
+//ACDC Power Source
+#define PPSMC_MSG_SetPptLimit                    0x32
+#define PPSMC_MSG_GetPptLimit                    0x33
+#define PPSMC_MSG_ReenableAcDcInterrupt          0x34
+#define PPSMC_MSG_NotifyPowerSource              0x35
+
+//BTC
+#define PPSMC_MSG_RunDcBtc                       0x36
+
+//                                               0x37
+
+//Others
+#define PPSMC_MSG_SetTemperatureInputSelect      0x38 // Can be removed
+#define PPSMC_MSG_SetFwDstatesMask               0x39
+#define PPSMC_MSG_SetThrottlerMask               0x3A
+
+#define PPSMC_MSG_SetExternalClientDfCstateAllow 0x3B
+
+#define PPSMC_MSG_SetMGpuFanBoostLimitRpm        0x3C
+
+//STB to dram log
+#define PPSMC_MSG_DumpSTBtoDram                  0x3D
+#define PPSMC_MSG_STBtoDramLogSetDramAddrHigh    0x3E
+#define PPSMC_MSG_STBtoDramLogSetDramAddrLow     0x3F
+#define PPSMC_MSG_STBtoDramLogSetDramSize        0x40
+#define PPSMC_MSG_SetOBMTraceBufferLogging       0x41
+
+#define PPSMC_MSG_AllowGfxDcs                    0x43
+#define PPSMC_MSG_DisallowGfxDcs                 0x44
+#define PPSMC_MSG_EnableAudioStutterWA           0x45
+#define PPSMC_MSG_PowerUpUmsch                   0x46
+#define PPSMC_MSG_PowerDownUmsch                 0x47
+#define PPSMC_MSG_SetDcsArch                     0x48
+#define PPSMC_MSG_TriggerVFFLR                   0x49
+#define PPSMC_MSG_SetNumBadMemoryPagesRetired    0x4A
+#define PPSMC_MSG_SetBadMemoryPagesRetiredFlagsPerChannel 0x4B
+#define PPSMC_MSG_SetPriorityDeltaGain           0x4C
+#define PPSMC_MSG_AllowIHHostInterrupt           0x4D
+#define PPSMC_MSG_Mode3Reset                     0x4F
+#define PPSMC_Message_Count                      0x50
+#endif
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
index af427cc7dbb8..c48214e3dc8e 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
@@ -445,4 +445,11 @@ enum smu_feature_mask {
 	SMU_FEATURE_COUNT,
 };
 
+/* Message category flags */
+#define SMU_MSG_VF_FLAG			(1U << 0)
+#define SMU_MSG_RAS_PRI			(1U << 1)
+
+/* Firmware capability flags */
+#define SMU_FW_CAP_RAS_PRI		(1U << 0)
+
 #endif
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h
index a0e5ad0381d6..c2ab336bb530 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h
@@ -237,7 +237,7 @@ int smu_v11_0_set_azalia_d3_pme(struct smu_context *smu);
 int smu_v11_0_get_max_sustainable_clocks_by_dc(struct smu_context *smu,
 		struct pp_smu_nv_clock_table *max_clocks);
 
-bool smu_v11_0_baco_is_support(struct smu_context *smu);
+int smu_v11_0_get_bamaco_support(struct smu_context *smu);
 
 enum smu_baco_state smu_v11_0_baco_get_state(struct smu_context *smu);
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
index fbd57fa1a004..d9700a3f28d2 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
@@ -210,7 +210,7 @@ int smu_v13_0_set_azalia_d3_pme(struct smu_context *smu);
 int smu_v13_0_get_max_sustainable_clocks_by_dc(struct smu_context *smu,
 					       struct pp_smu_nv_clock_table *max_clocks);
 
-bool smu_v13_0_baco_is_support(struct smu_context *smu);
+int smu_v13_0_get_bamaco_support(struct smu_context *smu);
 
 int smu_v13_0_baco_enter(struct smu_context *smu);
 int smu_v13_0_baco_exit(struct smu_context *smu);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h
index 4af1985ae446..1fc4557e6fb4 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h
@@ -28,7 +28,7 @@
 #define SMU14_DRIVER_IF_VERSION_INV 0xFFFFFFFF
 #define SMU14_DRIVER_IF_VERSION_SMU_V14_0_0 0x7
 #define SMU14_DRIVER_IF_VERSION_SMU_V14_0_1 0x6
-#define SMU14_DRIVER_IF_VERSION_SMU_V14_0_2 0x1
+#define SMU14_DRIVER_IF_VERSION_SMU_V14_0_2 0x25
 
 #define FEATURE_MASK(feature) (1ULL << feature)
 
@@ -39,7 +39,8 @@
 #define MP1_SRAM			0x03c00004
 
 /* address block */
-#define smnMP1_FIRMWARE_FLAGS		0x3010028
+#define smnMP1_FIRMWARE_FLAGS_14_0_0	0x3010028
+#define smnMP1_FIRMWARE_FLAGS		0x3010024
 #define smnMP1_PUB_CTRL			0x3010d10
 
 #define MAX_DPM_LEVELS 16
@@ -160,7 +161,7 @@ int smu_v14_0_register_irq_handler(struct smu_context *smu);
 int smu_v14_0_baco_set_armd3_sequence(struct smu_context *smu,
 				      enum smu_baco_seq baco_seq);
 
-bool smu_v14_0_baco_is_support(struct smu_context *smu);
+int smu_v14_0_get_bamaco_support(struct smu_context *smu);
 
 enum smu_baco_state smu_v14_0_baco_get_state(struct smu_context *smu);
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0_2_pptable.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0_2_pptable.h
new file mode 100644
index 000000000000..4a3fde89aed7
--- /dev/null
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0_2_pptable.h
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef SMU_14_0_2_PPTABLE_H
+#define SMU_14_0_2_PPTABLE_H
+
+
+#pragma pack(push, 1)
+
+#define SMU_14_0_2_TABLE_FORMAT_REVISION 3
+
+// POWERPLAYTABLE::ulPlatformCaps
+#define SMU_14_0_2_PP_PLATFORM_CAP_POWERPLAY        0x1     // This cap indicates whether CCC need to show Powerplay page.
+#define SMU_14_0_2_PP_PLATFORM_CAP_SBIOSPOWERSOURCE 0x2     // This cap indicates whether power source notificaiton is done by SBIOS instead of OS.
+#define SMU_14_0_2_PP_PLATFORM_CAP_HARDWAREDC       0x4     // This cap indicates whether DC mode notificaiton is done by GPIO pin directly.
+#define SMU_14_0_2_PP_PLATFORM_CAP_BACO             0x8     // This cap indicates whether board supports the BACO circuitry.
+#define SMU_14_0_2_PP_PLATFORM_CAP_MACO             0x10    // This cap indicates whether board supports the MACO circuitry.
+#define SMU_14_0_2_PP_PLATFORM_CAP_SHADOWPSTATE     0x20    // This cap indicates whether board supports the Shadow Pstate.
+#define SMU_14_0_2_PP_PLATFORM_CAP_LEDSUPPORTED     0x40    // This cap indicates whether board supports the LED.
+#define SMU_14_0_2_PP_PLATFORM_CAP_MOBILEOVERDRIVE  0x80    // This cap indicates whether board supports the Mobile Overdrive.
+
+// SMU_14_0_2_PP_THERMALCONTROLLER - Thermal Controller Type
+#define SMU_14_0_2_PP_THERMALCONTROLLER_NONE        0
+
+#define SMU_14_0_2_PP_OVERDRIVE_VERSION             0x1     // TODO: FIX OverDrive Version TBD
+#define SMU_14_0_2_PP_POWERSAVINGCLOCK_VERSION      0x01    // Power Saving Clock Table Version 1.00
+
+enum SMU_14_0_2_OD_SW_FEATURE_CAP
+{
+    SMU_14_0_2_ODCAP_AUTO_FAN_ACOUSTIC_LIMIT        = 0,
+    SMU_14_0_2_ODCAP_POWER_MODE                     = 1,
+    SMU_14_0_2_ODCAP_AUTO_UV_ENGINE                 = 2,
+    SMU_14_0_2_ODCAP_AUTO_OC_ENGINE                 = 3,
+    SMU_14_0_2_ODCAP_AUTO_OC_MEMORY                 = 4,
+    SMU_14_0_2_ODCAP_MEMORY_TIMING_TUNE             = 5,
+    SMU_14_0_2_ODCAP_MANUAL_AC_TIMING               = 6,
+    SMU_14_0_2_ODCAP_AUTO_VF_CURVE_OPTIMIZER        = 7,
+    SMU_14_0_2_ODCAP_AUTO_SOC_UV                    = 8,
+    SMU_14_0_2_ODCAP_COUNT                          = 9,
+};
+
+enum SMU_14_0_2_OD_SW_FEATURE_ID
+{
+    SMU_14_0_2_ODFEATURE_AUTO_FAN_ACOUSTIC_LIMIT      = 1 << SMU_14_0_2_ODCAP_AUTO_FAN_ACOUSTIC_LIMIT,      // Auto Fan Acoustic RPM
+    SMU_14_0_2_ODFEATURE_POWER_MODE                   = 1 << SMU_14_0_2_ODCAP_POWER_MODE,                   // Optimized GPU Power Mode
+    SMU_14_0_2_ODFEATURE_AUTO_UV_ENGINE               = 1 << SMU_14_0_2_ODCAP_AUTO_UV_ENGINE,               // Auto Under Volt GFXCLK
+    SMU_14_0_2_ODFEATURE_AUTO_OC_ENGINE               = 1 << SMU_14_0_2_ODCAP_AUTO_OC_ENGINE,               // Auto Over Clock GFXCLK
+    SMU_14_0_2_ODFEATURE_AUTO_OC_MEMORY               = 1 << SMU_14_0_2_ODCAP_AUTO_OC_MEMORY,               // Auto Over Clock MCLK
+    SMU_14_0_2_ODFEATURE_MEMORY_TIMING_TUNE           = 1 << SMU_14_0_2_ODCAP_MEMORY_TIMING_TUNE,           // Auto AC Timing Tuning
+    SMU_14_0_2_ODFEATURE_MANUAL_AC_TIMING             = 1 << SMU_14_0_2_ODCAP_MANUAL_AC_TIMING,             // Manual fine grain AC Timing tuning
+    SMU_14_0_2_ODFEATURE_AUTO_VF_CURVE_OPTIMIZER      = 1 << SMU_14_0_2_ODCAP_AUTO_VF_CURVE_OPTIMIZER,      // Fine grain auto VF curve tuning
+    SMU_14_0_2_ODFEATURE_AUTO_SOC_UV                  = 1 << SMU_14_0_2_ODCAP_AUTO_SOC_UV,                  // Auto Unver Volt VDDSOC
+};
+
+#define SMU_14_0_2_MAX_ODFEATURE 32 // Maximum Number of OD Features
+
+enum SMU_14_0_2_OD_SW_FEATURE_SETTING_ID
+{
+    SMU_14_0_2_ODSETTING_AUTO_FAN_ACOUSTIC_LIMIT    = 0,
+    SMU_14_0_2_ODSETTING_POWER_MODE                 = 1,
+    SMU_14_0_2_ODSETTING_AUTOUVENGINE               = 2,
+    SMU_14_0_2_ODSETTING_AUTOOCENGINE               = 3,
+    SMU_14_0_2_ODSETTING_AUTOOCMEMORY               = 4,
+    SMU_14_0_2_ODSETTING_ACTIMING                   = 5,
+    SMU_14_0_2_ODSETTING_MANUAL_AC_TIMING           = 6,
+    SMU_14_0_2_ODSETTING_AUTO_VF_CURVE_OPTIMIZER    = 7,
+    SMU_14_0_2_ODSETTING_AUTO_SOC_UV                = 8,
+    SMU_14_0_2_ODSETTING_COUNT                      = 9,
+};
+#define SMU_14_0_2_MAX_ODSETTING 64 // Maximum Number of ODSettings
+
+enum SMU_14_0_2_PWRMODE_SETTING
+{
+    SMU_14_0_2_PMSETTING_POWER_LIMIT_QUIET = 0,
+    SMU_14_0_2_PMSETTING_POWER_LIMIT_BALANCE,
+    SMU_14_0_2_PMSETTING_POWER_LIMIT_TURBO,
+    SMU_14_0_2_PMSETTING_POWER_LIMIT_RAGE,
+    SMU_14_0_2_PMSETTING_ACOUSTIC_TEMP_QUIET,
+    SMU_14_0_2_PMSETTING_ACOUSTIC_TEMP_BALANCE,
+    SMU_14_0_2_PMSETTING_ACOUSTIC_TEMP_TURBO,
+    SMU_14_0_2_PMSETTING_ACOUSTIC_TEMP_RAGE,
+    SMU_14_0_2_PMSETTING_ACOUSTIC_TARGET_RPM_QUIET,
+    SMU_14_0_2_PMSETTING_ACOUSTIC_TARGET_RPM_BALANCE,
+    SMU_14_0_2_PMSETTING_ACOUSTIC_TARGET_RPM_TURBO,
+    SMU_14_0_2_PMSETTING_ACOUSTIC_TARGET_RPM_RAGE,
+    SMU_14_0_2_PMSETTING_ACOUSTIC_LIMIT_RPM_QUIET,
+    SMU_14_0_2_PMSETTING_ACOUSTIC_LIMIT_RPM_BALANCE,
+    SMU_14_0_2_PMSETTING_ACOUSTIC_LIMIT_RPM_TURBO,
+    SMU_14_0_2_PMSETTING_ACOUSTIC_LIMIT_RPM_RAGE,
+};
+#define SMU_14_0_2_MAX_PMSETTING 32 // Maximum Number of PowerMode Settings
+
+enum SMU_14_0_2_overdrive_table_id
+{
+    SMU_14_0_2_OVERDRIVE_TABLE_BASIC    = 0,
+    SMU_14_0_2_OVERDRIVE_TABLE_ADVANCED = 1,
+    SMU_14_0_2_OVERDRIVE_TABLE_COUNT    = 2,
+};
+
+struct smu_14_0_2_overdrive_table
+{
+    uint8_t revision;                                                           // Revision = SMU_14_0_2_PP_OVERDRIVE_VERSION
+    uint8_t reserve[3];                                                         // Zero filled field reserved for future use
+    uint8_t cap[SMU_14_0_2_OVERDRIVE_TABLE_COUNT][SMU_14_0_2_MAX_ODFEATURE];    // OD feature support flags
+    int32_t max[SMU_14_0_2_OVERDRIVE_TABLE_COUNT][SMU_14_0_2_MAX_ODSETTING];    // maximum settings
+    int32_t min[SMU_14_0_2_OVERDRIVE_TABLE_COUNT][SMU_14_0_2_MAX_ODSETTING];    // minimum settings
+    int16_t pm_setting[SMU_14_0_2_MAX_PMSETTING];                               // Optimized power mode feature settings
+};
+
+struct smu_14_0_2_powerplay_table
+{
+    struct atom_common_table_header header;                 // header.format_revision = 3 (HAS TO MATCH SMU_14_0_2_TABLE_FORMAT_REVISION), header.content_revision = ? structuresize is calculated by PPGen.
+    uint8_t table_revision;                                 // PPGen use only: table_revision = 3
+    uint8_t padding;                                        // Padding 1 byte to align table_size offset to 6 bytes (pmfw_start_offset, for PMFW to know the starting offset of PPTable_t).
+    uint16_t pmfw_pptable_start_offset;                     // The start offset of the pmfw portion. i.e. start of PPTable_t (start of SkuTable_t)
+    uint16_t pmfw_pptable_size;                             // The total size of pmfw_pptable, i.e PPTable_t.
+    uint16_t pmfw_pfe_table_start_offset;                   // The start offset of the PFE_Settings_t within pmfw_pptable.
+    uint16_t pmfw_pfe_table_size;                           // The size of PFE_Settings_t.
+    uint16_t pmfw_board_table_start_offset;                 // The start offset of the BoardTable_t within pmfw_pptable.
+    uint16_t pmfw_board_table_size;                         // The size of BoardTable_t.
+    uint16_t pmfw_custom_sku_table_start_offset;            // The start offset of the CustomSkuTable_t within pmfw_pptable.
+    uint16_t pmfw_custom_sku_table_size;                    // The size of the CustomSkuTable_t.
+    uint32_t golden_pp_id;                                  // PPGen use only: PP Table ID on the Golden Data Base
+    uint32_t golden_revision;                               // PPGen use only: PP Table Revision on the Golden Data Base
+    uint16_t format_id;                                     // PPGen use only: PPTable for different ASICs.
+    uint32_t platform_caps;                                 // POWERPLAYTABLE::ulPlatformCaps
+
+    uint8_t thermal_controller_type;                        // one of smu_14_0_2_PP_THERMALCONTROLLER
+
+    uint16_t small_power_limit1;
+    uint16_t small_power_limit2;
+    uint16_t boost_power_limit;                             // For Gemini Board, when the slave adapter is in BACO mode, the master adapter will use this boost power limit instead of the default power limit to boost the power limit.
+    uint16_t software_shutdown_temp;
+
+    uint8_t reserve[143];                                   // Zero filled field reserved for future use
+
+    struct smu_14_0_2_overdrive_table overdrive_table;
+
+    PPTable_t smc_pptable;                          // PPTable_t in driver_if.h -- as requested by PMFW, this offset should start at a 32-byte boundary, and the table_size above should remain at offset=6 bytes
+};
+
+#pragma pack(pop)
+
+#endif
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c
index 0c2d04f978ac..6d334a2aff67 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c
@@ -2387,7 +2387,7 @@ static const struct pptable_funcs arcturus_ppt_funcs = {
 	.register_irq_handler = smu_v11_0_register_irq_handler,
 	.set_azalia_d3_pme = smu_v11_0_set_azalia_d3_pme,
 	.get_max_sustainable_clocks_by_dc = smu_v11_0_get_max_sustainable_clocks_by_dc,
-	.baco_is_support = smu_v11_0_baco_is_support,
+	.get_bamaco_support = smu_v11_0_get_bamaco_support,
 	.baco_enter = smu_v11_0_baco_enter,
 	.baco_exit = smu_v11_0_baco_exit,
 	.get_dpm_ultimate_freq = smu_v11_0_get_dpm_ultimate_freq,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
index 836b1df79928..5a68d365967f 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
@@ -3538,7 +3538,7 @@ static const struct pptable_funcs navi10_ppt_funcs = {
 	.register_irq_handler = smu_v11_0_register_irq_handler,
 	.set_azalia_d3_pme = smu_v11_0_set_azalia_d3_pme,
 	.get_max_sustainable_clocks_by_dc = smu_v11_0_get_max_sustainable_clocks_by_dc,
-	.baco_is_support = smu_v11_0_baco_is_support,
+	.get_bamaco_support = smu_v11_0_get_bamaco_support,
 	.baco_enter = navi10_baco_enter,
 	.baco_exit = navi10_baco_exit,
 	.get_dpm_ultimate_freq = smu_v11_0_get_dpm_ultimate_freq,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
index 1f18b61884f3..e426f457a017 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
@@ -4431,7 +4431,7 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = {
 	.register_irq_handler = smu_v11_0_register_irq_handler,
 	.set_azalia_d3_pme = smu_v11_0_set_azalia_d3_pme,
 	.get_max_sustainable_clocks_by_dc = smu_v11_0_get_max_sustainable_clocks_by_dc,
-	.baco_is_support = smu_v11_0_baco_is_support,
+	.get_bamaco_support = smu_v11_0_get_bamaco_support,
 	.baco_enter = sienna_cichlid_baco_enter,
 	.baco_exit = sienna_cichlid_baco_exit,
 	.mode1_reset_is_support = sienna_cichlid_is_mode1_reset_supported,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c
index f6545093bfc1..9d5ab2ea643a 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c
@@ -93,7 +93,7 @@ static void smu_v11_0_poll_baco_exit(struct smu_context *smu)
 int smu_v11_0_init_microcode(struct smu_context *smu)
 {
 	struct amdgpu_device *adev = smu->adev;
-	char ucode_prefix[30];
+	char ucode_prefix[25];
 	char fw_name[SMU_FW_NAME_LEN];
 	int err = 0;
 	const struct smc_firmware_header_v1_0 *hdr;
@@ -1557,23 +1557,27 @@ int smu_v11_0_baco_set_armd3_sequence(struct smu_context *smu,
 	return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_ArmD3, baco_seq, NULL);
 }
 
-bool smu_v11_0_baco_is_support(struct smu_context *smu)
+int smu_v11_0_get_bamaco_support(struct smu_context *smu)
 {
 	struct smu_baco_context *smu_baco = &smu->smu_baco;
+	int bamaco_support = 0;
 
 	if (amdgpu_sriov_vf(smu->adev) || !smu_baco->platform_support)
-		return false;
+		return 0;
+
+	if (smu_baco->maco_support)
+		bamaco_support |= MACO_SUPPORT;
 
 	/* return true if ASIC is in BACO state already */
 	if (smu_v11_0_baco_get_state(smu) == SMU_BACO_STATE_ENTER)
-		return true;
+		return bamaco_support |= BACO_SUPPORT;
 
 	/* Arcturus does not support this bit mask */
 	if (smu_cmn_feature_is_supported(smu, SMU_FEATURE_BACO_BIT) &&
 	   !smu_cmn_feature_is_enabled(smu, SMU_FEATURE_BACO_BIT))
-		return false;
+		return 0;
 
-	return true;
+	return (bamaco_support |= BACO_SUPPORT);
 }
 
 enum smu_baco_state smu_v11_0_baco_get_state(struct smu_context *smu)
@@ -1603,7 +1607,7 @@ int smu_v11_0_baco_set_state(struct smu_context *smu, enum smu_baco_state state)
 		case IP_VERSION(11, 0, 11):
 		case IP_VERSION(11, 0, 12):
 		case IP_VERSION(11, 0, 13):
-			if (amdgpu_runtime_pm == 2)
+			if (adev->pm.rpm_mode == AMDGPU_RUNPM_BAMACO)
 				ret = smu_cmn_send_smc_msg_with_param(smu,
 								      SMU_MSG_EnterBaco,
 								      D3HOT_BAMACO_SEQUENCE,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
index da1f43999d09..379e44eb0019 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
@@ -301,7 +301,7 @@ static int vangogh_get_legacy_smu_metrics_data(struct smu_context *smu,
 		*value = metrics->GfxActivity / 100;
 		break;
 	case METRICS_AVERAGE_VCNACTIVITY:
-		*value = metrics->UvdActivity;
+		*value = metrics->UvdActivity / 100;
 		break;
 	case METRICS_AVERAGE_SOCKETPOWER:
 		*value = (metrics->CurrentSocketPower << 8) /
@@ -1507,6 +1507,12 @@ static int vangogh_read_sensor(struct smu_context *smu,
 						   (uint32_t *)data);
 		*size = 4;
 		break;
+	case AMDGPU_PP_SENSOR_VCN_LOAD:
+		ret = vangogh_common_get_smu_metrics_data(smu,
+						METRICS_AVERAGE_VCNACTIVITY,
+						(uint32_t *)data);
+		*size = 4;
+		break;
 	case AMDGPU_PP_SENSOR_GPU_AVG_POWER:
 		ret = vangogh_common_get_smu_metrics_data(smu,
 						   METRICS_AVERAGE_SOCKETPOWER,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
index f41ac6465f2a..ce941fbb9cfb 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
@@ -759,8 +759,11 @@ static int aldebaran_emit_clk_levels(struct smu_context *smu,
 	switch (type) {
 
 	case SMU_OD_SCLK:
-		*offset += sysfs_emit_at(buf, *offset, "%s:\n", "GFXCLK");
-		fallthrough;
+		*offset += sysfs_emit_at(buf, *offset, "%s:\n", "OD_SCLK");
+		*offset += sysfs_emit_at(buf, *offset, "0: %uMhz\n1: %uMhz\n",
+				      pstate_table->gfxclk_pstate.curr.min,
+				      pstate_table->gfxclk_pstate.curr.max);
+		return 0;
 	case SMU_SCLK:
 		ret = aldebaran_get_current_clk_freq_by_table(smu, SMU_GFXCLK, &cur_value);
 		if (ret) {
@@ -788,8 +791,11 @@ static int aldebaran_emit_clk_levels(struct smu_context *smu,
 		break;
 
 	case SMU_OD_MCLK:
-		*offset += sysfs_emit_at(buf, *offset, "%s:\n", "MCLK");
-		fallthrough;
+		*offset += sysfs_emit_at(buf, *offset, "%s:\n", "OD_MCLK");
+		*offset += sysfs_emit_at(buf, *offset, "0: %uMhz\n1: %uMhz\n",
+				      pstate_table->uclk_pstate.curr.min,
+				      pstate_table->uclk_pstate.curr.max);
+		return 0;
 	case SMU_MCLK:
 		ret = aldebaran_get_current_clk_freq_by_table(smu, SMU_UCLK, &cur_value);
 		if (ret) {
@@ -850,7 +856,6 @@ static int aldebaran_emit_clk_levels(struct smu_context *smu,
 	}
 
 	switch (type) {
-	case SMU_OD_SCLK:
 	case SMU_SCLK:
 		for (i = 0; i < display_levels; i++) {
 			clock_mhz = freq_values[i];
@@ -863,7 +868,6 @@ static int aldebaran_emit_clk_levels(struct smu_context *smu,
 		}
 		break;
 
-	case SMU_OD_MCLK:
 	case SMU_MCLK:
 	case SMU_SOCCLK:
 	case SMU_FCLK:
@@ -1581,11 +1585,11 @@ out:
 	adev->unique_id = ((uint64_t)upper32 << 32) | lower32;
 }
 
-static bool aldebaran_is_baco_supported(struct smu_context *smu)
+static int aldebaran_get_bamaco_support(struct smu_context *smu)
 {
 	/* aldebaran is not support baco */
 
-	return false;
+	return 0;
 }
 
 static int aldebaran_set_df_cstate(struct smu_context *smu,
@@ -2059,7 +2063,7 @@ static const struct pptable_funcs aldebaran_ppt_funcs = {
 	.register_irq_handler = smu_v13_0_register_irq_handler,
 	.set_azalia_d3_pme = smu_v13_0_set_azalia_d3_pme,
 	.get_max_sustainable_clocks_by_dc = smu_v13_0_get_max_sustainable_clocks_by_dc,
-	.baco_is_support = aldebaran_is_baco_supported,
+	.get_bamaco_support = aldebaran_get_bamaco_support,
 	.get_dpm_ultimate_freq = smu_v13_0_get_dpm_ultimate_freq,
 	.set_soft_freq_limited_range = aldebaran_set_soft_freq_limited_range,
 	.od_edit_dpm_table = aldebaran_usr_edit_dpm_table,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
index 48170bb5112e..a8d34adc7d3f 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
@@ -93,7 +93,7 @@ int smu_v13_0_init_microcode(struct smu_context *smu)
 {
 	struct amdgpu_device *adev = smu->adev;
 	char fw_name[30];
-	char ucode_prefix[30];
+	char ucode_prefix[15];
 	int err = 0;
 	const struct smc_firmware_header_v1_0 *hdr;
 	const struct common_firmware_header *header;
@@ -2247,7 +2247,7 @@ static int smu_v13_0_baco_set_state(struct smu_context *smu,
 	if (state == SMU_BACO_STATE_ENTER) {
 		ret = smu_cmn_send_smc_msg_with_param(smu,
 						      SMU_MSG_EnterBaco,
-						      (smu_baco->maco_support && amdgpu_runtime_pm != 1) ?
+						      (adev->pm.rpm_mode == AMDGPU_RUNPM_BAMACO) ?
 						      BACO_SEQ_BAMACO : BACO_SEQ_BACO,
 						      NULL);
 	} else {
@@ -2268,33 +2268,36 @@ static int smu_v13_0_baco_set_state(struct smu_context *smu,
 	return ret;
 }
 
-bool smu_v13_0_baco_is_support(struct smu_context *smu)
+int smu_v13_0_get_bamaco_support(struct smu_context *smu)
 {
 	struct smu_baco_context *smu_baco = &smu->smu_baco;
+	int bamaco_support = 0;
 
 	if (amdgpu_sriov_vf(smu->adev) || !smu_baco->platform_support)
-		return false;
+		return 0;
+
+	if (smu_baco->maco_support)
+		bamaco_support |= MACO_SUPPORT;
 
 	/* return true if ASIC is in BACO state already */
 	if (smu_v13_0_baco_get_state(smu) == SMU_BACO_STATE_ENTER)
-		return true;
+		return bamaco_support |= BACO_SUPPORT;
 
 	if (smu_cmn_feature_is_supported(smu, SMU_FEATURE_BACO_BIT) &&
 	    !smu_cmn_feature_is_enabled(smu, SMU_FEATURE_BACO_BIT))
-		return false;
+		return 0;
 
-	return true;
+	return (bamaco_support |= BACO_SUPPORT);
 }
 
 int smu_v13_0_baco_enter(struct smu_context *smu)
 {
-	struct smu_baco_context *smu_baco = &smu->smu_baco;
 	struct amdgpu_device *adev = smu->adev;
 	int ret;
 
 	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) {
 		return smu_v13_0_baco_set_armd3_sequence(smu,
-				(smu_baco->maco_support && amdgpu_runtime_pm != 1) ?
+				(adev->pm.rpm_mode == AMDGPU_RUNPM_BAMACO) ?
 					BACO_SEQ_BAMACO : BACO_SEQ_BACO);
 	} else {
 		ret = smu_v13_0_baco_set_state(smu, SMU_BACO_STATE_ENTER);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
index 67117ced7c6a..1e09d5f2d82f 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
@@ -3076,7 +3076,7 @@ static const struct pptable_funcs smu_v13_0_0_ppt_funcs = {
 	.set_tool_table_location = smu_v13_0_set_tool_table_location,
 	.deep_sleep_control = smu_v13_0_deep_sleep_control,
 	.gfx_ulv_control = smu_v13_0_gfx_ulv_control,
-	.baco_is_support = smu_v13_0_baco_is_support,
+	.get_bamaco_support = smu_v13_0_get_bamaco_support,
 	.baco_enter = smu_v13_0_baco_enter,
 	.baco_exit = smu_v13_0_baco_exit,
 	.mode1_reset_is_support = smu_v13_0_0_is_mode1_reset_supported,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c
index 949131bd1ecb..bc241b593db1 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c
@@ -226,7 +226,7 @@ static int smu_v13_0_4_system_features_control(struct smu_context *smu, bool en)
 	struct amdgpu_device *adev = smu->adev;
 	int ret = 0;
 
-	if (!en && !adev->in_s0ix) {
+	if (!en && adev->in_s4) {
 		/* Adds a GFX reset as workaround just before sending the
 		 * MP1_UNLOAD message to prevent GC/RLC/PMFW from entering
 		 * an invalid state.
@@ -328,7 +328,7 @@ static int smu_v13_0_4_get_smu_metrics_data(struct smu_context *smu,
 		*value = metrics->GfxActivity / 100;
 		break;
 	case METRICS_AVERAGE_VCNACTIVITY:
-		*value = metrics->UvdActivity;
+		*value = metrics->UvdActivity / 100;
 		break;
 	case METRICS_AVERAGE_SOCKETPOWER:
 		*value = (metrics->AverageSocketPower << 8) / 1000;
@@ -582,6 +582,12 @@ static int smu_v13_0_4_read_sensor(struct smu_context *smu,
 						       (uint32_t *)data);
 		*size = 4;
 		break;
+	case AMDGPU_PP_SENSOR_VCN_LOAD:
+		ret = smu_v13_0_4_get_smu_metrics_data(smu,
+							METRICS_AVERAGE_VCNACTIVITY,
+							(uint32_t *)data);
+		*size = 4;
+		break;
 	case AMDGPU_PP_SENSOR_GPU_AVG_POWER:
 		ret = smu_v13_0_4_get_smu_metrics_data(smu,
 						       METRICS_AVERAGE_SOCKETPOWER,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c
index 0dce672ac1b9..218f209c3775 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c
@@ -286,7 +286,7 @@ static int smu_v13_0_5_get_smu_metrics_data(struct smu_context *smu,
 		*value = metrics->GfxActivity / 100;
 		break;
 	case METRICS_AVERAGE_VCNACTIVITY:
-		*value = metrics->UvdActivity;
+		*value = metrics->UvdActivity / 100;
 		break;
 	case METRICS_CURR_SOCKETPOWER:
 		*value = (metrics->CurrentSocketPower << 8) / 1000;
@@ -332,6 +332,12 @@ static int smu_v13_0_5_read_sensor(struct smu_context *smu,
 								(uint32_t *)data);
 		*size = 4;
 		break;
+	case AMDGPU_PP_SENSOR_VCN_LOAD:
+		ret = smu_v13_0_5_get_smu_metrics_data(smu,
+							METRICS_AVERAGE_VCNACTIVITY,
+							(uint32_t *)data);
+		*size = 4;
+		break;
 	case AMDGPU_PP_SENSOR_GPU_INPUT_POWER:
 		ret = smu_v13_0_5_get_smu_metrics_data(smu,
 								METRICS_CURR_SOCKETPOWER,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
index 3957af057d54..4d3eca2fc3f1 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
@@ -138,13 +138,13 @@ static const struct cmn2asic_msg_mapping smu_v13_0_6_message_map[SMU_MSG_MAX_COU
 	MSG_MAP(SetToolsDramAddrHigh,		     PPSMC_MSG_SetToolsDramAddrHigh,		0),
 	MSG_MAP(SetToolsDramAddrLow,		     PPSMC_MSG_SetToolsDramAddrLow,		0),
 	MSG_MAP(SetSoftMinByFreq,		     PPSMC_MSG_SetSoftMinByFreq,		0),
-	MSG_MAP(SetSoftMaxByFreq,		     PPSMC_MSG_SetSoftMaxByFreq,		0),
+	MSG_MAP(SetSoftMaxByFreq,		     PPSMC_MSG_SetSoftMaxByFreq,		1),
 	MSG_MAP(GetMinDpmFreq,			     PPSMC_MSG_GetMinDpmFreq,			1),
 	MSG_MAP(GetMaxDpmFreq,			     PPSMC_MSG_GetMaxDpmFreq,			1),
 	MSG_MAP(GetDpmFreqByIndex,		     PPSMC_MSG_GetDpmFreqByIndex,		1),
 	MSG_MAP(SetPptLimit,			     PPSMC_MSG_SetPptLimit,			0),
 	MSG_MAP(GetPptLimit,			     PPSMC_MSG_GetPptLimit,			1),
-	MSG_MAP(GfxDeviceDriverReset,		     PPSMC_MSG_GfxDriverReset,			0),
+	MSG_MAP(GfxDeviceDriverReset,		     PPSMC_MSG_GfxDriverReset,			SMU_MSG_RAS_PRI),
 	MSG_MAP(DramLogSetDramAddrHigh,		     PPSMC_MSG_DramLogSetDramAddrHigh,		0),
 	MSG_MAP(DramLogSetDramAddrLow,		     PPSMC_MSG_DramLogSetDramAddrLow,		0),
 	MSG_MAP(DramLogSetDramSize,		     PPSMC_MSG_DramLogSetDramSize,		0),
@@ -167,10 +167,10 @@ static const struct cmn2asic_msg_mapping smu_v13_0_6_message_map[SMU_MSG_MAX_COU
 	MSG_MAP(GetCTFLimit,                         PPSMC_MSG_GetCTFLimit,                     0),
 	MSG_MAP(GetThermalLimit,                     PPSMC_MSG_ReadThrottlerLimit,              0),
 	MSG_MAP(ClearMcaOnRead,	                     PPSMC_MSG_ClearMcaOnRead,                  0),
-	MSG_MAP(QueryValidMcaCount,                  PPSMC_MSG_QueryValidMcaCount,              0),
-	MSG_MAP(QueryValidMcaCeCount,                PPSMC_MSG_QueryValidMcaCeCount,            0),
-	MSG_MAP(McaBankDumpDW,                       PPSMC_MSG_McaBankDumpDW,                   0),
-	MSG_MAP(McaBankCeDumpDW,                     PPSMC_MSG_McaBankCeDumpDW,                 0),
+	MSG_MAP(QueryValidMcaCount,                  PPSMC_MSG_QueryValidMcaCount,              SMU_MSG_RAS_PRI),
+	MSG_MAP(QueryValidMcaCeCount,                PPSMC_MSG_QueryValidMcaCeCount,            SMU_MSG_RAS_PRI),
+	MSG_MAP(McaBankDumpDW,                       PPSMC_MSG_McaBankDumpDW,                   SMU_MSG_RAS_PRI),
+	MSG_MAP(McaBankCeDumpDW,                     PPSMC_MSG_McaBankCeDumpDW,                 SMU_MSG_RAS_PRI),
 	MSG_MAP(SelectPLPDMode,                      PPSMC_MSG_SelectPLPDMode,                  0),
 	MSG_MAP(RmaDueToBadPageThreshold,            PPSMC_MSG_RmaDueToBadPageThreshold,        0),
 };
@@ -1010,8 +1010,11 @@ static int smu_v13_0_6_print_clk_levels(struct smu_context *smu,
 
 	switch (type) {
 	case SMU_OD_SCLK:
-		size += sysfs_emit_at(buf, size, "%s:\n", "GFXCLK");
-		fallthrough;
+		size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK");
+		size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMhz\n",
+				      pstate_table->gfxclk_pstate.curr.min,
+				      pstate_table->gfxclk_pstate.curr.max);
+		break;
 	case SMU_SCLK:
 		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_GFXCLK,
 								&now);
@@ -1052,8 +1055,11 @@ static int smu_v13_0_6_print_clk_levels(struct smu_context *smu,
 		break;
 
 	case SMU_OD_MCLK:
-		size += sysfs_emit_at(buf, size, "%s:\n", "MCLK");
-		fallthrough;
+		size += sysfs_emit_at(buf, size, "%s:\n", "OD_MCLK");
+		size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMhz\n",
+				      pstate_table->uclk_pstate.curr.min,
+				      pstate_table->uclk_pstate.curr.max);
+		break;
 	case SMU_MCLK:
 		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_UCLK,
 								&now);
@@ -1670,6 +1676,11 @@ static int smu_v13_0_6_set_soft_freq_limited_range(struct smu_context *smu,
 		if (clk_type == SMU_UCLK) {
 			if (max == pstate_table->uclk_pstate.curr.max)
 				return 0;
+			/* For VF, only allowed in FW versions 85.102 or greater */
+			if (amdgpu_sriov_vf(adev) &&
+			    ((smu->smc_fw_version < 0x556600) ||
+			     (adev->flags & AMD_IS_APU)))
+				return -EOPNOTSUPP;
 			/* Only max clock limiting is allowed for UCLK */
 			ret = smu_v13_0_set_soft_freq_limited_range(
 				smu, SMU_UCLK, 0, max);
@@ -2077,11 +2088,11 @@ static void smu_v13_0_6_get_unique_id(struct smu_context *smu)
 	adev->unique_id = pptable->PublicSerialNumber_AID;
 }
 
-static bool smu_v13_0_6_is_baco_supported(struct smu_context *smu)
+static int smu_v13_0_6_get_bamaco_support(struct smu_context *smu)
 {
 	/* smu_13_0_6 does not support baco */
 
-	return false;
+	return 0;
 }
 
 static const char *const throttling_logging_label[] = {
@@ -2228,7 +2239,15 @@ static ssize_t smu_v13_0_6_get_gpu_metrics(struct smu_context *smu, void **table
 	gpu_metrics->gfxclk_lock_status = GET_METRIC_FIELD(GfxLockXCDMak) >> GET_INST(GC, 0);
 
 	if (!(adev->flags & AMD_IS_APU)) {
-		if (!amdgpu_sriov_vf(adev)) {
+		/*Check smu version, PCIE link speed and width will be reported from pmfw metric
+		 * table for both pf & one vf for smu version 85.99.0 or higher else report only
+		 * for pf from registers
+		 */
+		if (smu->smc_fw_version >= 0x556300) {
+			gpu_metrics->pcie_link_width = metrics_x->PCIeLinkWidth;
+			gpu_metrics->pcie_link_speed =
+				pcie_gen_to_speed(metrics_x->PCIeLinkSpeed);
+		} else if (!amdgpu_sriov_vf(adev)) {
 			link_width_level = smu_v13_0_6_get_current_pcie_link_width_level(smu);
 			if (link_width_level > MAX_LINK_WIDTH)
 				link_width_level = 0;
@@ -2238,6 +2257,7 @@ static ssize_t smu_v13_0_6_get_gpu_metrics(struct smu_context *smu, void **table
 			gpu_metrics->pcie_link_speed =
 				smu_v13_0_6_get_current_pcie_link_speed(smu);
 		}
+
 		gpu_metrics->pcie_bandwidth_acc =
 				SMUQ10_ROUND(metrics_x->PcieBandwidthAcc[0]);
 		gpu_metrics->pcie_bandwidth_inst =
@@ -2294,6 +2314,17 @@ static ssize_t smu_v13_0_6_get_gpu_metrics(struct smu_context *smu, void **table
 	return sizeof(*gpu_metrics);
 }
 
+static void smu_v13_0_6_restore_pci_config(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	int i;
+
+	for (i = 0; i < 16; i++)
+		pci_write_config_dword(adev->pdev, i * 4,
+				       adev->pdev->saved_config_space[i]);
+	pci_restore_msi_state(adev->pdev);
+}
+
 static int smu_v13_0_6_mode2_reset(struct smu_context *smu)
 {
 	int ret = 0, index;
@@ -2315,6 +2346,20 @@ static int smu_v13_0_6_mode2_reset(struct smu_context *smu)
 	/* Restore the config space saved during init */
 	amdgpu_device_load_pci_state(adev->pdev);
 
+	/* Certain platforms have switches which assign virtual BAR values to
+	 * devices. OS uses the virtual BAR values and device behind the switch
+	 * is assgined another BAR value. When device's config space registers
+	 * are queried, switch returns the virtual BAR values. When mode-2 reset
+	 * is performed, switch is unaware of it, and will continue to return
+	 * the same virtual values to the OS.This affects
+	 * pci_restore_config_space() API as it doesn't write the value saved if
+	 * the current value read from config space is the same as what is
+	 * saved. As a workaround, make sure the config space is restored
+	 * always.
+	 */
+	if (!(adev->flags & AMD_IS_APU))
+		smu_v13_0_6_restore_pci_config(smu);
+
 	dev_dbg(smu->adev->dev, "wait for reset ack\n");
 	do {
 		ret = smu_cmn_wait_for_response(smu);
@@ -2671,6 +2716,11 @@ static int mca_umc_mca_get_err_count(const struct mca_ras_info *mca_ras, struct
 	    umc_v12_0_is_correctable_error(adev, status0))
 		*count = (ext_error_code == 0) ? odecc_err_cnt : 1;
 
+	amdgpu_umc_update_ecc_status(adev,
+			entry->regs[MCA_REG_IDX_STATUS],
+			entry->regs[MCA_REG_IDX_IPID],
+			entry->regs[MCA_REG_IDX_ADDR]);
+
 	return 0;
 }
 
@@ -2684,7 +2734,8 @@ static int mca_pcs_xgmi_mca_get_err_count(const struct mca_ras_info *mca_ras, st
 	ext_error_code = MCA_REG__STATUS__ERRORCODEEXT(entry->regs[MCA_REG_IDX_STATUS]);
 	err_cnt = MCA_REG__MISC0__ERRCNT(entry->regs[MCA_REG_IDX_MISC0]);
 
-	if (type == AMDGPU_MCA_ERROR_TYPE_UE && ext_error_code == 0)
+	if (type == AMDGPU_MCA_ERROR_TYPE_UE &&
+	    (ext_error_code == 0 || ext_error_code == 9))
 		*count = err_cnt;
 	else if (type == AMDGPU_MCA_ERROR_TYPE_CE && ext_error_code == 6)
 		*count = err_cnt;
@@ -2975,7 +3026,7 @@ static int aca_smu_set_debug_mode(struct amdgpu_device *adev, bool enable)
 	return smu_v13_0_6_mca_set_debug_mode(smu, enable);
 }
 
-static int smu_v13_0_6_get_valid_aca_count(struct smu_context *smu, enum aca_error_type type, u32 *count)
+static int smu_v13_0_6_get_valid_aca_count(struct smu_context *smu, enum aca_smu_type type, u32 *count)
 {
 	uint32_t msg;
 	int ret;
@@ -2984,10 +3035,10 @@ static int smu_v13_0_6_get_valid_aca_count(struct smu_context *smu, enum aca_err
 		return -EINVAL;
 
 	switch (type) {
-	case ACA_ERROR_TYPE_UE:
+	case ACA_SMU_TYPE_UE:
 		msg = SMU_MSG_QueryValidMcaCount;
 		break;
-	case ACA_ERROR_TYPE_CE:
+	case ACA_SMU_TYPE_CE:
 		msg = SMU_MSG_QueryValidMcaCeCount;
 		break;
 	default:
@@ -3004,14 +3055,14 @@ static int smu_v13_0_6_get_valid_aca_count(struct smu_context *smu, enum aca_err
 }
 
 static int aca_smu_get_valid_aca_count(struct amdgpu_device *adev,
-				       enum aca_error_type type, u32 *count)
+				       enum aca_smu_type type, u32 *count)
 {
 	struct smu_context *smu = adev->powerplay.pp_handle;
 	int ret;
 
 	switch (type) {
-	case ACA_ERROR_TYPE_UE:
-	case ACA_ERROR_TYPE_CE:
+	case ACA_SMU_TYPE_UE:
+	case ACA_SMU_TYPE_CE:
 		ret = smu_v13_0_6_get_valid_aca_count(smu, type, count);
 		break;
 	default:
@@ -3022,16 +3073,16 @@ static int aca_smu_get_valid_aca_count(struct amdgpu_device *adev,
 	return ret;
 }
 
-static int __smu_v13_0_6_aca_bank_dump(struct smu_context *smu, enum aca_error_type type,
+static int __smu_v13_0_6_aca_bank_dump(struct smu_context *smu, enum aca_smu_type type,
 				       int idx, int offset, u32 *val)
 {
 	uint32_t msg, param;
 
 	switch (type) {
-	case ACA_ERROR_TYPE_UE:
+	case ACA_SMU_TYPE_UE:
 		msg = SMU_MSG_McaBankDumpDW;
 		break;
-	case ACA_ERROR_TYPE_CE:
+	case ACA_SMU_TYPE_CE:
 		msg = SMU_MSG_McaBankCeDumpDW;
 		break;
 	default:
@@ -3043,7 +3094,7 @@ static int __smu_v13_0_6_aca_bank_dump(struct smu_context *smu, enum aca_error_t
 	return smu_cmn_send_smc_msg_with_param(smu, msg, param, (uint32_t *)val);
 }
 
-static int smu_v13_0_6_aca_bank_dump(struct smu_context *smu, enum aca_error_type type,
+static int smu_v13_0_6_aca_bank_dump(struct smu_context *smu, enum aca_smu_type type,
 				     int idx, int offset, u32 *val, int count)
 {
 	int ret, i;
@@ -3060,7 +3111,7 @@ static int smu_v13_0_6_aca_bank_dump(struct smu_context *smu, enum aca_error_typ
 	return 0;
 }
 
-static int aca_bank_read_reg(struct amdgpu_device *adev, enum aca_error_type type,
+static int aca_bank_read_reg(struct amdgpu_device *adev, enum aca_smu_type type,
 			     int idx, int reg_idx, u64 *val)
 {
 	struct smu_context *smu = adev->powerplay.pp_handle;
@@ -3077,13 +3128,13 @@ static int aca_bank_read_reg(struct amdgpu_device *adev, enum aca_error_type typ
 	*val = (u64)data[1] << 32 | data[0];
 
 	dev_dbg(adev->dev, "mca read bank reg: type:%s, index: %d, reg_idx: %d, val: 0x%016llx\n",
-		type == ACA_ERROR_TYPE_UE ? "UE" : "CE", idx, reg_idx, *val);
+		type == ACA_SMU_TYPE_UE ? "UE" : "CE", idx, reg_idx, *val);
 
 	return 0;
 }
 
 static int aca_smu_get_valid_aca_bank(struct amdgpu_device *adev,
-				      enum aca_error_type type, int idx, struct aca_bank *bank)
+				      enum aca_smu_type type, int idx, struct aca_bank *bank)
 {
 	int i, ret, count;
 
@@ -3097,12 +3148,25 @@ static int aca_smu_get_valid_aca_bank(struct amdgpu_device *adev,
 	return 0;
 }
 
+static int aca_smu_parse_error_code(struct amdgpu_device *adev, struct aca_bank *bank)
+{
+	int error_code;
+
+	if (!(adev->flags & AMD_IS_APU) && adev->pm.fw_version >= 0x00555600)
+		error_code = ACA_REG__SYND__ERRORINFORMATION(bank->regs[ACA_REG_IDX_SYND]);
+	else
+		error_code = ACA_REG__STATUS__ERRORCODE(bank->regs[ACA_REG_IDX_STATUS]);
+
+	return error_code & 0xff;
+}
+
 static const struct aca_smu_funcs smu_v13_0_6_aca_smu_funcs = {
 	.max_ue_bank_count = 12,
 	.max_ce_bank_count = 12,
 	.set_debug_mode = aca_smu_set_debug_mode,
 	.get_valid_aca_count = aca_smu_get_valid_aca_count,
 	.get_valid_aca_bank = aca_smu_get_valid_aca_bank,
+	.parse_error_code = aca_smu_parse_error_code,
 };
 
 static int smu_v13_0_6_select_xgmi_plpd_policy(struct smu_context *smu,
@@ -3179,7 +3243,7 @@ static const struct pptable_funcs smu_v13_0_6_ppt_funcs = {
 	.enable_thermal_alert = smu_v13_0_enable_thermal_alert,
 	.disable_thermal_alert = smu_v13_0_disable_thermal_alert,
 	.setup_pptable = smu_v13_0_6_setup_pptable,
-	.baco_is_support = smu_v13_0_6_is_baco_supported,
+	.get_bamaco_support = smu_v13_0_6_get_bamaco_support,
 	.get_dpm_ultimate_freq = smu_v13_0_6_get_dpm_ultimate_freq,
 	.set_soft_freq_limited_range = smu_v13_0_6_set_soft_freq_limited_range,
 	.od_edit_dpm_table = smu_v13_0_6_usr_edit_dpm_table,
@@ -3208,6 +3272,7 @@ void smu_v13_0_6_set_ppt_funcs(struct smu_context *smu)
 	smu->feature_map = smu_v13_0_6_feature_mask_map;
 	smu->table_map = smu_v13_0_6_table_map;
 	smu->smc_driver_if_version = SMU13_0_6_DRIVER_IF_VERSION;
+	smu->smc_fw_caps |= SMU_FW_CAP_RAS_PRI;
 	smu_v13_0_set_smu_mailbox_registers(smu);
 	amdgpu_mca_smu_init_funcs(smu->adev, &smu_v13_0_6_mca_smu_funcs);
 	amdgpu_aca_set_smu_funcs(smu->adev, &smu_v13_0_6_aca_smu_funcs);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
index 7318964f1f14..e996a0a4d33e 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
@@ -2650,7 +2650,7 @@ static const struct pptable_funcs smu_v13_0_7_ppt_funcs = {
 	.set_tool_table_location = smu_v13_0_set_tool_table_location,
 	.get_pp_feature_mask = smu_cmn_get_pp_feature_mask,
 	.set_pp_feature_mask = smu_cmn_set_pp_feature_mask,
-	.baco_is_support = smu_v13_0_baco_is_support,
+	.get_bamaco_support = smu_v13_0_get_bamaco_support,
 	.baco_enter = smu_v13_0_baco_enter,
 	.baco_exit = smu_v13_0_baco_exit,
 	.mode1_reset_is_support = smu_v13_0_7_is_mode1_reset_supported,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c
index 2d1736234b4a..d8bcf765a803 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c
@@ -363,7 +363,7 @@ static int yellow_carp_get_smu_metrics_data(struct smu_context *smu,
 		*value = metrics->GfxActivity / 100;
 		break;
 	case METRICS_AVERAGE_VCNACTIVITY:
-		*value = metrics->UvdActivity;
+		*value = metrics->UvdActivity / 100;
 		break;
 	case METRICS_CURR_SOCKETPOWER:
 		*value = (metrics->CurrentSocketPower << 8) / 1000;
@@ -423,6 +423,12 @@ static int yellow_carp_read_sensor(struct smu_context *smu,
 								(uint32_t *)data);
 		*size = 4;
 		break;
+	case AMDGPU_PP_SENSOR_VCN_LOAD:
+		ret = yellow_carp_get_smu_metrics_data(smu,
+							METRICS_AVERAGE_VCNACTIVITY,
+							(uint32_t *)data);
+		*size = 4;
+		break;
 	case AMDGPU_PP_SENSOR_GPU_INPUT_POWER:
 		ret = yellow_carp_get_smu_metrics_data(smu,
 								METRICS_CURR_SOCKETPOWER,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/Makefile b/drivers/gpu/drm/amd/pm/swsmu/smu14/Makefile
index ddbac5c655f7..4593e29e8ff8 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu14/Makefile
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/Makefile
@@ -23,7 +23,7 @@
 # Makefile for the 'smu manager' sub-component of powerplay.
 # It provides the smu management services for the driver.
 
-SMU14_MGR = smu_v14_0.o smu_v14_0_0_ppt.o
+SMU14_MGR = smu_v14_0.o smu_v14_0_0_ppt.o smu_v14_0_2_ppt.o
 
 AMD_SWSMU_SMU14MGR = $(addprefix $(AMD_SWSMU_PATH)/smu14/,$(SMU14_MGR))
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c
index 07a65e005785..68b9bf822e8d 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c
@@ -38,8 +38,13 @@
 #include "amdgpu_ras.h"
 #include "smu_cmn.h"
 
-#include "asic_reg/mp/mp_14_0_0_offset.h"
-#include "asic_reg/mp/mp_14_0_0_sh_mask.h"
+#include "asic_reg/mp/mp_14_0_2_offset.h"
+#include "asic_reg/mp/mp_14_0_2_sh_mask.h"
+
+#define regMP1_SMN_IH_SW_INT_mp1_14_0_0			0x0341
+#define regMP1_SMN_IH_SW_INT_mp1_14_0_0_BASE_IDX        0
+#define regMP1_SMN_IH_SW_INT_CTRL_mp1_14_0_0            0x0342
+#define regMP1_SMN_IH_SW_INT_CTRL_mp1_14_0_0_BASE_IDX   0
 
 /*
  * DO NOT use these for err/warn/info/debug messages.
@@ -52,6 +57,7 @@
 #undef pr_debug
 
 MODULE_FIRMWARE("amdgpu/smu_14_0_2.bin");
+MODULE_FIRMWARE("amdgpu/smu_14_0_3.bin");
 
 #define ENABLE_IMU_ARG_GFXOFF_ENABLE		1
 
@@ -106,7 +112,6 @@ void smu_v14_0_fini_microcode(struct smu_context *smu)
 
 int smu_v14_0_load_microcode(struct smu_context *smu)
 {
-#if 0
 	struct amdgpu_device *adev = smu->adev;
 	const uint32_t *src;
 	const struct smc_firmware_header_v1_0 *hdr;
@@ -131,8 +136,13 @@ int smu_v14_0_load_microcode(struct smu_context *smu)
 		    1 & ~MP1_SMN_PUB_CTRL__LX3_RESET_MASK);
 
 	for (i = 0; i < adev->usec_timeout; i++) {
-		mp1_fw_flags = RREG32_PCIE(MP1_Public |
-					   (smnMP1_FIRMWARE_FLAGS & 0xffffffff));
+		if (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(14, 0, 0) ||
+			amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(14, 0, 1))
+			mp1_fw_flags = RREG32_PCIE(MP1_Public |
+						   (smnMP1_FIRMWARE_FLAGS_14_0_0 & 0xffffffff));
+		else
+			mp1_fw_flags = RREG32_PCIE(MP1_Public |
+						   (smnMP1_FIRMWARE_FLAGS & 0xffffffff));
 		if ((mp1_fw_flags & MP1_CRU1_MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) >>
 		    MP1_CRU1_MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT)
 			break;
@@ -142,9 +152,7 @@ int smu_v14_0_load_microcode(struct smu_context *smu)
 	if (i == adev->usec_timeout)
 		return -ETIME;
 
-#endif
 	return 0;
-
 }
 
 int smu_v14_0_init_pptable_microcode(struct smu_context *smu)
@@ -165,6 +173,10 @@ int smu_v14_0_init_pptable_microcode(struct smu_context *smu)
 	if (!adev->scpm_enabled)
 		return 0;
 
+	if ((amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(14, 0, 2)) ||
+	    (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(14, 0, 3)))
+		return 0;
+
 	/* override pptable_id from driver parameter */
 	if (amdgpu_smu_pptable_id >= 0) {
 		pptable_id = amdgpu_smu_pptable_id;
@@ -198,7 +210,12 @@ int smu_v14_0_check_fw_status(struct smu_context *smu)
 	struct amdgpu_device *adev = smu->adev;
 	uint32_t mp1_fw_flags;
 
-	mp1_fw_flags = RREG32_PCIE(MP1_Public |
+	if (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(14, 0, 0) ||
+		amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(14, 0, 1))
+		mp1_fw_flags = RREG32_PCIE(MP1_Public |
+					   (smnMP1_FIRMWARE_FLAGS_14_0_0 & 0xffffffff));
+	else
+		mp1_fw_flags = RREG32_PCIE(MP1_Public |
 					   (smnMP1_FIRMWARE_FLAGS & 0xffffffff));
 
 	if ((mp1_fw_flags & MP1_CRU1_MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) >>
@@ -227,16 +244,16 @@ int smu_v14_0_check_fw_version(struct smu_context *smu)
 		adev->pm.fw_version = smu_version;
 
 	switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
-	case IP_VERSION(14, 0, 2):
-		smu->smc_driver_if_version = SMU14_DRIVER_IF_VERSION_SMU_V14_0_2;
-		break;
 	case IP_VERSION(14, 0, 0):
 		smu->smc_driver_if_version = SMU14_DRIVER_IF_VERSION_SMU_V14_0_0;
 		break;
 	case IP_VERSION(14, 0, 1):
 		smu->smc_driver_if_version = SMU14_DRIVER_IF_VERSION_SMU_V14_0_1;
 		break;
-
+	case IP_VERSION(14, 0, 2):
+	case IP_VERSION(14, 0, 3):
+		smu->smc_driver_if_version = SMU14_DRIVER_IF_VERSION_SMU_V14_0_2;
+		break;
 	default:
 		dev_err(adev->dev, "smu unsupported IP version: 0x%x.\n",
 			amdgpu_ip_version(adev, MP1_HWIP, 0));
@@ -738,9 +755,9 @@ int smu_v14_0_gfx_off_control(struct smu_context *smu, bool enable)
 	struct amdgpu_device *adev = smu->adev;
 
 	switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
-	case IP_VERSION(14, 0, 2):
 	case IP_VERSION(14, 0, 0):
 	case IP_VERSION(14, 0, 1):
+	case IP_VERSION(14, 0, 2):
 		if (!(adev->pm.pp_feature & PP_GFXOFF_MASK))
 			return 0;
 		if (enable)
@@ -841,9 +858,16 @@ static int smu_v14_0_set_irq_state(struct amdgpu_device *adev,
 		// TODO
 
 		/* For MP1 SW irqs */
-		val = RREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_CTRL);
-		val = REG_SET_FIELD(val, MP1_SMN_IH_SW_INT_CTRL, INT_MASK, 1);
-		WREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_CTRL, val);
+		if (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(14, 0, 0) ||
+			amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(14, 0, 1)) {
+			val = RREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_CTRL_mp1_14_0_0);
+			val = REG_SET_FIELD(val, MP1_SMN_IH_SW_INT_CTRL, INT_MASK, 1);
+			WREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_CTRL_mp1_14_0_0, val);
+		} else {
+			val = RREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_CTRL);
+			val = REG_SET_FIELD(val, MP1_SMN_IH_SW_INT_CTRL, INT_MASK, 1);
+			WREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_CTRL, val);
+		}
 
 		break;
 	case AMDGPU_IRQ_STATE_ENABLE:
@@ -851,14 +875,26 @@ static int smu_v14_0_set_irq_state(struct amdgpu_device *adev,
 		// TODO
 
 		/* For MP1 SW irqs */
-		val = RREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT);
-		val = REG_SET_FIELD(val, MP1_SMN_IH_SW_INT, ID, 0xFE);
-		val = REG_SET_FIELD(val, MP1_SMN_IH_SW_INT, VALID, 0);
-		WREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT, val);
-
-		val = RREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_CTRL);
-		val = REG_SET_FIELD(val, MP1_SMN_IH_SW_INT_CTRL, INT_MASK, 0);
-		WREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_CTRL, val);
+		if (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(14, 0, 0) ||
+			amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(14, 0, 1)) {
+			val = RREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_mp1_14_0_0);
+			val = REG_SET_FIELD(val, MP1_SMN_IH_SW_INT, ID, 0xFE);
+			val = REG_SET_FIELD(val, MP1_SMN_IH_SW_INT, VALID, 0);
+			WREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_mp1_14_0_0, val);
+
+			val = RREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_CTRL_mp1_14_0_0);
+			val = REG_SET_FIELD(val, MP1_SMN_IH_SW_INT_CTRL, INT_MASK, 0);
+			WREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_CTRL_mp1_14_0_0, val);
+		} else {
+			val = RREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT);
+			val = REG_SET_FIELD(val, MP1_SMN_IH_SW_INT, ID, 0xFE);
+			val = REG_SET_FIELD(val, MP1_SMN_IH_SW_INT, VALID, 0);
+			WREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT, val);
+
+			val = RREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_CTRL);
+			val = REG_SET_FIELD(val, MP1_SMN_IH_SW_INT_CTRL, INT_MASK, 0);
+			WREG32_SOC15(MP1, 0, regMP1_SMN_IH_SW_INT_CTRL, val);
+		}
 
 		break;
 	default:
@@ -868,11 +904,32 @@ static int smu_v14_0_set_irq_state(struct amdgpu_device *adev,
 	return 0;
 }
 
+#define THM_11_0__SRCID__THM_DIG_THERM_L2H		0		/* ASIC_TEMP > CG_THERMAL_INT.DIG_THERM_INTH  */
+#define THM_11_0__SRCID__THM_DIG_THERM_H2L		1		/* ASIC_TEMP < CG_THERMAL_INT.DIG_THERM_INTL  */
+
 static int smu_v14_0_irq_process(struct amdgpu_device *adev,
 				 struct amdgpu_irq_src *source,
 				 struct amdgpu_iv_entry *entry)
 {
-	// TODO
+	struct smu_context *smu = adev->powerplay.pp_handle;
+	uint32_t client_id = entry->client_id;
+	uint32_t src_id = entry->src_id;
+
+	if (client_id == SOC15_IH_CLIENTID_THM) {
+		switch (src_id) {
+		case THM_11_0__SRCID__THM_DIG_THERM_L2H:
+			schedule_delayed_work(&smu->swctf_delayed_work,
+					      msecs_to_jiffies(AMDGPU_SWCTF_EXTRA_DELAY));
+			break;
+		case THM_11_0__SRCID__THM_DIG_THERM_H2L:
+			dev_emerg(adev->dev, "ERROR: GPU under temperature range detected\n");
+			break;
+		default:
+			dev_emerg(adev->dev, "ERROR: GPU under temperature range unknown src id (%d)\n",
+				  src_id);
+			break;
+		}
+	}
 
 	return 0;
 }
@@ -894,7 +951,17 @@ int smu_v14_0_register_irq_handler(struct smu_context *smu)
 	irq_src->num_types = 1;
 	irq_src->funcs = &smu_v14_0_irq_funcs;
 
-	// TODO: THM related
+	ret = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_THM,
+				THM_11_0__SRCID__THM_DIG_THERM_L2H,
+				irq_src);
+	if (ret)
+		return ret;
+
+	ret = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_THM,
+				THM_11_0__SRCID__THM_DIG_THERM_H2L,
+				irq_src);
+	if (ret)
+		return ret;
 
 	ret = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_MP1,
 				SMU_IH_INTERRUPT_ID_TO_DRIVER,
@@ -1590,23 +1657,27 @@ int smu_v14_0_baco_set_armd3_sequence(struct smu_context *smu,
 	return 0;
 }
 
-bool smu_v14_0_baco_is_support(struct smu_context *smu)
+int smu_v14_0_get_bamaco_support(struct smu_context *smu)
 {
 	struct smu_baco_context *smu_baco = &smu->smu_baco;
+	int bamaco_support = 0;
 
 	if (amdgpu_sriov_vf(smu->adev) ||
 	    !smu_baco->platform_support)
-		return false;
+		return 0;
+
+	if (smu_baco->maco_support)
+		bamaco_support |= MACO_SUPPORT;
 
 	/* return true if ASIC is in BACO state already */
 	if (smu_v14_0_baco_get_state(smu) == SMU_BACO_STATE_ENTER)
-		return true;
+		return (bamaco_support |= BACO_SUPPORT);
 
 	if (smu_cmn_feature_is_supported(smu, SMU_FEATURE_BACO_BIT) &&
 	    !smu_cmn_feature_is_enabled(smu, SMU_FEATURE_BACO_BIT))
-		return false;
+		return 0;
 
-	return true;
+	return (bamaco_support |= BACO_SUPPORT);
 }
 
 enum smu_baco_state smu_v14_0_baco_get_state(struct smu_context *smu)
@@ -1629,7 +1700,7 @@ int smu_v14_0_baco_set_state(struct smu_context *smu,
 	if (state == SMU_BACO_STATE_ENTER) {
 		ret = smu_cmn_send_smc_msg_with_param(smu,
 						      SMU_MSG_EnterBaco,
-						      smu_baco->maco_support ?
+						      (adev->pm.rpm_mode == AMDGPU_RUNPM_BAMACO) ?
 						      BACO_SEQ_BAMACO : BACO_SEQ_BACO,
 						      NULL);
 	} else {
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c
index 63399c00cc28..e4419e1561ef 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c
@@ -362,6 +362,12 @@ static int smu_v14_0_0_read_sensor(struct smu_context *smu,
 						       (uint32_t *)data);
 		*size = 4;
 		break;
+	case AMDGPU_PP_SENSOR_VCN_LOAD:
+		ret = smu_v14_0_0_get_smu_metrics_data(smu,
+							METRICS_AVERAGE_VCNACTIVITY,
+							(uint32_t *)data);
+		*size = 4;
+		break;
 	case AMDGPU_PP_SENSOR_GPU_AVG_POWER:
 		ret = smu_v14_0_0_get_smu_metrics_data(smu,
 						       METRICS_AVERAGE_SOCKETPOWER,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
new file mode 100644
index 000000000000..90703f4542ab
--- /dev/null
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
@@ -0,0 +1,1795 @@
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#define SWSMU_CODE_LAYER_L2
+
+#include <linux/firmware.h>
+#include <linux/pci.h>
+#include <linux/i2c.h>
+#include "amdgpu.h"
+#include "amdgpu_smu.h"
+#include "atomfirmware.h"
+#include "amdgpu_atomfirmware.h"
+#include "amdgpu_atombios.h"
+#include "smu_v14_0.h"
+#include "smu14_driver_if_v14_0.h"
+#include "soc15_common.h"
+#include "atom.h"
+#include "smu_v14_0_2_ppt.h"
+#include "smu_v14_0_2_pptable.h"
+#include "smu_v14_0_2_ppsmc.h"
+#include "mp/mp_14_0_2_offset.h"
+#include "mp/mp_14_0_2_sh_mask.h"
+
+#include "smu_cmn.h"
+#include "amdgpu_ras.h"
+
+/*
+ * DO NOT use these for err/warn/info/debug messages.
+ * Use dev_err, dev_warn, dev_info and dev_dbg instead.
+ * They are more MGPU friendly.
+ */
+#undef pr_err
+#undef pr_warn
+#undef pr_info
+#undef pr_debug
+
+#define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c))
+
+#define FEATURE_MASK(feature) (1ULL << feature)
+#define SMC_DPM_FEATURE ( \
+	FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT)     | \
+	FEATURE_MASK(FEATURE_DPM_UCLK_BIT)	 | \
+	FEATURE_MASK(FEATURE_DPM_LINK_BIT)       | \
+	FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT)     | \
+	FEATURE_MASK(FEATURE_DPM_FCLK_BIT))
+
+#define MP0_MP1_DATA_REGION_SIZE_COMBOPPTABLE	0x4000
+
+static struct cmn2asic_msg_mapping smu_v14_0_2_message_map[SMU_MSG_MAX_COUNT] = {
+	MSG_MAP(TestMessage,			PPSMC_MSG_TestMessage,                 1),
+	MSG_MAP(GetSmuVersion,			PPSMC_MSG_GetSmuVersion,               1),
+	MSG_MAP(GetDriverIfVersion,		PPSMC_MSG_GetDriverIfVersion,          1),
+	MSG_MAP(SetAllowedFeaturesMaskLow,	PPSMC_MSG_SetAllowedFeaturesMaskLow,   0),
+	MSG_MAP(SetAllowedFeaturesMaskHigh,	PPSMC_MSG_SetAllowedFeaturesMaskHigh,  0),
+	MSG_MAP(EnableAllSmuFeatures,		PPSMC_MSG_EnableAllSmuFeatures,        0),
+	MSG_MAP(DisableAllSmuFeatures,		PPSMC_MSG_DisableAllSmuFeatures,       0),
+	MSG_MAP(EnableSmuFeaturesLow,		PPSMC_MSG_EnableSmuFeaturesLow,        1),
+	MSG_MAP(EnableSmuFeaturesHigh,		PPSMC_MSG_EnableSmuFeaturesHigh,       1),
+	MSG_MAP(DisableSmuFeaturesLow,		PPSMC_MSG_DisableSmuFeaturesLow,       1),
+	MSG_MAP(DisableSmuFeaturesHigh,		PPSMC_MSG_DisableSmuFeaturesHigh,      1),
+	MSG_MAP(GetEnabledSmuFeaturesLow,       PPSMC_MSG_GetRunningSmuFeaturesLow,    1),
+	MSG_MAP(GetEnabledSmuFeaturesHigh,	PPSMC_MSG_GetRunningSmuFeaturesHigh,   1),
+	MSG_MAP(SetWorkloadMask,		PPSMC_MSG_SetWorkloadMask,             1),
+	MSG_MAP(SetPptLimit,			PPSMC_MSG_SetPptLimit,                 0),
+	MSG_MAP(SetDriverDramAddrHigh,		PPSMC_MSG_SetDriverDramAddrHigh,       1),
+	MSG_MAP(SetDriverDramAddrLow,		PPSMC_MSG_SetDriverDramAddrLow,        1),
+	MSG_MAP(SetToolsDramAddrHigh,		PPSMC_MSG_SetToolsDramAddrHigh,        0),
+	MSG_MAP(SetToolsDramAddrLow,		PPSMC_MSG_SetToolsDramAddrLow,         0),
+	MSG_MAP(TransferTableSmu2Dram,		PPSMC_MSG_TransferTableSmu2Dram,       1),
+	MSG_MAP(TransferTableDram2Smu,		PPSMC_MSG_TransferTableDram2Smu,       0),
+	MSG_MAP(UseDefaultPPTable,		PPSMC_MSG_UseDefaultPPTable,           0),
+	MSG_MAP(RunDcBtc,			PPSMC_MSG_RunDcBtc,                    0),
+	MSG_MAP(EnterBaco,			PPSMC_MSG_EnterBaco,                   0),
+	MSG_MAP(ExitBaco,			PPSMC_MSG_ExitBaco,                    0),
+	MSG_MAP(SetSoftMinByFreq,		PPSMC_MSG_SetSoftMinByFreq,            1),
+	MSG_MAP(SetSoftMaxByFreq,		PPSMC_MSG_SetSoftMaxByFreq,            1),
+	MSG_MAP(SetHardMinByFreq,		PPSMC_MSG_SetHardMinByFreq,            1),
+	MSG_MAP(SetHardMaxByFreq,		PPSMC_MSG_SetHardMaxByFreq,            0),
+	MSG_MAP(GetMinDpmFreq,			PPSMC_MSG_GetMinDpmFreq,               1),
+	MSG_MAP(GetMaxDpmFreq,			PPSMC_MSG_GetMaxDpmFreq,               1),
+	MSG_MAP(GetDpmFreqByIndex,		PPSMC_MSG_GetDpmFreqByIndex,           1),
+	MSG_MAP(PowerUpVcn,			PPSMC_MSG_PowerUpVcn,                  0),
+	MSG_MAP(PowerDownVcn,			PPSMC_MSG_PowerDownVcn,                0),
+	MSG_MAP(PowerUpJpeg,			PPSMC_MSG_PowerUpJpeg,                 0),
+	MSG_MAP(PowerDownJpeg,			PPSMC_MSG_PowerDownJpeg,               0),
+	MSG_MAP(GetDcModeMaxDpmFreq,		PPSMC_MSG_GetDcModeMaxDpmFreq,         1),
+	MSG_MAP(OverridePcieParameters,		PPSMC_MSG_OverridePcieParameters,      0),
+	MSG_MAP(DramLogSetDramAddrHigh,		PPSMC_MSG_DramLogSetDramAddrHigh,      0),
+	MSG_MAP(DramLogSetDramAddrLow,		PPSMC_MSG_DramLogSetDramAddrLow,       0),
+	MSG_MAP(DramLogSetDramSize,		PPSMC_MSG_DramLogSetDramSize,          0),
+	MSG_MAP(AllowGfxOff,			PPSMC_MSG_AllowGfxOff,                 0),
+	MSG_MAP(DisallowGfxOff,			PPSMC_MSG_DisallowGfxOff,              0),
+	MSG_MAP(SetMGpuFanBoostLimitRpm,	PPSMC_MSG_SetMGpuFanBoostLimitRpm,     0),
+	MSG_MAP(GetPptLimit,			PPSMC_MSG_GetPptLimit,                 0),
+	MSG_MAP(NotifyPowerSource,		PPSMC_MSG_NotifyPowerSource,           0),
+	MSG_MAP(Mode1Reset,			PPSMC_MSG_Mode1Reset,                  0),
+	MSG_MAP(PrepareMp1ForUnload,		PPSMC_MSG_PrepareMp1ForUnload,         0),
+	MSG_MAP(DFCstateControl,		PPSMC_MSG_SetExternalClientDfCstateAllow, 0),
+	MSG_MAP(ArmD3,				PPSMC_MSG_ArmD3,                       0),
+	MSG_MAP(SetNumBadMemoryPagesRetired,	PPSMC_MSG_SetNumBadMemoryPagesRetired,   0),
+	MSG_MAP(SetBadMemoryPagesRetiredFlagsPerChannel,
+			    PPSMC_MSG_SetBadMemoryPagesRetiredFlagsPerChannel,   0),
+	MSG_MAP(AllowIHHostInterrupt,		PPSMC_MSG_AllowIHHostInterrupt,       0),
+	MSG_MAP(ReenableAcDcInterrupt,		PPSMC_MSG_ReenableAcDcInterrupt,       0),
+};
+
+static struct cmn2asic_mapping smu_v14_0_2_clk_map[SMU_CLK_COUNT] = {
+	CLK_MAP(GFXCLK,		PPCLK_GFXCLK),
+	CLK_MAP(SCLK,		PPCLK_GFXCLK),
+	CLK_MAP(SOCCLK,		PPCLK_SOCCLK),
+	CLK_MAP(FCLK,		PPCLK_FCLK),
+	CLK_MAP(UCLK,		PPCLK_UCLK),
+	CLK_MAP(MCLK,		PPCLK_UCLK),
+	CLK_MAP(VCLK,		PPCLK_VCLK_0),
+	CLK_MAP(DCLK,		PPCLK_DCLK_0),
+};
+
+static struct cmn2asic_mapping smu_v14_0_2_feature_mask_map[SMU_FEATURE_COUNT] = {
+	FEA_MAP(FW_DATA_READ),
+	FEA_MAP(DPM_GFXCLK),
+	FEA_MAP(DPM_GFX_POWER_OPTIMIZER),
+	FEA_MAP(DPM_UCLK),
+	FEA_MAP(DPM_FCLK),
+	FEA_MAP(DPM_SOCCLK),
+	FEA_MAP(DPM_LINK),
+	FEA_MAP(DPM_DCN),
+	FEA_MAP(VMEMP_SCALING),
+	FEA_MAP(VDDIO_MEM_SCALING),
+	FEA_MAP(DS_GFXCLK),
+	FEA_MAP(DS_SOCCLK),
+	FEA_MAP(DS_FCLK),
+	FEA_MAP(DS_LCLK),
+	FEA_MAP(DS_DCFCLK),
+	FEA_MAP(DS_UCLK),
+	FEA_MAP(GFX_ULV),
+	FEA_MAP(FW_DSTATE),
+	FEA_MAP(GFXOFF),
+	FEA_MAP(BACO),
+	FEA_MAP(MM_DPM),
+	FEA_MAP(SOC_MPCLK_DS),
+	FEA_MAP(BACO_MPCLK_DS),
+	FEA_MAP(THROTTLERS),
+	FEA_MAP(SMARTSHIFT),
+	FEA_MAP(GTHR),
+	FEA_MAP(ACDC),
+	FEA_MAP(VR0HOT),
+	FEA_MAP(FW_CTF),
+	FEA_MAP(FAN_CONTROL),
+	FEA_MAP(GFX_DCS),
+	FEA_MAP(GFX_READ_MARGIN),
+	FEA_MAP(LED_DISPLAY),
+	FEA_MAP(GFXCLK_SPREAD_SPECTRUM),
+	FEA_MAP(OUT_OF_BAND_MONITOR),
+	FEA_MAP(OPTIMIZED_VMIN),
+	FEA_MAP(GFX_IMU),
+	FEA_MAP(BOOT_TIME_CAL),
+	FEA_MAP(GFX_PCC_DFLL),
+	FEA_MAP(SOC_CG),
+	FEA_MAP(DF_CSTATE),
+	FEA_MAP(GFX_EDC),
+	FEA_MAP(BOOT_POWER_OPT),
+	FEA_MAP(CLOCK_POWER_DOWN_BYPASS),
+	FEA_MAP(DS_VCN),
+	FEA_MAP(BACO_CG),
+	FEA_MAP(MEM_TEMP_READ),
+	FEA_MAP(ATHUB_MMHUB_PG),
+	FEA_MAP(SOC_PCC),
+	[SMU_FEATURE_DPM_VCLK_BIT] = {1, FEATURE_MM_DPM_BIT},
+	[SMU_FEATURE_DPM_DCLK_BIT] = {1, FEATURE_MM_DPM_BIT},
+	[SMU_FEATURE_PPT_BIT] = {1, FEATURE_THROTTLERS_BIT},
+};
+
+static struct cmn2asic_mapping smu_v14_0_2_table_map[SMU_TABLE_COUNT] = {
+	TAB_MAP(PPTABLE),
+	TAB_MAP(WATERMARKS),
+	TAB_MAP(AVFS_PSM_DEBUG),
+	TAB_MAP(PMSTATUSLOG),
+	TAB_MAP(SMU_METRICS),
+	TAB_MAP(DRIVER_SMU_CONFIG),
+	TAB_MAP(ACTIVITY_MONITOR_COEFF),
+	[SMU_TABLE_COMBO_PPTABLE] = {1, TABLE_COMBO_PPTABLE},
+	TAB_MAP(I2C_COMMANDS),
+	TAB_MAP(ECCINFO),
+};
+
+static struct cmn2asic_mapping smu_v14_0_2_pwr_src_map[SMU_POWER_SOURCE_COUNT] = {
+	PWR_MAP(AC),
+	PWR_MAP(DC),
+};
+
+static struct cmn2asic_mapping smu_v14_0_2_workload_map[PP_SMC_POWER_PROFILE_COUNT] = {
+	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT,	WORKLOAD_PPLIB_DEFAULT_BIT),
+	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_FULLSCREEN3D,		WORKLOAD_PPLIB_FULL_SCREEN_3D_BIT),
+	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_POWERSAVING,		WORKLOAD_PPLIB_POWER_SAVING_BIT),
+	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VIDEO,		WORKLOAD_PPLIB_VIDEO_BIT),
+	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VR,			WORKLOAD_PPLIB_VR_BIT),
+	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_COMPUTE,		WORKLOAD_PPLIB_COMPUTE_BIT),
+	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_CUSTOM,		WORKLOAD_PPLIB_CUSTOM_BIT),
+	WORKLOAD_MAP(PP_SMC_POWER_PROFILE_WINDOW3D,		WORKLOAD_PPLIB_WINDOW_3D_BIT),
+};
+
+#if 0
+static const uint8_t smu_v14_0_2_throttler_map[] = {
+	[THROTTLER_PPT0_BIT]		= (SMU_THROTTLER_PPT0_BIT),
+	[THROTTLER_PPT1_BIT]		= (SMU_THROTTLER_PPT1_BIT),
+	[THROTTLER_PPT2_BIT]		= (SMU_THROTTLER_PPT2_BIT),
+	[THROTTLER_PPT3_BIT]		= (SMU_THROTTLER_PPT3_BIT),
+	[THROTTLER_TDC_GFX_BIT]		= (SMU_THROTTLER_TDC_GFX_BIT),
+	[THROTTLER_TDC_SOC_BIT]		= (SMU_THROTTLER_TDC_SOC_BIT),
+	[THROTTLER_TEMP_EDGE_BIT]	= (SMU_THROTTLER_TEMP_EDGE_BIT),
+	[THROTTLER_TEMP_HOTSPOT_BIT]	= (SMU_THROTTLER_TEMP_HOTSPOT_BIT),
+	[THROTTLER_TEMP_MEM_BIT]	= (SMU_THROTTLER_TEMP_MEM_BIT),
+	[THROTTLER_TEMP_VR_GFX_BIT]	= (SMU_THROTTLER_TEMP_VR_GFX_BIT),
+	[THROTTLER_TEMP_VR_SOC_BIT]	= (SMU_THROTTLER_TEMP_VR_SOC_BIT),
+	[THROTTLER_TEMP_VR_MEM0_BIT]	= (SMU_THROTTLER_TEMP_VR_MEM0_BIT),
+	[THROTTLER_TEMP_VR_MEM1_BIT]	= (SMU_THROTTLER_TEMP_VR_MEM1_BIT),
+	[THROTTLER_TEMP_LIQUID0_BIT]	= (SMU_THROTTLER_TEMP_LIQUID0_BIT),
+	[THROTTLER_TEMP_LIQUID1_BIT]	= (SMU_THROTTLER_TEMP_LIQUID1_BIT),
+	[THROTTLER_GFX_APCC_PLUS_BIT]	= (SMU_THROTTLER_APCC_BIT),
+	[THROTTLER_FIT_BIT]		= (SMU_THROTTLER_FIT_BIT),
+};
+#endif
+
+static int
+smu_v14_0_2_get_allowed_feature_mask(struct smu_context *smu,
+				  uint32_t *feature_mask, uint32_t num)
+{
+	struct amdgpu_device *adev = smu->adev;
+	/*u32 smu_version;*/
+
+	if (num > 2)
+		return -EINVAL;
+
+	memset(feature_mask, 0xff, sizeof(uint32_t) * num);
+
+	if (adev->pm.pp_feature & PP_SCLK_DPM_MASK) {
+		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT);
+		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_IMU_BIT);
+	}
+#if 0
+	if (!(adev->pg_flags & AMD_PG_SUPPORT_ATHUB) ||
+	    !(adev->pg_flags & AMD_PG_SUPPORT_MMHUB))
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_ATHUB_MMHUB_PG_BIT);
+
+	if (!(adev->pm.pp_feature & PP_SOCCLK_DPM_MASK))
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT);
+
+	/* PMFW 78.58 contains a critical fix for gfxoff feature */
+	smu_cmn_get_smc_version(smu, NULL, &smu_version);
+	if ((smu_version < 0x004e3a00) ||
+	     !(adev->pm.pp_feature & PP_GFXOFF_MASK))
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFXOFF_BIT);
+
+	if (!(adev->pm.pp_feature & PP_MCLK_DPM_MASK)) {
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_UCLK_BIT);
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VMEMP_SCALING_BIT);
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VDDIO_MEM_SCALING_BIT);
+	}
+
+	if (!(adev->pm.pp_feature & PP_SCLK_DEEP_SLEEP_MASK))
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_GFXCLK_BIT);
+
+	if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK)) {
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_LINK_BIT);
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_LCLK_BIT);
+	}
+
+	if (!(adev->pm.pp_feature & PP_ULV_MASK))
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFX_ULV_BIT);
+#endif
+
+	return 0;
+}
+
+static int smu_v14_0_2_check_powerplay_table(struct smu_context *smu)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+	struct smu_14_0_2_powerplay_table *powerplay_table =
+		table_context->power_play_table;
+	struct smu_baco_context *smu_baco = &smu->smu_baco;
+	PPTable_t *pptable = smu->smu_table.driver_pptable;
+	const OverDriveLimits_t * const overdrive_upperlimits =
+				&pptable->SkuTable.OverDriveLimitsBasicMax;
+	const OverDriveLimits_t * const overdrive_lowerlimits =
+				&pptable->SkuTable.OverDriveLimitsBasicMin;
+
+	if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_HARDWAREDC)
+		smu->dc_controlled_by_gpio = true;
+
+	if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_BACO) {
+		smu_baco->platform_support = true;
+
+		if (powerplay_table->platform_caps & SMU_14_0_2_PP_PLATFORM_CAP_MACO)
+			smu_baco->maco_support = true;
+	}
+
+	if (!overdrive_lowerlimits->FeatureCtrlMask ||
+	    !overdrive_upperlimits->FeatureCtrlMask)
+		smu->od_enabled = false;
+
+	table_context->thermal_controller_type =
+		powerplay_table->thermal_controller_type;
+
+	/*
+	 * Instead of having its own buffer space and get overdrive_table copied,
+	 * smu->od_settings just points to the actual overdrive_table
+	 */
+	smu->od_settings = &powerplay_table->overdrive_table;
+
+	smu->adev->pm.no_fan =
+		!(pptable->PFE_Settings.FeaturesToRun[0] & (1 << FEATURE_FAN_CONTROL_BIT));
+
+	return 0;
+}
+
+static int smu_v14_0_2_store_powerplay_table(struct smu_context *smu)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+	struct smu_14_0_2_powerplay_table *powerplay_table =
+		table_context->power_play_table;
+
+	memcpy(table_context->driver_pptable, &powerplay_table->smc_pptable,
+	       sizeof(PPTable_t));
+
+	return 0;
+}
+
+#ifndef atom_smc_dpm_info_table_14_0_0
+struct atom_smc_dpm_info_table_14_0_0 {
+	struct atom_common_table_header table_header;
+	BoardTable_t BoardTable;
+};
+#endif
+
+static int smu_v14_0_2_append_powerplay_table(struct smu_context *smu)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+	PPTable_t *smc_pptable = table_context->driver_pptable;
+	struct atom_smc_dpm_info_table_14_0_0 *smc_dpm_table;
+	BoardTable_t *BoardTable = &smc_pptable->BoardTable;
+	int index, ret;
+
+	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+					    smc_dpm_info);
+
+	ret = amdgpu_atombios_get_data_table(smu->adev, index, NULL, NULL, NULL,
+					     (uint8_t **)&smc_dpm_table);
+	if (ret)
+		return ret;
+
+	memcpy(BoardTable, &smc_dpm_table->BoardTable, sizeof(BoardTable_t));
+
+	return 0;
+}
+
+#if 0
+static int smu_v14_0_2_get_pptable_from_pmfw(struct smu_context *smu,
+					     void **table,
+					     uint32_t *size)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	void *combo_pptable = smu_table->combo_pptable;
+	int ret = 0;
+
+	ret = smu_cmn_get_combo_pptable(smu);
+	if (ret)
+		return ret;
+
+	*table = combo_pptable;
+	*size = sizeof(struct smu_14_0_powerplay_table);
+
+	return 0;
+}
+#endif
+
+static int smu_v14_0_2_get_pptable_from_pmfw(struct smu_context *smu,
+					     void **table,
+					     uint32_t *size)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	void *combo_pptable = smu_table->combo_pptable;
+	int ret = 0;
+
+	ret = smu_cmn_get_combo_pptable(smu);
+	if (ret)
+		return ret;
+
+	*table = combo_pptable;
+	*size = sizeof(struct smu_14_0_2_powerplay_table);
+
+	return 0;
+}
+
+static int smu_v14_0_2_setup_pptable(struct smu_context *smu)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct amdgpu_device *adev = smu->adev;
+	int ret = 0;
+
+	if (amdgpu_sriov_vf(smu->adev))
+		return 0;
+
+	if (!adev->scpm_enabled)
+		ret = smu_v14_0_setup_pptable(smu);
+	else
+		ret = smu_v14_0_2_get_pptable_from_pmfw(smu,
+							&smu_table->power_play_table,
+							&smu_table->power_play_table_size);
+	if (ret)
+		return ret;
+
+	ret = smu_v14_0_2_store_powerplay_table(smu);
+	if (ret)
+		return ret;
+
+	/*
+	 * With SCPM enabled, the operation below will be handled
+	 * by PSP. Driver involvment is unnecessary and useless.
+	 */
+	if (!adev->scpm_enabled) {
+		ret = smu_v14_0_2_append_powerplay_table(smu);
+		if (ret)
+			return ret;
+	}
+
+	ret = smu_v14_0_2_check_powerplay_table(smu);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
+static int smu_v14_0_2_tables_init(struct smu_context *smu)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *tables = smu_table->tables;
+
+	SMU_TABLE_INIT(tables, SMU_TABLE_PPTABLE, sizeof(PPTable_t),
+		       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+	SMU_TABLE_INIT(tables, SMU_TABLE_WATERMARKS, sizeof(Watermarks_t),
+		       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+	SMU_TABLE_INIT(tables, SMU_TABLE_SMU_METRICS, sizeof(SmuMetricsExternal_t),
+		       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+	SMU_TABLE_INIT(tables, SMU_TABLE_I2C_COMMANDS, sizeof(SwI2cRequest_t),
+		       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+	SMU_TABLE_INIT(tables, SMU_TABLE_OVERDRIVE, sizeof(OverDriveTable_t),
+		       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+	SMU_TABLE_INIT(tables, SMU_TABLE_ACTIVITY_MONITOR_COEFF,
+		       sizeof(DpmActivityMonitorCoeffIntExternal_t), PAGE_SIZE,
+		       AMDGPU_GEM_DOMAIN_VRAM);
+	SMU_TABLE_INIT(tables, SMU_TABLE_COMBO_PPTABLE, MP0_MP1_DATA_REGION_SIZE_COMBOPPTABLE,
+			PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+	SMU_TABLE_INIT(tables, SMU_TABLE_ECCINFO, sizeof(EccInfoTable_t),
+			PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+
+	smu_table->metrics_table = kzalloc(sizeof(SmuMetricsExternal_t), GFP_KERNEL);
+	if (!smu_table->metrics_table)
+		goto err0_out;
+	smu_table->metrics_time = 0;
+
+	smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_3);
+	smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL);
+	if (!smu_table->gpu_metrics_table)
+		goto err1_out;
+
+	smu_table->watermarks_table = kzalloc(sizeof(Watermarks_t), GFP_KERNEL);
+	if (!smu_table->watermarks_table)
+		goto err2_out;
+
+	smu_table->ecc_table = kzalloc(tables[SMU_TABLE_ECCINFO].size, GFP_KERNEL);
+	if (!smu_table->ecc_table)
+		goto err3_out;
+
+	return 0;
+
+err3_out:
+	kfree(smu_table->watermarks_table);
+err2_out:
+	kfree(smu_table->gpu_metrics_table);
+err1_out:
+	kfree(smu_table->metrics_table);
+err0_out:
+	return -ENOMEM;
+}
+
+static int smu_v14_0_2_allocate_dpm_context(struct smu_context *smu)
+{
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+
+	smu_dpm->dpm_context = kzalloc(sizeof(struct smu_14_0_dpm_context),
+				       GFP_KERNEL);
+	if (!smu_dpm->dpm_context)
+		return -ENOMEM;
+
+	smu_dpm->dpm_context_size = sizeof(struct smu_14_0_dpm_context);
+
+	return 0;
+}
+
+static int smu_v14_0_2_init_smc_tables(struct smu_context *smu)
+{
+	int ret = 0;
+
+	ret = smu_v14_0_2_tables_init(smu);
+	if (ret)
+		return ret;
+
+	ret = smu_v14_0_2_allocate_dpm_context(smu);
+	if (ret)
+		return ret;
+
+	return smu_v14_0_init_smc_tables(smu);
+}
+
+static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu)
+{
+	struct smu_14_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
+	struct smu_table_context *table_context = &smu->smu_table;
+	PPTable_t *pptable = table_context->driver_pptable;
+	SkuTable_t *skutable = &pptable->SkuTable;
+	struct smu_14_0_dpm_table *dpm_table;
+	struct smu_14_0_pcie_table *pcie_table;
+	uint32_t link_level;
+	int ret = 0;
+
+	/* socclk dpm table setup */
+	dpm_table = &dpm_context->dpm_tables.soc_table;
+	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT)) {
+		ret = smu_v14_0_set_single_dpm_table(smu,
+						     SMU_SOCCLK,
+						     dpm_table);
+		if (ret)
+			return ret;
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100;
+		dpm_table->dpm_levels[0].enabled = true;
+		dpm_table->min = dpm_table->dpm_levels[0].value;
+		dpm_table->max = dpm_table->dpm_levels[0].value;
+	}
+
+	/* gfxclk dpm table setup */
+	dpm_table = &dpm_context->dpm_tables.gfx_table;
+	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) {
+		ret = smu_v14_0_set_single_dpm_table(smu,
+						     SMU_GFXCLK,
+						     dpm_table);
+		if (ret)
+			return ret;
+
+		/*
+		 * Update the reported maximum shader clock to the value
+		 * which can be guarded to be achieved on all cards. This
+		 * is aligned with Window setting. And considering that value
+		 * might be not the peak frequency the card can achieve, it
+		 * is normal some real-time clock frequency can overtake this
+		 * labelled maximum clock frequency(for example in pp_dpm_sclk
+		 * sysfs output).
+		 */
+		if (skutable->DriverReportedClocks.GameClockAc &&
+		    (dpm_table->dpm_levels[dpm_table->count - 1].value >
+		    skutable->DriverReportedClocks.GameClockAc)) {
+			dpm_table->dpm_levels[dpm_table->count - 1].value =
+				skutable->DriverReportedClocks.GameClockAc;
+			dpm_table->max = skutable->DriverReportedClocks.GameClockAc;
+		}
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100;
+		dpm_table->dpm_levels[0].enabled = true;
+		dpm_table->min = dpm_table->dpm_levels[0].value;
+		dpm_table->max = dpm_table->dpm_levels[0].value;
+	}
+
+	/* uclk dpm table setup */
+	dpm_table = &dpm_context->dpm_tables.uclk_table;
+	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT)) {
+		ret = smu_v14_0_set_single_dpm_table(smu,
+						     SMU_UCLK,
+						     dpm_table);
+		if (ret)
+			return ret;
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100;
+		dpm_table->dpm_levels[0].enabled = true;
+		dpm_table->min = dpm_table->dpm_levels[0].value;
+		dpm_table->max = dpm_table->dpm_levels[0].value;
+	}
+
+	/* fclk dpm table setup */
+	dpm_table = &dpm_context->dpm_tables.fclk_table;
+	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_FCLK_BIT)) {
+		ret = smu_v14_0_set_single_dpm_table(smu,
+						     SMU_FCLK,
+						     dpm_table);
+		if (ret)
+			return ret;
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.fclk / 100;
+		dpm_table->dpm_levels[0].enabled = true;
+		dpm_table->min = dpm_table->dpm_levels[0].value;
+		dpm_table->max = dpm_table->dpm_levels[0].value;
+	}
+
+	/* vclk dpm table setup */
+	dpm_table = &dpm_context->dpm_tables.vclk_table;
+	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_VCLK_BIT)) {
+		ret = smu_v14_0_set_single_dpm_table(smu,
+						     SMU_VCLK,
+						     dpm_table);
+		if (ret)
+			return ret;
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.vclk / 100;
+		dpm_table->dpm_levels[0].enabled = true;
+		dpm_table->min = dpm_table->dpm_levels[0].value;
+		dpm_table->max = dpm_table->dpm_levels[0].value;
+	}
+
+	/* dclk dpm table setup */
+	dpm_table = &dpm_context->dpm_tables.dclk_table;
+	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCLK_BIT)) {
+		ret = smu_v14_0_set_single_dpm_table(smu,
+						     SMU_DCLK,
+						     dpm_table);
+		if (ret)
+			return ret;
+	} else {
+		dpm_table->count = 1;
+		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dclk / 100;
+		dpm_table->dpm_levels[0].enabled = true;
+		dpm_table->min = dpm_table->dpm_levels[0].value;
+		dpm_table->max = dpm_table->dpm_levels[0].value;
+	}
+
+	/* lclk dpm table setup */
+	pcie_table = &dpm_context->dpm_tables.pcie_table;
+	pcie_table->num_of_link_levels = 0;
+	for (link_level = 0; link_level < NUM_LINK_LEVELS; link_level++) {
+		if (!skutable->PcieGenSpeed[link_level] &&
+		    !skutable->PcieLaneCount[link_level] &&
+		    !skutable->LclkFreq[link_level])
+			continue;
+
+		pcie_table->pcie_gen[pcie_table->num_of_link_levels] =
+					skutable->PcieGenSpeed[link_level];
+		pcie_table->pcie_lane[pcie_table->num_of_link_levels] =
+					skutable->PcieLaneCount[link_level];
+		pcie_table->clk_freq[pcie_table->num_of_link_levels] =
+					skutable->LclkFreq[link_level];
+		pcie_table->num_of_link_levels++;
+	}
+
+	return 0;
+}
+
+static bool smu_v14_0_2_is_dpm_running(struct smu_context *smu)
+{
+	int ret = 0;
+	uint64_t feature_enabled;
+
+	ret = smu_cmn_get_enabled_mask(smu, &feature_enabled);
+	if (ret)
+		return false;
+
+	return !!(feature_enabled & SMC_DPM_FEATURE);
+}
+
+static void smu_v14_0_2_dump_pptable(struct smu_context *smu)
+{
+       struct smu_table_context *table_context = &smu->smu_table;
+       PPTable_t *pptable = table_context->driver_pptable;
+       PFE_Settings_t *PFEsettings = &pptable->PFE_Settings;
+
+       dev_info(smu->adev->dev, "Dumped PPTable:\n");
+
+       dev_info(smu->adev->dev, "Version = 0x%08x\n", PFEsettings->Version);
+       dev_info(smu->adev->dev, "FeaturesToRun[0] = 0x%08x\n", PFEsettings->FeaturesToRun[0]);
+       dev_info(smu->adev->dev, "FeaturesToRun[1] = 0x%08x\n", PFEsettings->FeaturesToRun[1]);
+}
+
+static uint32_t smu_v14_0_2_get_throttler_status(SmuMetrics_t *metrics)
+{
+	uint32_t throttler_status = 0;
+	int i;
+
+	for (i = 0; i < THROTTLER_COUNT; i++)
+		throttler_status |=
+			(metrics->ThrottlingPercentage[i] ? 1U << i : 0);
+
+	return throttler_status;
+}
+
+#define SMU_14_0_2_BUSY_THRESHOLD	5
+static int smu_v14_0_2_get_smu_metrics_data(struct smu_context *smu,
+					    MetricsMember_t member,
+					    uint32_t *value)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	SmuMetrics_t *metrics =
+		&(((SmuMetricsExternal_t *)(smu_table->metrics_table))->SmuMetrics);
+	int ret = 0;
+
+	ret = smu_cmn_get_metrics_table(smu,
+					NULL,
+					false);
+	if (ret)
+		return ret;
+
+	switch (member) {
+	case METRICS_CURR_GFXCLK:
+		*value = metrics->CurrClock[PPCLK_GFXCLK];
+		break;
+	case METRICS_CURR_SOCCLK:
+		*value = metrics->CurrClock[PPCLK_SOCCLK];
+		break;
+	case METRICS_CURR_UCLK:
+		*value = metrics->CurrClock[PPCLK_UCLK];
+		break;
+	case METRICS_CURR_VCLK:
+		*value = metrics->CurrClock[PPCLK_VCLK_0];
+		break;
+	case METRICS_CURR_DCLK:
+		*value = metrics->CurrClock[PPCLK_DCLK_0];
+		break;
+	case METRICS_CURR_FCLK:
+		*value = metrics->CurrClock[PPCLK_FCLK];
+		break;
+	case METRICS_CURR_DCEFCLK:
+		*value = metrics->CurrClock[PPCLK_DCFCLK];
+		break;
+	case METRICS_AVERAGE_GFXCLK:
+		if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD)
+			*value = metrics->AverageGfxclkFrequencyPostDs;
+		else
+			*value = metrics->AverageGfxclkFrequencyPreDs;
+		break;
+	case METRICS_AVERAGE_FCLK:
+		if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD)
+			*value = metrics->AverageFclkFrequencyPostDs;
+		else
+			*value = metrics->AverageFclkFrequencyPreDs;
+		break;
+	case METRICS_AVERAGE_UCLK:
+		if (metrics->AverageUclkActivity <= SMU_14_0_2_BUSY_THRESHOLD)
+			*value = metrics->AverageMemclkFrequencyPostDs;
+		else
+			*value = metrics->AverageMemclkFrequencyPreDs;
+		break;
+	case METRICS_AVERAGE_VCLK:
+		*value = metrics->AverageVclk0Frequency;
+		break;
+	case METRICS_AVERAGE_DCLK:
+		*value = metrics->AverageDclk0Frequency;
+		break;
+	case METRICS_AVERAGE_VCLK1:
+		*value = metrics->AverageVclk1Frequency;
+		break;
+	case METRICS_AVERAGE_DCLK1:
+		*value = metrics->AverageDclk1Frequency;
+		break;
+	case METRICS_AVERAGE_GFXACTIVITY:
+		*value = metrics->AverageGfxActivity;
+		break;
+	case METRICS_AVERAGE_MEMACTIVITY:
+		*value = metrics->AverageUclkActivity;
+		break;
+	case METRICS_AVERAGE_SOCKETPOWER:
+		*value = metrics->AverageSocketPower << 8;
+		break;
+	case METRICS_TEMPERATURE_EDGE:
+		*value = metrics->AvgTemperature[TEMP_EDGE] *
+			SMU_TEMPERATURE_UNITS_PER_CENTIGRADES;
+		break;
+	case METRICS_TEMPERATURE_HOTSPOT:
+		*value = metrics->AvgTemperature[TEMP_HOTSPOT] *
+			SMU_TEMPERATURE_UNITS_PER_CENTIGRADES;
+		break;
+	case METRICS_TEMPERATURE_MEM:
+		*value = metrics->AvgTemperature[TEMP_MEM] *
+			SMU_TEMPERATURE_UNITS_PER_CENTIGRADES;
+		break;
+	case METRICS_TEMPERATURE_VRGFX:
+		*value = metrics->AvgTemperature[TEMP_VR_GFX] *
+			SMU_TEMPERATURE_UNITS_PER_CENTIGRADES;
+		break;
+	case METRICS_TEMPERATURE_VRSOC:
+		*value = metrics->AvgTemperature[TEMP_VR_SOC] *
+			SMU_TEMPERATURE_UNITS_PER_CENTIGRADES;
+		break;
+	case METRICS_THROTTLER_STATUS:
+		*value = smu_v14_0_2_get_throttler_status(metrics);
+		break;
+	case METRICS_CURR_FANSPEED:
+		*value = metrics->AvgFanRpm;
+		break;
+	case METRICS_CURR_FANPWM:
+		*value = metrics->AvgFanPwm;
+		break;
+	case METRICS_VOLTAGE_VDDGFX:
+		*value = metrics->AvgVoltage[SVI_PLANE_VDD_GFX];
+		break;
+	case METRICS_PCIE_RATE:
+		*value = metrics->PcieRate;
+		break;
+	case METRICS_PCIE_WIDTH:
+		*value = metrics->PcieWidth;
+		break;
+	default:
+		*value = UINT_MAX;
+		break;
+	}
+
+	return ret;
+}
+
+static int smu_v14_0_2_get_dpm_ultimate_freq(struct smu_context *smu,
+					     enum smu_clk_type clk_type,
+					     uint32_t *min,
+					     uint32_t *max)
+{
+	struct smu_14_0_dpm_context *dpm_context =
+		smu->smu_dpm.dpm_context;
+	struct smu_14_0_dpm_table *dpm_table;
+
+	switch (clk_type) {
+	case SMU_MCLK:
+	case SMU_UCLK:
+		/* uclk dpm table */
+		dpm_table = &dpm_context->dpm_tables.uclk_table;
+		break;
+	case SMU_GFXCLK:
+	case SMU_SCLK:
+		/* gfxclk dpm table */
+		dpm_table = &dpm_context->dpm_tables.gfx_table;
+		break;
+	case SMU_SOCCLK:
+		/* socclk dpm table */
+		dpm_table = &dpm_context->dpm_tables.soc_table;
+		break;
+	case SMU_FCLK:
+		/* fclk dpm table */
+		dpm_table = &dpm_context->dpm_tables.fclk_table;
+		break;
+	case SMU_VCLK:
+	case SMU_VCLK1:
+		/* vclk dpm table */
+		dpm_table = &dpm_context->dpm_tables.vclk_table;
+		break;
+	case SMU_DCLK:
+	case SMU_DCLK1:
+		/* dclk dpm table */
+		dpm_table = &dpm_context->dpm_tables.dclk_table;
+		break;
+	default:
+		dev_err(smu->adev->dev, "Unsupported clock type!\n");
+		return -EINVAL;
+	}
+
+	if (min)
+		*min = dpm_table->min;
+	if (max)
+		*max = dpm_table->max;
+
+	return 0;
+}
+
+static int smu_v14_0_2_read_sensor(struct smu_context *smu,
+				   enum amd_pp_sensors sensor,
+				   void *data,
+				   uint32_t *size)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+	PPTable_t *smc_pptable = table_context->driver_pptable;
+	int ret = 0;
+
+	switch (sensor) {
+	case AMDGPU_PP_SENSOR_MAX_FAN_RPM:
+		*(uint16_t *)data = smc_pptable->CustomSkuTable.FanMaximumRpm;
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_MEM_LOAD:
+		ret = smu_v14_0_2_get_smu_metrics_data(smu,
+						       METRICS_AVERAGE_MEMACTIVITY,
+						       (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_GPU_LOAD:
+		ret = smu_v14_0_2_get_smu_metrics_data(smu,
+						       METRICS_AVERAGE_GFXACTIVITY,
+						       (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_GPU_AVG_POWER:
+		ret = smu_v14_0_2_get_smu_metrics_data(smu,
+						       METRICS_AVERAGE_SOCKETPOWER,
+						       (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_HOTSPOT_TEMP:
+		ret = smu_v14_0_2_get_smu_metrics_data(smu,
+						       METRICS_TEMPERATURE_HOTSPOT,
+						       (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_EDGE_TEMP:
+		ret = smu_v14_0_2_get_smu_metrics_data(smu,
+						       METRICS_TEMPERATURE_EDGE,
+						       (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_MEM_TEMP:
+		ret = smu_v14_0_2_get_smu_metrics_data(smu,
+						       METRICS_TEMPERATURE_MEM,
+						       (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_GFX_MCLK:
+		ret = smu_v14_0_2_get_smu_metrics_data(smu,
+						       METRICS_CURR_UCLK,
+						       (uint32_t *)data);
+		*(uint32_t *)data *= 100;
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_GFX_SCLK:
+		ret = smu_v14_0_2_get_smu_metrics_data(smu,
+						       METRICS_AVERAGE_GFXCLK,
+						       (uint32_t *)data);
+		*(uint32_t *)data *= 100;
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_VDDGFX:
+		ret = smu_v14_0_2_get_smu_metrics_data(smu,
+						       METRICS_VOLTAGE_VDDGFX,
+						       (uint32_t *)data);
+		*size = 4;
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+		break;
+	}
+
+	return ret;
+}
+
+static int smu_v14_0_2_get_current_clk_freq_by_table(struct smu_context *smu,
+						     enum smu_clk_type clk_type,
+						     uint32_t *value)
+{
+	MetricsMember_t member_type;
+	int clk_id = 0;
+
+	clk_id = smu_cmn_to_asic_specific_index(smu,
+						CMN2ASIC_MAPPING_CLK,
+						clk_type);
+	if (clk_id < 0)
+		return -EINVAL;
+
+	switch (clk_id) {
+	case PPCLK_GFXCLK:
+		member_type = METRICS_AVERAGE_GFXCLK;
+		break;
+	case PPCLK_UCLK:
+		member_type = METRICS_CURR_UCLK;
+		break;
+	case PPCLK_FCLK:
+		member_type = METRICS_CURR_FCLK;
+		break;
+	case PPCLK_SOCCLK:
+		member_type = METRICS_CURR_SOCCLK;
+		break;
+	case PPCLK_VCLK_0:
+		member_type = METRICS_AVERAGE_VCLK;
+		break;
+	case PPCLK_DCLK_0:
+		member_type = METRICS_AVERAGE_DCLK;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return smu_v14_0_2_get_smu_metrics_data(smu,
+						member_type,
+						value);
+}
+
+static int smu_v14_0_2_print_clk_levels(struct smu_context *smu,
+					enum smu_clk_type clk_type,
+					char *buf)
+{
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+	struct smu_14_0_dpm_context *dpm_context = smu_dpm->dpm_context;
+	struct smu_14_0_dpm_table *single_dpm_table;
+	int i, curr_freq, size = 0;
+	int ret = 0;
+
+	smu_cmn_get_sysfs_buf(&buf, &size);
+
+	if (amdgpu_ras_intr_triggered()) {
+		size += sysfs_emit_at(buf, size, "unavailable\n");
+		return size;
+	}
+
+	switch (clk_type) {
+	case SMU_SCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.gfx_table);
+		break;
+	case SMU_MCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.uclk_table);
+		break;
+	case SMU_SOCCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.soc_table);
+		break;
+	case SMU_FCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.fclk_table);
+		break;
+	case SMU_VCLK:
+	case SMU_VCLK1:
+		single_dpm_table = &(dpm_context->dpm_tables.vclk_table);
+		break;
+	case SMU_DCLK:
+	case SMU_DCLK1:
+		single_dpm_table = &(dpm_context->dpm_tables.dclk_table);
+		break;
+	default:
+		break;
+	}
+
+	switch (clk_type) {
+	case SMU_SCLK:
+	case SMU_MCLK:
+	case SMU_SOCCLK:
+	case SMU_FCLK:
+	case SMU_VCLK:
+	case SMU_VCLK1:
+	case SMU_DCLK:
+	case SMU_DCLK1:
+		ret = smu_v14_0_2_get_current_clk_freq_by_table(smu, clk_type, &curr_freq);
+		if (ret) {
+			dev_err(smu->adev->dev, "Failed to get current clock freq!");
+			return ret;
+		}
+
+		if (single_dpm_table->is_fine_grained) {
+			/*
+			 * For fine grained dpms, there are only two dpm levels:
+			 *   - level 0 -> min clock freq
+			 *   - level 1 -> max clock freq
+			 * And the current clock frequency can be any value between them.
+			 * So, if the current clock frequency is not at level 0 or level 1,
+			 * we will fake it as three dpm levels:
+			 *   - level 0 -> min clock freq
+			 *   - level 1 -> current actual clock freq
+			 *   - level 2 -> max clock freq
+			 */
+			if ((single_dpm_table->dpm_levels[0].value != curr_freq) &&
+			     (single_dpm_table->dpm_levels[1].value != curr_freq)) {
+				size += sysfs_emit_at(buf, size, "0: %uMhz\n",
+						single_dpm_table->dpm_levels[0].value);
+				size += sysfs_emit_at(buf, size, "1: %uMhz *\n",
+						curr_freq);
+				size += sysfs_emit_at(buf, size, "2: %uMhz\n",
+						single_dpm_table->dpm_levels[1].value);
+			} else {
+				size += sysfs_emit_at(buf, size, "0: %uMhz %s\n",
+						single_dpm_table->dpm_levels[0].value,
+						single_dpm_table->dpm_levels[0].value == curr_freq ? "*" : "");
+				size += sysfs_emit_at(buf, size, "1: %uMhz %s\n",
+						single_dpm_table->dpm_levels[1].value,
+						single_dpm_table->dpm_levels[1].value == curr_freq ? "*" : "");
+			}
+		} else {
+			for (i = 0; i < single_dpm_table->count; i++)
+				size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
+						i, single_dpm_table->dpm_levels[i].value,
+						single_dpm_table->dpm_levels[i].value == curr_freq ? "*" : "");
+		}
+		break;
+	case SMU_PCIE:
+		// TODO
+		break;
+
+	default:
+		break;
+	}
+
+	return size;
+}
+
+static int smu_v14_0_2_force_clk_levels(struct smu_context *smu,
+					enum smu_clk_type clk_type,
+					uint32_t mask)
+{
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+	struct smu_14_0_dpm_context *dpm_context = smu_dpm->dpm_context;
+	struct smu_14_0_dpm_table *single_dpm_table;
+	uint32_t soft_min_level, soft_max_level;
+	uint32_t min_freq, max_freq;
+	int ret = 0;
+
+	soft_min_level = mask ? (ffs(mask) - 1) : 0;
+	soft_max_level = mask ? (fls(mask) - 1) : 0;
+
+	switch (clk_type) {
+	case SMU_GFXCLK:
+	case SMU_SCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.gfx_table);
+		break;
+	case SMU_MCLK:
+	case SMU_UCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.uclk_table);
+		break;
+	case SMU_SOCCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.soc_table);
+		break;
+	case SMU_FCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.fclk_table);
+		break;
+	case SMU_VCLK:
+	case SMU_VCLK1:
+		single_dpm_table = &(dpm_context->dpm_tables.vclk_table);
+		break;
+	case SMU_DCLK:
+	case SMU_DCLK1:
+		single_dpm_table = &(dpm_context->dpm_tables.dclk_table);
+		break;
+	default:
+		break;
+	}
+
+	switch (clk_type) {
+	case SMU_GFXCLK:
+	case SMU_SCLK:
+	case SMU_MCLK:
+	case SMU_UCLK:
+	case SMU_SOCCLK:
+	case SMU_FCLK:
+	case SMU_VCLK:
+	case SMU_VCLK1:
+	case SMU_DCLK:
+	case SMU_DCLK1:
+		if (single_dpm_table->is_fine_grained) {
+			/* There is only 2 levels for fine grained DPM */
+			soft_max_level = (soft_max_level >= 1 ? 1 : 0);
+			soft_min_level = (soft_min_level >= 1 ? 1 : 0);
+		} else {
+			if ((soft_max_level >= single_dpm_table->count) ||
+			    (soft_min_level >= single_dpm_table->count))
+				return -EINVAL;
+		}
+
+		min_freq = single_dpm_table->dpm_levels[soft_min_level].value;
+		max_freq = single_dpm_table->dpm_levels[soft_max_level].value;
+
+		ret = smu_v14_0_set_soft_freq_limited_range(smu,
+							    clk_type,
+							    min_freq,
+							    max_freq);
+		break;
+	case SMU_DCEFCLK:
+	case SMU_PCIE:
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static int smu_v14_0_2_update_pcie_parameters(struct smu_context *smu,
+					      uint8_t pcie_gen_cap,
+					      uint8_t pcie_width_cap)
+{
+	struct smu_14_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
+	struct smu_14_0_pcie_table *pcie_table =
+				&dpm_context->dpm_tables.pcie_table;
+	uint32_t smu_pcie_arg;
+	int ret, i;
+
+	for (i = 0; i < pcie_table->num_of_link_levels; i++) {
+		if (pcie_table->pcie_gen[i] > pcie_gen_cap)
+			pcie_table->pcie_gen[i] = pcie_gen_cap;
+		if (pcie_table->pcie_lane[i] > pcie_width_cap)
+			pcie_table->pcie_lane[i] = pcie_width_cap;
+
+		smu_pcie_arg = i << 16;
+		smu_pcie_arg |= pcie_table->pcie_gen[i] << 8;
+		smu_pcie_arg |= pcie_table->pcie_lane[i];
+
+		ret = smu_cmn_send_smc_msg_with_param(smu,
+						      SMU_MSG_OverridePcieParameters,
+						      smu_pcie_arg,
+						      NULL);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int smu_v14_0_2_get_thermal_temperature_range(struct smu_context *smu,
+						     struct smu_temperature_range *range)
+{
+	// TODO
+
+	return 0;
+}
+
+static int smu_v14_0_2_populate_umd_state_clk(struct smu_context *smu)
+{
+	// TODO
+
+	return 0;
+}
+
+static void smu_v14_0_2_get_unique_id(struct smu_context *smu)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	SmuMetrics_t *metrics =
+		&(((SmuMetricsExternal_t *)(smu_table->metrics_table))->SmuMetrics);
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t upper32 = 0, lower32 = 0;
+	int ret;
+
+	ret = smu_cmn_get_metrics_table(smu, NULL, false);
+	if (ret)
+		goto out;
+
+	upper32 = metrics->PublicSerialNumberUpper;
+	lower32 = metrics->PublicSerialNumberLower;
+
+out:
+	adev->unique_id = ((uint64_t)upper32 << 32) | lower32;
+}
+
+static int smu_v14_0_2_get_power_limit(struct smu_context *smu,
+				       uint32_t *current_power_limit,
+				       uint32_t *default_power_limit,
+				       uint32_t *max_power_limit,
+				       uint32_t *min_power_limit)
+{
+	// TODO
+
+	return 0;
+}
+
+static int smu_v14_0_2_get_power_profile_mode(struct smu_context *smu,
+					      char *buf)
+{
+	DpmActivityMonitorCoeffIntExternal_t activity_monitor_external;
+	DpmActivityMonitorCoeffInt_t *activity_monitor =
+		&(activity_monitor_external.DpmActivityMonitorCoeffInt);
+	static const char *title[] = {
+			"PROFILE_INDEX(NAME)",
+			"CLOCK_TYPE(NAME)",
+			"FPS",
+			"MinActiveFreqType",
+			"MinActiveFreq",
+			"BoosterFreqType",
+			"BoosterFreq",
+			"PD_Data_limit_c",
+			"PD_Data_error_coeff",
+			"PD_Data_error_rate_coeff"};
+	int16_t workload_type = 0;
+	uint32_t i, size = 0;
+	int result = 0;
+
+	if (!buf)
+		return -EINVAL;
+
+	size += sysfs_emit_at(buf, size, "%16s %s %s %s %s %s %s %s %s %s\n",
+			title[0], title[1], title[2], title[3], title[4], title[5],
+			title[6], title[7], title[8], title[9]);
+
+	for (i = 0; i < PP_SMC_POWER_PROFILE_COUNT; i++) {
+		/* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */
+		workload_type = smu_cmn_to_asic_specific_index(smu,
+							       CMN2ASIC_MAPPING_WORKLOAD,
+							       i);
+		if (workload_type == -ENOTSUPP)
+			continue;
+		else if (workload_type < 0)
+			return -EINVAL;
+
+		result = smu_cmn_update_table(smu,
+					      SMU_TABLE_ACTIVITY_MONITOR_COEFF,
+					      workload_type,
+					      (void *)(&activity_monitor_external),
+					      false);
+		if (result) {
+			dev_err(smu->adev->dev, "[%s] Failed to get activity monitor!", __func__);
+			return result;
+		}
+
+		size += sysfs_emit_at(buf, size, "%2d %14s%s:\n",
+			i, amdgpu_pp_profile_name[i], (i == smu->power_profile_mode) ? "*" : " ");
+
+		size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d\n",
+			" ",
+			0,
+			"GFXCLK",
+			activity_monitor->Gfx_FPS,
+			activity_monitor->Gfx_MinActiveFreqType,
+			activity_monitor->Gfx_MinActiveFreq,
+			activity_monitor->Gfx_BoosterFreqType,
+			activity_monitor->Gfx_BoosterFreq,
+			activity_monitor->Gfx_PD_Data_limit_c,
+			activity_monitor->Gfx_PD_Data_error_coeff,
+			activity_monitor->Gfx_PD_Data_error_rate_coeff);
+
+		size += sysfs_emit_at(buf, size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d\n",
+			" ",
+			1,
+			"FCLK",
+			activity_monitor->Fclk_FPS,
+			activity_monitor->Fclk_MinActiveFreqType,
+			activity_monitor->Fclk_MinActiveFreq,
+			activity_monitor->Fclk_BoosterFreqType,
+			activity_monitor->Fclk_BoosterFreq,
+			activity_monitor->Fclk_PD_Data_limit_c,
+			activity_monitor->Fclk_PD_Data_error_coeff,
+			activity_monitor->Fclk_PD_Data_error_rate_coeff);
+	}
+
+	return size;
+}
+
+static int smu_v14_0_2_set_power_profile_mode(struct smu_context *smu,
+					      long *input,
+					      uint32_t size)
+{
+	DpmActivityMonitorCoeffIntExternal_t activity_monitor_external;
+	DpmActivityMonitorCoeffInt_t *activity_monitor =
+		&(activity_monitor_external.DpmActivityMonitorCoeffInt);
+	int workload_type, ret = 0;
+
+	smu->power_profile_mode = input[size];
+
+	if (smu->power_profile_mode >= PP_SMC_POWER_PROFILE_COUNT) {
+		dev_err(smu->adev->dev, "Invalid power profile mode %d\n", smu->power_profile_mode);
+		return -EINVAL;
+	}
+
+	if (smu->power_profile_mode == PP_SMC_POWER_PROFILE_CUSTOM) {
+		ret = smu_cmn_update_table(smu,
+					   SMU_TABLE_ACTIVITY_MONITOR_COEFF,
+					   WORKLOAD_PPLIB_CUSTOM_BIT,
+					   (void *)(&activity_monitor_external),
+					   false);
+		if (ret) {
+			dev_err(smu->adev->dev, "[%s] Failed to get activity monitor!", __func__);
+			return ret;
+		}
+
+		switch (input[0]) {
+		case 0: /* Gfxclk */
+			activity_monitor->Gfx_FPS = input[1];
+			activity_monitor->Gfx_MinActiveFreqType = input[2];
+			activity_monitor->Gfx_MinActiveFreq = input[3];
+			activity_monitor->Gfx_BoosterFreqType = input[4];
+			activity_monitor->Gfx_BoosterFreq = input[5];
+			activity_monitor->Gfx_PD_Data_limit_c = input[6];
+			activity_monitor->Gfx_PD_Data_error_coeff = input[7];
+			activity_monitor->Gfx_PD_Data_error_rate_coeff = input[8];
+			break;
+		case 1: /* Fclk */
+			activity_monitor->Fclk_FPS = input[1];
+			activity_monitor->Fclk_MinActiveFreqType = input[2];
+			activity_monitor->Fclk_MinActiveFreq = input[3];
+			activity_monitor->Fclk_BoosterFreqType = input[4];
+			activity_monitor->Fclk_BoosterFreq = input[5];
+			activity_monitor->Fclk_PD_Data_limit_c = input[6];
+			activity_monitor->Fclk_PD_Data_error_coeff = input[7];
+			activity_monitor->Fclk_PD_Data_error_rate_coeff = input[8];
+			break;
+		}
+
+		ret = smu_cmn_update_table(smu,
+					   SMU_TABLE_ACTIVITY_MONITOR_COEFF,
+					   WORKLOAD_PPLIB_CUSTOM_BIT,
+					   (void *)(&activity_monitor_external),
+					   true);
+		if (ret) {
+			dev_err(smu->adev->dev, "[%s] Failed to set activity monitor!", __func__);
+			return ret;
+		}
+	}
+
+	/* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */
+	workload_type = smu_cmn_to_asic_specific_index(smu,
+						       CMN2ASIC_MAPPING_WORKLOAD,
+						       smu->power_profile_mode);
+	if (workload_type < 0)
+		return -EINVAL;
+
+	return smu_cmn_send_smc_msg_with_param(smu,
+					       SMU_MSG_SetWorkloadMask,
+					       1 << workload_type,
+					       NULL);
+}
+
+static int smu_v14_0_2_baco_enter(struct smu_context *smu)
+{
+	struct smu_baco_context *smu_baco = &smu->smu_baco;
+	struct amdgpu_device *adev = smu->adev;
+
+	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev))
+		return smu_v14_0_baco_set_armd3_sequence(smu,
+				smu_baco->maco_support ? BACO_SEQ_BAMACO : BACO_SEQ_BACO);
+	else
+		return smu_v14_0_baco_enter(smu);
+}
+
+static int smu_v14_0_2_baco_exit(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+
+	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) {
+		/* Wait for PMFW handling for the Dstate change */
+		usleep_range(10000, 11000);
+		return smu_v14_0_baco_set_armd3_sequence(smu, BACO_SEQ_ULPS);
+	} else {
+		return smu_v14_0_baco_exit(smu);
+	}
+}
+
+static bool smu_v14_0_2_is_mode1_reset_supported(struct smu_context *smu)
+{
+	// TODO
+
+	return true;
+}
+
+static int smu_v14_0_2_i2c_xfer(struct i2c_adapter *i2c_adap,
+				   struct i2c_msg *msg, int num_msgs)
+{
+	struct amdgpu_smu_i2c_bus *smu_i2c = i2c_get_adapdata(i2c_adap);
+	struct amdgpu_device *adev = smu_i2c->adev;
+	struct smu_context *smu = adev->powerplay.pp_handle;
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *table = &smu_table->driver_table;
+	SwI2cRequest_t *req, *res = (SwI2cRequest_t *)table->cpu_addr;
+	int i, j, r, c;
+	u16 dir;
+
+	if (!adev->pm.dpm_enabled)
+		return -EBUSY;
+
+	req = kzalloc(sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	req->I2CcontrollerPort = smu_i2c->port;
+	req->I2CSpeed = I2C_SPEED_FAST_400K;
+	req->SlaveAddress = msg[0].addr << 1; /* wants an 8-bit address */
+	dir = msg[0].flags & I2C_M_RD;
+
+	for (c = i = 0; i < num_msgs; i++) {
+		for (j = 0; j < msg[i].len; j++, c++) {
+			SwI2cCmd_t *cmd = &req->SwI2cCmds[c];
+
+			if (!(msg[i].flags & I2C_M_RD)) {
+				/* write */
+				cmd->CmdConfig |= CMDCONFIG_READWRITE_MASK;
+				cmd->ReadWriteData = msg[i].buf[j];
+			}
+
+			if ((dir ^ msg[i].flags) & I2C_M_RD) {
+				/* The direction changes.
+				 */
+				dir = msg[i].flags & I2C_M_RD;
+				cmd->CmdConfig |= CMDCONFIG_RESTART_MASK;
+			}
+
+			req->NumCmds++;
+
+			/*
+			 * Insert STOP if we are at the last byte of either last
+			 * message for the transaction or the client explicitly
+			 * requires a STOP at this particular message.
+			 */
+			if ((j == msg[i].len - 1) &&
+			    ((i == num_msgs - 1) || (msg[i].flags & I2C_M_STOP))) {
+				cmd->CmdConfig &= ~CMDCONFIG_RESTART_MASK;
+				cmd->CmdConfig |= CMDCONFIG_STOP_MASK;
+			}
+		}
+	}
+	mutex_lock(&adev->pm.mutex);
+	r = smu_cmn_update_table(smu, SMU_TABLE_I2C_COMMANDS, 0, req, true);
+	mutex_unlock(&adev->pm.mutex);
+	if (r)
+		goto fail;
+
+	for (c = i = 0; i < num_msgs; i++) {
+		if (!(msg[i].flags & I2C_M_RD)) {
+			c += msg[i].len;
+			continue;
+		}
+		for (j = 0; j < msg[i].len; j++, c++) {
+			SwI2cCmd_t *cmd = &res->SwI2cCmds[c];
+
+			msg[i].buf[j] = cmd->ReadWriteData;
+		}
+	}
+	r = num_msgs;
+fail:
+	kfree(req);
+	return r;
+}
+
+static u32 smu_v14_0_2_i2c_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm smu_v14_0_2_i2c_algo = {
+	.master_xfer = smu_v14_0_2_i2c_xfer,
+	.functionality = smu_v14_0_2_i2c_func,
+};
+
+static const struct i2c_adapter_quirks smu_v14_0_2_i2c_control_quirks = {
+	.flags = I2C_AQ_COMB | I2C_AQ_COMB_SAME_ADDR | I2C_AQ_NO_ZERO_LEN,
+	.max_read_len  = MAX_SW_I2C_COMMANDS,
+	.max_write_len = MAX_SW_I2C_COMMANDS,
+	.max_comb_1st_msg_len = 2,
+	.max_comb_2nd_msg_len = MAX_SW_I2C_COMMANDS - 2,
+};
+
+static int smu_v14_0_2_i2c_control_init(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	int res, i;
+
+	for (i = 0; i < MAX_SMU_I2C_BUSES; i++) {
+		struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i];
+		struct i2c_adapter *control = &smu_i2c->adapter;
+
+		smu_i2c->adev = adev;
+		smu_i2c->port = i;
+		mutex_init(&smu_i2c->mutex);
+		control->owner = THIS_MODULE;
+		control->dev.parent = &adev->pdev->dev;
+		control->algo = &smu_v14_0_2_i2c_algo;
+		snprintf(control->name, sizeof(control->name), "AMDGPU SMU %d", i);
+		control->quirks = &smu_v14_0_2_i2c_control_quirks;
+		i2c_set_adapdata(control, smu_i2c);
+
+		res = i2c_add_adapter(control);
+		if (res) {
+			DRM_ERROR("Failed to register hw i2c, err: %d\n", res);
+			goto Out_err;
+		}
+	}
+
+	/* assign the buses used for the FRU EEPROM and RAS EEPROM */
+	/* XXX ideally this would be something in a vbios data table */
+	adev->pm.ras_eeprom_i2c_bus = &adev->pm.smu_i2c[1].adapter;
+	adev->pm.fru_eeprom_i2c_bus = &adev->pm.smu_i2c[0].adapter;
+
+	return 0;
+Out_err:
+	for ( ; i >= 0; i--) {
+		struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i];
+		struct i2c_adapter *control = &smu_i2c->adapter;
+
+		i2c_del_adapter(control);
+	}
+	return res;
+}
+
+static void smu_v14_0_2_i2c_control_fini(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	int i;
+
+	for (i = 0; i < MAX_SMU_I2C_BUSES; i++) {
+		struct amdgpu_smu_i2c_bus *smu_i2c = &adev->pm.smu_i2c[i];
+		struct i2c_adapter *control = &smu_i2c->adapter;
+
+		i2c_del_adapter(control);
+	}
+	adev->pm.ras_eeprom_i2c_bus = NULL;
+	adev->pm.fru_eeprom_i2c_bus = NULL;
+}
+
+static int smu_v14_0_2_set_mp1_state(struct smu_context *smu,
+				     enum pp_mp1_state mp1_state)
+{
+	int ret;
+
+	switch (mp1_state) {
+	case PP_MP1_STATE_UNLOAD:
+		ret = smu_cmn_set_mp1_state(smu, mp1_state);
+		break;
+	default:
+		/* Ignore others */
+		ret = 0;
+	}
+
+	return ret;
+}
+
+static int smu_v14_0_2_set_df_cstate(struct smu_context *smu,
+				     enum pp_df_cstate state)
+{
+	return smu_cmn_send_smc_msg_with_param(smu,
+					       SMU_MSG_DFCstateControl,
+					       state,
+					       NULL);
+}
+
+static int smu_v14_0_2_mode1_reset(struct smu_context *smu)
+{
+	int ret = 0;
+
+	// TODO
+
+	return ret;
+}
+
+static int smu_v14_0_2_mode2_reset(struct smu_context *smu)
+{
+	int ret = 0;
+
+	// TODO
+
+	return ret;
+}
+
+static int smu_v14_0_2_enable_gfx_features(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+
+	if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(14, 0, 2))
+		return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_EnableAllSmuFeatures,
+										   FEATURE_PWR_GFX, NULL);
+	else
+		return -EOPNOTSUPP;
+}
+
+static void smu_v14_0_2_set_smu_mailbox_registers(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+
+	smu->param_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_82);
+	smu->msg_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_66);
+	smu->resp_reg = SOC15_REG_OFFSET(MP1, 0, regMP1_SMN_C2PMSG_90);
+}
+
+static int smu_v14_0_2_smu_send_bad_mem_page_num(struct smu_context *smu,
+		uint32_t size)
+{
+	int ret = 0;
+
+	/* message SMU to update the bad page number on SMUBUS */
+	ret = smu_cmn_send_smc_msg_with_param(smu,
+					  SMU_MSG_SetNumBadMemoryPagesRetired,
+					  size, NULL);
+	if (ret)
+		dev_err(smu->adev->dev,
+			  "[%s] failed to message SMU to update bad memory pages number\n",
+			  __func__);
+
+	return ret;
+}
+
+static int smu_v14_0_2_send_bad_mem_channel_flag(struct smu_context *smu,
+		uint32_t size)
+{
+	int ret = 0;
+
+	/* message SMU to update the bad channel info on SMUBUS */
+	ret = smu_cmn_send_smc_msg_with_param(smu,
+				  SMU_MSG_SetBadMemoryPagesRetiredFlagsPerChannel,
+				  size, NULL);
+	if (ret)
+		dev_err(smu->adev->dev,
+			  "[%s] failed to message SMU to update bad memory pages channel info\n",
+			  __func__);
+
+	return ret;
+}
+
+static ssize_t smu_v14_0_2_get_ecc_info(struct smu_context *smu,
+					void *table)
+{
+	int ret = 0;
+
+	// TODO
+
+	return ret;
+}
+
+static const struct pptable_funcs smu_v14_0_2_ppt_funcs = {
+	.get_allowed_feature_mask = smu_v14_0_2_get_allowed_feature_mask,
+	.set_default_dpm_table = smu_v14_0_2_set_default_dpm_table,
+	.i2c_init = smu_v14_0_2_i2c_control_init,
+	.i2c_fini = smu_v14_0_2_i2c_control_fini,
+	.is_dpm_running = smu_v14_0_2_is_dpm_running,
+	.dump_pptable = smu_v14_0_2_dump_pptable,
+	.init_microcode = smu_v14_0_init_microcode,
+	.load_microcode = smu_v14_0_load_microcode,
+	.fini_microcode = smu_v14_0_fini_microcode,
+	.init_smc_tables = smu_v14_0_2_init_smc_tables,
+	.fini_smc_tables = smu_v14_0_fini_smc_tables,
+	.init_power = smu_v14_0_init_power,
+	.fini_power = smu_v14_0_fini_power,
+	.check_fw_status = smu_v14_0_check_fw_status,
+	.setup_pptable = smu_v14_0_2_setup_pptable,
+	.check_fw_version = smu_v14_0_check_fw_version,
+	.write_pptable = smu_cmn_write_pptable,
+	.set_driver_table_location = smu_v14_0_set_driver_table_location,
+	.system_features_control = smu_v14_0_system_features_control,
+	.set_allowed_mask = smu_v14_0_set_allowed_mask,
+	.get_enabled_mask = smu_cmn_get_enabled_mask,
+	.dpm_set_vcn_enable = smu_v14_0_set_vcn_enable,
+	.dpm_set_jpeg_enable = smu_v14_0_set_jpeg_enable,
+	.get_dpm_ultimate_freq = smu_v14_0_2_get_dpm_ultimate_freq,
+	.get_vbios_bootup_values = smu_v14_0_get_vbios_bootup_values,
+	.read_sensor = smu_v14_0_2_read_sensor,
+	.feature_is_enabled = smu_cmn_feature_is_enabled,
+	.print_clk_levels = smu_v14_0_2_print_clk_levels,
+	.force_clk_levels = smu_v14_0_2_force_clk_levels,
+	.update_pcie_parameters = smu_v14_0_2_update_pcie_parameters,
+	.get_thermal_temperature_range = smu_v14_0_2_get_thermal_temperature_range,
+	.register_irq_handler = smu_v14_0_register_irq_handler,
+	.notify_memory_pool_location = smu_v14_0_notify_memory_pool_location,
+	.set_soft_freq_limited_range = smu_v14_0_set_soft_freq_limited_range,
+	.init_pptable_microcode = smu_v14_0_init_pptable_microcode,
+	.populate_umd_state_clk = smu_v14_0_2_populate_umd_state_clk,
+	.set_performance_level = smu_v14_0_set_performance_level,
+	.gfx_off_control = smu_v14_0_gfx_off_control,
+	.get_unique_id = smu_v14_0_2_get_unique_id,
+	.get_power_limit = smu_v14_0_2_get_power_limit,
+	.set_power_limit = smu_v14_0_set_power_limit,
+	.set_power_source = smu_v14_0_set_power_source,
+	.get_power_profile_mode = smu_v14_0_2_get_power_profile_mode,
+	.set_power_profile_mode = smu_v14_0_2_set_power_profile_mode,
+	.run_btc = smu_v14_0_run_btc,
+	.get_pp_feature_mask = smu_cmn_get_pp_feature_mask,
+	.set_pp_feature_mask = smu_cmn_set_pp_feature_mask,
+	.set_tool_table_location = smu_v14_0_set_tool_table_location,
+	.deep_sleep_control = smu_v14_0_deep_sleep_control,
+	.gfx_ulv_control = smu_v14_0_gfx_ulv_control,
+	.get_bamaco_support = smu_v14_0_get_bamaco_support,
+	.baco_get_state = smu_v14_0_baco_get_state,
+	.baco_set_state = smu_v14_0_baco_set_state,
+	.baco_enter = smu_v14_0_2_baco_enter,
+	.baco_exit = smu_v14_0_2_baco_exit,
+	.mode1_reset_is_support = smu_v14_0_2_is_mode1_reset_supported,
+	.mode1_reset = smu_v14_0_2_mode1_reset,
+	.mode2_reset = smu_v14_0_2_mode2_reset,
+	.enable_gfx_features = smu_v14_0_2_enable_gfx_features,
+	.set_mp1_state = smu_v14_0_2_set_mp1_state,
+	.set_df_cstate = smu_v14_0_2_set_df_cstate,
+	.send_hbm_bad_pages_num = smu_v14_0_2_smu_send_bad_mem_page_num,
+	.send_hbm_bad_channel_flag = smu_v14_0_2_send_bad_mem_channel_flag,
+	.gpo_control = smu_v14_0_gpo_control,
+	.get_ecc_info = smu_v14_0_2_get_ecc_info,
+};
+
+void smu_v14_0_2_set_ppt_funcs(struct smu_context *smu)
+{
+	smu->ppt_funcs = &smu_v14_0_2_ppt_funcs;
+	smu->message_map = smu_v14_0_2_message_map;
+	smu->clock_map = smu_v14_0_2_clk_map;
+	smu->feature_map = smu_v14_0_2_feature_mask_map;
+	smu->table_map = smu_v14_0_2_table_map;
+	smu->pwr_src_map = smu_v14_0_2_pwr_src_map;
+	smu->workload_map = smu_v14_0_2_workload_map;
+	smu_v14_0_2_set_smu_mailbox_registers(smu);
+}
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.h
new file mode 100644
index 000000000000..b83729e5d6f9
--- /dev/null
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SMU_V14_0_2_PPT_H__
+#define __SMU_V14_0_2_PPT_H__
+
+extern void smu_v14_0_2_set_ppt_funcs(struct smu_context *smu);
+
+#endif
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
index b8dbd4e25348..6d1c3af927ca 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
@@ -235,6 +235,50 @@ static void __smu_cmn_send_msg(struct smu_context *smu,
 	WREG32(smu->msg_reg, msg);
 }
 
+static inline uint32_t __smu_cmn_get_msg_flags(struct smu_context *smu,
+					       enum smu_message_type msg)
+{
+	return smu->message_map[msg].flags;
+}
+
+static int __smu_cmn_ras_filter_msg(struct smu_context *smu,
+				    enum smu_message_type msg, bool *poll)
+{
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t flags, resp;
+	bool fed_status;
+
+	flags = __smu_cmn_get_msg_flags(smu, msg);
+	*poll = true;
+
+	/* When there is RAS fatal error, FW won't process non-RAS priority
+	 * messages. Don't allow any messages other than RAS priority messages.
+	 */
+	fed_status = amdgpu_ras_get_fed_status(adev);
+	if (fed_status) {
+		if (!(flags & SMU_MSG_RAS_PRI)) {
+			dev_dbg(adev->dev,
+				"RAS error detected, skip sending %s",
+				smu_get_message_name(smu, msg));
+			return -EACCES;
+		}
+
+		/* FW will ignore non-priority messages when a RAS fatal error
+		 * is detected. Hence it is possible that a previous message
+		 * wouldn't have got response. Allow to continue without polling
+		 * for response status for priority messages.
+		 */
+		resp = RREG32(smu->resp_reg);
+		dev_dbg(adev->dev,
+			"Sending RAS priority message %s response status: %x",
+			smu_get_message_name(smu, msg), resp);
+		if (resp == 0)
+			*poll = false;
+	}
+
+	return 0;
+}
+
 static int __smu_cmn_send_debug_msg(struct smu_context *smu,
 			       u32 msg,
 			       u32 param)
@@ -354,6 +398,7 @@ int smu_cmn_send_smc_msg_with_param(struct smu_context *smu,
 {
 	struct amdgpu_device *adev = smu->adev;
 	int res, index;
+	bool poll = true;
 	u32 reg;
 
 	if (adev->no_hw_access)
@@ -366,12 +411,20 @@ int smu_cmn_send_smc_msg_with_param(struct smu_context *smu,
 		return index == -EACCES ? 0 : index;
 
 	mutex_lock(&smu->message_lock);
-	reg = __smu_cmn_poll_stat(smu);
-	res = __smu_cmn_reg2errno(smu, reg);
-	if (reg == SMU_RESP_NONE ||
-	    res == -EREMOTEIO) {
-		__smu_cmn_reg_print_error(smu, reg, index, param, msg);
-		goto Out;
+
+	if (smu->smc_fw_caps & SMU_FW_CAP_RAS_PRI) {
+		res = __smu_cmn_ras_filter_msg(smu, msg, &poll);
+		if (res)
+			goto Out;
+	}
+
+	if (poll) {
+		reg = __smu_cmn_poll_stat(smu);
+		res = __smu_cmn_reg2errno(smu, reg);
+		if (reg == SMU_RESP_NONE || res == -EREMOTEIO) {
+			__smu_cmn_reg_print_error(smu, reg, index, param, msg);
+			goto Out;
+		}
 	}
 	__smu_cmn_send_msg(smu, (uint16_t) index, param);
 	reg = __smu_cmn_poll_stat(smu);
@@ -437,7 +490,7 @@ int smu_cmn_to_asic_specific_index(struct smu_context *smu,
 			return -EINVAL;
 
 		if (amdgpu_sriov_vf(smu->adev) &&
-		    !msg_mapping.valid_in_vf)
+		    !(msg_mapping.flags & SMU_MSG_VF_FLAG))
 			return -EACCES;
 
 		return msg_mapping.map_to;
diff --git a/drivers/gpu/drm/arm/display/komeda/Makefile b/drivers/gpu/drm/arm/display/komeda/Makefile
index 1931a7fa1a14..cf5287fcbbc2 100644
--- a/drivers/gpu/drm/arm/display/komeda/Makefile
+++ b/drivers/gpu/drm/arm/display/komeda/Makefile
@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 
 ccflags-y := \
-	-I $(srctree)/$(src)/../include \
-	-I $(srctree)/$(src)
+	-I $(src)/../include \
+	-I $(src)
 
 komeda-y := \
 	komeda_drv.o \
diff --git a/drivers/gpu/drm/arm/display/komeda/d71/d71_component.c b/drivers/gpu/drm/arm/display/komeda/d71/d71_component.c
index 42510fdea27e..67e5d3b4190f 100644
--- a/drivers/gpu/drm/arm/display/komeda/d71/d71_component.c
+++ b/drivers/gpu/drm/arm/display/komeda/d71/d71_component.c
@@ -4,6 +4,8 @@
  * Author: James.Qian.Wang <james.qian.wang@arm.com>
  *
  */
+
+#include <linux/seq_file.h>
 #include "d71_dev.h"
 #include "komeda_kms.h"
 #include "malidp_io.h"
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c
index 4b7d94961527..00f5864a0495 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c
@@ -5,6 +5,7 @@
  *
  */
 #include <linux/of.h>
+#include <linux/seq_file.h>
 
 #include <drm/drm_print.h>
 
diff --git a/drivers/gpu/drm/arm/malidp_mw.c b/drivers/gpu/drm/arm/malidp_mw.c
index 626709bec6f5..2577f0cef8fc 100644
--- a/drivers/gpu/drm/arm/malidp_mw.c
+++ b/drivers/gpu/drm/arm/malidp_mw.c
@@ -72,7 +72,10 @@ static void malidp_mw_connector_reset(struct drm_connector *connector)
 		__drm_atomic_helper_connector_destroy_state(connector->state);
 
 	kfree(connector->state);
-	__drm_atomic_helper_connector_reset(connector, &mw_state->base);
+	connector->state = NULL;
+
+	if (mw_state)
+		__drm_atomic_helper_connector_reset(connector, &mw_state->base);
 }
 
 static enum drm_connector_status
diff --git a/drivers/gpu/drm/armada/armada_debugfs.c b/drivers/gpu/drm/armada/armada_debugfs.c
index 29f4b52e3c8d..a763349dd89f 100644
--- a/drivers/gpu/drm/armada/armada_debugfs.c
+++ b/drivers/gpu/drm/armada/armada_debugfs.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/ctype.h>
+#include <linux/debugfs.h>
 #include <linux/module.h>
 #include <linux/seq_file.h>
 #include <linux/uaccess.h>
diff --git a/drivers/gpu/drm/ast/Makefile b/drivers/gpu/drm/ast/Makefile
index 5a53ce51fb24..d794c076bc24 100644
--- a/drivers/gpu/drm/ast/Makefile
+++ b/drivers/gpu/drm/ast/Makefile
@@ -3,6 +3,14 @@
 # Makefile for the drm device driver.  This driver provides support for the
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
-ast-y := ast_drv.o ast_i2c.o ast_main.o ast_mm.o ast_mode.o ast_post.o ast_dp501.o ast_dp.o
+ast-y := \
+	ast_ddc.o \
+	ast_dp501.o \
+	ast_dp.o \
+	ast_drv.o \
+	ast_main.o \
+	ast_mm.o \
+	ast_mode.o \
+	ast_post.o
 
 obj-$(CONFIG_DRM_AST) := ast.o
diff --git a/drivers/gpu/drm/ast/ast_i2c.c b/drivers/gpu/drm/ast/ast_ddc.c
index e5d3f7121de4..29cf5d157f34 100644
--- a/drivers/gpu/drm/ast/ast_i2c.c
+++ b/drivers/gpu/drm/ast/ast_ddc.c
@@ -21,20 +21,31 @@
  * of the Software.
  */
 
+#include <linux/i2c-algo-bit.h>
+#include <linux/i2c.h>
+
 #include <drm/drm_managed.h>
 #include <drm/drm_print.h>
 
+#include "ast_ddc.h"
 #include "ast_drv.h"
 
-static void ast_i2c_setsda(void *i2c_priv, int data)
+struct ast_ddc {
+	struct ast_device *ast;
+
+	struct i2c_algo_bit_data bit;
+	struct i2c_adapter adapter;
+};
+
+static void ast_ddc_algo_bit_data_setsda(void *data, int state)
 {
-	struct ast_i2c_chan *i2c = i2c_priv;
-	struct ast_device *ast = to_ast_device(i2c->dev);
+	struct ast_ddc *ddc = data;
+	struct ast_device *ast = ddc->ast;
 	int i;
 	u8 ujcrb7, jtemp;
 
 	for (i = 0; i < 0x10000; i++) {
-		ujcrb7 = ((data & 0x01) ? 0 : 1) << 2;
+		ujcrb7 = ((state & 0x01) ? 0 : 1) << 2;
 		ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xb7, 0xf1, ujcrb7);
 		jtemp = ast_get_index_reg_mask(ast, AST_IO_VGACRI, 0xb7, 0x04);
 		if (ujcrb7 == jtemp)
@@ -42,15 +53,15 @@ static void ast_i2c_setsda(void *i2c_priv, int data)
 	}
 }
 
-static void ast_i2c_setscl(void *i2c_priv, int clock)
+static void ast_ddc_algo_bit_data_setscl(void *data, int state)
 {
-	struct ast_i2c_chan *i2c = i2c_priv;
-	struct ast_device *ast = to_ast_device(i2c->dev);
+	struct ast_ddc *ddc = data;
+	struct ast_device *ast = ddc->ast;
 	int i;
 	u8 ujcrb7, jtemp;
 
 	for (i = 0; i < 0x10000; i++) {
-		ujcrb7 = ((clock & 0x01) ? 0 : 1);
+		ujcrb7 = ((state & 0x01) ? 0 : 1);
 		ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xb7, 0xf4, ujcrb7);
 		jtemp = ast_get_index_reg_mask(ast, AST_IO_VGACRI, 0xb7, 0x01);
 		if (ujcrb7 == jtemp)
@@ -58,10 +69,32 @@ static void ast_i2c_setscl(void *i2c_priv, int clock)
 	}
 }
 
-static int ast_i2c_getsda(void *i2c_priv)
+static int ast_ddc_algo_bit_data_pre_xfer(struct i2c_adapter *adapter)
+{
+	struct ast_ddc *ddc = i2c_get_adapdata(adapter);
+	struct ast_device *ast = ddc->ast;
+
+	/*
+	 * Protect access to I/O registers from concurrent modesetting
+	 * by acquiring the I/O-register lock.
+	 */
+	mutex_lock(&ast->modeset_lock);
+
+	return 0;
+}
+
+static void ast_ddc_algo_bit_data_post_xfer(struct i2c_adapter *adapter)
+{
+	struct ast_ddc *ddc = i2c_get_adapdata(adapter);
+	struct ast_device *ast = ddc->ast;
+
+	mutex_unlock(&ast->modeset_lock);
+}
+
+static int ast_ddc_algo_bit_data_getsda(void *data)
 {
-	struct ast_i2c_chan *i2c = i2c_priv;
-	struct ast_device *ast = to_ast_device(i2c->dev);
+	struct ast_ddc *ddc = data;
+	struct ast_device *ast = ddc->ast;
 	uint32_t val, val2, count, pass;
 
 	count = 0;
@@ -80,10 +113,10 @@ static int ast_i2c_getsda(void *i2c_priv)
 	return val & 1 ? 1 : 0;
 }
 
-static int ast_i2c_getscl(void *i2c_priv)
+static int ast_ddc_algo_bit_data_getscl(void *data)
 {
-	struct ast_i2c_chan *i2c = i2c_priv;
-	struct ast_device *ast = to_ast_device(i2c->dev);
+	struct ast_ddc *ddc = data;
+	struct ast_device *ast = ddc->ast;
 	uint32_t val, val2, count, pass;
 
 	count = 0;
@@ -102,50 +135,53 @@ static int ast_i2c_getscl(void *i2c_priv)
 	return val & 1 ? 1 : 0;
 }
 
-static void ast_i2c_release(struct drm_device *dev, void *res)
+static void ast_ddc_release(struct drm_device *dev, void *res)
 {
-	struct ast_i2c_chan *i2c = res;
+	struct ast_ddc *ddc = res;
 
-	i2c_del_adapter(&i2c->adapter);
-	kfree(i2c);
+	i2c_del_adapter(&ddc->adapter);
 }
 
-struct ast_i2c_chan *ast_i2c_create(struct drm_device *dev)
+struct i2c_adapter *ast_ddc_create(struct ast_device *ast)
 {
-	struct ast_i2c_chan *i2c;
+	struct drm_device *dev = &ast->base;
+	struct ast_ddc *ddc;
+	struct i2c_adapter *adapter;
+	struct i2c_algo_bit_data *bit;
 	int ret;
 
-	i2c = kzalloc(sizeof(struct ast_i2c_chan), GFP_KERNEL);
-	if (!i2c)
-		return NULL;
-
-	i2c->adapter.owner = THIS_MODULE;
-	i2c->adapter.dev.parent = dev->dev;
-	i2c->dev = dev;
-	i2c_set_adapdata(&i2c->adapter, i2c);
-	snprintf(i2c->adapter.name, sizeof(i2c->adapter.name),
-		 "AST i2c bit bus");
-	i2c->adapter.algo_data = &i2c->bit;
-
-	i2c->bit.udelay = 20;
-	i2c->bit.timeout = 2;
-	i2c->bit.data = i2c;
-	i2c->bit.setsda = ast_i2c_setsda;
-	i2c->bit.setscl = ast_i2c_setscl;
-	i2c->bit.getsda = ast_i2c_getsda;
-	i2c->bit.getscl = ast_i2c_getscl;
-	ret = i2c_bit_add_bus(&i2c->adapter);
+	ddc = drmm_kzalloc(dev, sizeof(*ddc), GFP_KERNEL);
+	if (!ddc)
+		return ERR_PTR(-ENOMEM);
+	ddc->ast = ast;
+
+	bit = &ddc->bit;
+	bit->data = ddc;
+	bit->setsda = ast_ddc_algo_bit_data_setsda;
+	bit->setscl = ast_ddc_algo_bit_data_setscl;
+	bit->getsda = ast_ddc_algo_bit_data_getsda;
+	bit->getscl = ast_ddc_algo_bit_data_getscl;
+	bit->pre_xfer = ast_ddc_algo_bit_data_pre_xfer;
+	bit->post_xfer = ast_ddc_algo_bit_data_post_xfer;
+	bit->udelay = 20;
+	bit->timeout = usecs_to_jiffies(2200);
+
+	adapter = &ddc->adapter;
+	adapter->owner = THIS_MODULE;
+	adapter->algo_data = bit;
+	adapter->dev.parent = dev->dev;
+	snprintf(adapter->name, sizeof(adapter->name), "AST DDC bus");
+	i2c_set_adapdata(adapter, ddc);
+
+	ret = i2c_bit_add_bus(adapter);
 	if (ret) {
 		drm_err(dev, "Failed to register bit i2c\n");
-		goto out_kfree;
+		return ERR_PTR(ret);
 	}
 
-	ret = drmm_add_action_or_reset(dev, ast_i2c_release, i2c);
+	ret = drmm_add_action_or_reset(dev, ast_ddc_release, ddc);
 	if (ret)
-		return NULL;
-	return i2c;
+		return ERR_PTR(ret);
 
-out_kfree:
-	kfree(i2c);
-	return NULL;
+	return &ddc->adapter;
 }
diff --git a/drivers/gpu/drm/ast/ast_ddc.h b/drivers/gpu/drm/ast/ast_ddc.h
new file mode 100644
index 000000000000..85c93edc9ae1
--- /dev/null
+++ b/drivers/gpu/drm/ast/ast_ddc.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: MIT */
+
+#ifndef __AST_DDC_H__
+#define __AST_DDC_H__
+
+struct ast_device;
+struct i2c_adapter;
+
+struct i2c_adapter *ast_ddc_create(struct ast_device *ast);
+
+#endif
diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c
index 90bcb1eb9cd9..f8c49ba68e78 100644
--- a/drivers/gpu/drm/ast/ast_drv.c
+++ b/drivers/gpu/drm/ast/ast_drv.c
@@ -27,6 +27,7 @@
  */
 
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/pci.h>
 
 #include <drm/drm_aperture.h>
diff --git a/drivers/gpu/drm/ast/ast_drv.h b/drivers/gpu/drm/ast/ast_drv.h
index 3be5ccf1f5f4..ba3d86973995 100644
--- a/drivers/gpu/drm/ast/ast_drv.h
+++ b/drivers/gpu/drm/ast/ast_drv.h
@@ -28,8 +28,6 @@
 #ifndef __AST_DRV_H__
 #define __AST_DRV_H__
 
-#include <linux/i2c.h>
-#include <linux/i2c-algo-bit.h>
 #include <linux/io.h>
 #include <linux/types.h>
 
@@ -149,37 +147,9 @@ static inline struct ast_plane *to_ast_plane(struct drm_plane *plane)
 }
 
 /*
- * Connector with i2c channel
+ * BMC
  */
 
-struct ast_i2c_chan {
-	struct i2c_adapter adapter;
-	struct drm_device *dev;
-	struct i2c_algo_bit_data bit;
-};
-
-struct ast_vga_connector {
-	struct drm_connector base;
-	struct ast_i2c_chan *i2c;
-};
-
-static inline struct ast_vga_connector *
-to_ast_vga_connector(struct drm_connector *connector)
-{
-	return container_of(connector, struct ast_vga_connector, base);
-}
-
-struct ast_sil164_connector {
-	struct drm_connector base;
-	struct ast_i2c_chan *i2c;
-};
-
-static inline struct ast_sil164_connector *
-to_ast_sil164_connector(struct drm_connector *connector)
-{
-	return container_of(connector, struct ast_sil164_connector, base);
-}
-
 struct ast_bmc_connector {
 	struct drm_connector base;
 	struct drm_connector *physical_connector;
@@ -222,11 +192,11 @@ struct ast_device {
 	struct {
 		struct {
 			struct drm_encoder encoder;
-			struct ast_vga_connector vga_connector;
+			struct drm_connector connector;
 		} vga;
 		struct {
 			struct drm_encoder encoder;
-			struct ast_sil164_connector sil164_connector;
+			struct drm_connector connector;
 		} sil164;
 		struct {
 			struct drm_encoder encoder;
@@ -498,9 +468,6 @@ bool ast_dp501_read_edid(struct drm_device *dev, u8 *ediddata);
 u8 ast_get_dp501_max_clk(struct drm_device *dev);
 void ast_init_3rdtx(struct drm_device *dev);
 
-/* ast_i2c.c */
-struct ast_i2c_chan *ast_i2c_create(struct drm_device *dev);
-
 /* aspeed DP */
 bool ast_astdp_is_connected(struct ast_device *ast);
 int ast_astdp_read_edid(struct drm_device *dev, u8 *ediddata);
diff --git a/drivers/gpu/drm/ast/ast_main.c b/drivers/gpu/drm/ast/ast_main.c
index 2f3ad5f949fc..0637abb70361 100644
--- a/drivers/gpu/drm/ast/ast_main.c
+++ b/drivers/gpu/drm/ast/ast_main.c
@@ -26,6 +26,7 @@
  * Authors: Dave Airlie <airlied@redhat.com>
  */
 
+#include <linux/of.h>
 #include <linux/pci.h>
 
 #include <drm/drm_atomic_helper.h>
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index a718646a66b8..6695af70768f 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -43,9 +43,11 @@
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_gem_shmem_helper.h>
 #include <drm/drm_managed.h>
+#include <drm/drm_panic.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
 
+#include "ast_ddc.h"
 #include "ast_drv.h"
 #include "ast_tables.h"
 
@@ -700,12 +702,29 @@ static void ast_primary_plane_helper_atomic_disable(struct drm_plane *plane,
 	ast_set_index_reg_mask(ast, AST_IO_VGASRI, 0x1, 0xdf, 0x20);
 }
 
+static int ast_primary_plane_helper_get_scanout_buffer(struct drm_plane *plane,
+						       struct drm_scanout_buffer *sb)
+{
+	struct ast_plane *ast_plane = to_ast_plane(plane);
+
+	if (plane->state && plane->state->fb && ast_plane->vaddr) {
+		sb->format = plane->state->fb->format;
+		sb->width = plane->state->fb->width;
+		sb->height = plane->state->fb->height;
+		sb->pitch[0] = plane->state->fb->pitches[0];
+		iosys_map_set_vaddr_iomem(&sb->map[0], ast_plane->vaddr);
+		return 0;
+	}
+	return -ENODEV;
+}
+
 static const struct drm_plane_helper_funcs ast_primary_plane_helper_funcs = {
 	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
 	.atomic_check = ast_primary_plane_helper_atomic_check,
 	.atomic_update = ast_primary_plane_helper_atomic_update,
 	.atomic_enable = ast_primary_plane_helper_atomic_enable,
 	.atomic_disable = ast_primary_plane_helper_atomic_disable,
+	.get_scanout_buffer = ast_primary_plane_helper_get_scanout_buffer,
 };
 
 static const struct drm_plane_funcs ast_primary_plane_funcs = {
@@ -1343,43 +1362,9 @@ static int ast_crtc_init(struct drm_device *dev)
  * VGA Connector
  */
 
-static int ast_vga_connector_helper_get_modes(struct drm_connector *connector)
-{
-	struct ast_vga_connector *ast_vga_connector = to_ast_vga_connector(connector);
-	struct drm_device *dev = connector->dev;
-	struct ast_device *ast = to_ast_device(dev);
-	struct edid *edid;
-	int count;
-
-	if (!ast_vga_connector->i2c)
-		goto err_drm_connector_update_edid_property;
-
-	/*
-	 * Protect access to I/O registers from concurrent modesetting
-	 * by acquiring the I/O-register lock.
-	 */
-	mutex_lock(&ast->modeset_lock);
-
-	edid = drm_get_edid(connector, &ast_vga_connector->i2c->adapter);
-	if (!edid)
-		goto err_mutex_unlock;
-
-	mutex_unlock(&ast->modeset_lock);
-
-	count = drm_add_edid_modes(connector, edid);
-	kfree(edid);
-
-	return count;
-
-err_mutex_unlock:
-	mutex_unlock(&ast->modeset_lock);
-err_drm_connector_update_edid_property:
-	drm_connector_update_edid_property(connector, NULL);
-	return 0;
-}
-
 static const struct drm_connector_helper_funcs ast_vga_connector_helper_funcs = {
-	.get_modes = ast_vga_connector_helper_get_modes,
+	.get_modes = drm_connector_helper_get_modes,
+	.detect_ctx = drm_connector_helper_detect_from_ddc,
 };
 
 static const struct drm_connector_funcs ast_vga_connector_funcs = {
@@ -1390,23 +1375,21 @@ static const struct drm_connector_funcs ast_vga_connector_funcs = {
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
-static int ast_vga_connector_init(struct drm_device *dev,
-				  struct ast_vga_connector *ast_vga_connector)
+static int ast_vga_connector_init(struct drm_device *dev, struct drm_connector *connector)
 {
-	struct drm_connector *connector = &ast_vga_connector->base;
+	struct ast_device *ast = to_ast_device(dev);
+	struct i2c_adapter *ddc;
 	int ret;
 
-	ast_vga_connector->i2c = ast_i2c_create(dev);
-	if (!ast_vga_connector->i2c)
-		drm_err(dev, "failed to add ddc bus for connector\n");
+	ddc = ast_ddc_create(ast);
+	if (IS_ERR(ddc)) {
+		ret = PTR_ERR(ddc);
+		drm_err(dev, "failed to add DDC bus for connector; ret=%d\n", ret);
+		return ret;
+	}
 
-	if (ast_vga_connector->i2c)
-		ret = drm_connector_init_with_ddc(dev, connector, &ast_vga_connector_funcs,
-						  DRM_MODE_CONNECTOR_VGA,
-						  &ast_vga_connector->i2c->adapter);
-	else
-		ret = drm_connector_init(dev, connector, &ast_vga_connector_funcs,
-					 DRM_MODE_CONNECTOR_VGA);
+	ret = drm_connector_init_with_ddc(dev, connector, &ast_vga_connector_funcs,
+					  DRM_MODE_CONNECTOR_VGA, ddc);
 	if (ret)
 		return ret;
 
@@ -1415,7 +1398,7 @@ static int ast_vga_connector_init(struct drm_device *dev,
 	connector->interlace_allowed = 0;
 	connector->doublescan_allowed = 0;
 
-	connector->polled = DRM_CONNECTOR_POLL_CONNECT;
+	connector->polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
 
 	return 0;
 }
@@ -1425,8 +1408,7 @@ static int ast_vga_output_init(struct ast_device *ast)
 	struct drm_device *dev = &ast->base;
 	struct drm_crtc *crtc = &ast->crtc;
 	struct drm_encoder *encoder = &ast->output.vga.encoder;
-	struct ast_vga_connector *ast_vga_connector = &ast->output.vga.vga_connector;
-	struct drm_connector *connector = &ast_vga_connector->base;
+	struct drm_connector *connector = &ast->output.vga.connector;
 	int ret;
 
 	ret = drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_DAC);
@@ -1434,7 +1416,7 @@ static int ast_vga_output_init(struct ast_device *ast)
 		return ret;
 	encoder->possible_crtcs = drm_crtc_mask(crtc);
 
-	ret = ast_vga_connector_init(dev, ast_vga_connector);
+	ret = ast_vga_connector_init(dev, connector);
 	if (ret)
 		return ret;
 
@@ -1449,43 +1431,9 @@ static int ast_vga_output_init(struct ast_device *ast)
  * SIL164 Connector
  */
 
-static int ast_sil164_connector_helper_get_modes(struct drm_connector *connector)
-{
-	struct ast_sil164_connector *ast_sil164_connector = to_ast_sil164_connector(connector);
-	struct drm_device *dev = connector->dev;
-	struct ast_device *ast = to_ast_device(dev);
-	struct edid *edid;
-	int count;
-
-	if (!ast_sil164_connector->i2c)
-		goto err_drm_connector_update_edid_property;
-
-	/*
-	 * Protect access to I/O registers from concurrent modesetting
-	 * by acquiring the I/O-register lock.
-	 */
-	mutex_lock(&ast->modeset_lock);
-
-	edid = drm_get_edid(connector, &ast_sil164_connector->i2c->adapter);
-	if (!edid)
-		goto err_mutex_unlock;
-
-	mutex_unlock(&ast->modeset_lock);
-
-	count = drm_add_edid_modes(connector, edid);
-	kfree(edid);
-
-	return count;
-
-err_mutex_unlock:
-	mutex_unlock(&ast->modeset_lock);
-err_drm_connector_update_edid_property:
-	drm_connector_update_edid_property(connector, NULL);
-	return 0;
-}
-
 static const struct drm_connector_helper_funcs ast_sil164_connector_helper_funcs = {
-	.get_modes = ast_sil164_connector_helper_get_modes,
+	.get_modes = drm_connector_helper_get_modes,
+	.detect_ctx = drm_connector_helper_detect_from_ddc,
 };
 
 static const struct drm_connector_funcs ast_sil164_connector_funcs = {
@@ -1496,23 +1444,21 @@ static const struct drm_connector_funcs ast_sil164_connector_funcs = {
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
-static int ast_sil164_connector_init(struct drm_device *dev,
-				     struct ast_sil164_connector *ast_sil164_connector)
+static int ast_sil164_connector_init(struct drm_device *dev, struct drm_connector *connector)
 {
-	struct drm_connector *connector = &ast_sil164_connector->base;
+	struct ast_device *ast = to_ast_device(dev);
+	struct i2c_adapter *ddc;
 	int ret;
 
-	ast_sil164_connector->i2c = ast_i2c_create(dev);
-	if (!ast_sil164_connector->i2c)
-		drm_err(dev, "failed to add ddc bus for connector\n");
+	ddc = ast_ddc_create(ast);
+	if (IS_ERR(ddc)) {
+		ret = PTR_ERR(ddc);
+		drm_err(dev, "failed to add DDC bus for connector; ret=%d\n", ret);
+		return ret;
+	}
 
-	if (ast_sil164_connector->i2c)
-		ret = drm_connector_init_with_ddc(dev, connector, &ast_sil164_connector_funcs,
-						  DRM_MODE_CONNECTOR_DVII,
-						  &ast_sil164_connector->i2c->adapter);
-	else
-		ret = drm_connector_init(dev, connector, &ast_sil164_connector_funcs,
-					 DRM_MODE_CONNECTOR_DVII);
+	ret = drm_connector_init_with_ddc(dev, connector, &ast_sil164_connector_funcs,
+					  DRM_MODE_CONNECTOR_DVII, ddc);
 	if (ret)
 		return ret;
 
@@ -1521,7 +1467,7 @@ static int ast_sil164_connector_init(struct drm_device *dev,
 	connector->interlace_allowed = 0;
 	connector->doublescan_allowed = 0;
 
-	connector->polled = DRM_CONNECTOR_POLL_CONNECT;
+	connector->polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
 
 	return 0;
 }
@@ -1531,8 +1477,7 @@ static int ast_sil164_output_init(struct ast_device *ast)
 	struct drm_device *dev = &ast->base;
 	struct drm_crtc *crtc = &ast->crtc;
 	struct drm_encoder *encoder = &ast->output.sil164.encoder;
-	struct ast_sil164_connector *ast_sil164_connector = &ast->output.sil164.sil164_connector;
-	struct drm_connector *connector = &ast_sil164_connector->base;
+	struct drm_connector *connector = &ast->output.sil164.connector;
 	int ret;
 
 	ret = drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_TMDS);
@@ -1540,7 +1485,7 @@ static int ast_sil164_output_init(struct ast_device *ast)
 		return ret;
 	encoder->possible_crtcs = drm_crtc_mask(crtc);
 
-	ret = ast_sil164_connector_init(dev, ast_sil164_connector);
+	ret = ast_sil164_connector_init(dev, connector);
 	if (ret)
 		return ret;
 
@@ -1952,13 +1897,13 @@ int ast_mode_config_init(struct ast_device *ast)
 		ret = ast_vga_output_init(ast);
 		if (ret)
 			return ret;
-		physical_connector = &ast->output.vga.vga_connector.base;
+		physical_connector = &ast->output.vga.connector;
 	}
 	if (ast->tx_chip_types & AST_TX_SIL164_BIT) {
 		ret = ast_sil164_output_init(ast);
 		if (ret)
 			return ret;
-		physical_connector = &ast->output.sil164.sil164_connector.base;
+		physical_connector = &ast->output.sil164.connector;
 	}
 	if (ast->tx_chip_types & AST_TX_DP501_BIT) {
 		ret = ast_dp501_output_init(ast);
@@ -1978,7 +1923,9 @@ int ast_mode_config_init(struct ast_device *ast)
 
 	drm_mode_config_reset(dev);
 
-	drm_kms_helper_poll_init(dev);
+	ret = drmm_kms_helper_poll_init(dev);
+	if (ret)
+		return ret;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
index efd996f6c138..c621be1a99a8 100644
--- a/drivers/gpu/drm/bridge/Kconfig
+++ b/drivers/gpu/drm/bridge/Kconfig
@@ -96,9 +96,8 @@ config DRM_ITE_IT6505
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HDCP_HELPER
 	select DRM_DISPLAY_HELPER
-	select DRM_DP_AUX_BUS
+	select DRM_DISPLAY_DP_AUX_BUS
 	select DRM_KMS_HELPER
-	select DRM_DP_HELPER
 	select EXTCON
 	select CRYPTO
 	select CRYPTO_HASH
@@ -190,6 +189,13 @@ config DRM_MEGACHIPS_STDPXXXX_GE_B850V3_FW
 	  to DP++. This is used with the i.MX6 imx-ldb
 	  driver. You are likely to say N here.
 
+config DRM_MICROCHIP_LVDS_SERIALIZER
+	tristate "Microchip LVDS serializer support"
+	depends on OF
+	depends on DRM_ATMEL_HLCDC
+	help
+	  Support for Microchip's LVDS serializer.
+
 config DRM_NWL_MIPI_DSI
 	tristate "Northwest Logic MIPI DSI Host controller"
 	depends on DRM
@@ -229,7 +235,7 @@ config DRM_PARADE_PS8640
 	depends on OF
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HELPER
-	select DRM_DP_AUX_BUS
+	select DRM_DISPLAY_DP_AUX_BUS
 	select DRM_KMS_HELPER
 	select DRM_MIPI_DSI
 	select DRM_PANEL
@@ -389,7 +395,7 @@ config DRM_TI_SN65DSI86
 	select DRM_PANEL
 	select DRM_MIPI_DSI
 	select AUXILIARY_BUS
-	select DRM_DP_AUX_BUS
+	select DRM_DISPLAY_DP_AUX_BUS
 	help
 	  Texas Instruments SN65DSI86 DSI to eDP Bridge driver
 
diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile
index 017b5832733b..7df87b582dca 100644
--- a/drivers/gpu/drm/bridge/Makefile
+++ b/drivers/gpu/drm/bridge/Makefile
@@ -13,6 +13,7 @@ obj-$(CONFIG_DRM_LONTIUM_LT9611) += lontium-lt9611.o
 obj-$(CONFIG_DRM_LONTIUM_LT9611UXC) += lontium-lt9611uxc.o
 obj-$(CONFIG_DRM_LVDS_CODEC) += lvds-codec.o
 obj-$(CONFIG_DRM_MEGACHIPS_STDPXXXX_GE_B850V3_FW) += megachips-stdpxxxx-ge-b850v3-fw.o
+obj-$(CONFIG_DRM_MICROCHIP_LVDS_SERIALIZER) += microchip-lvds.o
 obj-$(CONFIG_DRM_NXP_PTN3460) += nxp-ptn3460.o
 obj-$(CONFIG_DRM_PARADE_PS8622) += parade-ps8622.o
 obj-$(CONFIG_DRM_PARADE_PS8640) += parade-ps8640.o
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511.h b/drivers/gpu/drm/bridge/adv7511/adv7511.h
index 39c9ece373b0..ea271f62b214 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511.h
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511.h
@@ -356,6 +356,7 @@ struct adv7511 {
 	enum drm_connector_status status;
 	bool powered;
 
+	struct drm_bridge *next_bridge;
 	struct drm_display_mode curr_mode;
 
 	unsigned int f_tmds;
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
index b5518ff97165..66ccb61e2a66 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
@@ -17,6 +17,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_of.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
@@ -477,6 +478,11 @@ static int adv7511_irq_process(struct adv7511 *adv7511, bool process_hpd)
 	if (ret < 0)
 		return ret;
 
+	/* If there is no IRQ to handle, exit indicating no IRQ data */
+	if (!(irq0 & (ADV7511_INT0_HPD | ADV7511_INT0_EDID_READY)) &&
+	    !(irq1 & ADV7511_INT1_DDC_ERROR))
+		return -ENODATA;
+
 	regmap_write(adv7511->regmap, ADV7511_REG_INT(0), irq0);
 	regmap_write(adv7511->regmap, ADV7511_REG_INT(1), irq1);
 
@@ -946,6 +952,13 @@ static int adv7511_bridge_attach(struct drm_bridge *bridge,
 	struct adv7511 *adv = bridge_to_adv7511(bridge);
 	int ret = 0;
 
+	if (adv->next_bridge) {
+		ret = drm_bridge_attach(bridge->encoder, adv->next_bridge, bridge,
+					flags | DRM_BRIDGE_ATTACH_NO_CONNECTOR);
+		if (ret)
+			return ret;
+	}
+
 	if (!(flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)) {
 		ret = adv7511_connector_init(adv);
 		if (ret < 0)
@@ -1216,6 +1229,11 @@ static int adv7511_probe(struct i2c_client *i2c)
 
 	memset(&link_config, 0, sizeof(link_config));
 
+	ret = drm_of_find_panel_or_bridge(dev->of_node, 1, -1, NULL,
+					  &adv7511->next_bridge);
+	if (ret && ret != -ENODEV)
+		return ret;
+
 	if (adv7511->info->link_config)
 		ret = adv7511_parse_dt(dev->of_node, &link_config);
 	else
@@ -1318,7 +1336,8 @@ static int adv7511_probe(struct i2c_client *i2c)
 
 		ret = devm_request_threaded_irq(dev, i2c->irq, NULL,
 						adv7511_irq_handler,
-						IRQF_ONESHOT, dev_name(dev),
+						IRQF_ONESHOT | IRQF_SHARED,
+						dev_name(dev),
 						adv7511);
 		if (ret)
 			goto err_unregister_audio;
diff --git a/drivers/gpu/drm/bridge/analogix/Kconfig b/drivers/gpu/drm/bridge/analogix/Kconfig
index 173dada218ec..4846b2e9be7c 100644
--- a/drivers/gpu/drm/bridge/analogix/Kconfig
+++ b/drivers/gpu/drm/bridge/analogix/Kconfig
@@ -37,7 +37,7 @@ config DRM_ANALOGIX_ANX7625
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HDCP_HELPER
 	select DRM_DISPLAY_HELPER
-	select DRM_DP_AUX_BUS
+	select DRM_DISPLAY_DP_AUX_BUS
 	select DRM_MIPI_DSI
 	help
 	  ANX7625 is an ultra-low power 4K mobile HD transmitter
diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.c b/drivers/gpu/drm/bridge/analogix/anx7625.c
index 9d96d28d6fe8..59e9ad349969 100644
--- a/drivers/gpu/drm/bridge/analogix/anx7625.c
+++ b/drivers/gpu/drm/bridge/analogix/anx7625.c
@@ -2066,10 +2066,8 @@ static int anx7625_setup_dsi_device(struct anx7625_data *ctx)
 	};
 
 	host = of_find_mipi_dsi_host_by_node(ctx->pdata.mipi_host_node);
-	if (!host) {
-		DRM_DEV_ERROR(dev, "fail to find dsi host.\n");
-		return -EPROBE_DEFER;
-	}
+	if (!host)
+		return dev_err_probe(dev, -EPROBE_DEFER, "fail to find dsi host.\n");
 
 	dsi = devm_mipi_dsi_device_register_full(dev, host, &info);
 	if (IS_ERR(dsi)) {
@@ -2471,15 +2469,22 @@ static void anx7625_bridge_atomic_disable(struct drm_bridge *bridge,
 	mutex_unlock(&ctx->aux_lock);
 }
 
+static void
+anx7625_audio_update_connector_status(struct anx7625_data *ctx,
+				      enum drm_connector_status status);
+
 static enum drm_connector_status
 anx7625_bridge_detect(struct drm_bridge *bridge)
 {
 	struct anx7625_data *ctx = bridge_to_anx7625(bridge);
 	struct device *dev = ctx->dev;
+	enum drm_connector_status status;
 
 	DRM_DEV_DEBUG_DRIVER(dev, "drm bridge detect\n");
 
-	return anx7625_sink_detect(ctx);
+	status = anx7625_sink_detect(ctx);
+	anx7625_audio_update_connector_status(ctx, status);
+	return status;
 }
 
 static const struct drm_edid *anx7625_bridge_edid_read(struct drm_bridge *bridge,
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
index e226acc5c15e..8a91ef0ae065 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
+++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
@@ -2059,6 +2059,9 @@ static void cdns_mhdp_atomic_enable(struct drm_bridge *bridge,
 	mhdp_state = to_cdns_mhdp_bridge_state(new_state);
 
 	mhdp_state->current_mode = drm_mode_duplicate(bridge->dev, mode);
+	if (!mhdp_state->current_mode)
+		return;
+
 	drm_mode_set_name(mhdp_state->current_mode);
 
 	dev_dbg(mhdp->dev, "%s: Enabling mode %s\n", __func__, mode->name);
diff --git a/drivers/gpu/drm/bridge/chipone-icn6211.c b/drivers/gpu/drm/bridge/chipone-icn6211.c
index 82d23e4df09e..9eecac457dcf 100644
--- a/drivers/gpu/drm/bridge/chipone-icn6211.c
+++ b/drivers/gpu/drm/bridge/chipone-icn6211.c
@@ -563,10 +563,8 @@ static int chipone_dsi_host_attach(struct chipone *icn)
 
 	host = of_find_mipi_dsi_host_by_node(host_node);
 	of_node_put(host_node);
-	if (!host) {
-		dev_err(dev, "failed to find dsi host\n");
-		return -EPROBE_DEFER;
-	}
+	if (!host)
+		return dev_err_probe(dev, -EPROBE_DEFER, "failed to find dsi host\n");
 
 	dsi = mipi_dsi_device_register_full(host, &info);
 	if (IS_ERR(dsi)) {
@@ -783,7 +781,6 @@ static struct mipi_dsi_driver chipone_dsi_driver = {
 	.remove = chipone_dsi_remove,
 	.driver = {
 		.name = "chipone-icn6211",
-		.owner = THIS_MODULE,
 		.of_match_table = chipone_of_match,
 	},
 };
diff --git a/drivers/gpu/drm/bridge/imx/Kconfig b/drivers/gpu/drm/bridge/imx/Kconfig
index 5965e8027529..8dd89efa8ea7 100644
--- a/drivers/gpu/drm/bridge/imx/Kconfig
+++ b/drivers/gpu/drm/bridge/imx/Kconfig
@@ -8,8 +8,8 @@ config DRM_IMX8MP_DW_HDMI_BRIDGE
 	depends on OF
 	depends on COMMON_CLK
 	select DRM_DW_HDMI
-	select DRM_IMX8MP_HDMI_PVI
-	select PHY_FSL_SAMSUNG_HDMI_PHY
+	imply DRM_IMX8MP_HDMI_PVI
+	imply PHY_FSL_SAMSUNG_HDMI_PHY
 	help
 	  Choose this to enable support for the internal HDMI encoder found
 	  on the i.MX8MP SoC.
diff --git a/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c
index f2a09c879e3d..073e64dc200c 100644
--- a/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c
+++ b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c
@@ -173,15 +173,13 @@ static int imx8mp_hdmi_pvi_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int imx8mp_hdmi_pvi_remove(struct platform_device *pdev)
+static void imx8mp_hdmi_pvi_remove(struct platform_device *pdev)
 {
 	struct imx8mp_hdmi_pvi *pvi = platform_get_drvdata(pdev);
 
 	drm_bridge_remove(&pvi->bridge);
 
 	pm_runtime_disable(&pdev->dev);
-
-	return 0;
 }
 
 static const struct of_device_id imx8mp_hdmi_pvi_match[] = {
@@ -195,7 +193,7 @@ MODULE_DEVICE_TABLE(of, imx8mp_hdmi_pvi_match);
 
 static struct platform_driver imx8mp_hdmi_pvi_driver = {
 	.probe	= imx8mp_hdmi_pvi_probe,
-	.remove	= imx8mp_hdmi_pvi_remove,
+	.remove_new = imx8mp_hdmi_pvi_remove,
 	.driver		= {
 		.name = "imx-hdmi-pvi",
 		.of_match_table	= imx8mp_hdmi_pvi_match,
diff --git a/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx.c b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx.c
index 89fc432ac611..13bc570c5473 100644
--- a/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx.c
+++ b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx.c
@@ -104,13 +104,11 @@ static int imx8mp_dw_hdmi_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int imx8mp_dw_hdmi_remove(struct platform_device *pdev)
+static void imx8mp_dw_hdmi_remove(struct platform_device *pdev)
 {
 	struct imx8mp_hdmi *hdmi = platform_get_drvdata(pdev);
 
 	dw_hdmi_remove(hdmi->dw_hdmi);
-
-	return 0;
 }
 
 static int __maybe_unused imx8mp_dw_hdmi_pm_suspend(struct device *dev)
@@ -140,7 +138,7 @@ MODULE_DEVICE_TABLE(of, imx8mp_dw_hdmi_of_table);
 
 static struct platform_driver imx8mp_dw_hdmi_platform_driver = {
 	.probe		= imx8mp_dw_hdmi_probe,
-	.remove		= imx8mp_dw_hdmi_remove,
+	.remove_new	= imx8mp_dw_hdmi_remove,
 	.driver		= {
 		.name	= "imx8mp-dw-hdmi-tx",
 		.of_match_table = imx8mp_dw_hdmi_of_table,
diff --git a/drivers/gpu/drm/bridge/ite-it6505.c b/drivers/gpu/drm/bridge/ite-it6505.c
index 27334173e911..3f68c82888c2 100644
--- a/drivers/gpu/drm/bridge/ite-it6505.c
+++ b/drivers/gpu/drm/bridge/ite-it6505.c
@@ -3,6 +3,7 @@
  * Copyright (c) 2020, The Linux Foundation. All rights reserved.
  */
 #include <linux/bits.h>
+#include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/err.h>
diff --git a/drivers/gpu/drm/bridge/ite-it66121.c b/drivers/gpu/drm/bridge/ite-it66121.c
index 1c3433b5e366..925e42f46cd8 100644
--- a/drivers/gpu/drm/bridge/ite-it66121.c
+++ b/drivers/gpu/drm/bridge/ite-it66121.c
@@ -1540,12 +1540,6 @@ static int it66121_probe(struct i2c_client *client)
 		return -EINVAL;
 	}
 
-	if (!of_device_is_available(ep)) {
-		of_node_put(ep);
-		dev_err(ctx->dev, "The remote device is disabled\n");
-		return -ENODEV;
-	}
-
 	ctx->next_bridge = of_drm_find_bridge(ep);
 	of_node_put(ep);
 	if (!ctx->next_bridge) {
@@ -1586,13 +1580,18 @@ static int it66121_probe(struct i2c_client *client)
 	ctx->bridge.funcs = &it66121_bridge_funcs;
 	ctx->bridge.of_node = dev->of_node;
 	ctx->bridge.type = DRM_MODE_CONNECTOR_HDMIA;
-	ctx->bridge.ops = DRM_BRIDGE_OP_DETECT | DRM_BRIDGE_OP_EDID | DRM_BRIDGE_OP_HPD;
-
-	ret = devm_request_threaded_irq(dev, client->irq, NULL,	it66121_irq_threaded_handler,
-					IRQF_ONESHOT, dev_name(dev), ctx);
-	if (ret < 0) {
-		dev_err(dev, "Failed to request irq %d:%d\n", client->irq, ret);
-		return ret;
+	ctx->bridge.ops = DRM_BRIDGE_OP_DETECT | DRM_BRIDGE_OP_EDID;
+	if (client->irq > 0) {
+		ctx->bridge.ops |= DRM_BRIDGE_OP_HPD;
+
+		ret = devm_request_threaded_irq(dev, client->irq, NULL,
+						it66121_irq_threaded_handler,
+						IRQF_ONESHOT, dev_name(dev),
+						ctx);
+		if (ret < 0) {
+			dev_err(dev, "Failed to request irq %d:%d\n", client->irq, ret);
+			return ret;
+		}
 	}
 
 	it66121_audio_codec_init(ctx, dev);
diff --git a/drivers/gpu/drm/bridge/lontium-lt8912b.c b/drivers/gpu/drm/bridge/lontium-lt8912b.c
index 4b2ae27f0a57..1a9defa15663 100644
--- a/drivers/gpu/drm/bridge/lontium-lt8912b.c
+++ b/drivers/gpu/drm/bridge/lontium-lt8912b.c
@@ -494,10 +494,8 @@ static int lt8912_attach_dsi(struct lt8912 *lt)
 						 };
 
 	host = of_find_mipi_dsi_host_by_node(lt->host_node);
-	if (!host) {
-		dev_err(dev, "failed to find dsi host\n");
-		return -EPROBE_DEFER;
-	}
+	if (!host)
+		return dev_err_probe(dev, -EPROBE_DEFER, "failed to find dsi host\n");
 
 	dsi = devm_mipi_dsi_device_register_full(dev, host, &info);
 	if (IS_ERR(dsi)) {
diff --git a/drivers/gpu/drm/bridge/lontium-lt9611.c b/drivers/gpu/drm/bridge/lontium-lt9611.c
index a9c7e2b07ea1..b99fe87ec738 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9611.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9611.c
@@ -761,10 +761,8 @@ static struct mipi_dsi_device *lt9611_attach_dsi(struct lt9611 *lt9611,
 	int ret;
 
 	host = of_find_mipi_dsi_host_by_node(dsi_node);
-	if (!host) {
-		dev_err(lt9611->dev, "failed to find dsi host\n");
-		return ERR_PTR(-EPROBE_DEFER);
-	}
+	if (!host)
+		return ERR_PTR(dev_err_probe(lt9611->dev, -EPROBE_DEFER, "failed to find dsi host\n"));
 
 	dsi = devm_mipi_dsi_device_register_full(dev, host, &info);
 	if (IS_ERR(dsi)) {
diff --git a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
index f4f593ad8f79..ab702471f3ab 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
@@ -266,10 +266,8 @@ static struct mipi_dsi_device *lt9611uxc_attach_dsi(struct lt9611uxc *lt9611uxc,
 	int ret;
 
 	host = of_find_mipi_dsi_host_by_node(dsi_node);
-	if (!host) {
-		dev_err(dev, "failed to find dsi host\n");
-		return ERR_PTR(-EPROBE_DEFER);
-	}
+	if (!host)
+		return ERR_PTR(dev_err_probe(dev, -EPROBE_DEFER, "failed to find dsi host\n"));
 
 	dsi = devm_mipi_dsi_device_register_full(dev, host, &info);
 	if (IS_ERR(dsi)) {
diff --git a/drivers/gpu/drm/bridge/microchip-lvds.c b/drivers/gpu/drm/bridge/microchip-lvds.c
new file mode 100644
index 000000000000..b8313dad6072
--- /dev/null
+++ b/drivers/gpu/drm/bridge/microchip-lvds.c
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2023 Microchip Technology Inc. and its subsidiaries
+ *
+ * Author: Manikandan Muralidharan <manikandan.m@microchip.com>
+ * Author: Dharma Balasubiramani <dharma.b@microchip.com>
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/delay.h>
+#include <linux/jiffies.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of_graph.h>
+#include <linux/pinctrl/devinfo.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_bridge.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_print.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_simple_kms_helper.h>
+
+#define LVDS_POLL_TIMEOUT_MS 1000
+
+/* LVDSC register offsets */
+#define LVDSC_CR	0x00
+#define LVDSC_CFGR	0x04
+#define LVDSC_SR	0x0C
+#define LVDSC_WPMR	0xE4
+
+/* Bitfields in LVDSC_CR (Control Register) */
+#define LVDSC_CR_SER_EN	BIT(0)
+
+/* Bitfields in LVDSC_CFGR (Configuration Register) */
+#define LVDSC_CFGR_PIXSIZE_24BITS	0
+#define LVDSC_CFGR_DEN_POL_HIGH		0
+#define LVDSC_CFGR_DC_UNBALANCED	0
+#define LVDSC_CFGR_MAPPING_JEIDA	BIT(6)
+
+/*Bitfields in LVDSC_SR */
+#define LVDSC_SR_CS	BIT(0)
+
+/* Bitfields in LVDSC_WPMR (Write Protection Mode Register) */
+#define LVDSC_WPMR_WPKEY_MASK	GENMASK(31, 8)
+#define LVDSC_WPMR_WPKEY_PSSWD	0x4C5644
+
+struct mchp_lvds {
+	struct device *dev;
+	void __iomem *regs;
+	struct clk *pclk;
+	struct drm_panel *panel;
+	struct drm_bridge bridge;
+	struct drm_bridge *panel_bridge;
+};
+
+static inline struct mchp_lvds *bridge_to_lvds(struct drm_bridge *bridge)
+{
+	return container_of(bridge, struct mchp_lvds, bridge);
+}
+
+static inline u32 lvds_readl(struct mchp_lvds *lvds, u32 offset)
+{
+	return readl_relaxed(lvds->regs + offset);
+}
+
+static inline void lvds_writel(struct mchp_lvds *lvds, u32 offset, u32 val)
+{
+	writel_relaxed(val, lvds->regs + offset);
+}
+
+static void lvds_serialiser_on(struct mchp_lvds *lvds)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(LVDS_POLL_TIMEOUT_MS);
+
+	/* The LVDSC registers can only be written if WPEN is cleared */
+	lvds_writel(lvds, LVDSC_WPMR, (LVDSC_WPMR_WPKEY_PSSWD &
+				LVDSC_WPMR_WPKEY_MASK));
+
+	/* Wait for the status of configuration registers to be changed */
+	while (lvds_readl(lvds, LVDSC_SR) & LVDSC_SR_CS) {
+		if (time_after(jiffies, timeout)) {
+			dev_err(lvds->dev, "%s: timeout error\n", __func__);
+			return;
+		}
+		usleep_range(1000, 2000);
+	}
+
+	/* Configure the LVDSC */
+	lvds_writel(lvds, LVDSC_CFGR, (LVDSC_CFGR_MAPPING_JEIDA |
+				LVDSC_CFGR_DC_UNBALANCED |
+				LVDSC_CFGR_DEN_POL_HIGH |
+				LVDSC_CFGR_PIXSIZE_24BITS));
+
+	/* Enable the LVDS serializer */
+	lvds_writel(lvds, LVDSC_CR, LVDSC_CR_SER_EN);
+}
+
+static int mchp_lvds_attach(struct drm_bridge *bridge,
+			    enum drm_bridge_attach_flags flags)
+{
+	struct mchp_lvds *lvds = bridge_to_lvds(bridge);
+
+	return drm_bridge_attach(bridge->encoder, lvds->panel_bridge,
+				 bridge, flags);
+}
+
+static void mchp_lvds_enable(struct drm_bridge *bridge)
+{
+	struct mchp_lvds *lvds = bridge_to_lvds(bridge);
+	int ret;
+
+	ret = clk_prepare_enable(lvds->pclk);
+	if (ret < 0) {
+		dev_err(lvds->dev, "failed to enable lvds pclk %d\n", ret);
+		return;
+	}
+
+	ret = pm_runtime_get_sync(lvds->dev);
+	if (ret < 0) {
+		dev_err(lvds->dev, "failed to get pm runtime: %d\n", ret);
+		return;
+	}
+
+	lvds_serialiser_on(lvds);
+}
+
+static void mchp_lvds_disable(struct drm_bridge *bridge)
+{
+	struct mchp_lvds *lvds = bridge_to_lvds(bridge);
+
+	pm_runtime_put(lvds->dev);
+	clk_disable_unprepare(lvds->pclk);
+}
+
+static const struct drm_bridge_funcs mchp_lvds_bridge_funcs = {
+	.attach = mchp_lvds_attach,
+	.enable = mchp_lvds_enable,
+	.disable = mchp_lvds_disable,
+};
+
+static int mchp_lvds_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct mchp_lvds *lvds;
+	struct device_node *port;
+	int ret;
+
+	if (!dev->of_node)
+		return -ENODEV;
+
+	lvds = devm_kzalloc(&pdev->dev, sizeof(*lvds), GFP_KERNEL);
+	if (!lvds)
+		return -ENOMEM;
+
+	lvds->dev = dev;
+
+	lvds->regs = devm_ioremap_resource(lvds->dev,
+			platform_get_resource(pdev, IORESOURCE_MEM, 0));
+	if (IS_ERR(lvds->regs))
+		return PTR_ERR(lvds->regs);
+
+	lvds->pclk = devm_clk_get(lvds->dev, "pclk");
+	if (IS_ERR(lvds->pclk))
+		return dev_err_probe(lvds->dev, PTR_ERR(lvds->pclk),
+				"could not get pclk_lvds\n");
+
+	port = of_graph_get_remote_node(dev->of_node, 1, 0);
+	if (!port) {
+		dev_err(dev,
+			"can't find port point, please init lvds panel port!\n");
+		return -ENODEV;
+	}
+
+	lvds->panel = of_drm_find_panel(port);
+	of_node_put(port);
+
+	if (IS_ERR(lvds->panel))
+		return -EPROBE_DEFER;
+
+	lvds->panel_bridge = devm_drm_of_get_bridge(dev, dev->of_node, 1, 0);
+
+	if (IS_ERR(lvds->panel_bridge))
+		return PTR_ERR(lvds->panel_bridge);
+
+	lvds->bridge.of_node = dev->of_node;
+	lvds->bridge.type = DRM_MODE_CONNECTOR_LVDS;
+	lvds->bridge.funcs = &mchp_lvds_bridge_funcs;
+
+	dev_set_drvdata(dev, lvds);
+	ret = devm_pm_runtime_enable(dev);
+	if (ret < 0) {
+		dev_err(lvds->dev, "failed to enable pm runtime: %d\n", ret);
+		return ret;
+	}
+
+	drm_bridge_add(&lvds->bridge);
+
+	return 0;
+}
+
+static const struct of_device_id mchp_lvds_dt_ids[] = {
+	{
+		.compatible = "microchip,sam9x75-lvds",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, mchp_lvds_dt_ids);
+
+static struct platform_driver mchp_lvds_driver = {
+	.probe = mchp_lvds_probe,
+	.driver = {
+		   .name = "microchip-lvds",
+		   .of_match_table = mchp_lvds_dt_ids,
+	},
+};
+module_platform_driver(mchp_lvds_driver);
+
+MODULE_AUTHOR("Manikandan Muralidharan <manikandan.m@microchip.com>");
+MODULE_AUTHOR("Dharma Balasubiramani <dharma.b@microchip.com>");
+MODULE_DESCRIPTION("Low Voltage Differential Signaling Controller Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/bridge/panel.c b/drivers/gpu/drm/bridge/panel.c
index 7f41525f7a6e..32506524d9a2 100644
--- a/drivers/gpu/drm/bridge/panel.c
+++ b/drivers/gpu/drm/bridge/panel.c
@@ -4,6 +4,8 @@
  * Copyright (C) 2017 Broadcom
  */
 
+#include <linux/debugfs.h>
+
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_connector.h>
diff --git a/drivers/gpu/drm/bridge/sii902x.c b/drivers/gpu/drm/bridge/sii902x.c
index 8f84e98249c7..2fbeda9025bf 100644
--- a/drivers/gpu/drm/bridge/sii902x.c
+++ b/drivers/gpu/drm/bridge/sii902x.c
@@ -1092,7 +1092,7 @@ static int sii902x_init(struct sii902x *sii902x)
 	}
 
 	sii902x->i2cmux->priv = sii902x;
-	ret = i2c_mux_add_adapter(sii902x->i2cmux, 0, 0, 0);
+	ret = i2c_mux_add_adapter(sii902x->i2cmux, 0, 0);
 	if (ret)
 		goto err_unreg_audio;
 
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
index cceb5aab6c83..9f2bc932c371 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
@@ -3291,40 +3291,17 @@ static void dw_hdmi_init_hw(struct dw_hdmi *hdmi)
 
 static int dw_hdmi_parse_dt(struct dw_hdmi *hdmi)
 {
-	struct device_node *endpoint;
 	struct device_node *remote;
 
 	if (!hdmi->plat_data->output_port)
 		return 0;
 
-	endpoint = of_graph_get_endpoint_by_regs(hdmi->dev->of_node,
-						 hdmi->plat_data->output_port,
-						 -1);
-	if (!endpoint) {
-		/*
-		 * On platforms whose bindings don't make the output port
-		 * mandatory (such as Rockchip) the plat_data->output_port
-		 * field isn't set, so it's safe to make this a fatal error.
-		 */
-		dev_err(hdmi->dev, "Missing endpoint in port@%u\n",
-			hdmi->plat_data->output_port);
-		return -ENODEV;
-	}
 
-	remote = of_graph_get_remote_port_parent(endpoint);
-	of_node_put(endpoint);
-	if (!remote) {
-		dev_err(hdmi->dev, "Endpoint in port@%u unconnected\n",
-			hdmi->plat_data->output_port);
+	remote = of_graph_get_remote_node(hdmi->dev->of_node,
+					  hdmi->plat_data->output_port,
+					  -1);
+	if (!remote)
 		return -ENODEV;
-	}
-
-	if (!of_device_is_available(remote)) {
-		dev_err(hdmi->dev, "port@%u remote device is disabled\n",
-			hdmi->plat_data->output_port);
-		of_node_put(remote);
-		return -ENODEV;
-	}
 
 	hdmi->next_bridge = of_drm_find_bridge(remote);
 	of_node_put(remote);
diff --git a/drivers/gpu/drm/bridge/tc358764.c b/drivers/gpu/drm/bridge/tc358764.c
index deccb3995022..3d3d135b4348 100644
--- a/drivers/gpu/drm/bridge/tc358764.c
+++ b/drivers/gpu/drm/bridge/tc358764.c
@@ -401,7 +401,6 @@ static struct mipi_dsi_driver tc358764_driver = {
 	.remove = tc358764_remove,
 	.driver = {
 		.name = "tc358764",
-		.owner = THIS_MODULE,
 		.of_match_table = tc358764_of_match,
 	},
 };
diff --git a/drivers/gpu/drm/bridge/tc358775.c b/drivers/gpu/drm/bridge/tc358775.c
index 90a89d70d832..3b7cc3be2ccd 100644
--- a/drivers/gpu/drm/bridge/tc358775.c
+++ b/drivers/gpu/drm/bridge/tc358775.c
@@ -15,6 +15,7 @@
 #include <linux/kernel.h>
 #include <linux/media-bus-format.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 #include <linux/regulator/consumer.h>
 #include <linux/slab.h>
 
@@ -107,6 +108,7 @@
 #define RDPKTLN         0x0404  /* Command Read Packet Length */
 
 #define VPCTRL          0x0450  /* Video Path Control */
+#define EVTMODE		BIT(5)  /* Video event mode enable, tc35876x only */
 #define HTIM1           0x0454  /* Horizontal Timing Control 1 */
 #define HTIM2           0x0458  /* Horizontal Timing Control 2 */
 #define VTIM1           0x045C  /* Vertical Timing Control 1 */
@@ -254,6 +256,11 @@ enum tc358775_ports {
 	TC358775_LVDS_OUT1,
 };
 
+enum tc3587x5_type {
+	TC358765 = 0x65,
+	TC358775 = 0x75,
+};
+
 struct tc_data {
 	struct i2c_client	*i2c;
 	struct device		*dev;
@@ -271,6 +278,8 @@ struct tc_data {
 	struct gpio_desc	*stby_gpio;
 	u8			lvds_link; /* single-link or dual-link */
 	u8			bpc;
+
+	enum tc3587x5_type	type;
 };
 
 static inline struct tc_data *bridge_to_tc(struct drm_bridge *b)
@@ -424,10 +433,16 @@ static void tc_bridge_enable(struct drm_bridge *bridge)
 	d2l_write(tc->i2c, PPI_STARTPPI, PPI_START_FUNCTION);
 	d2l_write(tc->i2c, DSI_STARTDSI, DSI_RX_START);
 
+	/* Video event mode vs pulse mode bit, does not exist for tc358775 */
+	if (tc->type == TC358765)
+		val = EVTMODE;
+	else
+		val = 0;
+
 	if (tc->bpc == 8)
-		val = TC358775_VPCTRL_OPXLFMT(1);
+		val |= TC358775_VPCTRL_OPXLFMT(1);
 	else /* bpc = 6; */
-		val = TC358775_VPCTRL_MSF(1);
+		val |= TC358775_VPCTRL_MSF(1);
 
 	dsiclk = mode->crtc_clock * 3 * tc->bpc / tc->num_dsi_lanes / 1000;
 	clkdiv = dsiclk / (tc->lvds_link == DUAL_LINK ? DIVIDE_BY_6 : DIVIDE_BY_3);
@@ -454,10 +469,6 @@ static void tc_bridge_enable(struct drm_bridge *bridge)
 	dev_dbg(tc->dev, "bus_formats %04x bpc %d\n",
 		connector->display_info.bus_formats[0],
 		tc->bpc);
-	/*
-	 * Default hardware register settings of tc358775 configured
-	 * with MEDIA_BUS_FMT_RGB888_1X7X4_JEIDA jeida-24 format
-	 */
 	if (connector->display_info.bus_formats[0] ==
 		MEDIA_BUS_FMT_RGB888_1X7X4_SPWG) {
 		/* VESA-24 */
@@ -468,14 +479,15 @@ static void tc_bridge_enable(struct drm_bridge *bridge)
 		d2l_write(tc->i2c, LV_MX1619, LV_MX(LVI_B6, LVI_B7, LVI_B1, LVI_B2));
 		d2l_write(tc->i2c, LV_MX2023, LV_MX(LVI_B3, LVI_B4, LVI_B5, LVI_L0));
 		d2l_write(tc->i2c, LV_MX2427, LV_MX(LVI_HS, LVI_VS, LVI_DE, LVI_R6));
-	} else { /*  MEDIA_BUS_FMT_RGB666_1X7X3_SPWG - JEIDA-18 */
-		d2l_write(tc->i2c, LV_MX0003, LV_MX(LVI_R0, LVI_R1, LVI_R2, LVI_R3));
-		d2l_write(tc->i2c, LV_MX0407, LV_MX(LVI_R4, LVI_L0, LVI_R5, LVI_G0));
-		d2l_write(tc->i2c, LV_MX0811, LV_MX(LVI_G1, LVI_G2, LVI_L0, LVI_L0));
-		d2l_write(tc->i2c, LV_MX1215, LV_MX(LVI_G3, LVI_G4, LVI_G5, LVI_B0));
-		d2l_write(tc->i2c, LV_MX1619, LV_MX(LVI_L0, LVI_L0, LVI_B1, LVI_B2));
-		d2l_write(tc->i2c, LV_MX2023, LV_MX(LVI_B3, LVI_B4, LVI_B5, LVI_L0));
-		d2l_write(tc->i2c, LV_MX2427, LV_MX(LVI_HS, LVI_VS, LVI_DE, LVI_L0));
+	} else {
+		/* JEIDA-18 and JEIDA-24 */
+		d2l_write(tc->i2c, LV_MX0003, LV_MX(LVI_R2, LVI_R3, LVI_R4, LVI_R5));
+		d2l_write(tc->i2c, LV_MX0407, LV_MX(LVI_R6, LVI_R1, LVI_R7, LVI_G2));
+		d2l_write(tc->i2c, LV_MX0811, LV_MX(LVI_G3, LVI_G4, LVI_G0, LVI_G1));
+		d2l_write(tc->i2c, LV_MX1215, LV_MX(LVI_G5, LVI_G6, LVI_G7, LVI_B2));
+		d2l_write(tc->i2c, LV_MX1619, LV_MX(LVI_B0, LVI_B1, LVI_B3, LVI_B4));
+		d2l_write(tc->i2c, LV_MX2023, LV_MX(LVI_B5, LVI_B6, LVI_B7, LVI_L0));
+		d2l_write(tc->i2c, LV_MX2427, LV_MX(LVI_HS, LVI_VS, LVI_DE, LVI_R0));
 	}
 
 	d2l_write(tc->i2c, VFUEN, VFUEN_EN);
@@ -528,27 +540,24 @@ tc_mode_valid(struct drm_bridge *bridge,
 static int tc358775_parse_dt(struct device_node *np, struct tc_data *tc)
 {
 	struct device_node *endpoint;
-	struct device_node *parent;
 	struct device_node *remote;
 	int dsi_lanes = -1;
 
-	/*
-	 * To get the data-lanes of dsi, we need to access the dsi0_out of port1
-	 *  of dsi0 endpoint from bridge port0 of d2l_in
-	 */
 	endpoint = of_graph_get_endpoint_by_regs(tc->dev->of_node,
 						 TC358775_DSI_IN, -1);
-	if (endpoint) {
-		/* dsi0_out node */
-		parent = of_graph_get_remote_port_parent(endpoint);
-		of_node_put(endpoint);
-		if (parent) {
-			/* dsi0 port 1 */
-			dsi_lanes = drm_of_get_data_lanes_count_ep(parent, 1, -1, 1, 4);
-			of_node_put(parent);
-		}
+	dsi_lanes = drm_of_get_data_lanes_count(endpoint, 1, 4);
+
+	/* Quirk old dtb: Use data lanes from the DSI host side instead of bridge */
+	if (dsi_lanes == -EINVAL || dsi_lanes == -ENODEV) {
+		remote = of_graph_get_remote_endpoint(endpoint);
+		dsi_lanes = drm_of_get_data_lanes_count(remote, 1, 4);
+		of_node_put(remote);
+		if (dsi_lanes >= 1)
+			dev_warn(tc->dev, "no dsi-lanes for the bridge, using host lanes\n");
 	}
 
+	of_node_put(endpoint);
+
 	if (dsi_lanes < 0)
 		return dsi_lanes;
 
@@ -610,10 +619,8 @@ static int tc_attach_host(struct tc_data *tc)
 						};
 
 	host = of_find_mipi_dsi_host_by_node(tc->host_node);
-	if (!host) {
-		dev_err(dev, "failed to find dsi host\n");
-		return -EPROBE_DEFER;
-	}
+	if (!host)
+		return dev_err_probe(dev, -EPROBE_DEFER, "failed to find dsi host\n");
 
 	dsi = devm_mipi_dsi_device_register_full(dev, host, &info);
 	if (IS_ERR(dsi)) {
@@ -625,7 +632,21 @@ static int tc_attach_host(struct tc_data *tc)
 
 	dsi->lanes = tc->num_dsi_lanes;
 	dsi->format = MIPI_DSI_FMT_RGB888;
-	dsi->mode_flags = MIPI_DSI_MODE_VIDEO;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+			  MIPI_DSI_MODE_LPM;
+
+	/*
+	 * The hs_rate and lp_rate are data rate values. The HS mode is
+	 * differential, while the LP mode is single ended. As the HS mode
+	 * uses DDR, the DSI clock frequency is half the hs_rate. The 10 Mbs
+	 * data rate for LP mode is not specified in the bridge data sheet,
+	 * but seems to be part of the MIPI DSI spec.
+	 */
+	if (tc->type == TC358765)
+		dsi->hs_rate = 800000000;
+	else
+		dsi->hs_rate = 1000000000;
+	dsi->lp_rate = 10000000;
 
 	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
@@ -648,6 +669,7 @@ static int tc_probe(struct i2c_client *client)
 
 	tc->dev = dev;
 	tc->i2c = client;
+	tc->type = (enum tc3587x5_type)(unsigned long)of_device_get_match_data(dev);
 
 	tc->panel_bridge = devm_drm_of_get_bridge(dev, dev->of_node,
 						  TC358775_LVDS_OUT0, 0);
@@ -672,12 +694,9 @@ static int tc_probe(struct i2c_client *client)
 		return ret;
 	}
 
-	tc->stby_gpio = devm_gpiod_get(dev, "stby", GPIOD_OUT_HIGH);
-	if (IS_ERR(tc->stby_gpio)) {
-		ret = PTR_ERR(tc->stby_gpio);
-		dev_err(dev, "cannot get stby-gpio %d\n", ret);
-		return ret;
-	}
+	tc->stby_gpio = devm_gpiod_get_optional(dev, "stby", GPIOD_OUT_HIGH);
+	if (IS_ERR(tc->stby_gpio))
+		return PTR_ERR(tc->stby_gpio);
 
 	tc->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(tc->reset_gpio)) {
@@ -688,6 +707,7 @@ static int tc_probe(struct i2c_client *client)
 
 	tc->bridge.funcs = &tc_bridge_funcs;
 	tc->bridge.of_node = dev->of_node;
+	tc->bridge.pre_enable_prev_first = true;
 	drm_bridge_add(&tc->bridge);
 
 	i2c_set_clientdata(client, tc);
@@ -711,13 +731,15 @@ static void tc_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tc358775_i2c_ids[] = {
-	{ "tc358775", 0 },
+	{ "tc358765", TC358765, },
+	{ "tc358775", TC358775, },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tc358775_i2c_ids);
 
 static const struct of_device_id tc358775_of_ids[] = {
-	{ .compatible = "toshiba,tc358775", },
+	{ .compatible = "toshiba,tc358765", .data = (void *)TC358765, },
+	{ .compatible = "toshiba,tc358775", .data = (void *)TC358775, },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, tc358775_of_ids);
diff --git a/drivers/gpu/drm/bridge/thc63lvd1024.c b/drivers/gpu/drm/bridge/thc63lvd1024.c
index d4c1a601bbb5..674efc489e3a 100644
--- a/drivers/gpu/drm/bridge/thc63lvd1024.c
+++ b/drivers/gpu/drm/bridge/thc63lvd1024.c
@@ -123,26 +123,11 @@ static int thc63_parse_dt(struct thc63_dev *thc63)
 	struct device_node *endpoint;
 	struct device_node *remote;
 
-	endpoint = of_graph_get_endpoint_by_regs(thc63->dev->of_node,
-						 THC63_RGB_OUT0, -1);
-	if (!endpoint) {
-		dev_err(thc63->dev, "Missing endpoint in port@%u\n",
-			THC63_RGB_OUT0);
-		return -ENODEV;
-	}
-
-	remote = of_graph_get_remote_port_parent(endpoint);
-	of_node_put(endpoint);
+	remote = of_graph_get_remote_node(thc63->dev->of_node,
+					  THC63_RGB_OUT0, -1);
 	if (!remote) {
-		dev_err(thc63->dev, "Endpoint in port@%u unconnected\n",
-			THC63_RGB_OUT0);
-		return -ENODEV;
-	}
-
-	if (!of_device_is_available(remote)) {
-		dev_err(thc63->dev, "port@%u remote endpoint is disabled\n",
+		dev_err(thc63->dev, "No remote endpoint for port@%u\n",
 			THC63_RGB_OUT0);
-		of_node_put(remote);
 		return -ENODEV;
 	}
 
diff --git a/drivers/gpu/drm/bridge/ti-dlpc3433.c b/drivers/gpu/drm/bridge/ti-dlpc3433.c
index ca3348109bcd..6b559e071301 100644
--- a/drivers/gpu/drm/bridge/ti-dlpc3433.c
+++ b/drivers/gpu/drm/bridge/ti-dlpc3433.c
@@ -319,12 +319,11 @@ static int dlpc_host_attach(struct dlpc *dlpc)
 		.channel = 0,
 		.node = NULL,
 	};
+	int ret;
 
 	host = of_find_mipi_dsi_host_by_node(dlpc->host_node);
-	if (!host) {
-		DRM_DEV_ERROR(dev, "failed to find dsi host\n");
-		return -EPROBE_DEFER;
-	}
+	if (!host)
+		return dev_err_probe(dev, -EPROBE_DEFER, "failed to find dsi host\n");
 
 	dlpc->dsi = mipi_dsi_device_register_full(host, &info);
 	if (IS_ERR(dlpc->dsi)) {
@@ -336,7 +335,11 @@ static int dlpc_host_attach(struct dlpc *dlpc)
 	dlpc->dsi->format = MIPI_DSI_FMT_RGB565;
 	dlpc->dsi->lanes = dlpc->dsi_lanes;
 
-	return devm_mipi_dsi_attach(dev, dlpc->dsi);
+	ret = devm_mipi_dsi_attach(dev, dlpc->dsi);
+	if (ret)
+		DRM_DEV_ERROR(dev, "failed to attach dsi host\n");
+
+	return ret;
 }
 
 static int dlpc3433_probe(struct i2c_client *client)
@@ -367,10 +370,8 @@ static int dlpc3433_probe(struct i2c_client *client)
 	drm_bridge_add(&dlpc->bridge);
 
 	ret = dlpc_host_attach(dlpc);
-	if (ret) {
-		DRM_DEV_ERROR(dev, "failed to attach dsi host\n");
+	if (ret)
 		goto err_remove_bridge;
-	}
 
 	return 0;
 
diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi83.c b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
index 4814b7b6d1fd..57a7ed13f996 100644
--- a/drivers/gpu/drm/bridge/ti-sn65dsi83.c
+++ b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
@@ -478,7 +478,6 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge,
 		dev_err(ctx->dev, "failed to lock PLL, ret=%i\n", ret);
 		/* On failure, disable PLL again and exit. */
 		regmap_write(ctx->regmap, REG_RC_PLL_EN, 0x00);
-		regulator_disable(ctx->vcc);
 		return;
 	}
 
diff --git a/drivers/gpu/drm/ci/arm64.config b/drivers/gpu/drm/ci/arm64.config
index 8dbce9919a57..4140303d6260 100644
--- a/drivers/gpu/drm/ci/arm64.config
+++ b/drivers/gpu/drm/ci/arm64.config
@@ -87,7 +87,7 @@ CONFIG_DRM_PARADE_PS8640=y
 CONFIG_DRM_LONTIUM_LT9611UXC=y
 CONFIG_PHY_QCOM_USB_HS=y
 CONFIG_QCOM_GPI_DMA=y
-CONFIG_USB_ONBOARD_HUB=y
+CONFIG_USB_ONBOARD_DEV=y
 CONFIG_NVMEM_QCOM_QFPROM=y
 CONFIG_PHY_QCOM_USB_SNPS_FEMTO_V2=y
 
@@ -97,7 +97,7 @@ CONFIG_USB_RTL8152=y
 # db820c ethernet
 CONFIG_ATL1C=y
 # Chromebooks ethernet
-CONFIG_USB_ONBOARD_HUB=y
+CONFIG_USB_ONBOARD_DEV=y
 # 888 HDK ethernet
 CONFIG_USB_LAN78XX=y
 
diff --git a/drivers/gpu/drm/ci/test.yml b/drivers/gpu/drm/ci/test.yml
index 0857773e5c5f..8bc63912fddb 100644
--- a/drivers/gpu/drm/ci/test.yml
+++ b/drivers/gpu/drm/ci/test.yml
@@ -252,11 +252,11 @@ i915:cml:
 i915:tgl:
   extends:
     - .i915
-  parallel: 8
+  parallel: 5
   variables:
-    DEVICE_TYPE: asus-cx9400-volteer
+    DEVICE_TYPE: acer-cp514-2h-1130g7-volteer
     GPU_VERSION: tgl
-    RUNNER_TAG: mesa-ci-x86-64-lava-asus-cx9400-volteer
+    RUNNER_TAG: mesa-ci-x86-64-lava-acer-cp514-2h-1130g7-volteer
 
 .amdgpu:
   extends:
diff --git a/drivers/gpu/drm/display/Kconfig b/drivers/gpu/drm/display/Kconfig
index c0f56888c328..864a6488bfdf 100644
--- a/drivers/gpu/drm/display/Kconfig
+++ b/drivers/gpu/drm/display/Kconfig
@@ -1,15 +1,36 @@
 # SPDX-License-Identifier: MIT
 
-config DRM_DP_AUX_BUS
+config DRM_DISPLAY_HELPER
 	tristate
 	depends on DRM
-	depends on OF || COMPILE_TEST
+	help
+	  DRM helpers for display adapters.
 
-config DRM_DISPLAY_HELPER
+config DRM_DISPLAY_DP_AUX_BUS
 	tristate
 	depends on DRM
+	depends on OF || COMPILE_TEST
+
+config DRM_DISPLAY_DP_AUX_CEC
+	bool "Enable DisplayPort CEC-Tunneling-over-AUX HDMI support"
+	depends on DRM && DRM_DISPLAY_HELPER
+	select DRM_DISPLAY_DP_HELPER
+	select CEC_CORE
 	help
-	  DRM helpers for display adapters.
+	  Choose this option if you want to enable HDMI CEC support for
+	  DisplayPort/USB-C to HDMI adapters.
+
+	  Note: not all adapters support this feature, and even for those
+	  that do support this they often do not hook up the CEC pin.
+
+config DRM_DISPLAY_DP_AUX_CHARDEV
+	bool "DRM DP AUX Interface"
+	depends on DRM && DRM_DISPLAY_HELPER
+	select DRM_DISPLAY_DP_HELPER
+	help
+	  Choose this option to enable a /dev/drm_dp_auxN node that allows to
+	  read and write values to arbitrary DPCD registers on the DP aux
+	  channel.
 
 config DRM_DISPLAY_DP_HELPER
 	bool
@@ -25,7 +46,7 @@ config DRM_DISPLAY_DP_TUNNEL
 	  DP tunnel features like the Bandwidth Allocation mode to maximize the
 	  BW utilization for display streams on Thunderbolt links.
 
-config DRM_DISPLAY_DEBUG_DP_TUNNEL_STATE
+config DRM_DISPLAY_DP_TUNNEL_STATE_DEBUG
 	bool "Enable debugging the DP tunnel state"
 	depends on REF_TRACKER
 	depends on DRM_DISPLAY_DP_TUNNEL
@@ -49,24 +70,3 @@ config DRM_DISPLAY_HDMI_HELPER
 	depends on DRM_DISPLAY_HELPER
 	help
 	  DRM display helpers for HDMI.
-
-config DRM_DP_AUX_CHARDEV
-	bool "DRM DP AUX Interface"
-	depends on DRM && DRM_DISPLAY_HELPER
-	select DRM_DISPLAY_DP_HELPER
-	help
-	  Choose this option to enable a /dev/drm_dp_auxN node that allows to
-	  read and write values to arbitrary DPCD registers on the DP aux
-	  channel.
-
-config DRM_DP_CEC
-	bool "Enable DisplayPort CEC-Tunneling-over-AUX HDMI support"
-	depends on DRM && DRM_DISPLAY_HELPER
-	select DRM_DISPLAY_DP_HELPER
-	select CEC_CORE
-	help
-	  Choose this option if you want to enable HDMI CEC support for
-	  DisplayPort/USB-C to HDMI adapters.
-
-	  Note: not all adapters support this feature, and even for those
-	  that do support this they often do not hook up the CEC pin.
diff --git a/drivers/gpu/drm/display/Makefile b/drivers/gpu/drm/display/Makefile
index 7ca61333c669..17d2cc73ff56 100644
--- a/drivers/gpu/drm/display/Makefile
+++ b/drivers/gpu/drm/display/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: MIT
 
-obj-$(CONFIG_DRM_DP_AUX_BUS) += drm_dp_aux_bus.o
+obj-$(CONFIG_DRM_DISPLAY_DP_AUX_BUS) += drm_dp_aux_bus.o
 
 drm_display_helper-y := drm_display_helper_mod.o
 drm_display_helper-$(CONFIG_DRM_DISPLAY_DP_HELPER) += \
@@ -14,7 +14,7 @@ drm_display_helper-$(CONFIG_DRM_DISPLAY_HDCP_HELPER) += drm_hdcp_helper.o
 drm_display_helper-$(CONFIG_DRM_DISPLAY_HDMI_HELPER) += \
 	drm_hdmi_helper.o \
 	drm_scdc_helper.o
-drm_display_helper-$(CONFIG_DRM_DP_AUX_CHARDEV) += drm_dp_aux_dev.o
-drm_display_helper-$(CONFIG_DRM_DP_CEC) += drm_dp_cec.o
+drm_display_helper-$(CONFIG_DRM_DISPLAY_DP_AUX_CHARDEV) += drm_dp_aux_dev.o
+drm_display_helper-$(CONFIG_DRM_DISPLAY_DP_AUX_CEC) += drm_dp_cec.o
 
 obj-$(CONFIG_DRM_DISPLAY_HELPER) += drm_display_helper.o
diff --git a/drivers/gpu/drm/display/drm_dp_helper.c b/drivers/gpu/drm/display/drm_dp_helper.c
index f5d4be897866..79a615667aab 100644
--- a/drivers/gpu/drm/display/drm_dp_helper.c
+++ b/drivers/gpu/drm/display/drm_dp_helper.c
@@ -2113,7 +2113,7 @@ EXPORT_SYMBOL(drm_dp_aux_init);
  * drm_dp_aux_register() in &drm_connector_funcs.late_register, and likewise to
  * call drm_dp_aux_unregister() in &drm_connector_funcs.early_unregister.
  * Functions which don't follow this will likely Oops when
- * %CONFIG_DRM_DP_AUX_CHARDEV is enabled.
+ * %CONFIG_DRM_DISPLAY_DP_AUX_CHARDEV is enabled.
  *
  * For devices where the AUX channel is a device that exists independently of
  * the &drm_device that uses it, such as SoCs and bridge devices, it is
@@ -2281,6 +2281,8 @@ static const struct dpcd_quirk dpcd_quirk_list[] = {
 	{ OUI(0x90, 0xCC, 0x24), DEVICE_ID_ANY, true, BIT(DP_DPCD_QUIRK_DSC_WITHOUT_VIRTUAL_DPCD) },
 	/* Synaptics DP1.4 MST hubs require DSC for some modes on which it applies HBLANK expansion. */
 	{ OUI(0x90, 0xCC, 0x24), DEVICE_ID_ANY, true, BIT(DP_DPCD_QUIRK_HBLANK_EXPANSION_REQUIRES_DSC) },
+	/* MediaTek panels (at least in U3224KBA) require DSC for modes with a short HBLANK on UHBR links. */
+	{ OUI(0x00, 0x0C, 0xE7), DEVICE_ID_ANY, false, BIT(DP_DPCD_QUIRK_HBLANK_EXPANSION_REQUIRES_DSC) },
 	/* Apple MacBookPro 2017 15 inch eDP Retina panel reports too low DP_MAX_LINK_RATE */
 	{ OUI(0x00, 0x10, 0xfa), DEVICE_ID(101, 68, 21, 101, 98, 97), false, BIT(DP_DPCD_QUIRK_CAN_DO_MAX_LINK_RATE_3_24_GBPS) },
 };
@@ -2948,6 +2950,43 @@ void drm_dp_vsc_sdp_log(struct drm_printer *p, const struct drm_dp_vsc_sdp *vsc)
 }
 EXPORT_SYMBOL(drm_dp_vsc_sdp_log);
 
+void drm_dp_as_sdp_log(struct drm_printer *p, const struct drm_dp_as_sdp *as_sdp)
+{
+	drm_printf(p, "DP SDP: AS_SDP, revision %u, length %u\n",
+		   as_sdp->revision, as_sdp->length);
+	drm_printf(p, "    vtotal: %d\n", as_sdp->vtotal);
+	drm_printf(p, "    target_rr: %d\n", as_sdp->target_rr);
+	drm_printf(p, "    duration_incr_ms: %d\n", as_sdp->duration_incr_ms);
+	drm_printf(p, "    duration_decr_ms: %d\n", as_sdp->duration_decr_ms);
+	drm_printf(p, "    operation_mode: %d\n", as_sdp->mode);
+}
+EXPORT_SYMBOL(drm_dp_as_sdp_log);
+
+/**
+ * drm_dp_as_sdp_supported() - check if adaptive sync sdp is supported
+ * @aux: DisplayPort AUX channel
+ * @dpcd: DisplayPort configuration data
+ *
+ * Returns true if adaptive sync sdp is supported, else returns false
+ */
+bool drm_dp_as_sdp_supported(struct drm_dp_aux *aux, const u8 dpcd[DP_RECEIVER_CAP_SIZE])
+{
+	u8 rx_feature;
+
+	if (dpcd[DP_DPCD_REV] < DP_DPCD_REV_13)
+		return false;
+
+	if (drm_dp_dpcd_readb(aux, DP_DPRX_FEATURE_ENUMERATION_LIST_CONT_1,
+			      &rx_feature) != 1) {
+		drm_dbg_dp(aux->drm_dev,
+			   "Failed to read DP_DPRX_FEATURE_ENUMERATION_LIST_CONT_1\n");
+		return false;
+	}
+
+	return (rx_feature & DP_ADAPTIVE_SYNC_SDP_SUPPORTED);
+}
+EXPORT_SYMBOL(drm_dp_as_sdp_supported);
+
 /**
  * drm_dp_vsc_sdp_supported() - check if vsc sdp is supported
  * @aux: DisplayPort AUX channel
diff --git a/drivers/gpu/drm/display/drm_dp_helper_internal.h b/drivers/gpu/drm/display/drm_dp_helper_internal.h
index 8917fc3af9ec..737949a2820f 100644
--- a/drivers/gpu/drm/display/drm_dp_helper_internal.h
+++ b/drivers/gpu/drm/display/drm_dp_helper_internal.h
@@ -5,7 +5,7 @@
 
 struct drm_dp_aux;
 
-#ifdef CONFIG_DRM_DP_AUX_CHARDEV
+#ifdef CONFIG_DRM_DISPLAY_DP_AUX_CHARDEV
 int drm_dp_aux_dev_init(void);
 void drm_dp_aux_dev_exit(void);
 int drm_dp_aux_register_devnode(struct drm_dp_aux *aux);
diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c
index 03d528209426..7f8e1cfbe19d 100644
--- a/drivers/gpu/drm/display/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c
@@ -2274,7 +2274,7 @@ drm_dp_mst_port_add_connector(struct drm_dp_mst_branch *mstb,
 
 	if (port->pdt != DP_PEER_DEVICE_NONE &&
 	    drm_dp_mst_is_end_device(port->pdt, port->mcs) &&
-	    port->port_num >= DP_MST_LOGICAL_PORT_0)
+	    drm_dp_mst_port_is_logical(port))
 		port->cached_edid = drm_edid_read_ddc(port->connector,
 						      &port->aux.ddc);
 
@@ -3421,7 +3421,6 @@ EXPORT_SYMBOL(drm_dp_remove_payload_part2);
 /**
  * drm_dp_add_payload_part2() - Execute payload update part 2
  * @mgr: Manager to use.
- * @state: The global atomic state
  * @payload: The payload to update
  *
  * If @payload was successfully assigned a starting time slot by drm_dp_add_payload_part1(), this
@@ -3430,14 +3429,13 @@ EXPORT_SYMBOL(drm_dp_remove_payload_part2);
  * Returns: 0 on success, negative error code on failure.
  */
 int drm_dp_add_payload_part2(struct drm_dp_mst_topology_mgr *mgr,
-			     struct drm_atomic_state *state,
 			     struct drm_dp_mst_atomic_payload *payload)
 {
 	int ret = 0;
 
 	/* Skip failed payloads */
 	if (payload->payload_allocation_status != DRM_DP_MST_PAYLOAD_ALLOCATION_DFP) {
-		drm_dbg_kms(state->dev, "Part 1 of payload creation for %s failed, skipping part 2\n",
+		drm_dbg_kms(mgr->dev, "Part 1 of payload creation for %s failed, skipping part 2\n",
 			    payload->port->connector->name);
 		return -EIO;
 	}
@@ -3608,24 +3606,30 @@ fixed20_12 drm_dp_get_vc_payload_bw(const struct drm_dp_mst_topology_mgr *mgr,
 EXPORT_SYMBOL(drm_dp_get_vc_payload_bw);
 
 /**
- * drm_dp_read_mst_cap() - check whether or not a sink supports MST
+ * drm_dp_read_mst_cap() - Read the sink's MST mode capability
  * @aux: The DP AUX channel to use
  * @dpcd: A cached copy of the DPCD capabilities for this sink
  *
- * Returns: %True if the sink supports MST, %false otherwise
+ * Returns: enum drm_dp_mst_mode to indicate MST mode capability
  */
-bool drm_dp_read_mst_cap(struct drm_dp_aux *aux,
-			 const u8 dpcd[DP_RECEIVER_CAP_SIZE])
+enum drm_dp_mst_mode drm_dp_read_mst_cap(struct drm_dp_aux *aux,
+					 const u8 dpcd[DP_RECEIVER_CAP_SIZE])
 {
 	u8 mstm_cap;
 
 	if (dpcd[DP_DPCD_REV] < DP_DPCD_REV_12)
-		return false;
+		return DRM_DP_SST;
 
 	if (drm_dp_dpcd_readb(aux, DP_MSTM_CAP, &mstm_cap) != 1)
-		return false;
+		return DRM_DP_SST;
+
+	if (mstm_cap & DP_MST_CAP)
+		return DRM_DP_MST;
 
-	return mstm_cap & DP_MST_CAP;
+	if (mstm_cap & DP_SINGLE_STREAM_SIDEBAND_MSG)
+		return DRM_DP_SST_SIDEBAND_MSG;
+
+	return DRM_DP_SST;
 }
 EXPORT_SYMBOL(drm_dp_read_mst_cap);
 
@@ -4213,7 +4217,7 @@ drm_dp_mst_detect_port(struct drm_connector *connector,
 	case DP_PEER_DEVICE_SST_SINK:
 		ret = connector_status_connected;
 		/* for logical ports - cache the EDID */
-		if (port->port_num >= DP_MST_LOGICAL_PORT_0 && !port->cached_edid)
+		if (drm_dp_mst_port_is_logical(port) && !port->cached_edid)
 			port->cached_edid = drm_edid_read_ddc(connector, &port->aux.ddc);
 		break;
 	case DP_PEER_DEVICE_DP_LEGACY_CONV:
@@ -5977,7 +5981,7 @@ static bool drm_dp_mst_is_virtual_dpcd(struct drm_dp_mst_port *port)
 		return false;
 
 	/* Virtual DP Sink (Internal Display Panel) */
-	if (port->port_num >= 8)
+	if (drm_dp_mst_port_is_logical(port))
 		return true;
 
 	/* DP-to-HDMI Protocol Converter */
@@ -6005,6 +6009,22 @@ static bool drm_dp_mst_is_virtual_dpcd(struct drm_dp_mst_port *port)
 }
 
 /**
+ * drm_dp_mst_aux_for_parent() - Get the AUX device for an MST port's parent
+ * @port: MST port whose parent's AUX device is returned
+ *
+ * Return the AUX device for @port's parent or NULL if port's parent is the
+ * root port.
+ */
+struct drm_dp_aux *drm_dp_mst_aux_for_parent(struct drm_dp_mst_port *port)
+{
+	if (!port->parent || !port->parent->port_parent)
+		return NULL;
+
+	return &port->parent->port_parent->aux;
+}
+EXPORT_SYMBOL(drm_dp_mst_aux_for_parent);
+
+/**
  * drm_dp_mst_dsc_aux_for_port() - Find the correct aux for DSC
  * @port: The port to check. A leaf of the MST tree with an attached display.
  *
diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology_internal.h b/drivers/gpu/drm/display/drm_dp_mst_topology_internal.h
index a785ccbfdd73..f41c34e26be2 100644
--- a/drivers/gpu/drm/display/drm_dp_mst_topology_internal.h
+++ b/drivers/gpu/drm/display/drm_dp_mst_topology_internal.h
@@ -10,7 +10,9 @@
 #ifndef _DRM_DP_MST_HELPER_INTERNAL_H_
 #define _DRM_DP_MST_HELPER_INTERNAL_H_
 
-#include <drm/display/drm_dp_mst_helper.h>
+struct drm_dp_sideband_msg_req_body;
+struct drm_dp_sideband_msg_tx;
+struct drm_printer;
 
 void
 drm_dp_encode_sideband_req(const struct drm_dp_sideband_msg_req_body *req,
diff --git a/drivers/gpu/drm/display/drm_dp_tunnel.c b/drivers/gpu/drm/display/drm_dp_tunnel.c
index 120e0de674c1..48b2df120086 100644
--- a/drivers/gpu/drm/display/drm_dp_tunnel.c
+++ b/drivers/gpu/drm/display/drm_dp_tunnel.c
@@ -191,7 +191,7 @@ struct drm_dp_tunnel_mgr {
 	struct drm_dp_tunnel_group *groups;
 	wait_queue_head_t bw_req_queue;
 
-#ifdef CONFIG_DRM_DISPLAY_DEBUG_DP_TUNNEL_STATE
+#ifdef CONFIG_DRM_DISPLAY_DP_TUNNEL_STATE_DEBUG
 	struct ref_tracker_dir ref_tracker;
 #endif
 };
@@ -385,7 +385,7 @@ static void tunnel_put(struct drm_dp_tunnel *tunnel)
 	kref_put(&tunnel->kref, free_tunnel);
 }
 
-#ifdef CONFIG_DRM_DISPLAY_DEBUG_DP_TUNNEL_STATE
+#ifdef CONFIG_DRM_DISPLAY_DP_TUNNEL_STATE_DEBUG
 static void track_tunnel_ref(struct drm_dp_tunnel *tunnel,
 			     struct ref_tracker **tracker)
 {
@@ -436,8 +436,8 @@ EXPORT_SYMBOL(drm_dp_tunnel_get);
 
 /**
  * drm_dp_tunnel_put - Put a reference for a DP tunnel
- * @tunnel - Tunnel object
- * @tracker - Debug tracker for the reference
+ * @tunnel: Tunnel object
+ * @tracker: Debug tracker for the reference
  *
  * Put a reference for @tunnel along with its debug *@tracker, which
  * was obtained with drm_dp_tunnel_get().
@@ -1170,7 +1170,7 @@ int drm_dp_tunnel_alloc_bw(struct drm_dp_tunnel *tunnel, int bw)
 EXPORT_SYMBOL(drm_dp_tunnel_alloc_bw);
 
 /**
- * drm_dp_tunnel_atomic_get_allocated_bw - Get the BW allocated for a DP tunnel
+ * drm_dp_tunnel_get_allocated_bw - Get the BW allocated for a DP tunnel
  * @tunnel: Tunnel object
  *
  * Get the current BW allocated for @tunnel. After the tunnel is created /
@@ -1603,7 +1603,7 @@ static void cleanup_group(struct drm_dp_tunnel_group *group)
 	drm_atomic_private_obj_fini(&group->base);
 }
 
-#ifdef CONFIG_DRM_DISPLAY_DEBUG_DP_TUNNEL_STATE
+#ifdef CONFIG_DRM_DISPLAY_DP_TUNNEL_STATE_DEBUG
 static void check_unique_stream_ids(const struct drm_dp_tunnel_group_state *group_state)
 {
 	const struct drm_dp_tunnel_state *tunnel_state;
@@ -1881,7 +1881,7 @@ static void destroy_mgr(struct drm_dp_tunnel_mgr *mgr)
 		drm_WARN_ON(mgr->dev, !list_empty(&mgr->groups[i].tunnels));
 	}
 
-#ifdef CONFIG_DRM_DISPLAY_DEBUG_DP_TUNNEL_STATE
+#ifdef CONFIG_DRM_DISPLAY_DP_TUNNEL_STATE_DEBUG
 	ref_tracker_dir_exit(&mgr->ref_tracker);
 #endif
 
@@ -1892,6 +1892,7 @@ static void destroy_mgr(struct drm_dp_tunnel_mgr *mgr)
 /**
  * drm_dp_tunnel_mgr_create - Create a DP tunnel manager
  * @dev: DRM device object
+ * @max_group_count: Maximum number of tunnel groups
  *
  * Creates a DP tunnel manager for @dev.
  *
@@ -1918,7 +1919,7 @@ drm_dp_tunnel_mgr_create(struct drm_device *dev, int max_group_count)
 		return NULL;
 	}
 
-#ifdef CONFIG_DRM_DISPLAY_DEBUG_DP_TUNNEL_STATE
+#ifdef CONFIG_DRM_DISPLAY_DP_TUNNEL_STATE_DEBUG
 	ref_tracker_dir_init(&mgr->ref_tracker, 16, "dptun");
 #endif
 
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index 39ef0a6addeb..fb97b51b38f1 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -38,6 +38,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_panic.h>
 #include <drm/drm_print.h>
 #include <drm/drm_self_refresh_helper.h>
 #include <drm/drm_vblank.h>
@@ -3016,6 +3017,7 @@ int drm_atomic_helper_swap_state(struct drm_atomic_state *state,
 				  bool stall)
 {
 	int i, ret;
+	unsigned long flags;
 	struct drm_connector *connector;
 	struct drm_connector_state *old_conn_state, *new_conn_state;
 	struct drm_crtc *crtc;
@@ -3099,6 +3101,7 @@ int drm_atomic_helper_swap_state(struct drm_atomic_state *state,
 		}
 	}
 
+	drm_panic_lock(state->dev, flags);
 	for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) {
 		WARN_ON(plane->state != old_plane_state);
 
@@ -3108,6 +3111,7 @@ int drm_atomic_helper_swap_state(struct drm_atomic_state *state,
 		state->planes[i].state = old_plane_state;
 		plane->state = new_plane_state;
 	}
+	drm_panic_unlock(state->dev, flags);
 
 	for_each_oldnew_private_obj_in_state(state, obj, old_obj_state, new_obj_state, i) {
 		WARN_ON(obj->state != old_obj_state);
diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c
index 29d4940188d4..fc16fddee5c5 100644
--- a/drivers/gpu/drm/drm_atomic_uapi.c
+++ b/drivers/gpu/drm/drm_atomic_uapi.c
@@ -145,10 +145,10 @@ int drm_atomic_set_mode_prop_for_crtc(struct drm_crtc_state *state,
 					     &state->mode, blob->data);
 		if (ret) {
 			drm_dbg_atomic(crtc->dev,
-				       "[CRTC:%d:%s] invalid mode (ret=%d, status=%s):\n",
+				       "[CRTC:%d:%s] invalid mode (%s, %pe): " DRM_MODE_FMT "\n",
 				       crtc->base.id, crtc->name,
-				       ret, drm_get_mode_status_name(state->mode.status));
-			drm_mode_debug_printmodeline(&state->mode);
+				       drm_get_mode_status_name(state->mode.status),
+				       ERR_PTR(ret), DRM_MODE_ARG(&state->mode));
 			return -EINVAL;
 		}
 
diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c
index 521a71c61b16..28abe9aa99ca 100644
--- a/drivers/gpu/drm/drm_bridge.c
+++ b/drivers/gpu/drm/drm_bridge.c
@@ -657,6 +657,13 @@ static void drm_atomic_bridge_call_post_disable(struct drm_bridge *bridge,
  * bridge will be called before the previous one to reverse the @pre_enable
  * calling direction.
  *
+ * Example:
+ * Bridge A ---> Bridge B ---> Bridge C ---> Bridge D ---> Bridge E
+ *
+ * With pre_enable_prev_first flag enable in Bridge B, D, E then the resulting
+ * @post_disable order would be,
+ * Bridge B, Bridge A, Bridge E, Bridge D, Bridge C.
+ *
  * Note: the bridge passed should be the one closest to the encoder
  */
 void drm_atomic_bridge_chain_post_disable(struct drm_bridge *bridge,
@@ -687,11 +694,17 @@ void drm_atomic_bridge_chain_post_disable(struct drm_bridge *bridge,
 				 */
 				list_for_each_entry_from(next, &encoder->bridge_chain,
 							 chain_node) {
-					if (next->pre_enable_prev_first) {
+					if (!next->pre_enable_prev_first) {
 						next = list_prev_entry(next, chain_node);
 						limit = next;
 						break;
 					}
+
+					if (list_is_last(&next->chain_node,
+							 &encoder->bridge_chain)) {
+						limit = next;
+						break;
+					}
 				}
 
 				/* Call these bridges in reverse order */
@@ -747,6 +760,13 @@ static void drm_atomic_bridge_call_pre_enable(struct drm_bridge *bridge,
  * If a bridge sets @pre_enable_prev_first, then the pre_enable for the
  * prev bridge will be called before pre_enable of this bridge.
  *
+ * Example:
+ * Bridge A ---> Bridge B ---> Bridge C ---> Bridge D ---> Bridge E
+ *
+ * With pre_enable_prev_first flag enable in Bridge B, D, E then the resulting
+ * @pre_enable order would be,
+ * Bridge C, Bridge D, Bridge E, Bridge A, Bridge B.
+ *
  * Note: the bridge passed should be the one closest to the encoder
  */
 void drm_atomic_bridge_chain_pre_enable(struct drm_bridge *bridge,
@@ -774,7 +794,7 @@ void drm_atomic_bridge_chain_pre_enable(struct drm_bridge *bridge,
 					/* Found first bridge that does NOT
 					 * request prev to be enabled first
 					 */
-					limit = list_prev_entry(next, chain_node);
+					limit = next;
 					break;
 				}
 			}
diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c
index 5ebdd6f8f36e..6a8e45e9d0ec 100644
--- a/drivers/gpu/drm/drm_buddy.c
+++ b/drivers/gpu/drm/drm_buddy.c
@@ -57,6 +57,16 @@ static void list_insert_sorted(struct drm_buddy *mm,
 	__list_add(&block->link, node->link.prev, &node->link);
 }
 
+static void clear_reset(struct drm_buddy_block *block)
+{
+	block->header &= ~DRM_BUDDY_HEADER_CLEAR;
+}
+
+static void mark_cleared(struct drm_buddy_block *block)
+{
+	block->header |= DRM_BUDDY_HEADER_CLEAR;
+}
+
 static void mark_allocated(struct drm_buddy_block *block)
 {
 	block->header &= ~DRM_BUDDY_HEADER_STATE;
@@ -82,6 +92,133 @@ static void mark_split(struct drm_buddy_block *block)
 	list_del(&block->link);
 }
 
+static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2)
+{
+	return s1 <= e2 && e1 >= s2;
+}
+
+static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2)
+{
+	return s1 <= s2 && e1 >= e2;
+}
+
+static struct drm_buddy_block *
+__get_buddy(struct drm_buddy_block *block)
+{
+	struct drm_buddy_block *parent;
+
+	parent = block->parent;
+	if (!parent)
+		return NULL;
+
+	if (parent->left == block)
+		return parent->right;
+
+	return parent->left;
+}
+
+static unsigned int __drm_buddy_free(struct drm_buddy *mm,
+				     struct drm_buddy_block *block,
+				     bool force_merge)
+{
+	struct drm_buddy_block *parent;
+	unsigned int order;
+
+	while ((parent = block->parent)) {
+		struct drm_buddy_block *buddy;
+
+		buddy = __get_buddy(block);
+
+		if (!drm_buddy_block_is_free(buddy))
+			break;
+
+		if (!force_merge) {
+			/*
+			 * Check the block and its buddy clear state and exit
+			 * the loop if they both have the dissimilar state.
+			 */
+			if (drm_buddy_block_is_clear(block) !=
+			    drm_buddy_block_is_clear(buddy))
+				break;
+
+			if (drm_buddy_block_is_clear(block))
+				mark_cleared(parent);
+		}
+
+		list_del(&buddy->link);
+		if (force_merge && drm_buddy_block_is_clear(buddy))
+			mm->clear_avail -= drm_buddy_block_size(mm, buddy);
+
+		drm_block_free(mm, block);
+		drm_block_free(mm, buddy);
+
+		block = parent;
+	}
+
+	order = drm_buddy_block_order(block);
+	mark_free(mm, block);
+
+	return order;
+}
+
+static int __force_merge(struct drm_buddy *mm,
+			 u64 start,
+			 u64 end,
+			 unsigned int min_order)
+{
+	unsigned int order;
+	int i;
+
+	if (!min_order)
+		return -ENOMEM;
+
+	if (min_order > mm->max_order)
+		return -EINVAL;
+
+	for (i = min_order - 1; i >= 0; i--) {
+		struct drm_buddy_block *block, *prev;
+
+		list_for_each_entry_safe_reverse(block, prev, &mm->free_list[i], link) {
+			struct drm_buddy_block *buddy;
+			u64 block_start, block_end;
+
+			if (!block->parent)
+				continue;
+
+			block_start = drm_buddy_block_offset(block);
+			block_end = block_start + drm_buddy_block_size(mm, block) - 1;
+
+			if (!contains(start, end, block_start, block_end))
+				continue;
+
+			buddy = __get_buddy(block);
+			if (!drm_buddy_block_is_free(buddy))
+				continue;
+
+			WARN_ON(drm_buddy_block_is_clear(block) ==
+				drm_buddy_block_is_clear(buddy));
+
+			/*
+			 * If the prev block is same as buddy, don't access the
+			 * block in the next iteration as we would free the
+			 * buddy block as part of the free function.
+			 */
+			if (prev == buddy)
+				prev = list_prev_entry(prev, link);
+
+			list_del(&block->link);
+			if (drm_buddy_block_is_clear(block))
+				mm->clear_avail -= drm_buddy_block_size(mm, block);
+
+			order = __drm_buddy_free(mm, block, true);
+			if (order >= min_order)
+				return 0;
+		}
+	}
+
+	return -ENOMEM;
+}
+
 /**
  * drm_buddy_init - init memory manager
  *
@@ -102,7 +239,7 @@ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size)
 	if (size < chunk_size)
 		return -EINVAL;
 
-	if (chunk_size < PAGE_SIZE)
+	if (chunk_size < SZ_4K)
 		return -EINVAL;
 
 	if (!is_power_of_2(chunk_size))
@@ -112,6 +249,7 @@ int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size)
 
 	mm->size = size;
 	mm->avail = size;
+	mm->clear_avail = 0;
 	mm->chunk_size = chunk_size;
 	mm->max_order = ilog2(size) - ilog2(chunk_size);
 
@@ -186,11 +324,21 @@ EXPORT_SYMBOL(drm_buddy_init);
  */
 void drm_buddy_fini(struct drm_buddy *mm)
 {
+	u64 root_size, size;
+	unsigned int order;
 	int i;
 
+	size = mm->size;
+
 	for (i = 0; i < mm->n_roots; ++i) {
+		order = ilog2(size) - ilog2(mm->chunk_size);
+		__force_merge(mm, 0, size, order);
+
 		WARN_ON(!drm_buddy_block_is_free(mm->roots[i]));
 		drm_block_free(mm, mm->roots[i]);
+
+		root_size = mm->chunk_size << order;
+		size -= root_size;
 	}
 
 	WARN_ON(mm->avail != mm->size);
@@ -223,26 +371,17 @@ static int split_block(struct drm_buddy *mm,
 	mark_free(mm, block->left);
 	mark_free(mm, block->right);
 
+	if (drm_buddy_block_is_clear(block)) {
+		mark_cleared(block->left);
+		mark_cleared(block->right);
+		clear_reset(block);
+	}
+
 	mark_split(block);
 
 	return 0;
 }
 
-static struct drm_buddy_block *
-__get_buddy(struct drm_buddy_block *block)
-{
-	struct drm_buddy_block *parent;
-
-	parent = block->parent;
-	if (!parent)
-		return NULL;
-
-	if (parent->left == block)
-		return parent->right;
-
-	return parent->left;
-}
-
 /**
  * drm_get_buddy - get buddy address
  *
@@ -260,30 +399,6 @@ drm_get_buddy(struct drm_buddy_block *block)
 }
 EXPORT_SYMBOL(drm_get_buddy);
 
-static void __drm_buddy_free(struct drm_buddy *mm,
-			     struct drm_buddy_block *block)
-{
-	struct drm_buddy_block *parent;
-
-	while ((parent = block->parent)) {
-		struct drm_buddy_block *buddy;
-
-		buddy = __get_buddy(block);
-
-		if (!drm_buddy_block_is_free(buddy))
-			break;
-
-		list_del(&buddy->link);
-
-		drm_block_free(mm, block);
-		drm_block_free(mm, buddy);
-
-		block = parent;
-	}
-
-	mark_free(mm, block);
-}
-
 /**
  * drm_buddy_free_block - free a block
  *
@@ -295,42 +410,74 @@ void drm_buddy_free_block(struct drm_buddy *mm,
 {
 	BUG_ON(!drm_buddy_block_is_allocated(block));
 	mm->avail += drm_buddy_block_size(mm, block);
-	__drm_buddy_free(mm, block);
+	if (drm_buddy_block_is_clear(block))
+		mm->clear_avail += drm_buddy_block_size(mm, block);
+
+	__drm_buddy_free(mm, block, false);
 }
 EXPORT_SYMBOL(drm_buddy_free_block);
 
-/**
- * drm_buddy_free_list - free blocks
- *
- * @mm: DRM buddy manager
- * @objects: input list head to free blocks
- */
-void drm_buddy_free_list(struct drm_buddy *mm, struct list_head *objects)
+static void __drm_buddy_free_list(struct drm_buddy *mm,
+				  struct list_head *objects,
+				  bool mark_clear,
+				  bool mark_dirty)
 {
 	struct drm_buddy_block *block, *on;
 
+	WARN_ON(mark_dirty && mark_clear);
+
 	list_for_each_entry_safe(block, on, objects, link) {
+		if (mark_clear)
+			mark_cleared(block);
+		else if (mark_dirty)
+			clear_reset(block);
 		drm_buddy_free_block(mm, block);
 		cond_resched();
 	}
 	INIT_LIST_HEAD(objects);
 }
-EXPORT_SYMBOL(drm_buddy_free_list);
 
-static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2)
+static void drm_buddy_free_list_internal(struct drm_buddy *mm,
+					 struct list_head *objects)
 {
-	return s1 <= e2 && e1 >= s2;
+	/*
+	 * Don't touch the clear/dirty bit, since allocation is still internal
+	 * at this point. For example we might have just failed part of the
+	 * allocation.
+	 */
+	__drm_buddy_free_list(mm, objects, false, false);
 }
 
-static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2)
+/**
+ * drm_buddy_free_list - free blocks
+ *
+ * @mm: DRM buddy manager
+ * @objects: input list head to free blocks
+ * @flags: optional flags like DRM_BUDDY_CLEARED
+ */
+void drm_buddy_free_list(struct drm_buddy *mm,
+			 struct list_head *objects,
+			 unsigned int flags)
 {
-	return s1 <= s2 && e1 >= e2;
+	bool mark_clear = flags & DRM_BUDDY_CLEARED;
+
+	__drm_buddy_free_list(mm, objects, mark_clear, !mark_clear);
+}
+EXPORT_SYMBOL(drm_buddy_free_list);
+
+static bool block_incompatible(struct drm_buddy_block *block, unsigned int flags)
+{
+	bool needs_clear = flags & DRM_BUDDY_CLEAR_ALLOCATION;
+
+	return needs_clear != drm_buddy_block_is_clear(block);
 }
 
 static struct drm_buddy_block *
-alloc_range_bias(struct drm_buddy *mm,
-		 u64 start, u64 end,
-		 unsigned int order)
+__alloc_range_bias(struct drm_buddy *mm,
+		   u64 start, u64 end,
+		   unsigned int order,
+		   unsigned long flags,
+		   bool fallback)
 {
 	u64 req_size = mm->chunk_size << order;
 	struct drm_buddy_block *block;
@@ -377,6 +524,9 @@ alloc_range_bias(struct drm_buddy *mm,
 				continue;
 		}
 
+		if (!fallback && block_incompatible(block, flags))
+			continue;
+
 		if (contains(start, end, block_start, block_end) &&
 		    order == drm_buddy_block_order(block)) {
 			/*
@@ -410,30 +560,57 @@ err_undo:
 	if (buddy &&
 	    (drm_buddy_block_is_free(block) &&
 	     drm_buddy_block_is_free(buddy)))
-		__drm_buddy_free(mm, block);
+		__drm_buddy_free(mm, block, false);
 	return ERR_PTR(err);
 }
 
 static struct drm_buddy_block *
-get_maxblock(struct drm_buddy *mm, unsigned int order)
+__drm_buddy_alloc_range_bias(struct drm_buddy *mm,
+			     u64 start, u64 end,
+			     unsigned int order,
+			     unsigned long flags)
+{
+	struct drm_buddy_block *block;
+	bool fallback = false;
+
+	block = __alloc_range_bias(mm, start, end, order,
+				   flags, fallback);
+	if (IS_ERR(block))
+		return __alloc_range_bias(mm, start, end, order,
+					  flags, !fallback);
+
+	return block;
+}
+
+static struct drm_buddy_block *
+get_maxblock(struct drm_buddy *mm, unsigned int order,
+	     unsigned long flags)
 {
-	struct drm_buddy_block *max_block = NULL, *node;
+	struct drm_buddy_block *max_block = NULL, *block = NULL;
 	unsigned int i;
 
 	for (i = order; i <= mm->max_order; ++i) {
-		if (!list_empty(&mm->free_list[i])) {
-			node = list_last_entry(&mm->free_list[i],
-					       struct drm_buddy_block,
-					       link);
-			if (!max_block) {
-				max_block = node;
+		struct drm_buddy_block *tmp_block;
+
+		list_for_each_entry_reverse(tmp_block, &mm->free_list[i], link) {
+			if (block_incompatible(tmp_block, flags))
 				continue;
-			}
 
-			if (drm_buddy_block_offset(node) >
-			    drm_buddy_block_offset(max_block)) {
-				max_block = node;
-			}
+			block = tmp_block;
+			break;
+		}
+
+		if (!block)
+			continue;
+
+		if (!max_block) {
+			max_block = block;
+			continue;
+		}
+
+		if (drm_buddy_block_offset(block) >
+		    drm_buddy_block_offset(max_block)) {
+			max_block = block;
 		}
 	}
 
@@ -450,12 +627,30 @@ alloc_from_freelist(struct drm_buddy *mm,
 	int err;
 
 	if (flags & DRM_BUDDY_TOPDOWN_ALLOCATION) {
-		block = get_maxblock(mm, order);
+		block = get_maxblock(mm, order, flags);
 		if (block)
 			/* Store the obtained block order */
 			tmp = drm_buddy_block_order(block);
 	} else {
 		for (tmp = order; tmp <= mm->max_order; ++tmp) {
+			struct drm_buddy_block *tmp_block;
+
+			list_for_each_entry_reverse(tmp_block, &mm->free_list[tmp], link) {
+				if (block_incompatible(tmp_block, flags))
+					continue;
+
+				block = tmp_block;
+				break;
+			}
+
+			if (block)
+				break;
+		}
+	}
+
+	if (!block) {
+		/* Fallback method */
+		for (tmp = order; tmp <= mm->max_order; ++tmp) {
 			if (!list_empty(&mm->free_list[tmp])) {
 				block = list_last_entry(&mm->free_list[tmp],
 							struct drm_buddy_block,
@@ -464,10 +659,10 @@ alloc_from_freelist(struct drm_buddy *mm,
 					break;
 			}
 		}
-	}
 
-	if (!block)
-		return ERR_PTR(-ENOSPC);
+		if (!block)
+			return ERR_PTR(-ENOSPC);
+	}
 
 	BUG_ON(!drm_buddy_block_is_free(block));
 
@@ -483,7 +678,7 @@ alloc_from_freelist(struct drm_buddy *mm,
 
 err_undo:
 	if (tmp != order)
-		__drm_buddy_free(mm, block);
+		__drm_buddy_free(mm, block, false);
 	return ERR_PTR(err);
 }
 
@@ -526,16 +721,18 @@ static int __alloc_range(struct drm_buddy *mm,
 		}
 
 		if (contains(start, end, block_start, block_end)) {
-			if (!drm_buddy_block_is_free(block)) {
+			if (drm_buddy_block_is_free(block)) {
+				mark_allocated(block);
+				total_allocated += drm_buddy_block_size(mm, block);
+				mm->avail -= drm_buddy_block_size(mm, block);
+				if (drm_buddy_block_is_clear(block))
+					mm->clear_avail -= drm_buddy_block_size(mm, block);
+				list_add_tail(&block->link, &allocated);
+				continue;
+			} else if (!mm->clear_avail) {
 				err = -ENOSPC;
 				goto err_free;
 			}
-
-			mark_allocated(block);
-			total_allocated += drm_buddy_block_size(mm, block);
-			mm->avail -= drm_buddy_block_size(mm, block);
-			list_add_tail(&block->link, &allocated);
-			continue;
 		}
 
 		if (!drm_buddy_block_is_split(block)) {
@@ -567,14 +764,14 @@ err_undo:
 	if (buddy &&
 	    (drm_buddy_block_is_free(block) &&
 	     drm_buddy_block_is_free(buddy)))
-		__drm_buddy_free(mm, block);
+		__drm_buddy_free(mm, block, false);
 
 err_free:
 	if (err == -ENOSPC && total_allocated_on_err) {
 		list_splice_tail(&allocated, blocks);
 		*total_allocated_on_err = total_allocated;
 	} else {
-		drm_buddy_free_list(mm, &allocated);
+		drm_buddy_free_list_internal(mm, &allocated);
 	}
 
 	return err;
@@ -640,11 +837,11 @@ static int __alloc_contig_try_harder(struct drm_buddy *mm,
 			list_splice(&blocks_lhs, blocks);
 			return 0;
 		} else if (err != -ENOSPC) {
-			drm_buddy_free_list(mm, blocks);
+			drm_buddy_free_list_internal(mm, blocks);
 			return err;
 		}
 		/* Free blocks for the next iteration */
-		drm_buddy_free_list(mm, blocks);
+		drm_buddy_free_list_internal(mm, blocks);
 	}
 
 	return -ENOSPC;
@@ -700,6 +897,8 @@ int drm_buddy_block_trim(struct drm_buddy *mm,
 	list_del(&block->link);
 	mark_free(mm, block);
 	mm->avail += drm_buddy_block_size(mm, block);
+	if (drm_buddy_block_is_clear(block))
+		mm->clear_avail += drm_buddy_block_size(mm, block);
 
 	/* Prevent recursively freeing this node */
 	parent = block->parent;
@@ -711,6 +910,8 @@ int drm_buddy_block_trim(struct drm_buddy *mm,
 	if (err) {
 		mark_allocated(block);
 		mm->avail -= drm_buddy_block_size(mm, block);
+		if (drm_buddy_block_is_clear(block))
+			mm->clear_avail -= drm_buddy_block_size(mm, block);
 		list_add(&block->link, blocks);
 	}
 
@@ -719,13 +920,28 @@ int drm_buddy_block_trim(struct drm_buddy *mm,
 }
 EXPORT_SYMBOL(drm_buddy_block_trim);
 
+static struct drm_buddy_block *
+__drm_buddy_alloc_blocks(struct drm_buddy *mm,
+			 u64 start, u64 end,
+			 unsigned int order,
+			 unsigned long flags)
+{
+	if (flags & DRM_BUDDY_RANGE_ALLOCATION)
+		/* Allocate traversing within the range */
+		return  __drm_buddy_alloc_range_bias(mm, start, end,
+						     order, flags);
+	else
+		/* Allocate from freelist */
+		return alloc_from_freelist(mm, order, flags);
+}
+
 /**
  * drm_buddy_alloc_blocks - allocate power-of-two blocks
  *
  * @mm: DRM buddy manager to allocate from
  * @start: start of the allowed range for this block
  * @end: end of the allowed range for this block
- * @size: size of the allocation
+ * @size: size of the allocation in bytes
  * @min_block_size: alignment of the allocation
  * @blocks: output list head to add allocated blocks
  * @flags: DRM_BUDDY_*_ALLOCATION flags
@@ -800,23 +1016,33 @@ int drm_buddy_alloc_blocks(struct drm_buddy *mm,
 		BUG_ON(order < min_order);
 
 		do {
-			if (flags & DRM_BUDDY_RANGE_ALLOCATION)
-				/* Allocate traversing within the range */
-				block = alloc_range_bias(mm, start, end, order);
-			else
-				/* Allocate from freelist */
-				block = alloc_from_freelist(mm, order, flags);
-
+			block = __drm_buddy_alloc_blocks(mm, start,
+							 end,
+							 order,
+							 flags);
 			if (!IS_ERR(block))
 				break;
 
 			if (order-- == min_order) {
+				/* Try allocation through force merge method */
+				if (mm->clear_avail &&
+				    !__force_merge(mm, start, end, min_order)) {
+					block = __drm_buddy_alloc_blocks(mm, start,
+									 end,
+									 min_order,
+									 flags);
+					if (!IS_ERR(block)) {
+						order = min_order;
+						break;
+					}
+				}
+
+				/*
+				 * Try contiguous block allocation through
+				 * try harder method.
+				 */
 				if (flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION &&
 				    !(flags & DRM_BUDDY_RANGE_ALLOCATION))
-					/*
-					 * Try contiguous block allocation through
-					 * try harder method
-					 */
 					return __alloc_contig_try_harder(mm,
 									 original_size,
 									 original_min_size,
@@ -828,6 +1054,8 @@ int drm_buddy_alloc_blocks(struct drm_buddy *mm,
 
 		mark_allocated(block);
 		mm->avail -= drm_buddy_block_size(mm, block);
+		if (drm_buddy_block_is_clear(block))
+			mm->clear_avail -= drm_buddy_block_size(mm, block);
 		kmemleak_update_trace(block);
 		list_add_tail(&block->link, &allocated);
 
@@ -866,7 +1094,7 @@ int drm_buddy_alloc_blocks(struct drm_buddy *mm,
 	return 0;
 
 err_free:
-	drm_buddy_free_list(mm, &allocated);
+	drm_buddy_free_list_internal(mm, &allocated);
 	return err;
 }
 EXPORT_SYMBOL(drm_buddy_alloc_blocks);
@@ -899,8 +1127,8 @@ void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p)
 {
 	int order;
 
-	drm_printf(p, "chunk_size: %lluKiB, total: %lluMiB, free: %lluMiB\n",
-		   mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20);
+	drm_printf(p, "chunk_size: %lluKiB, total: %lluMiB, free: %lluMiB, clear_free: %lluMiB\n",
+		   mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20);
 
 	for (order = mm->max_order; order >= 0; order--) {
 		struct drm_buddy_block *block;
diff --git a/drivers/gpu/drm/drm_client.c b/drivers/gpu/drm/drm_client.c
index 9403b3f576f7..2803ac111bbd 100644
--- a/drivers/gpu/drm/drm_client.c
+++ b/drivers/gpu/drm/drm_client.c
@@ -172,6 +172,18 @@ void drm_client_release(struct drm_client_dev *client)
 }
 EXPORT_SYMBOL(drm_client_release);
 
+/**
+ * drm_client_dev_unregister - Unregister clients
+ * @dev: DRM device
+ *
+ * This function releases all clients by calling each client's
+ * &drm_client_funcs.unregister callback. The callback function
+ * is responsibe for releaseing all resources including the client
+ * itself.
+ *
+ * The helper drm_dev_unregister() calls this function. Drivers
+ * that use it don't need to call this function themselves.
+ */
 void drm_client_dev_unregister(struct drm_device *dev)
 {
 	struct drm_client_dev *client, *tmp;
@@ -191,6 +203,7 @@ void drm_client_dev_unregister(struct drm_device *dev)
 	}
 	mutex_unlock(&dev->clientlist_mutex);
 }
+EXPORT_SYMBOL(drm_client_dev_unregister);
 
 /**
  * drm_client_dev_hotplug - Send hotplug event to clients
@@ -305,6 +318,66 @@ err_delete:
 }
 
 /**
+ * drm_client_buffer_vmap_local - Map DRM client buffer into address space
+ * @buffer: DRM client buffer
+ * @map_copy: Returns the mapped memory's address
+ *
+ * This function maps a client buffer into kernel address space. If the
+ * buffer is already mapped, it returns the existing mapping's address.
+ *
+ * Client buffer mappings are not ref'counted. Each call to
+ * drm_client_buffer_vmap_local() should be closely followed by a call to
+ * drm_client_buffer_vunmap_local(). See drm_client_buffer_vmap() for
+ * long-term mappings.
+ *
+ * The returned address is a copy of the internal value. In contrast to
+ * other vmap interfaces, you don't need it for the client's vunmap
+ * function. So you can modify it at will during blit and draw operations.
+ *
+ * Returns:
+ *	0 on success, or a negative errno code otherwise.
+ */
+int drm_client_buffer_vmap_local(struct drm_client_buffer *buffer,
+				 struct iosys_map *map_copy)
+{
+	struct drm_gem_object *gem = buffer->gem;
+	struct iosys_map *map = &buffer->map;
+	int ret;
+
+	drm_gem_lock(gem);
+
+	ret = drm_gem_vmap(gem, map);
+	if (ret)
+		goto err_drm_gem_vmap_unlocked;
+	*map_copy = *map;
+
+	return 0;
+
+err_drm_gem_vmap_unlocked:
+	drm_gem_unlock(gem);
+	return 0;
+}
+EXPORT_SYMBOL(drm_client_buffer_vmap_local);
+
+/**
+ * drm_client_buffer_vunmap_local - Unmap DRM client buffer
+ * @buffer: DRM client buffer
+ *
+ * This function removes a client buffer's memory mapping established
+ * with drm_client_buffer_vunmap_local(). Calling this function is only
+ * required by clients that manage their buffer mappings by themselves.
+ */
+void drm_client_buffer_vunmap_local(struct drm_client_buffer *buffer)
+{
+	struct drm_gem_object *gem = buffer->gem;
+	struct iosys_map *map = &buffer->map;
+
+	drm_gem_vunmap(gem, map);
+	drm_gem_unlock(gem);
+}
+EXPORT_SYMBOL(drm_client_buffer_vunmap_local);
+
+/**
  * drm_client_buffer_vmap - Map DRM client buffer into address space
  * @buffer: DRM client buffer
  * @map_copy: Returns the mapped memory's address
@@ -328,24 +401,30 @@ int
 drm_client_buffer_vmap(struct drm_client_buffer *buffer,
 		       struct iosys_map *map_copy)
 {
+	struct drm_gem_object *gem = buffer->gem;
 	struct iosys_map *map = &buffer->map;
 	int ret;
 
-	/*
-	 * FIXME: The dependency on GEM here isn't required, we could
-	 * convert the driver handle to a dma-buf instead and use the
-	 * backend-agnostic dma-buf vmap support instead. This would
-	 * require that the handle2fd prime ioctl is reworked to pull the
-	 * fd_install step out of the driver backend hooks, to make that
-	 * final step optional for internal users.
-	 */
-	ret = drm_gem_vmap_unlocked(buffer->gem, map);
+	drm_gem_lock(gem);
+
+	ret = drm_gem_pin_locked(gem);
 	if (ret)
-		return ret;
+		goto err_drm_gem_pin_locked;
+	ret = drm_gem_vmap(gem, map);
+	if (ret)
+		goto err_drm_gem_vmap;
+
+	drm_gem_unlock(gem);
 
 	*map_copy = *map;
 
 	return 0;
+
+err_drm_gem_vmap:
+	drm_gem_unpin_locked(buffer->gem);
+err_drm_gem_pin_locked:
+	drm_gem_unlock(gem);
+	return ret;
 }
 EXPORT_SYMBOL(drm_client_buffer_vmap);
 
@@ -359,9 +438,13 @@ EXPORT_SYMBOL(drm_client_buffer_vmap);
  */
 void drm_client_buffer_vunmap(struct drm_client_buffer *buffer)
 {
+	struct drm_gem_object *gem = buffer->gem;
 	struct iosys_map *map = &buffer->map;
 
-	drm_gem_vunmap_unlocked(buffer->gem, map);
+	drm_gem_lock(gem);
+	drm_gem_vunmap(gem, map);
+	drm_gem_unpin_locked(gem);
+	drm_gem_unlock(gem);
 }
 EXPORT_SYMBOL(drm_client_buffer_vunmap);
 
diff --git a/drivers/gpu/drm/drm_client_modeset.c b/drivers/gpu/drm/drm_client_modeset.c
index 0683a129b362..31af5cf37a09 100644
--- a/drivers/gpu/drm/drm_client_modeset.c
+++ b/drivers/gpu/drm/drm_client_modeset.c
@@ -242,8 +242,10 @@ static void drm_client_connectors_enabled(struct drm_connector **connectors,
 	for (i = 0; i < connector_count; i++) {
 		connector = connectors[i];
 		enabled[i] = drm_connector_enabled(connector, true);
-		DRM_DEBUG_KMS("connector %d enabled? %s\n", connector->base.id,
-			      connector->display_info.non_desktop ? "non desktop" : str_yes_no(enabled[i]));
+		drm_dbg_kms(connector->dev, "[CONNECTOR:%d:%s] enabled? %s\n",
+			    connector->base.id, connector->name,
+			    connector->display_info.non_desktop ?
+			    "non desktop" : str_yes_no(enabled[i]));
 
 		any_enabled |= enabled[i];
 	}
@@ -303,7 +305,7 @@ static bool drm_client_target_cloned(struct drm_device *dev,
 	}
 
 	if (can_clone) {
-		DRM_DEBUG_KMS("can clone using command line\n");
+		drm_dbg_kms(dev, "can clone using command line\n");
 		return true;
 	}
 
@@ -332,15 +334,16 @@ static bool drm_client_target_cloned(struct drm_device *dev,
 	kfree(dmt_mode);
 
 	if (can_clone) {
-		DRM_DEBUG_KMS("can clone using 1024x768\n");
+		drm_dbg_kms(dev, "can clone using 1024x768\n");
 		return true;
 	}
 fail:
-	DRM_INFO("kms: can't enable cloning when we probably wanted to.\n");
+	drm_info(dev, "kms: can't enable cloning when we probably wanted to.\n");
 	return false;
 }
 
-static int drm_client_get_tile_offsets(struct drm_connector **connectors,
+static int drm_client_get_tile_offsets(struct drm_device *dev,
+				       struct drm_connector **connectors,
 				       unsigned int connector_count,
 				       struct drm_display_mode **modes,
 				       struct drm_client_offset *offsets,
@@ -357,8 +360,9 @@ static int drm_client_get_tile_offsets(struct drm_connector **connectors,
 			continue;
 
 		if (!modes[i] && (h_idx || v_idx)) {
-			DRM_DEBUG_KMS("no modes for connector tiled %d %d\n", i,
-				      connector->base.id);
+			drm_dbg_kms(dev,
+				    "[CONNECTOR:%d:%s] no modes for connector tiled %d\n",
+				    connector->base.id, connector->name, i);
 			continue;
 		}
 		if (connector->tile_h_loc < h_idx)
@@ -369,11 +373,12 @@ static int drm_client_get_tile_offsets(struct drm_connector **connectors,
 	}
 	offsets[idx].x = hoffset;
 	offsets[idx].y = voffset;
-	DRM_DEBUG_KMS("returned %d %d for %d %d\n", hoffset, voffset, h_idx, v_idx);
+	drm_dbg_kms(dev, "returned %d %d for %d %d\n", hoffset, voffset, h_idx, v_idx);
 	return 0;
 }
 
-static bool drm_client_target_preferred(struct drm_connector **connectors,
+static bool drm_client_target_preferred(struct drm_device *dev,
+					struct drm_connector **connectors,
 					unsigned int connector_count,
 					struct drm_display_mode **modes,
 					struct drm_client_offset *offsets,
@@ -423,17 +428,19 @@ retry:
 			 * find the tile offsets for this pass - need to find
 			 * all tiles left and above
 			 */
-			drm_client_get_tile_offsets(connectors, connector_count, modes, offsets, i,
+			drm_client_get_tile_offsets(dev, connectors, connector_count,
+						    modes, offsets, i,
 						    connector->tile_h_loc, connector->tile_v_loc);
 		}
-		DRM_DEBUG_KMS("looking for cmdline mode on connector %d\n",
-			      connector->base.id);
+		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] looking for cmdline mode\n",
+			    connector->base.id, connector->name);
 
 		/* got for command line mode first */
 		modes[i] = drm_connector_pick_cmdline_mode(connector);
 		if (!modes[i]) {
-			DRM_DEBUG_KMS("looking for preferred mode on connector %d %d\n",
-				      connector->base.id, connector->tile_group ? connector->tile_group->id : 0);
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] looking for preferred mode, tile %d\n",
+				    connector->base.id, connector->name,
+				    connector->tile_group ? connector->tile_group->id : 0);
 			modes[i] = drm_connector_has_preferred_mode(connector, width, height);
 		}
 		/* No preferred modes, pick one off the list */
@@ -455,16 +462,18 @@ retry:
 			    (connector->tile_h_loc == 0 &&
 			     connector->tile_v_loc == 0 &&
 			     !drm_connector_get_tiled_mode(connector))) {
-				DRM_DEBUG_KMS("Falling back to non tiled mode on Connector %d\n",
-					      connector->base.id);
+				drm_dbg_kms(dev,
+					    "[CONNECTOR:%d:%s] Falling back to non-tiled mode\n",
+					    connector->base.id, connector->name);
 				modes[i] = drm_connector_fallback_non_tiled_mode(connector);
 			} else {
 				modes[i] = drm_connector_get_tiled_mode(connector);
 			}
 		}
 
-		DRM_DEBUG_KMS("found mode %s\n", modes[i] ? modes[i]->name :
-			  "none");
+		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Found mode %s\n",
+			    connector->base.id, connector->name,
+			    modes[i] ? modes[i]->name : "none");
 		conn_configured |= BIT_ULL(i);
 	}
 
@@ -585,7 +594,7 @@ static bool drm_client_firmware_config(struct drm_client_dev *client,
 	if (!drm_drv_uses_atomic_modeset(dev))
 		return false;
 
-	if (WARN_ON(count <= 0))
+	if (drm_WARN_ON(dev, count <= 0))
 		return false;
 
 	save_enabled = kcalloc(count, sizeof(bool), GFP_KERNEL);
@@ -624,26 +633,26 @@ retry:
 			num_connectors_detected++;
 
 		if (!enabled[i]) {
-			DRM_DEBUG_KMS("connector %s not enabled, skipping\n",
-				      connector->name);
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] not enabled, skipping\n",
+				    connector->base.id, connector->name);
 			conn_configured |= BIT(i);
 			continue;
 		}
 
 		if (connector->force == DRM_FORCE_OFF) {
-			DRM_DEBUG_KMS("connector %s is disabled by user, skipping\n",
-				      connector->name);
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] disabled by user, skipping\n",
+				    connector->base.id, connector->name);
 			enabled[i] = false;
 			continue;
 		}
 
 		encoder = connector->state->best_encoder;
-		if (!encoder || WARN_ON(!connector->state->crtc)) {
+		if (!encoder || drm_WARN_ON(dev, !connector->state->crtc)) {
 			if (connector->force > DRM_FORCE_OFF)
 				goto bail;
 
-			DRM_DEBUG_KMS("connector %s has no encoder or crtc, skipping\n",
-				      connector->name);
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] has no encoder or crtc, skipping\n",
+				    connector->base.id, connector->name);
 			enabled[i] = false;
 			conn_configured |= BIT(i);
 			continue;
@@ -660,28 +669,30 @@ retry:
 		 */
 		for (j = 0; j < count; j++) {
 			if (crtcs[j] == new_crtc) {
-				DRM_DEBUG_KMS("fallback: cloned configuration\n");
+				drm_dbg_kms(dev, "fallback: cloned configuration\n");
 				goto bail;
 			}
 		}
 
-		DRM_DEBUG_KMS("looking for cmdline mode on connector %s\n",
-			      connector->name);
+		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] looking for cmdline mode\n",
+			    connector->base.id, connector->name);
 
 		/* go for command line mode first */
 		modes[i] = drm_connector_pick_cmdline_mode(connector);
 
 		/* try for preferred next */
 		if (!modes[i]) {
-			DRM_DEBUG_KMS("looking for preferred mode on connector %s %d\n",
-				      connector->name, connector->has_tile);
+			drm_dbg_kms(dev,
+				    "[CONNECTOR:%d:%s] looking for preferred mode, has tile: %s\n",
+				    connector->base.id, connector->name,
+				    str_yes_no(connector->has_tile));
 			modes[i] = drm_connector_has_preferred_mode(connector, width, height);
 		}
 
 		/* No preferred mode marked by the EDID? Are there any modes? */
 		if (!modes[i] && !list_empty(&connector->modes)) {
-			DRM_DEBUG_KMS("using first mode listed on connector %s\n",
-				      connector->name);
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] using first listed mode\n",
+				    connector->base.id, connector->name);
 			modes[i] = list_first_entry(&connector->modes,
 						    struct drm_display_mode,
 						    head);
@@ -700,8 +711,8 @@ retry:
 			 * This is crtc->mode and not crtc->state->mode for the
 			 * fastboot check to work correctly.
 			 */
-			DRM_DEBUG_KMS("looking for current mode on connector %s\n",
-				      connector->name);
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] looking for current mode\n",
+				    connector->base.id, connector->name);
 			modes[i] = &connector->state->crtc->mode;
 		}
 		/*
@@ -710,18 +721,18 @@ retry:
 		 */
 		if (connector->has_tile &&
 		    num_tiled_conns < connector->num_h_tile * connector->num_v_tile) {
-			DRM_DEBUG_KMS("Falling back to non tiled mode on Connector %d\n",
-				      connector->base.id);
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Falling back to non-tiled mode\n",
+				    connector->base.id, connector->name);
 			modes[i] = drm_connector_fallback_non_tiled_mode(connector);
 		}
 		crtcs[i] = new_crtc;
 
-		DRM_DEBUG_KMS("connector %s on [CRTC:%d:%s]: %dx%d%s\n",
-			      connector->name,
-			      connector->state->crtc->base.id,
-			      connector->state->crtc->name,
-			      modes[i]->hdisplay, modes[i]->vdisplay,
-			      modes[i]->flags & DRM_MODE_FLAG_INTERLACE ? "i" : "");
+		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] on [CRTC:%d:%s]: %dx%d%s\n",
+			    connector->base.id, connector->name,
+			    connector->state->crtc->base.id,
+			    connector->state->crtc->name,
+			    modes[i]->hdisplay, modes[i]->vdisplay,
+			    modes[i]->flags & DRM_MODE_FLAG_INTERLACE ? "i" : "");
 
 		fallback = false;
 		conn_configured |= BIT(i);
@@ -737,15 +748,15 @@ retry:
 	 */
 	if (num_connectors_enabled != num_connectors_detected &&
 	    num_connectors_enabled < dev->mode_config.num_crtc) {
-		DRM_DEBUG_KMS("fallback: Not all outputs enabled\n");
-		DRM_DEBUG_KMS("Enabled: %i, detected: %i\n", num_connectors_enabled,
-			      num_connectors_detected);
+		drm_dbg_kms(dev, "fallback: Not all outputs enabled\n");
+		drm_dbg_kms(dev, "Enabled: %i, detected: %i\n",
+			    num_connectors_enabled, num_connectors_detected);
 		fallback = true;
 	}
 
 	if (fallback) {
 bail:
-		DRM_DEBUG_KMS("Not using firmware configuration\n");
+		drm_dbg_kms(dev, "Not using firmware configuration\n");
 		memcpy(enabled, save_enabled, count);
 		ret = false;
 	}
@@ -783,7 +794,7 @@ int drm_client_modeset_probe(struct drm_client_dev *client, unsigned int width,
 	int i, ret = 0;
 	bool *enabled;
 
-	DRM_DEBUG_KMS("\n");
+	drm_dbg_kms(dev, "\n");
 
 	if (!width)
 		width = dev->mode_config.max_width;
@@ -814,7 +825,6 @@ int drm_client_modeset_probe(struct drm_client_dev *client, unsigned int width,
 	offsets = kcalloc(connector_count, sizeof(*offsets), GFP_KERNEL);
 	enabled = kcalloc(connector_count, sizeof(bool), GFP_KERNEL);
 	if (!crtcs || !modes || !enabled || !offsets) {
-		DRM_ERROR("Memory allocation failed\n");
 		ret = -ENOMEM;
 		goto out;
 	}
@@ -825,7 +835,7 @@ int drm_client_modeset_probe(struct drm_client_dev *client, unsigned int width,
 	for (i = 0; i < connector_count; i++)
 		total_modes_count += connectors[i]->funcs->fill_modes(connectors[i], width, height);
 	if (!total_modes_count)
-		DRM_DEBUG_KMS("No connectors reported connected with modes\n");
+		drm_dbg_kms(dev, "No connectors reported connected with modes\n");
 	drm_client_connectors_enabled(connectors, connector_count, enabled);
 
 	if (!drm_client_firmware_config(client, connectors, connector_count, crtcs,
@@ -836,12 +846,12 @@ int drm_client_modeset_probe(struct drm_client_dev *client, unsigned int width,
 
 		if (!drm_client_target_cloned(dev, connectors, connector_count, modes,
 					      offsets, enabled, width, height) &&
-		    !drm_client_target_preferred(connectors, connector_count, modes,
+		    !drm_client_target_preferred(dev, connectors, connector_count, modes,
 						 offsets, enabled, width, height))
-			DRM_ERROR("Unable to find initial modes\n");
+			drm_err(dev, "Unable to find initial modes\n");
 
-		DRM_DEBUG_KMS("picking CRTCs for %dx%d config\n",
-			      width, height);
+		drm_dbg_kms(dev, "picking CRTCs for %dx%d config\n",
+			    width, height);
 
 		drm_client_pick_crtcs(client, connectors, connector_count,
 				      crtcs, modes, 0, width, height);
@@ -858,11 +868,12 @@ int drm_client_modeset_probe(struct drm_client_dev *client, unsigned int width,
 			struct drm_mode_set *modeset = drm_client_find_modeset(client, crtc);
 			struct drm_connector *connector = connectors[i];
 
-			DRM_DEBUG_KMS("desired mode %s set on crtc %d (%d,%d)\n",
-				      mode->name, crtc->base.id, offset->x, offset->y);
+			drm_dbg_kms(dev, "[CRTC:%d:%s] desired mode %s set (%d,%d)\n",
+				    crtc->base.id, crtc->name,
+				    mode->name, offset->x, offset->y);
 
-			if (WARN_ON_ONCE(modeset->num_connectors == DRM_CLIENT_MAX_CLONED_CONNECTORS ||
-					 (dev->mode_config.num_crtc > 1 && modeset->num_connectors == 1))) {
+			if (drm_WARN_ON_ONCE(dev, modeset->num_connectors == DRM_CLIENT_MAX_CLONED_CONNECTORS ||
+					     (dev->mode_config.num_crtc > 1 && modeset->num_connectors == 1))) {
 				ret = -EINVAL;
 				break;
 			}
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index b0516505f7ae..4d2df7f64dc5 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -2940,7 +2940,7 @@ int drm_mode_getconnector(struct drm_device *dev, void *data,
 						     dev->mode_config.max_width,
 						     dev->mode_config.max_height);
 		else
-			drm_dbg_kms(dev, "User-space requested a forced probe on [CONNECTOR:%d:%s] but is not the DRM master, demoting to read-only probe",
+			drm_dbg_kms(dev, "User-space requested a forced probe on [CONNECTOR:%d:%s] but is not the DRM master, demoting to read-only probe\n",
 				    connector->base.id, connector->name);
 	}
 
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 82c665d3e74b..483969b84a30 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -716,10 +716,10 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 
 	crtc = drm_crtc_find(dev, file_priv, crtc_req->crtc_id);
 	if (!crtc) {
-		DRM_DEBUG_KMS("Unknown CRTC ID %d\n", crtc_req->crtc_id);
+		drm_dbg_kms(dev, "Unknown CRTC ID %d\n", crtc_req->crtc_id);
 		return -ENOENT;
 	}
-	DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
+	drm_dbg_kms(dev, "[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
 
 	plane = crtc->primary;
 
@@ -742,7 +742,7 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 				old_fb = plane->fb;
 
 			if (!old_fb) {
-				DRM_DEBUG_KMS("CRTC doesn't have current FB\n");
+				drm_dbg_kms(dev, "CRTC doesn't have current FB\n");
 				ret = -EINVAL;
 				goto out;
 			}
@@ -753,8 +753,8 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 		} else {
 			fb = drm_framebuffer_lookup(dev, file_priv, crtc_req->fb_id);
 			if (!fb) {
-				DRM_DEBUG_KMS("Unknown FB ID%d\n",
-						crtc_req->fb_id);
+				drm_dbg_kms(dev, "Unknown FB ID%d\n",
+					    crtc_req->fb_id);
 				ret = -ENOENT;
 				goto out;
 			}
@@ -767,7 +767,7 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 		}
 		if (!file_priv->aspect_ratio_allowed &&
 		    (crtc_req->mode.flags & DRM_MODE_FLAG_PIC_AR_MASK) != DRM_MODE_FLAG_PIC_AR_NONE) {
-			DRM_DEBUG_KMS("Unexpected aspect-ratio flag bits\n");
+			drm_dbg_kms(dev, "Unexpected aspect-ratio flag bits\n");
 			ret = -EINVAL;
 			goto out;
 		}
@@ -775,9 +775,9 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 
 		ret = drm_mode_convert_umode(dev, mode, &crtc_req->mode);
 		if (ret) {
-			DRM_DEBUG_KMS("Invalid mode (ret=%d, status=%s)\n",
-				      ret, drm_get_mode_status_name(mode->status));
-			drm_mode_debug_printmodeline(mode);
+			drm_dbg_kms(dev, "Invalid mode (%s, %pe): " DRM_MODE_FMT "\n",
+				    drm_get_mode_status_name(mode->status),
+				    ERR_PTR(ret), DRM_MODE_ARG(mode));
 			goto out;
 		}
 
@@ -793,9 +793,8 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 							   fb->format->format,
 							   fb->modifier);
 			if (ret) {
-				DRM_DEBUG_KMS("Invalid pixel format %p4cc, modifier 0x%llx\n",
-					      &fb->format->format,
-					      fb->modifier);
+				drm_dbg_kms(dev, "Invalid pixel format %p4cc, modifier 0x%llx\n",
+					    &fb->format->format, fb->modifier);
 				goto out;
 			}
 		}
@@ -808,14 +807,14 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 	}
 
 	if (crtc_req->count_connectors == 0 && mode) {
-		DRM_DEBUG_KMS("Count connectors is 0 but mode set\n");
+		drm_dbg_kms(dev, "Count connectors is 0 but mode set\n");
 		ret = -EINVAL;
 		goto out;
 	}
 
 	if (crtc_req->count_connectors > 0 && (!mode || !fb)) {
-		DRM_DEBUG_KMS("Count connectors is %d but no mode or fb set\n",
-			  crtc_req->count_connectors);
+		drm_dbg_kms(dev, "Count connectors is %d but no mode or fb set\n",
+			    crtc_req->count_connectors);
 		ret = -EINVAL;
 		goto out;
 	}
@@ -847,14 +846,13 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 
 			connector = drm_connector_lookup(dev, file_priv, out_id);
 			if (!connector) {
-				DRM_DEBUG_KMS("Connector id %d unknown\n",
-						out_id);
+				drm_dbg_kms(dev, "Connector id %d unknown\n",
+					    out_id);
 				ret = -ENOENT;
 				goto out;
 			}
-			DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
-					connector->base.id,
-					connector->name);
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s]\n",
+				    connector->base.id, connector->name);
 
 			connector_set[i] = connector;
 			num_connectors++;
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 2dafc39a27cb..0955f1c385dd 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -110,15 +110,15 @@ bool drm_helper_encoder_in_use(struct drm_encoder *encoder)
 	struct drm_connector_list_iter conn_iter;
 	struct drm_device *dev = encoder->dev;
 
-	WARN_ON(drm_drv_uses_atomic_modeset(dev));
+	drm_WARN_ON(dev, drm_drv_uses_atomic_modeset(dev));
 
 	/*
 	 * We can expect this mutex to be locked if we are not panicking.
 	 * Locking is currently fubar in the panic handler.
 	 */
 	if (!oops_in_progress) {
-		WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
-		WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
+		drm_WARN_ON(dev, !mutex_is_locked(&dev->mode_config.mutex));
+		drm_WARN_ON(dev, !drm_modeset_is_locked(&dev->mode_config.connection_mutex));
 	}
 
 
@@ -150,14 +150,14 @@ bool drm_helper_crtc_in_use(struct drm_crtc *crtc)
 	struct drm_encoder *encoder;
 	struct drm_device *dev = crtc->dev;
 
-	WARN_ON(drm_drv_uses_atomic_modeset(dev));
+	drm_WARN_ON(dev, drm_drv_uses_atomic_modeset(dev));
 
 	/*
 	 * We can expect this mutex to be locked if we are not panicking.
 	 * Locking is currently fubar in the panic handler.
 	 */
 	if (!oops_in_progress)
-		WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
+		drm_WARN_ON(dev, !mutex_is_locked(&dev->mode_config.mutex));
 
 	drm_for_each_encoder(encoder, dev)
 		if (encoder->crtc == crtc && drm_helper_encoder_in_use(encoder))
@@ -230,7 +230,7 @@ static void __drm_helper_disable_unused_functions(struct drm_device *dev)
  */
 void drm_helper_disable_unused_functions(struct drm_device *dev)
 {
-	WARN_ON(drm_drv_uses_atomic_modeset(dev));
+	drm_WARN_ON(dev, drm_drv_uses_atomic_modeset(dev));
 
 	drm_modeset_lock_all(dev);
 	__drm_helper_disable_unused_functions(dev);
@@ -294,7 +294,7 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
 	struct drm_encoder *encoder;
 	bool ret = true;
 
-	WARN_ON(drm_drv_uses_atomic_modeset(dev));
+	drm_WARN_ON(dev, drm_drv_uses_atomic_modeset(dev));
 
 	drm_warn_on_modeset_not_all_locked(dev);
 
@@ -338,7 +338,8 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
 		if (encoder_funcs->mode_fixup) {
 			if (!(ret = encoder_funcs->mode_fixup(encoder, mode,
 							      adjusted_mode))) {
-				DRM_DEBUG_KMS("Encoder fixup failed\n");
+				drm_dbg_kms(dev, "[ENCODER:%d:%s] mode fixup failed\n",
+					    encoder->base.id, encoder->name);
 				goto done;
 			}
 		}
@@ -347,11 +348,12 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
 	if (crtc_funcs->mode_fixup) {
 		if (!(ret = crtc_funcs->mode_fixup(crtc, mode,
 						adjusted_mode))) {
-			DRM_DEBUG_KMS("CRTC fixup failed\n");
+			drm_dbg_kms(dev, "[CRTC:%d:%s] mode fixup failed\n",
+				    crtc->base.id, crtc->name);
 			goto done;
 		}
 	}
-	DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
+	drm_dbg_kms(dev, "[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
 
 	drm_mode_copy(&crtc->hwmode, adjusted_mode);
 
@@ -390,8 +392,8 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
 		if (!encoder_funcs)
 			continue;
 
-		DRM_DEBUG_KMS("[ENCODER:%d:%s] set [MODE:%s]\n",
-			encoder->base.id, encoder->name, mode->name);
+		drm_dbg_kms(dev, "[ENCODER:%d:%s] set [MODE:%s]\n",
+			    encoder->base.id, encoder->name, mode->name);
 		if (encoder_funcs->mode_set)
 			encoder_funcs->mode_set(encoder, mode, adjusted_mode);
 	}
@@ -503,7 +505,7 @@ drm_connector_get_single_encoder(struct drm_connector *connector)
 {
 	struct drm_encoder *encoder;
 
-	WARN_ON(hweight32(connector->possible_encoders) > 1);
+	drm_WARN_ON(connector->dev, hweight32(connector->possible_encoders) > 1);
 	drm_connector_for_each_possible_encoder(connector, encoder)
 		return encoder;
 
@@ -564,8 +566,6 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set,
 	int ret;
 	int i;
 
-	DRM_DEBUG_KMS("\n");
-
 	BUG_ON(!set);
 	BUG_ON(!set->crtc);
 	BUG_ON(!set->crtc->helper_private);
@@ -577,19 +577,22 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set,
 	crtc_funcs = set->crtc->helper_private;
 
 	dev = set->crtc->dev;
-	WARN_ON(drm_drv_uses_atomic_modeset(dev));
+
+	drm_dbg_kms(dev, "\n");
+
+	drm_WARN_ON(dev, drm_drv_uses_atomic_modeset(dev));
 
 	if (!set->mode)
 		set->fb = NULL;
 
 	if (set->fb) {
-		DRM_DEBUG_KMS("[CRTC:%d:%s] [FB:%d] #connectors=%d (x y) (%i %i)\n",
-			      set->crtc->base.id, set->crtc->name,
-			      set->fb->base.id,
-			      (int)set->num_connectors, set->x, set->y);
+		drm_dbg_kms(dev, "[CRTC:%d:%s] [FB:%d] #connectors=%d (x y) (%i %i)\n",
+			    set->crtc->base.id, set->crtc->name,
+			    set->fb->base.id,
+			    (int)set->num_connectors, set->x, set->y);
 	} else {
-		DRM_DEBUG_KMS("[CRTC:%d:%s] [NOFB]\n",
-			      set->crtc->base.id, set->crtc->name);
+		drm_dbg_kms(dev, "[CRTC:%d:%s] [NOFB]\n",
+			    set->crtc->base.id, set->crtc->name);
 		drm_crtc_helper_disable(set->crtc);
 		return 0;
 	}
@@ -639,7 +642,8 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set,
 	if (set->crtc->primary->fb != set->fb) {
 		/* If we have no fb then treat it as a full mode set */
 		if (set->crtc->primary->fb == NULL) {
-			DRM_DEBUG_KMS("crtc has no fb, full mode set\n");
+			drm_dbg_kms(dev, "[CRTC:%d:%s] no fb, full mode set\n",
+				    set->crtc->base.id, set->crtc->name);
 			mode_changed = true;
 		} else if (set->fb->format != set->crtc->primary->fb->format) {
 			mode_changed = true;
@@ -651,9 +655,10 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set,
 		fb_changed = true;
 
 	if (!drm_mode_equal(set->mode, &set->crtc->mode)) {
-		DRM_DEBUG_KMS("modes are different, full mode set\n");
-		drm_mode_debug_printmodeline(&set->crtc->mode);
-		drm_mode_debug_printmodeline(set->mode);
+		drm_dbg_kms(dev, "[CRTC:%d:%s] modes are different, full mode set:\n",
+			    set->crtc->base.id, set->crtc->name);
+		drm_dbg_kms(dev, DRM_MODE_FMT "\n", DRM_MODE_ARG(&set->crtc->mode));
+		drm_dbg_kms(dev, DRM_MODE_FMT "\n", DRM_MODE_ARG(set->mode));
 		mode_changed = true;
 	}
 
@@ -687,7 +692,8 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set,
 					fail = 1;
 
 				if (connector->dpms != DRM_MODE_DPMS_ON) {
-					DRM_DEBUG_KMS("connector dpms not on, full mode switch\n");
+					drm_dbg_kms(dev, "[CONNECTOR:%d:%s] DPMS not on, full mode switch\n",
+						    connector->base.id, connector->name);
 					mode_changed = true;
 				}
 
@@ -696,7 +702,8 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set,
 		}
 
 		if (new_encoder != connector->encoder) {
-			DRM_DEBUG_KMS("encoder changed, full mode switch\n");
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] encoder changed, full mode switch\n",
+				    connector->base.id, connector->name);
 			mode_changed = true;
 			/* If the encoder is reused for another connector, then
 			 * the appropriate crtc will be set later.
@@ -737,17 +744,18 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set,
 			goto fail;
 		}
 		if (new_crtc != connector->encoder->crtc) {
-			DRM_DEBUG_KMS("crtc changed, full mode switch\n");
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] CRTC changed, full mode switch\n",
+				    connector->base.id, connector->name);
 			mode_changed = true;
 			connector->encoder->crtc = new_crtc;
 		}
 		if (new_crtc) {
-			DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [CRTC:%d:%s]\n",
-				      connector->base.id, connector->name,
-				      new_crtc->base.id, new_crtc->name);
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] to [CRTC:%d:%s]\n",
+				    connector->base.id, connector->name,
+				    new_crtc->base.id, new_crtc->name);
 		} else {
-			DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [NOCRTC]\n",
-				      connector->base.id, connector->name);
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] to [NOCRTC]\n",
+				    connector->base.id, connector->name);
 		}
 	}
 	drm_connector_list_iter_end(&conn_iter);
@@ -758,23 +766,23 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set,
 
 	if (mode_changed) {
 		if (drm_helper_crtc_in_use(set->crtc)) {
-			DRM_DEBUG_KMS("attempting to set mode from"
-					" userspace\n");
-			drm_mode_debug_printmodeline(set->mode);
+			drm_dbg_kms(dev, "[CRTC:%d:%s] attempting to set mode from userspace: " DRM_MODE_FMT "\n",
+				    set->crtc->base.id, set->crtc->name, DRM_MODE_ARG(set->mode));
 			set->crtc->primary->fb = set->fb;
 			if (!drm_crtc_helper_set_mode(set->crtc, set->mode,
 						      set->x, set->y,
 						      save_set.fb)) {
-				DRM_ERROR("failed to set mode on [CRTC:%d:%s]\n",
-					  set->crtc->base.id, set->crtc->name);
+				drm_err(dev, "[CRTC:%d:%s] failed to set mode\n",
+					set->crtc->base.id, set->crtc->name);
 				set->crtc->primary->fb = save_set.fb;
 				ret = -EINVAL;
 				goto fail;
 			}
-			DRM_DEBUG_KMS("Setting connector DPMS state to on\n");
+			drm_dbg_kms(dev, "[CRTC:%d:%s] Setting connector DPMS state to on\n",
+				    set->crtc->base.id, set->crtc->name);
 			for (i = 0; i < set->num_connectors; i++) {
-				DRM_DEBUG_KMS("\t[CONNECTOR:%d:%s] set DPMS on\n", set->connectors[i]->base.id,
-					      set->connectors[i]->name);
+				drm_dbg_kms(dev, "\t[CONNECTOR:%d:%s] set DPMS on\n", set->connectors[i]->base.id,
+					    set->connectors[i]->name);
 				set->connectors[i]->funcs->dpms(set->connectors[i], DRM_MODE_DPMS_ON);
 			}
 		}
@@ -823,7 +831,7 @@ fail:
 	if (mode_changed &&
 	    !drm_crtc_helper_set_mode(save_set.crtc, save_set.mode, save_set.x,
 				      save_set.y, save_set.fb))
-		DRM_ERROR("failed to restore config after modeset failure\n");
+		drm_err(dev, "failed to restore config after modeset failure\n");
 
 	kfree(save_connector_encoders);
 	kfree(save_encoder_crtcs);
@@ -905,7 +913,7 @@ int drm_helper_connector_dpms(struct drm_connector *connector, int mode)
 	struct drm_crtc *crtc = encoder ? encoder->crtc : NULL;
 	int old_dpms, encoder_dpms = DRM_MODE_DPMS_OFF;
 
-	WARN_ON(drm_drv_uses_atomic_modeset(connector->dev));
+	drm_WARN_ON(connector->dev, drm_drv_uses_atomic_modeset(connector->dev));
 
 	if (mode == connector->dpms)
 		return 0;
@@ -980,7 +988,7 @@ void drm_helper_resume_force_mode(struct drm_device *dev)
 	int encoder_dpms;
 	bool ret;
 
-	WARN_ON(drm_drv_uses_atomic_modeset(dev));
+	drm_WARN_ON(dev, drm_drv_uses_atomic_modeset(dev));
 
 	drm_modeset_lock_all(dev);
 	drm_for_each_crtc(crtc, dev) {
@@ -993,7 +1001,7 @@ void drm_helper_resume_force_mode(struct drm_device *dev)
 
 		/* Restoring the old config should never fail! */
 		if (ret == false)
-			DRM_ERROR("failed to set mode on crtc %p\n", crtc);
+			drm_err(dev, "failed to set mode on crtc %p\n", crtc);
 
 		/* Turn off outputs that were already powered off */
 		if (drm_helper_choose_crtc_dpms(crtc)) {
diff --git a/drivers/gpu/drm/drm_crtc_helper_internal.h b/drivers/gpu/drm/drm_crtc_helper_internal.h
index 28e04e750130..8059f65c5d6c 100644
--- a/drivers/gpu/drm/drm_crtc_helper_internal.h
+++ b/drivers/gpu/drm/drm_crtc_helper_internal.h
@@ -26,10 +26,15 @@
  * implementation details and are not exported to drivers.
  */
 
-#include <drm/drm_connector.h>
-#include <drm/drm_crtc.h>
-#include <drm/drm_encoder.h>
-#include <drm/drm_modes.h>
+#ifndef __DRM_CRTC_HELPER_INTERNAL_H__
+#define __DRM_CRTC_HELPER_INTERNAL_H__
+
+enum drm_mode_status;
+struct drm_connector;
+struct drm_crtc;
+struct drm_display_mode;
+struct drm_encoder;
+struct drm_modeset_acquire_ctx;
 
 /* drm_probe_helper.c */
 enum drm_mode_status drm_crtc_mode_valid(struct drm_crtc *crtc,
@@ -44,3 +49,5 @@ drm_connector_mode_valid(struct drm_connector *connector,
 
 struct drm_encoder *
 drm_connector_get_single_encoder(struct drm_connector *connector);
+
+#endif /* __DRM_CRTC_HELPER_INTERNAL_H__ */
diff --git a/drivers/gpu/drm/drm_crtc_internal.h b/drivers/gpu/drm/drm_crtc_internal.h
index a514d5207e41..25aaae937ceb 100644
--- a/drivers/gpu/drm/drm_crtc_internal.h
+++ b/drivers/gpu/drm/drm_crtc_internal.h
@@ -32,6 +32,10 @@
  * and are not exported to drivers.
  */
 
+#ifndef __DRM_CRTC_INTERNAL_H__
+#define __DRM_CRTC_INTERNAL_H__
+
+#include <linux/err.h>
 #include <linux/types.h>
 
 enum drm_color_encoding;
@@ -39,12 +43,14 @@ enum drm_color_range;
 enum drm_connector_force;
 enum drm_mode_status;
 
+struct cea_sad;
 struct drm_atomic_state;
 struct drm_bridge;
 struct drm_connector;
 struct drm_crtc;
 struct drm_device;
 struct drm_display_mode;
+struct drm_edid;
 struct drm_file;
 struct drm_framebuffer;
 struct drm_mode_create_dumb;
@@ -54,6 +60,7 @@ struct drm_mode_object;
 struct drm_mode_set;
 struct drm_plane;
 struct drm_plane_state;
+struct drm_printer;
 struct drm_property;
 struct edid;
 struct fwnode_handle;
@@ -292,6 +299,10 @@ void drm_mode_fixup_1366x768(struct drm_display_mode *mode);
 int drm_edid_override_show(struct drm_connector *connector, struct seq_file *m);
 int drm_edid_override_set(struct drm_connector *connector, const void *edid, size_t size);
 int drm_edid_override_reset(struct drm_connector *connector);
+const u8 *drm_edid_find_extension(const struct drm_edid *drm_edid,
+				  int ext_id, int *ext_index);
+void drm_edid_cta_sad_get(const struct cea_sad *cta_sad, u8 *sad);
+void drm_edid_cta_sad_set(struct cea_sad *cta_sad, const u8 *sad);
 
 /* drm_edid_load.c */
 #ifdef CONFIG_DRM_LOAD_EDID_FIRMWARE
@@ -303,3 +314,5 @@ drm_edid_load_firmware(struct drm_connector *connector)
 	return ERR_PTR(-ENOENT);
 }
 #endif
+
+#endif /* __DRM_CRTC_INTERNAL_H__ */
diff --git a/drivers/gpu/drm/drm_displayid.c b/drivers/gpu/drm/drm_displayid.c
index 9edc111be7ee..9d01d762801f 100644
--- a/drivers/gpu/drm/drm_displayid.c
+++ b/drivers/gpu/drm/drm_displayid.c
@@ -3,10 +3,12 @@
  * Copyright © 2021 Intel Corporation
  */
 
-#include <drm/drm_displayid.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_print.h>
 
+#include "drm_crtc_internal.h"
+#include "drm_displayid_internal.h"
+
 static const struct displayid_header *
 displayid_get_header(const u8 *displayid, int length, int index)
 {
@@ -53,9 +55,10 @@ static const u8 *drm_find_displayid_extension(const struct drm_edid *drm_edid,
 					      int *length, int *idx,
 					      int *ext_index)
 {
-	const u8 *displayid = drm_find_edid_extension(drm_edid, DISPLAYID_EXT, ext_index);
 	const struct displayid_header *base;
+	const u8 *displayid;
 
+	displayid = drm_edid_find_extension(drm_edid, DISPLAYID_EXT, ext_index);
 	if (!displayid)
 		return NULL;
 
diff --git a/drivers/gpu/drm/drm_displayid_internal.h b/drivers/gpu/drm/drm_displayid_internal.h
new file mode 100644
index 000000000000..aee1b86a73c1
--- /dev/null
+++ b/drivers/gpu/drm/drm_displayid_internal.h
@@ -0,0 +1,170 @@
+/*
+ * Copyright © 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __DRM_DISPLAYID_INTERNAL_H__
+#define __DRM_DISPLAYID_INTERNAL_H__
+
+#include <linux/types.h>
+#include <linux/bits.h>
+
+struct drm_edid;
+
+#define VESA_IEEE_OUI				0x3a0292
+
+/* DisplayID Structure versions */
+#define DISPLAY_ID_STRUCTURE_VER_20		0x20
+
+/* DisplayID Structure v1r2 Data Blocks */
+#define DATA_BLOCK_PRODUCT_ID			0x00
+#define DATA_BLOCK_DISPLAY_PARAMETERS		0x01
+#define DATA_BLOCK_COLOR_CHARACTERISTICS	0x02
+#define DATA_BLOCK_TYPE_1_DETAILED_TIMING	0x03
+#define DATA_BLOCK_TYPE_2_DETAILED_TIMING	0x04
+#define DATA_BLOCK_TYPE_3_SHORT_TIMING		0x05
+#define DATA_BLOCK_TYPE_4_DMT_TIMING		0x06
+#define DATA_BLOCK_VESA_TIMING			0x07
+#define DATA_BLOCK_CEA_TIMING			0x08
+#define DATA_BLOCK_VIDEO_TIMING_RANGE		0x09
+#define DATA_BLOCK_PRODUCT_SERIAL_NUMBER	0x0a
+#define DATA_BLOCK_GP_ASCII_STRING		0x0b
+#define DATA_BLOCK_DISPLAY_DEVICE_DATA		0x0c
+#define DATA_BLOCK_INTERFACE_POWER_SEQUENCING	0x0d
+#define DATA_BLOCK_TRANSFER_CHARACTERISTICS	0x0e
+#define DATA_BLOCK_DISPLAY_INTERFACE		0x0f
+#define DATA_BLOCK_STEREO_DISPLAY_INTERFACE	0x10
+#define DATA_BLOCK_TILED_DISPLAY		0x12
+#define DATA_BLOCK_VENDOR_SPECIFIC		0x7f
+#define DATA_BLOCK_CTA				0x81
+
+/* DisplayID Structure v2r0 Data Blocks */
+#define DATA_BLOCK_2_PRODUCT_ID			0x20
+#define DATA_BLOCK_2_DISPLAY_PARAMETERS		0x21
+#define DATA_BLOCK_2_TYPE_7_DETAILED_TIMING	0x22
+#define DATA_BLOCK_2_TYPE_8_ENUMERATED_TIMING	0x23
+#define DATA_BLOCK_2_TYPE_9_FORMULA_TIMING	0x24
+#define DATA_BLOCK_2_DYNAMIC_VIDEO_TIMING	0x25
+#define DATA_BLOCK_2_DISPLAY_INTERFACE_FEATURES	0x26
+#define DATA_BLOCK_2_STEREO_DISPLAY_INTERFACE	0x27
+#define DATA_BLOCK_2_TILED_DISPLAY_TOPOLOGY	0x28
+#define DATA_BLOCK_2_CONTAINER_ID		0x29
+#define DATA_BLOCK_2_VENDOR_SPECIFIC		0x7e
+#define DATA_BLOCK_2_CTA_DISPLAY_ID		0x81
+
+/* DisplayID Structure v1r2 Product Type */
+#define PRODUCT_TYPE_EXTENSION			0
+#define PRODUCT_TYPE_TEST			1
+#define PRODUCT_TYPE_PANEL			2
+#define PRODUCT_TYPE_MONITOR			3
+#define PRODUCT_TYPE_TV				4
+#define PRODUCT_TYPE_REPEATER			5
+#define PRODUCT_TYPE_DIRECT_DRIVE		6
+
+/* DisplayID Structure v2r0 Display Product Primary Use Case (~Product Type) */
+#define PRIMARY_USE_EXTENSION			0
+#define PRIMARY_USE_TEST			1
+#define PRIMARY_USE_GENERIC			2
+#define PRIMARY_USE_TV				3
+#define PRIMARY_USE_DESKTOP_PRODUCTIVITY	4
+#define PRIMARY_USE_DESKTOP_GAMING		5
+#define PRIMARY_USE_PRESENTATION		6
+#define PRIMARY_USE_HEAD_MOUNTED_VR		7
+#define PRIMARY_USE_HEAD_MOUNTED_AR		8
+
+struct displayid_header {
+	u8 rev;
+	u8 bytes;
+	u8 prod_id;
+	u8 ext_count;
+} __packed;
+
+struct displayid_block {
+	u8 tag;
+	u8 rev;
+	u8 num_bytes;
+} __packed;
+
+struct displayid_tiled_block {
+	struct displayid_block base;
+	u8 tile_cap;
+	u8 topo[3];
+	u8 tile_size[4];
+	u8 tile_pixel_bezel[5];
+	u8 topology_id[8];
+} __packed;
+
+struct displayid_detailed_timings_1 {
+	u8 pixel_clock[3];
+	u8 flags;
+	u8 hactive[2];
+	u8 hblank[2];
+	u8 hsync[2];
+	u8 hsw[2];
+	u8 vactive[2];
+	u8 vblank[2];
+	u8 vsync[2];
+	u8 vsw[2];
+} __packed;
+
+struct displayid_detailed_timing_block {
+	struct displayid_block base;
+	struct displayid_detailed_timings_1 timings[];
+};
+
+#define DISPLAYID_VESA_MSO_OVERLAP	GENMASK(3, 0)
+#define DISPLAYID_VESA_MSO_MODE		GENMASK(6, 5)
+
+struct displayid_vesa_vendor_specific_block {
+	struct displayid_block base;
+	u8 oui[3];
+	u8 data_structure_type;
+	u8 mso;
+} __packed;
+
+/*
+ * DisplayID iteration.
+ *
+ * Do not access directly, this is private.
+ */
+struct displayid_iter {
+	const struct drm_edid *drm_edid;
+
+	const u8 *section;
+	int length;
+	int idx;
+	int ext_index;
+
+	u8 version;
+	u8 primary_use;
+};
+
+void displayid_iter_edid_begin(const struct drm_edid *drm_edid,
+			       struct displayid_iter *iter);
+const struct displayid_block *
+__displayid_iter_next(struct displayid_iter *iter);
+#define displayid_iter_for_each(__block, __iter) \
+	while (((__block) = __displayid_iter_next(__iter)))
+void displayid_iter_end(struct displayid_iter *iter);
+
+u8 displayid_version(const struct displayid_iter *iter);
+u8 displayid_primary_use(const struct displayid_iter *iter);
+
+#endif
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 243cacb3575c..535b624d4c9d 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -43,6 +43,7 @@
 #include <drm/drm_file.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mode_object.h>
+#include <drm/drm_panic.h>
 #include <drm/drm_print.h>
 #include <drm/drm_privacy_screen_machine.h>
 
@@ -638,6 +639,7 @@ static int drm_dev_init(struct drm_device *dev,
 	mutex_init(&dev->filelist_mutex);
 	mutex_init(&dev->clientlist_mutex);
 	mutex_init(&dev->master_mutex);
+	raw_spin_lock_init(&dev->mode_config.panic_lock);
 
 	ret = drmm_add_action_or_reset(dev, drm_dev_init_release, NULL);
 	if (ret)
@@ -943,6 +945,7 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags)
 		if (ret)
 			goto err_unload;
 	}
+	drm_panic_register(dev);
 
 	DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n",
 		 driver->name, driver->major, driver->minor,
@@ -987,6 +990,8 @@ void drm_dev_unregister(struct drm_device *dev)
 {
 	dev->registered = false;
 
+	drm_panic_unregister(dev);
+
 	drm_client_dev_unregister(dev);
 
 	if (drm_core_check_feature(dev, DRIVER_MODESET))
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 923c4423151c..4f54c91b31b2 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -29,16 +29,17 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/byteorder/generic.h>
 #include <linux/cec.h>
 #include <linux/hdmi.h>
 #include <linux/i2c.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/seq_buf.h>
 #include <linux/slab.h>
 #include <linux/vga_switcheroo.h>
 
-#include <drm/drm_displayid.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_eld.h>
@@ -46,6 +47,7 @@
 #include <drm/drm_print.h>
 
 #include "drm_crtc_internal.h"
+#include "drm_displayid_internal.h"
 #include "drm_internal.h"
 
 static int oui(u8 first, u8 second, u8 third)
@@ -102,6 +104,11 @@ struct detailed_mode_closure {
 	int modes;
 };
 
+struct drm_edid_match_closure {
+	const struct drm_edid_ident *ident;
+	bool matched;
+};
+
 #define LEVEL_DMT	0
 #define LEVEL_GTF	1
 #define LEVEL_GTF2	2
@@ -109,13 +116,15 @@ struct detailed_mode_closure {
 
 #define EDID_QUIRK(vend_chr_0, vend_chr_1, vend_chr_2, product_id, _quirks) \
 { \
-	.panel_id = drm_edid_encode_panel_id(vend_chr_0, vend_chr_1, vend_chr_2, \
-					     product_id), \
+	.ident = { \
+		.panel_id = drm_edid_encode_panel_id(vend_chr_0, vend_chr_1, \
+						     vend_chr_2, product_id), \
+	}, \
 	.quirks = _quirks \
 }
 
 static const struct edid_quirk {
-	u32 panel_id;
+	const struct drm_edid_ident ident;
 	u32 quirks;
 } edid_quirk_list[] = {
 	/* Acer AL1706 */
@@ -1811,36 +1820,25 @@ static bool edid_block_is_zero(const void *edid)
 	return !memchr_inv(edid, 0, EDID_LENGTH);
 }
 
-/**
- * drm_edid_are_equal - compare two edid blobs.
- * @edid1: pointer to first blob
- * @edid2: pointer to second blob
- * This helper can be used during probing to determine if
- * edid had changed.
- */
-bool drm_edid_are_equal(const struct edid *edid1, const struct edid *edid2)
+static bool drm_edid_eq(const struct drm_edid *drm_edid,
+			const void *raw_edid, size_t raw_edid_size)
 {
-	int edid1_len, edid2_len;
-	bool edid1_present = edid1 != NULL;
-	bool edid2_present = edid2 != NULL;
+	bool edid1_present = drm_edid && drm_edid->edid && drm_edid->size;
+	bool edid2_present = raw_edid && raw_edid_size;
 
 	if (edid1_present != edid2_present)
 		return false;
 
-	if (edid1) {
-		edid1_len = edid_size(edid1);
-		edid2_len = edid_size(edid2);
-
-		if (edid1_len != edid2_len)
+	if (edid1_present) {
+		if (drm_edid->size != raw_edid_size)
 			return false;
 
-		if (memcmp(edid1, edid2, edid1_len))
+		if (memcmp(drm_edid->edid, raw_edid, drm_edid->size))
 			return false;
 	}
 
 	return true;
 }
-EXPORT_SYMBOL(drm_edid_are_equal);
 
 enum edid_block_status {
 	EDID_BLOCK_OK = 0,
@@ -2749,8 +2747,84 @@ const struct drm_edid *drm_edid_read(struct drm_connector *connector)
 }
 EXPORT_SYMBOL(drm_edid_read);
 
-static u32 edid_extract_panel_id(const struct edid *edid)
+/**
+ * drm_edid_get_product_id - Get the vendor and product identification
+ * @drm_edid: EDID
+ * @id: Where to place the product id
+ */
+void drm_edid_get_product_id(const struct drm_edid *drm_edid,
+			     struct drm_edid_product_id *id)
+{
+	if (drm_edid && drm_edid->edid && drm_edid->size >= EDID_LENGTH)
+		memcpy(id, &drm_edid->edid->product_id, sizeof(*id));
+	else
+		memset(id, 0, sizeof(*id));
+}
+EXPORT_SYMBOL(drm_edid_get_product_id);
+
+static void decode_date(struct seq_buf *s, const struct drm_edid_product_id *id)
+{
+	int week = id->week_of_manufacture;
+	int year = id->year_of_manufacture + 1990;
+
+	if (week == 0xff)
+		seq_buf_printf(s, "model year: %d", year);
+	else if (!week)
+		seq_buf_printf(s, "year of manufacture: %d", year);
+	else
+		seq_buf_printf(s, "week/year of manufacture: %d/%d", week, year);
+}
+
+/**
+ * drm_edid_print_product_id - Print decoded product id to printer
+ * @p: drm printer
+ * @id: EDID product id
+ * @raw: If true, also print the raw hex
+ *
+ * See VESA E-EDID 1.4 section 3.4.
+ */
+void drm_edid_print_product_id(struct drm_printer *p,
+			       const struct drm_edid_product_id *id, bool raw)
+{
+	DECLARE_SEQ_BUF(date, 40);
+	char vend[4];
+
+	drm_edid_decode_mfg_id(be16_to_cpu(id->manufacturer_name), vend);
+
+	decode_date(&date, id);
+
+	drm_printf(p, "manufacturer name: %s, product code: %u, serial number: %u, %s\n",
+		   vend, le16_to_cpu(id->product_code),
+		   le32_to_cpu(id->serial_number), seq_buf_str(&date));
+
+	if (raw)
+		drm_printf(p, "raw product id: %*ph\n", (int)sizeof(*id), id);
+
+	WARN_ON(seq_buf_has_overflowed(&date));
+}
+EXPORT_SYMBOL(drm_edid_print_product_id);
+
+/**
+ * drm_edid_get_panel_id - Get a panel's ID from EDID
+ * @drm_edid: EDID that contains panel ID.
+ *
+ * This function uses the first block of the EDID of a panel and (assuming
+ * that the EDID is valid) extracts the ID out of it. The ID is a 32-bit value
+ * (16 bits of manufacturer ID and 16 bits of per-manufacturer ID) that's
+ * supposed to be different for each different modem of panel.
+ *
+ * Return: A 32-bit ID that should be different for each make/model of panel.
+ *         See the functions drm_edid_encode_panel_id() and
+ *         drm_edid_decode_panel_id() for some details on the structure of this
+ *         ID. Return 0 if the EDID size is less than a base block.
+ */
+u32 drm_edid_get_panel_id(const struct drm_edid *drm_edid)
 {
+	const struct edid *edid = drm_edid->edid;
+
+	if (drm_edid->size < EDID_LENGTH)
+		return 0;
+
 	/*
 	 * We represent the ID as a 32-bit number so it can easily be compared
 	 * with "==".
@@ -2768,60 +2842,54 @@ static u32 edid_extract_panel_id(const struct edid *edid)
 	       (u32)edid->mfg_id[1] << 16   |
 	       (u32)EDID_PRODUCT_ID(edid);
 }
+EXPORT_SYMBOL(drm_edid_get_panel_id);
 
 /**
- * drm_edid_get_panel_id - Get a panel's ID through DDC
+ * drm_edid_read_base_block - Get a panel's EDID base block
  * @adapter: I2C adapter to use for DDC
  *
- * This function reads the first block of the EDID of a panel and (assuming
- * that the EDID is valid) extracts the ID out of it. The ID is a 32-bit value
- * (16 bits of manufacturer ID and 16 bits of per-manufacturer ID) that's
- * supposed to be different for each different modem of panel.
+ * This function returns the drm_edid containing the first block of the EDID of
+ * a panel.
  *
  * This function is intended to be used during early probing on devices where
  * more than one panel might be present. Because of its intended use it must
- * assume that the EDID of the panel is correct, at least as far as the ID
- * is concerned (in other words, we don't process any overrides here).
+ * assume that the EDID of the panel is correct, at least as far as the base
+ * block is concerned (in other words, we don't process any overrides here).
+ *
+ * Caller should call drm_edid_free() after use.
  *
  * NOTE: it's expected that this function and drm_do_get_edid() will both
  * be read the EDID, but there is no caching between them. Since we're only
  * reading the first block, hopefully this extra overhead won't be too big.
  *
- * Return: A 32-bit ID that should be different for each make/model of panel.
- *         See the functions drm_edid_encode_panel_id() and
- *         drm_edid_decode_panel_id() for some details on the structure of this
- *         ID.
+ * WARNING: Only use this function when the connector is unknown. For example,
+ * during the early probe of panel. The EDID read from the function is temporary
+ * and should be replaced by the full EDID returned from other drm_edid_read.
+ *
+ * Return: Pointer to allocated EDID base block, or NULL on any failure.
  */
-
-u32 drm_edid_get_panel_id(struct i2c_adapter *adapter)
+const struct drm_edid *drm_edid_read_base_block(struct i2c_adapter *adapter)
 {
 	enum edid_block_status status;
 	void *base_block;
-	u32 panel_id = 0;
-
-	/*
-	 * There are no manufacturer IDs of 0, so if there is a problem reading
-	 * the EDID then we'll just return 0.
-	 */
 
 	base_block = kzalloc(EDID_LENGTH, GFP_KERNEL);
 	if (!base_block)
-		return 0;
+		return NULL;
 
 	status = edid_block_read(base_block, 0, drm_do_probe_ddc_edid, adapter);
 
 	edid_block_status_print(status, base_block, 0);
 
-	if (edid_block_status_valid(status, edid_block_tag(base_block)))
-		panel_id = edid_extract_panel_id(base_block);
-	else
+	if (!edid_block_status_valid(status, edid_block_tag(base_block))) {
 		edid_block_dump(KERN_NOTICE, base_block, 0);
+		kfree(base_block);
+		return NULL;
+	}
 
-	kfree(base_block);
-
-	return panel_id;
+	return _drm_edid_alloc(base_block, EDID_LENGTH);
 }
-EXPORT_SYMBOL(drm_edid_get_panel_id);
+EXPORT_SYMBOL(drm_edid_read_base_block);
 
 /**
  * drm_get_edid_switcheroo - get EDID data for a vga_switcheroo output
@@ -2903,16 +2971,17 @@ EXPORT_SYMBOL(drm_edid_duplicate);
  * @drm_edid: EDID to process
  *
  * This tells subsequent routines what fixes they need to apply.
+ *
+ * Return: A u32 represents the quirks to apply.
  */
 static u32 edid_get_quirks(const struct drm_edid *drm_edid)
 {
-	u32 panel_id = edid_extract_panel_id(drm_edid->edid);
 	const struct edid_quirk *quirk;
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(edid_quirk_list); i++) {
 		quirk = &edid_quirk_list[i];
-		if (quirk->panel_id == panel_id)
+		if (drm_edid_match(drm_edid, &quirk->ident))
 			return quirk->quirks;
 	}
 
@@ -4120,7 +4189,7 @@ static int add_detailed_modes(struct drm_connector *connector,
  *
  * FIXME: Prefer not returning pointers to raw EDID data.
  */
-const u8 *drm_find_edid_extension(const struct drm_edid *drm_edid,
+const u8 *drm_edid_find_extension(const struct drm_edid *drm_edid,
 				  int ext_id, int *ext_index)
 {
 	const u8 *edid_ext = NULL;
@@ -4150,11 +4219,21 @@ static bool drm_edid_has_cta_extension(const struct drm_edid *drm_edid)
 {
 	const struct displayid_block *block;
 	struct displayid_iter iter;
-	int ext_index = 0;
+	struct drm_edid_iter edid_iter;
+	const u8 *ext;
 	bool found = false;
 
 	/* Look for a top level CEA extension block */
-	if (drm_find_edid_extension(drm_edid, CEA_EXT, &ext_index))
+	drm_edid_iter_begin(drm_edid, &edid_iter);
+	drm_edid_iter_for_each(ext, &edid_iter) {
+		if (ext[0] == CEA_EXT) {
+			found = true;
+			break;
+		}
+	}
+	drm_edid_iter_end(&edid_iter);
+
+	if (found)
 		return true;
 
 	/* CEA blocks can also be found embedded in a DisplayID block */
@@ -5443,6 +5522,66 @@ drm_parse_hdmi_vsdb_audio(struct drm_connector *connector, const u8 *db)
 }
 
 static void
+match_identity(const struct detailed_timing *timing, void *data)
+{
+	struct drm_edid_match_closure *closure = data;
+	unsigned int i;
+	const char *name = closure->ident->name;
+	unsigned int name_len = strlen(name);
+	const char *desc = timing->data.other_data.data.str.str;
+	unsigned int desc_len = ARRAY_SIZE(timing->data.other_data.data.str.str);
+
+	if (name_len > desc_len ||
+	    !(is_display_descriptor(timing, EDID_DETAIL_MONITOR_NAME) ||
+	      is_display_descriptor(timing, EDID_DETAIL_MONITOR_STRING)))
+		return;
+
+	if (strncmp(name, desc, name_len))
+		return;
+
+	for (i = name_len; i < desc_len; i++) {
+		if (desc[i] == '\n')
+			break;
+		/* Allow white space before EDID string terminator. */
+		if (!isspace(desc[i]))
+			return;
+	}
+
+	closure->matched = true;
+}
+
+/**
+ * drm_edid_match - match drm_edid with given identity
+ * @drm_edid: EDID
+ * @ident: the EDID identity to match with
+ *
+ * Check if the EDID matches with the given identity.
+ *
+ * Return: True if the given identity matched with EDID, false otherwise.
+ */
+bool drm_edid_match(const struct drm_edid *drm_edid,
+		    const struct drm_edid_ident *ident)
+{
+	if (!drm_edid || drm_edid_get_panel_id(drm_edid) != ident->panel_id)
+		return false;
+
+	/* Match with name only if it's not NULL. */
+	if (ident->name) {
+		struct drm_edid_match_closure closure = {
+			.ident = ident,
+			.matched = false,
+		};
+
+		drm_for_each_detailed_block(drm_edid, match_identity, &closure);
+
+		return closure.matched;
+	}
+
+	return true;
+}
+EXPORT_SYMBOL(drm_edid_match);
+
+static void
 monitor_name(const struct detailed_timing *timing, void *data)
 {
 	const char **res = data;
@@ -6787,15 +6926,14 @@ static int _drm_edid_connector_property_update(struct drm_connector *connector,
 	int ret;
 
 	if (connector->edid_blob_ptr) {
-		const struct edid *old_edid = connector->edid_blob_ptr->data;
-
-		if (old_edid) {
-			if (!drm_edid_are_equal(drm_edid ? drm_edid->edid : NULL, old_edid)) {
-				connector->epoch_counter++;
-				drm_dbg_kms(dev, "[CONNECTOR:%d:%s] EDID changed, epoch counter %llu\n",
-					    connector->base.id, connector->name,
-					    connector->epoch_counter);
-			}
+		const void *old_edid = connector->edid_blob_ptr->data;
+		size_t old_edid_size = connector->edid_blob_ptr->length;
+
+		if (old_edid && !drm_edid_eq(drm_edid, old_edid, old_edid_size)) {
+			connector->epoch_counter++;
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] EDID changed, epoch counter %llu\n",
+				    connector->base.id, connector->name,
+				    connector->epoch_counter);
 		}
 	}
 
@@ -7324,7 +7462,7 @@ static void drm_parse_tiled_block(struct drm_connector *connector,
 static bool displayid_is_tiled_block(const struct displayid_iter *iter,
 				     const struct displayid_block *block)
 {
-	return (displayid_version(iter) == DISPLAY_ID_STRUCTURE_VER_12 &&
+	return (displayid_version(iter) < DISPLAY_ID_STRUCTURE_VER_20 &&
 		block->tag == DATA_BLOCK_TILED_DISPLAY) ||
 		(displayid_version(iter) == DISPLAY_ID_STRUCTURE_VER_20 &&
 		 block->tag == DATA_BLOCK_2_TILED_DISPLAY_TOPOLOGY);
diff --git a/drivers/gpu/drm/drm_eld.c b/drivers/gpu/drm/drm_eld.c
index 5177991aa272..c0428d07de53 100644
--- a/drivers/gpu/drm/drm_eld.c
+++ b/drivers/gpu/drm/drm_eld.c
@@ -3,10 +3,12 @@
  * Copyright © 2023 Intel Corporation
  */
 
+#include <linux/export.h>
+
 #include <drm/drm_edid.h>
 #include <drm/drm_eld.h>
 
-#include "drm_internal.h"
+#include "drm_crtc_internal.h"
 
 /**
  * drm_eld_sad_get - get SAD from ELD to struct cea_sad
diff --git a/drivers/gpu/drm/drm_fb_dma_helper.c b/drivers/gpu/drm/drm_fb_dma_helper.c
index 3b535ad1b07c..e1d61a65210b 100644
--- a/drivers/gpu/drm/drm_fb_dma_helper.c
+++ b/drivers/gpu/drm/drm_fb_dma_helper.c
@@ -15,6 +15,7 @@
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_panic.h>
 #include <drm/drm_plane.h>
 #include <linux/dma-mapping.h>
 #include <linux/module.h>
@@ -148,3 +149,47 @@ void drm_fb_dma_sync_non_coherent(struct drm_device *drm,
 	}
 }
 EXPORT_SYMBOL_GPL(drm_fb_dma_sync_non_coherent);
+
+/**
+ * drm_fb_dma_get_scanout_buffer - Provide a scanout buffer in case of panic
+ * @plane: DRM primary plane
+ * @sb: scanout buffer for the panic handler
+ * Returns: 0 or negative error code
+ *
+ * Generic get_scanout_buffer() implementation, for drivers that uses the
+ * drm_fb_dma_helper. It won't call vmap in the panic context, so the driver
+ * should make sure the primary plane is vmapped, otherwise the panic screen
+ * won't get displayed.
+ */
+int drm_fb_dma_get_scanout_buffer(struct drm_plane *plane,
+				  struct drm_scanout_buffer *sb)
+{
+	struct drm_gem_dma_object *dma_obj;
+	struct drm_framebuffer *fb;
+
+	if (!plane->state || !plane->state->fb)
+		return -EINVAL;
+
+	fb = plane->state->fb;
+	/* Only support linear modifier */
+	if (fb->modifier != DRM_FORMAT_MOD_LINEAR)
+		return -ENODEV;
+
+	dma_obj = drm_fb_dma_get_gem_obj(fb, 0);
+
+	/* Buffer should be accessible from the CPU */
+	if (dma_obj->base.import_attach)
+		return -ENODEV;
+
+	/* Buffer should be already mapped to CPU */
+	if (!dma_obj->vaddr)
+		return -ENODEV;
+
+	iosys_map_set_vaddr(&sb->map[0], dma_obj->vaddr);
+	sb->format = fb->format;
+	sb->height = fb->height;
+	sb->width = fb->width;
+	sb->pitch[0] = fb->pitches[0];
+	return 0;
+}
+EXPORT_SYMBOL(drm_fb_dma_get_scanout_buffer);
diff --git a/drivers/gpu/drm/drm_fbdev_generic.c b/drivers/gpu/drm/drm_fbdev_generic.c
index d647d89764cb..97e579c33d84 100644
--- a/drivers/gpu/drm/drm_fbdev_generic.c
+++ b/drivers/gpu/drm/drm_fbdev_generic.c
@@ -113,7 +113,6 @@ static int drm_fbdev_generic_helper_fb_probe(struct drm_fb_helper *fb_helper,
 	/* screen */
 	info->flags |= FBINFO_VIRTFB | FBINFO_READS_FAST;
 	info->screen_buffer = screen_buffer;
-	info->fix.smem_start = page_to_phys(vmalloc_to_page(info->screen_buffer));
 	info->fix.smem_len = screen_size;
 
 	/* deferred I/O */
@@ -197,14 +196,14 @@ static int drm_fbdev_generic_damage_blit(struct drm_fb_helper *fb_helper,
 	 */
 	mutex_lock(&fb_helper->lock);
 
-	ret = drm_client_buffer_vmap(buffer, &map);
+	ret = drm_client_buffer_vmap_local(buffer, &map);
 	if (ret)
 		goto out;
 
 	dst = map;
 	drm_fbdev_generic_damage_blit_real(fb_helper, clip, &dst);
 
-	drm_client_buffer_vunmap(buffer);
+	drm_client_buffer_vunmap_local(buffer);
 
 out:
 	mutex_unlock(&fb_helper->lock);
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 44a948b80ee1..d4bbc5d109c8 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -1161,7 +1161,7 @@ void drm_gem_print_info(struct drm_printer *p, unsigned int indent,
 		obj->funcs->print_info(p, indent, obj);
 }
 
-int drm_gem_pin(struct drm_gem_object *obj)
+int drm_gem_pin_locked(struct drm_gem_object *obj)
 {
 	if (obj->funcs->pin)
 		return obj->funcs->pin(obj);
@@ -1169,12 +1169,30 @@ int drm_gem_pin(struct drm_gem_object *obj)
 	return 0;
 }
 
-void drm_gem_unpin(struct drm_gem_object *obj)
+void drm_gem_unpin_locked(struct drm_gem_object *obj)
 {
 	if (obj->funcs->unpin)
 		obj->funcs->unpin(obj);
 }
 
+int drm_gem_pin(struct drm_gem_object *obj)
+{
+	int ret;
+
+	dma_resv_lock(obj->resv, NULL);
+	ret = drm_gem_pin_locked(obj);
+	dma_resv_unlock(obj->resv);
+
+	return ret;
+}
+
+void drm_gem_unpin(struct drm_gem_object *obj)
+{
+	dma_resv_lock(obj->resv, NULL);
+	drm_gem_unpin_locked(obj);
+	dma_resv_unlock(obj->resv);
+}
+
 int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map)
 {
 	int ret;
@@ -1209,6 +1227,18 @@ void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
 }
 EXPORT_SYMBOL(drm_gem_vunmap);
 
+void drm_gem_lock(struct drm_gem_object *obj)
+{
+	dma_resv_lock(obj->resv, NULL);
+}
+EXPORT_SYMBOL(drm_gem_lock);
+
+void drm_gem_unlock(struct drm_gem_object *obj)
+{
+	dma_resv_unlock(obj->resv);
+}
+EXPORT_SYMBOL(drm_gem_unlock);
+
 int drm_gem_vmap_unlocked(struct drm_gem_object *obj, struct iosys_map *map)
 {
 	int ret;
diff --git a/drivers/gpu/drm/drm_gem_atomic_helper.c b/drivers/gpu/drm/drm_gem_atomic_helper.c
index e440f458b663..93337543aac3 100644
--- a/drivers/gpu/drm/drm_gem_atomic_helper.c
+++ b/drivers/gpu/drm/drm_gem_atomic_helper.c
@@ -224,8 +224,8 @@ __drm_gem_duplicate_shadow_plane_state(struct drm_plane *plane,
 
 	__drm_atomic_helper_plane_duplicate_state(plane, &new_shadow_plane_state->base);
 
-	drm_format_conv_state_copy(&shadow_plane_state->fmtcnv_state,
-				   &new_shadow_plane_state->fmtcnv_state);
+	drm_format_conv_state_copy(&new_shadow_plane_state->fmtcnv_state,
+				   &shadow_plane_state->fmtcnv_state);
 }
 EXPORT_SYMBOL(__drm_gem_duplicate_shadow_plane_state);
 
diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
index e435f986cd13..53c003983ad1 100644
--- a/drivers/gpu/drm/drm_gem_shmem_helper.c
+++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
@@ -10,7 +10,6 @@
 #include <linux/shmem_fs.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
-#include <linux/module.h>
 
 #ifdef CONFIG_X86
 #include <asm/set_memory.h>
@@ -228,23 +227,27 @@ void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem)
 }
 EXPORT_SYMBOL(drm_gem_shmem_put_pages);
 
-static int drm_gem_shmem_pin_locked(struct drm_gem_shmem_object *shmem)
+int drm_gem_shmem_pin_locked(struct drm_gem_shmem_object *shmem)
 {
 	int ret;
 
 	dma_resv_assert_held(shmem->base.resv);
 
+	drm_WARN_ON(shmem->base.dev, shmem->base.import_attach);
+
 	ret = drm_gem_shmem_get_pages(shmem);
 
 	return ret;
 }
+EXPORT_SYMBOL(drm_gem_shmem_pin_locked);
 
-static void drm_gem_shmem_unpin_locked(struct drm_gem_shmem_object *shmem)
+void drm_gem_shmem_unpin_locked(struct drm_gem_shmem_object *shmem)
 {
 	dma_resv_assert_held(shmem->base.resv);
 
 	drm_gem_shmem_put_pages(shmem);
 }
+EXPORT_SYMBOL(drm_gem_shmem_unpin_locked);
 
 /**
  * drm_gem_shmem_pin - Pin backing pages for a shmem GEM object
@@ -610,6 +613,9 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct
 		return ret;
 	}
 
+	if (is_cow_mapping(vma->vm_flags))
+		return -EINVAL;
+
 	dma_resv_lock(shmem->base.resv, NULL);
 	ret = drm_gem_shmem_get_pages(shmem);
 	dma_resv_unlock(shmem->base.resv);
diff --git a/drivers/gpu/drm/drm_gem_vram_helper.c b/drivers/gpu/drm/drm_gem_vram_helper.c
index 1ac284a9e8ee..6027584406af 100644
--- a/drivers/gpu/drm/drm_gem_vram_helper.c
+++ b/drivers/gpu/drm/drm_gem_vram_helper.c
@@ -282,6 +282,8 @@ static int drm_gem_vram_pin_locked(struct drm_gem_vram_object *gbo,
 	struct ttm_operation_ctx ctx = { false, false };
 	int ret;
 
+	dma_resv_assert_held(gbo->bo.base.resv);
+
 	if (gbo->bo.pin_count)
 		goto out;
 
@@ -337,6 +339,8 @@ EXPORT_SYMBOL(drm_gem_vram_pin);
 
 static void drm_gem_vram_unpin_locked(struct drm_gem_vram_object *gbo)
 {
+	dma_resv_assert_held(gbo->bo.base.resv);
+
 	ttm_bo_unpin(&gbo->bo);
 }
 
@@ -363,11 +367,28 @@ int drm_gem_vram_unpin(struct drm_gem_vram_object *gbo)
 }
 EXPORT_SYMBOL(drm_gem_vram_unpin);
 
-static int drm_gem_vram_kmap_locked(struct drm_gem_vram_object *gbo,
-				    struct iosys_map *map)
+/**
+ * drm_gem_vram_vmap() - Pins and maps a GEM VRAM object into kernel address
+ *                       space
+ * @gbo: The GEM VRAM object to map
+ * @map: Returns the kernel virtual address of the VRAM GEM object's backing
+ *       store.
+ *
+ * The vmap function pins a GEM VRAM object to its current location, either
+ * system or video memory, and maps its buffer into kernel address space.
+ * As pinned object cannot be relocated, you should avoid pinning objects
+ * permanently. Call drm_gem_vram_vunmap() with the returned address to
+ * unmap and unpin the GEM VRAM object.
+ *
+ * Returns:
+ * 0 on success, or a negative error code otherwise.
+ */
+int drm_gem_vram_vmap(struct drm_gem_vram_object *gbo, struct iosys_map *map)
 {
 	int ret;
 
+	dma_resv_assert_held(gbo->bo.base.resv);
+
 	if (gbo->vmap_use_count > 0)
 		goto out;
 
@@ -388,12 +409,23 @@ out:
 
 	return 0;
 }
+EXPORT_SYMBOL(drm_gem_vram_vmap);
 
-static void drm_gem_vram_kunmap_locked(struct drm_gem_vram_object *gbo,
-				       struct iosys_map *map)
+/**
+ * drm_gem_vram_vunmap() - Unmaps and unpins a GEM VRAM object
+ * @gbo: The GEM VRAM object to unmap
+ * @map: Kernel virtual address where the VRAM GEM object was mapped
+ *
+ * A call to drm_gem_vram_vunmap() unmaps and unpins a GEM VRAM buffer. See
+ * the documentation for drm_gem_vram_vmap() for more information.
+ */
+void drm_gem_vram_vunmap(struct drm_gem_vram_object *gbo,
+			 struct iosys_map *map)
 {
 	struct drm_device *dev = gbo->bo.base.dev;
 
+	dma_resv_assert_held(gbo->bo.base.resv);
+
 	if (drm_WARN_ON_ONCE(dev, !gbo->vmap_use_count))
 		return;
 
@@ -410,60 +442,6 @@ static void drm_gem_vram_kunmap_locked(struct drm_gem_vram_object *gbo,
 	 * from memory. See drm_gem_vram_bo_driver_move_notify().
 	 */
 }
-
-/**
- * drm_gem_vram_vmap() - Pins and maps a GEM VRAM object into kernel address
- *                       space
- * @gbo: The GEM VRAM object to map
- * @map: Returns the kernel virtual address of the VRAM GEM object's backing
- *       store.
- *
- * The vmap function pins a GEM VRAM object to its current location, either
- * system or video memory, and maps its buffer into kernel address space.
- * As pinned object cannot be relocated, you should avoid pinning objects
- * permanently. Call drm_gem_vram_vunmap() with the returned address to
- * unmap and unpin the GEM VRAM object.
- *
- * Returns:
- * 0 on success, or a negative error code otherwise.
- */
-int drm_gem_vram_vmap(struct drm_gem_vram_object *gbo, struct iosys_map *map)
-{
-	int ret;
-
-	dma_resv_assert_held(gbo->bo.base.resv);
-
-	ret = drm_gem_vram_pin_locked(gbo, 0);
-	if (ret)
-		return ret;
-	ret = drm_gem_vram_kmap_locked(gbo, map);
-	if (ret)
-		goto err_drm_gem_vram_unpin_locked;
-
-	return 0;
-
-err_drm_gem_vram_unpin_locked:
-	drm_gem_vram_unpin_locked(gbo);
-	return ret;
-}
-EXPORT_SYMBOL(drm_gem_vram_vmap);
-
-/**
- * drm_gem_vram_vunmap() - Unmaps and unpins a GEM VRAM object
- * @gbo: The GEM VRAM object to unmap
- * @map: Kernel virtual address where the VRAM GEM object was mapped
- *
- * A call to drm_gem_vram_vunmap() unmaps and unpins a GEM VRAM buffer. See
- * the documentation for drm_gem_vram_vmap() for more information.
- */
-void drm_gem_vram_vunmap(struct drm_gem_vram_object *gbo,
-			 struct iosys_map *map)
-{
-	dma_resv_assert_held(gbo->bo.base.resv);
-
-	drm_gem_vram_kunmap_locked(gbo, map);
-	drm_gem_vram_unpin_locked(gbo);
-}
 EXPORT_SYMBOL(drm_gem_vram_vunmap);
 
 /**
@@ -768,7 +746,8 @@ static int drm_gem_vram_object_pin(struct drm_gem_object *gem)
 {
 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);
 
-	/* Fbdev console emulation is the use case of these PRIME
+	/*
+	 * Fbdev console emulation is the use case of these PRIME
 	 * helpers. This may involve updating a hardware buffer from
 	 * a shadow FB. We pin the buffer to it's current location
 	 * (either video RAM or system memory) to prevent it from
@@ -776,7 +755,7 @@ static int drm_gem_vram_object_pin(struct drm_gem_object *gem)
 	 * the buffer to be pinned to VRAM, implement a callback that
 	 * sets the flags accordingly.
 	 */
-	return drm_gem_vram_pin(gbo, 0);
+	return drm_gem_vram_pin_locked(gbo, 0);
 }
 
 /**
@@ -787,7 +766,7 @@ static void drm_gem_vram_object_unpin(struct drm_gem_object *gem)
 {
 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);
 
-	drm_gem_vram_unpin(gbo);
+	drm_gem_vram_unpin_locked(gbo);
 }
 
 /**
diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h
index 8e4faf0a28e6..690505a1f7a5 100644
--- a/drivers/gpu/drm/drm_internal.h
+++ b/drivers/gpu/drm/drm_internal.h
@@ -21,6 +21,9 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#ifndef __DRM_INTERNAL_H__
+#define __DRM_INTERNAL_H__
+
 #include <linux/kthread.h>
 #include <linux/types.h>
 
@@ -32,7 +35,6 @@
 
 #define DRM_IF_VERSION(maj, min) (maj << 16 | min)
 
-struct cea_sad;
 struct dentry;
 struct dma_buf;
 struct iosys_map;
@@ -170,6 +172,8 @@ void drm_gem_release(struct drm_device *dev, struct drm_file *file_private);
 void drm_gem_print_info(struct drm_printer *p, unsigned int indent,
 			const struct drm_gem_object *obj);
 
+int drm_gem_pin_locked(struct drm_gem_object *obj);
+void drm_gem_unpin_locked(struct drm_gem_object *obj);
 int drm_gem_pin(struct drm_gem_object *obj);
 void drm_gem_unpin(struct drm_gem_object *obj);
 int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map);
@@ -273,6 +277,4 @@ void drm_framebuffer_print_info(struct drm_printer *p, unsigned int indent,
 				const struct drm_framebuffer *fb);
 void drm_framebuffer_debugfs_init(struct drm_device *dev);
 
-/* drm_edid.c */
-void drm_edid_cta_sad_get(const struct cea_sad *cta_sad, u8 *sad);
-void drm_edid_cta_sad_set(struct cea_sad *cta_sad, const u8 *sad);
+#endif /* __DRM_INTERNAL_H__ */
diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c
index ef6e416522f8..795001bb7ff1 100644
--- a/drivers/gpu/drm/drm_mipi_dsi.c
+++ b/drivers/gpu/drm/drm_mipi_dsi.c
@@ -645,29 +645,56 @@ int mipi_dsi_set_maximum_return_packet_size(struct mipi_dsi_device *dsi,
 EXPORT_SYMBOL(mipi_dsi_set_maximum_return_packet_size);
 
 /**
- * mipi_dsi_compression_mode() - enable/disable DSC on the peripheral
+ * mipi_dsi_compression_mode_ext() - enable/disable DSC on the peripheral
  * @dsi: DSI peripheral device
  * @enable: Whether to enable or disable the DSC
+ * @algo: Selected compression algorithm
+ * @pps_selector: Select PPS from the table of pre-stored or uploaded PPS entries
  *
- * Enable or disable Display Stream Compression on the peripheral using the
- * default Picture Parameter Set and VESA DSC 1.1 algorithm.
+ * Enable or disable Display Stream Compression on the peripheral.
  *
  * Return: 0 on success or a negative error code on failure.
  */
-ssize_t mipi_dsi_compression_mode(struct mipi_dsi_device *dsi, bool enable)
+int mipi_dsi_compression_mode_ext(struct mipi_dsi_device *dsi, bool enable,
+				  enum mipi_dsi_compression_algo algo,
+				  unsigned int pps_selector)
 {
-	/* Note: Needs updating for non-default PPS or algorithm */
-	u8 tx[2] = { enable << 0, 0 };
+	u8 tx[2] = { };
 	struct mipi_dsi_msg msg = {
 		.channel = dsi->channel,
 		.type = MIPI_DSI_COMPRESSION_MODE,
 		.tx_len = sizeof(tx),
 		.tx_buf = tx,
 	};
-	int ret = mipi_dsi_device_transfer(dsi, &msg);
+	int ret;
+
+	if (algo > 3 || pps_selector > 3)
+		return -EINVAL;
+
+	tx[0] = (enable << 0) |
+		(algo << 1) |
+		(pps_selector << 4);
+
+	ret = mipi_dsi_device_transfer(dsi, &msg);
 
 	return (ret < 0) ? ret : 0;
 }
+EXPORT_SYMBOL(mipi_dsi_compression_mode_ext);
+
+/**
+ * mipi_dsi_compression_mode() - enable/disable DSC on the peripheral
+ * @dsi: DSI peripheral device
+ * @enable: Whether to enable or disable the DSC
+ *
+ * Enable or disable Display Stream Compression on the peripheral using the
+ * default Picture Parameter Set and VESA DSC 1.1 algorithm.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int mipi_dsi_compression_mode(struct mipi_dsi_device *dsi, bool enable)
+{
+	return mipi_dsi_compression_mode_ext(dsi, enable, MIPI_DSI_COMPRESSION_DSC, 0);
+}
 EXPORT_SYMBOL(mipi_dsi_compression_mode);
 
 /**
@@ -679,8 +706,8 @@ EXPORT_SYMBOL(mipi_dsi_compression_mode);
  *
  * Return: 0 on success or a negative error code on failure.
  */
-ssize_t mipi_dsi_picture_parameter_set(struct mipi_dsi_device *dsi,
-				       const struct drm_dsc_picture_parameter_set *pps)
+int mipi_dsi_picture_parameter_set(struct mipi_dsi_device *dsi,
+				   const struct drm_dsc_picture_parameter_set *pps)
 {
 	struct mipi_dsi_msg msg = {
 		.channel = dsi->channel,
diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c
index 48fd2d67f352..568972258222 100644
--- a/drivers/gpu/drm/drm_mode_config.c
+++ b/drivers/gpu/drm/drm_mode_config.c
@@ -372,6 +372,13 @@ static int drm_mode_create_standard_properties(struct drm_device *dev)
 		return -ENOMEM;
 	dev->mode_config.modifiers_property = prop;
 
+	prop = drm_property_create(dev,
+				   DRM_MODE_PROP_IMMUTABLE | DRM_MODE_PROP_BLOB,
+				   "SIZE_HINTS", 0);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.size_hints_property = prop;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index c4f88c3a93b7..2d8b0371619d 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -373,8 +373,8 @@ static int fill_analog_mode(struct drm_device *dev,
 	if (!bt601 &&
 	    (hact_duration_ns < params->hact_ns.min ||
 	     hact_duration_ns > params->hact_ns.max)) {
-		DRM_ERROR("Invalid horizontal active area duration: %uns (min: %u, max %u)\n",
-			  hact_duration_ns, params->hact_ns.min, params->hact_ns.max);
+		drm_err(dev, "Invalid horizontal active area duration: %uns (min: %u, max %u)\n",
+			hact_duration_ns, params->hact_ns.min, params->hact_ns.max);
 		return -EINVAL;
 	}
 
@@ -385,8 +385,8 @@ static int fill_analog_mode(struct drm_device *dev,
 	if (!bt601 &&
 	    (hblk_duration_ns < params->hblk_ns.min ||
 	     hblk_duration_ns > params->hblk_ns.max)) {
-		DRM_ERROR("Invalid horizontal blanking duration: %uns (min: %u, max %u)\n",
-			  hblk_duration_ns, params->hblk_ns.min, params->hblk_ns.max);
+		drm_err(dev, "Invalid horizontal blanking duration: %uns (min: %u, max %u)\n",
+			hblk_duration_ns, params->hblk_ns.min, params->hblk_ns.max);
 		return -EINVAL;
 	}
 
@@ -397,8 +397,8 @@ static int fill_analog_mode(struct drm_device *dev,
 	if (!bt601 &&
 	    (hslen_duration_ns < params->hslen_ns.min ||
 	     hslen_duration_ns > params->hslen_ns.max)) {
-		DRM_ERROR("Invalid horizontal sync duration: %uns (min: %u, max %u)\n",
-			  hslen_duration_ns, params->hslen_ns.min, params->hslen_ns.max);
+		drm_err(dev, "Invalid horizontal sync duration: %uns (min: %u, max %u)\n",
+			hslen_duration_ns, params->hslen_ns.min, params->hslen_ns.max);
 		return -EINVAL;
 	}
 
@@ -409,7 +409,8 @@ static int fill_analog_mode(struct drm_device *dev,
 	if (!bt601 &&
 	    (porches_duration_ns > (params->hfp_ns.max + params->hbp_ns.max) ||
 	     porches_duration_ns < (params->hfp_ns.min + params->hbp_ns.min))) {
-		DRM_ERROR("Invalid horizontal porches duration: %uns\n", porches_duration_ns);
+		drm_err(dev, "Invalid horizontal porches duration: %uns\n",
+			porches_duration_ns);
 		return -EINVAL;
 	}
 
@@ -431,8 +432,8 @@ static int fill_analog_mode(struct drm_device *dev,
 	if (!bt601 &&
 	    (hfp_duration_ns < params->hfp_ns.min ||
 	     hfp_duration_ns > params->hfp_ns.max)) {
-		DRM_ERROR("Invalid horizontal front porch duration: %uns (min: %u, max %u)\n",
-			  hfp_duration_ns, params->hfp_ns.min, params->hfp_ns.max);
+		drm_err(dev, "Invalid horizontal front porch duration: %uns (min: %u, max %u)\n",
+			hfp_duration_ns, params->hfp_ns.min, params->hfp_ns.max);
 		return -EINVAL;
 	}
 
@@ -443,8 +444,8 @@ static int fill_analog_mode(struct drm_device *dev,
 	if (!bt601 &&
 	    (hbp_duration_ns < params->hbp_ns.min ||
 	     hbp_duration_ns > params->hbp_ns.max)) {
-		DRM_ERROR("Invalid horizontal back porch duration: %uns (min: %u, max %u)\n",
-			  hbp_duration_ns, params->hbp_ns.min, params->hbp_ns.max);
+		drm_err(dev, "Invalid horizontal back porch duration: %uns (min: %u, max %u)\n",
+			hbp_duration_ns, params->hbp_ns.min, params->hbp_ns.max);
 		return -EINVAL;
 	}
 
@@ -495,8 +496,8 @@ static int fill_analog_mode(struct drm_device *dev,
 
 	vtotal = vactive + vfp + vslen + vbp;
 	if (params->num_lines != vtotal) {
-		DRM_ERROR("Invalid vertical total: %upx (expected %upx)\n",
-			  vtotal, params->num_lines);
+		drm_err(dev, "Invalid vertical total: %upx (expected %upx)\n",
+			vtotal, params->num_lines);
 		return -EINVAL;
 	}
 
@@ -1200,9 +1201,8 @@ int of_get_drm_display_mode(struct device_node *np,
 	if (bus_flags)
 		drm_bus_flags_from_videomode(&vm, bus_flags);
 
-	pr_debug("%pOF: got %dx%d display mode\n",
-		np, vm.hactive, vm.vactive);
-	drm_mode_debug_printmodeline(dmode);
+	pr_debug("%pOF: got %dx%d display mode: " DRM_MODE_FMT "\n",
+		 np, vm.hactive, vm.vactive, DRM_MODE_ARG(dmode));
 
 	return 0;
 }
@@ -1250,7 +1250,7 @@ int of_get_drm_panel_display_mode(struct device_node *np,
 	dmode->width_mm = width_mm;
 	dmode->height_mm = height_mm;
 
-	drm_mode_debug_printmodeline(dmode);
+	pr_debug(DRM_MODE_FMT "\n", DRM_MODE_ARG(dmode));
 
 	return 0;
 }
@@ -1812,10 +1812,8 @@ void drm_mode_prune_invalid(struct drm_device *dev,
 					 DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));
 			}
 			if (verbose) {
-				drm_mode_debug_printmodeline(mode);
-				DRM_DEBUG_KMS("Not using %s mode: %s\n",
-					      mode->name,
-					      drm_get_mode_status_name(mode->status));
+				drm_dbg_kms(dev, "Rejected mode: " DRM_MODE_FMT " (%s)\n",
+					    DRM_MODE_ARG(mode), drm_get_mode_status_name(mode->status));
 			}
 			drm_mode_destroy(dev, mode);
 		}
diff --git a/drivers/gpu/drm/drm_panic.c b/drivers/gpu/drm/drm_panic.c
new file mode 100644
index 000000000000..7ece67086cec
--- /dev/null
+++ b/drivers/gpu/drm/drm_panic.c
@@ -0,0 +1,585 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/*
+ * Copyright (c) 2023 Red Hat.
+ * Author: Jocelyn Falempe <jfalempe@redhat.com>
+ * inspired by the drm_log driver from David Herrmann <dh.herrmann@gmail.com>
+ * Tux Ascii art taken from cowsay written by Tony Monroe
+ */
+
+#include <linux/font.h>
+#include <linux/iosys-map.h>
+#include <linux/kdebug.h>
+#include <linux/kmsg_dump.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/types.h>
+
+#include <drm/drm_drv.h>
+#include <drm/drm_format_helper.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_panic.h>
+#include <drm/drm_plane.h>
+#include <drm/drm_print.h>
+
+MODULE_AUTHOR("Jocelyn Falempe");
+MODULE_DESCRIPTION("DRM panic handler");
+MODULE_LICENSE("GPL");
+
+/**
+ * DOC: overview
+ *
+ * To enable DRM panic for a driver, the primary plane must implement a
+ * &drm_plane_helper_funcs.get_scanout_buffer helper function. It is then
+ * automatically registered to the drm panic handler.
+ * When a panic occurs, the &drm_plane_helper_funcs.get_scanout_buffer will be
+ * called, and the driver can provide a framebuffer so the panic handler can
+ * draw the panic screen on it. Currently only linear buffer and a few color
+ * formats are supported.
+ * Optionally the driver can also provide a &drm_plane_helper_funcs.panic_flush
+ * callback, that will be called after that, to send additional commands to the
+ * hardware to make the scanout buffer visible.
+ */
+
+/*
+ * This module displays a user friendly message on screen when a kernel panic
+ * occurs. This is conflicting with fbcon, so you can only enable it when fbcon
+ * is disabled.
+ * It's intended for end-user, so have minimal technical/debug information.
+ *
+ * Implementation details:
+ *
+ * It is a panic handler, so it can't take lock, allocate memory, run tasks/irq,
+ * or attempt to sleep. It's a best effort, and it may not be able to display
+ * the message in all situations (like if the panic occurs in the middle of a
+ * modesetting).
+ * It will display only one static frame, so performance optimizations are low
+ * priority as the machine is already in an unusable state.
+ */
+
+struct drm_panic_line {
+	u32 len;
+	const char *txt;
+};
+
+#define PANIC_LINE(s) {.len = sizeof(s) - 1, .txt = s}
+
+static struct drm_panic_line panic_msg[] = {
+	PANIC_LINE("KERNEL PANIC !"),
+	PANIC_LINE(""),
+	PANIC_LINE("Please reboot your computer."),
+};
+
+static const struct drm_panic_line logo[] = {
+	PANIC_LINE("     .--.        _"),
+	PANIC_LINE("    |o_o |      | |"),
+	PANIC_LINE("    |:_/ |      | |"),
+	PANIC_LINE("   //   \\ \\     |_|"),
+	PANIC_LINE("  (|     | )     _"),
+	PANIC_LINE(" /'\\_   _/`\\    (_)"),
+	PANIC_LINE(" \\___)=(___/"),
+};
+
+/*
+ * Color conversion
+ */
+
+static u16 convert_xrgb8888_to_rgb565(u32 pix)
+{
+	return ((pix & 0x00F80000) >> 8) |
+	       ((pix & 0x0000FC00) >> 5) |
+	       ((pix & 0x000000F8) >> 3);
+}
+
+static u16 convert_xrgb8888_to_rgba5551(u32 pix)
+{
+	return ((pix & 0x00f80000) >> 8) |
+	       ((pix & 0x0000f800) >> 5) |
+	       ((pix & 0x000000f8) >> 2) |
+	       BIT(0); /* set alpha bit */
+}
+
+static u16 convert_xrgb8888_to_xrgb1555(u32 pix)
+{
+	return ((pix & 0x00f80000) >> 9) |
+	       ((pix & 0x0000f800) >> 6) |
+	       ((pix & 0x000000f8) >> 3);
+}
+
+static u16 convert_xrgb8888_to_argb1555(u32 pix)
+{
+	return BIT(15) | /* set alpha bit */
+	       ((pix & 0x00f80000) >> 9) |
+	       ((pix & 0x0000f800) >> 6) |
+	       ((pix & 0x000000f8) >> 3);
+}
+
+static u32 convert_xrgb8888_to_argb8888(u32 pix)
+{
+	return pix | GENMASK(31, 24); /* fill alpha bits */
+}
+
+static u32 convert_xrgb8888_to_xbgr8888(u32 pix)
+{
+	return ((pix & 0x00ff0000) >> 16) <<  0 |
+	       ((pix & 0x0000ff00) >>  8) <<  8 |
+	       ((pix & 0x000000ff) >>  0) << 16 |
+	       ((pix & 0xff000000) >> 24) << 24;
+}
+
+static u32 convert_xrgb8888_to_abgr8888(u32 pix)
+{
+	return ((pix & 0x00ff0000) >> 16) <<  0 |
+	       ((pix & 0x0000ff00) >>  8) <<  8 |
+	       ((pix & 0x000000ff) >>  0) << 16 |
+	       GENMASK(31, 24); /* fill alpha bits */
+}
+
+static u32 convert_xrgb8888_to_xrgb2101010(u32 pix)
+{
+	pix = ((pix & 0x000000FF) << 2) |
+	      ((pix & 0x0000FF00) << 4) |
+	      ((pix & 0x00FF0000) << 6);
+	return pix | ((pix >> 8) & 0x00300C03);
+}
+
+static u32 convert_xrgb8888_to_argb2101010(u32 pix)
+{
+	pix = ((pix & 0x000000FF) << 2) |
+	      ((pix & 0x0000FF00) << 4) |
+	      ((pix & 0x00FF0000) << 6);
+	return GENMASK(31, 30) /* set alpha bits */ | pix | ((pix >> 8) & 0x00300C03);
+}
+
+/*
+ * convert_from_xrgb8888 - convert one pixel from xrgb8888 to the desired format
+ * @color: input color, in xrgb8888 format
+ * @format: output format
+ *
+ * Returns:
+ * Color in the format specified, casted to u32.
+ * Or 0 if the format is not supported.
+ */
+static u32 convert_from_xrgb8888(u32 color, u32 format)
+{
+	switch (format) {
+	case DRM_FORMAT_RGB565:
+		return convert_xrgb8888_to_rgb565(color);
+	case DRM_FORMAT_RGBA5551:
+		return convert_xrgb8888_to_rgba5551(color);
+	case DRM_FORMAT_XRGB1555:
+		return convert_xrgb8888_to_xrgb1555(color);
+	case DRM_FORMAT_ARGB1555:
+		return convert_xrgb8888_to_argb1555(color);
+	case DRM_FORMAT_RGB888:
+	case DRM_FORMAT_XRGB8888:
+		return color;
+	case DRM_FORMAT_ARGB8888:
+		return convert_xrgb8888_to_argb8888(color);
+	case DRM_FORMAT_XBGR8888:
+		return convert_xrgb8888_to_xbgr8888(color);
+	case DRM_FORMAT_ABGR8888:
+		return convert_xrgb8888_to_abgr8888(color);
+	case DRM_FORMAT_XRGB2101010:
+		return convert_xrgb8888_to_xrgb2101010(color);
+	case DRM_FORMAT_ARGB2101010:
+		return convert_xrgb8888_to_argb2101010(color);
+	default:
+		WARN_ONCE(1, "Can't convert to %p4cc\n", &format);
+		return 0;
+	}
+}
+
+/*
+ * Blit & Fill
+ */
+static void drm_panic_blit16(struct iosys_map *dmap, unsigned int dpitch,
+			     const u8 *sbuf8, unsigned int spitch,
+			     unsigned int height, unsigned int width,
+			     u16 fg16, u16 bg16)
+{
+	unsigned int y, x;
+	u16 val16;
+
+	for (y = 0; y < height; y++) {
+		for (x = 0; x < width; x++) {
+			val16 = (sbuf8[(y * spitch) + x / 8] & (0x80 >> (x % 8))) ? fg16 : bg16;
+			iosys_map_wr(dmap, y * dpitch + x * sizeof(u16), u16, val16);
+		}
+	}
+}
+
+static void drm_panic_blit24(struct iosys_map *dmap, unsigned int dpitch,
+			     const u8 *sbuf8, unsigned int spitch,
+			     unsigned int height, unsigned int width,
+			     u32 fg32, u32 bg32)
+{
+	unsigned int y, x;
+	u32 val32;
+
+	for (y = 0; y < height; y++) {
+		for (x = 0; x < width; x++) {
+			u32 off = y * dpitch + x * 3;
+
+			val32 = (sbuf8[(y * spitch) + x / 8] & (0x80 >> (x % 8))) ? fg32 : bg32;
+
+			/* write blue-green-red to output in little endianness */
+			iosys_map_wr(dmap, off, u8, (val32 & 0x000000FF) >> 0);
+			iosys_map_wr(dmap, off + 1, u8, (val32 & 0x0000FF00) >> 8);
+			iosys_map_wr(dmap, off + 2, u8, (val32 & 0x00FF0000) >> 16);
+		}
+	}
+}
+
+static void drm_panic_blit32(struct iosys_map *dmap, unsigned int dpitch,
+			     const u8 *sbuf8, unsigned int spitch,
+			     unsigned int height, unsigned int width,
+			     u32 fg32, u32 bg32)
+{
+	unsigned int y, x;
+	u32 val32;
+
+	for (y = 0; y < height; y++) {
+		for (x = 0; x < width; x++) {
+			val32 = (sbuf8[(y * spitch) + x / 8] & (0x80 >> (x % 8))) ? fg32 : bg32;
+			iosys_map_wr(dmap, y * dpitch + x * sizeof(u32), u32, val32);
+		}
+	}
+}
+
+/*
+ * drm_panic_blit - convert a monochrome image to a linear framebuffer
+ * @dmap: destination iosys_map
+ * @dpitch: destination pitch in bytes
+ * @sbuf8: source buffer, in monochrome format, 8 pixels per byte.
+ * @spitch: source pitch in bytes
+ * @height: height of the image to copy, in pixels
+ * @width: width of the image to copy, in pixels
+ * @fg_color: foreground color, in destination format
+ * @bg_color: background color, in destination format
+ * @pixel_width: pixel width in bytes.
+ *
+ * This can be used to draw a font character, which is a monochrome image, to a
+ * framebuffer in other supported format.
+ */
+static void drm_panic_blit(struct iosys_map *dmap, unsigned int dpitch,
+			   const u8 *sbuf8, unsigned int spitch,
+			   unsigned int height, unsigned int width,
+			   u32 fg_color, u32 bg_color,
+			   unsigned int pixel_width)
+{
+	switch (pixel_width) {
+	case 2:
+		drm_panic_blit16(dmap, dpitch, sbuf8, spitch,
+				 height, width, fg_color, bg_color);
+	break;
+	case 3:
+		drm_panic_blit24(dmap, dpitch, sbuf8, spitch,
+				 height, width, fg_color, bg_color);
+	break;
+	case 4:
+		drm_panic_blit32(dmap, dpitch, sbuf8, spitch,
+				 height, width, fg_color, bg_color);
+	break;
+	default:
+		WARN_ONCE(1, "Can't blit with pixel width %d\n", pixel_width);
+	}
+}
+
+static void drm_panic_fill16(struct iosys_map *dmap, unsigned int dpitch,
+			     unsigned int height, unsigned int width,
+			     u16 color)
+{
+	unsigned int y, x;
+
+	for (y = 0; y < height; y++)
+		for (x = 0; x < width; x++)
+			iosys_map_wr(dmap, y * dpitch + x * sizeof(u16), u16, color);
+}
+
+static void drm_panic_fill24(struct iosys_map *dmap, unsigned int dpitch,
+			     unsigned int height, unsigned int width,
+			     u32 color)
+{
+	unsigned int y, x;
+
+	for (y = 0; y < height; y++) {
+		for (x = 0; x < width; x++) {
+			unsigned int off = y * dpitch + x * 3;
+
+			/* write blue-green-red to output in little endianness */
+			iosys_map_wr(dmap, off, u8, (color & 0x000000FF) >> 0);
+			iosys_map_wr(dmap, off + 1, u8, (color & 0x0000FF00) >> 8);
+			iosys_map_wr(dmap, off + 2, u8, (color & 0x00FF0000) >> 16);
+		}
+	}
+}
+
+static void drm_panic_fill32(struct iosys_map *dmap, unsigned int dpitch,
+			     unsigned int height, unsigned int width,
+			     u32 color)
+{
+	unsigned int y, x;
+
+	for (y = 0; y < height; y++)
+		for (x = 0; x < width; x++)
+			iosys_map_wr(dmap, y * dpitch + x * sizeof(u32), u32, color);
+}
+
+/*
+ * drm_panic_fill - Fill a rectangle with a color
+ * @dmap: destination iosys_map, pointing to the top left corner of the rectangle
+ * @dpitch: destination pitch in bytes
+ * @height: height of the rectangle, in pixels
+ * @width: width of the rectangle, in pixels
+ * @color: color to fill the rectangle.
+ * @pixel_width: pixel width in bytes
+ *
+ * Fill a rectangle with a color, in a linear framebuffer.
+ */
+static void drm_panic_fill(struct iosys_map *dmap, unsigned int dpitch,
+			   unsigned int height, unsigned int width,
+			   u32 color, unsigned int pixel_width)
+{
+	switch (pixel_width) {
+	case 2:
+		drm_panic_fill16(dmap, dpitch, height, width, color);
+	break;
+	case 3:
+		drm_panic_fill24(dmap, dpitch, height, width, color);
+	break;
+	case 4:
+		drm_panic_fill32(dmap, dpitch, height, width, color);
+	break;
+	default:
+		WARN_ONCE(1, "Can't fill with pixel width %d\n", pixel_width);
+	}
+}
+
+static const u8 *get_char_bitmap(const struct font_desc *font, char c, size_t font_pitch)
+{
+	return font->data + (c * font->height) * font_pitch;
+}
+
+static unsigned int get_max_line_len(const struct drm_panic_line *lines, int len)
+{
+	int i;
+	unsigned int max = 0;
+
+	for (i = 0; i < len; i++)
+		max = max(lines[i].len, max);
+	return max;
+}
+
+/*
+ * Draw a text in a rectangle on a framebuffer. The text is truncated if it overflows the rectangle
+ */
+static void draw_txt_rectangle(struct drm_scanout_buffer *sb,
+			       const struct font_desc *font,
+			       const struct drm_panic_line *msg,
+			       unsigned int msg_lines,
+			       bool centered,
+			       struct drm_rect *clip,
+			       u32 fg_color,
+			       u32 bg_color)
+{
+	int i, j;
+	const u8 *src;
+	size_t font_pitch = DIV_ROUND_UP(font->width, 8);
+	struct iosys_map dst;
+	unsigned int px_width = sb->format->cpp[0];
+	int left = 0;
+
+	msg_lines = min(msg_lines,  drm_rect_height(clip) / font->height);
+	for (i = 0; i < msg_lines; i++) {
+		size_t line_len = min(msg[i].len, drm_rect_width(clip) / font->width);
+
+		if (centered)
+			left = (drm_rect_width(clip) - (line_len * font->width)) / 2;
+
+		dst = sb->map[0];
+		iosys_map_incr(&dst, (clip->y1 + i * font->height) * sb->pitch[0] +
+				     (clip->x1 + left) * px_width);
+		for (j = 0; j < line_len; j++) {
+			src = get_char_bitmap(font, msg[i].txt[j], font_pitch);
+			drm_panic_blit(&dst, sb->pitch[0], src, font_pitch,
+				       font->height, font->width,
+				       fg_color, bg_color, px_width);
+			iosys_map_incr(&dst, font->width * px_width);
+		}
+	}
+}
+
+/*
+ * Draw the panic message at the center of the screen
+ */
+static void draw_panic_static(struct drm_scanout_buffer *sb)
+{
+	size_t msg_lines = ARRAY_SIZE(panic_msg);
+	size_t logo_lines = ARRAY_SIZE(logo);
+	u32 fg_color = CONFIG_DRM_PANIC_FOREGROUND_COLOR;
+	u32 bg_color = CONFIG_DRM_PANIC_BACKGROUND_COLOR;
+	const struct font_desc *font = get_default_font(sb->width, sb->height, NULL, NULL);
+	struct drm_rect r_logo, r_msg;
+
+	if (!font)
+		return;
+
+	fg_color = convert_from_xrgb8888(fg_color, sb->format->format);
+	bg_color = convert_from_xrgb8888(bg_color, sb->format->format);
+
+	r_logo = DRM_RECT_INIT(0, 0,
+			       get_max_line_len(logo, logo_lines) * font->width,
+			       logo_lines * font->height);
+	r_msg = DRM_RECT_INIT(0, 0,
+			      min(get_max_line_len(panic_msg, msg_lines) * font->width, sb->width),
+			      min(msg_lines * font->height, sb->height));
+
+	/* Center the panic message */
+	drm_rect_translate(&r_msg, (sb->width - r_msg.x2) / 2, (sb->height - r_msg.y2) / 2);
+
+	/* Fill with the background color, and draw text on top */
+	drm_panic_fill(&sb->map[0], sb->pitch[0], sb->height, sb->width,
+		       bg_color, sb->format->cpp[0]);
+
+	if ((r_msg.x1 >= drm_rect_width(&r_logo) || r_msg.y1 >= drm_rect_height(&r_logo)) &&
+	    drm_rect_width(&r_logo) < sb->width && drm_rect_height(&r_logo) < sb->height) {
+		draw_txt_rectangle(sb, font, logo, logo_lines, false, &r_logo, fg_color, bg_color);
+	}
+	draw_txt_rectangle(sb, font, panic_msg, msg_lines, true, &r_msg, fg_color, bg_color);
+}
+
+/*
+ * drm_panic_is_format_supported()
+ * @format: a fourcc color code
+ * Returns: true if supported, false otherwise.
+ *
+ * Check if drm_panic will be able to use this color format.
+ */
+static bool drm_panic_is_format_supported(const struct drm_format_info *format)
+{
+	if (format->num_planes != 1)
+		return false;
+	return convert_from_xrgb8888(0xffffff, format->format) != 0;
+}
+
+static void draw_panic_plane(struct drm_plane *plane)
+{
+	struct drm_scanout_buffer sb;
+	int ret;
+	unsigned long flags;
+
+	if (!drm_panic_trylock(plane->dev, flags))
+		return;
+
+	ret = plane->helper_private->get_scanout_buffer(plane, &sb);
+
+	if (!ret && drm_panic_is_format_supported(sb.format)) {
+		draw_panic_static(&sb);
+		if (plane->helper_private->panic_flush)
+			plane->helper_private->panic_flush(plane);
+	}
+	drm_panic_unlock(plane->dev, flags);
+}
+
+static struct drm_plane *to_drm_plane(struct kmsg_dumper *kd)
+{
+	return container_of(kd, struct drm_plane, kmsg_panic);
+}
+
+static void drm_panic(struct kmsg_dumper *dumper, enum kmsg_dump_reason reason)
+{
+	struct drm_plane *plane = to_drm_plane(dumper);
+
+	if (reason == KMSG_DUMP_PANIC)
+		draw_panic_plane(plane);
+}
+
+
+/*
+ * DEBUG FS, This is currently unsafe.
+ * Create one file per plane, so it's possible to debug one plane at a time.
+ * TODO: It would be better to emulate an NMI context.
+ */
+#ifdef CONFIG_DRM_PANIC_DEBUG
+#include <linux/debugfs.h>
+
+static ssize_t debugfs_trigger_write(struct file *file, const char __user *user_buf,
+				     size_t count, loff_t *ppos)
+{
+	bool run;
+
+	if (kstrtobool_from_user(user_buf, count, &run) == 0 && run) {
+		struct drm_plane *plane = file->private_data;
+
+		draw_panic_plane(plane);
+	}
+	return count;
+}
+
+static const struct file_operations dbg_drm_panic_ops = {
+	.owner = THIS_MODULE,
+	.write = debugfs_trigger_write,
+	.open = simple_open,
+};
+
+static void debugfs_register_plane(struct drm_plane *plane, int index)
+{
+	char fname[32];
+
+	snprintf(fname, 32, "drm_panic_plane_%d", index);
+	debugfs_create_file(fname, 0200, plane->dev->debugfs_root,
+			    plane, &dbg_drm_panic_ops);
+}
+#else
+static void debugfs_register_plane(struct drm_plane *plane, int index) {}
+#endif /* CONFIG_DRM_PANIC_DEBUG */
+
+/**
+ * drm_panic_register() - Initialize DRM panic for a device
+ * @dev: the drm device on which the panic screen will be displayed.
+ */
+void drm_panic_register(struct drm_device *dev)
+{
+	struct drm_plane *plane;
+	int registered_plane = 0;
+
+	if (!dev->mode_config.num_total_plane)
+		return;
+
+	drm_for_each_plane(plane, dev) {
+		if (!plane->helper_private || !plane->helper_private->get_scanout_buffer)
+			continue;
+		plane->kmsg_panic.dump = drm_panic;
+		plane->kmsg_panic.max_reason = KMSG_DUMP_PANIC;
+		if (kmsg_dump_register(&plane->kmsg_panic))
+			drm_warn(dev, "Failed to register panic handler\n");
+		else {
+			debugfs_register_plane(plane, registered_plane);
+			registered_plane++;
+		}
+	}
+	if (registered_plane)
+		drm_info(dev, "Registered %d planes with drm panic\n", registered_plane);
+}
+EXPORT_SYMBOL(drm_panic_register);
+
+/**
+ * drm_panic_unregister()
+ * @dev: the drm device previously registered.
+ */
+void drm_panic_unregister(struct drm_device *dev)
+{
+	struct drm_plane *plane;
+
+	if (!dev->mode_config.num_total_plane)
+		return;
+
+	drm_for_each_plane(plane, dev) {
+		if (!plane->helper_private || !plane->helper_private->get_scanout_buffer)
+			continue;
+		kmsg_dump_unregister(&plane->kmsg_panic);
+	}
+}
+EXPORT_SYMBOL(drm_panic_unregister);
diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c
index 672c655c7a8e..57662a1fd345 100644
--- a/drivers/gpu/drm/drm_plane.c
+++ b/drivers/gpu/drm/drm_plane.c
@@ -140,6 +140,25 @@
  *     DRM_FORMAT_MOD_LINEAR. Before linux kernel release v5.1 there have been
  *     various bugs in this area with inconsistencies between the capability
  *     flag and per-plane properties.
+ *
+ * SIZE_HINTS:
+ *     Blob property which contains the set of recommended plane size
+ *     which can used for simple "cursor like" use cases (eg. no scaling).
+ *     Using these hints frees userspace from extensive probing of
+ *     supported plane sizes through atomic/setcursor ioctls.
+ *
+ *     The blob contains an array of struct drm_plane_size_hint, in
+ *     order of preference. For optimal usage userspace should pick
+ *     the first size that satisfies its own requirements.
+ *
+ *     Drivers should only attach this property to planes that
+ *     support a very limited set of sizes.
+ *
+ *     Note that property value 0 (ie. no blob) is reserved for potential
+ *     future use. Current userspace is expected to ignore the property
+ *     if the value is 0, and fall back to some other means (eg.
+ *     &DRM_CAP_CURSOR_WIDTH and &DRM_CAP_CURSOR_HEIGHT) to determine
+ *     the appropriate plane size to use.
  */
 
 static unsigned int drm_num_planes(struct drm_device *dev)
@@ -1729,3 +1748,40 @@ int drm_plane_create_scaling_filter_property(struct drm_plane *plane,
 	return 0;
 }
 EXPORT_SYMBOL(drm_plane_create_scaling_filter_property);
+
+/**
+ * drm_plane_add_size_hints_property - create a size hints property
+ *
+ * @plane: drm plane
+ * @hints: size hints
+ * @num_hints: number of size hints
+ *
+ * Create a size hints property for the plane.
+ *
+ * RETURNS:
+ * Zero for success or -errno
+ */
+int drm_plane_add_size_hints_property(struct drm_plane *plane,
+				      const struct drm_plane_size_hint *hints,
+				      int num_hints)
+{
+	struct drm_device *dev = plane->dev;
+	struct drm_mode_config *config = &dev->mode_config;
+	struct drm_property_blob *blob;
+
+	/* extending to other plane types needs actual thought */
+	if (drm_WARN_ON(dev, plane->type != DRM_PLANE_TYPE_CURSOR))
+		return -EINVAL;
+
+	blob = drm_property_create_blob(dev,
+					array_size(sizeof(hints[0]), num_hints),
+					hints);
+	if (IS_ERR(blob))
+		return PTR_ERR(blob);
+
+	drm_object_attach_property(&plane->base, config->size_hints_property,
+				   blob->base.id);
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_plane_add_size_hints_property);
diff --git a/drivers/gpu/drm/drm_print.c b/drivers/gpu/drm/drm_print.c
index 699b7dbffd7b..cf2efb44722c 100644
--- a/drivers/gpu/drm/drm_print.c
+++ b/drivers/gpu/drm/drm_print.c
@@ -23,13 +23,13 @@
  * Rob Clark <robdclark@gmail.com>
  */
 
-#include <linux/stdarg.h>
-
+#include <linux/debugfs.h>
+#include <linux/dynamic_debug.h>
 #include <linux/io.h>
 #include <linux/moduleparam.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
-#include <linux/dynamic_debug.h>
+#include <linux/stdarg.h>
 
 #include <drm/drm.h>
 #include <drm/drm_drv.h>
diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c
index bf2dd1f46b6c..4f75a1cfd820 100644
--- a/drivers/gpu/drm/drm_probe_helper.c
+++ b/drivers/gpu/drm/drm_probe_helper.c
@@ -37,6 +37,7 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_fourcc.h>
+#include <drm/drm_managed.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
@@ -566,8 +567,8 @@ int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
 
 	drm_modeset_acquire_init(&ctx, 0);
 
-	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id,
-			connector->name);
+	drm_dbg_kms(dev, "[CONNECTOR:%d:%s]\n", connector->base.id,
+		    connector->name);
 
 retry:
 	ret = drm_modeset_lock(&dev->mode_config.connection_mutex, &ctx);
@@ -610,11 +611,10 @@ retry:
 	 * check here, and if anything changed start the hotplug code.
 	 */
 	if (old_status != connector->status) {
-		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n",
-			      connector->base.id,
-			      connector->name,
-			      drm_get_connector_status_name(old_status),
-			      drm_get_connector_status_name(connector->status));
+		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] status updated from %s to %s\n",
+			    connector->base.id, connector->name,
+			    drm_get_connector_status_name(old_status),
+			    drm_get_connector_status_name(connector->status));
 
 		/*
 		 * The hotplug event code might call into the fb
@@ -637,8 +637,8 @@ retry:
 		drm_kms_helper_poll_enable(dev);
 
 	if (connector->status == connector_status_disconnected) {
-		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] disconnected\n",
-			connector->base.id, connector->name);
+		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] disconnected\n",
+			    connector->base.id, connector->name);
 		drm_connector_update_edid_property(connector, NULL);
 		drm_mode_prune_invalid(dev, &connector->modes, false);
 		goto exit;
@@ -696,11 +696,13 @@ exit:
 
 	drm_mode_sort(&connector->modes);
 
-	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] probed modes :\n", connector->base.id,
-			connector->name);
+	drm_dbg_kms(dev, "[CONNECTOR:%d:%s] probed modes:\n",
+		    connector->base.id, connector->name);
+
 	list_for_each_entry(mode, &connector->modes, head) {
 		drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V);
-		drm_mode_debug_printmodeline(mode);
+		drm_dbg_kms(dev, "Probed mode: " DRM_MODE_FMT "\n",
+			    DRM_MODE_ARG(mode));
 	}
 
 	return count;
@@ -833,14 +835,12 @@ static void output_poll_execute(struct work_struct *work)
 			old = drm_get_connector_status_name(old_status);
 			new = drm_get_connector_status_name(connector->status);
 
-			DRM_DEBUG_KMS("[CONNECTOR:%d:%s] "
-				      "status updated from %s to %s\n",
-				      connector->base.id,
-				      connector->name,
-				      old, new);
-			DRM_DEBUG_KMS("[CONNECTOR:%d:%s] epoch counter %llu -> %llu\n",
-				      connector->base.id, connector->name,
-				      old_epoch_counter, connector->epoch_counter);
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] status updated from %s to %s\n",
+				    connector->base.id, connector->name,
+				    old, new);
+			drm_dbg_kms(dev, "[CONNECTOR:%d:%s] epoch counter %llu -> %llu\n",
+				    connector->base.id, connector->name,
+				    old_epoch_counter, connector->epoch_counter);
 
 			changed = true;
 		}
@@ -951,6 +951,32 @@ void drm_kms_helper_poll_fini(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_kms_helper_poll_fini);
 
+static void drm_kms_helper_poll_init_release(struct drm_device *dev, void *res)
+{
+	drm_kms_helper_poll_fini(dev);
+}
+
+/**
+ * drmm_kms_helper_poll_init - initialize and enable output polling
+ * @dev: drm_device
+ *
+ * This function initializes and then also enables output polling support for
+ * @dev similar to drm_kms_helper_poll_init(). Polling will automatically be
+ * cleaned up when the DRM device goes away.
+ *
+ * See drm_kms_helper_poll_init() for more information.
+ *
+ * Returns:
+ * 0 on success, or a negative errno code otherwise.
+ */
+int drmm_kms_helper_poll_init(struct drm_device *dev)
+{
+	drm_kms_helper_poll_init(dev);
+
+	return drmm_add_action_or_reset(dev, drm_kms_helper_poll_init_release, dev);
+}
+EXPORT_SYMBOL(drmm_kms_helper_poll_init);
+
 static bool check_connector_changed(struct drm_connector *connector)
 {
 	struct drm_device *dev = connector->dev;
@@ -1279,3 +1305,32 @@ int drm_connector_helper_tv_get_modes(struct drm_connector *connector)
 	return i;
 }
 EXPORT_SYMBOL(drm_connector_helper_tv_get_modes);
+
+/**
+ * drm_connector_helper_detect_from_ddc - Read EDID and detect connector status.
+ * @connector: The connector
+ * @ctx: Acquire context
+ * @force: Perform screen-destructive operations, if necessary
+ *
+ * Detects the connector status by reading the EDID using drm_probe_ddc(),
+ * which requires connector->ddc to be set. Returns connector_status_connected
+ * on success or connector_status_disconnected on failure.
+ *
+ * Returns:
+ * The connector status as defined by enum drm_connector_status.
+ */
+int drm_connector_helper_detect_from_ddc(struct drm_connector *connector,
+					 struct drm_modeset_acquire_ctx *ctx,
+					 bool force)
+{
+	struct i2c_adapter *ddc = connector->ddc;
+
+	if (!ddc)
+		return connector_status_unknown;
+
+	if (drm_probe_ddc(ddc))
+		return connector_status_connected;
+
+	return connector_status_disconnected;
+}
+EXPORT_SYMBOL(drm_connector_helper_detect_from_ddc);
diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c
index a953f69a34b6..bd9b8ab4f82b 100644
--- a/drivers/gpu/drm/drm_sysfs.c
+++ b/drivers/gpu/drm/drm_sysfs.c
@@ -209,10 +209,9 @@ static ssize_t status_store(struct device *device,
 		ret = -EINVAL;
 
 	if (old_force != connector->force || !connector->force) {
-		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] force updated from %d to %d or reprobing\n",
-			      connector->base.id,
-			      connector->name,
-			      old_force, connector->force);
+		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] force updated from %d to %d or reprobing\n",
+			    connector->base.id, connector->name,
+			    old_force, connector->force);
 
 		connector->funcs->fill_modes(connector,
 					     dev->mode_config.max_width,
@@ -383,8 +382,8 @@ int drm_sysfs_connector_add(struct drm_connector *connector)
 	if (r)
 		goto err_free;
 
-	DRM_DEBUG("adding \"%s\" to sysfs\n",
-		  connector->name);
+	drm_dbg_kms(dev, "[CONNECTOR:%d:%s] adding connector to sysfs\n",
+		    connector->base.id, connector->name);
 
 	r = device_add(kdev);
 	if (r) {
@@ -430,8 +429,9 @@ void drm_sysfs_connector_remove(struct drm_connector *connector)
 	if (dev_fwnode(connector->kdev))
 		component_del(connector->kdev, &typec_connector_ops);
 
-	DRM_DEBUG("removing \"%s\" from sysfs\n",
-		  connector->name);
+	drm_dbg_kms(connector->dev,
+		    "[CONNECTOR:%d:%s] removing connector from sysfs\n",
+		    connector->base.id, connector->name);
 
 	device_unregister(connector->kdev);
 	connector->kdev = NULL;
@@ -442,7 +442,7 @@ void drm_sysfs_lease_event(struct drm_device *dev)
 	char *event_string = "LEASE=1";
 	char *envp[] = { event_string, NULL };
 
-	DRM_DEBUG("generating lease event\n");
+	drm_dbg_lease(dev, "generating lease event\n");
 
 	kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, envp);
 }
@@ -463,7 +463,7 @@ void drm_sysfs_hotplug_event(struct drm_device *dev)
 	char *event_string = "HOTPLUG=1";
 	char *envp[] = { event_string, NULL };
 
-	DRM_DEBUG("generating hotplug event\n");
+	drm_dbg_kms(dev, "generating hotplug event\n");
 
 	kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, envp);
 }
diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c
index 702a12bc93bd..cc3571e25a9a 100644
--- a/drivers/gpu/drm/drm_vblank.c
+++ b/drivers/gpu/drm/drm_vblank.c
@@ -166,11 +166,24 @@ module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600)
 MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
 MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
 
+static struct drm_vblank_crtc *
+drm_vblank_crtc(struct drm_device *dev, unsigned int pipe)
+{
+	return &dev->vblank[pipe];
+}
+
+struct drm_vblank_crtc *
+drm_crtc_vblank_crtc(struct drm_crtc *crtc)
+{
+	return drm_vblank_crtc(crtc->dev, drm_crtc_index(crtc));
+}
+EXPORT_SYMBOL(drm_crtc_vblank_crtc);
+
 static void store_vblank(struct drm_device *dev, unsigned int pipe,
 			 u32 vblank_count_inc,
 			 ktime_t t_vblank, u32 last)
 {
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 
 	assert_spin_locked(&dev->vblank_time_lock);
 
@@ -184,7 +197,7 @@ static void store_vblank(struct drm_device *dev, unsigned int pipe,
 
 static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
 {
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 
 	return vblank->max_vblank_count ?: dev->max_vblank_count;
 }
@@ -273,7 +286,7 @@ static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe
 static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
 				    bool in_vblank_irq)
 {
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 	u32 cur_vblank, diff;
 	bool rc;
 	ktime_t t_vblank;
@@ -364,7 +377,7 @@ static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
 
 u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
 {
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 	u64 count;
 
 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
@@ -438,7 +451,7 @@ static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
  */
 void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
 {
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 	unsigned long irqflags;
 
 	assert_spin_locked(&dev->vbl_lock);
@@ -600,7 +613,7 @@ void drm_calc_timestamping_constants(struct drm_crtc *crtc,
 {
 	struct drm_device *dev = crtc->dev;
 	unsigned int pipe = drm_crtc_index(crtc);
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
 	int linedur_ns = 0, framedur_ns = 0;
 	int dotclock = mode->crtc_clock;
 
@@ -930,7 +943,7 @@ EXPORT_SYMBOL(drm_crtc_vblank_count);
 static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
 				     ktime_t *vblanktime)
 {
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 	u64 vblank_count;
 	unsigned int seq;
 
@@ -985,7 +998,6 @@ EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
  */
 int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime)
 {
-	unsigned int pipe = drm_crtc_index(crtc);
 	struct drm_vblank_crtc *vblank;
 	struct drm_display_mode *mode;
 	u64 vblank_start;
@@ -993,7 +1005,7 @@ int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime)
 	if (!drm_dev_has_vblank(crtc->dev))
 		return -EINVAL;
 
-	vblank = &crtc->dev->vblank[pipe];
+	vblank = drm_crtc_vblank_crtc(crtc);
 	mode = &vblank->hwmode;
 
 	if (!vblank->framedur_ns || !vblank->linedur_ns)
@@ -1147,7 +1159,7 @@ static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
 
 static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
 {
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 	int ret = 0;
 
 	assert_spin_locked(&dev->vbl_lock);
@@ -1185,7 +1197,7 @@ static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
 
 int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
 {
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 	unsigned long irqflags;
 	int ret = 0;
 
@@ -1228,7 +1240,7 @@ EXPORT_SYMBOL(drm_crtc_vblank_get);
 
 void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
 {
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 
 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
 		return;
@@ -1274,7 +1286,7 @@ EXPORT_SYMBOL(drm_crtc_vblank_put);
  */
 void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
 {
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 	int ret;
 	u64 last;
 
@@ -1327,7 +1339,7 @@ void drm_crtc_vblank_off(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
 	unsigned int pipe = drm_crtc_index(crtc);
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
 	struct drm_pending_vblank_event *e, *t;
 	ktime_t now;
 	u64 seq;
@@ -1405,8 +1417,7 @@ EXPORT_SYMBOL(drm_crtc_vblank_off);
 void drm_crtc_vblank_reset(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
-	unsigned int pipe = drm_crtc_index(crtc);
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
 
 	spin_lock_irq(&dev->vbl_lock);
 	/*
@@ -1445,8 +1456,7 @@ void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
 				   u32 max_vblank_count)
 {
 	struct drm_device *dev = crtc->dev;
-	unsigned int pipe = drm_crtc_index(crtc);
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
 
 	drm_WARN_ON(dev, dev->max_vblank_count);
 	drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset));
@@ -1469,7 +1479,7 @@ void drm_crtc_vblank_on(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
 	unsigned int pipe = drm_crtc_index(crtc);
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
 
 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
 		return;
@@ -1512,7 +1522,7 @@ static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
 	assert_spin_locked(&dev->vbl_lock);
 	assert_spin_locked(&dev->vblank_time_lock);
 
-	vblank = &dev->vblank[pipe];
+	vblank = drm_vblank_crtc(dev, pipe);
 	drm_WARN_ONCE(dev,
 		      drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,
 		      "Cannot compute missed vblanks without frame duration\n");
@@ -1564,7 +1574,7 @@ static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
 				  union drm_wait_vblank *vblwait,
 				  struct drm_file *file_priv)
 {
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 	struct drm_pending_vblank_event *e;
 	ktime_t now;
 	u64 seq;
@@ -1872,7 +1882,7 @@ static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
  */
 bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
 {
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 	unsigned long irqflags;
 	bool disable_irq;
 
@@ -1981,7 +1991,7 @@ int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
 
 	pipe = drm_crtc_index(crtc);
 
-	vblank = &dev->vblank[pipe];
+	vblank = drm_crtc_vblank_crtc(crtc);
 	vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
 
 	if (!vblank_enabled) {
@@ -2046,7 +2056,7 @@ int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
 
 	pipe = drm_crtc_index(crtc);
 
-	vblank = &dev->vblank[pipe];
+	vblank = drm_crtc_vblank_crtc(crtc);
 
 	e = kzalloc(sizeof(*e), GFP_KERNEL);
 	if (e == NULL)
diff --git a/drivers/gpu/drm/drm_vblank_work.c b/drivers/gpu/drm/drm_vblank_work.c
index 43cd5c0f4f6f..4fe9b1d3b00f 100644
--- a/drivers/gpu/drm/drm_vblank_work.c
+++ b/drivers/gpu/drm/drm_vblank_work.c
@@ -245,7 +245,7 @@ void drm_vblank_work_init(struct drm_vblank_work *work, struct drm_crtc *crtc,
 {
 	kthread_init_work(&work->base, func);
 	INIT_LIST_HEAD(&work->node);
-	work->vblank = &crtc->dev->vblank[drm_crtc_index(crtc)];
+	work->vblank = drm_crtc_vblank_crtc(crtc);
 }
 EXPORT_SYMBOL(drm_vblank_work_init);
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index 734412aae94d..a9bf426f69b3 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -164,26 +164,6 @@ int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value)
 		*value = gpu->identity.eco_id;
 		break;
 
-	case ETNAVIV_PARAM_GPU_NN_CORE_COUNT:
-		*value = gpu->identity.nn_core_count;
-		break;
-
-	case ETNAVIV_PARAM_GPU_NN_MAD_PER_CORE:
-		*value = gpu->identity.nn_mad_per_core;
-		break;
-
-	case ETNAVIV_PARAM_GPU_TP_CORE_COUNT:
-		*value = gpu->identity.tp_core_count;
-		break;
-
-	case ETNAVIV_PARAM_GPU_ON_CHIP_SRAM_SIZE:
-		*value = gpu->identity.on_chip_sram_size;
-		break;
-
-	case ETNAVIV_PARAM_GPU_AXI_SRAM_SIZE:
-		*value = gpu->identity.axi_sram_size;
-		break;
-
 	default:
 		DBG("%s: invalid param: %u", dev_name(gpu->dev), param);
 		return -EINVAL;
@@ -663,8 +643,8 @@ static void etnaviv_gpu_enable_mlcg(struct etnaviv_gpu *gpu)
 	/* Disable TX clock gating on affected core revisions. */
 	if (etnaviv_is_model_rev(gpu, GC4000, 0x5222) ||
 	    etnaviv_is_model_rev(gpu, GC2000, 0x5108) ||
-	    etnaviv_is_model_rev(gpu, GC2000, 0x6202) ||
-	    etnaviv_is_model_rev(gpu, GC2000, 0x6203))
+	    etnaviv_is_model_rev(gpu, GC7000, 0x6202) ||
+	    etnaviv_is_model_rev(gpu, GC7000, 0x6203))
 		pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_TX;
 
 	/* Disable SE and RA clock gating on affected core revisions. */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
index 7d5e9158e13c..197e0037732e 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
@@ -54,18 +54,6 @@ struct etnaviv_chip_identity {
 	/* Number of Neural Network cores. */
 	u32 nn_core_count;
 
-	/* Number of MAD units per Neural Network core. */
-	u32 nn_mad_per_core;
-
-	/* Number of Tensor Processing cores. */
-	u32 tp_core_count;
-
-	/* Size in bytes of the SRAM inside the NPU. */
-	u32 on_chip_sram_size;
-
-	/* Size in bytes of the SRAM across the AXI bus. */
-	u32 axi_sram_size;
-
 	/* Size of the vertex cache. */
 	u32 vertex_cache_size;
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c b/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c
index d8e7334de8ce..8665f2658d51 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c
@@ -17,10 +17,6 @@ static const struct etnaviv_chip_identity etnaviv_chip_identities[] = {
 		.thread_count = 128,
 		.shader_core_count = 1,
 		.nn_core_count = 0,
-		.nn_mad_per_core = 0,
-		.tp_core_count = 0,
-		.on_chip_sram_size = 0,
-		.axi_sram_size = 0,
 		.vertex_cache_size = 8,
 		.vertex_output_buffer_size = 1024,
 		.pixel_pipes = 1,
@@ -52,11 +48,6 @@ static const struct etnaviv_chip_identity etnaviv_chip_identities[] = {
 		.register_max = 64,
 		.thread_count = 256,
 		.shader_core_count = 1,
-		.nn_core_count = 0,
-		.nn_mad_per_core = 0,
-		.tp_core_count = 0,
-		.on_chip_sram_size = 0,
-		.axi_sram_size = 0,
 		.vertex_cache_size = 8,
 		.vertex_output_buffer_size = 512,
 		.pixel_pipes = 1,
@@ -89,10 +80,6 @@ static const struct etnaviv_chip_identity etnaviv_chip_identities[] = {
 		.thread_count = 512,
 		.shader_core_count = 2,
 		.nn_core_count = 0,
-		.nn_mad_per_core = 0,
-		.tp_core_count = 0,
-		.on_chip_sram_size = 0,
-		.axi_sram_size = 0,
 		.vertex_cache_size = 16,
 		.vertex_output_buffer_size = 1024,
 		.pixel_pipes = 1,
@@ -125,10 +112,6 @@ static const struct etnaviv_chip_identity etnaviv_chip_identities[] = {
 		.thread_count = 512,
 		.shader_core_count = 2,
 		.nn_core_count = 0,
-		.nn_mad_per_core = 0,
-		.tp_core_count = 0,
-		.on_chip_sram_size = 0,
-		.axi_sram_size = 0,
 		.vertex_cache_size = 16,
 		.vertex_output_buffer_size = 1024,
 		.pixel_pipes = 1,
@@ -160,11 +143,6 @@ static const struct etnaviv_chip_identity etnaviv_chip_identities[] = {
 		.register_max = 64,
 		.thread_count = 512,
 		.shader_core_count = 2,
-		.nn_core_count = 0,
-		.nn_mad_per_core = 0,
-		.tp_core_count = 0,
-		.on_chip_sram_size = 0,
-		.axi_sram_size = 0,
 		.vertex_cache_size = 16,
 		.vertex_output_buffer_size = 1024,
 		.pixel_pipes = 1,
@@ -197,10 +175,6 @@ static const struct etnaviv_chip_identity etnaviv_chip_identities[] = {
 		.thread_count = 1024,
 		.shader_core_count = 4,
 		.nn_core_count = 0,
-		.nn_mad_per_core = 0,
-		.tp_core_count = 0,
-		.on_chip_sram_size = 0,
-		.axi_sram_size = 0,
 		.vertex_cache_size = 16,
 		.vertex_output_buffer_size = 1024,
 		.pixel_pipes = 2,
@@ -233,10 +207,6 @@ static const struct etnaviv_chip_identity etnaviv_chip_identities[] = {
 		.thread_count = 256,
 		.shader_core_count = 1,
 		.nn_core_count = 8,
-		.nn_mad_per_core = 64,
-		.tp_core_count = 4,
-		.on_chip_sram_size = 524288,
-		.axi_sram_size = 1048576,
 		.vertex_cache_size = 16,
 		.vertex_output_buffer_size = 1024,
 		.pixel_pipes = 1,
@@ -269,10 +239,6 @@ static const struct etnaviv_chip_identity etnaviv_chip_identities[] = {
 		.thread_count = 256,
 		.shader_core_count = 1,
 		.nn_core_count = 6,
-		.nn_mad_per_core = 64,
-		.tp_core_count = 3,
-		.on_chip_sram_size = 262144,
-		.axi_sram_size = 0,
 		.vertex_cache_size = 16,
 		.vertex_output_buffer_size = 1024,
 		.pixel_pipes = 1,
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dsi.c b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
index 2fe0e5f3f638..bf16deaae68b 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dsi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
@@ -184,7 +184,6 @@ struct platform_driver dsi_driver = {
 	.remove_new = samsung_dsim_remove,
 	.driver = {
 		   .name = "exynos-dsi",
-		   .owner = THIS_MODULE,
 		   .pm = &samsung_dsim_pm_ops,
 		   .of_match_table = exynos_dsi_of_match,
 	},
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimc.c b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
index e81a576de398..142184c8c3bc 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
@@ -1412,7 +1412,6 @@ struct platform_driver fimc_driver = {
 	.driver		= {
 		.of_match_table = fimc_of_match,
 		.name	= "exynos-drm-fimc",
-		.owner	= THIS_MODULE,
 		.pm	= pm_ptr(&fimc_pm_ops),
 	},
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
index f2145227a1e0..f57df8c48139 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
@@ -1326,7 +1326,6 @@ struct platform_driver fimd_driver = {
 	.remove_new	= fimd_remove,
 	.driver		= {
 		.name	= "exynos4-fb",
-		.owner	= THIS_MODULE,
 		.pm	= pm_ptr(&exynos_fimd_pm_ops),
 		.of_match_table = fimd_driver_dt_match,
 	},
diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
index f3138423612e..3a3b2c00e400 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
@@ -1610,7 +1610,6 @@ struct platform_driver g2d_driver = {
 	.remove_new	= g2d_remove,
 	.driver		= {
 		.name	= "exynos-drm-g2d",
-		.owner	= THIS_MODULE,
 		.pm	= pm_ptr(&g2d_pm_ops),
 		.of_match_table = exynos_g2d_match,
 	},
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gsc.c b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
index 180507a47700..1b111e2c3347 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gsc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
@@ -1423,7 +1423,6 @@ struct platform_driver gsc_driver = {
 	.remove_new	= gsc_remove,
 	.driver		= {
 		.name	= "exynos-drm-gsc",
-		.owner	= THIS_MODULE,
 		.pm	= &gsc_pm_ops,
 		.of_match_table = exynos_drm_gsc_of_match,
 	},
diff --git a/drivers/gpu/drm/exynos/exynos_drm_mic.c b/drivers/gpu/drm/exynos/exynos_drm_mic.c
index e2920960180f..d61ec451807c 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_mic.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_mic.c
@@ -464,7 +464,6 @@ struct platform_driver mic_driver = {
 	.driver		= {
 		.name	= "exynos-mic",
 		.pm	= pm_ptr(&exynos_mic_pm_ops),
-		.owner	= THIS_MODULE,
 		.of_match_table = exynos_mic_of_match,
 	},
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_rotator.c b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
index 5f7516655b08..2eb0b701672f 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_rotator.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
@@ -454,7 +454,6 @@ struct platform_driver rotator_driver = {
 	.remove_new	= rotator_remove,
 	.driver		= {
 		.name	= "exynos-rotator",
-		.owner	= THIS_MODULE,
 		.pm	= pm_ptr(&rotator_pm_ops),
 		.of_match_table = exynos_rotator_match,
 	},
diff --git a/drivers/gpu/drm/exynos/exynos_drm_scaler.c b/drivers/gpu/drm/exynos/exynos_drm_scaler.c
index 392f721f13ab..a9d469896824 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_scaler.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_scaler.c
@@ -722,7 +722,6 @@ struct platform_driver scaler_driver = {
 	.remove_new	= scaler_remove,
 	.driver		= {
 		.name	= "exynos-scaler",
-		.owner	= THIS_MODULE,
 		.pm	= pm_ptr(&scaler_pm_ops),
 		.of_match_table = exynos_scaler_match,
 	},
diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
index f5bbba9ad225..fab135308b70 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
@@ -479,7 +479,6 @@ struct platform_driver vidi_driver = {
 	.remove_new	= vidi_remove,
 	.driver		= {
 		.name	= "exynos-drm-vidi",
-		.owner	= THIS_MODULE,
 		.dev_groups = vidi_groups,
 	},
 };
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index b1d02dec3774..e968824a4c72 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -1919,10 +1919,9 @@ static int hdmi_get_ddc_adapter(struct hdmi_context *hdata)
 static int hdmi_get_phy_io(struct hdmi_context *hdata)
 {
 	const char *compatible_str = "samsung,exynos4212-hdmiphy";
-	struct device_node *np;
-	int ret = 0;
+	struct device_node *np __free(device_node) =
+		of_find_compatible_node(NULL, NULL, compatible_str);
 
-	np = of_find_compatible_node(NULL, NULL, compatible_str);
 	if (!np) {
 		np = of_parse_phandle(hdata->dev->of_node, "phy", 0);
 		if (!np) {
@@ -1937,21 +1936,17 @@ static int hdmi_get_phy_io(struct hdmi_context *hdata)
 		if (!hdata->regs_hdmiphy) {
 			DRM_DEV_ERROR(hdata->dev,
 				      "failed to ioremap hdmi phy\n");
-			ret = -ENOMEM;
-			goto out;
+			return -ENOMEM;
 		}
 	} else {
 		hdata->hdmiphy_port = of_find_i2c_device_by_node(np);
 		if (!hdata->hdmiphy_port) {
 			DRM_INFO("Failed to get hdmi phy i2c client\n");
-			ret = -EPROBE_DEFER;
-			goto out;
+			return -EPROBE_DEFER;
 		}
 	}
 
-out:
-	of_node_put(np);
-	return ret;
+	return 0;
 }
 
 static int hdmi_probe(struct platform_device *pdev)
@@ -2126,7 +2121,6 @@ struct platform_driver hdmi_driver = {
 	.remove_new	= hdmi_remove,
 	.driver		= {
 		.name	= "exynos-hdmi",
-		.owner	= THIS_MODULE,
 		.pm	= &exynos_hdmi_pm_ops,
 		.of_match_table = hdmi_match_types,
 	},
diff --git a/drivers/gpu/drm/exynos/exynos_mixer.c b/drivers/gpu/drm/exynos/exynos_mixer.c
index 6822333fd0e6..1db955f00044 100644
--- a/drivers/gpu/drm/exynos/exynos_mixer.c
+++ b/drivers/gpu/drm/exynos/exynos_mixer.c
@@ -1331,7 +1331,6 @@ static const struct dev_pm_ops exynos_mixer_pm_ops = {
 struct platform_driver mixer_driver = {
 	.driver = {
 		.name = "exynos-mixer",
-		.owner = THIS_MODULE,
 		.pm = &exynos_mixer_pm_ops,
 		.of_match_table = mixer_match_types,
 	},
diff --git a/drivers/gpu/drm/gma500/Makefile b/drivers/gpu/drm/gma500/Makefile
index 4f302cd5e1a6..58fed80c7392 100644
--- a/drivers/gpu/drm/gma500/Makefile
+++ b/drivers/gpu/drm/gma500/Makefile
@@ -34,7 +34,6 @@ gma500_gfx-y += \
 	  psb_intel_lvds.o \
 	  psb_intel_modes.o \
 	  psb_intel_sdvo.o \
-	  psb_lid.o \
 	  psb_irq.o
 
 gma500_gfx-$(CONFIG_ACPI) +=  opregion.o
diff --git a/drivers/gpu/drm/gma500/mmu.c b/drivers/gpu/drm/gma500/mmu.c
index a70b01ccdf70..4d78b33eaa82 100644
--- a/drivers/gpu/drm/gma500/mmu.c
+++ b/drivers/gpu/drm/gma500/mmu.c
@@ -5,6 +5,7 @@
  **************************************************************************/
 
 #include <linux/highmem.h>
+#include <linux/vmalloc.h>
 
 #include "mmu.h"
 #include "psb_drv.h"
diff --git a/drivers/gpu/drm/gma500/oaktrail_lvds.c b/drivers/gpu/drm/gma500/oaktrail_lvds.c
index d974d0c60d2a..72191d6f0d06 100644
--- a/drivers/gpu/drm/gma500/oaktrail_lvds.c
+++ b/drivers/gpu/drm/gma500/oaktrail_lvds.c
@@ -11,8 +11,6 @@
 #include <linux/i2c.h>
 #include <linux/pm_runtime.h>
 
-#include <asm/intel-mid.h>
-
 #include <drm/drm_edid.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_simple_kms_helper.h>
diff --git a/drivers/gpu/drm/gma500/psb_device.c b/drivers/gpu/drm/gma500/psb_device.c
index dcfcd7b89d4a..6dece8f0e380 100644
--- a/drivers/gpu/drm/gma500/psb_device.c
+++ b/drivers/gpu/drm/gma500/psb_device.c
@@ -73,8 +73,7 @@ static int psb_backlight_setup(struct drm_device *dev)
 	}
 
 	psb_intel_lvds_set_brightness(dev, PSB_MAX_BRIGHTNESS);
-	/* This must occur after the backlight is properly initialised */
-	psb_lid_timer_init(dev_priv);
+
 	return 0;
 }
 
@@ -259,8 +258,6 @@ static int psb_chip_setup(struct drm_device *dev)
 
 static void psb_chip_teardown(struct drm_device *dev)
 {
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	psb_lid_timer_takedown(dev_priv);
 	gma_intel_teardown_gmbus(dev);
 }
 
diff --git a/drivers/gpu/drm/gma500/psb_drv.h b/drivers/gpu/drm/gma500/psb_drv.h
index c5edfa4aa4cc..83c17689c454 100644
--- a/drivers/gpu/drm/gma500/psb_drv.h
+++ b/drivers/gpu/drm/gma500/psb_drv.h
@@ -162,7 +162,6 @@
 #define PSB_NUM_VBLANKS 2
 
 #define PSB_WATCHDOG_DELAY (HZ * 2)
-#define PSB_LID_DELAY (HZ / 10)
 
 #define PSB_MAX_BRIGHTNESS		100
 
@@ -491,11 +490,7 @@ struct drm_psb_private {
 	/* Hotplug handling */
 	struct work_struct hotplug_work;
 
-	/* LID-Switch */
-	spinlock_t lid_lock;
-	struct timer_list lid_timer;
 	struct psb_intel_opregion opregion;
-	u32 lid_last_state;
 
 	/* Watchdog */
 	uint32_t apm_reg;
@@ -591,10 +586,6 @@ struct psb_ops {
 	int i2c_bus;		/* I2C bus identifier for Moorestown */
 };
 
-/* psb_lid.c */
-extern void psb_lid_timer_init(struct drm_psb_private *dev_priv);
-extern void psb_lid_timer_takedown(struct drm_psb_private *dev_priv);
-
 /* modesetting */
 extern void psb_modeset_init(struct drm_device *dev);
 extern void psb_modeset_cleanup(struct drm_device *dev);
diff --git a/drivers/gpu/drm/gma500/psb_lid.c b/drivers/gpu/drm/gma500/psb_lid.c
deleted file mode 100644
index 58a7fe392636..000000000000
--- a/drivers/gpu/drm/gma500/psb_lid.c
+++ /dev/null
@@ -1,80 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/**************************************************************************
- * Copyright (c) 2007, Intel Corporation.
- *
- * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
- **************************************************************************/
-
-#include <linux/spinlock.h>
-
-#include "psb_drv.h"
-#include "psb_intel_reg.h"
-#include "psb_reg.h"
-
-static void psb_lid_timer_func(struct timer_list *t)
-{
-	struct drm_psb_private *dev_priv = from_timer(dev_priv, t, lid_timer);
-	struct drm_device *dev = (struct drm_device *)&dev_priv->dev;
-	struct timer_list *lid_timer = &dev_priv->lid_timer;
-	unsigned long irq_flags;
-	u32 __iomem *lid_state = dev_priv->opregion.lid_state;
-	u32 pp_status;
-
-	if (readl(lid_state) == dev_priv->lid_last_state)
-		goto lid_timer_schedule;
-
-	if ((readl(lid_state)) & 0x01) {
-		/*lid state is open*/
-		REG_WRITE(PP_CONTROL, REG_READ(PP_CONTROL) | POWER_TARGET_ON);
-		do {
-			pp_status = REG_READ(PP_STATUS);
-		} while ((pp_status & PP_ON) == 0 &&
-			 (pp_status & PP_SEQUENCE_MASK) != 0);
-
-		if (REG_READ(PP_STATUS) & PP_ON) {
-			/*FIXME: should be backlight level before*/
-			psb_intel_lvds_set_brightness(dev, 100);
-		} else {
-			DRM_DEBUG("LVDS panel never powered up");
-			return;
-		}
-	} else {
-		psb_intel_lvds_set_brightness(dev, 0);
-
-		REG_WRITE(PP_CONTROL, REG_READ(PP_CONTROL) & ~POWER_TARGET_ON);
-		do {
-			pp_status = REG_READ(PP_STATUS);
-		} while ((pp_status & PP_ON) == 0);
-	}
-	dev_priv->lid_last_state =  readl(lid_state);
-
-lid_timer_schedule:
-	spin_lock_irqsave(&dev_priv->lid_lock, irq_flags);
-	if (!timer_pending(lid_timer)) {
-		lid_timer->expires = jiffies + PSB_LID_DELAY;
-		add_timer(lid_timer);
-	}
-	spin_unlock_irqrestore(&dev_priv->lid_lock, irq_flags);
-}
-
-void psb_lid_timer_init(struct drm_psb_private *dev_priv)
-{
-	struct timer_list *lid_timer = &dev_priv->lid_timer;
-	unsigned long irq_flags;
-
-	spin_lock_init(&dev_priv->lid_lock);
-	spin_lock_irqsave(&dev_priv->lid_lock, irq_flags);
-
-	timer_setup(lid_timer, psb_lid_timer_func, 0);
-
-	lid_timer->expires = jiffies + PSB_LID_DELAY;
-
-	add_timer(lid_timer);
-	spin_unlock_irqrestore(&dev_priv->lid_lock, irq_flags);
-}
-
-void psb_lid_timer_takedown(struct drm_psb_private *dev_priv)
-{
-	del_timer_sync(&dev_priv->lid_timer);
-}
-
diff --git a/drivers/gpu/drm/gud/gud_connector.c b/drivers/gpu/drm/gud/gud_connector.c
index 034e78360d4f..0f07d77c5d52 100644
--- a/drivers/gpu/drm/gud/gud_connector.c
+++ b/drivers/gpu/drm/gud/gud_connector.c
@@ -221,7 +221,7 @@ static int gud_connector_get_modes(struct drm_connector *connector)
 	struct gud_display_mode_req *reqmodes = NULL;
 	struct gud_connector_get_edid_ctx edid_ctx;
 	unsigned int i, num_modes = 0;
-	struct edid *edid = NULL;
+	const struct drm_edid *drm_edid = NULL;
 	int idx, ret;
 
 	if (!drm_dev_enter(connector->dev, &idx))
@@ -238,13 +238,13 @@ static int gud_connector_get_modes(struct drm_connector *connector)
 		gud_conn_err(connector, "Invalid EDID size", ret);
 	} else if (ret > 0) {
 		edid_ctx.len = ret;
-		edid = drm_do_get_edid(connector, gud_connector_get_edid_block, &edid_ctx);
+		drm_edid = drm_edid_read_custom(connector, gud_connector_get_edid_block, &edid_ctx);
 	}
 
 	kfree(edid_ctx.buf);
-	drm_connector_update_edid_property(connector, edid);
+	drm_edid_connector_update(connector, drm_edid);
 
-	if (edid && edid_ctx.edid_override)
+	if (drm_edid && edid_ctx.edid_override)
 		goto out;
 
 	reqmodes = kmalloc_array(GUD_CONNECTOR_MAX_NUM_MODES, sizeof(*reqmodes), GFP_KERNEL);
@@ -276,10 +276,10 @@ static int gud_connector_get_modes(struct drm_connector *connector)
 	}
 out:
 	if (!num_modes)
-		num_modes = drm_add_edid_modes(connector, edid);
+		num_modes = drm_edid_connector_add_modes(connector);
 
 	kfree(reqmodes);
-	kfree(edid);
+	drm_edid_free(drm_edid);
 	drm_dev_exit(idx);
 
 	return num_modes;
diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index bc18e2d9ea05..d8397065c3f0 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -27,8 +27,8 @@ config DRM_I915_DEBUG
 	select REF_TRACKER
 	select STACKDEPOT
 	select STACKTRACE
-	select DRM_DP_AUX_CHARDEV
-	select DRM_DISPLAY_DEBUG_DP_TUNNEL_STATE if DRM_I915_DP_TUNNEL
+	select DRM_DISPLAY_DP_AUX_CHARDEV
+	select DRM_DISPLAY_DP_TUNNEL_STATE_DEBUG if DRM_I915_DP_TUNNEL
 	select X86_MSR # used by igt/pm_rpm
 	select DRM_VGEM # used by igt/prime_vgem (dmabuf interop checks)
 	select DRM_DEBUG_MM if DRM=y
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index fba73c38e235..c8c8b31da4fb 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -32,16 +32,11 @@ endif
 # Enable -Werror in CI and development
 subdir-ccflags-$(CONFIG_DRM_I915_WERROR) += -Werror
 
-# Fine grained warnings disable
-CFLAGS_i915_pci.o = -Wno-override-init
-CFLAGS_display/intel_display_device.o = -Wno-override-init
-CFLAGS_display/intel_fbdev.o = -Wno-override-init
-
 # Support compiling the display code separately for both i915 and xe
 # drivers. Define I915 when building i915.
 subdir-ccflags-y += -DI915
 
-subdir-ccflags-y += -I$(srctree)/$(src)
+subdir-ccflags-y += -I$(src)
 
 # Please keep these build lists sorted!
 
@@ -271,6 +266,7 @@ i915-y += \
 	display/intel_display_rps.o \
 	display/intel_display_wa.o \
 	display/intel_dmc.o \
+	display/intel_dmc_wl.o \
 	display/intel_dpio_phy.o \
 	display/intel_dpll.o \
 	display/intel_dpll_mgr.o \
@@ -434,7 +430,7 @@ no-header-test := \
 
 always-$(CONFIG_DRM_I915_WERROR) += \
 	$(patsubst %.h,%.hdrtest, $(filter-out $(no-header-test), \
-		$(shell cd $(srctree)/$(src) && find * -name '*.h')))
+		$(shell cd $(src) && find * -name '*.h')))
 
 quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@)
       cmd_hdrtest = $(CC) $(filter-out $(CFLAGS_GCOV), $(c_flags)) -S -o /dev/null -x c /dev/null -include $<; \
diff --git a/drivers/gpu/drm/i915/display/bxt_dpio_phy_regs.h b/drivers/gpu/drm/i915/display/bxt_dpio_phy_regs.h
new file mode 100644
index 000000000000..275f4d9c3fb0
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/bxt_dpio_phy_regs.h
@@ -0,0 +1,273 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __BXT_DPIO_PHY_REGS_H__
+#define __BXT_DPIO_PHY_REGS_H__
+
+#include "intel_display_reg_defs.h"
+
+/* BXT PHY registers */
+#define _BXT_PHY0_BASE			0x6C000
+#define _BXT_PHY1_BASE			0x162000
+#define _BXT_PHY2_BASE			0x163000
+#define BXT_PHY_BASE(phy) \
+	 _PICK_EVEN_2RANGES(phy, 1, \
+			    _BXT_PHY0_BASE, _BXT_PHY0_BASE, \
+			    _BXT_PHY1_BASE, _BXT_PHY2_BASE)
+
+#define _BXT_PHY(phy, reg) \
+	_MMIO(BXT_PHY_BASE(phy) - _BXT_PHY0_BASE + (reg))
+
+#define _BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1) \
+	(BXT_PHY_BASE(phy) + _PIPE((ch), (reg_ch0) - _BXT_PHY0_BASE, \
+					 (reg_ch1) - _BXT_PHY0_BASE))
+#define _MMIO_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1) \
+	_MMIO(_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1))
+#define _BXT_LANE_OFFSET(lane)           (((lane) >> 1) * 0x200 + \
+					  ((lane) & 1) * 0x80)
+#define _MMIO_BXT_PHY_CH_LN(phy, ch, lane, reg_ch0, reg_ch1) \
+	_MMIO(_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1) + _BXT_LANE_OFFSET(lane))
+
+/* BXT PHY PLL registers */
+#define _PORT_PLL_A			0x46074
+#define _PORT_PLL_B			0x46078
+#define _PORT_PLL_C			0x4607c
+#define   PORT_PLL_ENABLE		REG_BIT(31)
+#define   PORT_PLL_LOCK			REG_BIT(30)
+#define   PORT_PLL_REF_SEL		REG_BIT(27)
+#define   PORT_PLL_POWER_ENABLE		REG_BIT(26)
+#define   PORT_PLL_POWER_STATE		REG_BIT(25)
+#define BXT_PORT_PLL_ENABLE(port)	_MMIO_PORT(port, _PORT_PLL_A, _PORT_PLL_B)
+
+#define _PORT_PLL_EBB_0_A		0x162034
+#define _PORT_PLL_EBB_0_B		0x6C034
+#define _PORT_PLL_EBB_0_C		0x6C340
+#define   PORT_PLL_P1_MASK		REG_GENMASK(15, 13)
+#define   PORT_PLL_P1(p1)		REG_FIELD_PREP(PORT_PLL_P1_MASK, (p1))
+#define   PORT_PLL_P2_MASK		REG_GENMASK(12, 8)
+#define   PORT_PLL_P2(p2)		REG_FIELD_PREP(PORT_PLL_P2_MASK, (p2))
+#define BXT_PORT_PLL_EBB_0(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PLL_EBB_0_B, \
+							 _PORT_PLL_EBB_0_C)
+
+#define _PORT_PLL_EBB_4_A		0x162038
+#define _PORT_PLL_EBB_4_B		0x6C038
+#define _PORT_PLL_EBB_4_C		0x6C344
+#define   PORT_PLL_RECALIBRATE		REG_BIT(14)
+#define   PORT_PLL_10BIT_CLK_ENABLE	REG_BIT(13)
+#define BXT_PORT_PLL_EBB_4(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PLL_EBB_4_B, \
+							 _PORT_PLL_EBB_4_C)
+
+#define _PORT_PLL_0_A			0x162100
+#define _PORT_PLL_0_B			0x6C100
+#define _PORT_PLL_0_C			0x6C380
+/* PORT_PLL_0_A */
+#define   PORT_PLL_M2_INT_MASK		REG_GENMASK(7, 0)
+#define   PORT_PLL_M2_INT(m2_int)	REG_FIELD_PREP(PORT_PLL_M2_INT_MASK, (m2_int))
+/* PORT_PLL_1_A */
+#define   PORT_PLL_N_MASK		REG_GENMASK(11, 8)
+#define   PORT_PLL_N(n)			REG_FIELD_PREP(PORT_PLL_N_MASK, (n))
+/* PORT_PLL_2_A */
+#define   PORT_PLL_M2_FRAC_MASK		REG_GENMASK(21, 0)
+#define   PORT_PLL_M2_FRAC(m2_frac)	REG_FIELD_PREP(PORT_PLL_M2_FRAC_MASK, (m2_frac))
+/* PORT_PLL_3_A */
+#define   PORT_PLL_M2_FRAC_ENABLE	REG_BIT(16)
+/* PORT_PLL_6_A */
+#define   PORT_PLL_GAIN_CTL_MASK	REG_GENMASK(18, 16)
+#define   PORT_PLL_GAIN_CTL(x)		REG_FIELD_PREP(PORT_PLL_GAIN_CTL_MASK, (x))
+#define   PORT_PLL_INT_COEFF_MASK	REG_GENMASK(12, 8)
+#define   PORT_PLL_INT_COEFF(x)		REG_FIELD_PREP(PORT_PLL_INT_COEFF_MASK, (x))
+#define   PORT_PLL_PROP_COEFF_MASK	REG_GENMASK(3, 0)
+#define   PORT_PLL_PROP_COEFF(x)	REG_FIELD_PREP(PORT_PLL_PROP_COEFF_MASK, (x))
+/* PORT_PLL_8_A */
+#define   PORT_PLL_TARGET_CNT_MASK	REG_GENMASK(9, 0)
+#define   PORT_PLL_TARGET_CNT(x)	REG_FIELD_PREP(PORT_PLL_TARGET_CNT_MASK, (x))
+/* PORT_PLL_9_A */
+#define  PORT_PLL_LOCK_THRESHOLD_MASK	REG_GENMASK(3, 1)
+#define  PORT_PLL_LOCK_THRESHOLD(x)	REG_FIELD_PREP(PORT_PLL_LOCK_THRESHOLD_MASK, (x))
+/* PORT_PLL_10_A */
+#define  PORT_PLL_DCO_AMP_OVR_EN_H	REG_BIT(27)
+#define  PORT_PLL_DCO_AMP_MASK		REG_GENMASK(13, 10)
+#define  PORT_PLL_DCO_AMP(x)		REG_FIELD_PREP(PORT_PLL_DCO_AMP_MASK, (x))
+#define _PORT_PLL_BASE(phy, ch)		_BXT_PHY_CH(phy, ch, \
+						    _PORT_PLL_0_B, \
+						    _PORT_PLL_0_C)
+#define BXT_PORT_PLL(phy, ch, idx)	_MMIO(_PORT_PLL_BASE(phy, ch) + \
+					      (idx) * 4)
+
+/* BXT PHY common lane registers */
+#define _PORT_CL1CM_DW0_A		0x162000
+#define _PORT_CL1CM_DW0_BC		0x6C000
+#define   PHY_POWER_GOOD		REG_BIT(16)
+#define   PHY_RESERVED			REG_BIT(7)
+#define BXT_PORT_CL1CM_DW0(phy)		_BXT_PHY((phy), _PORT_CL1CM_DW0_BC)
+
+#define _PORT_CL1CM_DW9_A		0x162024
+#define _PORT_CL1CM_DW9_BC		0x6C024
+#define   IREF0RC_OFFSET_MASK		REG_GENMASK(15, 8)
+#define   IREF0RC_OFFSET(x)		REG_FIELD_PREP(IREF0RC_OFFSET_MASK, (x))
+#define BXT_PORT_CL1CM_DW9(phy)		_BXT_PHY((phy), _PORT_CL1CM_DW9_BC)
+
+#define _PORT_CL1CM_DW10_A		0x162028
+#define _PORT_CL1CM_DW10_BC		0x6C028
+#define   IREF1RC_OFFSET_MASK		REG_GENMASK(15, 8)
+#define   IREF1RC_OFFSET(x)		REG_FIELD_PREP(IREF1RC_OFFSET_MASK, (x))
+#define BXT_PORT_CL1CM_DW10(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW10_BC)
+
+#define _PORT_CL1CM_DW28_A		0x162070
+#define _PORT_CL1CM_DW28_BC		0x6C070
+#define   OCL1_POWER_DOWN_EN		REG_BIT(23)
+#define   DW28_OLDO_DYN_PWR_DOWN_EN	REG_BIT(22)
+#define   SUS_CLK_CONFIG		REG_GENMASK(1, 0)
+#define BXT_PORT_CL1CM_DW28(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW28_BC)
+
+#define _PORT_CL1CM_DW30_A		0x162078
+#define _PORT_CL1CM_DW30_BC		0x6C078
+#define   OCL2_LDOFUSE_PWR_DIS		REG_BIT(6)
+#define BXT_PORT_CL1CM_DW30(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW30_BC)
+
+/* The spec defines this only for BXT PHY0, but lets assume that this
+ * would exist for PHY1 too if it had a second channel.
+ */
+#define _PORT_CL2CM_DW6_A		0x162358
+#define _PORT_CL2CM_DW6_BC		0x6C358
+#define BXT_PORT_CL2CM_DW6(phy)		_BXT_PHY((phy), _PORT_CL2CM_DW6_BC)
+#define   DW6_OLDO_DYN_PWR_DOWN_EN	REG_BIT(28)
+
+/* BXT PHY Ref registers */
+#define _PORT_REF_DW3_A			0x16218C
+#define _PORT_REF_DW3_BC		0x6C18C
+#define   GRC_DONE			REG_BIT(22)
+#define BXT_PORT_REF_DW3(phy)		_BXT_PHY((phy), _PORT_REF_DW3_BC)
+
+#define _PORT_REF_DW6_A			0x162198
+#define _PORT_REF_DW6_BC		0x6C198
+#define   GRC_CODE_MASK			REG_GENMASK(31, 24)
+#define   GRC_CODE(x)			REG_FIELD_PREP(GRC_CODE_MASK, (x))
+#define   GRC_CODE_FAST_MASK		REG_GENMASK(23, 16)
+#define   GRC_CODE_FAST(x)		REG_FIELD_PREP(GRC_CODE_FAST_MASK, (x))
+#define   GRC_CODE_SLOW_MASK		REG_GENMASK(15, 8)
+#define   GRC_CODE_SLOW(x)		REG_FIELD_PREP(GRC_CODE_SLOW_MASK, (x))
+#define   GRC_CODE_NOM_MASK		REG_GENMASK(7, 0)
+#define   GRC_CODE_NOM(x)		REG_FIELD_PREP(GRC_CODE_NOM_MASK, (x))
+#define BXT_PORT_REF_DW6(phy)		_BXT_PHY((phy), _PORT_REF_DW6_BC)
+
+#define _PORT_REF_DW8_A			0x1621A0
+#define _PORT_REF_DW8_BC		0x6C1A0
+#define   GRC_DIS			REG_BIT(15)
+#define   GRC_RDY_OVRD			REG_BIT(1)
+#define BXT_PORT_REF_DW8(phy)		_BXT_PHY((phy), _PORT_REF_DW8_BC)
+
+/* BXT PHY PCS registers */
+#define _PORT_PCS_DW10_LN01_A		0x162428
+#define _PORT_PCS_DW10_LN01_B		0x6C428
+#define _PORT_PCS_DW10_LN01_C		0x6C828
+#define _PORT_PCS_DW10_GRP_A		0x162C28
+#define _PORT_PCS_DW10_GRP_B		0x6CC28
+#define _PORT_PCS_DW10_GRP_C		0x6CE28
+#define BXT_PORT_PCS_DW10_LN01(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PCS_DW10_LN01_B, \
+							 _PORT_PCS_DW10_LN01_C)
+#define BXT_PORT_PCS_DW10_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PCS_DW10_GRP_B, \
+							 _PORT_PCS_DW10_GRP_C)
+
+#define   TX2_SWING_CALC_INIT		REG_BIT(31)
+#define   TX1_SWING_CALC_INIT		REG_BIT(30)
+
+#define _PORT_PCS_DW12_LN01_A		0x162430
+#define _PORT_PCS_DW12_LN01_B		0x6C430
+#define _PORT_PCS_DW12_LN01_C		0x6C830
+#define _PORT_PCS_DW12_LN23_A		0x162630
+#define _PORT_PCS_DW12_LN23_B		0x6C630
+#define _PORT_PCS_DW12_LN23_C		0x6CA30
+#define _PORT_PCS_DW12_GRP_A		0x162c30
+#define _PORT_PCS_DW12_GRP_B		0x6CC30
+#define _PORT_PCS_DW12_GRP_C		0x6CE30
+#define   LANESTAGGER_STRAP_OVRD	REG_BIT(6)
+#define   LANE_STAGGER_MASK		REG_GENMASK(4, 0)
+#define BXT_PORT_PCS_DW12_LN01(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PCS_DW12_LN01_B, \
+							 _PORT_PCS_DW12_LN01_C)
+#define BXT_PORT_PCS_DW12_LN23(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PCS_DW12_LN23_B, \
+							 _PORT_PCS_DW12_LN23_C)
+#define BXT_PORT_PCS_DW12_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PCS_DW12_GRP_B, \
+							 _PORT_PCS_DW12_GRP_C)
+
+/* BXT PHY TX registers */
+#define _PORT_TX_DW2_LN0_A		0x162508
+#define _PORT_TX_DW2_LN0_B		0x6C508
+#define _PORT_TX_DW2_LN0_C		0x6C908
+#define _PORT_TX_DW2_GRP_A		0x162D08
+#define _PORT_TX_DW2_GRP_B		0x6CD08
+#define _PORT_TX_DW2_GRP_C		0x6CF08
+#define BXT_PORT_TX_DW2_LN(phy, ch, lane)	_MMIO_BXT_PHY_CH_LN(phy, ch, lane, \
+								    _PORT_TX_DW2_LN0_B,	\
+								    _PORT_TX_DW2_LN0_C)
+#define BXT_PORT_TX_DW2_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_TX_DW2_GRP_B, \
+							 _PORT_TX_DW2_GRP_C)
+#define   MARGIN_000_MASK		REG_GENMASK(23, 16)
+#define   MARGIN_000(x)			REG_FIELD_PREP(MARGIN_000_MASK, (x))
+#define   UNIQ_TRANS_SCALE_MASK		REG_GENMASK(15, 8)
+#define   UNIQ_TRANS_SCALE(x)		REG_FIELD_PREP(UNIQ_TRANS_SCALE_MASK, (x))
+
+#define _PORT_TX_DW3_LN0_A		0x16250C
+#define _PORT_TX_DW3_LN0_B		0x6C50C
+#define _PORT_TX_DW3_LN0_C		0x6C90C
+#define _PORT_TX_DW3_GRP_A		0x162D0C
+#define _PORT_TX_DW3_GRP_B		0x6CD0C
+#define _PORT_TX_DW3_GRP_C		0x6CF0C
+#define BXT_PORT_TX_DW3_LN(phy, ch, lane)	_MMIO_BXT_PHY_CH_LN(phy, ch, lane, \
+								    _PORT_TX_DW3_LN0_B, \
+								    _PORT_TX_DW3_LN0_C)
+#define BXT_PORT_TX_DW3_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_TX_DW3_GRP_B, \
+							 _PORT_TX_DW3_GRP_C)
+#define   SCALE_DCOMP_METHOD		REG_BIT(26)
+#define   UNIQUE_TRANGE_EN_METHOD	REG_BIT(27)
+
+#define _PORT_TX_DW4_LN0_A		0x162510
+#define _PORT_TX_DW4_LN0_B		0x6C510
+#define _PORT_TX_DW4_LN0_C		0x6C910
+#define _PORT_TX_DW4_GRP_A		0x162D10
+#define _PORT_TX_DW4_GRP_B		0x6CD10
+#define _PORT_TX_DW4_GRP_C		0x6CF10
+#define BXT_PORT_TX_DW4_LN(phy, ch, lane)	_MMIO_BXT_PHY_CH_LN(phy, ch, lane, \
+								    _PORT_TX_DW4_LN0_B, \
+								    _PORT_TX_DW4_LN0_C)
+#define BXT_PORT_TX_DW4_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_TX_DW4_GRP_B, \
+							 _PORT_TX_DW4_GRP_C)
+#define   DE_EMPHASIS_MASK		REG_GENMASK(31, 24)
+#define   DE_EMPHASIS(x)		REG_FIELD_PREP(DE_EMPHASIS_MASK, (x))
+
+#define _PORT_TX_DW5_LN0_A		0x162514
+#define _PORT_TX_DW5_LN0_B		0x6C514
+#define _PORT_TX_DW5_LN0_C		0x6C914
+#define _PORT_TX_DW5_GRP_A		0x162D14
+#define _PORT_TX_DW5_GRP_B		0x6CD14
+#define _PORT_TX_DW5_GRP_C		0x6CF14
+#define BXT_PORT_TX_DW5_LN(phy, ch, lane)	_MMIO_BXT_PHY_CH_LN(phy, ch, lane, \
+								    _PORT_TX_DW5_LN0_B, \
+								    _PORT_TX_DW5_LN0_C)
+#define BXT_PORT_TX_DW5_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_TX_DW5_GRP_B, \
+							 _PORT_TX_DW5_GRP_C)
+#define   DCC_DELAY_RANGE_1		REG_BIT(9)
+#define   DCC_DELAY_RANGE_2		REG_BIT(8)
+
+#define _PORT_TX_DW14_LN0_A		0x162538
+#define _PORT_TX_DW14_LN0_B		0x6C538
+#define _PORT_TX_DW14_LN0_C		0x6C938
+#define   LATENCY_OPTIM			REG_BIT(30)
+#define BXT_PORT_TX_DW14_LN(phy, ch, lane)	_MMIO_BXT_PHY_CH_LN(phy, ch, lane, \
+								    _PORT_TX_DW14_LN0_B, \
+								    _PORT_TX_DW14_LN0_C)
+
+#endif /* __BXT_DPIO_PHY_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c
index ac456a2275db..79ecfc339430 100644
--- a/drivers/gpu/drm/i915/display/icl_dsi.c
+++ b/drivers/gpu/drm/i915/display/icl_dsi.c
@@ -1616,8 +1616,7 @@ static int gen11_dsi_compute_config(struct intel_encoder *encoder,
 				    struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi,
-						   base);
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	struct intel_connector *intel_connector = intel_dsi->attached_connector;
 	struct drm_display_mode *adjusted_mode =
 		&pipe_config->hw.adjusted_mode;
diff --git a/drivers/gpu/drm/i915/display/intel_atomic.c b/drivers/gpu/drm/i915/display/intel_atomic.c
index 2bb270f82932..7a77ae3dc394 100644
--- a/drivers/gpu/drm/i915/display/intel_atomic.c
+++ b/drivers/gpu/drm/i915/display/intel_atomic.c
@@ -62,7 +62,7 @@ int intel_digital_connector_atomic_get_property(struct drm_connector *connector,
 {
 	struct drm_device *dev = connector->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_digital_connector_state *intel_conn_state =
+	const struct intel_digital_connector_state *intel_conn_state =
 		to_intel_digital_connector_state(state);
 
 	if (property == dev_priv->display.properties.force_audio)
diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c
index 07e0c73204f3..40e7d862675e 100644
--- a/drivers/gpu/drm/i915/display/intel_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_audio.c
@@ -76,19 +76,6 @@ struct intel_audio_funcs {
 				       struct intel_crtc_state *crtc_state);
 };
 
-/* DP N/M table */
-#define LC_810M	810000
-#define LC_540M	540000
-#define LC_270M	270000
-#define LC_162M	162000
-
-struct dp_aud_n_m {
-	int sample_rate;
-	int clock;
-	u16 m;
-	u16 n;
-};
-
 struct hdmi_aud_ncts {
 	int sample_rate;
 	int clock;
@@ -96,60 +83,6 @@ struct hdmi_aud_ncts {
 	int cts;
 };
 
-/* Values according to DP 1.4 Table 2-104 */
-static const struct dp_aud_n_m dp_aud_n_m[] = {
-	{ 32000, LC_162M, 1024, 10125 },
-	{ 44100, LC_162M, 784, 5625 },
-	{ 48000, LC_162M, 512, 3375 },
-	{ 64000, LC_162M, 2048, 10125 },
-	{ 88200, LC_162M, 1568, 5625 },
-	{ 96000, LC_162M, 1024, 3375 },
-	{ 128000, LC_162M, 4096, 10125 },
-	{ 176400, LC_162M, 3136, 5625 },
-	{ 192000, LC_162M, 2048, 3375 },
-	{ 32000, LC_270M, 1024, 16875 },
-	{ 44100, LC_270M, 784, 9375 },
-	{ 48000, LC_270M, 512, 5625 },
-	{ 64000, LC_270M, 2048, 16875 },
-	{ 88200, LC_270M, 1568, 9375 },
-	{ 96000, LC_270M, 1024, 5625 },
-	{ 128000, LC_270M, 4096, 16875 },
-	{ 176400, LC_270M, 3136, 9375 },
-	{ 192000, LC_270M, 2048, 5625 },
-	{ 32000, LC_540M, 1024, 33750 },
-	{ 44100, LC_540M, 784, 18750 },
-	{ 48000, LC_540M, 512, 11250 },
-	{ 64000, LC_540M, 2048, 33750 },
-	{ 88200, LC_540M, 1568, 18750 },
-	{ 96000, LC_540M, 1024, 11250 },
-	{ 128000, LC_540M, 4096, 33750 },
-	{ 176400, LC_540M, 3136, 18750 },
-	{ 192000, LC_540M, 2048, 11250 },
-	{ 32000, LC_810M, 1024, 50625 },
-	{ 44100, LC_810M, 784, 28125 },
-	{ 48000, LC_810M, 512, 16875 },
-	{ 64000, LC_810M, 2048, 50625 },
-	{ 88200, LC_810M, 1568, 28125 },
-	{ 96000, LC_810M, 1024, 16875 },
-	{ 128000, LC_810M, 4096, 50625 },
-	{ 176400, LC_810M, 3136, 28125 },
-	{ 192000, LC_810M, 2048, 16875 },
-};
-
-static const struct dp_aud_n_m *
-audio_config_dp_get_n_m(const struct intel_crtc_state *crtc_state, int rate)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(dp_aud_n_m); i++) {
-		if (rate == dp_aud_n_m[i].sample_rate &&
-		    crtc_state->port_clock == dp_aud_n_m[i].clock)
-			return &dp_aud_n_m[i];
-	}
-
-	return NULL;
-}
-
 static const struct {
 	int clock;
 	u32 config;
@@ -387,47 +320,17 @@ hsw_dp_audio_config_update(struct intel_encoder *encoder,
 			   const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	struct i915_audio_component *acomp = i915->display.audio.component;
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
-	enum port port = encoder->port;
-	const struct dp_aud_n_m *nm;
-	int rate;
-	u32 tmp;
-
-	rate = acomp ? acomp->aud_sample_rate[port] : 0;
-	nm = audio_config_dp_get_n_m(crtc_state, rate);
-	if (nm)
-		drm_dbg_kms(&i915->drm, "using Maud %u, Naud %u\n", nm->m,
-			    nm->n);
-	else
-		drm_dbg_kms(&i915->drm, "using automatic Maud, Naud\n");
-
-	tmp = intel_de_read(i915, HSW_AUD_CFG(cpu_transcoder));
-	tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
-	tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
-	tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
-	tmp |= AUD_CONFIG_N_VALUE_INDEX;
-
-	if (nm) {
-		tmp &= ~AUD_CONFIG_N_MASK;
-		tmp |= AUD_CONFIG_N(nm->n);
-		tmp |= AUD_CONFIG_N_PROG_ENABLE;
-	}
-
-	intel_de_write(i915, HSW_AUD_CFG(cpu_transcoder), tmp);
-
-	tmp = intel_de_read(i915, HSW_AUD_M_CTS_ENABLE(cpu_transcoder));
-	tmp &= ~AUD_CONFIG_M_MASK;
-	tmp &= ~AUD_M_CTS_M_VALUE_INDEX;
-	tmp &= ~AUD_M_CTS_M_PROG_ENABLE;
 
-	if (nm) {
-		tmp |= nm->m;
-		tmp |= AUD_M_CTS_M_VALUE_INDEX;
-		tmp |= AUD_M_CTS_M_PROG_ENABLE;
-	}
+	/* Enable time stamps. Let HW calculate Maud/Naud values */
+	intel_de_rmw(i915, HSW_AUD_CFG(cpu_transcoder),
+		     AUD_CONFIG_N_VALUE_INDEX |
+		     AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK |
+		     AUD_CONFIG_UPPER_N_MASK |
+		     AUD_CONFIG_LOWER_N_MASK |
+		     AUD_CONFIG_N_PROG_ENABLE,
+		     AUD_CONFIG_N_VALUE_INDEX);
 
-	intel_de_write(i915, HSW_AUD_M_CTS_ENABLE(cpu_transcoder), tmp);
 }
 
 static void
@@ -1349,17 +1252,6 @@ static const struct component_ops i915_audio_component_bind_ops = {
 static void i915_audio_component_init(struct drm_i915_private *i915)
 {
 	u32 aud_freq, aud_freq_init;
-	int ret;
-
-	ret = component_add_typed(i915->drm.dev,
-				  &i915_audio_component_bind_ops,
-				  I915_COMPONENT_AUDIO);
-	if (ret < 0) {
-		drm_err(&i915->drm,
-			"failed to add audio component (%d)\n", ret);
-		/* continue with reduced functionality */
-		return;
-	}
 
 	if (DISPLAY_VER(i915) >= 9) {
 		aud_freq_init = intel_de_read(i915, AUD_FREQ_CNTRL);
@@ -1382,6 +1274,21 @@ static void i915_audio_component_init(struct drm_i915_private *i915)
 
 	/* init with current cdclk */
 	intel_audio_cdclk_change_post(i915);
+}
+
+static void i915_audio_component_register(struct drm_i915_private *i915)
+{
+	int ret;
+
+	ret = component_add_typed(i915->drm.dev,
+				  &i915_audio_component_bind_ops,
+				  I915_COMPONENT_AUDIO);
+	if (ret < 0) {
+		drm_err(&i915->drm,
+			"failed to add audio component (%d)\n", ret);
+		/* continue with reduced functionality */
+		return;
+	}
 
 	i915->display.audio.component_registered = true;
 }
@@ -1414,6 +1321,12 @@ void intel_audio_init(struct drm_i915_private *i915)
 		i915_audio_component_init(i915);
 }
 
+void intel_audio_register(struct drm_i915_private *i915)
+{
+	if (!i915->display.audio.lpe.platdev)
+		i915_audio_component_register(i915);
+}
+
 /**
  * intel_audio_deinit() - deinitialize the audio driver
  * @i915: the i915 drm device private data
diff --git a/drivers/gpu/drm/i915/display/intel_audio.h b/drivers/gpu/drm/i915/display/intel_audio.h
index 9327954b801e..576c061d72a4 100644
--- a/drivers/gpu/drm/i915/display/intel_audio.h
+++ b/drivers/gpu/drm/i915/display/intel_audio.h
@@ -28,6 +28,7 @@ void intel_audio_codec_get_config(struct intel_encoder *encoder,
 void intel_audio_cdclk_change_pre(struct drm_i915_private *dev_priv);
 void intel_audio_cdclk_change_post(struct drm_i915_private *dev_priv);
 void intel_audio_init(struct drm_i915_private *dev_priv);
+void intel_audio_register(struct drm_i915_private *i915);
 void intel_audio_deinit(struct drm_i915_private *dev_priv);
 void intel_audio_sdp_split_update(const struct intel_crtc_state *crtc_state);
 
diff --git a/drivers/gpu/drm/i915/display/intel_audio_regs.h b/drivers/gpu/drm/i915/display/intel_audio_regs.h
index 616e7b1275c4..88ea2740365d 100644
--- a/drivers/gpu/drm/i915/display/intel_audio_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_audio_regs.h
@@ -148,4 +148,20 @@
 #define HBLANK_START_COUNT_96	4
 #define HBLANK_START_COUNT_128	5
 
+/* LPE Audio */
+#define I915_HDMI_LPE_AUDIO_BASE	(VLV_DISPLAY_BASE + 0x65000)
+#define I915_HDMI_LPE_AUDIO_SIZE	0x1000
+
+#define VLV_AUD_CHICKEN_BIT_REG		_MMIO(VLV_DISPLAY_BASE + 0x62F38)
+#define VLV_CHICKEN_BIT_DBG_ENABLE	(1 << 0)
+
+#define _VLV_AUD_PORT_EN_B_DBG		0x62F20
+#define _VLV_AUD_PORT_EN_C_DBG		0x62F30
+#define _VLV_AUD_PORT_EN_D_DBG		0x62F34
+#define VLV_AUD_PORT_EN_DBG(port)	_MMIO_BASE_PORT3(VLV_DISPLAY_BASE, (port) - PORT_B, \
+							 _VLV_AUD_PORT_EN_B_DBG, \
+							 _VLV_AUD_PORT_EN_C_DBG, \
+							 _VLV_AUD_PORT_EN_D_DBG)
+#define VLV_AMP_MUTE		        (1 << 1)
+
 #endif /* __INTEL_AUDIO_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_backlight.c b/drivers/gpu/drm/i915/display/intel_backlight.c
index 1946d7fb3c2e..071668bfe5d1 100644
--- a/drivers/gpu/drm/i915/display/intel_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_backlight.c
@@ -83,16 +83,16 @@ static u32 scale_hw_to_user(struct intel_connector *connector,
 
 u32 intel_backlight_invert_pwm_level(struct intel_connector *connector, u32 val)
 {
-	struct drm_i915_private *i915 = to_i915(connector->base.dev);
+	struct intel_display *display = to_intel_display(connector);
 	struct intel_panel *panel = &connector->panel;
 
-	drm_WARN_ON(&i915->drm, panel->backlight.pwm_level_max == 0);
+	drm_WARN_ON(display->drm, panel->backlight.pwm_level_max == 0);
 
-	if (i915->display.params.invert_brightness < 0)
+	if (display->params.invert_brightness < 0)
 		return val;
 
-	if (i915->display.params.invert_brightness > 0 ||
-	    intel_has_quirk(i915, QUIRK_INVERT_BRIGHTNESS)) {
+	if (display->params.invert_brightness > 0 ||
+	    intel_has_quirk(display, QUIRK_INVERT_BRIGHTNESS)) {
 		return panel->backlight.pwm_level_max - val + panel->backlight.pwm_level_min;
 	}
 
@@ -126,15 +126,15 @@ u32 intel_backlight_level_to_pwm(struct intel_connector *connector, u32 val)
 
 u32 intel_backlight_level_from_pwm(struct intel_connector *connector, u32 val)
 {
-	struct drm_i915_private *i915 = to_i915(connector->base.dev);
+	struct intel_display *display = to_intel_display(connector);
 	struct intel_panel *panel = &connector->panel;
 
-	drm_WARN_ON_ONCE(&i915->drm,
+	drm_WARN_ON_ONCE(display->drm,
 			 panel->backlight.max == 0 || panel->backlight.pwm_level_max == 0);
 
-	if (i915->display.params.invert_brightness > 0 ||
-	    (i915->display.params.invert_brightness == 0 &&
-	     intel_has_quirk(i915, QUIRK_INVERT_BRIGHTNESS)))
+	if (display->params.invert_brightness > 0 ||
+	    (display->params.invert_brightness == 0 &&
+	     intel_has_quirk(display, QUIRK_INVERT_BRIGHTNESS)))
 		val = panel->backlight.pwm_level_max - (val - panel->backlight.pwm_level_min);
 
 	return scale(val, panel->backlight.pwm_level_min, panel->backlight.pwm_level_max,
@@ -761,8 +761,8 @@ static void __intel_backlight_enable(const struct intel_crtc_state *crtc_state,
 
 	WARN_ON(panel->backlight.max == 0);
 
-	if (panel->backlight.level <= panel->backlight.min) {
-		panel->backlight.level = panel->backlight.max;
+	if (panel->backlight.level < panel->backlight.min) {
+		panel->backlight.level = panel->backlight.min;
 		if (panel->backlight.device)
 			panel->backlight.device->props.brightness =
 				scale_hw_to_user(connector,
@@ -949,7 +949,7 @@ int intel_backlight_device_register(struct intel_connector *connector)
 	else
 		props.power = FB_BLANK_POWERDOWN;
 
-	name = kstrdup("intel_backlight", GFP_KERNEL);
+	name = kstrdup_const("intel_backlight", GFP_KERNEL);
 	if (!name)
 		return -ENOMEM;
 
@@ -963,7 +963,7 @@ int intel_backlight_device_register(struct intel_connector *connector)
 		 * compatibility. Use unique names for subsequent backlight devices as a
 		 * fallback when the default name already exists.
 		 */
-		kfree(name);
+		kfree_const(name);
 		name = kasprintf(GFP_KERNEL, "card%d-%s-backlight",
 				 i915->drm.primary->index, connector->base.name);
 		if (!name)
@@ -987,7 +987,7 @@ int intel_backlight_device_register(struct intel_connector *connector)
 		    connector->base.base.id, connector->base.name, name);
 
 out:
-	kfree(name);
+	kfree_const(name);
 
 	return ret;
 }
@@ -1642,17 +1642,17 @@ void intel_backlight_update(struct intel_atomic_state *state,
 
 int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe)
 {
-	struct drm_i915_private *i915 = to_i915(connector->base.dev);
+	struct intel_display *display = to_intel_display(connector);
 	struct intel_panel *panel = &connector->panel;
 	int ret;
 
 	if (!connector->panel.vbt.backlight.present) {
-		if (intel_has_quirk(i915, QUIRK_BACKLIGHT_PRESENT)) {
-			drm_dbg_kms(&i915->drm,
+		if (intel_has_quirk(display, QUIRK_BACKLIGHT_PRESENT)) {
+			drm_dbg_kms(display->drm,
 				    "[CONNECTOR:%d:%s] no backlight present per VBT, but present per quirk\n",
 				    connector->base.base.id, connector->base.name);
 		} else {
-			drm_dbg_kms(&i915->drm,
+			drm_dbg_kms(display->drm,
 				    "[CONNECTOR:%d:%s] no backlight present per VBT\n",
 				    connector->base.base.id, connector->base.name);
 			return 0;
@@ -1660,16 +1660,16 @@ int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe)
 	}
 
 	/* ensure intel_panel has been initialized first */
-	if (drm_WARN_ON(&i915->drm, !panel->backlight.funcs))
+	if (drm_WARN_ON(display->drm, !panel->backlight.funcs))
 		return -ENODEV;
 
 	/* set level and max in panel struct */
-	mutex_lock(&i915->display.backlight.lock);
+	mutex_lock(&display->backlight.lock);
 	ret = panel->backlight.funcs->setup(connector, pipe);
-	mutex_unlock(&i915->display.backlight.lock);
+	mutex_unlock(&display->backlight.lock);
 
 	if (ret) {
-		drm_dbg_kms(&i915->drm,
+		drm_dbg_kms(display->drm,
 			    "[CONNECTOR:%d:%s] failed to setup backlight\n",
 			    connector->base.base.id, connector->base.name);
 		return ret;
@@ -1677,7 +1677,7 @@ int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe)
 
 	panel->backlight.present = true;
 
-	drm_dbg_kms(&i915->drm,
+	drm_dbg_kms(display->drm,
 		    "[CONNECTOR:%d:%s] backlight initialized, %s, brightness %u/%u\n",
 		    connector->base.base.id, connector->base.name,
 		    str_enabled_disabled(panel->backlight.enabled),
@@ -1821,7 +1821,7 @@ void intel_backlight_init_funcs(struct intel_panel *panel)
 		if (intel_dp_aux_init_backlight_funcs(connector) == 0)
 			return;
 
-		if (!intel_has_quirk(i915, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK))
+		if (!intel_has_quirk(&i915->display, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK))
 			connector->panel.backlight.power = intel_pps_backlight_power;
 	}
 
diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c
index 52bd3576835b..5fb48b6129b6 100644
--- a/drivers/gpu/drm/i915/display/intel_bios.c
+++ b/drivers/gpu/drm/i915/display/intel_bios.c
@@ -25,6 +25,8 @@
  *
  */
 
+#include <linux/firmware.h>
+
 #include <drm/display/drm_dp_helper.h>
 #include <drm/display/drm_dsc_helper.h>
 #include <drm/drm_edid.h>
@@ -593,11 +595,14 @@ get_lvds_fp_timing(const struct bdb_lvds_lfp_data *data,
 	return (const void *)data + ptrs->ptr[index].fp_timing.offset;
 }
 
-static const struct lvds_pnp_id *
+static const struct drm_edid_product_id *
 get_lvds_pnp_id(const struct bdb_lvds_lfp_data *data,
 		const struct bdb_lvds_lfp_data_ptrs *ptrs,
 		int index)
 {
+	/* These two are supposed to have the same layout in memory. */
+	BUILD_BUG_ON(sizeof(struct lvds_pnp_id) != sizeof(struct drm_edid_product_id));
+
 	return (const void *)data + ptrs->ptr[index].panel_pnp_id.offset;
 }
 
@@ -611,19 +616,6 @@ get_lfp_data_tail(const struct bdb_lvds_lfp_data *data,
 		return NULL;
 }
 
-static void dump_pnp_id(struct drm_i915_private *i915,
-			const struct lvds_pnp_id *pnp_id,
-			const char *name)
-{
-	u16 mfg_name = be16_to_cpu((__force __be16)pnp_id->mfg_name);
-	char vend[4];
-
-	drm_dbg_kms(&i915->drm, "%s PNPID mfg: %s (0x%x), prod: %u, serial: %u, week: %d, year: %d\n",
-		    name, drm_edid_decode_mfg_id(mfg_name, vend),
-		    pnp_id->mfg_name, pnp_id->product_code, pnp_id->serial,
-		    pnp_id->mfg_week, pnp_id->mfg_year + 1990);
-}
-
 static int opregion_get_panel_type(struct drm_i915_private *i915,
 				   const struct intel_bios_encoder_data *devdata,
 				   const struct drm_edid *drm_edid, bool use_fallback)
@@ -662,21 +654,21 @@ static int pnpid_get_panel_type(struct drm_i915_private *i915,
 {
 	const struct bdb_lvds_lfp_data *data;
 	const struct bdb_lvds_lfp_data_ptrs *ptrs;
-	const struct lvds_pnp_id *edid_id;
-	struct lvds_pnp_id edid_id_nodate;
-	const struct edid *edid = drm_edid_raw(drm_edid); /* FIXME */
+	struct drm_edid_product_id product_id, product_id_nodate;
+	struct drm_printer p;
 	int i, best = -1;
 
-	if (!edid)
+	if (!drm_edid)
 		return -1;
 
-	edid_id = (const void *)&edid->mfg_id[0];
+	drm_edid_get_product_id(drm_edid, &product_id);
 
-	edid_id_nodate = *edid_id;
-	edid_id_nodate.mfg_week = 0;
-	edid_id_nodate.mfg_year = 0;
+	product_id_nodate = product_id;
+	product_id_nodate.week_of_manufacture = 0;
+	product_id_nodate.year_of_manufacture = 0;
 
-	dump_pnp_id(i915, edid_id, "EDID");
+	p = drm_dbg_printer(&i915->drm, DRM_UT_KMS, "EDID");
+	drm_edid_print_product_id(&p, &product_id, true);
 
 	ptrs = bdb_find_section(i915, BDB_LVDS_LFP_DATA_PTRS);
 	if (!ptrs)
@@ -687,11 +679,11 @@ static int pnpid_get_panel_type(struct drm_i915_private *i915,
 		return -1;
 
 	for (i = 0; i < 16; i++) {
-		const struct lvds_pnp_id *vbt_id =
+		const struct drm_edid_product_id *vbt_id =
 			get_lvds_pnp_id(data, ptrs, i);
 
 		/* full match? */
-		if (!memcmp(vbt_id, edid_id, sizeof(*vbt_id)))
+		if (!memcmp(vbt_id, &product_id, sizeof(*vbt_id)))
 			return i;
 
 		/*
@@ -699,7 +691,7 @@ static int pnpid_get_panel_type(struct drm_i915_private *i915,
 		 * and the VBT entry does not specify a date.
 		 */
 		if (best < 0 &&
-		    !memcmp(vbt_id, &edid_id_nodate, sizeof(*vbt_id)))
+		    !memcmp(vbt_id, &product_id_nodate, sizeof(*vbt_id)))
 			best = i;
 	}
 
@@ -885,7 +877,8 @@ parse_lfp_data(struct drm_i915_private *i915,
 	const struct bdb_lvds_lfp_data *data;
 	const struct bdb_lvds_lfp_data_tail *tail;
 	const struct bdb_lvds_lfp_data_ptrs *ptrs;
-	const struct lvds_pnp_id *pnp_id;
+	const struct drm_edid_product_id *pnp_id;
+	struct drm_printer p;
 	int panel_type = panel->vbt.panel_type;
 
 	ptrs = bdb_find_section(i915, BDB_LVDS_LFP_DATA_PTRS);
@@ -900,7 +893,9 @@ parse_lfp_data(struct drm_i915_private *i915,
 		parse_lfp_panel_dtd(i915, panel, data, ptrs);
 
 	pnp_id = get_lvds_pnp_id(data, ptrs, panel_type);
-	dump_pnp_id(i915, pnp_id, "Panel");
+
+	p = drm_dbg_printer(&i915->drm, DRM_UT_KMS, "Panel");
+	drm_edid_print_product_id(&p, pnp_id, false);
 
 	tail = get_lfp_data_tail(data, ptrs);
 	if (!tail)
@@ -1042,22 +1037,11 @@ parse_lfp_backlight(struct drm_i915_private *i915,
 	panel->vbt.backlight.type = INTEL_BACKLIGHT_DISPLAY_DDI;
 	panel->vbt.backlight.controller = 0;
 	if (i915->display.vbt.version >= 191) {
-		size_t exp_size;
-
-		if (i915->display.vbt.version >= 236)
-			exp_size = sizeof(struct bdb_lfp_backlight_data);
-		else if (i915->display.vbt.version >= 234)
-			exp_size = EXP_BDB_LFP_BL_DATA_SIZE_REV_234;
-		else
-			exp_size = EXP_BDB_LFP_BL_DATA_SIZE_REV_191;
-
-		if (get_blocksize(backlight_data) >= exp_size) {
-			const struct lfp_backlight_control_method *method;
+		const struct lfp_backlight_control_method *method;
 
-			method = &backlight_data->backlight_control[panel_type];
-			panel->vbt.backlight.type = method->type;
-			panel->vbt.backlight.controller = method->controller;
-		}
+		method = &backlight_data->backlight_control[panel_type];
+		panel->vbt.backlight.type = method->type;
+		panel->vbt.backlight.controller = method->controller;
 	}
 
 	panel->vbt.backlight.pwm_freq_hz = entry->pwm_freq_hz;
@@ -2730,6 +2714,57 @@ static void parse_ddi_ports(struct drm_i915_private *i915)
 		print_ddi_port(devdata);
 }
 
+static int child_device_expected_size(u16 version)
+{
+	BUILD_BUG_ON(sizeof(struct child_device_config) < 40);
+
+	if (version > 256)
+		return -ENOENT;
+	else if (version >= 256)
+		return 40;
+	else if (version >= 216)
+		return 39;
+	else if (version >= 196)
+		return 38;
+	else if (version >= 195)
+		return 37;
+	else if (version >= 111)
+		return LEGACY_CHILD_DEVICE_CONFIG_SIZE;
+	else if (version >= 106)
+		return 27;
+	else
+		return 22;
+}
+
+static bool child_device_size_valid(struct drm_i915_private *i915, int size)
+{
+	int expected_size;
+
+	expected_size = child_device_expected_size(i915->display.vbt.version);
+	if (expected_size < 0) {
+		expected_size = sizeof(struct child_device_config);
+		drm_dbg(&i915->drm,
+			"Expected child device config size for VBT version %u not known; assuming %d\n",
+			i915->display.vbt.version, expected_size);
+	}
+
+	/* Flag an error for unexpected size, but continue anyway. */
+	if (size != expected_size)
+		drm_err(&i915->drm,
+			"Unexpected child device config size %d (expected %d for VBT version %u)\n",
+			size, expected_size, i915->display.vbt.version);
+
+	/* The legacy sized child device config is the minimum we need. */
+	if (size < LEGACY_CHILD_DEVICE_CONFIG_SIZE) {
+		drm_dbg_kms(&i915->drm,
+			    "Child device config size %d is too small.\n",
+			    size);
+		return false;
+	}
+
+	return true;
+}
+
 static void
 parse_general_definitions(struct drm_i915_private *i915)
 {
@@ -2737,7 +2772,6 @@ parse_general_definitions(struct drm_i915_private *i915)
 	struct intel_bios_encoder_data *devdata;
 	const struct child_device_config *child;
 	int i, child_device_num;
-	u8 expected_size;
 	u16 block_size;
 	int bus_pin;
 
@@ -2761,39 +2795,8 @@ parse_general_definitions(struct drm_i915_private *i915)
 	if (intel_gmbus_is_valid_pin(i915, bus_pin))
 		i915->display.vbt.crt_ddc_pin = bus_pin;
 
-	if (i915->display.vbt.version < 106) {
-		expected_size = 22;
-	} else if (i915->display.vbt.version < 111) {
-		expected_size = 27;
-	} else if (i915->display.vbt.version < 195) {
-		expected_size = LEGACY_CHILD_DEVICE_CONFIG_SIZE;
-	} else if (i915->display.vbt.version == 195) {
-		expected_size = 37;
-	} else if (i915->display.vbt.version <= 215) {
-		expected_size = 38;
-	} else if (i915->display.vbt.version <= 250) {
-		expected_size = 39;
-	} else {
-		expected_size = sizeof(*child);
-		BUILD_BUG_ON(sizeof(*child) < 39);
-		drm_dbg(&i915->drm,
-			"Expected child device config size for VBT version %u not known; assuming %u\n",
-			i915->display.vbt.version, expected_size);
-	}
-
-	/* Flag an error for unexpected size, but continue anyway. */
-	if (defs->child_dev_size != expected_size)
-		drm_err(&i915->drm,
-			"Unexpected child device config size %u (expected %u for VBT version %u)\n",
-			defs->child_dev_size, expected_size, i915->display.vbt.version);
-
-	/* The legacy sized child device config is the minimum we need. */
-	if (defs->child_dev_size < LEGACY_CHILD_DEVICE_CONFIG_SIZE) {
-		drm_dbg_kms(&i915->drm,
-			    "Child device config size %u is too small.\n",
-			    defs->child_dev_size);
+	if (!child_device_size_valid(i915, defs->child_dev_size))
 		return;
-	}
 
 	/* get the number of child device */
 	child_device_num = (block_size - sizeof(*defs)) / defs->child_dev_size;
@@ -2869,9 +2872,8 @@ init_vbt_panel_defaults(struct intel_panel *panel)
 static void
 init_vbt_missing_defaults(struct drm_i915_private *i915)
 {
+	unsigned int ports = DISPLAY_RUNTIME_INFO(i915)->port_mask;
 	enum port port;
-	int ports = BIT(PORT_A) | BIT(PORT_B) | BIT(PORT_C) |
-		    BIT(PORT_D) | BIT(PORT_E) | BIT(PORT_F);
 
 	if (!HAS_DDI(i915) && !IS_CHERRYVIEW(i915))
 		return;
@@ -2981,6 +2983,43 @@ bool intel_bios_is_valid_vbt(struct drm_i915_private *i915,
 	return vbt;
 }
 
+static struct vbt_header *firmware_get_vbt(struct drm_i915_private *i915,
+					   size_t *size)
+{
+	struct vbt_header *vbt = NULL;
+	const struct firmware *fw = NULL;
+	const char *name = i915->display.params.vbt_firmware;
+	int ret;
+
+	if (!name || !*name)
+		return NULL;
+
+	ret = request_firmware(&fw, name, i915->drm.dev);
+	if (ret) {
+		drm_err(&i915->drm,
+			"Requesting VBT firmware \"%s\" failed (%d)\n",
+			name, ret);
+		return NULL;
+	}
+
+	if (intel_bios_is_valid_vbt(i915, fw->data, fw->size)) {
+		vbt = kmemdup(fw->data, fw->size, GFP_KERNEL);
+		if (vbt) {
+			drm_dbg_kms(&i915->drm,
+				    "Found valid VBT firmware \"%s\"\n", name);
+			if (size)
+				*size = fw->size;
+		}
+	} else {
+		drm_dbg_kms(&i915->drm, "Invalid VBT firmware \"%s\"\n",
+			    name);
+	}
+
+	release_firmware(fw);
+
+	return vbt;
+}
+
 static u32 intel_spi_read(struct intel_uncore *uncore, u32 offset)
 {
 	intel_uncore_write(uncore, PRIMARY_SPI_ADDRESS, offset);
@@ -2988,7 +3027,8 @@ static u32 intel_spi_read(struct intel_uncore *uncore, u32 offset)
 	return intel_uncore_read(uncore, PRIMARY_SPI_TRIGGER);
 }
 
-static struct vbt_header *spi_oprom_get_vbt(struct drm_i915_private *i915)
+static struct vbt_header *spi_oprom_get_vbt(struct drm_i915_private *i915,
+					    size_t *size)
 {
 	u32 count, data, found, store = 0;
 	u32 static_region, oprom_offset;
@@ -3031,6 +3071,9 @@ static struct vbt_header *spi_oprom_get_vbt(struct drm_i915_private *i915)
 
 	drm_dbg_kms(&i915->drm, "Found valid VBT in SPI flash\n");
 
+	if (size)
+		*size = vbt_size;
+
 	return (struct vbt_header *)vbt;
 
 err_free_vbt:
@@ -3039,7 +3082,8 @@ err_not_found:
 	return NULL;
 }
 
-static struct vbt_header *oprom_get_vbt(struct drm_i915_private *i915)
+static struct vbt_header *oprom_get_vbt(struct drm_i915_private *i915,
+					size_t *sizep)
 {
 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
 	void __iomem *p = NULL, *oprom;
@@ -3088,6 +3132,9 @@ static struct vbt_header *oprom_get_vbt(struct drm_i915_private *i915)
 
 	pci_unmap_rom(pdev, oprom);
 
+	if (sizep)
+		*sizep = vbt_size;
+
 	drm_dbg_kms(&i915->drm, "Found valid VBT in PCI ROM\n");
 
 	return vbt;
@@ -3100,6 +3147,32 @@ err_unmap_oprom:
 	return NULL;
 }
 
+static const struct vbt_header *intel_bios_get_vbt(struct drm_i915_private *i915,
+						   size_t *sizep)
+{
+	const struct vbt_header *vbt = NULL;
+	intel_wakeref_t wakeref;
+
+	vbt = firmware_get_vbt(i915, sizep);
+
+	if (!vbt)
+		vbt = intel_opregion_get_vbt(i915, sizep);
+
+	/*
+	 * If the OpRegion does not have VBT, look in SPI flash
+	 * through MMIO or PCI mapping
+	 */
+	if (!vbt && IS_DGFX(i915))
+		with_intel_runtime_pm(&i915->runtime_pm, wakeref)
+			vbt = spi_oprom_get_vbt(i915, sizep);
+
+	if (!vbt)
+		with_intel_runtime_pm(&i915->runtime_pm, wakeref)
+			vbt = oprom_get_vbt(i915, sizep);
+
+	return vbt;
+}
+
 /**
  * intel_bios_init - find VBT and initialize settings from the BIOS
  * @i915: i915 device instance
@@ -3111,7 +3184,6 @@ err_unmap_oprom:
 void intel_bios_init(struct drm_i915_private *i915)
 {
 	const struct vbt_header *vbt;
-	struct vbt_header *oprom_vbt = NULL;
 	const struct bdb_header *bdb;
 
 	INIT_LIST_HEAD(&i915->display.vbt.display_devices);
@@ -3125,21 +3197,7 @@ void intel_bios_init(struct drm_i915_private *i915)
 
 	init_vbt_defaults(i915);
 
-	vbt = intel_opregion_get_vbt(i915, NULL);
-
-	/*
-	 * If the OpRegion does not have VBT, look in SPI flash through MMIO or
-	 * PCI mapping
-	 */
-	if (!vbt && IS_DGFX(i915)) {
-		oprom_vbt = spi_oprom_get_vbt(i915);
-		vbt = oprom_vbt;
-	}
-
-	if (!vbt) {
-		oprom_vbt = oprom_get_vbt(i915);
-		vbt = oprom_vbt;
-	}
+	vbt = intel_bios_get_vbt(i915, NULL);
 
 	if (!vbt)
 		goto out;
@@ -3172,7 +3230,7 @@ out:
 	parse_sdvo_device_mapping(i915);
 	parse_ddi_ports(i915);
 
-	kfree(oprom_vbt);
+	kfree(vbt);
 }
 
 static void intel_bios_init_panel(struct drm_i915_private *i915,
@@ -3344,8 +3402,7 @@ bool intel_bios_is_lvds_present(struct drm_i915_private *i915, u8 *i2c_pin)
 		 * additional data.  Trust that if the VBT was written into
 		 * the OpRegion then they have validated the LVDS's existence.
 		 */
-		if (intel_opregion_get_vbt(i915, NULL))
-			return true;
+		return intel_opregion_vbt_present(i915);
 	}
 
 	return false;
@@ -3706,13 +3763,12 @@ static int intel_bios_vbt_show(struct seq_file *m, void *unused)
 	const void *vbt;
 	size_t vbt_size;
 
-	/*
-	 * FIXME: VBT might originate from other places than opregion, and then
-	 * this would be incorrect.
-	 */
-	vbt = intel_opregion_get_vbt(i915, &vbt_size);
-	if (vbt)
+	vbt = intel_bios_get_vbt(i915, &vbt_size);
+
+	if (vbt) {
 		seq_write(m, vbt, vbt_size);
+		kfree(vbt);
+	}
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c
index 7f2a50b4f494..972ea887e232 100644
--- a/drivers/gpu/drm/i915/display/intel_bw.c
+++ b/drivers/gpu/drm/i915/display/intel_bw.c
@@ -162,7 +162,9 @@ int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
 				1);
 
 	if (ret < 0) {
-		drm_err(&dev_priv->drm, "Failed to disable qgv points (%d) points: 0x%x\n", ret, points_mask);
+		drm_err(&dev_priv->drm,
+			"Failed to disable qgv points (0x%x) points: 0x%x\n",
+			ret, points_mask);
 		return ret;
 	}
 
@@ -290,8 +292,10 @@ static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
 		struct intel_qgv_point *sp = &qi->points[i];
 
 		ret = intel_read_qgv_point_info(dev_priv, sp, i);
-		if (ret)
+		if (ret) {
+			drm_dbg_kms(&dev_priv->drm, "Could not read QGV %d info\n", i);
 			return ret;
+		}
 
 		drm_dbg_kms(&dev_priv->drm,
 			    "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
@@ -659,6 +663,22 @@ static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv,
 	return bi->psf_bw[psf_gv_point];
 }
 
+static unsigned int icl_qgv_bw(struct drm_i915_private *i915,
+			       int num_active_planes, int qgv_point)
+{
+	unsigned int idx;
+
+	if (DISPLAY_VER(i915) >= 12)
+		idx = tgl_max_bw_index(i915, num_active_planes, qgv_point);
+	else
+		idx = icl_max_bw_index(i915, num_active_planes, qgv_point);
+
+	if (idx >= ARRAY_SIZE(i915->display.bw.max))
+		return 0;
+
+	return i915->display.bw.max[idx].deratedbw[qgv_point];
+}
+
 void intel_bw_init_hw(struct drm_i915_private *dev_priv)
 {
 	if (!HAS_DISPLAY(dev_priv))
@@ -735,6 +755,7 @@ void intel_bw_crtc_update(struct intel_bw_state *bw_state,
 		intel_bw_crtc_data_rate(crtc_state);
 	bw_state->num_active_planes[crtc->pipe] =
 		intel_bw_crtc_num_active_planes(crtc_state);
+	bw_state->force_check_qgv = true;
 
 	drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n",
 		    pipe_name(crtc->pipe),
@@ -804,6 +825,80 @@ intel_atomic_get_bw_state(struct intel_atomic_state *state)
 	return to_intel_bw_state(bw_state);
 }
 
+static unsigned int icl_max_bw_qgv_point_mask(struct drm_i915_private *i915,
+					      int num_active_planes)
+{
+	unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
+	unsigned int max_bw_point = 0;
+	unsigned int max_bw = 0;
+	int i;
+
+	for (i = 0; i < num_qgv_points; i++) {
+		unsigned int max_data_rate =
+			icl_qgv_bw(i915, num_active_planes, i);
+
+		/*
+		 * We need to know which qgv point gives us
+		 * maximum bandwidth in order to disable SAGV
+		 * if we find that we exceed SAGV block time
+		 * with watermarks. By that moment we already
+		 * have those, as it is calculated earlier in
+		 * intel_atomic_check,
+		 */
+		if (max_data_rate > max_bw) {
+			max_bw_point = BIT(i);
+			max_bw = max_data_rate;
+		}
+	}
+
+	return max_bw_point;
+}
+
+static u16 icl_prepare_qgv_points_mask(struct drm_i915_private *i915,
+				       unsigned int qgv_points,
+				       unsigned int psf_points)
+{
+	return ~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
+		 ADLS_PCODE_REQ_PSF_PT(psf_points)) & icl_qgv_points_mask(i915);
+}
+
+static unsigned int icl_max_bw_psf_gv_point_mask(struct drm_i915_private *i915)
+{
+	unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
+	unsigned int max_bw_point_mask = 0;
+	unsigned int max_bw = 0;
+	int i;
+
+	for (i = 0; i < num_psf_gv_points; i++) {
+		unsigned int max_data_rate = adl_psf_bw(i915, i);
+
+		if (max_data_rate > max_bw) {
+			max_bw_point_mask = BIT(i);
+			max_bw = max_data_rate;
+		} else if (max_data_rate == max_bw) {
+			max_bw_point_mask |= BIT(i);
+		}
+	}
+
+	return max_bw_point_mask;
+}
+
+static void icl_force_disable_sagv(struct drm_i915_private *i915,
+				   struct intel_bw_state *bw_state)
+{
+	unsigned int qgv_points = icl_max_bw_qgv_point_mask(i915, 0);
+	unsigned int psf_points = icl_max_bw_psf_gv_point_mask(i915);
+
+	bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(i915,
+								qgv_points,
+								psf_points);
+
+	drm_dbg_kms(&i915->drm, "Forcing SAGV disable: mask 0x%x\n",
+		    bw_state->qgv_points_mask);
+
+	icl_pcode_restrict_qgv_points(i915, bw_state->qgv_points_mask);
+}
+
 static int mtl_find_qgv_points(struct drm_i915_private *i915,
 			       unsigned int data_rate,
 			       unsigned int num_active_planes,
@@ -881,8 +976,6 @@ static int icl_find_qgv_points(struct drm_i915_private *i915,
 			       const struct intel_bw_state *old_bw_state,
 			       struct intel_bw_state *new_bw_state)
 {
-	unsigned int max_bw_point = 0;
-	unsigned int max_bw = 0;
 	unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
 	unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
 	u16 psf_points = 0;
@@ -895,31 +988,8 @@ static int icl_find_qgv_points(struct drm_i915_private *i915,
 		return ret;
 
 	for (i = 0; i < num_qgv_points; i++) {
-		unsigned int idx;
-		unsigned int max_data_rate;
-
-		if (DISPLAY_VER(i915) >= 12)
-			idx = tgl_max_bw_index(i915, num_active_planes, i);
-		else
-			idx = icl_max_bw_index(i915, num_active_planes, i);
-
-		if (idx >= ARRAY_SIZE(i915->display.bw.max))
-			continue;
-
-		max_data_rate = i915->display.bw.max[idx].deratedbw[i];
-
-		/*
-		 * We need to know which qgv point gives us
-		 * maximum bandwidth in order to disable SAGV
-		 * if we find that we exceed SAGV block time
-		 * with watermarks. By that moment we already
-		 * have those, as it is calculated earlier in
-		 * intel_atomic_check,
-		 */
-		if (max_data_rate > max_bw) {
-			max_bw_point = i;
-			max_bw = max_data_rate;
-		}
+		unsigned int max_data_rate = icl_qgv_bw(i915,
+							num_active_planes, i);
 		if (max_data_rate >= data_rate)
 			qgv_points |= BIT(i);
 
@@ -963,20 +1033,18 @@ static int icl_find_qgv_points(struct drm_i915_private *i915,
 	 * cause.
 	 */
 	if (!intel_can_enable_sagv(i915, new_bw_state)) {
-		qgv_points = BIT(max_bw_point);
-		drm_dbg_kms(&i915->drm, "No SAGV, using single QGV point %d\n",
-			    max_bw_point);
+		qgv_points = icl_max_bw_qgv_point_mask(i915, num_active_planes);
+		drm_dbg_kms(&i915->drm, "No SAGV, using single QGV point mask 0x%x\n",
+			    qgv_points);
 	}
 
 	/*
 	 * We store the ones which need to be masked as that is what PCode
 	 * actually accepts as a parameter.
 	 */
-	new_bw_state->qgv_points_mask =
-		~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
-		  ADLS_PCODE_REQ_PSF_PT(psf_points)) &
-		icl_qgv_points_mask(i915);
-
+	new_bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(i915,
+								    qgv_points,
+								    psf_points);
 	/*
 	 * If the actual mask had changed we need to make sure that
 	 * the commits are serialized(in case this is a nomodeset, nonblocking)
@@ -1272,8 +1340,9 @@ int intel_bw_atomic_check(struct intel_atomic_state *state)
 	new_bw_state = intel_atomic_get_new_bw_state(state);
 
 	if (new_bw_state &&
-	    intel_can_enable_sagv(i915, old_bw_state) !=
-	    intel_can_enable_sagv(i915, new_bw_state))
+	    (intel_can_enable_sagv(i915, old_bw_state) !=
+	     intel_can_enable_sagv(i915, new_bw_state) ||
+	     new_bw_state->force_check_qgv))
 		changed = true;
 
 	/*
@@ -1287,6 +1356,8 @@ int intel_bw_atomic_check(struct intel_atomic_state *state)
 	if (ret)
 		return ret;
 
+	new_bw_state->force_check_qgv = false;
+
 	return 0;
 }
 
@@ -1313,7 +1384,7 @@ static const struct intel_global_state_funcs intel_bw_funcs = {
 	.atomic_destroy_state = intel_bw_destroy_state,
 };
 
-int intel_bw_init(struct drm_i915_private *dev_priv)
+int intel_bw_init(struct drm_i915_private *i915)
 {
 	struct intel_bw_state *state;
 
@@ -1321,8 +1392,15 @@ int intel_bw_init(struct drm_i915_private *dev_priv)
 	if (!state)
 		return -ENOMEM;
 
-	intel_atomic_global_obj_init(dev_priv, &dev_priv->display.bw.obj,
+	intel_atomic_global_obj_init(i915, &i915->display.bw.obj,
 				     &state->base, &intel_bw_funcs);
 
+	/*
+	 * Limit this only if we have SAGV. And for Display version 14 onwards
+	 * sagv is handled though pmdemand requests
+	 */
+	if (intel_has_sagv(i915) && IS_DISPLAY_VER(i915, 11, 13))
+		icl_force_disable_sagv(i915, state);
+
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_bw.h b/drivers/gpu/drm/i915/display/intel_bw.h
index 59cb4fc5db76..161813cca473 100644
--- a/drivers/gpu/drm/i915/display/intel_bw.h
+++ b/drivers/gpu/drm/i915/display/intel_bw.h
@@ -47,12 +47,19 @@ struct intel_bw_state {
 	 */
 	u16 qgv_points_mask;
 
+	/*
+	 * Flag to force the QGV comparison in atomic check right after the
+	 * hw state readout
+	 */
+	bool force_check_qgv;
+
 	int min_cdclk[I915_MAX_PIPES];
 	unsigned int data_rate[I915_MAX_PIPES];
 	u8 num_active_planes[I915_MAX_PIPES];
 };
 
-#define to_intel_bw_state(x) container_of((x), struct intel_bw_state, base)
+#define to_intel_bw_state(global_state) \
+	container_of_const((global_state), struct intel_bw_state, base)
 
 struct intel_bw_state *
 intel_atomic_get_old_bw_state(struct intel_atomic_state *state);
diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c
index f672bfd70d45..7a833b5f2de2 100644
--- a/drivers/gpu/drm/i915/display/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/display/intel_cdclk.c
@@ -39,6 +39,8 @@
 #include "intel_pcode.h"
 #include "intel_psr.h"
 #include "intel_vdsc.h"
+#include "skl_watermark.h"
+#include "skl_watermark_regs.h"
 #include "vlv_sideband.h"
 
 /**
@@ -63,6 +65,32 @@
  * DMC will not change the active CDCLK frequency however, so that part
  * will still be performed by the driver directly.
  *
+ * There are multiple components involved in the generation of the CDCLK
+ * frequency:
+ *
+ * - We have the CDCLK PLL, which generates an output clock based on a
+ *   reference clock and a ratio parameter.
+ * - The CD2X Divider, which divides the output of the PLL based on a
+ *   divisor selected from a set of pre-defined choices.
+ * - The CD2X Squasher, which further divides the output based on a
+ *   waveform represented as a sequence of bits where each zero
+ *   "squashes out" a clock cycle.
+ * - And, finally, a fixed divider that divides the output frequency by 2.
+ *
+ * As such, the resulting CDCLK frequency can be calculated with the
+ * following formula:
+ *
+ *     cdclk = vco / cd2x_div / (sq_len / sq_div) / 2
+ *
+ * , where vco is the frequency generated by the PLL; cd2x_div
+ * represents the CD2X Divider; sq_len and sq_div are the bit length
+ * and the number of high bits for the CD2X Squasher waveform, respectively;
+ * and 2 represents the fixed divider.
+ *
+ * Note that some older platforms do not contain the CD2X Divider
+ * and/or CD2X Squasher, in which case we can ignore their respective
+ * factors in the formula above.
+ *
  * Several methods exist to change the CDCLK frequency, which ones are
  * supported depends on the platform:
  *
@@ -993,15 +1021,14 @@ static int skl_cdclk_decimal(int cdclk)
 	return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
 }
 
-static void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv,
-					int vco)
+static void skl_set_preferred_cdclk_vco(struct drm_i915_private *i915, int vco)
 {
-	bool changed = dev_priv->skl_preferred_vco_freq != vco;
+	bool changed = i915->display.cdclk.skl_preferred_vco_freq != vco;
 
-	dev_priv->skl_preferred_vco_freq = vco;
+	i915->display.cdclk.skl_preferred_vco_freq = vco;
 
 	if (changed)
-		intel_update_max_cdclk(dev_priv);
+		intel_update_max_cdclk(i915);
 }
 
 static u32 skl_dpll0_link_rate(struct drm_i915_private *dev_priv, int vco)
@@ -1205,7 +1232,7 @@ static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
 		 * Use the current vco as our initial
 		 * guess as to what the preferred vco is.
 		 */
-		if (dev_priv->skl_preferred_vco_freq == 0)
+		if (dev_priv->display.cdclk.skl_preferred_vco_freq == 0)
 			skl_set_preferred_cdclk_vco(dev_priv,
 						    dev_priv->display.cdclk.hw.vco);
 		return;
@@ -1213,7 +1240,7 @@ static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
 
 	cdclk_config = dev_priv->display.cdclk.hw;
 
-	cdclk_config.vco = dev_priv->skl_preferred_vco_freq;
+	cdclk_config.vco = dev_priv->display.cdclk.skl_preferred_vco_freq;
 	if (cdclk_config.vco == 0)
 		cdclk_config.vco = 8100000;
 	cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
@@ -1391,7 +1418,7 @@ static const struct intel_cdclk_vals mtl_cdclk_table[] = {
 	{}
 };
 
-static const struct intel_cdclk_vals lnl_cdclk_table[] = {
+static const struct intel_cdclk_vals xe2lpd_cdclk_table[] = {
 	{ .refclk = 38400, .cdclk = 153600, .ratio = 16, .waveform = 0xaaaa },
 	{ .refclk = 38400, .cdclk = 172800, .ratio = 16, .waveform = 0xad5a },
 	{ .refclk = 38400, .cdclk = 192000, .ratio = 16, .waveform = 0xb6b6 },
@@ -1656,6 +1683,8 @@ static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
 	}
 
  out:
+	if (DISPLAY_VER(dev_priv) >= 20)
+		cdclk_config->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;
 	/*
 	 * Can't read this out :( Let's assume it's
 	 * at least what the CDCLK frequency requires.
@@ -1850,6 +1879,37 @@ static bool cdclk_pll_is_unknown(unsigned int vco)
 	return vco == ~0;
 }
 
+static bool mdclk_source_is_cdclk_pll(struct drm_i915_private *i915)
+{
+	return DISPLAY_VER(i915) >= 20;
+}
+
+static u32 xe2lpd_mdclk_source_sel(struct drm_i915_private *i915)
+{
+	if (mdclk_source_is_cdclk_pll(i915))
+		return MDCLK_SOURCE_SEL_CDCLK_PLL;
+
+	return MDCLK_SOURCE_SEL_CD2XCLK;
+}
+
+int intel_mdclk_cdclk_ratio(struct drm_i915_private *i915,
+			    const struct intel_cdclk_config *cdclk_config)
+{
+	if (mdclk_source_is_cdclk_pll(i915))
+		return DIV_ROUND_UP(cdclk_config->vco, cdclk_config->cdclk);
+
+	/* Otherwise, source for MDCLK is CD2XCLK. */
+	return 2;
+}
+
+static void xe2lpd_mdclk_cdclk_ratio_program(struct drm_i915_private *i915,
+					     const struct intel_cdclk_config *cdclk_config)
+{
+	intel_dbuf_mdclk_cdclk_ratio_update(i915,
+					    intel_mdclk_cdclk_ratio(i915, cdclk_config),
+					    cdclk_config->joined_mbus);
+}
+
 static bool cdclk_compute_crawl_and_squash_midpoint(struct drm_i915_private *i915,
 						    const struct intel_cdclk_config *old_cdclk_config,
 						    const struct intel_cdclk_config *new_cdclk_config,
@@ -1954,7 +2014,7 @@ static u32 bxt_cdclk_ctl(struct drm_i915_private *i915,
 		val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
 
 	if (DISPLAY_VER(i915) >= 20)
-		val |= MDCLK_SOURCE_SEL_CDCLK_PLL;
+		val |= xe2lpd_mdclk_source_sel(i915);
 	else
 		val |= skl_cdclk_decimal(cdclk);
 
@@ -1967,7 +2027,6 @@ static void _bxt_set_cdclk(struct drm_i915_private *dev_priv,
 {
 	int cdclk = cdclk_config->cdclk;
 	int vco = cdclk_config->vco;
-	u16 waveform;
 
 	if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->display.cdclk.hw.vco > 0 && vco > 0 &&
 	    !cdclk_pll_is_unknown(dev_priv->display.cdclk.hw.vco)) {
@@ -1982,10 +2041,11 @@ static void _bxt_set_cdclk(struct drm_i915_private *dev_priv,
 	} else
 		bxt_cdclk_pll_update(dev_priv, vco);
 
-	waveform = cdclk_squash_waveform(dev_priv, cdclk);
+	if (HAS_CDCLK_SQUASH(dev_priv)) {
+		u16 waveform = cdclk_squash_waveform(dev_priv, cdclk);
 
-	if (HAS_CDCLK_SQUASH(dev_priv))
 		dg2_cdclk_squash_program(dev_priv, waveform);
+	}
 
 	intel_de_write(dev_priv, CDCLK_CTL, bxt_cdclk_ctl(dev_priv, cdclk_config, pipe));
 
@@ -2030,6 +2090,9 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
 		return;
 	}
 
+	if (DISPLAY_VER(dev_priv) >= 20 && cdclk < dev_priv->display.cdclk.hw.cdclk)
+		xe2lpd_mdclk_cdclk_ratio_program(dev_priv, cdclk_config);
+
 	if (cdclk_compute_crawl_and_squash_midpoint(dev_priv, &dev_priv->display.cdclk.hw,
 						    cdclk_config, &mid_cdclk_config)) {
 		_bxt_set_cdclk(dev_priv, &mid_cdclk_config, pipe);
@@ -2038,6 +2101,9 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
 		_bxt_set_cdclk(dev_priv, cdclk_config, pipe);
 	}
 
+	if (DISPLAY_VER(dev_priv) >= 20 && cdclk > dev_priv->display.cdclk.hw.cdclk)
+		xe2lpd_mdclk_cdclk_ratio_program(dev_priv, cdclk_config);
+
 	if (DISPLAY_VER(dev_priv) >= 14)
 		/*
 		 * NOOP - No Pcode communication needed for
@@ -2260,16 +2326,15 @@ static bool intel_cdclk_can_squash(struct drm_i915_private *dev_priv,
 }
 
 /**
- * intel_cdclk_needs_modeset - Determine if changong between the CDCLK
- *                             configurations requires a modeset on all pipes
+ * intel_cdclk_clock_changed - Check whether the clock changed
  * @a: first CDCLK configuration
  * @b: second CDCLK configuration
  *
  * Returns:
- * True if changing between the two CDCLK configurations
- * requires all pipes to be off, false if not.
+ * True if CDCLK changed in a way that requires re-programming and
+ * False otherwise.
  */
-bool intel_cdclk_needs_modeset(const struct intel_cdclk_config *a,
+bool intel_cdclk_clock_changed(const struct intel_cdclk_config *a,
 			       const struct intel_cdclk_config *b)
 {
 	return a->cdclk != b->cdclk ||
@@ -2322,7 +2387,7 @@ static bool intel_cdclk_can_cd2x_update(struct drm_i915_private *dev_priv,
 static bool intel_cdclk_changed(const struct intel_cdclk_config *a,
 				const struct intel_cdclk_config *b)
 {
-	return intel_cdclk_needs_modeset(a, b) ||
+	return intel_cdclk_clock_changed(a, b) ||
 		a->voltage_level != b->voltage_level;
 }
 
@@ -2368,18 +2433,9 @@ static void intel_pcode_notify(struct drm_i915_private *i915,
 			ret);
 }
 
-/**
- * intel_set_cdclk - Push the CDCLK configuration to the hardware
- * @dev_priv: i915 device
- * @cdclk_config: new CDCLK configuration
- * @pipe: pipe with which to synchronize the update
- *
- * Program the hardware based on the passed in CDCLK state,
- * if necessary.
- */
 static void intel_set_cdclk(struct drm_i915_private *dev_priv,
 			    const struct intel_cdclk_config *cdclk_config,
-			    enum pipe pipe)
+			    enum pipe pipe, const char *context)
 {
 	struct intel_encoder *encoder;
 
@@ -2389,7 +2445,7 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv,
 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.funcs.cdclk->set_cdclk))
 		return;
 
-	intel_cdclk_dump_config(dev_priv, cdclk_config, "Changing CDCLK to");
+	intel_cdclk_dump_config(dev_priv, cdclk_config, context);
 
 	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
@@ -2519,6 +2575,17 @@ static void intel_cdclk_pcode_post_notify(struct intel_atomic_state *state)
 			   update_cdclk, update_pipe_count);
 }
 
+bool intel_cdclk_is_decreasing_later(struct intel_atomic_state *state)
+{
+	const struct intel_cdclk_state *old_cdclk_state =
+		intel_atomic_get_old_cdclk_state(state);
+	const struct intel_cdclk_state *new_cdclk_state =
+		intel_atomic_get_new_cdclk_state(state);
+
+	return new_cdclk_state && !new_cdclk_state->disable_pipes &&
+		new_cdclk_state->actual.cdclk < old_cdclk_state->actual.cdclk;
+}
+
 /**
  * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
  * @state: intel atomic state
@@ -2560,9 +2627,16 @@ intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
 						 old_cdclk_state->actual.voltage_level);
 	}
 
+	/*
+	 * mbus joining will be changed later by
+	 * intel_dbuf_mbus_{pre,post}_ddb_update()
+	 */
+	cdclk_config.joined_mbus = old_cdclk_state->actual.joined_mbus;
+
 	drm_WARN_ON(&i915->drm, !new_cdclk_state->base.changed);
 
-	intel_set_cdclk(i915, &cdclk_config, pipe);
+	intel_set_cdclk(i915, &cdclk_config, pipe,
+			"Pre changing CDCLK to");
 }
 
 /**
@@ -2597,7 +2671,8 @@ intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
 
 	drm_WARN_ON(&i915->drm, !new_cdclk_state->base.changed);
 
-	intel_set_cdclk(i915, &new_cdclk_state->actual, pipe);
+	intel_set_cdclk(i915, &new_cdclk_state->actual, pipe,
+			"Post changing CDCLK to");
 }
 
 static int intel_pixel_rate_to_cdclk(const struct intel_crtc_state *crtc_state)
@@ -2748,25 +2823,6 @@ int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
 	if (crtc_state->dsc.compression_enable)
 		min_cdclk = max(min_cdclk, intel_vdsc_min_cdclk(crtc_state));
 
-	/*
-	 * HACK. Currently for TGL/DG2 platforms we calculate
-	 * min_cdclk initially based on pixel_rate divided
-	 * by 2, accounting for also plane requirements,
-	 * however in some cases the lowest possible CDCLK
-	 * doesn't work and causing the underruns.
-	 * Explicitly stating here that this seems to be currently
-	 * rather a Hack, than final solution.
-	 */
-	if (IS_TIGERLAKE(dev_priv) || IS_DG2(dev_priv)) {
-		/*
-		 * Clamp to max_cdclk_freq in case pixel rate is higher,
-		 * in order not to break an 8K, but still leave W/A at place.
-		 */
-		min_cdclk = max_t(int, min_cdclk,
-				  min_t(int, crtc_state->pixel_rate,
-					dev_priv->display.cdclk.max_cdclk_freq));
-	}
-
 	return min_cdclk;
 }
 
@@ -2954,7 +3010,7 @@ static int skl_dpll0_vco(struct intel_cdclk_state *cdclk_state)
 
 	vco = cdclk_state->logical.vco;
 	if (!vco)
-		vco = dev_priv->skl_preferred_vco_freq;
+		vco = dev_priv->display.cdclk.skl_preferred_vco_freq;
 
 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
 		if (!crtc_state->hw.enable)
@@ -3139,6 +3195,20 @@ int intel_cdclk_atomic_check(struct intel_atomic_state *state,
 	return 0;
 }
 
+int intel_cdclk_state_set_joined_mbus(struct intel_atomic_state *state, bool joined_mbus)
+{
+	struct intel_cdclk_state *cdclk_state;
+
+	cdclk_state = intel_atomic_get_cdclk_state(state);
+	if (IS_ERR(cdclk_state))
+		return PTR_ERR(cdclk_state);
+
+	cdclk_state->actual.joined_mbus = joined_mbus;
+	cdclk_state->logical.joined_mbus = joined_mbus;
+
+	return intel_atomic_lock_global_state(&cdclk_state->base);
+}
+
 int intel_cdclk_init(struct drm_i915_private *dev_priv)
 {
 	struct intel_cdclk_state *cdclk_state;
@@ -3247,7 +3317,7 @@ int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
 		drm_dbg_kms(&dev_priv->drm,
 			    "Can change cdclk cd2x divider with pipe %c active\n",
 			    pipe_name(pipe));
-	} else if (intel_cdclk_needs_modeset(&old_cdclk_state->actual,
+	} else if (intel_cdclk_clock_changed(&old_cdclk_state->actual,
 					     &new_cdclk_state->actual)) {
 		/* All pipes must be switched off while we change the cdclk. */
 		ret = intel_modeset_all_pipes_late(state, "CDCLK change");
@@ -3260,6 +3330,15 @@ int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
 			    "Modeset required for cdclk change\n");
 	}
 
+	if (intel_mdclk_cdclk_ratio(dev_priv, &old_cdclk_state->actual) !=
+	    intel_mdclk_cdclk_ratio(dev_priv, &new_cdclk_state->actual)) {
+		int ratio = intel_mdclk_cdclk_ratio(dev_priv, &new_cdclk_state->actual);
+
+		ret = intel_dbuf_state_set_mdclk_cdclk_ratio(state, ratio);
+		if (ret)
+			return ret;
+	}
+
 	drm_dbg_kms(&dev_priv->drm,
 		    "New cdclk calculated to be logical %u kHz, actual %u kHz\n",
 		    new_cdclk_state->logical.cdclk,
@@ -3317,7 +3396,7 @@ void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
 		u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
 		int max_cdclk, vco;
 
-		vco = dev_priv->skl_preferred_vco_freq;
+		vco = dev_priv->display.cdclk.skl_preferred_vco_freq;
 		drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
 
 		/*
@@ -3359,13 +3438,13 @@ void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
 		dev_priv->display.cdclk.max_cdclk_freq = dev_priv->display.cdclk.hw.cdclk;
 	}
 
-	dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
+	dev_priv->display.cdclk.max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
 
 	drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
 		dev_priv->display.cdclk.max_cdclk_freq);
 
 	drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
-		dev_priv->max_dotclk_freq);
+		dev_priv->display.cdclk.max_dotclk_freq);
 }
 
 /**
@@ -3539,7 +3618,7 @@ static int i915_cdclk_info_show(struct seq_file *m, void *unused)
 
 	seq_printf(m, "Current CD clock frequency: %d kHz\n", i915->display.cdclk.hw.cdclk);
 	seq_printf(m, "Max CD clock frequency: %d kHz\n", i915->display.cdclk.max_cdclk_freq);
-	seq_printf(m, "Max pixel clock frequency: %d kHz\n", i915->max_dotclk_freq);
+	seq_printf(m, "Max pixel clock frequency: %d kHz\n", i915->display.cdclk.max_dotclk_freq);
 
 	return 0;
 }
@@ -3554,13 +3633,6 @@ void intel_cdclk_debugfs_register(struct drm_i915_private *i915)
 			    i915, &i915_cdclk_info_fops);
 }
 
-static const struct intel_cdclk_funcs mtl_cdclk_funcs = {
-	.get_cdclk = bxt_get_cdclk,
-	.set_cdclk = bxt_set_cdclk,
-	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
-	.calc_voltage_level = rplu_calc_voltage_level,
-};
-
 static const struct intel_cdclk_funcs rplu_cdclk_funcs = {
 	.get_cdclk = bxt_get_cdclk,
 	.set_cdclk = bxt_set_cdclk,
@@ -3704,10 +3776,10 @@ static const struct intel_cdclk_funcs i830_cdclk_funcs = {
 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
 {
 	if (DISPLAY_VER(dev_priv) >= 20) {
-		dev_priv->display.funcs.cdclk = &mtl_cdclk_funcs;
-		dev_priv->display.cdclk.table = lnl_cdclk_table;
+		dev_priv->display.funcs.cdclk = &rplu_cdclk_funcs;
+		dev_priv->display.cdclk.table = xe2lpd_cdclk_table;
 	} else if (DISPLAY_VER(dev_priv) >= 14) {
-		dev_priv->display.funcs.cdclk = &mtl_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &rplu_cdclk_funcs;
 		dev_priv->display.cdclk.table = mtl_cdclk_table;
 	} else if (IS_DG2(dev_priv)) {
 		dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.h b/drivers/gpu/drm/i915/display/intel_cdclk.h
index 71bc032bfef1..cfdcdec07a4d 100644
--- a/drivers/gpu/drm/i915/display/intel_cdclk.h
+++ b/drivers/gpu/drm/i915/display/intel_cdclk.h
@@ -18,6 +18,8 @@ struct intel_crtc_state;
 struct intel_cdclk_config {
 	unsigned int cdclk, vco, ref, bypass;
 	u8 voltage_level;
+	/* This field is only valid for Xe2LPD and above. */
+	bool joined_mbus;
 };
 
 struct intel_cdclk_state {
@@ -63,8 +65,11 @@ void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv);
 void intel_update_max_cdclk(struct drm_i915_private *dev_priv);
 void intel_update_cdclk(struct drm_i915_private *dev_priv);
 u32 intel_read_rawclk(struct drm_i915_private *dev_priv);
-bool intel_cdclk_needs_modeset(const struct intel_cdclk_config *a,
+bool intel_cdclk_clock_changed(const struct intel_cdclk_config *a,
 			       const struct intel_cdclk_config *b);
+int intel_mdclk_cdclk_ratio(struct drm_i915_private *i915,
+			    const struct intel_cdclk_config *cdclk_config);
+bool intel_cdclk_is_decreasing_later(struct intel_atomic_state *state);
 void intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state);
 void intel_set_cdclk_post_plane_update(struct intel_atomic_state *state);
 void intel_cdclk_dump_config(struct drm_i915_private *i915,
@@ -75,10 +80,13 @@ void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv,
 			   struct intel_cdclk_config *cdclk_config);
 int intel_cdclk_atomic_check(struct intel_atomic_state *state,
 			     bool *need_cdclk_calc);
+int intel_cdclk_state_set_joined_mbus(struct intel_atomic_state *state, bool joined_mbus);
 struct intel_cdclk_state *
 intel_atomic_get_cdclk_state(struct intel_atomic_state *state);
 
-#define to_intel_cdclk_state(x) container_of((x), struct intel_cdclk_state, base)
+#define to_intel_cdclk_state(global_state) \
+	container_of_const((global_state), struct intel_cdclk_state, base)
+
 #define intel_atomic_get_old_cdclk_state(state) \
 	to_intel_cdclk_state(intel_atomic_get_old_global_obj_state(state, &to_i915(state->base.dev)->display.cdclk.obj))
 #define intel_atomic_get_new_cdclk_state(state) \
diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c
index ca7112b32cb3..d23163dc64d4 100644
--- a/drivers/gpu/drm/i915/display/intel_color.c
+++ b/drivers/gpu/drm/i915/display/intel_color.c
@@ -616,19 +616,19 @@ static void vlv_load_wgc_csc(struct intel_crtc *crtc,
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 
-	intel_de_write_fw(dev_priv, PIPE_WGC_C01_C00(pipe),
+	intel_de_write_fw(dev_priv, PIPE_WGC_C01_C00(dev_priv, pipe),
 			  csc->coeff[1] << 16 | csc->coeff[0]);
-	intel_de_write_fw(dev_priv, PIPE_WGC_C02(pipe),
+	intel_de_write_fw(dev_priv, PIPE_WGC_C02(dev_priv, pipe),
 			  csc->coeff[2]);
 
-	intel_de_write_fw(dev_priv, PIPE_WGC_C11_C10(pipe),
+	intel_de_write_fw(dev_priv, PIPE_WGC_C11_C10(dev_priv, pipe),
 			  csc->coeff[4] << 16 | csc->coeff[3]);
-	intel_de_write_fw(dev_priv, PIPE_WGC_C12(pipe),
+	intel_de_write_fw(dev_priv, PIPE_WGC_C12(dev_priv, pipe),
 			  csc->coeff[5]);
 
-	intel_de_write_fw(dev_priv, PIPE_WGC_C21_C20(pipe),
+	intel_de_write_fw(dev_priv, PIPE_WGC_C21_C20(dev_priv, pipe),
 			  csc->coeff[7] << 16 | csc->coeff[6]);
-	intel_de_write_fw(dev_priv, PIPE_WGC_C22(pipe),
+	intel_de_write_fw(dev_priv, PIPE_WGC_C22(dev_priv, pipe),
 			  csc->coeff[8]);
 }
 
@@ -639,25 +639,25 @@ static void vlv_read_wgc_csc(struct intel_crtc *crtc,
 	enum pipe pipe = crtc->pipe;
 	u32 tmp;
 
-	tmp = intel_de_read_fw(dev_priv, PIPE_WGC_C01_C00(pipe));
+	tmp = intel_de_read_fw(dev_priv, PIPE_WGC_C01_C00(dev_priv, pipe));
 	csc->coeff[0] = tmp & 0xffff;
 	csc->coeff[1] = tmp >> 16;
 
-	tmp = intel_de_read_fw(dev_priv, PIPE_WGC_C02(pipe));
+	tmp = intel_de_read_fw(dev_priv, PIPE_WGC_C02(dev_priv, pipe));
 	csc->coeff[2] = tmp & 0xffff;
 
-	tmp = intel_de_read_fw(dev_priv, PIPE_WGC_C11_C10(pipe));
+	tmp = intel_de_read_fw(dev_priv, PIPE_WGC_C11_C10(dev_priv, pipe));
 	csc->coeff[3] = tmp & 0xffff;
 	csc->coeff[4] = tmp >> 16;
 
-	tmp = intel_de_read_fw(dev_priv, PIPE_WGC_C12(pipe));
+	tmp = intel_de_read_fw(dev_priv, PIPE_WGC_C12(dev_priv, pipe));
 	csc->coeff[5] = tmp & 0xffff;
 
-	tmp = intel_de_read_fw(dev_priv, PIPE_WGC_C21_C20(pipe));
+	tmp = intel_de_read_fw(dev_priv, PIPE_WGC_C21_C20(dev_priv, pipe));
 	csc->coeff[6] = tmp & 0xffff;
 	csc->coeff[7] = tmp >> 16;
 
-	tmp = intel_de_read_fw(dev_priv, PIPE_WGC_C22(pipe));
+	tmp = intel_de_read_fw(dev_priv, PIPE_WGC_C22(dev_priv, pipe));
 	csc->coeff[8] = tmp & 0xffff;
 }
 
@@ -1227,7 +1227,7 @@ static void i9xx_load_lut_8(struct intel_crtc *crtc,
 	lut = blob->data;
 
 	for (i = 0; i < 256; i++)
-		intel_de_write_fw(dev_priv, PALETTE(pipe, i),
+		intel_de_write_fw(dev_priv, PALETTE(dev_priv, pipe, i),
 				  i9xx_lut_8(&lut[i]));
 }
 
@@ -1240,9 +1240,11 @@ static void i9xx_load_lut_10(struct intel_crtc *crtc,
 	enum pipe pipe = crtc->pipe;
 
 	for (i = 0; i < lut_size - 1; i++) {
-		intel_de_write_fw(dev_priv, PALETTE(pipe, 2 * i + 0),
+		intel_de_write_fw(dev_priv,
+				  PALETTE(dev_priv, pipe, 2 * i + 0),
 				  i9xx_lut_10_ldw(&lut[i]));
-		intel_de_write_fw(dev_priv, PALETTE(pipe, 2 * i + 1),
+		intel_de_write_fw(dev_priv,
+				  PALETTE(dev_priv, pipe, 2 * i + 1),
 				  i9xx_lut_10_udw(&lut[i]));
 	}
 }
@@ -1274,9 +1276,11 @@ static void i965_load_lut_10p6(struct intel_crtc *crtc,
 	enum pipe pipe = crtc->pipe;
 
 	for (i = 0; i < lut_size - 1; i++) {
-		intel_de_write_fw(dev_priv, PALETTE(pipe, 2 * i + 0),
+		intel_de_write_fw(dev_priv,
+				  PALETTE(dev_priv, pipe, 2 * i + 0),
 				  i965_lut_10p6_ldw(&lut[i]));
-		intel_de_write_fw(dev_priv, PALETTE(pipe, 2 * i + 1),
+		intel_de_write_fw(dev_priv,
+				  PALETTE(dev_priv, pipe, 2 * i + 1),
 				  i965_lut_10p6_udw(&lut[i]));
 	}
 
@@ -3150,7 +3154,8 @@ static struct drm_property_blob *i9xx_read_lut_8(struct intel_crtc *crtc)
 	lut = blob->data;
 
 	for (i = 0; i < LEGACY_LUT_LENGTH; i++) {
-		u32 val = intel_de_read_fw(dev_priv, PALETTE(pipe, i));
+		u32 val = intel_de_read_fw(dev_priv,
+					   PALETTE(dev_priv, pipe, i));
 
 		i9xx_lut_8_pack(&lut[i], val);
 	}
@@ -3176,8 +3181,10 @@ static struct drm_property_blob *i9xx_read_lut_10(struct intel_crtc *crtc)
 	lut = blob->data;
 
 	for (i = 0; i < lut_size - 1; i++) {
-		ldw = intel_de_read_fw(dev_priv, PALETTE(pipe, 2 * i + 0));
-		udw = intel_de_read_fw(dev_priv, PALETTE(pipe, 2 * i + 1));
+		ldw = intel_de_read_fw(dev_priv,
+				       PALETTE(dev_priv, pipe, 2 * i + 0));
+		udw = intel_de_read_fw(dev_priv,
+				       PALETTE(dev_priv, pipe, 2 * i + 1));
 
 		i9xx_lut_10_pack(&lut[i], ldw, udw);
 	}
@@ -3224,8 +3231,10 @@ static struct drm_property_blob *i965_read_lut_10p6(struct intel_crtc *crtc)
 	lut = blob->data;
 
 	for (i = 0; i < lut_size - 1; i++) {
-		u32 ldw = intel_de_read_fw(dev_priv, PALETTE(pipe, 2 * i + 0));
-		u32 udw = intel_de_read_fw(dev_priv, PALETTE(pipe, 2 * i + 1));
+		u32 ldw = intel_de_read_fw(dev_priv,
+					   PALETTE(dev_priv, pipe, 2 * i + 0));
+		u32 udw = intel_de_read_fw(dev_priv,
+					   PALETTE(dev_priv, pipe, 2 * i + 1));
 
 		i965_lut_10p6_pack(&lut[i], ldw, udw);
 	}
diff --git a/drivers/gpu/drm/i915/display/intel_color_regs.h b/drivers/gpu/drm/i915/display/intel_color_regs.h
index 9f4ae58f3e7e..bb99ea533842 100644
--- a/drivers/gpu/drm/i915/display/intel_color_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_color_regs.h
@@ -8,7 +8,35 @@
 
 #include "intel_display_reg_defs.h"
 
-/* legacy palette */
+/* GMCH palette */
+#define _PALETTE_A		0xa000
+#define _PALETTE_B		0xa800
+#define _CHV_PALETTE_C		0xc000
+/* 8bit mode / i965+ 10.6 interpolated mode ldw/udw */
+#define   PALETTE_RED_MASK		REG_GENMASK(23, 16)
+#define   PALETTE_GREEN_MASK		REG_GENMASK(15, 8)
+#define   PALETTE_BLUE_MASK		REG_GENMASK(7, 0)
+/* pre-i965 10bit interpolated mode ldw */
+#define   PALETTE_10BIT_RED_LDW_MASK	REG_GENMASK(23, 16)
+#define   PALETTE_10BIT_GREEN_LDW_MASK	REG_GENMASK(15, 8)
+#define   PALETTE_10BIT_BLUE_LDW_MASK	REG_GENMASK(7, 0)
+/* pre-i965 10bit interpolated mode udw */
+#define   PALETTE_10BIT_RED_EXP_MASK	REG_GENMASK(23, 22)
+#define   PALETTE_10BIT_RED_MANT_MASK	REG_GENMASK(21, 18)
+#define   PALETTE_10BIT_RED_UDW_MASK	REG_GENMASK(17, 16)
+#define   PALETTE_10BIT_GREEN_EXP_MASK	REG_GENMASK(15, 14)
+#define   PALETTE_10BIT_GREEN_MANT_MASK	REG_GENMASK(13, 10)
+#define   PALETTE_10BIT_GREEN_UDW_MASK	REG_GENMASK(9, 8)
+#define   PALETTE_10BIT_BLUE_EXP_MASK	REG_GENMASK(7, 6)
+#define   PALETTE_10BIT_BLUE_MANT_MASK	REG_GENMASK(5, 2)
+#define   PALETTE_10BIT_BLUE_UDW_MASK	REG_GENMASK(1, 0)
+#define PALETTE(dev_priv, pipe, i) _MMIO(DISPLAY_MMIO_BASE(dev_priv) +			\
+			       _PICK_EVEN_2RANGES(pipe, 2,			\
+						  _PALETTE_A, _PALETTE_B,	\
+						  _CHV_PALETTE_C, _CHV_PALETTE_C) + \
+						  (i) * 4)
+
+/* ilk+ palette */
 #define _LGC_PALETTE_A           0x4a000
 #define _LGC_PALETTE_B           0x4a800
 /* see PALETTE_* for the bits */
@@ -228,12 +256,12 @@
 #define _PIPE_A_WGC_C21_C20	0x600C0 /* s2.10 */
 #define _PIPE_A_WGC_C22		0x600C4 /* s2.10 */
 
-#define PIPE_WGC_C01_C00(pipe)		_MMIO_TRANS2(pipe, _PIPE_A_WGC_C01_C00)
-#define PIPE_WGC_C02(pipe)		_MMIO_TRANS2(pipe, _PIPE_A_WGC_C02)
-#define PIPE_WGC_C11_C10(pipe)		_MMIO_TRANS2(pipe, _PIPE_A_WGC_C11_C10)
-#define PIPE_WGC_C12(pipe)		_MMIO_TRANS2(pipe, _PIPE_A_WGC_C12)
-#define PIPE_WGC_C21_C20(pipe)		_MMIO_TRANS2(pipe, _PIPE_A_WGC_C21_C20)
-#define PIPE_WGC_C22(pipe)		_MMIO_TRANS2(pipe, _PIPE_A_WGC_C22)
+#define PIPE_WGC_C01_C00(dev_priv, pipe)		_MMIO_TRANS2(dev_priv, pipe, _PIPE_A_WGC_C01_C00)
+#define PIPE_WGC_C02(dev_priv, pipe)		_MMIO_TRANS2(dev_priv, pipe, _PIPE_A_WGC_C02)
+#define PIPE_WGC_C11_C10(dev_priv, pipe)		_MMIO_TRANS2(dev_priv, pipe, _PIPE_A_WGC_C11_C10)
+#define PIPE_WGC_C12(dev_priv, pipe)		_MMIO_TRANS2(dev_priv, pipe, _PIPE_A_WGC_C12)
+#define PIPE_WGC_C21_C20(dev_priv, pipe)		_MMIO_TRANS2(dev_priv, pipe, _PIPE_A_WGC_C21_C20)
+#define PIPE_WGC_C22(dev_priv, pipe)		_MMIO_TRANS2(dev_priv, pipe, _PIPE_A_WGC_C22)
 
 /* pipe CSC & degamma/gamma LUTs on CHV */
 #define _CGM_PIPE_A_CSC_COEFF01	(VLV_DISPLAY_BASE + 0x67900)
diff --git a/drivers/gpu/drm/i915/display/intel_combo_phy_regs.h b/drivers/gpu/drm/i915/display/intel_combo_phy_regs.h
index b0983edccf3f..0964e392d02c 100644
--- a/drivers/gpu/drm/i915/display/intel_combo_phy_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_combo_phy_regs.h
@@ -25,28 +25,26 @@
 						 4 * (dw))
 
 #define ICL_PORT_CL_DW5(phy)			_MMIO(_ICL_PORT_CL_DW(5, phy))
-#define   CL_POWER_DOWN_ENABLE			(1 << 4)
-#define   SUS_CLOCK_CONFIG			(3 << 0)
+#define   CL_POWER_DOWN_ENABLE			REG_BIT(4)
+#define   SUS_CLOCK_CONFIG			REG_GENMASK(1, 0)
 
 #define ICL_PORT_CL_DW10(phy)			_MMIO(_ICL_PORT_CL_DW(10, phy))
-#define  PG_SEQ_DELAY_OVERRIDE_MASK		(3 << 25)
-#define  PG_SEQ_DELAY_OVERRIDE_SHIFT		25
-#define  PG_SEQ_DELAY_OVERRIDE_ENABLE		(1 << 24)
-#define  PWR_UP_ALL_LANES			(0x0 << 4)
-#define  PWR_DOWN_LN_3_2_1			(0xe << 4)
-#define  PWR_DOWN_LN_3_2			(0xc << 4)
-#define  PWR_DOWN_LN_3				(0x8 << 4)
-#define  PWR_DOWN_LN_2_1_0			(0x7 << 4)
-#define  PWR_DOWN_LN_1_0			(0x3 << 4)
-#define  PWR_DOWN_LN_3_1			(0xa << 4)
-#define  PWR_DOWN_LN_3_1_0			(0xb << 4)
-#define  PWR_DOWN_LN_MASK			(0xf << 4)
-#define  PWR_DOWN_LN_SHIFT			4
-#define  EDP4K2K_MODE_OVRD_EN			(1 << 3)
-#define  EDP4K2K_MODE_OVRD_OPTIMIZED		(1 << 2)
+#define  PG_SEQ_DELAY_OVERRIDE_MASK		REG_GENMASK(26, 25)
+#define  PG_SEQ_DELAY_OVERRIDE_ENABLE		REG_BIT(24)
+#define  PWR_DOWN_LN_MASK			REG_GENMASK(7, 4)
+#define  PWR_UP_ALL_LANES			REG_FIELD_PREP(PWR_DOWN_LN_MASK, 0x0)
+#define  PWR_DOWN_LN_3_2_1			REG_FIELD_PREP(PWR_DOWN_LN_MASK, 0xe)
+#define  PWR_DOWN_LN_3_2			REG_FIELD_PREP(PWR_DOWN_LN_MASK, 0xc)
+#define  PWR_DOWN_LN_3				REG_FIELD_PREP(PWR_DOWN_LN_MASK, 0x8)
+#define  PWR_DOWN_LN_2_1_0			REG_FIELD_PREP(PWR_DOWN_LN_MASK, 0x7)
+#define  PWR_DOWN_LN_1_0			REG_FIELD_PREP(PWR_DOWN_LN_MASK, 0x3)
+#define  PWR_DOWN_LN_3_1			REG_FIELD_PREP(PWR_DOWN_LN_MASK, 0xa)
+#define  PWR_DOWN_LN_3_1_0			REG_FIELD_PREP(PWR_DOWN_LN_MASK, 0xb)
+#define  EDP4K2K_MODE_OVRD_EN			REG_BIT(3)
+#define  EDP4K2K_MODE_OVRD_OPTIMIZED		REG_BIT(2)
 
 #define ICL_PORT_CL_DW12(phy)			_MMIO(_ICL_PORT_CL_DW(12, phy))
-#define   ICL_LANE_ENABLE_AUX			(1 << 0)
+#define   ICL_LANE_ENABLE_AUX			REG_BIT(0)
 
 /* ICL Port COMP_DW registers */
 #define _ICL_PORT_COMP				0x100
@@ -54,24 +52,22 @@
 						 _ICL_PORT_COMP + 4 * (dw))
 
 #define ICL_PORT_COMP_DW0(phy)			_MMIO(_ICL_PORT_COMP_DW(0, phy))
-#define   COMP_INIT				(1 << 31)
+#define   COMP_INIT				REG_BIT(31)
 
 #define ICL_PORT_COMP_DW1(phy)			_MMIO(_ICL_PORT_COMP_DW(1, phy))
 
 #define ICL_PORT_COMP_DW3(phy)			_MMIO(_ICL_PORT_COMP_DW(3, phy))
-#define   PROCESS_INFO_DOT_0			(0 << 26)
-#define   PROCESS_INFO_DOT_1			(1 << 26)
-#define   PROCESS_INFO_DOT_4			(2 << 26)
-#define   PROCESS_INFO_MASK			(7 << 26)
-#define   PROCESS_INFO_SHIFT			26
-#define   VOLTAGE_INFO_0_85V			(0 << 24)
-#define   VOLTAGE_INFO_0_95V			(1 << 24)
-#define   VOLTAGE_INFO_1_05V			(2 << 24)
-#define   VOLTAGE_INFO_MASK			(3 << 24)
-#define   VOLTAGE_INFO_SHIFT			24
+#define   PROCESS_INFO_MASK			REG_GENMASK(28, 26)
+#define   PROCESS_INFO_DOT_0			REG_FIELD_PREP(PROCESS_INFO_MASK, 0)
+#define   PROCESS_INFO_DOT_1			REG_FIELD_PREP(PROCESS_INFO_MASK, 1)
+#define   PROCESS_INFO_DOT_4			REG_FIELD_PREP(PROCESS_INFO_MASK, 2)
+#define   VOLTAGE_INFO_MASK			REG_GENMASK(25, 24)
+#define   VOLTAGE_INFO_0_85V			REG_FIELD_PREP(VOLTAGE_INFO_MASK, 0)
+#define   VOLTAGE_INFO_0_95V			REG_FIELD_PREP(VOLTAGE_INFO_MASK, 1)
+#define   VOLTAGE_INFO_1_05V			REG_FIELD_PREP(VOLTAGE_INFO_MASK, 2)
 
 #define ICL_PORT_COMP_DW8(phy)			_MMIO(_ICL_PORT_COMP_DW(8, phy))
-#define   IREFGEN				(1 << 24)
+#define   IREFGEN				REG_BIT(24)
 
 #define ICL_PORT_COMP_DW9(phy)			_MMIO(_ICL_PORT_COMP_DW(9, phy))
 
@@ -92,9 +88,9 @@
 #define ICL_PORT_PCS_DW1_LN(ln, phy)		_MMIO(_ICL_PORT_PCS_DW_LN(1, ln, phy))
 #define   DCC_MODE_SELECT_MASK			REG_GENMASK(21, 20)
 #define   RUN_DCC_ONCE				REG_FIELD_PREP(DCC_MODE_SELECT_MASK, 0)
-#define   COMMON_KEEPER_EN			(1 << 26)
-#define   LATENCY_OPTIM_MASK			(0x3 << 2)
-#define   LATENCY_OPTIM_VAL(x)			((x) << 2)
+#define   COMMON_KEEPER_EN			REG_BIT(26)
+#define   LATENCY_OPTIM_MASK			REG_GENMASK(3, 2)
+#define   LATENCY_OPTIM_VAL(x)			REG_FIELD_PREP(LATENCY_OPTIM_MASK, (x))
 
 /* ICL Port TX registers */
 #define _ICL_PORT_TX_AUX			0x380
@@ -111,42 +107,49 @@
 #define ICL_PORT_TX_DW2_AUX(phy)		_MMIO(_ICL_PORT_TX_DW_AUX(2, phy))
 #define ICL_PORT_TX_DW2_GRP(phy)		_MMIO(_ICL_PORT_TX_DW_GRP(2, phy))
 #define ICL_PORT_TX_DW2_LN(ln, phy)		_MMIO(_ICL_PORT_TX_DW_LN(2, ln, phy))
-#define   SWING_SEL_UPPER(x)			(((x) >> 3) << 15)
-#define   SWING_SEL_UPPER_MASK			(1 << 15)
-#define   SWING_SEL_LOWER(x)			(((x) & 0x7) << 11)
-#define   SWING_SEL_LOWER_MASK			(0x7 << 11)
-#define   FRC_LATENCY_OPTIM_MASK		(0x7 << 8)
-#define   FRC_LATENCY_OPTIM_VAL(x)		((x) << 8)
-#define   RCOMP_SCALAR(x)			((x) << 0)
-#define   RCOMP_SCALAR_MASK			(0xFF << 0)
+#define   SWING_SEL_UPPER_MASK			REG_BIT(15)
+#define   SWING_SEL_UPPER(x)			REG_FIELD_PREP(SWING_SEL_UPPER_MASK, (x) >> 3)
+#define   SWING_SEL_LOWER_MASK			REG_GENMASK(13, 11)
+#define   SWING_SEL_LOWER(x)			REG_FIELD_PREP(SWING_SEL_LOWER_MASK, (x) & 0x7)
+#define   FRC_LATENCY_OPTIM_MASK		REG_GENMASK(10, 8)
+#define   FRC_LATENCY_OPTIM_VAL(x)		REG_FIELD_PREP(FRC_LATENCY_OPTIM_MASK, (x))
+#define   RCOMP_SCALAR_MASK			REG_GENMASK(7, 0)
+#define   RCOMP_SCALAR(x)			REG_FIELD_PREP(RCOMP_SCALAR_MASK, (x))
 
 #define ICL_PORT_TX_DW4_AUX(phy)		_MMIO(_ICL_PORT_TX_DW_AUX(4, phy))
 #define ICL_PORT_TX_DW4_GRP(phy)		_MMIO(_ICL_PORT_TX_DW_GRP(4, phy))
 #define ICL_PORT_TX_DW4_LN(ln, phy)		_MMIO(_ICL_PORT_TX_DW_LN(4, ln, phy))
-#define   LOADGEN_SELECT			(1 << 31)
-#define   POST_CURSOR_1(x)			((x) << 12)
-#define   POST_CURSOR_1_MASK			(0x3F << 12)
-#define   POST_CURSOR_2(x)			((x) << 6)
-#define   POST_CURSOR_2_MASK			(0x3F << 6)
-#define   CURSOR_COEFF(x)			((x) << 0)
-#define   CURSOR_COEFF_MASK			(0x3F << 0)
+#define   LOADGEN_SELECT			REG_BIT(31)
+#define   POST_CURSOR_1_MASK			REG_GENMASK(17, 12)
+#define   POST_CURSOR_1(x)			REG_FIELD_PREP(POST_CURSOR_1_MASK, (x))
+#define   POST_CURSOR_2_MASK			REG_GENMASK(11, 6)
+#define   POST_CURSOR_2(x)			REG_FIELD_PREP(POST_CURSOR_2_MASK, (x))
+#define   CURSOR_COEFF_MASK			REG_GENMASK(5, 0)
+#define   CURSOR_COEFF(x)			REG_FIELD_PREP(CURSOR_COEFF_MASK, (x))
 
 #define ICL_PORT_TX_DW5_AUX(phy)		_MMIO(_ICL_PORT_TX_DW_AUX(5, phy))
 #define ICL_PORT_TX_DW5_GRP(phy)		_MMIO(_ICL_PORT_TX_DW_GRP(5, phy))
 #define ICL_PORT_TX_DW5_LN(ln, phy)		_MMIO(_ICL_PORT_TX_DW_LN(5, ln, phy))
-#define   TX_TRAINING_EN			(1 << 31)
-#define   TAP2_DISABLE				(1 << 30)
-#define   TAP3_DISABLE				(1 << 29)
-#define   SCALING_MODE_SEL(x)			((x) << 18)
-#define   SCALING_MODE_SEL_MASK			(0x7 << 18)
-#define   RTERM_SELECT(x)			((x) << 3)
-#define   RTERM_SELECT_MASK			(0x7 << 3)
+#define   TX_TRAINING_EN			REG_BIT(31)
+#define   TAP2_DISABLE				REG_BIT(30)
+#define   TAP3_DISABLE				REG_BIT(29)
+#define   SCALING_MODE_SEL_MASK			REG_GENMASK(20, 18)
+#define   SCALING_MODE_SEL(x)			REG_FIELD_PREP(SCALING_MODE_SEL_MASK, (x))
+#define   RTERM_SELECT_MASK			REG_GENMASK(5, 3)
+#define   RTERM_SELECT(x)			REG_FIELD_PREP(RTERM_SELECT_MASK, (x))
+
+#define ICL_PORT_TX_DW6_AUX(phy)		_MMIO(_ICL_PORT_TX_DW_AUX(6, phy))
+#define ICL_PORT_TX_DW6_GRP(phy)		_MMIO(_ICL_PORT_TX_DW_GRP(6, phy))
+#define ICL_PORT_TX_DW6_LN(ln, phy)		_MMIO(_ICL_PORT_TX_DW_LN(6, ln, phy))
+#define   O_FUNC_OVRD_EN			REG_BIT(7)
+#define   O_LDO_REF_SEL_CRI			REG_GENMASK(6, 1)
+#define   O_LDO_BYPASS_CRI			REG_BIT(0)
 
 #define ICL_PORT_TX_DW7_AUX(phy)		_MMIO(_ICL_PORT_TX_DW_AUX(7, phy))
 #define ICL_PORT_TX_DW7_GRP(phy)		_MMIO(_ICL_PORT_TX_DW_GRP(7, phy))
 #define ICL_PORT_TX_DW7_LN(ln, phy)		_MMIO(_ICL_PORT_TX_DW_LN(7, ln, phy))
-#define   N_SCALAR(x)				((x) << 24)
-#define   N_SCALAR_MASK				(0x7F << 24)
+#define   N_SCALAR_MASK				REG_GENMASK(30, 24)
+#define   N_SCALAR(x)				REG_FIELD_PREP(N_SCALAR_MASK, (x))
 
 #define ICL_PORT_TX_DW8_AUX(phy)		_MMIO(_ICL_PORT_TX_DW_AUX(8, phy))
 #define ICL_PORT_TX_DW8_GRP(phy)		_MMIO(_ICL_PORT_TX_DW_GRP(8, phy))
diff --git a/drivers/gpu/drm/i915/display/intel_crt.c b/drivers/gpu/drm/i915/display/intel_crt.c
index 93479db0f89f..10e95dc425a6 100644
--- a/drivers/gpu/drm/i915/display/intel_crt.c
+++ b/drivers/gpu/drm/i915/display/intel_crt.c
@@ -348,7 +348,7 @@ intel_crt_mode_valid(struct drm_connector *connector,
 {
 	struct drm_device *dev = connector->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	int max_dotclk = dev_priv->max_dotclk_freq;
+	int max_dotclk = dev_priv->display.cdclk.max_dotclk_freq;
 	enum drm_mode_status status;
 	int max_clock;
 
@@ -356,9 +356,6 @@ intel_crt_mode_valid(struct drm_connector *connector,
 	if (status != MODE_OK)
 		return status;
 
-	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
-		return MODE_NO_DBLESCAN;
-
 	if (mode->clock < 25000)
 		return MODE_CLOCK_LOW;
 
diff --git a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
index 4bcf446c75f4..ccaa4cb2809b 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
+++ b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
@@ -12,33 +12,31 @@
 #include "intel_hdmi.h"
 #include "intel_vrr.h"
 
-static void intel_dump_crtc_timings(struct drm_i915_private *i915,
+static void intel_dump_crtc_timings(struct drm_printer *p,
 				    const struct drm_display_mode *mode)
 {
-	drm_dbg_kms(&i915->drm, "crtc timings: clock=%d, "
-		    "hd=%d hb=%d-%d hs=%d-%d ht=%d, "
-		    "vd=%d vb=%d-%d vs=%d-%d vt=%d, "
-		    "flags=0x%x\n",
-		    mode->crtc_clock,
-		    mode->crtc_hdisplay, mode->crtc_hblank_start, mode->crtc_hblank_end,
-		    mode->crtc_hsync_start, mode->crtc_hsync_end, mode->crtc_htotal,
-		    mode->crtc_vdisplay, mode->crtc_vblank_start, mode->crtc_vblank_end,
-		    mode->crtc_vsync_start, mode->crtc_vsync_end, mode->crtc_vtotal,
-		    mode->flags);
+	drm_printf(p, "crtc timings: clock=%d, "
+		   "hd=%d hb=%d-%d hs=%d-%d ht=%d, "
+		   "vd=%d vb=%d-%d vs=%d-%d vt=%d, "
+		   "flags=0x%x\n",
+		   mode->crtc_clock,
+		   mode->crtc_hdisplay, mode->crtc_hblank_start, mode->crtc_hblank_end,
+		   mode->crtc_hsync_start, mode->crtc_hsync_end, mode->crtc_htotal,
+		   mode->crtc_vdisplay, mode->crtc_vblank_start, mode->crtc_vblank_end,
+		   mode->crtc_vsync_start, mode->crtc_vsync_end, mode->crtc_vtotal,
+		   mode->flags);
 }
 
 static void
-intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
+intel_dump_m_n_config(struct drm_printer *p,
+		      const struct intel_crtc_state *pipe_config,
 		      const char *id, unsigned int lane_count,
 		      const struct intel_link_m_n *m_n)
 {
-	struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
-
-	drm_dbg_kms(&i915->drm,
-		    "%s: lanes: %i; data_m: %u, data_n: %u, link_m: %u, link_n: %u, tu: %u\n",
-		    id, lane_count,
-		    m_n->data_m, m_n->data_n,
-		    m_n->link_m, m_n->link_n, m_n->tu);
+	drm_printf(p, "%s: lanes: %i; data_m: %u, data_n: %u, link_m: %u, link_n: %u, tu: %u\n",
+		   id, lane_count,
+		   m_n->data_m, m_n->data_n,
+		   m_n->link_m, m_n->link_n, m_n->tu);
 }
 
 static void
@@ -52,17 +50,7 @@ intel_dump_infoframe(struct drm_i915_private *i915,
 }
 
 static void
-intel_dump_dp_vsc_sdp(struct drm_i915_private *i915,
-		      const struct drm_dp_vsc_sdp *vsc)
-{
-	struct drm_printer p = drm_dbg_printer(&i915->drm, DRM_UT_KMS, NULL);
-
-	drm_dp_vsc_sdp_log(&p, vsc);
-}
-
-static void
-intel_dump_buffer(struct drm_i915_private *i915,
-		  const char *prefix, const u8 *buf, size_t len)
+intel_dump_buffer(const char *prefix, const u8 *buf, size_t len)
 {
 	if (!drm_debug_enabled(DRM_UT_KMS))
 		return;
@@ -130,71 +118,66 @@ const char *intel_output_format_name(enum intel_output_format format)
 	return output_format_str[format];
 }
 
-static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
+static void intel_dump_plane_state(struct drm_printer *p,
+				   const struct intel_plane_state *plane_state)
 {
 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
-	struct drm_i915_private *i915 = to_i915(plane->base.dev);
 	const struct drm_framebuffer *fb = plane_state->hw.fb;
 
 	if (!fb) {
-		drm_dbg_kms(&i915->drm,
-			    "[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
-			    plane->base.base.id, plane->base.name,
-			    str_yes_no(plane_state->uapi.visible));
+		drm_printf(p, "[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
+			   plane->base.base.id, plane->base.name,
+			   str_yes_no(plane_state->uapi.visible));
 		return;
 	}
 
-	drm_dbg_kms(&i915->drm,
-		    "[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %p4cc modifier = 0x%llx, visible: %s\n",
-		    plane->base.base.id, plane->base.name,
-		    fb->base.id, fb->width, fb->height, &fb->format->format,
-		    fb->modifier, str_yes_no(plane_state->uapi.visible));
-	drm_dbg_kms(&i915->drm, "\trotation: 0x%x, scaler: %d, scaling_filter: %d\n",
-		    plane_state->hw.rotation, plane_state->scaler_id, plane_state->hw.scaling_filter);
+	drm_printf(p, "[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %p4cc modifier = 0x%llx, visible: %s\n",
+		   plane->base.base.id, plane->base.name,
+		   fb->base.id, fb->width, fb->height, &fb->format->format,
+		   fb->modifier, str_yes_no(plane_state->uapi.visible));
+	drm_printf(p, "\trotation: 0x%x, scaler: %d, scaling_filter: %d\n",
+		   plane_state->hw.rotation, plane_state->scaler_id, plane_state->hw.scaling_filter);
 	if (plane_state->uapi.visible)
-		drm_dbg_kms(&i915->drm,
-			    "\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
-			    DRM_RECT_FP_ARG(&plane_state->uapi.src),
-			    DRM_RECT_ARG(&plane_state->uapi.dst));
+		drm_printf(p, "\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
+			   DRM_RECT_FP_ARG(&plane_state->uapi.src),
+			   DRM_RECT_ARG(&plane_state->uapi.dst));
 }
 
 static void
-ilk_dump_csc(struct drm_i915_private *i915, const char *name,
+ilk_dump_csc(struct drm_i915_private *i915,
+	     struct drm_printer *p,
+	     const char *name,
 	     const struct intel_csc_matrix *csc)
 {
 	int i;
 
-	drm_dbg_kms(&i915->drm,
-		    "%s: pre offsets: 0x%04x 0x%04x 0x%04x\n", name,
-		    csc->preoff[0], csc->preoff[1], csc->preoff[2]);
+	drm_printf(p, "%s: pre offsets: 0x%04x 0x%04x 0x%04x\n", name,
+		   csc->preoff[0], csc->preoff[1], csc->preoff[2]);
 
 	for (i = 0; i < 3; i++)
-		drm_dbg_kms(&i915->drm,
-			    "%s: coefficients: 0x%04x 0x%04x 0x%04x\n", name,
-			    csc->coeff[3 * i + 0],
-			    csc->coeff[3 * i + 1],
-			    csc->coeff[3 * i + 2]);
+		drm_printf(p, "%s: coefficients: 0x%04x 0x%04x 0x%04x\n", name,
+			   csc->coeff[3 * i + 0],
+			   csc->coeff[3 * i + 1],
+			   csc->coeff[3 * i + 2]);
 
 	if (DISPLAY_VER(i915) < 7)
 		return;
 
-	drm_dbg_kms(&i915->drm,
-		    "%s: post offsets: 0x%04x 0x%04x 0x%04x\n", name,
-		    csc->postoff[0], csc->postoff[1], csc->postoff[2]);
+	drm_printf(p, "%s: post offsets: 0x%04x 0x%04x 0x%04x\n", name,
+		   csc->postoff[0], csc->postoff[1], csc->postoff[2]);
 }
 
 static void
-vlv_dump_csc(struct drm_i915_private *i915, const char *name,
+vlv_dump_csc(struct drm_printer *p, const char *name,
 	     const struct intel_csc_matrix *csc)
 {
 	int i;
 
 	for (i = 0; i < 3; i++)
-		drm_dbg_kms(&i915->drm,
-			    "%s: coefficients: 0x%04x 0x%04x 0x%04x\n", name,
-			    csc->coeff[3 * i + 0],
-			    csc->coeff[3 * i + 1],
-			    csc->coeff[3 * i + 2]);
+		drm_printf(p, "%s: coefficients: 0x%04x 0x%04x 0x%04x\n", name,
+			   csc->coeff[3 * i + 0],
+			   csc->coeff[3 * i + 1],
+			   csc->coeff[3 * i + 2]);
 }
 
 void intel_crtc_state_dump(const struct intel_crtc_state *pipe_config,
@@ -205,85 +188,86 @@ void intel_crtc_state_dump(const struct intel_crtc_state *pipe_config,
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 	const struct intel_plane_state *plane_state;
 	struct intel_plane *plane;
+	struct drm_printer p;
 	char buf[64];
 	int i;
 
-	drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] enable: %s [%s]\n",
-		    crtc->base.base.id, crtc->base.name,
-		    str_yes_no(pipe_config->hw.enable), context);
+	if (!drm_debug_enabled(DRM_UT_KMS))
+		return;
+
+	p = drm_dbg_printer(&i915->drm, DRM_UT_KMS, NULL);
+
+	drm_printf(&p, "[CRTC:%d:%s] enable: %s [%s]\n",
+		   crtc->base.base.id, crtc->base.name,
+		   str_yes_no(pipe_config->hw.enable), context);
 
 	if (!pipe_config->hw.enable)
 		goto dump_planes;
 
 	snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
-	drm_dbg_kms(&i915->drm,
-		    "active: %s, output_types: %s (0x%x), output format: %s, sink format: %s\n",
-		    str_yes_no(pipe_config->hw.active),
-		    buf, pipe_config->output_types,
-		    intel_output_format_name(pipe_config->output_format),
-		    intel_output_format_name(pipe_config->sink_format));
-
-	drm_dbg_kms(&i915->drm,
-		    "cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
-		    transcoder_name(pipe_config->cpu_transcoder),
-		    pipe_config->pipe_bpp, pipe_config->dither);
-
-	drm_dbg_kms(&i915->drm, "MST master transcoder: %s\n",
-		    transcoder_name(pipe_config->mst_master_transcoder));
-
-	drm_dbg_kms(&i915->drm,
-		    "port sync: master transcoder: %s, slave transcoder bitmask = 0x%x\n",
-		    transcoder_name(pipe_config->master_transcoder),
-		    pipe_config->sync_mode_slaves_mask);
-
-	drm_dbg_kms(&i915->drm, "bigjoiner: %s, pipes: 0x%x\n",
-		    intel_crtc_is_bigjoiner_slave(pipe_config) ? "slave" :
-		    intel_crtc_is_bigjoiner_master(pipe_config) ? "master" : "no",
-		    pipe_config->bigjoiner_pipes);
-
-	drm_dbg_kms(&i915->drm, "splitter: %s, link count %d, overlap %d\n",
-		    str_enabled_disabled(pipe_config->splitter.enable),
-		    pipe_config->splitter.link_count,
-		    pipe_config->splitter.pixel_overlap);
+	drm_printf(&p, "active: %s, output_types: %s (0x%x), output format: %s, sink format: %s\n",
+		   str_yes_no(pipe_config->hw.active),
+		   buf, pipe_config->output_types,
+		   intel_output_format_name(pipe_config->output_format),
+		   intel_output_format_name(pipe_config->sink_format));
+
+	drm_printf(&p, "cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
+		   transcoder_name(pipe_config->cpu_transcoder),
+		   pipe_config->pipe_bpp, pipe_config->dither);
+
+	drm_printf(&p, "MST master transcoder: %s\n",
+		   transcoder_name(pipe_config->mst_master_transcoder));
+
+	drm_printf(&p, "port sync: master transcoder: %s, slave transcoder bitmask = 0x%x\n",
+		   transcoder_name(pipe_config->master_transcoder),
+		   pipe_config->sync_mode_slaves_mask);
+
+	drm_printf(&p, "bigjoiner: %s, pipes: 0x%x\n",
+		   intel_crtc_is_bigjoiner_slave(pipe_config) ? "slave" :
+		   intel_crtc_is_bigjoiner_master(pipe_config) ? "master" : "no",
+		   pipe_config->bigjoiner_pipes);
+
+	drm_printf(&p, "splitter: %s, link count %d, overlap %d\n",
+		   str_enabled_disabled(pipe_config->splitter.enable),
+		   pipe_config->splitter.link_count,
+		   pipe_config->splitter.pixel_overlap);
 
 	if (pipe_config->has_pch_encoder)
-		intel_dump_m_n_config(pipe_config, "fdi",
+		intel_dump_m_n_config(&p, pipe_config, "fdi",
 				      pipe_config->fdi_lanes,
 				      &pipe_config->fdi_m_n);
 
 	if (intel_crtc_has_dp_encoder(pipe_config)) {
-		intel_dump_m_n_config(pipe_config, "dp m_n",
+		intel_dump_m_n_config(&p, pipe_config, "dp m_n",
 				      pipe_config->lane_count,
 				      &pipe_config->dp_m_n);
-		intel_dump_m_n_config(pipe_config, "dp m2_n2",
+		intel_dump_m_n_config(&p, pipe_config, "dp m2_n2",
 				      pipe_config->lane_count,
 				      &pipe_config->dp_m2_n2);
-		drm_dbg_kms(&i915->drm, "fec: %s, enhanced framing: %s\n",
-			    str_enabled_disabled(pipe_config->fec_enable),
-			    str_enabled_disabled(pipe_config->enhanced_framing));
-
-		drm_dbg_kms(&i915->drm, "sdp split: %s\n",
-			    str_enabled_disabled(pipe_config->sdp_split_enable));
-
-		drm_dbg_kms(&i915->drm, "psr: %s, psr2: %s, panel replay: %s, selective fetch: %s\n",
-			    str_enabled_disabled(pipe_config->has_psr),
-			    str_enabled_disabled(pipe_config->has_psr2),
-			    str_enabled_disabled(pipe_config->has_panel_replay),
-			    str_enabled_disabled(pipe_config->enable_psr2_sel_fetch));
+		drm_printf(&p, "fec: %s, enhanced framing: %s\n",
+			   str_enabled_disabled(pipe_config->fec_enable),
+			   str_enabled_disabled(pipe_config->enhanced_framing));
+
+		drm_printf(&p, "sdp split: %s\n",
+			   str_enabled_disabled(pipe_config->sdp_split_enable));
+
+		drm_printf(&p, "psr: %s, psr2: %s, panel replay: %s, selective fetch: %s\n",
+			   str_enabled_disabled(pipe_config->has_psr),
+			   str_enabled_disabled(pipe_config->has_psr2),
+			   str_enabled_disabled(pipe_config->has_panel_replay),
+			   str_enabled_disabled(pipe_config->enable_psr2_sel_fetch));
 	}
 
-	drm_dbg_kms(&i915->drm, "framestart delay: %d, MSA timing delay: %d\n",
-		    pipe_config->framestart_delay, pipe_config->msa_timing_delay);
+	drm_printf(&p, "framestart delay: %d, MSA timing delay: %d\n",
+		   pipe_config->framestart_delay, pipe_config->msa_timing_delay);
 
-	drm_dbg_kms(&i915->drm,
-		    "audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
-		    pipe_config->has_audio, pipe_config->has_infoframe,
-		    pipe_config->infoframes.enable);
+	drm_printf(&p, "audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
+		   pipe_config->has_audio, pipe_config->has_infoframe,
+		   pipe_config->infoframes.enable);
 
 	if (pipe_config->infoframes.enable &
 	    intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
-		drm_dbg_kms(&i915->drm, "GCP: 0x%x\n",
-			    pipe_config->infoframes.gcp);
+		drm_printf(&p, "GCP: 0x%x\n", pipe_config->infoframes.gcp);
 	if (pipe_config->infoframes.enable &
 	    intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
 		intel_dump_infoframe(i915, &pipe_config->infoframes.avi);
@@ -301,91 +285,88 @@ void intel_crtc_state_dump(const struct intel_crtc_state *pipe_config,
 		intel_dump_infoframe(i915, &pipe_config->infoframes.drm);
 	if (pipe_config->infoframes.enable &
 	    intel_hdmi_infoframe_enable(DP_SDP_VSC))
-		intel_dump_dp_vsc_sdp(i915, &pipe_config->infoframes.vsc);
+		drm_dp_vsc_sdp_log(&p, &pipe_config->infoframes.vsc);
+	if (pipe_config->infoframes.enable &
+	    intel_hdmi_infoframe_enable(DP_SDP_ADAPTIVE_SYNC))
+		drm_dp_as_sdp_log(&p, &pipe_config->infoframes.as_sdp);
 
 	if (pipe_config->has_audio)
-		intel_dump_buffer(i915, "ELD: ", pipe_config->eld,
+		intel_dump_buffer("ELD: ", pipe_config->eld,
 				  drm_eld_size(pipe_config->eld));
 
-	drm_dbg_kms(&i915->drm, "vrr: %s, vmin: %d, vmax: %d, pipeline full: %d, guardband: %d flipline: %d, vmin vblank: %d, vmax vblank: %d\n",
-		    str_yes_no(pipe_config->vrr.enable),
-		    pipe_config->vrr.vmin, pipe_config->vrr.vmax,
-		    pipe_config->vrr.pipeline_full, pipe_config->vrr.guardband,
-		    pipe_config->vrr.flipline,
-		    intel_vrr_vmin_vblank_start(pipe_config),
-		    intel_vrr_vmax_vblank_start(pipe_config));
-
-	drm_dbg_kms(&i915->drm, "requested mode: " DRM_MODE_FMT "\n",
-		    DRM_MODE_ARG(&pipe_config->hw.mode));
-	drm_dbg_kms(&i915->drm, "adjusted mode: " DRM_MODE_FMT "\n",
-		    DRM_MODE_ARG(&pipe_config->hw.adjusted_mode));
-	intel_dump_crtc_timings(i915, &pipe_config->hw.adjusted_mode);
-	drm_dbg_kms(&i915->drm, "pipe mode: " DRM_MODE_FMT "\n",
-		    DRM_MODE_ARG(&pipe_config->hw.pipe_mode));
-	intel_dump_crtc_timings(i915, &pipe_config->hw.pipe_mode);
-	drm_dbg_kms(&i915->drm,
-		    "port clock: %d, pipe src: " DRM_RECT_FMT ", pixel rate %d\n",
-		    pipe_config->port_clock, DRM_RECT_ARG(&pipe_config->pipe_src),
-		    pipe_config->pixel_rate);
-
-	drm_dbg_kms(&i915->drm, "linetime: %d, ips linetime: %d\n",
-		    pipe_config->linetime, pipe_config->ips_linetime);
+	drm_printf(&p, "vrr: %s, vmin: %d, vmax: %d, pipeline full: %d, guardband: %d flipline: %d, vmin vblank: %d, vmax vblank: %d\n",
+		   str_yes_no(pipe_config->vrr.enable),
+		   pipe_config->vrr.vmin, pipe_config->vrr.vmax,
+		   pipe_config->vrr.pipeline_full, pipe_config->vrr.guardband,
+		   pipe_config->vrr.flipline,
+		   intel_vrr_vmin_vblank_start(pipe_config),
+		   intel_vrr_vmax_vblank_start(pipe_config));
+
+	drm_printf(&p, "requested mode: " DRM_MODE_FMT "\n",
+		   DRM_MODE_ARG(&pipe_config->hw.mode));
+	drm_printf(&p, "adjusted mode: " DRM_MODE_FMT "\n",
+		   DRM_MODE_ARG(&pipe_config->hw.adjusted_mode));
+	intel_dump_crtc_timings(&p, &pipe_config->hw.adjusted_mode);
+	drm_printf(&p, "pipe mode: " DRM_MODE_FMT "\n",
+		   DRM_MODE_ARG(&pipe_config->hw.pipe_mode));
+	intel_dump_crtc_timings(&p, &pipe_config->hw.pipe_mode);
+	drm_printf(&p, "port clock: %d, pipe src: " DRM_RECT_FMT ", pixel rate %d\n",
+		   pipe_config->port_clock, DRM_RECT_ARG(&pipe_config->pipe_src),
+		   pipe_config->pixel_rate);
+
+	drm_printf(&p, "linetime: %d, ips linetime: %d\n",
+		   pipe_config->linetime, pipe_config->ips_linetime);
 
 	if (DISPLAY_VER(i915) >= 9)
-		drm_dbg_kms(&i915->drm,
-			    "num_scalers: %d, scaler_users: 0x%x, scaler_id: %d, scaling_filter: %d\n",
-			    crtc->num_scalers,
-			    pipe_config->scaler_state.scaler_users,
-			    pipe_config->scaler_state.scaler_id,
-			    pipe_config->hw.scaling_filter);
+		drm_printf(&p, "num_scalers: %d, scaler_users: 0x%x, scaler_id: %d, scaling_filter: %d\n",
+			   crtc->num_scalers,
+			   pipe_config->scaler_state.scaler_users,
+			   pipe_config->scaler_state.scaler_id,
+			   pipe_config->hw.scaling_filter);
 
 	if (HAS_GMCH(i915))
-		drm_dbg_kms(&i915->drm,
-			    "gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
-			    pipe_config->gmch_pfit.control,
-			    pipe_config->gmch_pfit.pgm_ratios,
-			    pipe_config->gmch_pfit.lvds_border_bits);
+		drm_printf(&p, "gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
+			   pipe_config->gmch_pfit.control,
+			   pipe_config->gmch_pfit.pgm_ratios,
+			   pipe_config->gmch_pfit.lvds_border_bits);
 	else
-		drm_dbg_kms(&i915->drm,
-			    "pch pfit: " DRM_RECT_FMT ", %s, force thru: %s\n",
-			    DRM_RECT_ARG(&pipe_config->pch_pfit.dst),
-			    str_enabled_disabled(pipe_config->pch_pfit.enabled),
-			    str_yes_no(pipe_config->pch_pfit.force_thru));
+		drm_printf(&p, "pch pfit: " DRM_RECT_FMT ", %s, force thru: %s\n",
+			   DRM_RECT_ARG(&pipe_config->pch_pfit.dst),
+			   str_enabled_disabled(pipe_config->pch_pfit.enabled),
+			   str_yes_no(pipe_config->pch_pfit.force_thru));
 
-	drm_dbg_kms(&i915->drm, "ips: %i, double wide: %i, drrs: %i\n",
-		    pipe_config->ips_enabled, pipe_config->double_wide,
-		    pipe_config->has_drrs);
+	drm_printf(&p, "ips: %i, double wide: %i, drrs: %i\n",
+		   pipe_config->ips_enabled, pipe_config->double_wide,
+		   pipe_config->has_drrs);
 
-	intel_dpll_dump_hw_state(i915, &pipe_config->dpll_hw_state);
+	intel_dpll_dump_hw_state(i915, &p, &pipe_config->dpll_hw_state);
 
 	if (IS_CHERRYVIEW(i915))
-		drm_dbg_kms(&i915->drm,
-			    "cgm_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
-			    pipe_config->cgm_mode, pipe_config->gamma_mode,
-			    pipe_config->gamma_enable, pipe_config->csc_enable);
+		drm_printf(&p, "cgm_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
+			   pipe_config->cgm_mode, pipe_config->gamma_mode,
+			   pipe_config->gamma_enable, pipe_config->csc_enable);
 	else
-		drm_dbg_kms(&i915->drm,
-			    "csc_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
-			    pipe_config->csc_mode, pipe_config->gamma_mode,
-			    pipe_config->gamma_enable, pipe_config->csc_enable);
-
-	drm_dbg_kms(&i915->drm, "pre csc lut: %s%d entries, post csc lut: %d entries\n",
-		    pipe_config->pre_csc_lut && pipe_config->pre_csc_lut ==
-		    i915->display.color.glk_linear_degamma_lut ? "(linear) " : "",
-		    pipe_config->pre_csc_lut ?
-		    drm_color_lut_size(pipe_config->pre_csc_lut) : 0,
-		    pipe_config->post_csc_lut ?
-		    drm_color_lut_size(pipe_config->post_csc_lut) : 0);
+		drm_printf(&p, "csc_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
+			   pipe_config->csc_mode, pipe_config->gamma_mode,
+			   pipe_config->gamma_enable, pipe_config->csc_enable);
+
+	drm_printf(&p, "pre csc lut: %s%d entries, post csc lut: %d entries\n",
+		   pipe_config->pre_csc_lut && pipe_config->pre_csc_lut ==
+		   i915->display.color.glk_linear_degamma_lut ? "(linear) " : "",
+		   pipe_config->pre_csc_lut ?
+		   drm_color_lut_size(pipe_config->pre_csc_lut) : 0,
+		   pipe_config->post_csc_lut ?
+		   drm_color_lut_size(pipe_config->post_csc_lut) : 0);
 
 	if (DISPLAY_VER(i915) >= 11)
-		ilk_dump_csc(i915, "output csc", &pipe_config->output_csc);
+		ilk_dump_csc(i915, &p, "output csc", &pipe_config->output_csc);
 
 	if (!HAS_GMCH(i915))
-		ilk_dump_csc(i915, "pipe csc", &pipe_config->csc);
+		ilk_dump_csc(i915, &p, "pipe csc", &pipe_config->csc);
 	else if (IS_CHERRYVIEW(i915))
-		vlv_dump_csc(i915, "cgm csc", &pipe_config->csc);
+		vlv_dump_csc(&p, "cgm csc", &pipe_config->csc);
 	else if (IS_VALLEYVIEW(i915))
-		vlv_dump_csc(i915, "wgc csc", &pipe_config->csc);
+		vlv_dump_csc(&p, "wgc csc", &pipe_config->csc);
 
 dump_planes:
 	if (!state)
@@ -393,6 +374,6 @@ dump_planes:
 
 	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
 		if (plane->pipe == crtc->pipe)
-			intel_dump_plane_state(plane_state);
+			intel_dump_plane_state(&p, plane_state);
 	}
 }
diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c
index 0d3da55e1c24..23a122ee20c9 100644
--- a/drivers/gpu/drm/i915/display/intel_cursor.c
+++ b/drivers/gpu/drm/i915/display/intel_cursor.c
@@ -509,6 +509,24 @@ static void i9xx_cursor_disable_sel_fetch_arm(struct intel_plane *plane,
 	intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_CTL(pipe, plane->id), 0);
 }
 
+static void wa_16021440873(struct intel_plane *plane,
+			   const struct intel_crtc_state *crtc_state,
+			   const struct intel_plane_state *plane_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	u32 ctl = plane_state->ctl;
+	int et_y_position = drm_rect_height(&crtc_state->pipe_src) + 1;
+	enum pipe pipe = plane->pipe;
+
+	ctl &= ~MCURSOR_MODE_MASK;
+	ctl |= MCURSOR_MODE_64_2B;
+
+	intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_CTL(pipe, plane->id), ctl);
+
+	intel_de_write(dev_priv, PIPE_SRCSZ_ERLY_TPT(pipe),
+		       PIPESRC_HEIGHT(et_y_position));
+}
+
 static void i9xx_cursor_update_sel_fetch_arm(struct intel_plane *plane,
 					     const struct intel_crtc_state *crtc_state,
 					     const struct intel_plane_state *plane_state)
@@ -529,7 +547,11 @@ static void i9xx_cursor_update_sel_fetch_arm(struct intel_plane *plane,
 		intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_CTL(pipe, plane->id),
 				  plane_state->ctl);
 	} else {
-		i9xx_cursor_disable_sel_fetch_arm(plane, crtc_state);
+		/* Wa_16021440873 */
+		if (crtc_state->enable_psr2_su_region_et)
+			wa_16021440873(plane, crtc_state, plane_state);
+		else
+			i9xx_cursor_disable_sel_fetch_arm(plane, crtc_state);
 	}
 }
 
@@ -821,6 +843,28 @@ static const struct drm_plane_funcs intel_cursor_plane_funcs = {
 	.format_mod_supported = intel_cursor_format_mod_supported,
 };
 
+static void intel_cursor_add_size_hints_property(struct intel_plane *plane)
+{
+	struct drm_i915_private *i915 = to_i915(plane->base.dev);
+	const struct drm_mode_config *config = &i915->drm.mode_config;
+	struct drm_plane_size_hint hints[4];
+	int size, max_size, num_hints = 0;
+
+	max_size = min(config->cursor_width, config->cursor_height);
+
+	/* for simplicity only enumerate the supported square+POT sizes */
+	for (size = 64; size <= max_size; size *= 2) {
+		if (drm_WARN_ON(&i915->drm, num_hints >= ARRAY_SIZE(hints)))
+			break;
+
+		hints[num_hints].width = size;
+		hints[num_hints].height = size;
+		num_hints++;
+	}
+
+	drm_plane_add_size_hints_property(&plane->base, hints, num_hints);
+}
+
 struct intel_plane *
 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
 			  enum pipe pipe)
@@ -879,6 +923,8 @@ intel_cursor_plane_create(struct drm_i915_private *dev_priv,
 						   DRM_MODE_ROTATE_0 |
 						   DRM_MODE_ROTATE_180);
 
+	intel_cursor_add_size_hints_property(cursor);
+
 	zpos = DISPLAY_RUNTIME_INFO(dev_priv)->num_sprites[pipe] + 1;
 	drm_plane_create_zpos_immutable_property(&cursor->base, zpos);
 
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.c b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
index 64e0f820a789..8e3b13884bb8 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
@@ -29,8 +29,11 @@
 #define INTEL_CX0_LANE1		BIT(1)
 #define INTEL_CX0_BOTH_LANES	(INTEL_CX0_LANE1 | INTEL_CX0_LANE0)
 
-bool intel_is_c10phy(struct drm_i915_private *i915, enum phy phy)
+bool intel_encoder_is_c10phy(struct intel_encoder *encoder)
 {
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum phy phy = intel_encoder_to_phy(encoder);
+
 	if ((IS_LUNARLAKE(i915) || IS_METEORLAKE(i915)) && phy < PHY_C)
 		return true;
 
@@ -46,8 +49,7 @@ static int lane_mask_to_lane(u8 lane_mask)
 	return ilog2(lane_mask);
 }
 
-static u8 intel_cx0_get_owned_lane_mask(struct drm_i915_private *i915,
-					struct intel_encoder *encoder)
+static u8 intel_cx0_get_owned_lane_mask(struct intel_encoder *encoder)
 {
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 
@@ -114,16 +116,20 @@ static void intel_cx0_phy_transaction_end(struct intel_encoder *encoder, intel_w
 	intel_display_power_put(i915, POWER_DOMAIN_DC_OFF, wakeref);
 }
 
-static void intel_clear_response_ready_flag(struct drm_i915_private *i915,
-					    enum port port, int lane)
+static void intel_clear_response_ready_flag(struct intel_encoder *encoder,
+					    int lane)
 {
-	intel_de_rmw(i915, XELPDP_PORT_P2M_MSGBUS_STATUS(i915, port, lane),
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+
+	intel_de_rmw(i915, XELPDP_PORT_P2M_MSGBUS_STATUS(i915, encoder->port, lane),
 		     0, XELPDP_PORT_P2M_RESPONSE_READY | XELPDP_PORT_P2M_ERROR_SET);
 }
 
-static void intel_cx0_bus_reset(struct drm_i915_private *i915, enum port port, int lane)
+static void intel_cx0_bus_reset(struct intel_encoder *encoder, int lane)
 {
-	enum phy phy = intel_port_to_phy(i915, port);
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	intel_de_write(i915, XELPDP_PORT_M2P_MSGBUS_CTL(i915, port, lane),
 		       XELPDP_PORT_M2P_TRANSACTION_RESET);
@@ -135,20 +141,22 @@ static void intel_cx0_bus_reset(struct drm_i915_private *i915, enum port port, i
 		return;
 	}
 
-	intel_clear_response_ready_flag(i915, port, lane);
+	intel_clear_response_ready_flag(encoder, lane);
 }
 
-static int intel_cx0_wait_for_ack(struct drm_i915_private *i915, enum port port,
+static int intel_cx0_wait_for_ack(struct intel_encoder *encoder,
 				  int command, int lane, u32 *val)
 {
-	enum phy phy = intel_port_to_phy(i915, port);
-
-	if (__intel_de_wait_for_register(i915,
-					 XELPDP_PORT_P2M_MSGBUS_STATUS(i915, port, lane),
-					 XELPDP_PORT_P2M_RESPONSE_READY,
-					 XELPDP_PORT_P2M_RESPONSE_READY,
-					 XELPDP_MSGBUS_TIMEOUT_FAST_US,
-					 XELPDP_MSGBUS_TIMEOUT_SLOW, val)) {
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
+	enum phy phy = intel_encoder_to_phy(encoder);
+
+	if (intel_de_wait_custom(i915,
+				 XELPDP_PORT_P2M_MSGBUS_STATUS(i915, port, lane),
+				 XELPDP_PORT_P2M_RESPONSE_READY,
+				 XELPDP_PORT_P2M_RESPONSE_READY,
+				 XELPDP_MSGBUS_TIMEOUT_FAST_US,
+				 XELPDP_MSGBUS_TIMEOUT_SLOW, val)) {
 		drm_dbg_kms(&i915->drm, "PHY %c Timeout waiting for message ACK. Status: 0x%x\n",
 			    phy_name(phy), *val);
 
@@ -158,31 +166,33 @@ static int intel_cx0_wait_for_ack(struct drm_i915_private *i915, enum port port,
 				    "PHY %c Hardware did not detect a timeout\n",
 				    phy_name(phy));
 
-		intel_cx0_bus_reset(i915, port, lane);
+		intel_cx0_bus_reset(encoder, lane);
 		return -ETIMEDOUT;
 	}
 
 	if (*val & XELPDP_PORT_P2M_ERROR_SET) {
 		drm_dbg_kms(&i915->drm, "PHY %c Error occurred during %s command. Status: 0x%x\n", phy_name(phy),
 			    command == XELPDP_PORT_P2M_COMMAND_READ_ACK ? "read" : "write", *val);
-		intel_cx0_bus_reset(i915, port, lane);
+		intel_cx0_bus_reset(encoder, lane);
 		return -EINVAL;
 	}
 
 	if (REG_FIELD_GET(XELPDP_PORT_P2M_COMMAND_TYPE_MASK, *val) != command) {
 		drm_dbg_kms(&i915->drm, "PHY %c Not a %s response. MSGBUS Status: 0x%x.\n", phy_name(phy),
 			    command == XELPDP_PORT_P2M_COMMAND_READ_ACK ? "read" : "write", *val);
-		intel_cx0_bus_reset(i915, port, lane);
+		intel_cx0_bus_reset(encoder, lane);
 		return -EINVAL;
 	}
 
 	return 0;
 }
 
-static int __intel_cx0_read_once(struct drm_i915_private *i915, enum port port,
+static int __intel_cx0_read_once(struct intel_encoder *encoder,
 				 int lane, u16 addr)
 {
-	enum phy phy = intel_port_to_phy(i915, port);
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
+	enum phy phy = intel_encoder_to_phy(encoder);
 	int ack;
 	u32 val;
 
@@ -191,7 +201,7 @@ static int __intel_cx0_read_once(struct drm_i915_private *i915, enum port port,
 				    XELPDP_MSGBUS_TIMEOUT_SLOW)) {
 		drm_dbg_kms(&i915->drm,
 			    "PHY %c Timeout waiting for previous transaction to complete. Reset the bus and retry.\n", phy_name(phy));
-		intel_cx0_bus_reset(i915, port, lane);
+		intel_cx0_bus_reset(encoder, lane);
 		return -ETIMEDOUT;
 	}
 
@@ -200,33 +210,34 @@ static int __intel_cx0_read_once(struct drm_i915_private *i915, enum port port,
 		       XELPDP_PORT_M2P_COMMAND_READ |
 		       XELPDP_PORT_M2P_ADDRESS(addr));
 
-	ack = intel_cx0_wait_for_ack(i915, port, XELPDP_PORT_P2M_COMMAND_READ_ACK, lane, &val);
+	ack = intel_cx0_wait_for_ack(encoder, XELPDP_PORT_P2M_COMMAND_READ_ACK, lane, &val);
 	if (ack < 0)
 		return ack;
 
-	intel_clear_response_ready_flag(i915, port, lane);
+	intel_clear_response_ready_flag(encoder, lane);
 
 	/*
 	 * FIXME: Workaround to let HW to settle
 	 * down and let the message bus to end up
 	 * in a known state
 	 */
-	intel_cx0_bus_reset(i915, port, lane);
+	intel_cx0_bus_reset(encoder, lane);
 
 	return REG_FIELD_GET(XELPDP_PORT_P2M_DATA_MASK, val);
 }
 
-static u8 __intel_cx0_read(struct drm_i915_private *i915, enum port port,
+static u8 __intel_cx0_read(struct intel_encoder *encoder,
 			   int lane, u16 addr)
 {
-	enum phy phy = intel_port_to_phy(i915, port);
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	int i, status;
 
 	assert_dc_off(i915);
 
 	/* 3 tries is assumed to be enough to read successfully */
 	for (i = 0; i < 3; i++) {
-		status = __intel_cx0_read_once(i915, port, lane, addr);
+		status = __intel_cx0_read_once(encoder, lane, addr);
 
 		if (status >= 0)
 			return status;
@@ -238,18 +249,20 @@ static u8 __intel_cx0_read(struct drm_i915_private *i915, enum port port,
 	return 0;
 }
 
-static u8 intel_cx0_read(struct drm_i915_private *i915, enum port port,
+static u8 intel_cx0_read(struct intel_encoder *encoder,
 			 u8 lane_mask, u16 addr)
 {
 	int lane = lane_mask_to_lane(lane_mask);
 
-	return __intel_cx0_read(i915, port, lane, addr);
+	return __intel_cx0_read(encoder, lane, addr);
 }
 
-static int __intel_cx0_write_once(struct drm_i915_private *i915, enum port port,
+static int __intel_cx0_write_once(struct intel_encoder *encoder,
 				  int lane, u16 addr, u8 data, bool committed)
 {
-	enum phy phy = intel_port_to_phy(i915, port);
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
+	enum phy phy = intel_encoder_to_phy(encoder);
 	int ack;
 	u32 val;
 
@@ -258,7 +271,7 @@ static int __intel_cx0_write_once(struct drm_i915_private *i915, enum port port,
 				    XELPDP_MSGBUS_TIMEOUT_SLOW)) {
 		drm_dbg_kms(&i915->drm,
 			    "PHY %c Timeout waiting for previous transaction to complete. Resetting the bus.\n", phy_name(phy));
-		intel_cx0_bus_reset(i915, port, lane);
+		intel_cx0_bus_reset(encoder, lane);
 		return -ETIMEDOUT;
 	}
 
@@ -274,45 +287,46 @@ static int __intel_cx0_write_once(struct drm_i915_private *i915, enum port port,
 				    XELPDP_MSGBUS_TIMEOUT_SLOW)) {
 		drm_dbg_kms(&i915->drm,
 			    "PHY %c Timeout waiting for write to complete. Resetting the bus.\n", phy_name(phy));
-		intel_cx0_bus_reset(i915, port, lane);
+		intel_cx0_bus_reset(encoder, lane);
 		return -ETIMEDOUT;
 	}
 
 	if (committed) {
-		ack = intel_cx0_wait_for_ack(i915, port, XELPDP_PORT_P2M_COMMAND_WRITE_ACK, lane, &val);
+		ack = intel_cx0_wait_for_ack(encoder, XELPDP_PORT_P2M_COMMAND_WRITE_ACK, lane, &val);
 		if (ack < 0)
 			return ack;
 	} else if ((intel_de_read(i915, XELPDP_PORT_P2M_MSGBUS_STATUS(i915, port, lane)) &
 		    XELPDP_PORT_P2M_ERROR_SET)) {
 		drm_dbg_kms(&i915->drm,
 			    "PHY %c Error occurred during write command.\n", phy_name(phy));
-		intel_cx0_bus_reset(i915, port, lane);
+		intel_cx0_bus_reset(encoder, lane);
 		return -EINVAL;
 	}
 
-	intel_clear_response_ready_flag(i915, port, lane);
+	intel_clear_response_ready_flag(encoder, lane);
 
 	/*
 	 * FIXME: Workaround to let HW to settle
 	 * down and let the message bus to end up
 	 * in a known state
 	 */
-	intel_cx0_bus_reset(i915, port, lane);
+	intel_cx0_bus_reset(encoder, lane);
 
 	return 0;
 }
 
-static void __intel_cx0_write(struct drm_i915_private *i915, enum port port,
+static void __intel_cx0_write(struct intel_encoder *encoder,
 			      int lane, u16 addr, u8 data, bool committed)
 {
-	enum phy phy = intel_port_to_phy(i915, port);
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	int i, status;
 
 	assert_dc_off(i915);
 
 	/* 3 tries is assumed to be enough to write successfully */
 	for (i = 0; i < 3; i++) {
-		status = __intel_cx0_write_once(i915, port, lane, addr, data, committed);
+		status = __intel_cx0_write_once(encoder, lane, addr, data, committed);
 
 		if (status == 0)
 			return;
@@ -322,63 +336,66 @@ static void __intel_cx0_write(struct drm_i915_private *i915, enum port port,
 		     "PHY %c Write %04x failed after %d retries.\n", phy_name(phy), addr, i);
 }
 
-static void intel_cx0_write(struct drm_i915_private *i915, enum port port,
+static void intel_cx0_write(struct intel_encoder *encoder,
 			    u8 lane_mask, u16 addr, u8 data, bool committed)
 {
 	int lane;
 
 	for_each_cx0_lane_in_mask(lane_mask, lane)
-		__intel_cx0_write(i915, port, lane, addr, data, committed);
+		__intel_cx0_write(encoder, lane, addr, data, committed);
 }
 
-static void intel_c20_sram_write(struct drm_i915_private *i915, enum port port,
+static void intel_c20_sram_write(struct intel_encoder *encoder,
 				 int lane, u16 addr, u16 data)
 {
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+
 	assert_dc_off(i915);
 
-	intel_cx0_write(i915, port, lane, PHY_C20_WR_ADDRESS_H, addr >> 8, 0);
-	intel_cx0_write(i915, port, lane, PHY_C20_WR_ADDRESS_L, addr & 0xff, 0);
+	intel_cx0_write(encoder, lane, PHY_C20_WR_ADDRESS_H, addr >> 8, 0);
+	intel_cx0_write(encoder, lane, PHY_C20_WR_ADDRESS_L, addr & 0xff, 0);
 
-	intel_cx0_write(i915, port, lane, PHY_C20_WR_DATA_H, data >> 8, 0);
-	intel_cx0_write(i915, port, lane, PHY_C20_WR_DATA_L, data & 0xff, 1);
+	intel_cx0_write(encoder, lane, PHY_C20_WR_DATA_H, data >> 8, 0);
+	intel_cx0_write(encoder, lane, PHY_C20_WR_DATA_L, data & 0xff, 1);
 }
 
-static u16 intel_c20_sram_read(struct drm_i915_private *i915, enum port port,
+static u16 intel_c20_sram_read(struct intel_encoder *encoder,
 			       int lane, u16 addr)
 {
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	u16 val;
 
 	assert_dc_off(i915);
 
-	intel_cx0_write(i915, port, lane, PHY_C20_RD_ADDRESS_H, addr >> 8, 0);
-	intel_cx0_write(i915, port, lane, PHY_C20_RD_ADDRESS_L, addr & 0xff, 1);
+	intel_cx0_write(encoder, lane, PHY_C20_RD_ADDRESS_H, addr >> 8, 0);
+	intel_cx0_write(encoder, lane, PHY_C20_RD_ADDRESS_L, addr & 0xff, 1);
 
-	val = intel_cx0_read(i915, port, lane, PHY_C20_RD_DATA_H);
+	val = intel_cx0_read(encoder, lane, PHY_C20_RD_DATA_H);
 	val <<= 8;
-	val |= intel_cx0_read(i915, port, lane, PHY_C20_RD_DATA_L);
+	val |= intel_cx0_read(encoder, lane, PHY_C20_RD_DATA_L);
 
 	return val;
 }
 
-static void __intel_cx0_rmw(struct drm_i915_private *i915, enum port port,
+static void __intel_cx0_rmw(struct intel_encoder *encoder,
 			    int lane, u16 addr, u8 clear, u8 set, bool committed)
 {
 	u8 old, val;
 
-	old = __intel_cx0_read(i915, port, lane, addr);
+	old = __intel_cx0_read(encoder, lane, addr);
 	val = (old & ~clear) | set;
 
 	if (val != old)
-		__intel_cx0_write(i915, port, lane, addr, val, committed);
+		__intel_cx0_write(encoder, lane, addr, val, committed);
 }
 
-static void intel_cx0_rmw(struct drm_i915_private *i915, enum port port,
+static void intel_cx0_rmw(struct intel_encoder *encoder,
 			  u8 lane_mask, u16 addr, u8 clear, u8 set, bool committed)
 {
 	u8 lane;
 
 	for_each_cx0_lane_in_mask(lane_mask, lane)
-		__intel_cx0_rmw(i915, port, lane, addr, clear, set, committed);
+		__intel_cx0_rmw(encoder, lane, addr, clear, set, committed);
 }
 
 static u8 intel_c10_get_tx_vboost_lvl(const struct intel_crtc_state *crtc_state)
@@ -414,7 +431,6 @@ void intel_cx0_phy_set_signal_levels(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	const struct intel_ddi_buf_trans *trans;
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
 	u8 owned_lane_mask;
 	intel_wakeref_t wakeref;
 	int n_entries, ln;
@@ -423,7 +439,7 @@ void intel_cx0_phy_set_signal_levels(struct intel_encoder *encoder,
 	if (intel_tc_port_in_tbt_alt_mode(dig_port))
 		return;
 
-	owned_lane_mask = intel_cx0_get_owned_lane_mask(i915, encoder);
+	owned_lane_mask = intel_cx0_get_owned_lane_mask(encoder);
 
 	wakeref = intel_cx0_phy_transaction_begin(encoder);
 
@@ -433,14 +449,14 @@ void intel_cx0_phy_set_signal_levels(struct intel_encoder *encoder,
 		return;
 	}
 
-	if (intel_is_c10phy(i915, phy)) {
-		intel_cx0_rmw(i915, encoder->port, owned_lane_mask, PHY_C10_VDR_CONTROL(1),
+	if (intel_encoder_is_c10phy(encoder)) {
+		intel_cx0_rmw(encoder, owned_lane_mask, PHY_C10_VDR_CONTROL(1),
 			      0, C10_VDR_CTRL_MSGBUS_ACCESS, MB_WRITE_COMMITTED);
-		intel_cx0_rmw(i915, encoder->port, owned_lane_mask, PHY_C10_VDR_CMN(3),
+		intel_cx0_rmw(encoder, owned_lane_mask, PHY_C10_VDR_CMN(3),
 			      C10_CMN3_TXVBOOST_MASK,
 			      C10_CMN3_TXVBOOST(intel_c10_get_tx_vboost_lvl(crtc_state)),
 			      MB_WRITE_UNCOMMITTED);
-		intel_cx0_rmw(i915, encoder->port, owned_lane_mask, PHY_C10_VDR_TX(1),
+		intel_cx0_rmw(encoder, owned_lane_mask, PHY_C10_VDR_TX(1),
 			      C10_TX1_TERMCTL_MASK,
 			      C10_TX1_TERMCTL(intel_c10_get_tx_term_ctl(crtc_state)),
 			      MB_WRITE_COMMITTED);
@@ -455,27 +471,27 @@ void intel_cx0_phy_set_signal_levels(struct intel_encoder *encoder,
 		if (!(lane_mask & owned_lane_mask))
 			continue;
 
-		intel_cx0_rmw(i915, encoder->port, lane_mask, PHY_CX0_VDROVRD_CTL(lane, tx, 0),
+		intel_cx0_rmw(encoder, lane_mask, PHY_CX0_VDROVRD_CTL(lane, tx, 0),
 			      C10_PHY_OVRD_LEVEL_MASK,
 			      C10_PHY_OVRD_LEVEL(trans->entries[level].snps.pre_cursor),
 			      MB_WRITE_COMMITTED);
-		intel_cx0_rmw(i915, encoder->port, lane_mask, PHY_CX0_VDROVRD_CTL(lane, tx, 1),
+		intel_cx0_rmw(encoder, lane_mask, PHY_CX0_VDROVRD_CTL(lane, tx, 1),
 			      C10_PHY_OVRD_LEVEL_MASK,
 			      C10_PHY_OVRD_LEVEL(trans->entries[level].snps.vswing),
 			      MB_WRITE_COMMITTED);
-		intel_cx0_rmw(i915, encoder->port, lane_mask, PHY_CX0_VDROVRD_CTL(lane, tx, 2),
+		intel_cx0_rmw(encoder, lane_mask, PHY_CX0_VDROVRD_CTL(lane, tx, 2),
 			      C10_PHY_OVRD_LEVEL_MASK,
 			      C10_PHY_OVRD_LEVEL(trans->entries[level].snps.post_cursor),
 			      MB_WRITE_COMMITTED);
 	}
 
 	/* Write Override enables in 0xD71 */
-	intel_cx0_rmw(i915, encoder->port, owned_lane_mask, PHY_C10_VDR_OVRD,
+	intel_cx0_rmw(encoder, owned_lane_mask, PHY_C10_VDR_OVRD,
 		      0, PHY_C10_VDR_OVRD_TX1 | PHY_C10_VDR_OVRD_TX2,
 		      MB_WRITE_COMMITTED);
 
-	if (intel_is_c10phy(i915, phy))
-		intel_cx0_rmw(i915, encoder->port, owned_lane_mask, PHY_C10_VDR_CONTROL(1),
+	if (intel_encoder_is_c10phy(encoder))
+		intel_cx0_rmw(encoder, owned_lane_mask, PHY_C10_VDR_CONTROL(1),
 			      0, C10_VDR_CTRL_UPDATE_CFG, MB_WRITE_COMMITTED);
 
 	intel_cx0_phy_transaction_end(encoder, wakeref);
@@ -1811,7 +1827,7 @@ static void intel_c10pll_update_pll(struct intel_crtc_state *crtc_state,
 				    struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	struct intel_cx0pll_state *pll_state = &crtc_state->cx0pll_state;
+	struct intel_cx0pll_state *pll_state = &crtc_state->dpll_hw_state.cx0pll;
 	int i;
 
 	if (intel_crtc_has_dp_encoder(crtc_state)) {
@@ -1843,7 +1859,7 @@ static int intel_c10pll_calc_state(struct intel_crtc_state *crtc_state,
 
 	for (i = 0; tables[i]; i++) {
 		if (crtc_state->port_clock == tables[i]->clock) {
-			crtc_state->cx0pll_state.c10 = *tables[i];
+			crtc_state->dpll_hw_state.cx0pll.c10 = *tables[i];
 			intel_c10pll_update_pll(crtc_state, encoder);
 
 			return 0;
@@ -1856,7 +1872,6 @@ static int intel_c10pll_calc_state(struct intel_crtc_state *crtc_state,
 static void intel_c10pll_readout_hw_state(struct intel_encoder *encoder,
 					  struct intel_c10pll_state *pll_state)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	u8 lane = INTEL_CX0_LANE0;
 	intel_wakeref_t wakeref;
 	int i;
@@ -1867,16 +1882,15 @@ static void intel_c10pll_readout_hw_state(struct intel_encoder *encoder,
 	 * According to C10 VDR Register programming Sequence we need
 	 * to do this to read PHY internal registers from MsgBus.
 	 */
-	intel_cx0_rmw(i915, encoder->port, lane, PHY_C10_VDR_CONTROL(1),
+	intel_cx0_rmw(encoder, lane, PHY_C10_VDR_CONTROL(1),
 		      0, C10_VDR_CTRL_MSGBUS_ACCESS,
 		      MB_WRITE_COMMITTED);
 
 	for (i = 0; i < ARRAY_SIZE(pll_state->pll); i++)
-		pll_state->pll[i] = intel_cx0_read(i915, encoder->port, lane,
-						   PHY_C10_VDR_PLL(i));
+		pll_state->pll[i] = intel_cx0_read(encoder, lane, PHY_C10_VDR_PLL(i));
 
-	pll_state->cmn = intel_cx0_read(i915, encoder->port, lane, PHY_C10_VDR_CMN(0));
-	pll_state->tx = intel_cx0_read(i915, encoder->port, lane, PHY_C10_VDR_TX(0));
+	pll_state->cmn = intel_cx0_read(encoder, lane, PHY_C10_VDR_CMN(0));
+	pll_state->tx = intel_cx0_read(encoder, lane, PHY_C10_VDR_TX(0));
 
 	intel_cx0_phy_transaction_end(encoder, wakeref);
 }
@@ -1885,31 +1899,31 @@ static void intel_c10_pll_program(struct drm_i915_private *i915,
 				  const struct intel_crtc_state *crtc_state,
 				  struct intel_encoder *encoder)
 {
-	const struct intel_c10pll_state *pll_state = &crtc_state->cx0pll_state.c10;
+	const struct intel_c10pll_state *pll_state = &crtc_state->dpll_hw_state.cx0pll.c10;
 	int i;
 
-	intel_cx0_rmw(i915, encoder->port, INTEL_CX0_BOTH_LANES, PHY_C10_VDR_CONTROL(1),
+	intel_cx0_rmw(encoder, INTEL_CX0_BOTH_LANES, PHY_C10_VDR_CONTROL(1),
 		      0, C10_VDR_CTRL_MSGBUS_ACCESS,
 		      MB_WRITE_COMMITTED);
 
 	/* Custom width needs to be programmed to 0 for both the phy lanes */
-	intel_cx0_rmw(i915, encoder->port, INTEL_CX0_BOTH_LANES, PHY_C10_VDR_CUSTOM_WIDTH,
+	intel_cx0_rmw(encoder, INTEL_CX0_BOTH_LANES, PHY_C10_VDR_CUSTOM_WIDTH,
 		      C10_VDR_CUSTOM_WIDTH_MASK, C10_VDR_CUSTOM_WIDTH_8_10,
 		      MB_WRITE_COMMITTED);
-	intel_cx0_rmw(i915, encoder->port, INTEL_CX0_BOTH_LANES, PHY_C10_VDR_CONTROL(1),
+	intel_cx0_rmw(encoder, INTEL_CX0_BOTH_LANES, PHY_C10_VDR_CONTROL(1),
 		      0, C10_VDR_CTRL_UPDATE_CFG,
 		      MB_WRITE_COMMITTED);
 
 	/* Program the pll values only for the master lane */
 	for (i = 0; i < ARRAY_SIZE(pll_state->pll); i++)
-		intel_cx0_write(i915, encoder->port, INTEL_CX0_LANE0, PHY_C10_VDR_PLL(i),
+		intel_cx0_write(encoder, INTEL_CX0_LANE0, PHY_C10_VDR_PLL(i),
 				pll_state->pll[i],
 				(i % 4) ? MB_WRITE_UNCOMMITTED : MB_WRITE_COMMITTED);
 
-	intel_cx0_write(i915, encoder->port, INTEL_CX0_LANE0, PHY_C10_VDR_CMN(0), pll_state->cmn, MB_WRITE_COMMITTED);
-	intel_cx0_write(i915, encoder->port, INTEL_CX0_LANE0, PHY_C10_VDR_TX(0), pll_state->tx, MB_WRITE_COMMITTED);
+	intel_cx0_write(encoder, INTEL_CX0_LANE0, PHY_C10_VDR_CMN(0), pll_state->cmn, MB_WRITE_COMMITTED);
+	intel_cx0_write(encoder, INTEL_CX0_LANE0, PHY_C10_VDR_TX(0), pll_state->tx, MB_WRITE_COMMITTED);
 
-	intel_cx0_rmw(i915, encoder->port, INTEL_CX0_LANE0, PHY_C10_VDR_CONTROL(1),
+	intel_cx0_rmw(encoder, INTEL_CX0_LANE0, PHY_C10_VDR_CONTROL(1),
 		      0, C10_VDR_CTRL_MASTER_LANE | C10_VDR_CTRL_UPDATE_CFG,
 		      MB_WRITE_COMMITTED);
 }
@@ -2037,10 +2051,8 @@ static int intel_c20_phy_check_hdmi_link_rate(int clock)
 int intel_cx0_phy_check_hdmi_link_rate(struct intel_hdmi *hdmi, int clock)
 {
 	struct intel_digital_port *dig_port = hdmi_to_dig_port(hdmi);
-	struct drm_i915_private *i915 = intel_hdmi_to_i915(hdmi);
-	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
 
-	if (intel_is_c10phy(i915, phy))
+	if (intel_encoder_is_c10phy(&dig_port->base))
 		return intel_c10_phy_check_hdmi_link_rate(clock);
 	return intel_c20_phy_check_hdmi_link_rate(clock);
 }
@@ -2067,7 +2079,7 @@ static int intel_c20pll_calc_state(struct intel_crtc_state *crtc_state,
 	/* try computed C20 HDMI tables before using consolidated tables */
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
 		if (intel_c20_compute_hdmi_tmds_pll(crtc_state->port_clock,
-						    &crtc_state->cx0pll_state.c20) == 0)
+						    &crtc_state->dpll_hw_state.cx0pll.c20) == 0)
 			return 0;
 	}
 
@@ -2077,7 +2089,7 @@ static int intel_c20pll_calc_state(struct intel_crtc_state *crtc_state,
 
 	for (i = 0; tables[i]; i++) {
 		if (crtc_state->port_clock == tables[i]->clock) {
-			crtc_state->cx0pll_state.c20 = *tables[i];
+			crtc_state->dpll_hw_state.cx0pll.c20 = *tables[i];
 			return 0;
 		}
 	}
@@ -2088,10 +2100,7 @@ static int intel_c20pll_calc_state(struct intel_crtc_state *crtc_state,
 int intel_cx0pll_calc_state(struct intel_crtc_state *crtc_state,
 			    struct intel_encoder *encoder)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
-
-	if (intel_is_c10phy(i915, phy))
+	if (intel_encoder_is_c10phy(encoder))
 		return intel_c10pll_calc_state(crtc_state, encoder);
 	return intel_c20pll_calc_state(crtc_state, encoder);
 }
@@ -2149,7 +2158,6 @@ static int intel_c20pll_calc_port_clock(struct intel_encoder *encoder,
 static void intel_c20pll_readout_hw_state(struct intel_encoder *encoder,
 					  struct intel_c20pll_state *pll_state)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	bool cntx;
 	intel_wakeref_t wakeref;
 	int i;
@@ -2157,25 +2165,25 @@ static void intel_c20pll_readout_hw_state(struct intel_encoder *encoder,
 	wakeref = intel_cx0_phy_transaction_begin(encoder);
 
 	/* 1. Read current context selection */
-	cntx = intel_cx0_read(i915, encoder->port, INTEL_CX0_LANE0, PHY_C20_VDR_CUSTOM_SERDES_RATE) & PHY_C20_CONTEXT_TOGGLE;
+	cntx = intel_cx0_read(encoder, INTEL_CX0_LANE0, PHY_C20_VDR_CUSTOM_SERDES_RATE) & PHY_C20_CONTEXT_TOGGLE;
 
 	/* Read Tx configuration */
 	for (i = 0; i < ARRAY_SIZE(pll_state->tx); i++) {
 		if (cntx)
-			pll_state->tx[i] = intel_c20_sram_read(i915, encoder->port, INTEL_CX0_LANE0,
+			pll_state->tx[i] = intel_c20_sram_read(encoder, INTEL_CX0_LANE0,
 							       PHY_C20_B_TX_CNTX_CFG(i));
 		else
-			pll_state->tx[i] = intel_c20_sram_read(i915, encoder->port, INTEL_CX0_LANE0,
+			pll_state->tx[i] = intel_c20_sram_read(encoder, INTEL_CX0_LANE0,
 							       PHY_C20_A_TX_CNTX_CFG(i));
 	}
 
 	/* Read common configuration */
 	for (i = 0; i < ARRAY_SIZE(pll_state->cmn); i++) {
 		if (cntx)
-			pll_state->cmn[i] = intel_c20_sram_read(i915, encoder->port, INTEL_CX0_LANE0,
+			pll_state->cmn[i] = intel_c20_sram_read(encoder, INTEL_CX0_LANE0,
 								PHY_C20_B_CMN_CNTX_CFG(i));
 		else
-			pll_state->cmn[i] = intel_c20_sram_read(i915, encoder->port, INTEL_CX0_LANE0,
+			pll_state->cmn[i] = intel_c20_sram_read(encoder, INTEL_CX0_LANE0,
 								PHY_C20_A_CMN_CNTX_CFG(i));
 	}
 
@@ -2183,20 +2191,20 @@ static void intel_c20pll_readout_hw_state(struct intel_encoder *encoder,
 		/* MPLLB configuration */
 		for (i = 0; i < ARRAY_SIZE(pll_state->mpllb); i++) {
 			if (cntx)
-				pll_state->mpllb[i] = intel_c20_sram_read(i915, encoder->port, INTEL_CX0_LANE0,
+				pll_state->mpllb[i] = intel_c20_sram_read(encoder, INTEL_CX0_LANE0,
 									  PHY_C20_B_MPLLB_CNTX_CFG(i));
 			else
-				pll_state->mpllb[i] = intel_c20_sram_read(i915, encoder->port, INTEL_CX0_LANE0,
+				pll_state->mpllb[i] = intel_c20_sram_read(encoder, INTEL_CX0_LANE0,
 									  PHY_C20_A_MPLLB_CNTX_CFG(i));
 		}
 	} else {
 		/* MPLLA configuration */
 		for (i = 0; i < ARRAY_SIZE(pll_state->mplla); i++) {
 			if (cntx)
-				pll_state->mplla[i] = intel_c20_sram_read(i915, encoder->port, INTEL_CX0_LANE0,
+				pll_state->mplla[i] = intel_c20_sram_read(encoder, INTEL_CX0_LANE0,
 									  PHY_C20_B_MPLLA_CNTX_CFG(i));
 			else
-				pll_state->mplla[i] = intel_c20_sram_read(i915, encoder->port, INTEL_CX0_LANE0,
+				pll_state->mplla[i] = intel_c20_sram_read(encoder, INTEL_CX0_LANE0,
 									  PHY_C20_A_MPLLA_CNTX_CFG(i));
 		}
 	}
@@ -2327,7 +2335,7 @@ static void intel_c20_pll_program(struct drm_i915_private *i915,
 				  const struct intel_crtc_state *crtc_state,
 				  struct intel_encoder *encoder)
 {
-	const struct intel_c20pll_state *pll_state = &crtc_state->cx0pll_state.c20;
+	const struct intel_c20pll_state *pll_state = &crtc_state->dpll_hw_state.cx0pll.c20;
 	bool dp = false;
 	int lane = crtc_state->lane_count > 2 ? INTEL_CX0_BOTH_LANES : INTEL_CX0_LANE0;
 	u32 clock = crtc_state->port_clock;
@@ -2338,7 +2346,7 @@ static void intel_c20_pll_program(struct drm_i915_private *i915,
 		dp = true;
 
 	/* 1. Read current context selection */
-	cntx = intel_cx0_read(i915, encoder->port, INTEL_CX0_LANE0, PHY_C20_VDR_CUSTOM_SERDES_RATE) & BIT(0);
+	cntx = intel_cx0_read(encoder, INTEL_CX0_LANE0, PHY_C20_VDR_CUSTOM_SERDES_RATE) & BIT(0);
 
 	/*
 	 * 2. If there is a protocol switch from HDMI to DP or vice versa, clear
@@ -2347,7 +2355,7 @@ static void intel_c20_pll_program(struct drm_i915_private *i915,
 	 */
 	if (intel_c20_protocol_switch_valid(encoder)) {
 		for (i = 0; i < 4; i++)
-			intel_c20_sram_write(i915, encoder->port, INTEL_CX0_LANE0, RAWLANEAONX_DIG_TX_MPLLB_CAL_DONE_BANK(i), 0);
+			intel_c20_sram_write(encoder, INTEL_CX0_LANE0, RAWLANEAONX_DIG_TX_MPLLB_CAL_DONE_BANK(i), 0);
 		usleep_range(4000, 4100);
 	}
 
@@ -2355,63 +2363,63 @@ static void intel_c20_pll_program(struct drm_i915_private *i915,
 	/* 3.1 Tx configuration */
 	for (i = 0; i < ARRAY_SIZE(pll_state->tx); i++) {
 		if (cntx)
-			intel_c20_sram_write(i915, encoder->port, INTEL_CX0_LANE0, PHY_C20_A_TX_CNTX_CFG(i), pll_state->tx[i]);
+			intel_c20_sram_write(encoder, INTEL_CX0_LANE0, PHY_C20_A_TX_CNTX_CFG(i), pll_state->tx[i]);
 		else
-			intel_c20_sram_write(i915, encoder->port, INTEL_CX0_LANE0, PHY_C20_B_TX_CNTX_CFG(i), pll_state->tx[i]);
+			intel_c20_sram_write(encoder, INTEL_CX0_LANE0, PHY_C20_B_TX_CNTX_CFG(i), pll_state->tx[i]);
 	}
 
 	/* 3.2 common configuration */
 	for (i = 0; i < ARRAY_SIZE(pll_state->cmn); i++) {
 		if (cntx)
-			intel_c20_sram_write(i915, encoder->port, INTEL_CX0_LANE0, PHY_C20_A_CMN_CNTX_CFG(i), pll_state->cmn[i]);
+			intel_c20_sram_write(encoder, INTEL_CX0_LANE0, PHY_C20_A_CMN_CNTX_CFG(i), pll_state->cmn[i]);
 		else
-			intel_c20_sram_write(i915, encoder->port, INTEL_CX0_LANE0, PHY_C20_B_CMN_CNTX_CFG(i), pll_state->cmn[i]);
+			intel_c20_sram_write(encoder, INTEL_CX0_LANE0, PHY_C20_B_CMN_CNTX_CFG(i), pll_state->cmn[i]);
 	}
 
 	/* 3.3 mpllb or mplla configuration */
 	if (intel_c20phy_use_mpllb(pll_state)) {
 		for (i = 0; i < ARRAY_SIZE(pll_state->mpllb); i++) {
 			if (cntx)
-				intel_c20_sram_write(i915, encoder->port, INTEL_CX0_LANE0,
+				intel_c20_sram_write(encoder, INTEL_CX0_LANE0,
 						     PHY_C20_A_MPLLB_CNTX_CFG(i),
 						     pll_state->mpllb[i]);
 			else
-				intel_c20_sram_write(i915, encoder->port, INTEL_CX0_LANE0,
+				intel_c20_sram_write(encoder, INTEL_CX0_LANE0,
 						     PHY_C20_B_MPLLB_CNTX_CFG(i),
 						     pll_state->mpllb[i]);
 		}
 	} else {
 		for (i = 0; i < ARRAY_SIZE(pll_state->mplla); i++) {
 			if (cntx)
-				intel_c20_sram_write(i915, encoder->port, INTEL_CX0_LANE0,
+				intel_c20_sram_write(encoder, INTEL_CX0_LANE0,
 						     PHY_C20_A_MPLLA_CNTX_CFG(i),
 						     pll_state->mplla[i]);
 			else
-				intel_c20_sram_write(i915, encoder->port, INTEL_CX0_LANE0,
+				intel_c20_sram_write(encoder, INTEL_CX0_LANE0,
 						     PHY_C20_B_MPLLA_CNTX_CFG(i),
 						     pll_state->mplla[i]);
 		}
 	}
 
 	/* 4. Program custom width to match the link protocol */
-	intel_cx0_rmw(i915, encoder->port, lane, PHY_C20_VDR_CUSTOM_WIDTH,
+	intel_cx0_rmw(encoder, lane, PHY_C20_VDR_CUSTOM_WIDTH,
 		      PHY_C20_CUSTOM_WIDTH_MASK,
 		      PHY_C20_CUSTOM_WIDTH(intel_get_c20_custom_width(clock, dp)),
 		      MB_WRITE_COMMITTED);
 
 	/* 5. For DP or 6. For HDMI */
 	if (dp) {
-		intel_cx0_rmw(i915, encoder->port, lane, PHY_C20_VDR_CUSTOM_SERDES_RATE,
+		intel_cx0_rmw(encoder, lane, PHY_C20_VDR_CUSTOM_SERDES_RATE,
 			      BIT(6) | PHY_C20_CUSTOM_SERDES_MASK,
 			      BIT(6) | PHY_C20_CUSTOM_SERDES(intel_c20_get_dp_rate(clock)),
 			      MB_WRITE_COMMITTED);
 	} else {
-		intel_cx0_rmw(i915, encoder->port, lane, PHY_C20_VDR_CUSTOM_SERDES_RATE,
+		intel_cx0_rmw(encoder, lane, PHY_C20_VDR_CUSTOM_SERDES_RATE,
 			      BIT(7) | PHY_C20_CUSTOM_SERDES_MASK,
 			      is_hdmi_frl(clock) ? BIT(7) : 0,
 			      MB_WRITE_COMMITTED);
 
-		intel_cx0_write(i915, encoder->port, INTEL_CX0_BOTH_LANES, PHY_C20_VDR_HDMI_RATE,
+		intel_cx0_write(encoder, INTEL_CX0_BOTH_LANES, PHY_C20_VDR_HDMI_RATE,
 				intel_c20_get_hdmi_rate(clock),
 				MB_WRITE_COMMITTED);
 	}
@@ -2420,7 +2428,7 @@ static void intel_c20_pll_program(struct drm_i915_private *i915,
 	 * 7. Write Vendor specific registers to toggle context setting to load
 	 * the updated programming toggle context bit
 	 */
-	intel_cx0_rmw(i915, encoder->port, lane, PHY_C20_VDR_CUSTOM_SERDES_RATE,
+	intel_cx0_rmw(encoder, lane, PHY_C20_VDR_CUSTOM_SERDES_RATE,
 		      BIT(0), cntx ? 0 : 1, MB_WRITE_COMMITTED);
 }
 
@@ -2476,9 +2484,9 @@ static void intel_program_port_clock_ctl(struct intel_encoder *encoder,
 	/* TODO: HDMI FRL */
 	/* DP2.0 10G and 20G rates enable MPLLA*/
 	if (crtc_state->port_clock == 1000000 || crtc_state->port_clock == 2000000)
-		val |= crtc_state->cx0pll_state.ssc_enabled ? XELPDP_SSC_ENABLE_PLLA : 0;
+		val |= crtc_state->dpll_hw_state.cx0pll.ssc_enabled ? XELPDP_SSC_ENABLE_PLLA : 0;
 	else
-		val |= crtc_state->cx0pll_state.ssc_enabled ? XELPDP_SSC_ENABLE_PLLB : 0;
+		val |= crtc_state->dpll_hw_state.cx0pll.ssc_enabled ? XELPDP_SSC_ENABLE_PLLB : 0;
 
 	intel_de_rmw(i915, XELPDP_PORT_CLOCK_CTL(i915, encoder->port),
 		     XELPDP_LANE1_PHY_CLOCK_SELECT | XELPDP_FORWARD_CLOCK_UNGATE |
@@ -2508,11 +2516,12 @@ static u32 intel_cx0_get_powerdown_state(u8 lane_mask, u8 state)
 	return val;
 }
 
-static void intel_cx0_powerdown_change_sequence(struct drm_i915_private *i915,
-						enum port port,
+static void intel_cx0_powerdown_change_sequence(struct intel_encoder *encoder,
 						u8 lane_mask, u8 state)
 {
-	enum phy phy = intel_port_to_phy(i915, port);
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
+	enum phy phy = intel_encoder_to_phy(encoder);
 	i915_reg_t buf_ctl2_reg = XELPDP_PORT_BUF_CTL2(i915, port);
 	int lane;
 
@@ -2528,7 +2537,7 @@ static void intel_cx0_powerdown_change_sequence(struct drm_i915_private *i915,
 			drm_dbg_kms(&i915->drm,
 				    "PHY %c Timeout waiting for previous transaction to complete. Reset the bus.\n",
 				    phy_name(phy));
-			intel_cx0_bus_reset(i915, port, lane);
+			intel_cx0_bus_reset(encoder, lane);
 		}
 
 	intel_de_rmw(i915, buf_ctl2_reg,
@@ -2536,15 +2545,18 @@ static void intel_cx0_powerdown_change_sequence(struct drm_i915_private *i915,
 		     intel_cx0_get_powerdown_update(lane_mask));
 
 	/* Update Timeout Value */
-	if (__intel_de_wait_for_register(i915, buf_ctl2_reg,
-					 intel_cx0_get_powerdown_update(lane_mask), 0,
-					 XELPDP_PORT_POWERDOWN_UPDATE_TIMEOUT_US, 0, NULL))
+	if (intel_de_wait_custom(i915, buf_ctl2_reg,
+				 intel_cx0_get_powerdown_update(lane_mask), 0,
+				 XELPDP_PORT_POWERDOWN_UPDATE_TIMEOUT_US, 0, NULL))
 		drm_warn(&i915->drm, "PHY %c failed to bring out of Lane reset after %dus.\n",
 			 phy_name(phy), XELPDP_PORT_RESET_START_TIMEOUT_US);
 }
 
-static void intel_cx0_setup_powerdown(struct drm_i915_private *i915, enum port port)
+static void intel_cx0_setup_powerdown(struct intel_encoder *encoder)
 {
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
+
 	intel_de_rmw(i915, XELPDP_PORT_BUF_CTL2(i915, port),
 		     XELPDP_POWER_STATE_READY_MASK,
 		     XELPDP_POWER_STATE_READY(CX0_P2_STATE_READY));
@@ -2577,13 +2589,13 @@ static u32 intel_cx0_get_pclk_refclk_ack(u8 lane_mask)
 	return val;
 }
 
-static void intel_cx0_phy_lane_reset(struct drm_i915_private *i915,
-				     struct intel_encoder *encoder,
+static void intel_cx0_phy_lane_reset(struct intel_encoder *encoder,
 				     bool lane_reversal)
 {
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	enum port port = encoder->port;
-	enum phy phy = intel_port_to_phy(i915, port);
-	u8 owned_lane_mask = intel_cx0_get_owned_lane_mask(i915, encoder);
+	enum phy phy = intel_encoder_to_phy(encoder);
+	u8 owned_lane_mask = intel_cx0_get_owned_lane_mask(encoder);
 	u8 lane_mask = lane_reversal ? INTEL_CX0_LANE1 : INTEL_CX0_LANE0;
 	u32 lane_pipe_reset = owned_lane_mask == INTEL_CX0_BOTH_LANES
 				? XELPDP_LANE_PIPE_RESET(0) | XELPDP_LANE_PIPE_RESET(1)
@@ -2593,19 +2605,19 @@ static void intel_cx0_phy_lane_reset(struct drm_i915_private *i915,
 					   XELPDP_LANE_PHY_CURRENT_STATUS(1))
 					: XELPDP_LANE_PHY_CURRENT_STATUS(0);
 
-	if (__intel_de_wait_for_register(i915, XELPDP_PORT_BUF_CTL1(i915, port),
-					 XELPDP_PORT_BUF_SOC_PHY_READY,
-					 XELPDP_PORT_BUF_SOC_PHY_READY,
-					 XELPDP_PORT_BUF_SOC_READY_TIMEOUT_US, 0, NULL))
+	if (intel_de_wait_custom(i915, XELPDP_PORT_BUF_CTL1(i915, port),
+				 XELPDP_PORT_BUF_SOC_PHY_READY,
+				 XELPDP_PORT_BUF_SOC_PHY_READY,
+				 XELPDP_PORT_BUF_SOC_READY_TIMEOUT_US, 0, NULL))
 		drm_warn(&i915->drm, "PHY %c failed to bring out of SOC reset after %dus.\n",
 			 phy_name(phy), XELPDP_PORT_BUF_SOC_READY_TIMEOUT_US);
 
 	intel_de_rmw(i915, XELPDP_PORT_BUF_CTL2(i915, port), lane_pipe_reset,
 		     lane_pipe_reset);
 
-	if (__intel_de_wait_for_register(i915, XELPDP_PORT_BUF_CTL2(i915, port),
-					 lane_phy_current_status, lane_phy_current_status,
-					 XELPDP_PORT_RESET_START_TIMEOUT_US, 0, NULL))
+	if (intel_de_wait_custom(i915, XELPDP_PORT_BUF_CTL2(i915, port),
+				 lane_phy_current_status, lane_phy_current_status,
+				 XELPDP_PORT_RESET_START_TIMEOUT_US, 0, NULL))
 		drm_warn(&i915->drm, "PHY %c failed to bring out of Lane reset after %dus.\n",
 			 phy_name(phy), XELPDP_PORT_RESET_START_TIMEOUT_US);
 
@@ -2613,16 +2625,16 @@ static void intel_cx0_phy_lane_reset(struct drm_i915_private *i915,
 		     intel_cx0_get_pclk_refclk_request(owned_lane_mask),
 		     intel_cx0_get_pclk_refclk_request(lane_mask));
 
-	if (__intel_de_wait_for_register(i915, XELPDP_PORT_CLOCK_CTL(i915, port),
-					 intel_cx0_get_pclk_refclk_ack(owned_lane_mask),
-					 intel_cx0_get_pclk_refclk_ack(lane_mask),
-					 XELPDP_REFCLK_ENABLE_TIMEOUT_US, 0, NULL))
+	if (intel_de_wait_custom(i915, XELPDP_PORT_CLOCK_CTL(i915, port),
+				 intel_cx0_get_pclk_refclk_ack(owned_lane_mask),
+				 intel_cx0_get_pclk_refclk_ack(lane_mask),
+				 XELPDP_REFCLK_ENABLE_TIMEOUT_US, 0, NULL))
 		drm_warn(&i915->drm, "PHY %c failed to request refclk after %dus.\n",
 			 phy_name(phy), XELPDP_REFCLK_ENABLE_TIMEOUT_US);
 
-	intel_cx0_powerdown_change_sequence(i915, port, INTEL_CX0_BOTH_LANES,
+	intel_cx0_powerdown_change_sequence(encoder, INTEL_CX0_BOTH_LANES,
 					    CX0_P2_STATE_RESET);
-	intel_cx0_setup_powerdown(i915, port);
+	intel_cx0_setup_powerdown(encoder);
 
 	intel_de_rmw(i915, XELPDP_PORT_BUF_CTL2(i915, port), lane_pipe_reset, 0);
 
@@ -2640,11 +2652,10 @@ static void intel_cx0_program_phy_lane(struct drm_i915_private *i915,
 	int i;
 	u8 disables;
 	bool dp_alt_mode = intel_tc_port_in_dp_alt_mode(enc_to_dig_port(encoder));
-	u8 owned_lane_mask = intel_cx0_get_owned_lane_mask(i915, encoder);
-	enum port port = encoder->port;
+	u8 owned_lane_mask = intel_cx0_get_owned_lane_mask(encoder);
 
-	if (intel_is_c10phy(i915, intel_port_to_phy(i915, port)))
-		intel_cx0_rmw(i915, port, owned_lane_mask,
+	if (intel_encoder_is_c10phy(encoder))
+		intel_cx0_rmw(encoder, owned_lane_mask,
 			      PHY_C10_VDR_CONTROL(1), 0,
 			      C10_VDR_CTRL_MSGBUS_ACCESS,
 			      MB_WRITE_COMMITTED);
@@ -2666,14 +2677,14 @@ static void intel_cx0_program_phy_lane(struct drm_i915_private *i915,
 		if (!(owned_lane_mask & lane_mask))
 			continue;
 
-		intel_cx0_rmw(i915, port, lane_mask, PHY_CX0_TX_CONTROL(tx, 2),
+		intel_cx0_rmw(encoder, lane_mask, PHY_CX0_TX_CONTROL(tx, 2),
 			      CONTROL2_DISABLE_SINGLE_TX,
 			      disables & BIT(i) ? CONTROL2_DISABLE_SINGLE_TX : 0,
 			      MB_WRITE_COMMITTED);
 	}
 
-	if (intel_is_c10phy(i915, intel_port_to_phy(i915, port)))
-		intel_cx0_rmw(i915, port, owned_lane_mask,
+	if (intel_encoder_is_c10phy(encoder))
+		intel_cx0_rmw(encoder, owned_lane_mask,
 			      PHY_C10_VDR_CONTROL(1), 0,
 			      C10_VDR_CTRL_UPDATE_CFG,
 			      MB_WRITE_COMMITTED);
@@ -2705,7 +2716,7 @@ static void intel_cx0pll_enable(struct intel_encoder *encoder,
 				const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	bool lane_reversal = dig_port->saved_port_bits & DDI_BUF_PORT_REVERSAL;
 	u8 maxpclk_lane = lane_reversal ? INTEL_CX0_LANE1 :
@@ -2719,13 +2730,13 @@ static void intel_cx0pll_enable(struct intel_encoder *encoder,
 	intel_program_port_clock_ctl(encoder, crtc_state, lane_reversal);
 
 	/* 2. Bring PHY out of reset. */
-	intel_cx0_phy_lane_reset(i915, encoder, lane_reversal);
+	intel_cx0_phy_lane_reset(encoder, lane_reversal);
 
 	/*
 	 * 3. Change Phy power state to Ready.
 	 * TODO: For DP alt mode use only one lane.
 	 */
-	intel_cx0_powerdown_change_sequence(i915, encoder->port, INTEL_CX0_BOTH_LANES,
+	intel_cx0_powerdown_change_sequence(encoder, INTEL_CX0_BOTH_LANES,
 					    CX0_P2_STATE_READY);
 
 	/*
@@ -2735,7 +2746,7 @@ static void intel_cx0pll_enable(struct intel_encoder *encoder,
 	 */
 
 	/* 5. Program PHY internal PLL internal registers. */
-	if (intel_is_c10phy(i915, phy))
+	if (intel_encoder_is_c10phy(encoder))
 		intel_c10_pll_program(i915, crtc_state, encoder);
 	else
 		intel_c20_pll_program(i915, crtc_state, encoder);
@@ -2767,10 +2778,10 @@ static void intel_cx0pll_enable(struct intel_encoder *encoder,
 		     intel_cx0_get_pclk_pll_request(maxpclk_lane));
 
 	/* 10. Poll on PORT_CLOCK_CTL PCLK PLL Ack LN<Lane for maxPCLK> == "1". */
-	if (__intel_de_wait_for_register(i915, XELPDP_PORT_CLOCK_CTL(i915, encoder->port),
-					 intel_cx0_get_pclk_pll_ack(INTEL_CX0_BOTH_LANES),
-					 intel_cx0_get_pclk_pll_ack(maxpclk_lane),
-					 XELPDP_PCLK_PLL_ENABLE_TIMEOUT_US, 0, NULL))
+	if (intel_de_wait_custom(i915, XELPDP_PORT_CLOCK_CTL(i915, encoder->port),
+				 intel_cx0_get_pclk_pll_ack(INTEL_CX0_BOTH_LANES),
+				 intel_cx0_get_pclk_pll_ack(maxpclk_lane),
+				 XELPDP_PCLK_PLL_ENABLE_TIMEOUT_US, 0, NULL))
 		drm_warn(&i915->drm, "Port %c PLL not locked after %dus.\n",
 			 phy_name(phy), XELPDP_PCLK_PLL_ENABLE_TIMEOUT_US);
 
@@ -2831,7 +2842,7 @@ static void intel_mtl_tbt_pll_enable(struct intel_encoder *encoder,
 				     const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	u32 val = 0;
 
 	/*
@@ -2858,10 +2869,10 @@ static void intel_mtl_tbt_pll_enable(struct intel_encoder *encoder,
 	intel_de_write(i915, XELPDP_PORT_CLOCK_CTL(i915, encoder->port), val);
 
 	/* 5. Poll on PORT_CLOCK_CTL TBT CLOCK Ack == "1". */
-	if (__intel_de_wait_for_register(i915, XELPDP_PORT_CLOCK_CTL(i915, encoder->port),
-					 XELPDP_TBT_CLOCK_ACK,
-					 XELPDP_TBT_CLOCK_ACK,
-					 100, 0, NULL))
+	if (intel_de_wait_custom(i915, XELPDP_PORT_CLOCK_CTL(i915, encoder->port),
+				 XELPDP_TBT_CLOCK_ACK,
+				 XELPDP_TBT_CLOCK_ACK,
+				 100, 0, NULL))
 		drm_warn(&i915->drm, "[ENCODER:%d:%s][%c] PHY PLL not locked after 100us.\n",
 			 encoder->base.base.id, encoder->base.name, phy_name(phy));
 
@@ -2892,12 +2903,12 @@ void intel_mtl_pll_enable(struct intel_encoder *encoder,
 static void intel_cx0pll_disable(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
-	bool is_c10 = intel_is_c10phy(i915, phy);
+	enum phy phy = intel_encoder_to_phy(encoder);
+	bool is_c10 = intel_encoder_is_c10phy(encoder);
 	intel_wakeref_t wakeref = intel_cx0_phy_transaction_begin(encoder);
 
 	/* 1. Change owned PHY lane power to Disable state. */
-	intel_cx0_powerdown_change_sequence(i915, encoder->port, INTEL_CX0_BOTH_LANES,
+	intel_cx0_powerdown_change_sequence(encoder, INTEL_CX0_BOTH_LANES,
 					    is_c10 ? CX0_P2PG_STATE_DISABLE :
 					    CX0_P4PG_STATE_DISABLE);
 
@@ -2920,10 +2931,10 @@ static void intel_cx0pll_disable(struct intel_encoder *encoder)
 	/*
 	 * 5. Poll on PORT_CLOCK_CTL PCLK PLL Ack LN<Lane for maxPCLK**> == "0".
 	 */
-	if (__intel_de_wait_for_register(i915, XELPDP_PORT_CLOCK_CTL(i915, encoder->port),
-					 intel_cx0_get_pclk_pll_ack(INTEL_CX0_BOTH_LANES) |
-					 intel_cx0_get_pclk_refclk_ack(INTEL_CX0_BOTH_LANES), 0,
-					 XELPDP_PCLK_PLL_DISABLE_TIMEOUT_US, 0, NULL))
+	if (intel_de_wait_custom(i915, XELPDP_PORT_CLOCK_CTL(i915, encoder->port),
+				 intel_cx0_get_pclk_pll_ack(INTEL_CX0_BOTH_LANES) |
+				 intel_cx0_get_pclk_refclk_ack(INTEL_CX0_BOTH_LANES), 0,
+				 XELPDP_PCLK_PLL_DISABLE_TIMEOUT_US, 0, NULL))
 		drm_warn(&i915->drm, "Port %c PLL not unlocked after %dus.\n",
 			 phy_name(phy), XELPDP_PCLK_PLL_DISABLE_TIMEOUT_US);
 
@@ -2944,7 +2955,7 @@ static void intel_cx0pll_disable(struct intel_encoder *encoder)
 static void intel_mtl_tbt_pll_disable(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	/*
 	 * 1. Follow the Display Voltage Frequency Switching Sequence Before
@@ -2958,8 +2969,8 @@ static void intel_mtl_tbt_pll_disable(struct intel_encoder *encoder)
 		     XELPDP_TBT_CLOCK_REQUEST, 0);
 
 	/* 3. Poll on PORT_CLOCK_CTL TBT CLOCK Ack == "0". */
-	if (__intel_de_wait_for_register(i915, XELPDP_PORT_CLOCK_CTL(i915, encoder->port),
-					 XELPDP_TBT_CLOCK_ACK, 0, 10, 0, NULL))
+	if (intel_de_wait_custom(i915, XELPDP_PORT_CLOCK_CTL(i915, encoder->port),
+				 XELPDP_TBT_CLOCK_ACK, 0, 10, 0, NULL))
 		drm_warn(&i915->drm, "[ENCODER:%d:%s][%c] PHY PLL not unlocked after 10us.\n",
 			 encoder->base.base.id, encoder->base.name, phy_name(phy));
 
@@ -3014,7 +3025,7 @@ static void intel_c10pll_state_verify(const struct intel_crtc_state *state,
 				      struct intel_c10pll_state *mpllb_hw_state)
 {
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
-	const struct intel_c10pll_state *mpllb_sw_state = &state->cx0pll_state.c10;
+	const struct intel_c10pll_state *mpllb_sw_state = &state->dpll_hw_state.cx0pll.c10;
 	int i;
 
 	if (intel_crtc_needs_fastset(state))
@@ -3043,10 +3054,7 @@ static void intel_c10pll_state_verify(const struct intel_crtc_state *state,
 void intel_cx0pll_readout_hw_state(struct intel_encoder *encoder,
 				   struct intel_cx0pll_state *pll_state)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
-
-	if (intel_is_c10phy(i915, phy))
+	if (intel_encoder_is_c10phy(encoder))
 		intel_c10pll_readout_hw_state(encoder, &pll_state->c10);
 	else
 		intel_c20pll_readout_hw_state(encoder, &pll_state->c20);
@@ -3055,10 +3063,7 @@ void intel_cx0pll_readout_hw_state(struct intel_encoder *encoder,
 int intel_cx0pll_calc_port_clock(struct intel_encoder *encoder,
 				 const struct intel_cx0pll_state *pll_state)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
-
-	if (intel_is_c10phy(i915, phy))
+	if (intel_encoder_is_c10phy(encoder))
 		return intel_c10pll_calc_port_clock(encoder, &pll_state->c10);
 
 	return intel_c20pll_calc_port_clock(encoder, &pll_state->c20);
@@ -3070,7 +3075,7 @@ static void intel_c20pll_state_verify(const struct intel_crtc_state *state,
 				      struct intel_c20pll_state *mpll_hw_state)
 {
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
-	const struct intel_c20pll_state *mpll_sw_state = &state->cx0pll_state.c20;
+	const struct intel_c20pll_state *mpll_sw_state = &state->dpll_hw_state.cx0pll.c20;
 	bool sw_use_mpllb = intel_c20phy_use_mpllb(mpll_sw_state);
 	bool hw_use_mpllb = intel_c20phy_use_mpllb(mpll_hw_state);
 	int i;
@@ -3124,7 +3129,6 @@ void intel_cx0pll_state_verify(struct intel_atomic_state *state,
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_encoder *encoder;
 	struct intel_cx0pll_state mpll_hw_state = {};
-	enum phy phy;
 
 	if (DISPLAY_VER(i915) < 14)
 		return;
@@ -3138,14 +3142,13 @@ void intel_cx0pll_state_verify(struct intel_atomic_state *state,
 		return;
 
 	encoder = intel_get_crtc_new_encoder(state, new_crtc_state);
-	phy = intel_port_to_phy(i915, encoder->port);
 
 	if (intel_tc_port_in_tbt_alt_mode(enc_to_dig_port(encoder)))
 		return;
 
 	intel_cx0pll_readout_hw_state(encoder, &mpll_hw_state);
 
-	if (intel_is_c10phy(i915, phy))
+	if (intel_encoder_is_c10phy(encoder))
 		intel_c10pll_state_verify(new_crtc_state, crtc, encoder, &mpll_hw_state.c10);
 	else
 		intel_c20pll_state_verify(new_crtc_state, crtc, encoder, &mpll_hw_state.c20);
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.h b/drivers/gpu/drm/i915/display/intel_cx0_phy.h
index c6682677253a..3e03af3e006c 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy.h
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.h
@@ -11,7 +11,6 @@
 #include <linux/bits.h>
 
 enum icl_port_dpll_id;
-enum phy;
 struct drm_i915_private;
 struct intel_atomic_state;
 struct intel_c10pll_state;
@@ -22,7 +21,7 @@ struct intel_crtc_state;
 struct intel_encoder;
 struct intel_hdmi;
 
-bool intel_is_c10phy(struct drm_i915_private *dev_priv, enum phy phy);
+bool intel_encoder_is_c10phy(struct intel_encoder *encoder);
 void intel_mtl_pll_enable(struct intel_encoder *encoder,
 			  const struct intel_crtc_state *crtc_state);
 void intel_mtl_pll_disable(struct intel_encoder *encoder);
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c
index c17462b4c2ac..3c3fc53376ce 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi.c
@@ -200,10 +200,10 @@ void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv,
 			port_name(port));
 }
 
-static void intel_wait_ddi_buf_active(struct drm_i915_private *dev_priv,
-				      enum port port)
+static void intel_wait_ddi_buf_active(struct intel_encoder *encoder)
 {
-	enum phy phy = intel_port_to_phy(dev_priv, port);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
 	int timeout_us;
 	int ret;
 
@@ -218,7 +218,7 @@ static void intel_wait_ddi_buf_active(struct drm_i915_private *dev_priv,
 	} else if (IS_DG2(dev_priv)) {
 		timeout_us = 1200;
 	} else if (DISPLAY_VER(dev_priv) >= 12) {
-		if (intel_phy_is_tc(dev_priv, phy))
+		if (intel_encoder_is_tc(encoder))
 			timeout_us = 3000;
 		else
 			timeout_us = 1000;
@@ -331,7 +331,6 @@ static void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder,
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
 
 	/* DDI_BUF_CTL_ENABLE will be set by intel_ddi_prepare_link_retrain() later */
 	intel_dp->DP = dig_port->saved_port_bits |
@@ -345,7 +344,7 @@ static void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder,
 			intel_dp->DP |= DDI_BUF_PORT_DATA_10BIT;
 	}
 
-	if (IS_ALDERLAKE_P(i915) && intel_phy_is_tc(i915, phy)) {
+	if (IS_ALDERLAKE_P(i915) && intel_encoder_is_tc(encoder)) {
 		intel_dp->DP |= ddi_buf_phy_link_rate(crtc_state->port_clock);
 		if (!intel_tc_port_in_tbt_alt_mode(dig_port))
 			intel_dp->DP |= DDI_BUF_CTL_TC_PHY_OWNERSHIP;
@@ -632,6 +631,7 @@ intel_ddi_config_transcoder_func(struct intel_encoder *encoder,
 
 void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state)
 {
+	struct intel_display *display = to_intel_display(crtc_state);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
@@ -662,10 +662,9 @@ void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state
 
 	intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder), ctl);
 
-	if (intel_has_quirk(dev_priv, QUIRK_INCREASE_DDI_DISABLED_TIME) &&
+	if (intel_has_quirk(display, QUIRK_INCREASE_DDI_DISABLED_TIME) &&
 	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
-		drm_dbg_kms(&dev_priv->drm,
-			    "Quirk Increase DDI disabled time\n");
+		drm_dbg_kms(display->drm, "Quirk Increase DDI disabled time\n");
 		/* Quirk time at 100ms for reliable operation */
 		msleep(100);
 	}
@@ -895,7 +894,6 @@ intel_ddi_main_link_aux_domain(struct intel_digital_port *dig_port,
 			       const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
 
 	/*
 	 * ICL+ HW requires corresponding AUX IOs to be powered up for PSR with
@@ -914,7 +912,7 @@ intel_ddi_main_link_aux_domain(struct intel_digital_port *dig_port,
 		return intel_display_power_aux_io_domain(i915, dig_port->aux_ch);
 	else if (DISPLAY_VER(i915) < 14 &&
 		 (intel_crtc_has_dp_encoder(crtc_state) ||
-		  intel_phy_is_tc(i915, phy)))
+		  intel_encoder_is_tc(&dig_port->base)))
 		return intel_aux_power_domain(dig_port);
 	else
 		return POWER_DOMAIN_INVALID;
@@ -984,7 +982,7 @@ void intel_ddi_enable_transcoder_clock(struct intel_encoder *encoder,
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
-	enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	u32 val;
 
 	if (cpu_transcoder == TRANSCODER_EDP)
@@ -1113,7 +1111,7 @@ static void icl_ddi_combo_vswing_program(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	const struct intel_ddi_buf_trans *trans;
-	enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	int n_entries, ln;
 	u32 val;
 
@@ -1176,7 +1174,7 @@ static void icl_combo_phy_set_signal_levels(struct intel_encoder *encoder,
 					    const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	u32 val;
 	int ln;
 
@@ -1227,7 +1225,7 @@ static void icl_mg_phy_set_signal_levels(struct intel_encoder *encoder,
 					 const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum tc_port tc_port = intel_port_to_tc(dev_priv, encoder->port);
+	enum tc_port tc_port = intel_encoder_to_tc(encoder);
 	const struct intel_ddi_buf_trans *trans;
 	int n_entries, ln;
 
@@ -1328,7 +1326,7 @@ static void tgl_dkl_phy_set_signal_levels(struct intel_encoder *encoder,
 					  const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum tc_port tc_port = intel_port_to_tc(dev_priv, encoder->port);
+	enum tc_port tc_port = intel_encoder_to_tc(encoder);
 	const struct intel_ddi_buf_trans *trans;
 	int n_entries, ln;
 
@@ -1526,7 +1524,7 @@ static void adls_ddi_enable_clock(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	const struct intel_shared_dpll *pll = crtc_state->shared_dpll;
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	if (drm_WARN_ON(&i915->drm, !pll))
 		return;
@@ -1540,7 +1538,7 @@ static void adls_ddi_enable_clock(struct intel_encoder *encoder,
 static void adls_ddi_disable_clock(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	_icl_ddi_disable_clock(i915, ADLS_DPCLKA_CFGCR(phy),
 			       ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy));
@@ -1549,7 +1547,7 @@ static void adls_ddi_disable_clock(struct intel_encoder *encoder)
 static bool adls_ddi_is_clock_enabled(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	return _icl_ddi_is_clock_enabled(i915, ADLS_DPCLKA_CFGCR(phy),
 					 ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy));
@@ -1558,7 +1556,7 @@ static bool adls_ddi_is_clock_enabled(struct intel_encoder *encoder)
 static struct intel_shared_dpll *adls_ddi_get_pll(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	return _icl_ddi_get_pll(i915, ADLS_DPCLKA_CFGCR(phy),
 				ADLS_DPCLKA_CFGCR_DDI_CLK_SEL_MASK(phy),
@@ -1570,7 +1568,7 @@ static void rkl_ddi_enable_clock(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	const struct intel_shared_dpll *pll = crtc_state->shared_dpll;
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	if (drm_WARN_ON(&i915->drm, !pll))
 		return;
@@ -1584,7 +1582,7 @@ static void rkl_ddi_enable_clock(struct intel_encoder *encoder,
 static void rkl_ddi_disable_clock(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	_icl_ddi_disable_clock(i915, ICL_DPCLKA_CFGCR0,
 			       RKL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy));
@@ -1593,7 +1591,7 @@ static void rkl_ddi_disable_clock(struct intel_encoder *encoder)
 static bool rkl_ddi_is_clock_enabled(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	return _icl_ddi_is_clock_enabled(i915, ICL_DPCLKA_CFGCR0,
 					 RKL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy));
@@ -1602,7 +1600,7 @@ static bool rkl_ddi_is_clock_enabled(struct intel_encoder *encoder)
 static struct intel_shared_dpll *rkl_ddi_get_pll(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	return _icl_ddi_get_pll(i915, ICL_DPCLKA_CFGCR0,
 				RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy),
@@ -1614,7 +1612,7 @@ static void dg1_ddi_enable_clock(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	const struct intel_shared_dpll *pll = crtc_state->shared_dpll;
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	if (drm_WARN_ON(&i915->drm, !pll))
 		return;
@@ -1637,7 +1635,7 @@ static void dg1_ddi_enable_clock(struct intel_encoder *encoder,
 static void dg1_ddi_disable_clock(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	_icl_ddi_disable_clock(i915, DG1_DPCLKA_CFGCR0(phy),
 			       DG1_DPCLKA_CFGCR0_DDI_CLK_OFF(phy));
@@ -1646,7 +1644,7 @@ static void dg1_ddi_disable_clock(struct intel_encoder *encoder)
 static bool dg1_ddi_is_clock_enabled(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	return _icl_ddi_is_clock_enabled(i915, DG1_DPCLKA_CFGCR0(phy),
 					 DG1_DPCLKA_CFGCR0_DDI_CLK_OFF(phy));
@@ -1655,7 +1653,7 @@ static bool dg1_ddi_is_clock_enabled(struct intel_encoder *encoder)
 static struct intel_shared_dpll *dg1_ddi_get_pll(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	enum intel_dpll_id id;
 	u32 val;
 
@@ -1680,7 +1678,7 @@ static void icl_ddi_combo_enable_clock(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	const struct intel_shared_dpll *pll = crtc_state->shared_dpll;
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	if (drm_WARN_ON(&i915->drm, !pll))
 		return;
@@ -1694,7 +1692,7 @@ static void icl_ddi_combo_enable_clock(struct intel_encoder *encoder,
 static void icl_ddi_combo_disable_clock(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	_icl_ddi_disable_clock(i915, ICL_DPCLKA_CFGCR0,
 			       ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy));
@@ -1703,7 +1701,7 @@ static void icl_ddi_combo_disable_clock(struct intel_encoder *encoder)
 static bool icl_ddi_combo_is_clock_enabled(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	return _icl_ddi_is_clock_enabled(i915, ICL_DPCLKA_CFGCR0,
 					 ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy));
@@ -1712,7 +1710,7 @@ static bool icl_ddi_combo_is_clock_enabled(struct intel_encoder *encoder)
 struct intel_shared_dpll *icl_ddi_combo_get_pll(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	return _icl_ddi_get_pll(i915, ICL_DPCLKA_CFGCR0,
 				ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy),
@@ -1767,7 +1765,7 @@ static void icl_ddi_tc_enable_clock(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	const struct intel_shared_dpll *pll = crtc_state->shared_dpll;
-	enum tc_port tc_port = intel_port_to_tc(i915, encoder->port);
+	enum tc_port tc_port = intel_encoder_to_tc(encoder);
 	enum port port = encoder->port;
 
 	if (drm_WARN_ON(&i915->drm, !pll))
@@ -1787,7 +1785,7 @@ static void icl_ddi_tc_enable_clock(struct intel_encoder *encoder,
 static void icl_ddi_tc_disable_clock(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum tc_port tc_port = intel_port_to_tc(i915, encoder->port);
+	enum tc_port tc_port = intel_encoder_to_tc(encoder);
 	enum port port = encoder->port;
 
 	mutex_lock(&i915->display.dpll.lock);
@@ -1803,7 +1801,7 @@ static void icl_ddi_tc_disable_clock(struct intel_encoder *encoder)
 static bool icl_ddi_tc_is_clock_enabled(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum tc_port tc_port = intel_port_to_tc(i915, encoder->port);
+	enum tc_port tc_port = intel_encoder_to_tc(encoder);
 	enum port port = encoder->port;
 	u32 tmp;
 
@@ -1820,7 +1818,7 @@ static bool icl_ddi_tc_is_clock_enabled(struct intel_encoder *encoder)
 static struct intel_shared_dpll *icl_ddi_tc_get_pll(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum tc_port tc_port = intel_port_to_tc(i915, encoder->port);
+	enum tc_port tc_port = intel_encoder_to_tc(encoder);
 	enum port port = encoder->port;
 	enum intel_dpll_id id;
 	u32 tmp;
@@ -2086,12 +2084,11 @@ icl_program_mg_dp_mode(struct intel_digital_port *dig_port,
 		       const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-	enum tc_port tc_port = intel_port_to_tc(dev_priv, dig_port->base.port);
-	enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
+	enum tc_port tc_port = intel_encoder_to_tc(&dig_port->base);
 	u32 ln0, ln1, pin_assignment;
 	u8 width;
 
-	if (!intel_phy_is_tc(dev_priv, phy) ||
+	if (!intel_encoder_is_tc(&dig_port->base) ||
 	    intel_tc_port_in_tbt_alt_mode(dig_port))
 		return;
 
@@ -2327,9 +2324,9 @@ static void intel_ddi_power_up_lanes(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
 
-	if (intel_phy_is_combo(i915, phy)) {
+	if (intel_encoder_is_combo(encoder)) {
+		enum phy phy = intel_encoder_to_phy(encoder);
 		bool lane_reversal =
 			dig_port->saved_port_bits & DDI_BUF_PORT_REVERSAL;
 
@@ -2339,10 +2336,15 @@ static void intel_ddi_power_up_lanes(struct intel_encoder *encoder,
 	}
 }
 
-/* Splitter enable for eDP MSO is limited to certain pipes. */
+/*
+ * Splitter enable for eDP MSO is limited to certain pipes, on certain
+ * platforms.
+ */
 static u8 intel_ddi_splitter_pipe_mask(struct drm_i915_private *i915)
 {
-	if (IS_ALDERLAKE_P(i915))
+	if (DISPLAY_VER(i915) > 20)
+		return ~0;
+	else if (IS_ALDERLAKE_P(i915))
 		return BIT(PIPE_A) | BIT(PIPE_B);
 	else
 		return BIT(PIPE_A);
@@ -2812,15 +2814,14 @@ static void intel_ddi_pre_enable_dp(struct intel_atomic_state *state,
 				    const struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 
-	if (HAS_DP20(dev_priv)) {
+	if (HAS_DP20(dev_priv))
 		intel_dp_128b132b_sdp_crc16(enc_to_intel_dp(encoder),
 					    crtc_state);
-		if (crtc_state->has_panel_replay)
-			drm_dp_dpcd_writeb(&intel_dp->aux, PANEL_REPLAY_CONFIG,
-					   DP_PANEL_REPLAY_ENABLE);
-	}
+
+	/* Panel replay has to be enabled in sink dpcd before link training. */
+	if (crtc_state->has_panel_replay)
+		intel_psr_enable_sink(enc_to_intel_dp(encoder), crtc_state);
 
 	if (DISPLAY_VER(dev_priv) >= 14)
 		mtl_ddi_pre_enable_dp(state, encoder, crtc_state, conn_state);
@@ -3095,39 +3096,48 @@ static void intel_ddi_post_disable_hdmi(struct intel_atomic_state *state,
 	intel_dp_dual_mode_set_tmds_output(intel_hdmi, false);
 }
 
-static void intel_ddi_post_disable(struct intel_atomic_state *state,
-				   struct intel_encoder *encoder,
-				   const struct intel_crtc_state *old_crtc_state,
-				   const struct drm_connector_state *old_conn_state)
+static void intel_ddi_post_disable_hdmi_or_sst(struct intel_atomic_state *state,
+					       struct intel_encoder *encoder,
+					       const struct intel_crtc_state *old_crtc_state,
+					       const struct drm_connector_state *old_conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_crtc *slave_crtc;
+	struct intel_crtc *pipe_crtc;
 
-	if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST)) {
-		intel_crtc_vblank_off(old_crtc_state);
+	for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, pipe_crtc,
+					 intel_crtc_joined_pipe_mask(old_crtc_state)) {
+		const struct intel_crtc_state *old_pipe_crtc_state =
+			intel_atomic_get_old_crtc_state(state, pipe_crtc);
 
-		intel_disable_transcoder(old_crtc_state);
+		intel_crtc_vblank_off(old_pipe_crtc_state);
+	}
 
-		intel_ddi_disable_transcoder_func(old_crtc_state);
+	intel_disable_transcoder(old_crtc_state);
 
-		intel_dsc_disable(old_crtc_state);
+	intel_ddi_disable_transcoder_func(old_crtc_state);
+
+	for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, pipe_crtc,
+					 intel_crtc_joined_pipe_mask(old_crtc_state)) {
+		const struct intel_crtc_state *old_pipe_crtc_state =
+			intel_atomic_get_old_crtc_state(state, pipe_crtc);
+
+		intel_dsc_disable(old_pipe_crtc_state);
 
 		if (DISPLAY_VER(dev_priv) >= 9)
-			skl_scaler_disable(old_crtc_state);
+			skl_scaler_disable(old_pipe_crtc_state);
 		else
-			ilk_pfit_disable(old_crtc_state);
+			ilk_pfit_disable(old_pipe_crtc_state);
 	}
+}
 
-	for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, slave_crtc,
-					 intel_crtc_bigjoiner_slave_pipes(old_crtc_state)) {
-		const struct intel_crtc_state *old_slave_crtc_state =
-			intel_atomic_get_old_crtc_state(state, slave_crtc);
-
-		intel_crtc_vblank_off(old_slave_crtc_state);
-
-		intel_dsc_disable(old_slave_crtc_state);
-		skl_scaler_disable(old_slave_crtc_state);
-	}
+static void intel_ddi_post_disable(struct intel_atomic_state *state,
+				   struct intel_encoder *encoder,
+				   const struct intel_crtc_state *old_crtc_state,
+				   const struct drm_connector_state *old_conn_state)
+{
+	if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
+		intel_ddi_post_disable_hdmi_or_sst(state, encoder, old_crtc_state,
+						   old_conn_state);
 
 	/*
 	 * When called from DP MST code:
@@ -3155,14 +3165,11 @@ static void intel_ddi_post_pll_disable(struct intel_atomic_state *state,
 				       const struct intel_crtc_state *old_crtc_state,
 				       const struct drm_connector_state *old_conn_state)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
-	bool is_tc_port = intel_phy_is_tc(i915, phy);
 
 	main_link_aux_power_domain_put(dig_port, old_crtc_state);
 
-	if (is_tc_port)
+	if (intel_encoder_is_tc(encoder))
 		intel_tc_port_put_link(dig_port);
 }
 
@@ -3263,7 +3270,6 @@ static void intel_enable_ddi_hdmi(struct intel_atomic_state *state,
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	struct drm_connector *connector = conn_state->connector;
 	enum port port = encoder->port;
-	enum phy phy = intel_port_to_phy(dev_priv, port);
 	u32 buf_ctl;
 
 	if (!intel_hdmi_handle_sink_scrambling(encoder, connector,
@@ -3347,14 +3353,14 @@ static void intel_enable_ddi_hdmi(struct intel_atomic_state *state,
 
 		if (DISPLAY_VER(dev_priv) >= 20)
 			buf_ctl |= XE2LPD_DDI_BUF_D2D_LINK_ENABLE;
-	} else if (IS_ALDERLAKE_P(dev_priv) && intel_phy_is_tc(dev_priv, phy)) {
+	} else if (IS_ALDERLAKE_P(dev_priv) && intel_encoder_is_tc(encoder)) {
 		drm_WARN_ON(&dev_priv->drm, !intel_tc_port_in_legacy_mode(dig_port));
 		buf_ctl |= DDI_BUF_CTL_TC_PHY_OWNERSHIP;
 	}
 
 	intel_de_write(dev_priv, DDI_BUF_CTL(port), buf_ctl);
 
-	intel_wait_ddi_buf_active(dev_priv, port);
+	intel_wait_ddi_buf_active(encoder);
 }
 
 static void intel_enable_ddi(struct intel_atomic_state *state,
@@ -3362,10 +3368,10 @@ static void intel_enable_ddi(struct intel_atomic_state *state,
 			     const struct intel_crtc_state *crtc_state,
 			     const struct drm_connector_state *conn_state)
 {
-	drm_WARN_ON(state->base.dev, crtc_state->has_pch_encoder);
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	struct intel_crtc *pipe_crtc;
 
-	if (!intel_crtc_is_bigjoiner_slave(crtc_state))
-		intel_ddi_enable_transcoder_func(encoder, crtc_state);
+	intel_ddi_enable_transcoder_func(encoder, crtc_state);
 
 	/* Enable/Disable DP2.0 SDP split config before transcoder */
 	intel_audio_sdp_split_update(crtc_state);
@@ -3374,7 +3380,13 @@ static void intel_enable_ddi(struct intel_atomic_state *state,
 
 	intel_ddi_wait_for_fec_status(encoder, crtc_state, true);
 
-	intel_crtc_vblank_on(crtc_state);
+	for_each_intel_crtc_in_pipe_mask_reverse(&i915->drm, pipe_crtc,
+						 intel_crtc_joined_pipe_mask(crtc_state)) {
+		const struct intel_crtc_state *pipe_crtc_state =
+			intel_atomic_get_new_crtc_state(state, pipe_crtc);
+
+		intel_crtc_vblank_on(pipe_crtc_state);
+	}
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
 		intel_enable_ddi_hdmi(state, encoder, crtc_state, conn_state);
@@ -3470,19 +3482,17 @@ void intel_ddi_update_active_dpll(struct intel_atomic_state *state,
 				  struct intel_crtc *crtc)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	struct intel_crtc_state *crtc_state =
+	const struct intel_crtc_state *crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
-	struct intel_crtc *slave_crtc;
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	struct intel_crtc *pipe_crtc;
 
 	/* FIXME: Add MTL pll_mgr */
-	if (DISPLAY_VER(i915) >= 14 || !intel_phy_is_tc(i915, phy))
+	if (DISPLAY_VER(i915) >= 14 || !intel_encoder_is_tc(encoder))
 		return;
 
-	intel_update_active_dpll(state, crtc, encoder);
-	for_each_intel_crtc_in_pipe_mask(&i915->drm, slave_crtc,
-					 intel_crtc_bigjoiner_slave_pipes(crtc_state))
-		intel_update_active_dpll(state, slave_crtc, encoder);
+	for_each_intel_crtc_in_pipe_mask(&i915->drm, pipe_crtc,
+					 intel_crtc_joined_pipe_mask(crtc_state))
+		intel_update_active_dpll(state, pipe_crtc, encoder);
 }
 
 static void
@@ -3493,8 +3503,7 @@ intel_ddi_pre_pll_enable(struct intel_atomic_state *state,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
-	enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
-	bool is_tc_port = intel_phy_is_tc(dev_priv, phy);
+	bool is_tc_port = intel_encoder_is_tc(encoder);
 
 	if (is_tc_port) {
 		struct intel_crtc *master_crtc =
@@ -3513,14 +3522,14 @@ intel_ddi_pre_pll_enable(struct intel_atomic_state *state,
 		 */
 		intel_tc_port_set_fia_lane_count(dig_port, crtc_state->lane_count);
 	else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
-		bxt_ddi_phy_set_lane_optim_mask(encoder,
-						crtc_state->lane_lat_optim_mask);
+		bxt_dpio_phy_set_lane_optim_mask(encoder,
+						 crtc_state->lane_lat_optim_mask);
 }
 
 static void adlp_tbt_to_dp_alt_switch_wa(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum tc_port tc_port = intel_port_to_tc(i915, encoder->port);
+	enum tc_port tc_port = intel_encoder_to_tc(encoder);
 	int ln;
 
 	for (ln = 0; ln < 2; ln++)
@@ -3574,7 +3583,7 @@ static void mtl_ddi_prepare_link_retrain(struct intel_dp *intel_dp,
 	intel_de_posting_read(dev_priv, DDI_BUF_CTL(port));
 
 	/* 6.j Poll for PORT_BUF_CTL Idle Status == 0, timeout after 100 us */
-	intel_wait_ddi_buf_active(dev_priv, port);
+	intel_wait_ddi_buf_active(encoder);
 }
 
 static void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp,
@@ -3624,7 +3633,7 @@ static void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp,
 	intel_de_write(dev_priv, DDI_BUF_CTL(port), intel_dp->DP);
 	intel_de_posting_read(dev_priv, DDI_BUF_CTL(port));
 
-	intel_wait_ddi_buf_active(dev_priv, port);
+	intel_wait_ddi_buf_active(encoder);
 }
 
 static void intel_ddi_set_link_train(struct intel_dp *intel_dp,
@@ -3681,7 +3690,7 @@ static void intel_ddi_set_idle_link_train(struct intel_dp *intel_dp,
 
 	if (intel_de_wait_for_set(dev_priv,
 				  dp_tp_status_reg(encoder, crtc_state),
-				  DP_TP_STATUS_IDLE_DONE, 1))
+				  DP_TP_STATUS_IDLE_DONE, 2))
 		drm_err(&dev_priv->drm,
 			"Timed out waiting for DP idle patterns\n");
 }
@@ -3946,7 +3955,7 @@ static void intel_ddi_get_config(struct intel_encoder *encoder,
 
 	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
 		pipe_config->lane_lat_optim_mask =
-			bxt_ddi_phy_get_lane_lat_optim_mask(encoder);
+			bxt_dpio_phy_get_lane_lat_optim_mask(encoder);
 
 	intel_ddi_compute_min_voltage_level(pipe_config);
 
@@ -3972,6 +3981,7 @@ static void intel_ddi_get_config(struct intel_encoder *encoder,
 
 	intel_read_dp_sdp(encoder, pipe_config, HDMI_PACKET_TYPE_GAMUT_METADATA);
 	intel_read_dp_sdp(encoder, pipe_config, DP_SDP_VSC);
+	intel_read_dp_sdp(encoder, pipe_config, DP_SDP_ADAPTIVE_SYNC);
 
 	intel_audio_codec_get_config(encoder, pipe_config);
 }
@@ -4006,8 +4016,8 @@ static void mtl_ddi_get_config(struct intel_encoder *encoder,
 	if (intel_tc_port_in_tbt_alt_mode(dig_port)) {
 		crtc_state->port_clock = intel_mtl_tbt_calc_port_clock(encoder);
 	} else {
-		intel_cx0pll_readout_hw_state(encoder, &crtc_state->cx0pll_state);
-		crtc_state->port_clock = intel_cx0pll_calc_port_clock(encoder, &crtc_state->cx0pll_state);
+		intel_cx0pll_readout_hw_state(encoder, &crtc_state->dpll_hw_state.cx0pll);
+		crtc_state->port_clock = intel_cx0pll_calc_port_clock(encoder, &crtc_state->dpll_hw_state.cx0pll);
 	}
 
 	intel_ddi_get_config(encoder, crtc_state);
@@ -4016,8 +4026,8 @@ static void mtl_ddi_get_config(struct intel_encoder *encoder,
 static void dg2_ddi_get_config(struct intel_encoder *encoder,
 				struct intel_crtc_state *crtc_state)
 {
-	intel_mpllb_readout_hw_state(encoder, &crtc_state->mpllb_state);
-	crtc_state->port_clock = intel_mpllb_calc_port_clock(encoder, &crtc_state->mpllb_state);
+	intel_mpllb_readout_hw_state(encoder, &crtc_state->dpll_hw_state.mpllb);
+	crtc_state->port_clock = intel_mpllb_calc_port_clock(encoder, &crtc_state->dpll_hw_state.mpllb);
 
 	intel_ddi_get_config(encoder, crtc_state);
 }
@@ -4144,10 +4154,7 @@ void hsw_ddi_get_config(struct intel_encoder *encoder,
 static void intel_ddi_sync_state(struct intel_encoder *encoder,
 				 const struct intel_crtc_state *crtc_state)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
-
-	if (intel_phy_is_tc(i915, phy))
+	if (intel_encoder_is_tc(encoder))
 		intel_tc_port_sanitize_mode(enc_to_dig_port(encoder),
 					    crtc_state);
 
@@ -4159,10 +4166,9 @@ static bool intel_ddi_initial_fastset_check(struct intel_encoder *encoder,
 					    struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
 	bool fastset = true;
 
-	if (intel_phy_is_tc(i915, phy)) {
+	if (intel_encoder_is_tc(encoder)) {
 		drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s] Forcing full modeset to compute TC port DPLLs\n",
 			    encoder->base.base.id, encoder->base.name);
 		crtc_state->uapi.mode_changed = true;
@@ -4226,7 +4232,7 @@ static int intel_ddi_compute_config(struct intel_encoder *encoder,
 
 	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
 		pipe_config->lane_lat_optim_mask =
-			bxt_ddi_phy_calc_lane_lat_optim_mask(pipe_config->lane_count);
+			bxt_dpio_phy_calc_lane_lat_optim_mask(pipe_config->lane_count);
 
 	intel_ddi_compute_min_voltage_level(pipe_config);
 
@@ -4353,10 +4359,9 @@ static void intel_ddi_encoder_destroy(struct drm_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->dev);
 	struct intel_digital_port *dig_port = enc_to_dig_port(to_intel_encoder(encoder));
-	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
 
 	intel_dp_encoder_flush_work(encoder);
-	if (intel_phy_is_tc(i915, phy))
+	if (intel_encoder_is_tc(&dig_port->base))
 		intel_tc_port_cleanup(dig_port);
 	intel_display_power_flush_work(i915);
 
@@ -4367,16 +4372,14 @@ static void intel_ddi_encoder_destroy(struct drm_encoder *encoder)
 
 static void intel_ddi_encoder_reset(struct drm_encoder *encoder)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->dev);
 	struct intel_dp *intel_dp = enc_to_intel_dp(to_intel_encoder(encoder));
 	struct intel_digital_port *dig_port = enc_to_dig_port(to_intel_encoder(encoder));
-	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
 
 	intel_dp->reset_link_params = true;
 
 	intel_pps_encoder_reset(intel_dp);
 
-	if (intel_phy_is_tc(i915, phy))
+	if (intel_encoder_is_tc(&dig_port->base))
 		intel_tc_port_init_mode(dig_port);
 }
 
@@ -4543,11 +4546,9 @@ static enum intel_hotplug_state
 intel_ddi_hotplug(struct intel_encoder *encoder,
 		  struct intel_connector *connector)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	struct intel_dp *intel_dp = &dig_port->dp;
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
-	bool is_tc = intel_phy_is_tc(i915, phy);
+	bool is_tc = intel_encoder_is_tc(encoder);
 	struct drm_modeset_acquire_ctx ctx;
 	enum intel_hotplug_state state;
 	int ret;
@@ -4829,10 +4830,7 @@ static bool port_strap_detected(struct drm_i915_private *i915, enum port port)
 
 static bool need_aux_ch(struct intel_encoder *encoder, bool init_dp)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
-
-	return init_dp || intel_phy_is_tc(i915, phy);
+	return init_dp || intel_encoder_is_tc(encoder);
 }
 
 static bool assert_has_icl_dsi(struct drm_i915_private *i915)
@@ -5076,17 +5074,17 @@ void intel_ddi_init(struct drm_i915_private *dev_priv,
 	} else if (IS_DG2(dev_priv)) {
 		encoder->set_signal_levels = intel_snps_phy_set_signal_levels;
 	} else if (DISPLAY_VER(dev_priv) >= 12) {
-		if (intel_phy_is_combo(dev_priv, phy))
+		if (intel_encoder_is_combo(encoder))
 			encoder->set_signal_levels = icl_combo_phy_set_signal_levels;
 		else
 			encoder->set_signal_levels = tgl_dkl_phy_set_signal_levels;
 	} else if (DISPLAY_VER(dev_priv) >= 11) {
-		if (intel_phy_is_combo(dev_priv, phy))
+		if (intel_encoder_is_combo(encoder))
 			encoder->set_signal_levels = icl_combo_phy_set_signal_levels;
 		else
 			encoder->set_signal_levels = icl_mg_phy_set_signal_levels;
 	} else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
-		encoder->set_signal_levels = bxt_ddi_phy_set_signal_levels;
+		encoder->set_signal_levels = bxt_dpio_phy_set_signal_levels;
 	} else {
 		encoder->set_signal_levels = hsw_set_signal_levels;
 	}
@@ -5131,7 +5129,7 @@ void intel_ddi_init(struct drm_i915_private *dev_priv,
 			goto err;
 	}
 
-	if (intel_phy_is_tc(dev_priv, phy)) {
+	if (intel_encoder_is_tc(encoder)) {
 		bool is_legacy =
 			!intel_bios_encoder_supports_typec_usb(devdata) &&
 			!intel_bios_encoder_supports_tbt(devdata);
@@ -5160,7 +5158,7 @@ void intel_ddi_init(struct drm_i915_private *dev_priv,
 	dig_port->ddi_io_power_domain = intel_display_power_ddi_io_domain(dev_priv, port);
 
 	if (DISPLAY_VER(dev_priv) >= 11) {
-		if (intel_phy_is_tc(dev_priv, phy))
+		if (intel_encoder_is_tc(encoder))
 			dig_port->connected = intel_tc_port_connected;
 		else
 			dig_port->connected = lpt_digital_port_connected;
diff --git a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c
index de809e2d9cac..4d21ce734343 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c
@@ -1691,14 +1691,11 @@ mtl_get_cx0_buf_trans(struct intel_encoder *encoder,
 		      const struct intel_crtc_state *crtc_state,
 		      int *n_entries)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
-
 	if (intel_crtc_has_dp_encoder(crtc_state) && crtc_state->port_clock >= 1000000)
 		return intel_get_buf_trans(&mtl_c20_trans_uhbr, n_entries);
-	else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) && !(intel_is_c10phy(i915, phy)))
+	else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) && !(intel_encoder_is_c10phy(encoder)))
 		return intel_get_buf_trans(&mtl_c20_trans_hdmi, n_entries);
-	else if (!intel_is_c10phy(i915, phy))
+	else if (!intel_encoder_is_c10phy(encoder))
 		return intel_get_buf_trans(&mtl_c20_trans_dp14, n_entries);
 	else
 		return intel_get_buf_trans(&mtl_c10_trans_dp14, n_entries);
@@ -1707,14 +1704,13 @@ mtl_get_cx0_buf_trans(struct intel_encoder *encoder,
 void intel_ddi_buf_trans_init(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
 
 	if (DISPLAY_VER(i915) >= 14) {
 		encoder->get_buf_trans = mtl_get_cx0_buf_trans;
 	} else if (IS_DG2(i915)) {
 		encoder->get_buf_trans = dg2_get_snps_buf_trans;
 	} else if (IS_ALDERLAKE_P(i915)) {
-		if (intel_phy_is_combo(i915, phy))
+		if (intel_encoder_is_combo(encoder))
 			encoder->get_buf_trans = adlp_get_combo_buf_trans;
 		else
 			encoder->get_buf_trans = adlp_get_dkl_buf_trans;
@@ -1725,16 +1721,16 @@ void intel_ddi_buf_trans_init(struct intel_encoder *encoder)
 	} else if (IS_DG1(i915)) {
 		encoder->get_buf_trans = dg1_get_combo_buf_trans;
 	} else if (DISPLAY_VER(i915) >= 12) {
-		if (intel_phy_is_combo(i915, phy))
+		if (intel_encoder_is_combo(encoder))
 			encoder->get_buf_trans = tgl_get_combo_buf_trans;
 		else
 			encoder->get_buf_trans = tgl_get_dkl_buf_trans;
 	} else if (DISPLAY_VER(i915) == 11) {
-		if (IS_PLATFORM(i915, INTEL_JASPERLAKE))
+		if (IS_JASPERLAKE(i915))
 			encoder->get_buf_trans = jsl_get_combo_buf_trans;
-		else if (IS_PLATFORM(i915, INTEL_ELKHARTLAKE))
+		else if (IS_ELKHARTLAKE(i915))
 			encoder->get_buf_trans = ehl_get_combo_buf_trans;
-		else if (intel_phy_is_combo(i915, phy))
+		else if (intel_encoder_is_combo(encoder))
 			encoder->get_buf_trans = icl_get_combo_buf_trans;
 		else
 			encoder->get_buf_trans = icl_get_mg_buf_trans;
diff --git a/drivers/gpu/drm/i915/display/intel_de.h b/drivers/gpu/drm/i915/display/intel_de.h
index 42552d8c151e..e881bfeafb47 100644
--- a/drivers/gpu/drm/i915/display/intel_de.h
+++ b/drivers/gpu/drm/i915/display/intel_de.h
@@ -10,80 +10,185 @@
 #include "i915_trace.h"
 #include "intel_uncore.h"
 
+static inline struct intel_uncore *__to_uncore(struct intel_display *display)
+{
+	return &to_i915(display->drm)->uncore;
+}
+
 static inline u32
-intel_de_read(struct drm_i915_private *i915, i915_reg_t reg)
+__intel_de_read(struct intel_display *display, i915_reg_t reg)
 {
-	return intel_uncore_read(&i915->uncore, reg);
+	u32 val;
+
+	intel_dmc_wl_get(display, reg);
+
+	val = intel_uncore_read(__to_uncore(display), reg);
+
+	intel_dmc_wl_put(display, reg);
+
+	return val;
 }
+#define intel_de_read(p,...) __intel_de_read(__to_intel_display(p), __VA_ARGS__)
 
 static inline u8
-intel_de_read8(struct drm_i915_private *i915, i915_reg_t reg)
+__intel_de_read8(struct intel_display *display, i915_reg_t reg)
 {
-	return intel_uncore_read8(&i915->uncore, reg);
+	u8 val;
+
+	intel_dmc_wl_get(display, reg);
+
+	val = intel_uncore_read8(__to_uncore(display), reg);
+
+	intel_dmc_wl_put(display, reg);
+
+	return val;
 }
+#define intel_de_read8(p,...) __intel_de_read8(__to_intel_display(p), __VA_ARGS__)
 
 static inline u64
-intel_de_read64_2x32(struct drm_i915_private *i915,
-		     i915_reg_t lower_reg, i915_reg_t upper_reg)
+__intel_de_read64_2x32(struct intel_display *display,
+		       i915_reg_t lower_reg, i915_reg_t upper_reg)
 {
-	return intel_uncore_read64_2x32(&i915->uncore, lower_reg, upper_reg);
+	u64 val;
+
+	intel_dmc_wl_get(display, lower_reg);
+	intel_dmc_wl_get(display, upper_reg);
+
+	val = intel_uncore_read64_2x32(__to_uncore(display), lower_reg,
+				       upper_reg);
+
+	intel_dmc_wl_put(display, upper_reg);
+	intel_dmc_wl_put(display, lower_reg);
+
+	return val;
 }
+#define intel_de_read64_2x32(p,...) __intel_de_read64_2x32(__to_intel_display(p), __VA_ARGS__)
 
 static inline void
-intel_de_posting_read(struct drm_i915_private *i915, i915_reg_t reg)
+__intel_de_posting_read(struct intel_display *display, i915_reg_t reg)
 {
-	intel_uncore_posting_read(&i915->uncore, reg);
+	intel_dmc_wl_get(display, reg);
+
+	intel_uncore_posting_read(__to_uncore(display), reg);
+
+	intel_dmc_wl_put(display, reg);
 }
+#define intel_de_posting_read(p,...) __intel_de_posting_read(__to_intel_display(p), __VA_ARGS__)
 
 static inline void
-intel_de_write(struct drm_i915_private *i915, i915_reg_t reg, u32 val)
+__intel_de_write(struct intel_display *display, i915_reg_t reg, u32 val)
 {
-	intel_uncore_write(&i915->uncore, reg, val);
+	intel_dmc_wl_get(display, reg);
+
+	intel_uncore_write(__to_uncore(display), reg, val);
+
+	intel_dmc_wl_put(display, reg);
 }
+#define intel_de_write(p,...) __intel_de_write(__to_intel_display(p), __VA_ARGS__)
 
 static inline u32
-intel_de_rmw(struct drm_i915_private *i915, i915_reg_t reg, u32 clear, u32 set)
+____intel_de_rmw_nowl(struct intel_display *display, i915_reg_t reg,
+		      u32 clear, u32 set)
 {
-	return intel_uncore_rmw(&i915->uncore, reg, clear, set);
+	return intel_uncore_rmw(__to_uncore(display), reg, clear, set);
 }
+#define __intel_de_rmw_nowl(p,...) ____intel_de_rmw_nowl(__to_intel_display(p), __VA_ARGS__)
+
+static inline u32
+__intel_de_rmw(struct intel_display *display, i915_reg_t reg, u32 clear,
+	       u32 set)
+{
+	u32 val;
+
+	intel_dmc_wl_get(display, reg);
+
+	val = __intel_de_rmw_nowl(display, reg, clear, set);
+
+	intel_dmc_wl_put(display, reg);
+
+	return val;
+}
+#define intel_de_rmw(p,...) __intel_de_rmw(__to_intel_display(p), __VA_ARGS__)
 
 static inline int
-intel_de_wait_for_register(struct drm_i915_private *i915, i915_reg_t reg,
-			   u32 mask, u32 value, unsigned int timeout)
+____intel_de_wait_for_register_nowl(struct intel_display *display,
+				    i915_reg_t reg,
+				    u32 mask, u32 value, unsigned int timeout)
 {
-	return intel_wait_for_register(&i915->uncore, reg, mask, value, timeout);
+	return intel_wait_for_register(__to_uncore(display), reg, mask,
+				       value, timeout);
 }
+#define __intel_de_wait_for_register_nowl(p,...) ____intel_de_wait_for_register_nowl(__to_intel_display(p), __VA_ARGS__)
 
 static inline int
-intel_de_wait_for_register_fw(struct drm_i915_private *i915, i915_reg_t reg,
-			      u32 mask, u32 value, unsigned int timeout)
+__intel_de_wait(struct intel_display *display, i915_reg_t reg,
+		u32 mask, u32 value, unsigned int timeout)
 {
-	return intel_wait_for_register_fw(&i915->uncore, reg, mask, value, timeout);
+	int ret;
+
+	intel_dmc_wl_get(display, reg);
+
+	ret = __intel_de_wait_for_register_nowl(display, reg, mask, value,
+						timeout);
+
+	intel_dmc_wl_put(display, reg);
+
+	return ret;
 }
+#define intel_de_wait(p,...) __intel_de_wait(__to_intel_display(p), __VA_ARGS__)
 
 static inline int
-__intel_de_wait_for_register(struct drm_i915_private *i915, i915_reg_t reg,
-			     u32 mask, u32 value,
-			     unsigned int fast_timeout_us,
-			     unsigned int slow_timeout_ms, u32 *out_value)
+__intel_de_wait_fw(struct intel_display *display, i915_reg_t reg,
+		   u32 mask, u32 value, unsigned int timeout)
 {
-	return __intel_wait_for_register(&i915->uncore, reg, mask, value,
-					 fast_timeout_us, slow_timeout_ms, out_value);
+	int ret;
+
+	intel_dmc_wl_get(display, reg);
+
+	ret = intel_wait_for_register_fw(__to_uncore(display), reg, mask,
+					 value, timeout);
+
+	intel_dmc_wl_put(display, reg);
+
+	return ret;
 }
+#define intel_de_wait_fw(p,...) __intel_de_wait_fw(__to_intel_display(p), __VA_ARGS__)
 
 static inline int
-intel_de_wait_for_set(struct drm_i915_private *i915, i915_reg_t reg,
-		      u32 mask, unsigned int timeout)
+__intel_de_wait_custom(struct intel_display *display, i915_reg_t reg,
+		       u32 mask, u32 value,
+		       unsigned int fast_timeout_us,
+		       unsigned int slow_timeout_ms, u32 *out_value)
 {
-	return intel_de_wait_for_register(i915, reg, mask, mask, timeout);
+	int ret;
+
+	intel_dmc_wl_get(display, reg);
+
+	ret = __intel_wait_for_register(__to_uncore(display), reg, mask,
+					value,
+					fast_timeout_us, slow_timeout_ms, out_value);
+
+	intel_dmc_wl_put(display, reg);
+
+	return ret;
 }
+#define intel_de_wait_custom(p,...) __intel_de_wait_custom(__to_intel_display(p), __VA_ARGS__)
 
 static inline int
-intel_de_wait_for_clear(struct drm_i915_private *i915, i915_reg_t reg,
+__intel_de_wait_for_set(struct intel_display *display, i915_reg_t reg,
 			u32 mask, unsigned int timeout)
 {
-	return intel_de_wait_for_register(i915, reg, mask, 0, timeout);
+	return intel_de_wait(display, reg, mask, mask, timeout);
+}
+#define intel_de_wait_for_set(p,...) __intel_de_wait_for_set(__to_intel_display(p), __VA_ARGS__)
+
+static inline int
+__intel_de_wait_for_clear(struct intel_display *display, i915_reg_t reg,
+			  u32 mask, unsigned int timeout)
+{
+	return intel_de_wait(display, reg, mask, 0, timeout);
 }
+#define intel_de_wait_for_clear(p,...) __intel_de_wait_for_clear(__to_intel_display(p), __VA_ARGS__)
 
 /*
  * Unlocked mmio-accessors, think carefully before using these.
@@ -94,33 +199,38 @@ intel_de_wait_for_clear(struct drm_i915_private *i915, i915_reg_t reg,
  * a more localised lock guarding all access to that bank of registers.
  */
 static inline u32
-intel_de_read_fw(struct drm_i915_private *i915, i915_reg_t reg)
+__intel_de_read_fw(struct intel_display *display, i915_reg_t reg)
 {
 	u32 val;
 
-	val = intel_uncore_read_fw(&i915->uncore, reg);
+	val = intel_uncore_read_fw(__to_uncore(display), reg);
 	trace_i915_reg_rw(false, reg, val, sizeof(val), true);
 
 	return val;
 }
+#define intel_de_read_fw(p,...) __intel_de_read_fw(__to_intel_display(p), __VA_ARGS__)
 
 static inline void
-intel_de_write_fw(struct drm_i915_private *i915, i915_reg_t reg, u32 val)
+__intel_de_write_fw(struct intel_display *display, i915_reg_t reg, u32 val)
 {
 	trace_i915_reg_rw(true, reg, val, sizeof(val), true);
-	intel_uncore_write_fw(&i915->uncore, reg, val);
+	intel_uncore_write_fw(__to_uncore(display), reg, val);
 }
+#define intel_de_write_fw(p,...) __intel_de_write_fw(__to_intel_display(p), __VA_ARGS__)
 
 static inline u32
-intel_de_read_notrace(struct drm_i915_private *i915, i915_reg_t reg)
+__intel_de_read_notrace(struct intel_display *display, i915_reg_t reg)
 {
-	return intel_uncore_read_notrace(&i915->uncore, reg);
+	return intel_uncore_read_notrace(__to_uncore(display), reg);
 }
+#define intel_de_read_notrace(p,...) __intel_de_read_notrace(__to_intel_display(p), __VA_ARGS__)
 
 static inline void
-intel_de_write_notrace(struct drm_i915_private *i915, i915_reg_t reg, u32 val)
+__intel_de_write_notrace(struct intel_display *display, i915_reg_t reg,
+			 u32 val)
 {
-	intel_uncore_write_notrace(&i915->uncore, reg, val);
+	intel_uncore_write_notrace(__to_uncore(display), reg, val);
 }
+#define intel_de_write_notrace(p,...) __intel_de_write_notrace(__to_intel_display(p), __VA_ARGS__)
 
 #endif /* __INTEL_DE_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
index 8af9e6128277..273323f30ae2 100644
--- a/drivers/gpu/drm/i915/display/intel_display.c
+++ b/drivers/gpu/drm/i915/display/intel_display.c
@@ -85,7 +85,6 @@
 #include "intel_dvo.h"
 #include "intel_fb.h"
 #include "intel_fbc.h"
-#include "intel_fbdev.h"
 #include "intel_fdi.h"
 #include "intel_fifo_underrun.h"
 #include "intel_frontbuffer.h"
@@ -120,6 +119,7 @@
 #include "skl_scaler.h"
 #include "skl_universal_plane.h"
 #include "skl_watermark.h"
+#include "vlv_dpio_phy_regs.h"
 #include "vlv_dsi.h"
 #include "vlv_dsi_pll.h"
 #include "vlv_dsi_regs.h"
@@ -275,6 +275,13 @@ static int intel_bigjoiner_num_pipes(const struct intel_crtc_state *crtc_state)
 	return hweight8(crtc_state->bigjoiner_pipes);
 }
 
+u8 intel_crtc_joined_pipe_mask(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+
+	return BIT(crtc->pipe) | crtc_state->bigjoiner_pipes;
+}
+
 struct intel_crtc *intel_master_crtc(const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
@@ -383,8 +390,7 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 		break;
 	}
 
-	if (intel_de_wait_for_register(dev_priv, dpll_reg,
-				       port_mask, expected_mask, 1000))
+	if (intel_de_wait(dev_priv, dpll_reg, port_mask, expected_mask, 1000))
 		drm_WARN(&dev_priv->drm, 1,
 			 "timed out waiting for [ENCODER:%d:%s] port ready: got 0x%x, expected 0x%x\n",
 			 dig_port->base.base.base.id, dig_port->base.base.name,
@@ -430,6 +436,18 @@ void intel_enable_transcoder(const struct intel_crtc_state *new_crtc_state)
 		intel_de_rmw(dev_priv, PIPE_ARB_CTL(pipe),
 			     0, PIPE_ARB_USE_PROG_SLOTS);
 
+	if (DISPLAY_VER(dev_priv) >= 14) {
+		u32 clear = DP_DSC_INSERT_SF_AT_EOL_WA;
+		u32 set = 0;
+
+		if (DISPLAY_VER(dev_priv) == 14)
+			set |= DP_FEC_BS_JITTER_WA;
+
+		intel_de_rmw(dev_priv,
+			     hsw_chicken_trans_reg(dev_priv, cpu_transcoder),
+			     clear, set);
+	}
+
 	val = intel_de_read(dev_priv, TRANSCONF(cpu_transcoder));
 	if (val & TRANSCONF_ENABLE) {
 		/* we keep both pipes enabled on 830 */
@@ -437,6 +455,14 @@ void intel_enable_transcoder(const struct intel_crtc_state *new_crtc_state)
 		return;
 	}
 
+	/* Wa_1409098942:adlp+ */
+	if (DISPLAY_VER(dev_priv) >= 13 &&
+	    new_crtc_state->dsc.compression_enable) {
+		val &= ~TRANSCONF_PIXEL_COUNT_SCALING_MASK;
+		val |= REG_FIELD_PREP(TRANSCONF_PIXEL_COUNT_SCALING_MASK,
+				      TRANSCONF_PIXEL_COUNT_SCALING_X4);
+	}
+
 	intel_de_write(dev_priv, TRANSCONF(cpu_transcoder),
 		       val | TRANSCONF_ENABLE);
 	intel_de_posting_read(dev_priv, TRANSCONF(cpu_transcoder));
@@ -483,6 +509,11 @@ void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state)
 	if (!IS_I830(dev_priv))
 		val &= ~TRANSCONF_ENABLE;
 
+	/* Wa_1409098942:adlp+ */
+	if (DISPLAY_VER(dev_priv) >= 13 &&
+	    old_crtc_state->dsc.compression_enable)
+		val &= ~TRANSCONF_PIXEL_COUNT_SCALING_MASK;
+
 	intel_de_write(dev_priv, TRANSCONF(cpu_transcoder), val);
 
 	if (DISPLAY_VER(dev_priv) >= 12)
@@ -535,7 +566,7 @@ bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 
 	return DISPLAY_VER(dev_priv) < 4 ||
-		(plane->fbc &&
+		(plane->fbc && !plane_state->no_fbc_reason &&
 		 plane_state->view.gtt.type == I915_GTT_VIEW_NORMAL);
 }
 
@@ -1552,18 +1583,21 @@ static void ilk_crtc_enable(struct intel_atomic_state *state,
 	intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
 }
 
-static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
-					    enum pipe pipe, bool apply)
+/* Display WA #1180: WaDisableScalarClockGating: glk */
+static bool glk_need_scaler_clock_gating_wa(const struct intel_crtc_state *crtc_state)
 {
-	u32 val = intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe));
-	u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
 
-	if (apply)
-		val |= mask;
-	else
-		val &= ~mask;
+	return DISPLAY_VER(i915) == 10 && crtc_state->pch_pfit.enabled;
+}
 
-	intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe), val);
+static void glk_pipe_scaler_clock_gating_wa(struct intel_crtc *crtc, bool enable)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
+
+	intel_de_rmw(i915, CLKGATE_DIS_PSL(crtc->pipe),
+		     mask, enable ? mask : 0);
 }
 
 static void hsw_set_linetime_wm(const struct intel_crtc_state *crtc_state)
@@ -1586,24 +1620,6 @@ static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
 		     HSW_FRAME_START_DELAY(crtc_state->framestart_delay - 1));
 }
 
-static void icl_ddi_bigjoiner_pre_enable(struct intel_atomic_state *state,
-					 const struct intel_crtc_state *crtc_state)
-{
-	struct intel_crtc *master_crtc = intel_master_crtc(crtc_state);
-
-	/*
-	 * Enable sequence steps 1-7 on bigjoiner master
-	 */
-	if (intel_crtc_is_bigjoiner_slave(crtc_state))
-		intel_encoders_pre_pll_enable(state, master_crtc);
-
-	if (crtc_state->shared_dpll)
-		intel_enable_shared_dpll(crtc_state);
-
-	if (intel_crtc_is_bigjoiner_slave(crtc_state))
-		intel_encoders_pre_enable(state, master_crtc);
-}
-
 static void hsw_configure_cpu_transcoder(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
@@ -1639,90 +1655,107 @@ static void hsw_crtc_enable(struct intel_atomic_state *state,
 	const struct intel_crtc_state *new_crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum pipe pipe = crtc->pipe, hsw_workaround_pipe;
 	enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
-	bool psl_clkgate_wa;
+	struct intel_crtc *pipe_crtc;
 
 	if (drm_WARN_ON(&dev_priv->drm, crtc->active))
 		return;
 
-	intel_dmc_enable_pipe(dev_priv, crtc->pipe);
+	for_each_intel_crtc_in_pipe_mask_reverse(&dev_priv->drm, pipe_crtc,
+						 intel_crtc_joined_pipe_mask(new_crtc_state))
+		intel_dmc_enable_pipe(dev_priv, pipe_crtc->pipe);
 
-	if (!new_crtc_state->bigjoiner_pipes) {
-		intel_encoders_pre_pll_enable(state, crtc);
+	intel_encoders_pre_pll_enable(state, crtc);
 
-		if (new_crtc_state->shared_dpll)
-			intel_enable_shared_dpll(new_crtc_state);
+	for_each_intel_crtc_in_pipe_mask_reverse(&dev_priv->drm, pipe_crtc,
+						 intel_crtc_joined_pipe_mask(new_crtc_state)) {
+		const struct intel_crtc_state *pipe_crtc_state =
+			intel_atomic_get_new_crtc_state(state, pipe_crtc);
 
-		intel_encoders_pre_enable(state, crtc);
-	} else {
-		icl_ddi_bigjoiner_pre_enable(state, new_crtc_state);
+		if (pipe_crtc_state->shared_dpll)
+			intel_enable_shared_dpll(pipe_crtc_state);
 	}
 
-	intel_dsc_enable(new_crtc_state);
+	intel_encoders_pre_enable(state, crtc);
 
-	if (DISPLAY_VER(dev_priv) >= 13)
-		intel_uncompressed_joiner_enable(new_crtc_state);
+	for_each_intel_crtc_in_pipe_mask_reverse(&dev_priv->drm, pipe_crtc,
+						 intel_crtc_joined_pipe_mask(new_crtc_state)) {
+		const struct intel_crtc_state *pipe_crtc_state =
+			intel_atomic_get_new_crtc_state(state, pipe_crtc);
 
-	intel_set_pipe_src_size(new_crtc_state);
-	if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
-		bdw_set_pipe_misc(new_crtc_state);
+		intel_dsc_enable(pipe_crtc_state);
+
+		if (DISPLAY_VER(dev_priv) >= 13)
+			intel_uncompressed_joiner_enable(pipe_crtc_state);
+
+		intel_set_pipe_src_size(pipe_crtc_state);
 
-	if (!intel_crtc_is_bigjoiner_slave(new_crtc_state) &&
-	    !transcoder_is_dsi(cpu_transcoder))
+		if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
+			bdw_set_pipe_misc(pipe_crtc_state);
+	}
+
+	if (!transcoder_is_dsi(cpu_transcoder))
 		hsw_configure_cpu_transcoder(new_crtc_state);
 
-	crtc->active = true;
+	for_each_intel_crtc_in_pipe_mask_reverse(&dev_priv->drm, pipe_crtc,
+						 intel_crtc_joined_pipe_mask(new_crtc_state)) {
+		const struct intel_crtc_state *pipe_crtc_state =
+			intel_atomic_get_new_crtc_state(state, pipe_crtc);
 
-	/* Display WA #1180: WaDisableScalarClockGating: glk */
-	psl_clkgate_wa = DISPLAY_VER(dev_priv) == 10 &&
-		new_crtc_state->pch_pfit.enabled;
-	if (psl_clkgate_wa)
-		glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
+		pipe_crtc->active = true;
 
-	if (DISPLAY_VER(dev_priv) >= 9)
-		skl_pfit_enable(new_crtc_state);
-	else
-		ilk_pfit_enable(new_crtc_state);
+		if (glk_need_scaler_clock_gating_wa(pipe_crtc_state))
+			glk_pipe_scaler_clock_gating_wa(pipe_crtc, true);
 
-	/*
-	 * On ILK+ LUT must be loaded before the pipe is running but with
-	 * clocks enabled
-	 */
-	intel_color_load_luts(new_crtc_state);
-	intel_color_commit_noarm(new_crtc_state);
-	intel_color_commit_arm(new_crtc_state);
-	/* update DSPCNTR to configure gamma/csc for pipe bottom color */
-	if (DISPLAY_VER(dev_priv) < 9)
-		intel_disable_primary_plane(new_crtc_state);
+		if (DISPLAY_VER(dev_priv) >= 9)
+			skl_pfit_enable(pipe_crtc_state);
+		else
+			ilk_pfit_enable(pipe_crtc_state);
 
-	hsw_set_linetime_wm(new_crtc_state);
+		/*
+		 * On ILK+ LUT must be loaded before the pipe is running but with
+		 * clocks enabled
+		 */
+		intel_color_load_luts(pipe_crtc_state);
+		intel_color_commit_noarm(pipe_crtc_state);
+		intel_color_commit_arm(pipe_crtc_state);
+		/* update DSPCNTR to configure gamma/csc for pipe bottom color */
+		if (DISPLAY_VER(dev_priv) < 9)
+			intel_disable_primary_plane(pipe_crtc_state);
 
-	if (DISPLAY_VER(dev_priv) >= 11)
-		icl_set_pipe_chicken(new_crtc_state);
+		hsw_set_linetime_wm(pipe_crtc_state);
 
-	intel_initial_watermarks(state, crtc);
+		if (DISPLAY_VER(dev_priv) >= 11)
+			icl_set_pipe_chicken(pipe_crtc_state);
 
-	if (intel_crtc_is_bigjoiner_slave(new_crtc_state))
-		intel_crtc_vblank_on(new_crtc_state);
+		intel_initial_watermarks(state, pipe_crtc);
+	}
 
 	intel_encoders_enable(state, crtc);
 
-	if (psl_clkgate_wa) {
-		intel_crtc_wait_for_next_vblank(crtc);
-		glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
-	}
+	for_each_intel_crtc_in_pipe_mask_reverse(&dev_priv->drm, pipe_crtc,
+						 intel_crtc_joined_pipe_mask(new_crtc_state)) {
+		const struct intel_crtc_state *pipe_crtc_state =
+			intel_atomic_get_new_crtc_state(state, pipe_crtc);
+		enum pipe hsw_workaround_pipe;
 
-	/* If we change the relative order between pipe/planes enabling, we need
-	 * to change the workaround. */
-	hsw_workaround_pipe = new_crtc_state->hsw_workaround_pipe;
-	if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
-		struct intel_crtc *wa_crtc;
+		if (glk_need_scaler_clock_gating_wa(pipe_crtc_state)) {
+			intel_crtc_wait_for_next_vblank(pipe_crtc);
+			glk_pipe_scaler_clock_gating_wa(pipe_crtc, false);
+		}
 
-		wa_crtc = intel_crtc_for_pipe(dev_priv, hsw_workaround_pipe);
+		/*
+		 * If we change the relative order between pipe/planes
+		 * enabling, we need to change the workaround.
+		 */
+		hsw_workaround_pipe = pipe_crtc_state->hsw_workaround_pipe;
+		if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
+			struct intel_crtc *wa_crtc =
+				intel_crtc_for_pipe(dev_priv, hsw_workaround_pipe);
 
-		intel_crtc_wait_for_next_vblank(wa_crtc);
-		intel_crtc_wait_for_next_vblank(wa_crtc);
+			intel_crtc_wait_for_next_vblank(wa_crtc);
+			intel_crtc_wait_for_next_vblank(wa_crtc);
+		}
 	}
 }
 
@@ -1786,29 +1819,28 @@ static void hsw_crtc_disable(struct intel_atomic_state *state,
 	const struct intel_crtc_state *old_crtc_state =
 		intel_atomic_get_old_crtc_state(state, crtc);
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	struct intel_crtc *pipe_crtc;
 
 	/*
 	 * FIXME collapse everything to one hook.
 	 * Need care with mst->ddi interactions.
 	 */
-	if (!intel_crtc_is_bigjoiner_slave(old_crtc_state)) {
-		intel_encoders_disable(state, crtc);
-		intel_encoders_post_disable(state, crtc);
-	}
-
-	intel_disable_shared_dpll(old_crtc_state);
+	intel_encoders_disable(state, crtc);
+	intel_encoders_post_disable(state, crtc);
 
-	if (!intel_crtc_is_bigjoiner_slave(old_crtc_state)) {
-		struct intel_crtc *slave_crtc;
+	for_each_intel_crtc_in_pipe_mask(&i915->drm, pipe_crtc,
+					 intel_crtc_joined_pipe_mask(old_crtc_state)) {
+		const struct intel_crtc_state *old_pipe_crtc_state =
+			intel_atomic_get_old_crtc_state(state, pipe_crtc);
 
-		intel_encoders_post_pll_disable(state, crtc);
+		intel_disable_shared_dpll(old_pipe_crtc_state);
+	}
 
-		intel_dmc_disable_pipe(i915, crtc->pipe);
+	intel_encoders_post_pll_disable(state, crtc);
 
-		for_each_intel_crtc_in_pipe_mask(&i915->drm, slave_crtc,
-						 intel_crtc_bigjoiner_slave_pipes(old_crtc_state))
-			intel_dmc_disable_pipe(i915, slave_crtc->pipe);
-	}
+	for_each_intel_crtc_in_pipe_mask(&i915->drm, pipe_crtc,
+					 intel_crtc_joined_pipe_mask(old_crtc_state))
+		intel_dmc_disable_pipe(i915, pipe_crtc->pipe);
 }
 
 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
@@ -1836,6 +1868,7 @@ static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
 	intel_de_write(dev_priv, BCLRPAT(crtc->pipe), 0);
 }
 
+/* Prefer intel_encoder_is_combo() */
 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
 {
 	if (phy == PHY_NONE)
@@ -1857,6 +1890,7 @@ bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
 		return false;
 }
 
+/* Prefer intel_encoder_is_tc() */
 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
 {
 	/*
@@ -1877,6 +1911,7 @@ bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
 	return false;
 }
 
+/* Prefer intel_encoder_is_snps() */
 bool intel_phy_is_snps(struct drm_i915_private *dev_priv, enum phy phy)
 {
 	/*
@@ -1886,6 +1921,7 @@ bool intel_phy_is_snps(struct drm_i915_private *dev_priv, enum phy phy)
 	return IS_DG2(dev_priv) && phy > PHY_NONE && phy <= PHY_E;
 }
 
+/* Prefer intel_encoder_to_phy() */
 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
 {
 	if (DISPLAY_VER(i915) >= 13 && port >= PORT_D_XELPD)
@@ -1903,6 +1939,7 @@ enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
 	return PHY_A + port - PORT_A;
 }
 
+/* Prefer intel_encoder_to_tc() */
 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
 {
 	if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
@@ -1914,6 +1951,41 @@ enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
 		return TC_PORT_1 + port - PORT_C;
 }
 
+enum phy intel_encoder_to_phy(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+
+	return intel_port_to_phy(i915, encoder->port);
+}
+
+bool intel_encoder_is_combo(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+
+	return intel_phy_is_combo(i915, intel_encoder_to_phy(encoder));
+}
+
+bool intel_encoder_is_snps(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+
+	return intel_phy_is_snps(i915, intel_encoder_to_phy(encoder));
+}
+
+bool intel_encoder_is_tc(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+
+	return intel_phy_is_tc(i915, intel_encoder_to_phy(encoder));
+}
+
+enum tc_port intel_encoder_to_tc(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+
+	return intel_port_to_tc(i915, encoder->port);
+}
+
 enum intel_display_power_domain
 intel_aux_power_domain(struct intel_digital_port *dig_port)
 {
@@ -2381,7 +2453,7 @@ static int intel_crtc_compute_pipe_mode(struct intel_crtc_state *crtc_state)
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
 	struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
-	int clock_limit = i915->max_dotclk_freq;
+	int clock_limit = i915->display.cdclk.max_dotclk_freq;
 
 	/*
 	 * Start with the adjusted_mode crtc timings, which
@@ -2405,7 +2477,7 @@ static int intel_crtc_compute_pipe_mode(struct intel_crtc_state *crtc_state)
 		 */
 		if (intel_crtc_supports_double_wide(crtc) &&
 		    pipe_mode->crtc_clock > clock_limit) {
-			clock_limit = i915->max_dotclk_freq;
+			clock_limit = i915->display.cdclk.max_dotclk_freq;
 			crtc_state->double_wide = true;
 		}
 	}
@@ -2999,19 +3071,16 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 
 	i9xx_get_pfit_config(pipe_config);
 
+	i9xx_dpll_get_hw_state(crtc, &pipe_config->dpll_hw_state);
+
 	if (DISPLAY_VER(dev_priv) >= 4) {
-		/* No way to read it out on pipes B and C */
-		if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
-			tmp = dev_priv->display.state.chv_dpll_md[crtc->pipe];
-		else
-			tmp = intel_de_read(dev_priv, DPLL_MD(crtc->pipe));
+		tmp = pipe_config->dpll_hw_state.i9xx.dpll_md;
 		pipe_config->pixel_multiplier =
 			((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
 			 >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
-		pipe_config->dpll_hw_state.dpll_md = tmp;
 	} else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
 		   IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
-		tmp = intel_de_read(dev_priv, DPLL(crtc->pipe));
+		tmp = pipe_config->dpll_hw_state.i9xx.dpll;
 		pipe_config->pixel_multiplier =
 			((tmp & SDVO_MULTIPLIER_MASK)
 			 >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
@@ -3021,26 +3090,13 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 		 * function. */
 		pipe_config->pixel_multiplier = 1;
 	}
-	pipe_config->dpll_hw_state.dpll = intel_de_read(dev_priv,
-							DPLL(crtc->pipe));
-	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
-		pipe_config->dpll_hw_state.fp0 = intel_de_read(dev_priv,
-							       FP0(crtc->pipe));
-		pipe_config->dpll_hw_state.fp1 = intel_de_read(dev_priv,
-							       FP1(crtc->pipe));
-	} else {
-		/* Mask out read-only status bits. */
-		pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
-						     DPLL_PORTC_READY_MASK |
-						     DPLL_PORTB_READY_MASK);
-	}
 
 	if (IS_CHERRYVIEW(dev_priv))
-		chv_crtc_clock_get(crtc, pipe_config);
+		chv_crtc_clock_get(pipe_config);
 	else if (IS_VALLEYVIEW(dev_priv))
-		vlv_crtc_clock_get(crtc, pipe_config);
+		vlv_crtc_clock_get(pipe_config);
 	else
-		i9xx_crtc_clock_get(crtc, pipe_config);
+		i9xx_crtc_clock_get(pipe_config);
 
 	/*
 	 * Normally the dotclock is filled in by the encoder .get_config()
@@ -3666,8 +3722,8 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
 					 struct intel_crtc_state *pipe_config,
 					 struct intel_display_power_domain_set *power_domain_set)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_display *display = to_intel_display(crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum transcoder cpu_transcoder;
 	enum port port;
 	u32 tmp;
@@ -3693,11 +3749,11 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
 			break;
 
 		/* XXX: this works for video mode only */
-		tmp = intel_de_read(dev_priv, BXT_MIPI_PORT_CTRL(port));
+		tmp = intel_de_read(display, BXT_MIPI_PORT_CTRL(port));
 		if (!(tmp & DPI_ENABLE))
 			continue;
 
-		tmp = intel_de_read(dev_priv, MIPI_CTRL(port));
+		tmp = intel_de_read(display, MIPI_CTRL(display, port));
 		if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
 			continue;
 
@@ -4714,8 +4770,6 @@ intel_modeset_pipe_config_late(struct intel_atomic_state *state,
 	struct drm_connector *connector;
 	int i;
 
-	intel_bigjoiner_adjust_pipe_src(crtc_state);
-
 	for_each_new_connector_in_state(&state->base, connector,
 					conn_state, i) {
 		struct intel_encoder *encoder =
@@ -4783,42 +4837,92 @@ intel_compare_dp_vsc_sdp(const struct drm_dp_vsc_sdp *a,
 }
 
 static bool
+intel_compare_dp_as_sdp(const struct drm_dp_as_sdp *a,
+			const struct drm_dp_as_sdp *b)
+{
+	return a->vtotal == b->vtotal &&
+		a->target_rr == b->target_rr &&
+		a->duration_incr_ms == b->duration_incr_ms &&
+		a->duration_decr_ms == b->duration_decr_ms &&
+		a->mode == b->mode;
+}
+
+static bool
 intel_compare_buffer(const u8 *a, const u8 *b, size_t len)
 {
 	return memcmp(a, b, len) == 0;
 }
 
+static void __printf(5, 6)
+pipe_config_mismatch(struct drm_printer *p, bool fastset,
+		     const struct intel_crtc *crtc,
+		     const char *name, const char *format, ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	va_start(args, format);
+	vaf.fmt = format;
+	vaf.va = &args;
+
+	if (fastset)
+		drm_printf(p, "[CRTC:%d:%s] fastset requirement not met in %s %pV\n",
+			   crtc->base.base.id, crtc->base.name, name, &vaf);
+	else
+		drm_printf(p, "[CRTC:%d:%s] mismatch in %s %pV\n",
+			   crtc->base.base.id, crtc->base.name, name, &vaf);
+
+	va_end(args);
+}
+
 static void
-pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
-			       bool fastset, const char *name,
+pipe_config_infoframe_mismatch(struct drm_printer *p, bool fastset,
+			       const struct intel_crtc *crtc,
+			       const char *name,
 			       const union hdmi_infoframe *a,
 			       const union hdmi_infoframe *b)
 {
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	const char *loglevel;
+
 	if (fastset) {
 		if (!drm_debug_enabled(DRM_UT_KMS))
 			return;
 
-		drm_dbg_kms(&dev_priv->drm,
-			    "fastset requirement not met in %s infoframe\n", name);
-		drm_dbg_kms(&dev_priv->drm, "expected:\n");
-		hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
-		drm_dbg_kms(&dev_priv->drm, "found:\n");
-		hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
+		loglevel = KERN_DEBUG;
 	} else {
-		drm_err(&dev_priv->drm, "mismatch in %s infoframe\n", name);
-		drm_err(&dev_priv->drm, "expected:\n");
-		hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
-		drm_err(&dev_priv->drm, "found:\n");
-		hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
+		loglevel = KERN_ERR;
 	}
+
+	pipe_config_mismatch(p, fastset, crtc, name, "infoframe");
+
+	drm_printf(p, "expected:\n");
+	hdmi_infoframe_log(loglevel, i915->drm.dev, a);
+	drm_printf(p, "found:\n");
+	hdmi_infoframe_log(loglevel, i915->drm.dev, b);
 }
 
 static void
-pipe_config_dp_vsc_sdp_mismatch(struct drm_i915_private *i915,
-				bool fastset, const char *name,
+pipe_config_dp_vsc_sdp_mismatch(struct drm_printer *p, bool fastset,
+				const struct intel_crtc *crtc,
+				const char *name,
 				const struct drm_dp_vsc_sdp *a,
 				const struct drm_dp_vsc_sdp *b)
 {
+	pipe_config_mismatch(p, fastset, crtc, name, "dp sdp");
+
+	drm_printf(p, "expected:\n");
+	drm_dp_vsc_sdp_log(p, a);
+	drm_printf(p, "found:\n");
+	drm_dp_vsc_sdp_log(p, b);
+}
+
+static void
+pipe_config_dp_as_sdp_mismatch(struct drm_i915_private *i915,
+			       bool fastset, const char *name,
+			       const struct drm_dp_as_sdp *a,
+			       const struct drm_dp_as_sdp *b)
+{
 	struct drm_printer p;
 
 	if (fastset) {
@@ -4832,9 +4936,9 @@ pipe_config_dp_vsc_sdp_mismatch(struct drm_i915_private *i915,
 	}
 
 	drm_printf(&p, "expected:\n");
-	drm_dp_vsc_sdp_log(&p, a);
+	drm_dp_as_sdp_log(&p, a);
 	drm_printf(&p, "found:\n");
-	drm_dp_vsc_sdp_log(&p, b);
+	drm_dp_as_sdp_log(&p, b);
 }
 
 /* Returns the length up to and including the last differing byte */
@@ -4852,64 +4956,35 @@ memcmp_diff_len(const u8 *a, const u8 *b, size_t len)
 }
 
 static void
-pipe_config_buffer_mismatch(bool fastset, const struct intel_crtc *crtc,
+pipe_config_buffer_mismatch(struct drm_printer *p, bool fastset,
+			    const struct intel_crtc *crtc,
 			    const char *name,
 			    const u8 *a, const u8 *b, size_t len)
 {
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	const char *loglevel;
 
 	if (fastset) {
 		if (!drm_debug_enabled(DRM_UT_KMS))
 			return;
 
-		/* only dump up to the last difference */
-		len = memcmp_diff_len(a, b, len);
-
-		drm_dbg_kms(&dev_priv->drm,
-			    "[CRTC:%d:%s] fastset requirement not met in %s buffer\n",
-			    crtc->base.base.id, crtc->base.name, name);
-		print_hex_dump(KERN_DEBUG, "expected: ", DUMP_PREFIX_NONE,
-			       16, 0, a, len, false);
-		print_hex_dump(KERN_DEBUG, "found: ", DUMP_PREFIX_NONE,
-			       16, 0, b, len, false);
+		loglevel = KERN_DEBUG;
 	} else {
-		/* only dump up to the last difference */
-		len = memcmp_diff_len(a, b, len);
-
-		drm_err(&dev_priv->drm, "[CRTC:%d:%s] mismatch in %s buffer\n",
-			crtc->base.base.id, crtc->base.name, name);
-		print_hex_dump(KERN_ERR, "expected: ", DUMP_PREFIX_NONE,
-			       16, 0, a, len, false);
-		print_hex_dump(KERN_ERR, "found: ", DUMP_PREFIX_NONE,
-			       16, 0, b, len, false);
+		loglevel = KERN_ERR;
 	}
-}
 
-static void __printf(4, 5)
-pipe_config_mismatch(bool fastset, const struct intel_crtc *crtc,
-		     const char *name, const char *format, ...)
-{
-	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
-	struct va_format vaf;
-	va_list args;
+	pipe_config_mismatch(p, fastset, crtc, name, "buffer");
 
-	va_start(args, format);
-	vaf.fmt = format;
-	vaf.va = &args;
+	/* only dump up to the last difference */
+	len = memcmp_diff_len(a, b, len);
 
-	if (fastset)
-		drm_dbg_kms(&i915->drm,
-			    "[CRTC:%d:%s] fastset requirement not met in %s %pV\n",
-			    crtc->base.base.id, crtc->base.name, name, &vaf);
-	else
-		drm_err(&i915->drm, "[CRTC:%d:%s] mismatch in %s %pV\n",
-			crtc->base.base.id, crtc->base.name, name, &vaf);
-
-	va_end(args);
+	print_hex_dump(loglevel, "expected: ", DUMP_PREFIX_NONE,
+		       16, 0, a, len, false);
+	print_hex_dump(loglevel, "found: ", DUMP_PREFIX_NONE,
+		       16, 0, b, len, false);
 }
 
 static void
-pipe_config_pll_mismatch(bool fastset,
+pipe_config_pll_mismatch(struct drm_printer *p, bool fastset,
 			 const struct intel_crtc *crtc,
 			 const char *name,
 			 const struct intel_dpll_hw_state *a,
@@ -4917,25 +4992,12 @@ pipe_config_pll_mismatch(bool fastset,
 {
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 
-	if (fastset) {
-		if (!drm_debug_enabled(DRM_UT_KMS))
-			return;
+	pipe_config_mismatch(p, fastset, crtc, name, " "); /* stupid -Werror=format-zero-length */
 
-		drm_dbg_kms(&i915->drm,
-			    "[CRTC:%d:%s] fastset requirement not met in %s\n",
-			    crtc->base.base.id, crtc->base.name, name);
-		drm_dbg_kms(&i915->drm, "expected:\n");
-		intel_dpll_dump_hw_state(i915, a);
-		drm_dbg_kms(&i915->drm, "found:\n");
-		intel_dpll_dump_hw_state(i915, b);
-	} else {
-		drm_err(&i915->drm, "[CRTC:%d:%s] mismatch in %s buffer\n",
-			crtc->base.base.id, crtc->base.name, name);
-		drm_err(&i915->drm, "expected:\n");
-		intel_dpll_dump_hw_state(i915, a);
-		drm_err(&i915->drm, "found:\n");
-		intel_dpll_dump_hw_state(i915, b);
-	}
+	drm_printf(p, "expected:\n");
+	intel_dpll_dump_hw_state(i915, p, a);
+	drm_printf(p, "found:\n");
+	intel_dpll_dump_hw_state(i915, p, b);
 }
 
 bool
@@ -4945,13 +5007,19 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 {
 	struct drm_i915_private *dev_priv = to_i915(current_config->uapi.crtc->dev);
 	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
+	struct drm_printer p;
 	bool ret = true;
 
+	if (fastset)
+		p = drm_dbg_printer(&dev_priv->drm, DRM_UT_KMS, NULL);
+	else
+		p = drm_err_printer(&dev_priv->drm, NULL);
+
 #define PIPE_CONF_CHECK_X(name) do { \
 	if (current_config->name != pipe_config->name) { \
 		BUILD_BUG_ON_MSG(__same_type(current_config->name, bool), \
 				 __stringify(name) " is bool");	\
-		pipe_config_mismatch(fastset, crtc, __stringify(name), \
+		pipe_config_mismatch(&p, fastset, crtc, __stringify(name), \
 				     "(expected 0x%08x, found 0x%08x)", \
 				     current_config->name, \
 				     pipe_config->name); \
@@ -4963,7 +5031,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	if ((current_config->name & (mask)) != (pipe_config->name & (mask))) { \
 		BUILD_BUG_ON_MSG(__same_type(current_config->name, bool), \
 				 __stringify(name) " is bool");	\
-		pipe_config_mismatch(fastset, crtc, __stringify(name), \
+		pipe_config_mismatch(&p, fastset, crtc, __stringify(name), \
 				     "(expected 0x%08x, found 0x%08x)", \
 				     current_config->name & (mask), \
 				     pipe_config->name & (mask)); \
@@ -4975,7 +5043,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	if (current_config->name != pipe_config->name) { \
 		BUILD_BUG_ON_MSG(__same_type(current_config->name, bool), \
 				 __stringify(name) " is bool");	\
-		pipe_config_mismatch(fastset, crtc, __stringify(name), \
+		pipe_config_mismatch(&p, fastset, crtc, __stringify(name), \
 				     "(expected %i, found %i)", \
 				     current_config->name, \
 				     pipe_config->name); \
@@ -4987,7 +5055,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	if (current_config->name != pipe_config->name) { \
 		BUILD_BUG_ON_MSG(!__same_type(current_config->name, bool), \
 				 __stringify(name) " is not bool");	\
-		pipe_config_mismatch(fastset, crtc,  __stringify(name), \
+		pipe_config_mismatch(&p, fastset, crtc,  __stringify(name), \
 				     "(expected %s, found %s)", \
 				     str_yes_no(current_config->name), \
 				     str_yes_no(pipe_config->name)); \
@@ -4997,7 +5065,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 
 #define PIPE_CONF_CHECK_P(name) do { \
 	if (current_config->name != pipe_config->name) { \
-		pipe_config_mismatch(fastset, crtc, __stringify(name), \
+		pipe_config_mismatch(&p, fastset, crtc, __stringify(name), \
 				     "(expected %p, found %p)", \
 				     current_config->name, \
 				     pipe_config->name); \
@@ -5008,7 +5076,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 #define PIPE_CONF_CHECK_M_N(name) do { \
 	if (!intel_compare_link_m_n(&current_config->name, \
 				    &pipe_config->name)) { \
-		pipe_config_mismatch(fastset, crtc, __stringify(name), \
+		pipe_config_mismatch(&p, fastset, crtc, __stringify(name), \
 				     "(expected tu %i data %i/%i link %i/%i, " \
 				     "found tu %i, data %i/%i link %i/%i)", \
 				     current_config->name.tu, \
@@ -5028,7 +5096,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 #define PIPE_CONF_CHECK_PLL(name) do { \
 	if (!intel_dpll_compare_hw_state(dev_priv, &current_config->name, \
 					 &pipe_config->name)) { \
-		pipe_config_pll_mismatch(fastset, crtc, __stringify(name), \
+		pipe_config_pll_mismatch(&p, fastset, crtc, __stringify(name), \
 					 &current_config->name, \
 					 &pipe_config->name); \
 		ret = false; \
@@ -5061,7 +5129,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 
 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
 	if ((current_config->name ^ pipe_config->name) & (mask)) { \
-		pipe_config_mismatch(fastset, crtc, __stringify(name), \
+		pipe_config_mismatch(&p, fastset, crtc, __stringify(name), \
 				     "(%x) (expected %i, found %i)", \
 				     (mask), \
 				     current_config->name & (mask), \
@@ -5073,7 +5141,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
 	if (!intel_compare_infoframe(&current_config->infoframes.name, \
 				     &pipe_config->infoframes.name)) { \
-		pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
+		pipe_config_infoframe_mismatch(&p, fastset, crtc, __stringify(name), \
 					       &current_config->infoframes.name, \
 					       &pipe_config->infoframes.name); \
 		ret = false; \
@@ -5083,7 +5151,17 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 #define PIPE_CONF_CHECK_DP_VSC_SDP(name) do { \
 	if (!intel_compare_dp_vsc_sdp(&current_config->infoframes.name, \
 				      &pipe_config->infoframes.name)) { \
-		pipe_config_dp_vsc_sdp_mismatch(dev_priv, fastset, __stringify(name), \
+		pipe_config_dp_vsc_sdp_mismatch(&p, fastset, crtc, __stringify(name), \
+						&current_config->infoframes.name, \
+						&pipe_config->infoframes.name); \
+		ret = false; \
+	} \
+} while (0)
+
+#define PIPE_CONF_CHECK_DP_AS_SDP(name) do { \
+	if (!intel_compare_dp_as_sdp(&current_config->infoframes.name, \
+				      &pipe_config->infoframes.name)) { \
+		pipe_config_dp_as_sdp_mismatch(dev_priv, fastset, __stringify(name), \
 						&current_config->infoframes.name, \
 						&pipe_config->infoframes.name); \
 		ret = false; \
@@ -5094,7 +5172,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	BUILD_BUG_ON(sizeof(current_config->name) != (len)); \
 	BUILD_BUG_ON(sizeof(pipe_config->name) != (len)); \
 	if (!intel_compare_buffer(current_config->name, pipe_config->name, (len))) { \
-		pipe_config_buffer_mismatch(fastset, crtc, __stringify(name), \
+		pipe_config_buffer_mismatch(&p, fastset, crtc, __stringify(name), \
 					    current_config->name, \
 					    pipe_config->name, \
 					    (len)); \
@@ -5107,7 +5185,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	    !intel_color_lut_equal(current_config, \
 				   current_config->lut, pipe_config->lut, \
 				   is_pre_csc_lut)) {	\
-		pipe_config_mismatch(fastset, crtc, __stringify(lut), \
+		pipe_config_mismatch(&p, fastset, crtc, __stringify(lut), \
 				     "hw_state doesn't match sw_state"); \
 		ret = false; \
 	} \
@@ -5236,6 +5314,18 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 		PIPE_CONF_CHECK_CSC(output_csc);
 	}
 
+	/*
+	 * Panel replay has to be enabled before link training. PSR doesn't have
+	 * this requirement -> check these only if using panel replay
+	 */
+	if (current_config->has_panel_replay || pipe_config->has_panel_replay) {
+		PIPE_CONF_CHECK_BOOL(has_psr);
+		PIPE_CONF_CHECK_BOOL(has_psr2);
+		PIPE_CONF_CHECK_BOOL(enable_psr2_sel_fetch);
+		PIPE_CONF_CHECK_BOOL(enable_psr2_su_region_et);
+		PIPE_CONF_CHECK_BOOL(has_panel_replay);
+	}
+
 	PIPE_CONF_CHECK_BOOL(double_wide);
 
 	if (dev_priv->display.dpll.mgr)
@@ -5271,6 +5361,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	PIPE_CONF_CHECK_INFOFRAME(hdmi);
 	PIPE_CONF_CHECK_INFOFRAME(drm);
 	PIPE_CONF_CHECK_DP_VSC_SDP(vsc);
+	PIPE_CONF_CHECK_DP_AS_SDP(as_sdp);
 
 	PIPE_CONF_CHECK_X(sync_mode_slaves_mask);
 	PIPE_CONF_CHECK_I(master_transcoder);
@@ -5322,6 +5413,8 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 		PIPE_CONF_CHECK_I(vrr.flipline);
 		PIPE_CONF_CHECK_I(vrr.pipeline_full);
 		PIPE_CONF_CHECK_I(vrr.guardband);
+		PIPE_CONF_CHECK_I(vrr.vsync_start);
+		PIPE_CONF_CHECK_I(vrr.vsync_end);
 	}
 
 #undef PIPE_CONF_CHECK_X
@@ -5567,14 +5660,16 @@ static int intel_modeset_checks(struct intel_atomic_state *state)
 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
 				     struct intel_crtc_state *new_crtc_state)
 {
-	struct drm_i915_private *i915 = to_i915(old_crtc_state->uapi.crtc->dev);
+	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 
 	/* only allow LRR when the timings stay within the VRR range */
 	if (old_crtc_state->vrr.in_range != new_crtc_state->vrr.in_range)
 		new_crtc_state->update_lrr = false;
 
 	if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
-		drm_dbg_kms(&i915->drm, "fastset requirement not met, forcing full modeset\n");
+		drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] fastset requirement not met, forcing full modeset\n",
+			    crtc->base.base.id, crtc->base.name);
 	else
 		new_crtc_state->uapi.mode_changed = false;
 
@@ -6228,27 +6323,37 @@ static int intel_atomic_check_config(struct intel_atomic_state *state,
 			continue;
 		}
 
-		if (intel_crtc_is_bigjoiner_slave(new_crtc_state)) {
-			drm_WARN_ON(&i915->drm, new_crtc_state->uapi.enable);
+		if (drm_WARN_ON(&i915->drm, intel_crtc_is_bigjoiner_slave(new_crtc_state)))
 			continue;
-		}
 
 		ret = intel_crtc_prepare_cleared_state(state, crtc);
 		if (ret)
-			break;
+			goto fail;
 
 		if (!new_crtc_state->hw.enable)
 			continue;
 
 		ret = intel_modeset_pipe_config(state, crtc, limits);
 		if (ret)
-			break;
+			goto fail;
+	}
 
-		ret = intel_atomic_check_bigjoiner(state, crtc);
+	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
+		if (!intel_crtc_needs_modeset(new_crtc_state))
+			continue;
+
+		if (drm_WARN_ON(&i915->drm, intel_crtc_is_bigjoiner_slave(new_crtc_state)))
+			continue;
+
+		if (!new_crtc_state->hw.enable)
+			continue;
+
+		ret = intel_modeset_pipe_config_late(state, crtc);
 		if (ret)
-			break;
+			goto fail;
 	}
 
+fail:
 	if (ret)
 		*failed_pipe = crtc->pipe;
 
@@ -6344,16 +6449,26 @@ int intel_atomic_check(struct drm_device *dev,
 	if (ret)
 		goto fail;
 
+	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
+		if (!intel_crtc_needs_modeset(new_crtc_state))
+			continue;
+
+		if (intel_crtc_is_bigjoiner_slave(new_crtc_state)) {
+			drm_WARN_ON(&dev_priv->drm, new_crtc_state->uapi.enable);
+			continue;
+		}
+
+		ret = intel_atomic_check_bigjoiner(state, crtc);
+		if (ret)
+			goto fail;
+	}
+
 	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
 					    new_crtc_state, i) {
 		if (!intel_crtc_needs_modeset(new_crtc_state))
 			continue;
 
-		if (new_crtc_state->hw.enable) {
-			ret = intel_modeset_pipe_config_late(state, crtc);
-			if (ret)
-				goto fail;
-		}
+		intel_bigjoiner_adjust_pipe_src(new_crtc_state);
 
 		intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
 	}
@@ -6635,17 +6750,21 @@ static void intel_enable_crtc(struct intel_atomic_state *state,
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 	const struct intel_crtc_state *new_crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
+	struct intel_crtc *pipe_crtc;
 
 	if (!intel_crtc_needs_modeset(new_crtc_state))
 		return;
 
-	/* VRR will be enable later, if required */
-	intel_crtc_update_active_timings(new_crtc_state, false);
+	for_each_intel_crtc_in_pipe_mask_reverse(&dev_priv->drm, pipe_crtc,
+						 intel_crtc_joined_pipe_mask(new_crtc_state)) {
+		const struct intel_crtc_state *pipe_crtc_state =
+			intel_atomic_get_new_crtc_state(state, pipe_crtc);
 
-	dev_priv->display.funcs.display->crtc_enable(state, crtc);
+		/* VRR will be enable later, if required */
+		intel_crtc_update_active_timings(pipe_crtc_state, false);
+	}
 
-	if (intel_crtc_is_bigjoiner_slave(new_crtc_state))
-		return;
+	dev_priv->display.funcs.display->crtc_enable(state, crtc);
 
 	/* vblanks work again, re-enable pipe CRC. */
 	intel_crtc_enable_pipe_crc(crtc);
@@ -6737,31 +6856,42 @@ static void intel_update_crtc(struct intel_atomic_state *state,
 }
 
 static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
-					  struct intel_crtc_state *old_crtc_state,
-					  struct intel_crtc_state *new_crtc_state,
 					  struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
+	const struct intel_crtc_state *old_crtc_state =
+		intel_atomic_get_old_crtc_state(state, crtc);
+	struct intel_crtc *pipe_crtc;
 
 	/*
 	 * We need to disable pipe CRC before disabling the pipe,
 	 * or we race against vblank off.
 	 */
-	intel_crtc_disable_pipe_crc(crtc);
+	for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, pipe_crtc,
+					 intel_crtc_joined_pipe_mask(old_crtc_state))
+		intel_crtc_disable_pipe_crc(pipe_crtc);
 
 	dev_priv->display.funcs.display->crtc_disable(state, crtc);
-	crtc->active = false;
-	intel_fbc_disable(crtc);
 
-	if (!new_crtc_state->hw.active)
-		intel_initial_watermarks(state, crtc);
+	for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, pipe_crtc,
+					 intel_crtc_joined_pipe_mask(old_crtc_state)) {
+		const struct intel_crtc_state *new_pipe_crtc_state =
+			intel_atomic_get_new_crtc_state(state, pipe_crtc);
+
+		pipe_crtc->active = false;
+		intel_fbc_disable(pipe_crtc);
+
+		if (!new_pipe_crtc_state->hw.active)
+			intel_initial_watermarks(state, pipe_crtc);
+	}
 }
 
 static void intel_commit_modeset_disables(struct intel_atomic_state *state)
 {
-	struct intel_crtc_state *new_crtc_state, *old_crtc_state;
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
 	struct intel_crtc *crtc;
-	u32 handled = 0;
+	u8 disable_pipes = 0;
 	int i;
 
 	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
@@ -6769,21 +6899,31 @@ static void intel_commit_modeset_disables(struct intel_atomic_state *state)
 		if (!intel_crtc_needs_modeset(new_crtc_state))
 			continue;
 
+		/*
+		 * Needs to be done even for pipes
+		 * that weren't enabled previously.
+		 */
 		intel_pre_plane_update(state, crtc);
 
 		if (!old_crtc_state->hw.active)
 			continue;
 
+		disable_pipes |= BIT(crtc->pipe);
+	}
+
+	for_each_old_intel_crtc_in_state(state, crtc, old_crtc_state, i) {
+		if ((disable_pipes & BIT(crtc->pipe)) == 0)
+			continue;
+
 		intel_crtc_disable_planes(state, crtc);
 	}
 
 	/* Only disable port sync and MST slaves */
-	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
-					    new_crtc_state, i) {
-		if (!intel_crtc_needs_modeset(new_crtc_state))
+	for_each_old_intel_crtc_in_state(state, crtc, old_crtc_state, i) {
+		if ((disable_pipes & BIT(crtc->pipe)) == 0)
 			continue;
 
-		if (!old_crtc_state->hw.active)
+		if (intel_crtc_is_bigjoiner_slave(old_crtc_state))
 			continue;
 
 		/* In case of Transcoder port Sync master slave CRTCs can be
@@ -6792,28 +6932,28 @@ static void intel_commit_modeset_disables(struct intel_atomic_state *state)
 		 * Slave vblanks are masked till Master Vblanks.
 		 */
 		if (!is_trans_port_sync_slave(old_crtc_state) &&
-		    !intel_dp_mst_is_slave_trans(old_crtc_state) &&
-		    !intel_crtc_is_bigjoiner_slave(old_crtc_state))
+		    !intel_dp_mst_is_slave_trans(old_crtc_state))
 			continue;
 
-		intel_old_crtc_state_disables(state, old_crtc_state,
-					      new_crtc_state, crtc);
-		handled |= BIT(crtc->pipe);
+		intel_old_crtc_state_disables(state, crtc);
+
+		disable_pipes &= ~intel_crtc_joined_pipe_mask(old_crtc_state);
 	}
 
 	/* Disable everything else left on */
-	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
-					    new_crtc_state, i) {
-		if (!intel_crtc_needs_modeset(new_crtc_state) ||
-		    (handled & BIT(crtc->pipe)))
+	for_each_old_intel_crtc_in_state(state, crtc, old_crtc_state, i) {
+		if ((disable_pipes & BIT(crtc->pipe)) == 0)
 			continue;
 
-		if (!old_crtc_state->hw.active)
+		if (intel_crtc_is_bigjoiner_slave(old_crtc_state))
 			continue;
 
-		intel_old_crtc_state_disables(state, old_crtc_state,
-					      new_crtc_state, crtc);
+		intel_old_crtc_state_disables(state, crtc);
+
+		disable_pipes &= ~intel_crtc_joined_pipe_mask(old_crtc_state);
 	}
+
+	drm_WARN_ON(&i915->drm, disable_pipes);
 }
 
 static void intel_commit_modeset_enables(struct intel_atomic_state *state)
@@ -6880,9 +7020,15 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state)
 		intel_pre_update_crtc(state, crtc);
 	}
 
+	intel_dbuf_mbus_pre_ddb_update(state);
+
 	while (update_pipes) {
-		for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
-						    new_crtc_state, i) {
+		/*
+		 * Commit in reverse order to make bigjoiner master
+		 * send the uapi events after slaves are done.
+		 */
+		for_each_oldnew_intel_crtc_in_state_reverse(state, crtc, old_crtc_state,
+							    new_crtc_state, i) {
 			enum pipe pipe = crtc->pipe;
 
 			if ((update_pipes & BIT(pipe)) == 0)
@@ -6910,6 +7056,8 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state)
 		}
 	}
 
+	intel_dbuf_mbus_post_ddb_update(state);
+
 	update_pipes = modeset_pipes;
 
 	/*
@@ -6922,12 +7070,14 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state)
 		if ((modeset_pipes & BIT(pipe)) == 0)
 			continue;
 
+		if (intel_crtc_is_bigjoiner_slave(new_crtc_state))
+			continue;
+
 		if (intel_dp_mst_is_slave_trans(new_crtc_state) ||
-		    is_trans_port_sync_master(new_crtc_state) ||
-		    intel_crtc_is_bigjoiner_master(new_crtc_state))
+		    is_trans_port_sync_master(new_crtc_state))
 			continue;
 
-		modeset_pipes &= ~BIT(pipe);
+		modeset_pipes &= ~intel_crtc_joined_pipe_mask(new_crtc_state);
 
 		intel_enable_crtc(state, crtc);
 	}
@@ -6942,7 +7092,10 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state)
 		if ((modeset_pipes & BIT(pipe)) == 0)
 			continue;
 
-		modeset_pipes &= ~BIT(pipe);
+		if (intel_crtc_is_bigjoiner_slave(new_crtc_state))
+			continue;
+
+		modeset_pipes &= ~intel_crtc_joined_pipe_mask(new_crtc_state);
 
 		intel_enable_crtc(state, crtc);
 	}
@@ -6959,7 +7112,11 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state)
 		intel_pre_update_crtc(state, crtc);
 	}
 
-	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
+	/*
+	 * Commit in reverse order to make bigjoiner master
+	 * send the uapi events after slaves are done.
+	 */
+	for_each_new_intel_crtc_in_state_reverse(state, crtc, new_crtc_state, i) {
 		enum pipe pipe = crtc->pipe;
 
 		if ((update_pipes & BIT(pipe)) == 0)
@@ -7156,7 +7313,6 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state)
 	intel_encoders_update_prepare(state);
 
 	intel_dbuf_pre_plane_update(state);
-	intel_mbus_dbox_update(state);
 
 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
 		if (new_crtc_state->do_async_flip)
@@ -7681,7 +7837,7 @@ void intel_setup_outputs(struct drm_i915_private *dev_priv)
 
 static int max_dotclock(struct drm_i915_private *i915)
 {
-	int max_dotclock = i915->max_dotclk_freq;
+	int max_dotclock = i915->display.cdclk.max_dotclk_freq;
 
 	/* icl+ might use bigjoiner */
 	if (DISPLAY_VER(i915) >= 11)
diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h
index f4a0773f0fca..56d1c0e3e62c 100644
--- a/drivers/gpu/drm/i915/display/intel_display.h
+++ b/drivers/gpu/drm/i915/display/intel_display.h
@@ -280,6 +280,12 @@ enum phy_fia {
 			    base.head)					\
 		for_each_if((pipe_mask) & BIT(intel_crtc->pipe))
 
+#define for_each_intel_crtc_in_pipe_mask_reverse(dev, intel_crtc, pipe_mask)	\
+	list_for_each_entry_reverse((intel_crtc),				\
+				    &(dev)->mode_config.crtc_list,		\
+				    base.head)					\
+		for_each_if((pipe_mask) & BIT((intel_crtc)->pipe))
+
 #define for_each_intel_encoder(dev, intel_encoder)		\
 	list_for_each_entry(intel_encoder,			\
 			    &(dev)->mode_config.encoder_list,	\
@@ -344,6 +350,14 @@ enum phy_fia {
 	     (__i)++) \
 		for_each_if(crtc)
 
+#define for_each_new_intel_crtc_in_state_reverse(__state, crtc, new_crtc_state, __i) \
+	for ((__i) = (__state)->base.dev->mode_config.num_crtc - 1; \
+	     (__i) >= 0  && \
+	     ((crtc) = to_intel_crtc((__state)->base.crtcs[__i].ptr), \
+	      (new_crtc_state) = to_intel_crtc_state((__state)->base.crtcs[__i].new_state), 1); \
+	     (__i)--) \
+		for_each_if(crtc)
+
 #define for_each_oldnew_intel_plane_in_state(__state, plane, old_plane_state, new_plane_state, __i) \
 	for ((__i) = 0; \
 	     (__i) < (__state)->base.dev->mode_config.num_total_plane && \
@@ -408,6 +422,7 @@ intel_cpu_transcoder_mode_valid(struct drm_i915_private *i915,
 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port);
 bool is_trans_port_sync_mode(const struct intel_crtc_state *state);
 bool is_trans_port_sync_master(const struct intel_crtc_state *state);
+u8 intel_crtc_joined_pipe_mask(const struct intel_crtc_state *crtc_state);
 bool intel_crtc_is_bigjoiner_slave(const struct intel_crtc_state *crtc_state);
 bool intel_crtc_is_bigjoiner_master(const struct intel_crtc_state *crtc_state);
 u8 intel_crtc_bigjoiner_slave_pipes(const struct intel_crtc_state *crtc_state);
@@ -448,6 +463,13 @@ bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy);
 bool intel_phy_is_snps(struct drm_i915_private *dev_priv, enum phy phy);
 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv,
 			      enum port port);
+
+enum phy intel_encoder_to_phy(struct intel_encoder *encoder);
+bool intel_encoder_is_combo(struct intel_encoder *encoder);
+bool intel_encoder_is_snps(struct intel_encoder *encoder);
+bool intel_encoder_is_tc(struct intel_encoder *encoder);
+enum tc_port intel_encoder_to_tc(struct intel_encoder *encoder);
+
 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
 				      struct drm_file *file_priv);
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_conversion.h b/drivers/gpu/drm/i915/display/intel_display_conversion.h
new file mode 100644
index 000000000000..ad8545c8055d
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_display_conversion.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2024 Intel Corporation */
+
+/*
+ * This header is for transitional struct intel_display conversion helpers only.
+ */
+
+#ifndef __INTEL_DISPLAY_CONVERSION__
+#define __INTEL_DISPLAY_CONVERSION__
+
+/*
+ * Transitional macro to optionally convert struct drm_i915_private * to struct
+ * intel_display *, also accepting the latter.
+ */
+#define __to_intel_display(p)						\
+	_Generic(p,							\
+		 const struct drm_i915_private *: (&((const struct drm_i915_private *)(p))->display), \
+		 struct drm_i915_private *: (&((struct drm_i915_private *)(p))->display), \
+		 const struct intel_display *: (p),			\
+		 struct intel_display *: (p))
+
+#endif /* __INTEL_DISPLAY_CONVERSION__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_core.h b/drivers/gpu/drm/i915/display/intel_display_core.h
index 2167dbee5eea..7715fc329057 100644
--- a/drivers/gpu/drm/i915/display/intel_display_core.h
+++ b/drivers/gpu/drm/i915/display/intel_display_core.h
@@ -26,6 +26,7 @@
 #include "intel_global_state.h"
 #include "intel_gmbus.h"
 #include "intel_opregion.h"
+#include "intel_dmc_wl.h"
 #include "intel_wm_types.h"
 
 struct task_struct;
@@ -282,6 +283,9 @@ struct intel_wm {
 };
 
 struct intel_display {
+	/* drm device backpointer */
+	struct drm_device *drm;
+
 	/* Display functions */
 	struct {
 		/* Top level crtc-ish functions */
@@ -345,6 +349,8 @@ struct intel_display {
 		struct intel_global_obj obj;
 
 		unsigned int max_cdclk_freq;
+		unsigned int max_dotclk_freq;
+		unsigned int skl_preferred_vco_freq;
 	} cdclk;
 
 	struct {
@@ -446,6 +452,16 @@ struct intel_display {
 	} ips;
 
 	struct {
+		bool display_irqs_enabled;
+
+		/* For i915gm/i945gm vblank irq workaround */
+		u8 vblank_enabled;
+
+		u32 de_irq_mask[I915_MAX_PIPES];
+		u32 pipestat_irq_mask[I915_MAX_PIPES];
+	} irq;
+
+	struct {
 		wait_queue_head_t waitqueue;
 
 		/* mutex to protect pmdemand programming sequence */
@@ -534,6 +550,7 @@ struct intel_display {
 	struct intel_overlay *overlay;
 	struct intel_display_params params;
 	struct intel_vbt_data vbt;
+	struct intel_dmc_wl wl;
 	struct intel_wm wm;
 };
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
index b99c024b0934..35f9f86ef70f 100644
--- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c
+++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
@@ -31,6 +31,7 @@
 #include "intel_hdmi.h"
 #include "intel_hotplug.h"
 #include "intel_panel.h"
+#include "intel_pps.h"
 #include "intel_psr.h"
 #include "intel_psr_regs.h"
 #include "intel_wm.h"
@@ -191,7 +192,7 @@ static void intel_hdcp_info(struct seq_file *m,
 			    struct intel_connector *intel_connector,
 			    bool remote_req)
 {
-	bool hdcp_cap, hdcp2_cap;
+	bool hdcp_cap = false, hdcp2_cap = false;
 
 	if (!intel_connector->hdcp.shim) {
 		seq_puts(m, "No Connector Support");
@@ -252,9 +253,6 @@ static void intel_connector_info(struct seq_file *m,
 				 struct drm_connector *connector)
 {
 	struct intel_connector *intel_connector = to_intel_connector(connector);
-	const struct drm_connector_state *conn_state = connector->state;
-	struct intel_encoder *encoder =
-		to_intel_encoder(conn_state->best_encoder);
 	const struct drm_display_mode *mode;
 
 	seq_printf(m, "[CONNECTOR:%d:%s]: status: %s\n",
@@ -271,28 +269,23 @@ static void intel_connector_info(struct seq_file *m,
 		   drm_get_subpixel_order_name(connector->display_info.subpixel_order));
 	seq_printf(m, "\tCEA rev: %d\n", connector->display_info.cea_rev);
 
-	if (!encoder)
-		return;
-
 	switch (connector->connector_type) {
 	case DRM_MODE_CONNECTOR_DisplayPort:
 	case DRM_MODE_CONNECTOR_eDP:
-		if (encoder->type == INTEL_OUTPUT_DP_MST)
+		if (intel_connector->mst_port)
 			intel_dp_mst_info(m, intel_connector);
 		else
 			intel_dp_info(m, intel_connector);
 		break;
 	case DRM_MODE_CONNECTOR_HDMIA:
-		if (encoder->type == INTEL_OUTPUT_HDMI ||
-		    encoder->type == INTEL_OUTPUT_DDI)
-			intel_hdmi_info(m, intel_connector);
+		intel_hdmi_info(m, intel_connector);
 		break;
 	default:
 		break;
 	}
 
 	seq_puts(m, "\tHDCP version: ");
-	if (intel_encoder_is_mst(encoder)) {
+	if (intel_connector->mst_port) {
 		intel_hdcp_info(m, intel_connector, true);
 		seq_puts(m, "\tMST Hub HDCP version: ");
 	}
@@ -645,51 +638,24 @@ static int i915_display_capabilities(struct seq_file *m, void *unused)
 static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_printer p = drm_seq_file_printer(m);
 	struct intel_shared_dpll *pll;
 	int i;
 
 	drm_modeset_lock_all(&dev_priv->drm);
 
-	seq_printf(m, "PLL refclks: non-SSC: %d kHz, SSC: %d kHz\n",
+	drm_printf(&p, "PLL refclks: non-SSC: %d kHz, SSC: %d kHz\n",
 		   dev_priv->display.dpll.ref_clks.nssc,
 		   dev_priv->display.dpll.ref_clks.ssc);
 
 	for_each_shared_dpll(dev_priv, pll, i) {
-		seq_printf(m, "DPLL%i: %s, id: %i\n", pll->index,
+		drm_printf(&p, "DPLL%i: %s, id: %i\n", pll->index,
 			   pll->info->name, pll->info->id);
-		seq_printf(m, " pipe_mask: 0x%x, active: 0x%x, on: %s\n",
+		drm_printf(&p, " pipe_mask: 0x%x, active: 0x%x, on: %s\n",
 			   pll->state.pipe_mask, pll->active_mask,
 			   str_yes_no(pll->on));
-		seq_printf(m, " tracked hardware state:\n");
-		seq_printf(m, " dpll:    0x%08x\n", pll->state.hw_state.dpll);
-		seq_printf(m, " dpll_md: 0x%08x\n",
-			   pll->state.hw_state.dpll_md);
-		seq_printf(m, " fp0:     0x%08x\n", pll->state.hw_state.fp0);
-		seq_printf(m, " fp1:     0x%08x\n", pll->state.hw_state.fp1);
-		seq_printf(m, " wrpll:   0x%08x\n", pll->state.hw_state.wrpll);
-		seq_printf(m, " cfgcr0:  0x%08x\n", pll->state.hw_state.cfgcr0);
-		seq_printf(m, " cfgcr1:  0x%08x\n", pll->state.hw_state.cfgcr1);
-		seq_printf(m, " div0:    0x%08x\n", pll->state.hw_state.div0);
-		seq_printf(m, " mg_refclkin_ctl:        0x%08x\n",
-			   pll->state.hw_state.mg_refclkin_ctl);
-		seq_printf(m, " mg_clktop2_coreclkctl1: 0x%08x\n",
-			   pll->state.hw_state.mg_clktop2_coreclkctl1);
-		seq_printf(m, " mg_clktop2_hsclkctl:    0x%08x\n",
-			   pll->state.hw_state.mg_clktop2_hsclkctl);
-		seq_printf(m, " mg_pll_div0:  0x%08x\n",
-			   pll->state.hw_state.mg_pll_div0);
-		seq_printf(m, " mg_pll_div1:  0x%08x\n",
-			   pll->state.hw_state.mg_pll_div1);
-		seq_printf(m, " mg_pll_lf:    0x%08x\n",
-			   pll->state.hw_state.mg_pll_lf);
-		seq_printf(m, " mg_pll_frac_lock: 0x%08x\n",
-			   pll->state.hw_state.mg_pll_frac_lock);
-		seq_printf(m, " mg_pll_ssc:   0x%08x\n",
-			   pll->state.hw_state.mg_pll_ssc);
-		seq_printf(m, " mg_pll_bias:  0x%08x\n",
-			   pll->state.hw_state.mg_pll_bias);
-		seq_printf(m, " mg_pll_tdc_coldst_bias: 0x%08x\n",
-			   pll->state.hw_state.mg_pll_tdc_coldst_bias);
+		drm_printf(&p, " tracked hardware state:\n");
+		intel_dpll_dump_hw_state(dev_priv, &p, &pll->state.hw_state);
 	}
 	drm_modeset_unlock_all(&dev_priv->drm);
 
@@ -1103,27 +1069,6 @@ void intel_display_debugfs_register(struct drm_i915_private *i915)
 	intel_display_debugfs_params(i915);
 }
 
-static int i915_panel_show(struct seq_file *m, void *data)
-{
-	struct intel_connector *connector = m->private;
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
-
-	if (connector->base.status != connector_status_connected)
-		return -ENODEV;
-
-	seq_printf(m, "Panel power up delay: %d\n",
-		   intel_dp->pps.panel_power_up_delay);
-	seq_printf(m, "Panel power down delay: %d\n",
-		   intel_dp->pps.panel_power_down_delay);
-	seq_printf(m, "Backlight on delay: %d\n",
-		   intel_dp->pps.backlight_on_delay);
-	seq_printf(m, "Backlight off delay: %d\n",
-		   intel_dp->pps.backlight_off_delay);
-
-	return 0;
-}
-DEFINE_SHOW_ATTRIBUTE(i915_panel);
-
 static int i915_hdcp_sink_capability_show(struct seq_file *m, void *data)
 {
 	struct intel_connector *connector = m->private;
@@ -1402,20 +1347,6 @@ out:	drm_modeset_unlock(&i915->drm.mode_config.connection_mutex);
 	return ret;
 }
 
-static int i915_bigjoiner_enable_show(struct seq_file *m, void *data)
-{
-	struct intel_connector *connector = m->private;
-	struct drm_crtc *crtc;
-
-	crtc = connector->base.state->crtc;
-	if (connector->base.status != connector_status_connected || !crtc)
-		return -ENODEV;
-
-	seq_printf(m, "Bigjoiner enable: %d\n", connector->force_bigjoiner_enable);
-
-	return 0;
-}
-
 static ssize_t i915_dsc_output_format_write(struct file *file,
 					    const char __user *ubuf,
 					    size_t len, loff_t *offp)
@@ -1437,30 +1368,6 @@ static ssize_t i915_dsc_output_format_write(struct file *file,
 	return len;
 }
 
-static ssize_t i915_bigjoiner_enable_write(struct file *file,
-					   const char __user *ubuf,
-					   size_t len, loff_t *offp)
-{
-	struct seq_file *m = file->private_data;
-	struct intel_connector *connector = m->private;
-	struct drm_crtc *crtc;
-	bool bigjoiner_en = 0;
-	int ret;
-
-	crtc = connector->base.state->crtc;
-	if (connector->base.status != connector_status_connected || !crtc)
-		return -ENODEV;
-
-	ret = kstrtobool_from_user(ubuf, len, &bigjoiner_en);
-	if (ret < 0)
-		return ret;
-
-	connector->force_bigjoiner_enable = bigjoiner_en;
-	*offp += len;
-
-	return len;
-}
-
 static int i915_dsc_output_format_open(struct inode *inode,
 				       struct file *file)
 {
@@ -1554,8 +1461,6 @@ static const struct file_operations i915_dsc_fractional_bpp_fops = {
 	.write = i915_dsc_fractional_bpp_write
 };
 
-DEFINE_SHOW_STORE_ATTRIBUTE(i915_bigjoiner_enable);
-
 /*
  * Returns the Current CRTC's bpc.
  * Example usage: cat /sys/kernel/debug/dri/0/crtc-0/i915_current_bpc
@@ -1608,12 +1513,9 @@ void intel_connector_debugfs_add(struct intel_connector *connector)
 		return;
 
 	intel_drrs_connector_debugfs_add(connector);
+	intel_pps_connector_debugfs_add(connector);
 	intel_psr_connector_debugfs_add(connector);
 
-	if (connector_type == DRM_MODE_CONNECTOR_eDP)
-		debugfs_create_file("i915_panel_timings", 0444, root,
-				    connector, &i915_panel_fops);
-
 	if (connector_type == DRM_MODE_CONNECTOR_DisplayPort ||
 	    connector_type == DRM_MODE_CONNECTOR_HDMIA ||
 	    connector_type == DRM_MODE_CONNECTOR_HDMIB) {
@@ -1640,8 +1542,8 @@ void intel_connector_debugfs_add(struct intel_connector *connector)
 	if (DISPLAY_VER(i915) >= 11 &&
 	    (connector_type == DRM_MODE_CONNECTOR_DisplayPort ||
 	     connector_type == DRM_MODE_CONNECTOR_eDP)) {
-		debugfs_create_file("i915_bigjoiner_force_enable", 0644, root,
-				    connector, &i915_bigjoiner_enable_fops);
+		debugfs_create_bool("i915_bigjoiner_force_enable", 0644, root,
+				    &connector->force_bigjoiner_enable);
 	}
 
 	if (connector_type == DRM_MODE_CONNECTOR_DSI ||
diff --git a/drivers/gpu/drm/i915/display/intel_display_device.c b/drivers/gpu/drm/i915/display/intel_display_device.c
index c02d79b50006..120e209ee74a 100644
--- a/drivers/gpu/drm/i915/display/intel_display_device.c
+++ b/drivers/gpu/drm/i915/display/intel_display_device.c
@@ -17,6 +17,9 @@
 #include "intel_display_reg_defs.h"
 #include "intel_fbc.h"
 
+__diag_push();
+__diag_ignore_all("-Woverride-init", "Allow field initialization overrides for display info");
+
 static const struct intel_display_device_info no_display = {};
 
 #define PIPE_A_OFFSET		0x70000
@@ -768,6 +771,8 @@ static const struct intel_display_device_info xe2_lpd_display = {
 		BIT(INTEL_FBC_C) | BIT(INTEL_FBC_D),
 };
 
+__diag_pop();
+
 /*
  * Separate detection for no display cases to keep the display id array simple.
  *
@@ -922,6 +927,9 @@ void intel_display_device_probe(struct drm_i915_private *i915)
 	const struct intel_display_device_info *info;
 	u16 ver, rel, step;
 
+	/* Add drm device backpointer as early as possible. */
+	i915->display.drm = &i915->drm;
+
 	if (HAS_GMD_ID(i915))
 		info = probe_gmdid_display(i915, &ver, &rel, &step);
 	else
diff --git a/drivers/gpu/drm/i915/display/intel_display_device.h b/drivers/gpu/drm/i915/display/intel_display_device.h
index 9b1bce2624b9..17ddf82f0b6e 100644
--- a/drivers/gpu/drm/i915/display/intel_display_device.h
+++ b/drivers/gpu/drm/i915/display/intel_display_device.h
@@ -8,6 +8,7 @@
 
 #include <linux/types.h>
 
+#include "intel_display_conversion.h"
 #include "intel_display_limits.h"
 
 struct drm_i915_private;
@@ -69,6 +70,7 @@ struct drm_printer;
 #define HAS_TRANSCODER(i915, trans)	((DISPLAY_RUNTIME_INFO(i915)->cpu_transcoder_mask & \
 					  BIT(trans)) != 0)
 #define HAS_VRR(i915)			(DISPLAY_VER(i915) >= 11)
+#define HAS_AS_SDP(i915)		(DISPLAY_VER(i915) >= 13)
 #define INTEL_NUM_PIPES(i915)		(hweight8(DISPLAY_RUNTIME_INFO(i915)->pipe_mask))
 #define I915_HAS_HOTPLUG(i915)		(DISPLAY_INFO(i915)->has_hotplug)
 #define OVERLAY_NEEDS_PHYSICAL(i915)	(DISPLAY_INFO(i915)->overlay_needs_physical)
@@ -99,8 +101,8 @@ struct drm_printer;
 	(IS_DISPLAY_IP_RANGE((__i915), (ipver), (ipver)) && \
 	 IS_DISPLAY_STEP((__i915), (from), (until)))
 
-#define DISPLAY_INFO(i915)		((i915)->display.info.__device_info)
-#define DISPLAY_RUNTIME_INFO(i915)	(&(i915)->display.info.__runtime_info)
+#define DISPLAY_INFO(i915)		(__to_intel_display(i915)->info.__device_info)
+#define DISPLAY_RUNTIME_INFO(i915)	(&__to_intel_display(i915)->info.__runtime_info)
 
 #define DISPLAY_VER(i915)	(DISPLAY_RUNTIME_INFO(i915)->ip.ver)
 #define DISPLAY_VER_FULL(i915)	IP_VER(DISPLAY_RUNTIME_INFO(i915)->ip.ver, \
diff --git a/drivers/gpu/drm/i915/display/intel_display_driver.c b/drivers/gpu/drm/i915/display/intel_display_driver.c
index 87dd07e0d138..794b4af38055 100644
--- a/drivers/gpu/drm/i915/display/intel_display_driver.c
+++ b/drivers/gpu/drm/i915/display/intel_display_driver.c
@@ -11,6 +11,7 @@
 #include <acpi/video.h>
 #include <drm/display/drm_dp_mst_helper.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_client.h>
 #include <drm/drm_mode_config.h>
 #include <drm/drm_privacy_screen_consumer.h>
 #include <drm/drm_probe_helper.h>
@@ -98,7 +99,6 @@ void intel_display_driver_init_hw(struct drm_i915_private *i915)
 static const struct drm_mode_config_funcs intel_mode_funcs = {
 	.fb_create = intel_user_framebuffer_create,
 	.get_format_info = intel_fb_get_format_info,
-	.output_poll_changed = intel_fbdev_output_poll_changed,
 	.mode_valid = intel_mode_valid,
 	.atomic_check = intel_atomic_check,
 	.atomic_commit = intel_atomic_commit,
@@ -198,11 +198,13 @@ void intel_display_driver_early_probe(struct drm_i915_private *i915)
 	intel_dpll_init_clock_hook(i915);
 	intel_init_display_hooks(i915);
 	intel_fdi_init_hook(i915);
+	intel_dmc_wl_init(&i915->display);
 }
 
 /* part #1: call before irq install */
 int intel_display_driver_probe_noirq(struct drm_i915_private *i915)
 {
+	struct intel_display *display = &i915->display;
 	int ret;
 
 	if (i915_inject_probe_failure(i915))
@@ -261,7 +263,7 @@ int intel_display_driver_probe_noirq(struct drm_i915_private *i915)
 	if (ret)
 		goto cleanup_vga_client_pw_domain_dmc;
 
-	intel_init_quirks(i915);
+	intel_init_quirks(display);
 
 	intel_fbc_init(i915);
 
@@ -514,10 +516,6 @@ int intel_display_driver_probe(struct drm_i915_private *i915)
 
 	intel_overlay_setup(i915);
 
-	ret = intel_fbdev_init(&i915->drm);
-	if (ret)
-		return ret;
-
 	/* Only enable hotplug handling once the fbdev is fully set up. */
 	intel_hpd_init(i915);
 
@@ -542,17 +540,9 @@ void intel_display_driver_register(struct drm_i915_private *i915)
 
 	intel_display_driver_enable_user_access(i915);
 
-	intel_display_debugfs_register(i915);
+	intel_audio_register(i915);
 
-	/*
-	 * Some ports require correctly set-up hpd registers for
-	 * detection to work properly (leading to ghost connected
-	 * connector status), e.g. VGA on gm45.  Hence we can only set
-	 * up the initial fbdev config after hpd irqs are fully
-	 * enabled. We do it last so that the async config cannot run
-	 * before the connectors are registered.
-	 */
-	intel_fbdev_initial_config_async(i915);
+	intel_display_debugfs_register(i915);
 
 	/*
 	 * We need to coordinate the hotplugs with the asynchronous
@@ -562,6 +552,8 @@ void intel_display_driver_register(struct drm_i915_private *i915)
 	drm_kms_helper_poll_init(&i915->drm);
 	intel_hpd_poll_disable(i915);
 
+	intel_fbdev_setup(i915);
+
 	intel_display_device_info_print(DISPLAY_INFO(i915),
 					DISPLAY_RUNTIME_INFO(i915), &p);
 }
@@ -597,9 +589,6 @@ void intel_display_driver_remove_noirq(struct drm_i915_private *i915)
 	 */
 	intel_hpd_poll_fini(i915);
 
-	/* poll work can call into fbdev, hence clean that up afterwards */
-	intel_fbdev_fini(i915);
-
 	intel_unregister_dsm_handler();
 
 	/* flush any delayed tasks or pending work */
@@ -638,7 +627,8 @@ void intel_display_driver_unregister(struct drm_i915_private *i915)
 	if (!HAS_DISPLAY(i915))
 		return;
 
-	intel_fbdev_unregister(i915);
+	drm_client_dev_unregister(&i915->drm);
+
 	/*
 	 * After flushing the fbdev (incl. a late async config which
 	 * will have delayed queuing of a hotplug event), then flush
diff --git a/drivers/gpu/drm/i915/display/intel_display_irq.c b/drivers/gpu/drm/i915/display/intel_display_irq.c
index f846c5b108b5..c337e0597541 100644
--- a/drivers/gpu/drm/i915/display/intel_display_irq.c
+++ b/drivers/gpu/drm/i915/display/intel_display_irq.c
@@ -117,13 +117,14 @@ static void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
 	if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
 		return;
 
-	new_val = dev_priv->de_irq_mask[pipe];
+	new_val = dev_priv->display.irq.de_irq_mask[pipe];
 	new_val &= ~interrupt_mask;
 	new_val |= (~enabled_irq_mask & interrupt_mask);
 
-	if (new_val != dev_priv->de_irq_mask[pipe]) {
-		dev_priv->de_irq_mask[pipe] = new_val;
-		intel_uncore_write(&dev_priv->uncore, GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
+	if (new_val != dev_priv->display.irq.de_irq_mask[pipe]) {
+		dev_priv->display.irq.de_irq_mask[pipe] = new_val;
+		intel_uncore_write(&dev_priv->uncore, GEN8_DE_PIPE_IMR(pipe),
+				   dev_priv->display.irq.de_irq_mask[pipe]);
 		intel_uncore_posting_read(&dev_priv->uncore, GEN8_DE_PIPE_IMR(pipe));
 	}
 }
@@ -179,7 +180,7 @@ void ibx_disable_display_interrupt(struct drm_i915_private *i915, u32 bits)
 u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv,
 			      enum pipe pipe)
 {
-	u32 status_mask = dev_priv->pipestat_irq_mask[pipe];
+	u32 status_mask = dev_priv->display.irq.pipestat_irq_mask[pipe];
 	u32 enable_mask = status_mask << 16;
 
 	lockdep_assert_held(&dev_priv->irq_lock);
@@ -233,10 +234,10 @@ void i915_enable_pipestat(struct drm_i915_private *dev_priv,
 	lockdep_assert_held(&dev_priv->irq_lock);
 	drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
 
-	if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == status_mask)
+	if ((dev_priv->display.irq.pipestat_irq_mask[pipe] & status_mask) == status_mask)
 		return;
 
-	dev_priv->pipestat_irq_mask[pipe] |= status_mask;
+	dev_priv->display.irq.pipestat_irq_mask[pipe] |= status_mask;
 	enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
 
 	intel_uncore_write(&dev_priv->uncore, reg, enable_mask | status_mask);
@@ -256,10 +257,10 @@ void i915_disable_pipestat(struct drm_i915_private *dev_priv,
 	lockdep_assert_held(&dev_priv->irq_lock);
 	drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
 
-	if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == 0)
+	if ((dev_priv->display.irq.pipestat_irq_mask[pipe] & status_mask) == 0)
 		return;
 
-	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
+	dev_priv->display.irq.pipestat_irq_mask[pipe] &= ~status_mask;
 	enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
 
 	intel_uncore_write(&dev_priv->uncore, reg, enable_mask | status_mask);
@@ -401,7 +402,7 @@ void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
 				   PIPESTAT_INT_STATUS_MASK |
 				   PIPE_FIFO_UNDERRUN_STATUS);
 
-		dev_priv->pipestat_irq_mask[pipe] = 0;
+		dev_priv->display.irq.pipestat_irq_mask[pipe] = 0;
 	}
 }
 
@@ -412,7 +413,7 @@ void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv,
 
 	spin_lock(&dev_priv->irq_lock);
 
-	if (!dev_priv->display_irqs_enabled) {
+	if (!dev_priv->display.irq.display_irqs_enabled) {
 		spin_unlock(&dev_priv->irq_lock);
 		return;
 	}
@@ -445,7 +446,7 @@ void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv,
 			break;
 		}
 		if (iir & iir_bit)
-			status_mask |= dev_priv->pipestat_irq_mask[pipe];
+			status_mask |= dev_priv->display.irq.pipestat_irq_mask[pipe];
 
 		if (!status_mask)
 			continue;
@@ -1203,7 +1204,7 @@ int i8xx_enable_vblank(struct drm_crtc *crtc)
 
 int i915gm_enable_vblank(struct drm_crtc *crtc)
 {
-	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
+	struct drm_i915_private *i915 = to_i915(crtc->dev);
 
 	/*
 	 * Vblank interrupts fail to wake the device up from C2+.
@@ -1211,8 +1212,8 @@ int i915gm_enable_vblank(struct drm_crtc *crtc)
 	 * the problem. There is a small power cost so we do this
 	 * only when vblank interrupts are actually enabled.
 	 */
-	if (dev_priv->vblank_enabled++ == 0)
-		intel_uncore_write(&dev_priv->uncore, SCPD0, _MASKED_BIT_ENABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
+	if (i915->display.irq.vblank_enabled++ == 0)
+		intel_uncore_write(&i915->uncore, SCPD0, _MASKED_BIT_ENABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
 
 	return i8xx_enable_vblank(crtc);
 }
@@ -1315,12 +1316,12 @@ void i8xx_disable_vblank(struct drm_crtc *crtc)
 
 void i915gm_disable_vblank(struct drm_crtc *crtc)
 {
-	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
+	struct drm_i915_private *i915 = to_i915(crtc->dev);
 
 	i8xx_disable_vblank(crtc);
 
-	if (--dev_priv->vblank_enabled == 0)
-		intel_uncore_write(&dev_priv->uncore, SCPD0, _MASKED_BIT_DISABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
+	if (--i915->display.irq.vblank_enabled == 0)
+		intel_uncore_write(&i915->uncore, SCPD0, _MASKED_BIT_DISABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
 }
 
 void i965_disable_vblank(struct drm_crtc *crtc)
@@ -1497,8 +1498,8 @@ void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
 
 	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
 		GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
-				  dev_priv->de_irq_mask[pipe],
-				  ~dev_priv->de_irq_mask[pipe] | extra_ier);
+				  dev_priv->display.irq.de_irq_mask[pipe],
+				  ~dev_priv->display.irq.de_irq_mask[pipe] | extra_ier);
 
 	spin_unlock_irq(&dev_priv->irq_lock);
 }
@@ -1558,10 +1559,10 @@ void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
 {
 	lockdep_assert_held(&dev_priv->irq_lock);
 
-	if (dev_priv->display_irqs_enabled)
+	if (dev_priv->display.irq.display_irqs_enabled)
 		return;
 
-	dev_priv->display_irqs_enabled = true;
+	dev_priv->display.irq.display_irqs_enabled = true;
 
 	if (intel_irqs_enabled(dev_priv)) {
 		vlv_display_irq_reset(dev_priv);
@@ -1573,10 +1574,10 @@ void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
 {
 	lockdep_assert_held(&dev_priv->irq_lock);
 
-	if (!dev_priv->display_irqs_enabled)
+	if (!dev_priv->display.irq.display_irqs_enabled)
 		return;
 
-	dev_priv->display_irqs_enabled = false;
+	dev_priv->display.irq.display_irqs_enabled = false;
 
 	if (intel_irqs_enabled(dev_priv))
 		vlv_display_irq_reset(dev_priv);
@@ -1694,12 +1695,12 @@ void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
 	}
 
 	for_each_pipe(dev_priv, pipe) {
-		dev_priv->de_irq_mask[pipe] = ~de_pipe_masked;
+		dev_priv->display.irq.de_irq_mask[pipe] = ~de_pipe_masked;
 
 		if (intel_display_power_is_enabled(dev_priv,
 						   POWER_DOMAIN_PIPE(pipe)))
 			GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
-					  dev_priv->de_irq_mask[pipe],
+					  dev_priv->display.irq.de_irq_mask[pipe],
 					  de_pipe_enables);
 	}
 
@@ -1770,9 +1771,9 @@ void intel_display_irq_init(struct drm_i915_private *i915)
 	 * domain. We defer setting up the display irqs in this case to the
 	 * runtime pm.
 	 */
-	i915->display_irqs_enabled = true;
+	i915->display.irq.display_irqs_enabled = true;
 	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
-		i915->display_irqs_enabled = false;
+		i915->display.irq.display_irqs_enabled = false;
 
 	intel_hotplug_irq_init(i915);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display_params.c b/drivers/gpu/drm/i915/display/intel_display_params.c
index 11e03cfb774d..1799a6643128 100644
--- a/drivers/gpu/drm/i915/display/intel_display_params.c
+++ b/drivers/gpu/drm/i915/display/intel_display_params.c
@@ -27,6 +27,10 @@ static struct intel_display_params intel_display_modparams __read_mostly = {
  * debugfs mode to 0.
  */
 
+intel_display_param_named_unsafe(dmc_firmware_path, charp, 0400,
+	"DMC firmware path to use instead of the default one. "
+	"Use /dev/null to disable DMC and runtime PM.");
+
 intel_display_param_named_unsafe(vbt_firmware, charp, 0400,
 	"Load VBT from specified file under /lib/firmware");
 
@@ -116,6 +120,11 @@ intel_display_param_named_unsafe(enable_psr2_sel_fetch, bool, 0400,
 	"(0=disabled, 1=enabled) "
 	"Default: 1");
 
+intel_display_param_named_unsafe(enable_dmc_wl, bool, 0400,
+	"Enable DMC wakelock "
+	"(0=disabled, 1=enabled) "
+	"Default: 0");
+
 __maybe_unused
 static void _param_print_bool(struct drm_printer *p, const char *driver_name,
 			      const char *name, bool val)
diff --git a/drivers/gpu/drm/i915/display/intel_display_params.h b/drivers/gpu/drm/i915/display/intel_display_params.h
index 6206cc51df04..1208a62c16d2 100644
--- a/drivers/gpu/drm/i915/display/intel_display_params.h
+++ b/drivers/gpu/drm/i915/display/intel_display_params.h
@@ -24,6 +24,7 @@ struct drm_i915_private;
  *       debugfs file
  */
 #define INTEL_DISPLAY_PARAMS_FOR_EACH(param) \
+	param(char *, dmc_firmware_path, NULL, 0400) \
 	param(char *, vbt_firmware, NULL, 0400) \
 	param(int, lvds_channel_mode, 0, 0400) \
 	param(int, panel_use_ssc, -1, 0600) \
@@ -46,6 +47,7 @@ struct drm_i915_private;
 	param(int, enable_psr, -1, 0600) \
 	param(bool, psr_safest_params, false, 0400) \
 	param(bool, enable_psr2_sel_fetch, true, 0400) \
+	param(bool, enable_dmc_wl, false, 0400) \
 
 #define MEMBER(T, member, ...) T member;
 struct intel_display_params {
diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c
index 6fd4fa52253a..03dc7edcc443 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power.c
@@ -640,13 +640,7 @@ release_async_put_domains(struct i915_power_domains *power_domains,
 	enum intel_display_power_domain domain;
 	intel_wakeref_t wakeref;
 
-	/*
-	 * The caller must hold already raw wakeref, upgrade that to a proper
-	 * wakeref to make the state checker happy about the HW access during
-	 * power well disabling.
-	 */
-	assert_rpm_raw_wakeref_held(rpm);
-	wakeref = intel_runtime_pm_get(rpm);
+	wakeref = intel_runtime_pm_get_noresume(rpm);
 
 	for_each_power_domain(domain, mask) {
 		/* Clear before put, so put's sanity check is happy. */
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c
index 06900ff307b2..83f616097a29 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_well.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c
@@ -17,6 +17,7 @@
 #include "intel_dkl_phy.h"
 #include "intel_dkl_phy_regs.h"
 #include "intel_dmc.h"
+#include "intel_dmc_wl.h"
 #include "intel_dp_aux_regs.h"
 #include "intel_dpio_phy.h"
 #include "intel_dpll.h"
@@ -26,6 +27,7 @@
 #include "intel_tc.h"
 #include "intel_vga.h"
 #include "skl_watermark.h"
+#include "vlv_dpio_phy_regs.h"
 #include "vlv_sideband.h"
 #include "vlv_sideband_reg.h"
 
@@ -199,6 +201,9 @@ static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
 		gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 }
 
+#define ICL_AUX_PW_TO_PHY(pw_idx)	\
+	((pw_idx) - ICL_PW_CTL_IDX_AUX_A + PHY_A)
+
 #define ICL_AUX_PW_TO_CH(pw_idx)	\
 	((pw_idx) - ICL_PW_CTL_IDX_AUX_A + AUX_CH_A)
 
@@ -217,27 +222,22 @@ static struct intel_digital_port *
 aux_ch_to_digital_port(struct drm_i915_private *dev_priv,
 		       enum aux_ch aux_ch)
 {
-	struct intel_digital_port *dig_port = NULL;
 	struct intel_encoder *encoder;
 
 	for_each_intel_encoder(&dev_priv->drm, encoder) {
+		struct intel_digital_port *dig_port;
+
 		/* We'll check the MST primary port */
 		if (encoder->type == INTEL_OUTPUT_DP_MST)
 			continue;
 
 		dig_port = enc_to_dig_port(encoder);
-		if (!dig_port)
-			continue;
-
-		if (dig_port->aux_ch != aux_ch) {
-			dig_port = NULL;
-			continue;
-		}
 
-		break;
+		if (dig_port && dig_port->aux_ch == aux_ch)
+			return dig_port;
 	}
 
-	return dig_port;
+	return NULL;
 }
 
 static enum phy icl_aux_pw_to_phy(struct drm_i915_private *i915,
@@ -253,7 +253,7 @@ static enum phy icl_aux_pw_to_phy(struct drm_i915_private *i915,
 	 * as HDMI-only and routed to a combo PHY, the encoder either won't be
 	 * present at all or it will not have an aux_ch assigned.
 	 */
-	return dig_port ? intel_port_to_phy(i915, dig_port->base.port) : PHY_NONE;
+	return dig_port ? intel_encoder_to_phy(&dig_port->base) : PHY_NONE;
 }
 
 static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
@@ -396,17 +396,11 @@ static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 	hsw_wait_for_power_well_disable(dev_priv, power_well);
 }
 
-static bool intel_port_is_edp(struct drm_i915_private *i915, enum port port)
+static bool intel_aux_ch_is_edp(struct drm_i915_private *i915, enum aux_ch aux_ch)
 {
-	struct intel_encoder *encoder;
-
-	for_each_intel_encoder(&i915->drm, encoder) {
-		if (encoder->type == INTEL_OUTPUT_EDP &&
-		    encoder->port == port)
-			return true;
-	}
+	struct intel_digital_port *dig_port = aux_ch_to_digital_port(i915, aux_ch);
 
-	return false;
+	return dig_port && dig_port->base.type == INTEL_OUTPUT_EDP;
 }
 
 static void
@@ -415,24 +409,25 @@ icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 {
 	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
-	enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
 
 	drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
 
 	intel_de_rmw(dev_priv, regs->driver, 0, HSW_PWR_WELL_CTL_REQ(pw_idx));
 
-	/* FIXME this is a mess */
-	if (phy != PHY_NONE)
-		intel_de_rmw(dev_priv, ICL_PORT_CL_DW12(phy),
-			     0, ICL_LANE_ENABLE_AUX);
+	/*
+	 * FIXME not sure if we should derive the PHY from the pw_idx, or
+	 * from the VBT defined AUX_CH->DDI->PHY mapping.
+	 */
+	intel_de_rmw(dev_priv, ICL_PORT_CL_DW12(ICL_AUX_PW_TO_PHY(pw_idx)),
+		     0, ICL_LANE_ENABLE_AUX);
 
 	hsw_wait_for_power_well_enable(dev_priv, power_well, false);
 
 	/* Display WA #1178: icl */
 	if (pw_idx >= ICL_PW_CTL_IDX_AUX_A && pw_idx <= ICL_PW_CTL_IDX_AUX_B &&
-	    !intel_port_is_edp(dev_priv, (enum port)phy))
-		intel_de_rmw(dev_priv, ICL_AUX_ANAOVRD1(pw_idx),
-			     0, ICL_AUX_ANAOVRD1_ENABLE | ICL_AUX_ANAOVRD1_LDO_BYPASS);
+	    !intel_aux_ch_is_edp(dev_priv, ICL_AUX_PW_TO_CH(pw_idx)))
+		intel_de_rmw(dev_priv, ICL_PORT_TX_DW6_AUX(ICL_AUX_PW_TO_PHY(pw_idx)),
+			     0, O_FUNC_OVRD_EN | O_LDO_BYPASS_CRI);
 }
 
 static void
@@ -441,14 +436,15 @@ icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
 {
 	const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
 	int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
-	enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
 
 	drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
 
-	/* FIXME this is a mess */
-	if (phy != PHY_NONE)
-		intel_de_rmw(dev_priv, ICL_PORT_CL_DW12(phy),
-			     ICL_LANE_ENABLE_AUX, 0);
+	/*
+	 * FIXME not sure if we should derive the PHY from the pw_idx, or
+	 * from the VBT defined AUX_CH->DDI->PHY mapping.
+	 */
+	intel_de_rmw(dev_priv, ICL_PORT_CL_DW12(ICL_AUX_PW_TO_PHY(pw_idx)),
+		     ICL_LANE_ENABLE_AUX, 0);
 
 	intel_de_rmw(dev_priv, regs->driver, HSW_PWR_WELL_CTL_REQ(pw_idx), 0);
 
@@ -827,6 +823,8 @@ void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 		intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
 			     0, SKL_SELECT_ALTERNATE_DC_EXIT);
 
+	intel_dmc_wl_enable(&dev_priv->display);
+
 	gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
 }
 
@@ -856,6 +854,8 @@ void skl_enable_dc6(struct drm_i915_private *dev_priv)
 		intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
 			     0, SKL_SELECT_ALTERNATE_DC_EXIT);
 
+	intel_dmc_wl_enable(&dev_priv->display);
+
 	gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 }
 
@@ -906,39 +906,39 @@ static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 					   struct i915_power_well *power_well)
 {
-	bxt_ddi_phy_init(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
+	bxt_dpio_phy_init(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
 }
 
 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 					    struct i915_power_well *power_well)
 {
-	bxt_ddi_phy_uninit(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
+	bxt_dpio_phy_uninit(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
 }
 
 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
 					    struct i915_power_well *power_well)
 {
-	return bxt_ddi_phy_is_enabled(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
+	return bxt_dpio_phy_is_enabled(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
 }
 
-static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
+static void bxt_verify_dpio_phy_power_wells(struct drm_i915_private *dev_priv)
 {
 	struct i915_power_well *power_well;
 
 	power_well = lookup_power_well(dev_priv, BXT_DISP_PW_DPIO_CMN_A);
 	if (intel_power_well_refcount(power_well) > 0)
-		bxt_ddi_phy_verify_state(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
+		bxt_dpio_phy_verify_state(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
 
 	power_well = lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
 	if (intel_power_well_refcount(power_well) > 0)
-		bxt_ddi_phy_verify_state(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
+		bxt_dpio_phy_verify_state(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
 
 	if (IS_GEMINILAKE(dev_priv)) {
 		power_well = lookup_power_well(dev_priv,
 					       GLK_DISP_PW_DPIO_CMN_C);
 		if (intel_power_well_refcount(power_well) > 0)
-			bxt_ddi_phy_verify_state(dev_priv,
-						 i915_power_well_instance(power_well)->bxt.phy);
+			bxt_dpio_phy_verify_state(dev_priv,
+						  i915_power_well_instance(power_well)->bxt.phy);
 	}
 }
 
@@ -976,16 +976,18 @@ void gen9_disable_dc_states(struct drm_i915_private *dev_priv)
 	if (!HAS_DISPLAY(dev_priv))
 		return;
 
+	intel_dmc_wl_disable(&dev_priv->display);
+
 	intel_cdclk_get_cdclk(dev_priv, &cdclk_config);
 	/* Can't read out voltage_level so can't use intel_cdclk_changed() */
 	drm_WARN_ON(&dev_priv->drm,
-		    intel_cdclk_needs_modeset(&dev_priv->display.cdclk.hw,
+		    intel_cdclk_clock_changed(&dev_priv->display.cdclk.hw,
 					      &cdclk_config));
 
 	gen9_assert_dbuf_enabled(dev_priv);
 
 	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
-		bxt_verify_ddi_phy_power_wells(dev_priv);
+		bxt_verify_dpio_phy_power_wells(dev_priv);
 
 	if (DISPLAY_VER(dev_priv) >= 11)
 		/*
@@ -1396,8 +1398,8 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
 	 * The PHY may be busy with some initial calibration and whatnot,
 	 * so the power state can take a while to actually change.
 	 */
-	if (intel_de_wait_for_register(dev_priv, DISPLAY_PHY_STATUS,
-				       phy_status_mask, phy_status, 10))
+	if (intel_de_wait(dev_priv, DISPLAY_PHY_STATUS,
+			  phy_status_mask, phy_status, 10))
 		drm_err(&dev_priv->drm,
 			"Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
 			intel_de_read(dev_priv, DISPLAY_PHY_STATUS) & phy_status_mask,
@@ -1441,9 +1443,9 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW28, tmp);
 
 	if (id == VLV_DISP_PW_DPIO_CMN_BC) {
-		tmp = vlv_dpio_read(dev_priv, phy, _CHV_CMN_DW6_CH1);
+		tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW6_CH1);
 		tmp |= DPIO_DYNPWRDOWNEN_CH1;
-		vlv_dpio_write(dev_priv, phy, _CHV_CMN_DW6_CH1, tmp);
+		vlv_dpio_write(dev_priv, phy, CHV_CMN_DW6_CH1, tmp);
 	} else {
 		/*
 		 * Force the non-existing CL2 off. BXT does this
@@ -1519,9 +1521,9 @@ static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpi
 		return;
 
 	if (ch == DPIO_CH0)
-		reg = _CHV_CMN_DW0_CH0;
+		reg = CHV_CMN_DW0_CH0;
 	else
-		reg = _CHV_CMN_DW6_CH1;
+		reg = CHV_CMN_DW6_CH1;
 
 	vlv_dpio_get(dev_priv);
 	val = vlv_dpio_read(dev_priv, phy, reg);
@@ -1552,10 +1554,11 @@ static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpi
 	}
 
 	if (ch == DPIO_CH0)
-		actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
+		actual = REG_FIELD_GET(DPIO_ANYDL_POWERDOWN_CH0 |
+				       DPIO_ALLDL_POWERDOWN_CH0, val);
 	else
-		actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
-	actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
+		actual = REG_FIELD_GET(DPIO_ANYDL_POWERDOWN_CH1 |
+				       DPIO_ALLDL_POWERDOWN_CH1, val);
 
 	drm_WARN(&dev_priv->drm, actual != expected,
 		 "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
diff --git a/drivers/gpu/drm/i915/display/intel_display_reg_defs.h b/drivers/gpu/drm/i915/display/intel_display_reg_defs.h
index 2f07b7afa3bf..b83ad06f2ea7 100644
--- a/drivers/gpu/drm/i915/display/intel_display_reg_defs.h
+++ b/drivers/gpu/drm/i915/display/intel_display_reg_defs.h
@@ -29,21 +29,21 @@
 #define _MMIO_PLL(pll, a, b)		_MMIO(_PLL(pll, a, b))
 #define _MMIO_PHY(phy, a, b)		_MMIO(_PHY(phy, a, b))
 
-#define _MMIO_PIPE3(pipe, a, b, c)	_MMIO(_PICK_EVEN_2RANGES(pipe, 1, a, a, b, c))
-#define _MMIO_PORT3(pipe, a, b, c)	_MMIO(_PICK_EVEN_2RANGES(pipe, 1, a, a, b, c))
+#define _MMIO_BASE_PIPE3(base, pipe, a, b, c)	_MMIO((base) + _PICK_EVEN_2RANGES(pipe, 1, a, a, b, c))
+#define _MMIO_BASE_PORT3(base, pipe, a, b, c)	_MMIO((base) + _PICK_EVEN_2RANGES(pipe, 1, a, a, b, c))
 
 /*
  * Device info offset array based helpers for groups of registers with unevenly
  * spaced base offsets.
  */
-#define _MMIO_PIPE2(pipe, reg)		_MMIO(DISPLAY_INFO(dev_priv)->pipe_offsets[(pipe)] - \
-					      DISPLAY_INFO(dev_priv)->pipe_offsets[PIPE_A] + \
-					      DISPLAY_MMIO_BASE(dev_priv) + (reg))
-#define _MMIO_TRANS2(tran, reg)		_MMIO(DISPLAY_INFO(dev_priv)->trans_offsets[(tran)] - \
-					      DISPLAY_INFO(dev_priv)->trans_offsets[TRANSCODER_A] + \
-					      DISPLAY_MMIO_BASE(dev_priv) + (reg))
-#define _MMIO_CURSOR2(pipe, reg)	_MMIO(DISPLAY_INFO(dev_priv)->cursor_offsets[(pipe)] - \
-					      DISPLAY_INFO(dev_priv)->cursor_offsets[PIPE_A] + \
-					      DISPLAY_MMIO_BASE(dev_priv) + (reg))
+#define _MMIO_PIPE2(display, pipe, reg)		_MMIO(DISPLAY_INFO(display)->pipe_offsets[(pipe)] - \
+						      DISPLAY_INFO(display)->pipe_offsets[PIPE_A] + \
+						      DISPLAY_MMIO_BASE(display) + (reg))
+#define _MMIO_TRANS2(display, tran, reg)	_MMIO(DISPLAY_INFO(display)->trans_offsets[(tran)] - \
+						      DISPLAY_INFO(display)->trans_offsets[TRANSCODER_A] + \
+						      DISPLAY_MMIO_BASE(display) + (reg))
+#define _MMIO_CURSOR2(display, pipe, reg)	_MMIO(DISPLAY_INFO(display)->cursor_offsets[(pipe)] - \
+						      DISPLAY_INFO(display)->cursor_offsets[PIPE_A] + \
+						      DISPLAY_MMIO_BASE(display) + (reg))
 
 #endif /* __INTEL_DISPLAY_REG_DEFS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_trace.h b/drivers/gpu/drm/i915/display/intel_display_trace.h
index 7862e7cefe02..49a5e6d9dc0d 100644
--- a/drivers/gpu/drm/i915/display/intel_display_trace.h
+++ b/drivers/gpu/drm/i915/display/intel_display_trace.h
@@ -34,7 +34,7 @@ TRACE_EVENT(intel_pipe_enable,
 	    TP_fast_assign(
 			   struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 			   struct intel_crtc *it__;
-			   __assign_str(dev, __dev_name_kms(crtc));
+			   __assign_str(dev);
 			   for_each_intel_crtc(&dev_priv->drm, it__) {
 				   __entry->frame[it__->pipe] = intel_crtc_get_vblank_counter(it__);
 				   __entry->scanline[it__->pipe] = intel_get_crtc_scanline(it__);
@@ -63,7 +63,7 @@ TRACE_EVENT(intel_pipe_disable,
 	    TP_fast_assign(
 			   struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 			   struct intel_crtc *it__;
-			   __assign_str(dev, __dev_name_kms(crtc));
+			   __assign_str(dev);
 			   for_each_intel_crtc(&dev_priv->drm, it__) {
 				   __entry->frame[it__->pipe] = intel_crtc_get_vblank_counter(it__);
 				   __entry->scanline[it__->pipe] = intel_get_crtc_scanline(it__);
@@ -91,7 +91,7 @@ TRACE_EVENT(intel_pipe_crc,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_kms(crtc));
+			   __assign_str(dev);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -119,7 +119,7 @@ TRACE_EVENT(intel_cpu_fifo_underrun,
 
 	    TP_fast_assign(
 			    struct intel_crtc *crtc = intel_crtc_for_pipe(dev_priv, pipe);
-			   __assign_str(dev, __dev_name_kms(crtc));
+			   __assign_str(dev);
 			   __entry->pipe = pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -144,7 +144,7 @@ TRACE_EVENT(intel_pch_fifo_underrun,
 	    TP_fast_assign(
 			   enum pipe pipe = pch_transcoder;
 			   struct intel_crtc *crtc = intel_crtc_for_pipe(dev_priv, pipe);
-			   __assign_str(dev, __dev_name_i915(dev_priv));
+			   __assign_str(dev);
 			   __entry->pipe = pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -169,7 +169,7 @@ TRACE_EVENT(intel_memory_cxsr,
 
 	    TP_fast_assign(
 			   struct intel_crtc *crtc;
-			   __assign_str(dev, __dev_name_i915(dev_priv));
+			   __assign_str(dev);
 			   for_each_intel_crtc(&dev_priv->drm, crtc) {
 				   __entry->frame[crtc->pipe] = intel_crtc_get_vblank_counter(crtc);
 				   __entry->scanline[crtc->pipe] = intel_get_crtc_scanline(crtc);
@@ -209,7 +209,7 @@ TRACE_EVENT(g4x_wm,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_kms(crtc));
+			   __assign_str(dev);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -256,7 +256,7 @@ TRACE_EVENT(vlv_wm,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_kms(crtc));
+			   __assign_str(dev);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -293,7 +293,7 @@ TRACE_EVENT(vlv_fifo_size,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_kms(crtc));
+			   __assign_str(dev);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -323,8 +323,8 @@ TRACE_EVENT(intel_plane_update_noarm,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_kms(plane));
-			   __assign_str(name, plane->base.name);
+			   __assign_str(dev);
+			   __assign_str(name);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -354,8 +354,8 @@ TRACE_EVENT(intel_plane_update_arm,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_kms(plane));
-			   __assign_str(name, plane->base.name);
+			   __assign_str(dev);
+			   __assign_str(name);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -383,8 +383,8 @@ TRACE_EVENT(intel_plane_disable_arm,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_kms(plane));
-			   __assign_str(name, plane->base.name);
+			   __assign_str(dev);
+			   __assign_str(name);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -410,8 +410,8 @@ TRACE_EVENT(intel_fbc_activate,
 	    TP_fast_assign(
 			   struct intel_crtc *crtc = intel_crtc_for_pipe(to_i915(plane->base.dev),
 									 plane->pipe);
-			   __assign_str(dev, __dev_name_kms(plane));
-			   __assign_str(name, plane->base.name);
+			   __assign_str(dev);
+			   __assign_str(name);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -437,8 +437,8 @@ TRACE_EVENT(intel_fbc_deactivate,
 	    TP_fast_assign(
 			   struct intel_crtc *crtc = intel_crtc_for_pipe(to_i915(plane->base.dev),
 									 plane->pipe);
-			   __assign_str(dev, __dev_name_kms(plane));
-			   __assign_str(name, plane->base.name);
+			   __assign_str(dev);
+			   __assign_str(name);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -464,8 +464,8 @@ TRACE_EVENT(intel_fbc_nuke,
 	    TP_fast_assign(
 			   struct intel_crtc *crtc = intel_crtc_for_pipe(to_i915(plane->base.dev),
 									 plane->pipe);
-			   __assign_str(dev, __dev_name_kms(plane));
-			   __assign_str(name, plane->base.name);
+			   __assign_str(dev);
+			   __assign_str(name);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -488,7 +488,7 @@ TRACE_EVENT(intel_crtc_vblank_work_start,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_kms(crtc));
+			   __assign_str(dev);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -511,7 +511,7 @@ TRACE_EVENT(intel_crtc_vblank_work_end,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_kms(crtc));
+			   __assign_str(dev);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -536,7 +536,7 @@ TRACE_EVENT(intel_pipe_update_start,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_kms(crtc));
+			   __assign_str(dev);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = intel_crtc_get_vblank_counter(crtc);
 			   __entry->scanline = intel_get_crtc_scanline(crtc);
@@ -564,7 +564,7 @@ TRACE_EVENT(intel_pipe_update_vblank_evaded,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_kms(crtc));
+			   __assign_str(dev);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = crtc->debug.start_vbl_count;
 			   __entry->scanline = crtc->debug.scanline_start;
@@ -590,7 +590,7 @@ TRACE_EVENT(intel_pipe_update_end,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_kms(crtc));
+			   __assign_str(dev);
 			   __entry->pipe = crtc->pipe;
 			   __entry->frame = frame;
 			   __entry->scanline = scanline_end;
@@ -613,7 +613,7 @@ TRACE_EVENT(intel_frontbuffer_invalidate,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_i915(i915));
+			   __assign_str(dev);
 			   __entry->frontbuffer_bits = frontbuffer_bits;
 			   __entry->origin = origin;
 			   ),
@@ -634,7 +634,7 @@ TRACE_EVENT(intel_frontbuffer_flush,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(dev, __dev_name_i915(i915));
+			   __assign_str(dev);
 			   __entry->frontbuffer_bits = frontbuffer_bits;
 			   __entry->origin = origin;
 			   ),
diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h
index bf3f942e19c3..62f7a30c37dc 100644
--- a/drivers/gpu/drm/i915/display/intel_display_types.h
+++ b/drivers/gpu/drm/i915/display/intel_display_types.h
@@ -661,7 +661,8 @@ struct intel_digital_connector_state {
 	int broadcast_rgb;
 };
 
-#define to_intel_digital_connector_state(x) container_of(x, struct intel_digital_connector_state, base)
+#define to_intel_digital_connector_state(conn_state) \
+	container_of_const((conn_state), struct intel_digital_connector_state, base)
 
 struct dpll {
 	/* given values */
@@ -1003,18 +1004,6 @@ enum intel_output_format {
 	INTEL_OUTPUT_FORMAT_YCBCR444,
 };
 
-struct intel_mpllb_state {
-	u32 clock; /* in KHz */
-	u32 ref_control;
-	u32 mpllb_cp;
-	u32 mpllb_div;
-	u32 mpllb_div2;
-	u32 mpllb_fracn1;
-	u32 mpllb_fracn2;
-	u32 mpllb_sscen;
-	u32 mpllb_sscstep;
-};
-
 /* Used by dp and fdi links */
 struct intel_link_m_n {
 	u32 tu;
@@ -1030,31 +1019,6 @@ struct intel_csc_matrix {
 	u16 postoff[3];
 };
 
-struct intel_c10pll_state {
-	u32 clock; /* in KHz */
-	u8 tx;
-	u8 cmn;
-	u8 pll[20];
-};
-
-struct intel_c20pll_state {
-	u32 clock; /* in kHz */
-	u16 tx[3];
-	u16 cmn[4];
-	union {
-		u16 mplla[10];
-		u16 mpllb[11];
-	};
-};
-
-struct intel_cx0pll_state {
-	union {
-		struct intel_c10pll_state c10;
-		struct intel_c20pll_state c20;
-	};
-	bool ssc_enabled;
-};
-
 struct intel_crtc_state {
 	/*
 	 * uapi (drm) state. This is the software state shown to userspace.
@@ -1199,11 +1163,7 @@ struct intel_crtc_state {
 	struct intel_shared_dpll *shared_dpll;
 
 	/* Actual register state of the dpll, for shared dpll cross-checking. */
-	union {
-		struct intel_dpll_hw_state dpll_hw_state;
-		struct intel_mpllb_state mpllb_state;
-		struct intel_cx0pll_state cx0pll_state;
-	};
+	struct intel_dpll_hw_state dpll_hw_state;
 
 	/*
 	 * ICL reserved DPLLs for the CRTC/port. The active PLL is selected by
@@ -1346,6 +1306,7 @@ struct intel_crtc_state {
 		union hdmi_infoframe hdmi;
 		union hdmi_infoframe drm;
 		struct drm_dp_vsc_sdp vsc;
+		struct drm_dp_as_sdp as_sdp;
 	} infoframes;
 
 	u8 eld[MAX_ELD_BYTES];
@@ -1432,6 +1393,7 @@ struct intel_crtc_state {
 		bool enable, in_range;
 		u8 pipeline_full;
 		u16 flipline, vmin, vmax, guardband;
+		u32 vsync_end, vsync_start;
 	} vrr;
 
 	/* Stream Splitter for eDP MSO */
@@ -1620,12 +1582,17 @@ struct intel_watermark_params {
 
 #define to_intel_atomic_state(x) container_of(x, struct intel_atomic_state, base)
 #define to_intel_crtc(x) container_of(x, struct intel_crtc, base)
-#define to_intel_crtc_state(x) container_of(x, struct intel_crtc_state, uapi)
 #define to_intel_connector(x) container_of(x, struct intel_connector, base)
 #define to_intel_encoder(x) container_of(x, struct intel_encoder, base)
-#define to_intel_framebuffer(x) container_of(x, struct intel_framebuffer, base)
 #define to_intel_plane(x) container_of(x, struct intel_plane, base)
-#define to_intel_plane_state(x) container_of(x, struct intel_plane_state, uapi)
+
+#define to_intel_crtc_state(crtc_state) \
+	container_of_const((crtc_state), struct intel_crtc_state, uapi)
+#define to_intel_plane_state(plane_state) \
+	container_of_const((plane_state), struct intel_plane_state, uapi)
+#define to_intel_framebuffer(fb) \
+	container_of_const((fb), struct intel_framebuffer, base)
+
 #define intel_fb_obj(x) ((x) ? to_intel_bo((x)->obj[0]) : NULL)
 
 struct intel_hdmi {
@@ -1740,6 +1707,8 @@ struct intel_psr {
 
 		/* LNL and beyond */
 		u8 check_entry_lines;
+		u8 silence_period_sym_clocks;
+		u8 lfps_half_cycle_num_of_syms;
 	} alpm_parameters;
 
 	ktime_t last_entry_attempt;
@@ -1801,6 +1770,7 @@ struct intel_dp {
 
 	bool is_mst;
 	int active_mst_links;
+	enum drm_dp_mst_mode mst_detect;
 
 	/* connector directly attached - won't be use for modeset in mst world */
 	struct intel_connector *attached_connector;
@@ -2186,4 +2156,41 @@ static inline int to_bpp_x16(int bpp)
 	return bpp << 4;
 }
 
+/*
+ * Conversion functions/macros from various pointer types to struct
+ * intel_display pointer.
+ */
+#define __drm_device_to_intel_display(p) \
+	(&to_i915(p)->display)
+#define __intel_connector_to_intel_display(p)		\
+	__drm_device_to_intel_display((p)->base.dev)
+#define __intel_crtc_to_intel_display(p)		\
+	__drm_device_to_intel_display((p)->base.dev)
+#define __intel_crtc_state_to_intel_display(p)			\
+	__drm_device_to_intel_display((p)->uapi.crtc->dev)
+#define __intel_digital_port_to_intel_display(p)		\
+	__drm_device_to_intel_display((p)->base.base.dev)
+#define __intel_dp_to_intel_display(p)	\
+	__drm_device_to_intel_display(dp_to_dig_port(p)->base.base.dev)
+#define __intel_encoder_to_intel_display(p)		\
+	__drm_device_to_intel_display((p)->base.dev)
+#define __intel_hdmi_to_intel_display(p)	\
+	__drm_device_to_intel_display(hdmi_to_dig_port(p)->base.base.dev)
+
+/* Helper for generic association. Map types to conversion functions/macros. */
+#define __assoc(type, p) \
+	struct type: __##type##_to_intel_display((struct type *)(p))
+
+/* Convert various pointer types to struct intel_display pointer. */
+#define to_intel_display(p)				\
+	_Generic(*p,					\
+		 __assoc(drm_device, p),		\
+		 __assoc(intel_connector, p),		\
+		 __assoc(intel_crtc, p),		\
+		 __assoc(intel_crtc_state, p),		\
+		 __assoc(intel_digital_port, p),	\
+		 __assoc(intel_dp, p),			\
+		 __assoc(intel_encoder, p),		\
+		 __assoc(intel_hdmi, p))
+
 #endif /*  __INTEL_DISPLAY_TYPES_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_wa.c b/drivers/gpu/drm/i915/display/intel_display_wa.c
index ac136fd992ba..e5a8022db664 100644
--- a/drivers/gpu/drm/i915/display/intel_display_wa.c
+++ b/drivers/gpu/drm/i915/display/intel_display_wa.c
@@ -10,20 +10,12 @@
 
 static void gen11_display_wa_apply(struct drm_i915_private *i915)
 {
-	/* Wa_1409120013 */
-	intel_de_write(i915, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-		       DPFC_CHICKEN_COMP_DUMMY_PIXEL);
-
 	/* Wa_14010594013 */
 	intel_de_rmw(i915, GEN8_CHICKEN_DCPR_1, 0, ICL_DELAY_PMRSP);
 }
 
 static void xe_d_display_wa_apply(struct drm_i915_private *i915)
 {
-	/* Wa_1409120013 */
-	intel_de_write(i915, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-		       DPFC_CHICKEN_COMP_DUMMY_PIXEL);
-
 	/* Wa_14013723622 */
 	intel_de_rmw(i915, CLKREQ_POLICY, CLKREQ_POLICY_MEM_UP_OVRD, 0);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c
index 835781624482..cbd2ac5671b1 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc.c
+++ b/drivers/gpu/drm/i915/display/intel_dmc.c
@@ -22,6 +22,7 @@
  *
  */
 
+#include <linux/debugfs.h>
 #include <linux/firmware.h>
 
 #include "i915_drv.h"
@@ -38,6 +39,8 @@
  * low-power state and comes back to normal.
  */
 
+#define INTEL_DMC_FIRMWARE_URL "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git"
+
 enum intel_dmc_id {
 	DMC_FW_MAIN = 0,
 	DMC_FW_PIPEA,
@@ -71,6 +74,21 @@ static struct intel_dmc *i915_to_dmc(struct drm_i915_private *i915)
 	return i915->display.dmc.dmc;
 }
 
+static const char *dmc_firmware_param(struct drm_i915_private *i915)
+{
+	const char *p = i915->display.params.dmc_firmware_path;
+
+	return p && *p ? p : NULL;
+}
+
+static bool dmc_firmware_param_disabled(struct drm_i915_private *i915)
+{
+	const char *p = dmc_firmware_param(i915);
+
+	/* Magic path to indicate disabled */
+	return p && !strcmp(p, "/dev/null");
+}
+
 #define DMC_VERSION(major, minor)	((major) << 16 | (minor))
 #define DMC_VERSION_MAJOR(version)	((version) >> 16)
 #define DMC_VERSION_MINOR(version)	((version) & 0xffff)
@@ -89,10 +107,14 @@ static struct intel_dmc *i915_to_dmc(struct drm_i915_private *i915)
 	__stringify(major) "_"			\
 	__stringify(minor) ".bin"
 
+#define XE2LPD_DMC_MAX_FW_SIZE		0x8000
 #define XELPDP_DMC_MAX_FW_SIZE		0x7000
 #define DISPLAY_VER13_DMC_MAX_FW_SIZE	0x20000
 #define DISPLAY_VER12_DMC_MAX_FW_SIZE	ICL_DMC_MAX_FW_SIZE
 
+#define XE2LPD_DMC_PATH			DMC_PATH(xe2lpd)
+MODULE_FIRMWARE(XE2LPD_DMC_PATH);
+
 #define MTL_DMC_PATH			DMC_PATH(mtl)
 MODULE_FIRMWARE(MTL_DMC_PATH);
 
@@ -136,6 +158,59 @@ MODULE_FIRMWARE(SKL_DMC_PATH);
 #define BXT_DMC_MAX_FW_SIZE		0x3000
 MODULE_FIRMWARE(BXT_DMC_PATH);
 
+static const char *dmc_firmware_default(struct drm_i915_private *i915, u32 *size)
+{
+	const char *fw_path = NULL;
+	u32 max_fw_size = 0;
+
+	if (DISPLAY_VER_FULL(i915) == IP_VER(20, 0)) {
+		fw_path = XE2LPD_DMC_PATH;
+		max_fw_size = XE2LPD_DMC_MAX_FW_SIZE;
+	} else if (DISPLAY_VER_FULL(i915) == IP_VER(14, 0)) {
+		fw_path = MTL_DMC_PATH;
+		max_fw_size = XELPDP_DMC_MAX_FW_SIZE;
+	} else if (IS_DG2(i915)) {
+		fw_path = DG2_DMC_PATH;
+		max_fw_size = DISPLAY_VER13_DMC_MAX_FW_SIZE;
+	} else if (IS_ALDERLAKE_P(i915)) {
+		fw_path = ADLP_DMC_PATH;
+		max_fw_size = DISPLAY_VER13_DMC_MAX_FW_SIZE;
+	} else if (IS_ALDERLAKE_S(i915)) {
+		fw_path = ADLS_DMC_PATH;
+		max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
+	} else if (IS_DG1(i915)) {
+		fw_path = DG1_DMC_PATH;
+		max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
+	} else if (IS_ROCKETLAKE(i915)) {
+		fw_path = RKL_DMC_PATH;
+		max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
+	} else if (IS_TIGERLAKE(i915)) {
+		fw_path = TGL_DMC_PATH;
+		max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
+	} else if (DISPLAY_VER(i915) == 11) {
+		fw_path = ICL_DMC_PATH;
+		max_fw_size = ICL_DMC_MAX_FW_SIZE;
+	} else if (IS_GEMINILAKE(i915)) {
+		fw_path = GLK_DMC_PATH;
+		max_fw_size = GLK_DMC_MAX_FW_SIZE;
+	} else if (IS_KABYLAKE(i915) ||
+		   IS_COFFEELAKE(i915) ||
+		   IS_COMETLAKE(i915)) {
+		fw_path = KBL_DMC_PATH;
+		max_fw_size = KBL_DMC_MAX_FW_SIZE;
+	} else if (IS_SKYLAKE(i915)) {
+		fw_path = SKL_DMC_PATH;
+		max_fw_size = SKL_DMC_MAX_FW_SIZE;
+	} else if (IS_BROXTON(i915)) {
+		fw_path = BXT_DMC_PATH;
+		max_fw_size = BXT_DMC_MAX_FW_SIZE;
+	}
+
+	*size = max_fw_size;
+
+	return fw_path;
+}
+
 #define DMC_DEFAULT_FW_OFFSET		0xFFFFFFFF
 #define PACKAGE_MAX_FW_INFO_ENTRIES	20
 #define PACKAGE_V2_MAX_FW_INFO_ENTRIES	32
@@ -546,6 +621,8 @@ void intel_dmc_disable_program(struct drm_i915_private *i915)
 	pipedmc_clock_gating_wa(i915, true);
 	disable_all_event_handlers(i915);
 	pipedmc_clock_gating_wa(i915, false);
+
+	intel_dmc_wl_disable(&i915->display);
 }
 
 void assert_dmc_loaded(struct drm_i915_private *i915)
@@ -845,7 +922,7 @@ static u32 parse_dmc_fw_css(struct intel_dmc *dmc,
 	return sizeof(struct intel_css_header);
 }
 
-static void parse_dmc_fw(struct intel_dmc *dmc, const struct firmware *fw)
+static int parse_dmc_fw(struct intel_dmc *dmc, const struct firmware *fw)
 {
 	struct drm_i915_private *i915 = dmc->i915;
 	struct intel_css_header *css_header;
@@ -858,13 +935,13 @@ static void parse_dmc_fw(struct intel_dmc *dmc, const struct firmware *fw)
 	u32 r, offset;
 
 	if (!fw)
-		return;
+		return -EINVAL;
 
 	/* Extract CSS Header information */
 	css_header = (struct intel_css_header *)fw->data;
 	r = parse_dmc_fw_css(dmc, css_header, fw->size);
 	if (!r)
-		return;
+		return -EINVAL;
 
 	readcount += r;
 
@@ -872,7 +949,7 @@ static void parse_dmc_fw(struct intel_dmc *dmc, const struct firmware *fw)
 	package_header = (struct intel_package_header *)&fw->data[readcount];
 	r = parse_dmc_fw_package(dmc, package_header, si, fw->size - readcount);
 	if (!r)
-		return;
+		return -EINVAL;
 
 	readcount += r;
 
@@ -889,6 +966,13 @@ static void parse_dmc_fw(struct intel_dmc *dmc, const struct firmware *fw)
 		dmc_header = (struct intel_dmc_header_base *)&fw->data[offset];
 		parse_dmc_fw_header(dmc, dmc_header, fw->size - offset, dmc_id);
 	}
+
+	if (!intel_dmc_has_payload(i915)) {
+		drm_err(&i915->drm, "DMC firmware main program not found\n");
+		return -ENOENT;
+	}
+
+	return 0;
 }
 
 static void intel_dmc_runtime_pm_get(struct drm_i915_private *i915)
@@ -923,7 +1007,7 @@ static void dmc_load_work_fn(struct work_struct *work)
 
 	err = request_firmware(&fw, dmc->fw_path, i915->drm.dev);
 
-	if (err == -ENOENT && !i915->params.dmc_firmware_path) {
+	if (err == -ENOENT && !dmc_firmware_param(i915)) {
 		fallback_path = dmc_fallback_path(i915);
 		if (fallback_path) {
 			drm_dbg_kms(&i915->drm, "%s not found, falling back to %s\n",
@@ -934,24 +1018,31 @@ static void dmc_load_work_fn(struct work_struct *work)
 		}
 	}
 
-	parse_dmc_fw(dmc, fw);
-
-	if (intel_dmc_has_payload(i915)) {
-		intel_dmc_load_program(i915);
-		intel_dmc_runtime_pm_put(i915);
-
-		drm_info(&i915->drm, "Finished loading DMC firmware %s (v%u.%u)\n",
-			 dmc->fw_path, DMC_VERSION_MAJOR(dmc->version),
-			 DMC_VERSION_MINOR(dmc->version));
-	} else {
+	if (err) {
 		drm_notice(&i915->drm,
-			   "Failed to load DMC firmware %s."
-			   " Disabling runtime power management.\n",
-			   dmc->fw_path);
+			   "Failed to load DMC firmware %s (%pe). Disabling runtime power management.\n",
+			   dmc->fw_path, ERR_PTR(err));
 		drm_notice(&i915->drm, "DMC firmware homepage: %s",
-			   INTEL_UC_FIRMWARE_URL);
+			   INTEL_DMC_FIRMWARE_URL);
+		return;
 	}
 
+	err = parse_dmc_fw(dmc, fw);
+	if (err) {
+		drm_notice(&i915->drm,
+			   "Failed to parse DMC firmware %s (%pe). Disabling runtime power management.\n",
+			   dmc->fw_path, ERR_PTR(err));
+		goto out;
+	}
+
+	intel_dmc_load_program(i915);
+	intel_dmc_runtime_pm_put(i915);
+
+	drm_info(&i915->drm, "Finished loading DMC firmware %s (v%u.%u)\n",
+		 dmc->fw_path, DMC_VERSION_MAJOR(dmc->version),
+		 DMC_VERSION_MINOR(dmc->version));
+
+out:
 	release_firmware(fw);
 }
 
@@ -987,56 +1078,16 @@ void intel_dmc_init(struct drm_i915_private *i915)
 
 	INIT_WORK(&dmc->work, dmc_load_work_fn);
 
-	if (DISPLAY_VER_FULL(i915) == IP_VER(14, 0)) {
-		dmc->fw_path = MTL_DMC_PATH;
-		dmc->max_fw_size = XELPDP_DMC_MAX_FW_SIZE;
-	} else if (IS_DG2(i915)) {
-		dmc->fw_path = DG2_DMC_PATH;
-		dmc->max_fw_size = DISPLAY_VER13_DMC_MAX_FW_SIZE;
-	} else if (IS_ALDERLAKE_P(i915)) {
-		dmc->fw_path = ADLP_DMC_PATH;
-		dmc->max_fw_size = DISPLAY_VER13_DMC_MAX_FW_SIZE;
-	} else if (IS_ALDERLAKE_S(i915)) {
-		dmc->fw_path = ADLS_DMC_PATH;
-		dmc->max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
-	} else if (IS_DG1(i915)) {
-		dmc->fw_path = DG1_DMC_PATH;
-		dmc->max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
-	} else if (IS_ROCKETLAKE(i915)) {
-		dmc->fw_path = RKL_DMC_PATH;
-		dmc->max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
-	} else if (IS_TIGERLAKE(i915)) {
-		dmc->fw_path = TGL_DMC_PATH;
-		dmc->max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
-	} else if (DISPLAY_VER(i915) == 11) {
-		dmc->fw_path = ICL_DMC_PATH;
-		dmc->max_fw_size = ICL_DMC_MAX_FW_SIZE;
-	} else if (IS_GEMINILAKE(i915)) {
-		dmc->fw_path = GLK_DMC_PATH;
-		dmc->max_fw_size = GLK_DMC_MAX_FW_SIZE;
-	} else if (IS_KABYLAKE(i915) ||
-		   IS_COFFEELAKE(i915) ||
-		   IS_COMETLAKE(i915)) {
-		dmc->fw_path = KBL_DMC_PATH;
-		dmc->max_fw_size = KBL_DMC_MAX_FW_SIZE;
-	} else if (IS_SKYLAKE(i915)) {
-		dmc->fw_path = SKL_DMC_PATH;
-		dmc->max_fw_size = SKL_DMC_MAX_FW_SIZE;
-	} else if (IS_BROXTON(i915)) {
-		dmc->fw_path = BXT_DMC_PATH;
-		dmc->max_fw_size = BXT_DMC_MAX_FW_SIZE;
-	}
-
-	if (i915->params.dmc_firmware_path) {
-		if (strlen(i915->params.dmc_firmware_path) == 0) {
-			drm_info(&i915->drm,
-				 "Disabling DMC firmware and runtime PM\n");
-			goto out;
-		}
+	dmc->fw_path = dmc_firmware_default(i915, &dmc->max_fw_size);
 
-		dmc->fw_path = i915->params.dmc_firmware_path;
+	if (dmc_firmware_param_disabled(i915)) {
+		drm_info(&i915->drm, "Disabling DMC firmware and runtime PM\n");
+		goto out;
 	}
 
+	if (dmc_firmware_param(i915))
+		dmc->fw_path = dmc_firmware_param(i915);
+
 	if (!dmc->fw_path) {
 		drm_dbg_kms(&i915->drm,
 			    "No known DMC firmware for platform, disabling runtime PM\n");
@@ -1072,6 +1123,8 @@ void intel_dmc_suspend(struct drm_i915_private *i915)
 	if (dmc)
 		flush_work(&dmc->work);
 
+	intel_dmc_wl_disable(&i915->display);
+
 	/* Drop the reference held in case DMC isn't loaded. */
 	if (!intel_dmc_has_payload(i915))
 		intel_dmc_runtime_pm_put(i915);
diff --git a/drivers/gpu/drm/i915/display/intel_dmc_regs.h b/drivers/gpu/drm/i915/display/intel_dmc_regs.h
index 90d0dbb41cfe..1bf446f96a10 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_dmc_regs.h
@@ -97,4 +97,10 @@
 #define TGL_DMC_DEBUG3		_MMIO(0x101090)
 #define DG1_DMC_DEBUG3		_MMIO(0x13415c)
 
+#define DMC_WAKELOCK_CFG	_MMIO(0x8F1B0)
+#define  DMC_WAKELOCK_CFG_ENABLE REG_BIT(31)
+#define DMC_WAKELOCK1_CTL	_MMIO(0x8F140)
+#define  DMC_WAKELOCK_CTL_REQ	 REG_BIT(31)
+#define  DMC_WAKELOCK_CTL_ACK	 REG_BIT(15)
+
 #endif /* __INTEL_DMC_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dmc_wl.c b/drivers/gpu/drm/i915/display/intel_dmc_wl.c
new file mode 100644
index 000000000000..d9864b9cc429
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dmc_wl.c
@@ -0,0 +1,264 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+
+#include <linux/kernel.h>
+
+#include "intel_de.h"
+#include "intel_dmc.h"
+#include "intel_dmc_regs.h"
+#include "intel_dmc_wl.h"
+
+/**
+ * DOC: DMC wakelock support
+ *
+ * Wake lock is the mechanism to cause display engine to exit DC
+ * states to allow programming to registers that are powered down in
+ * those states. Previous projects exited DC states automatically when
+ * detecting programming. Now software controls the exit by
+ * programming the wake lock. This improves system performance and
+ * system interactions and better fits the flip queue style of
+ * programming. Wake lock is only required when DC5, DC6, or DC6v have
+ * been enabled in DC_STATE_EN and the wake lock mode of operation has
+ * been enabled.
+ *
+ * The wakelock mechanism in DMC allows the display engine to exit DC
+ * states explicitly before programming registers that may be powered
+ * down.  In earlier hardware, this was done automatically and
+ * implicitly when the display engine accessed a register.  With the
+ * wakelock implementation, the driver asserts a wakelock in DMC,
+ * which forces it to exit the DC state until the wakelock is
+ * deasserted.
+ *
+ * The mechanism can be enabled and disabled by writing to the
+ * DMC_WAKELOCK_CFG register.  There are also 13 control registers
+ * that can be used to hold and release different wakelocks.  In the
+ * current implementation, we only need one wakelock, so only
+ * DMC_WAKELOCK1_CTL is used.  The other definitions are here for
+ * potential future use.
+ */
+
+#define DMC_WAKELOCK_CTL_TIMEOUT 5
+#define DMC_WAKELOCK_HOLD_TIME 50
+
+struct intel_dmc_wl_range {
+	u32 start;
+	u32 end;
+};
+
+static struct intel_dmc_wl_range lnl_wl_range[] = {
+	{ .start = 0x60000, .end = 0x7ffff },
+};
+
+static void __intel_dmc_wl_release(struct intel_display *display)
+{
+	struct drm_i915_private *i915 = to_i915(display->drm);
+	struct intel_dmc_wl *wl = &display->wl;
+
+	WARN_ON(refcount_read(&wl->refcount));
+
+	queue_delayed_work(i915->unordered_wq, &wl->work,
+			   msecs_to_jiffies(DMC_WAKELOCK_HOLD_TIME));
+}
+
+static void intel_dmc_wl_work(struct work_struct *work)
+{
+	struct intel_dmc_wl *wl =
+		container_of(work, struct intel_dmc_wl, work.work);
+	struct intel_display *display =
+		container_of(wl, struct intel_display, wl);
+	unsigned long flags;
+
+	spin_lock_irqsave(&wl->lock, flags);
+
+	/* Bail out if refcount reached zero while waiting for the spinlock */
+	if (!refcount_read(&wl->refcount))
+		goto out_unlock;
+
+	__intel_de_rmw_nowl(display, DMC_WAKELOCK1_CTL, DMC_WAKELOCK_CTL_REQ, 0);
+
+	if (__intel_de_wait_for_register_nowl(display, DMC_WAKELOCK1_CTL,
+					      DMC_WAKELOCK_CTL_ACK, 0,
+					      DMC_WAKELOCK_CTL_TIMEOUT)) {
+		WARN_RATELIMIT(1, "DMC wakelock release timed out");
+		goto out_unlock;
+	}
+
+	wl->taken = false;
+
+out_unlock:
+	spin_unlock_irqrestore(&wl->lock, flags);
+}
+
+static bool intel_dmc_wl_check_range(u32 address)
+{
+	int i;
+	bool wl_needed = false;
+
+	for (i = 0; i < ARRAY_SIZE(lnl_wl_range); i++) {
+		if (address >= lnl_wl_range[i].start &&
+		    address <= lnl_wl_range[i].end) {
+			wl_needed = true;
+			break;
+		}
+	}
+
+	return wl_needed;
+}
+
+static bool __intel_dmc_wl_supported(struct intel_display *display)
+{
+	struct drm_i915_private *i915 = to_i915(display->drm);
+
+	if (DISPLAY_VER(display) < 20 ||
+	    !intel_dmc_has_payload(i915) ||
+	    !display->params.enable_dmc_wl)
+		return false;
+
+	return true;
+}
+
+void intel_dmc_wl_init(struct intel_display *display)
+{
+	struct intel_dmc_wl *wl = &display->wl;
+
+	/* don't call __intel_dmc_wl_supported(), DMC is not loaded yet */
+	if (DISPLAY_VER(display) < 20 || !display->params.enable_dmc_wl)
+		return;
+
+	INIT_DELAYED_WORK(&wl->work, intel_dmc_wl_work);
+	spin_lock_init(&wl->lock);
+	refcount_set(&wl->refcount, 0);
+}
+
+void intel_dmc_wl_enable(struct intel_display *display)
+{
+	struct intel_dmc_wl *wl = &display->wl;
+	unsigned long flags;
+
+	if (!__intel_dmc_wl_supported(display))
+		return;
+
+	spin_lock_irqsave(&wl->lock, flags);
+
+	if (wl->enabled)
+		goto out_unlock;
+
+	/*
+	 * Enable wakelock in DMC.  We shouldn't try to take the
+	 * wakelock, because we're just enabling it, so call the
+	 * non-locking version directly here.
+	 */
+	__intel_de_rmw_nowl(display, DMC_WAKELOCK_CFG, 0, DMC_WAKELOCK_CFG_ENABLE);
+
+	wl->enabled = true;
+	wl->taken = false;
+
+out_unlock:
+	spin_unlock_irqrestore(&wl->lock, flags);
+}
+
+void intel_dmc_wl_disable(struct intel_display *display)
+{
+	struct intel_dmc_wl *wl = &display->wl;
+	unsigned long flags;
+
+	if (!__intel_dmc_wl_supported(display))
+		return;
+
+	flush_delayed_work(&wl->work);
+
+	spin_lock_irqsave(&wl->lock, flags);
+
+	if (!wl->enabled)
+		goto out_unlock;
+
+	/* Disable wakelock in DMC */
+	__intel_de_rmw_nowl(display, DMC_WAKELOCK_CFG, DMC_WAKELOCK_CFG_ENABLE, 0);
+
+	refcount_set(&wl->refcount, 0);
+	wl->enabled = false;
+	wl->taken = false;
+
+out_unlock:
+	spin_unlock_irqrestore(&wl->lock, flags);
+}
+
+void intel_dmc_wl_get(struct intel_display *display, i915_reg_t reg)
+{
+	struct intel_dmc_wl *wl = &display->wl;
+	unsigned long flags;
+
+	if (!__intel_dmc_wl_supported(display))
+		return;
+
+	if (!intel_dmc_wl_check_range(reg.reg))
+		return;
+
+	spin_lock_irqsave(&wl->lock, flags);
+
+	if (!wl->enabled)
+		goto out_unlock;
+
+	cancel_delayed_work(&wl->work);
+
+	if (refcount_inc_not_zero(&wl->refcount))
+		goto out_unlock;
+
+	refcount_set(&wl->refcount, 1);
+
+	/*
+	 * Only try to take the wakelock if it's not marked as taken
+	 * yet.  It may be already taken at this point if we have
+	 * already released the last reference, but the work has not
+	 * run yet.
+	 */
+	if (!wl->taken) {
+		__intel_de_rmw_nowl(display, DMC_WAKELOCK1_CTL, 0,
+				    DMC_WAKELOCK_CTL_REQ);
+
+		if (__intel_de_wait_for_register_nowl(display, DMC_WAKELOCK1_CTL,
+						      DMC_WAKELOCK_CTL_ACK,
+						      DMC_WAKELOCK_CTL_ACK,
+						      DMC_WAKELOCK_CTL_TIMEOUT)) {
+			WARN_RATELIMIT(1, "DMC wakelock ack timed out");
+			goto out_unlock;
+		}
+
+		wl->taken = true;
+	}
+
+out_unlock:
+	spin_unlock_irqrestore(&wl->lock, flags);
+}
+
+void intel_dmc_wl_put(struct intel_display *display, i915_reg_t reg)
+{
+	struct intel_dmc_wl *wl = &display->wl;
+	unsigned long flags;
+
+	if (!__intel_dmc_wl_supported(display))
+		return;
+
+	if (!intel_dmc_wl_check_range(reg.reg))
+		return;
+
+	spin_lock_irqsave(&wl->lock, flags);
+
+	if (!wl->enabled)
+		goto out_unlock;
+
+	if (WARN_RATELIMIT(!refcount_read(&wl->refcount),
+			   "Tried to put wakelock with refcount zero\n"))
+		goto out_unlock;
+
+	if (refcount_dec_and_test(&wl->refcount)) {
+		__intel_dmc_wl_release(display);
+
+		goto out_unlock;
+	}
+
+out_unlock:
+	spin_unlock_irqrestore(&wl->lock, flags);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_dmc_wl.h b/drivers/gpu/drm/i915/display/intel_dmc_wl.h
new file mode 100644
index 000000000000..adab51208d0a
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dmc_wl.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright (C) 2024 Intel Corporation
+ */
+
+#ifndef __INTEL_WAKELOCK_H__
+#define __INTEL_WAKELOCK_H__
+
+#include <linux/types.h>
+#include <linux/workqueue.h>
+#include <linux/refcount.h>
+
+#include "i915_reg_defs.h"
+
+struct intel_display;
+
+struct intel_dmc_wl {
+	spinlock_t lock; /* protects enabled, taken  and refcount */
+	bool enabled;
+	bool taken;
+	refcount_t refcount;
+	struct delayed_work work;
+};
+
+void intel_dmc_wl_init(struct intel_display *display);
+void intel_dmc_wl_enable(struct intel_display *display);
+void intel_dmc_wl_disable(struct intel_display *display);
+void intel_dmc_wl_get(struct intel_display *display, i915_reg_t reg);
+void intel_dmc_wl_put(struct intel_display *display, i915_reg_t reg);
+
+#endif /* __INTEL_WAKELOCK_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index e583515f9b25..e05e25cd4a94 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -89,6 +89,9 @@
 #define DP_DSC_MAX_ENC_THROUGHPUT_0		340000
 #define DP_DSC_MAX_ENC_THROUGHPUT_1		400000
 
+/* Max DSC line buffer depth supported by HW. */
+#define INTEL_DP_DSC_MAX_LINE_BUF_DEPTH		13
+
 /* DP DSC FEC Overhead factor in ppm = 1/(0.972261) = 1.028530 */
 #define DP_DSC_FEC_OVERHEAD_FACTOR		1028530
 
@@ -123,6 +126,14 @@ bool intel_dp_is_edp(struct intel_dp *intel_dp)
 	return dig_port->base.type == INTEL_OUTPUT_EDP;
 }
 
+bool intel_dp_as_sdp_supported(struct intel_dp *intel_dp)
+{
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
+
+	return HAS_AS_SDP(i915) &&
+		drm_dp_as_sdp_supported(&intel_dp->aux, intel_dp->dpcd);
+}
+
 static void intel_dp_unset_edid(struct intel_dp *intel_dp);
 
 /* Is link rate UHBR and thus 128b/132b? */
@@ -214,7 +225,7 @@ static void intel_dp_set_dpcd_sink_rates(struct intel_dp *intel_dp)
 	 * Sink rates for 128b/132b. If set, sink should support all 8b/10b
 	 * rates and 10 Gbps.
 	 */
-	if (intel_dp->dpcd[DP_MAIN_LINK_CHANNEL_CODING] & DP_CAP_ANSI_128B132B) {
+	if (drm_dp_128b132b_supported(intel_dp->dpcd)) {
 		u8 uhbr_rates = 0;
 
 		BUILD_BUG_ON(ARRAY_SIZE(intel_dp->sink_rates) < ARRAY_SIZE(dp_rates) + 3);
@@ -425,7 +436,7 @@ int intel_dp_max_link_data_rate(struct intel_dp *intel_dp,
 	return max_rate;
 }
 
-bool intel_dp_can_bigjoiner(struct intel_dp *intel_dp)
+bool intel_dp_has_bigjoiner(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct intel_encoder *encoder = &intel_dig_port->base;
@@ -443,11 +454,9 @@ static int dg2_max_source_rate(struct intel_dp *intel_dp)
 
 static int icl_max_source_rate(struct intel_dp *intel_dp)
 {
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-	enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
+	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 
-	if (intel_phy_is_combo(dev_priv, phy) && !intel_dp_is_edp(intel_dp))
+	if (intel_encoder_is_combo(encoder) && !intel_dp_is_edp(intel_dp))
 		return 540000;
 
 	return 810000;
@@ -463,11 +472,9 @@ static int ehl_max_source_rate(struct intel_dp *intel_dp)
 
 static int mtl_max_source_rate(struct intel_dp *intel_dp)
 {
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
+	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 
-	if (intel_is_c10phy(i915, phy))
+	if (intel_encoder_is_c10phy(encoder))
 		return 810000;
 
 	return 2000000;
@@ -1198,15 +1205,15 @@ intel_dp_mode_valid_downstream(struct intel_connector *connector,
 }
 
 bool intel_dp_need_bigjoiner(struct intel_dp *intel_dp,
+			     struct intel_connector *connector,
 			     int hdisplay, int clock)
 {
 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
-	struct intel_connector *connector = intel_dp->attached_connector;
 
-	if (!intel_dp_can_bigjoiner(intel_dp))
+	if (!intel_dp_has_bigjoiner(intel_dp))
 		return false;
 
-	return clock > i915->max_dotclk_freq || hdisplay > 5120 ||
+	return clock > i915->display.cdclk.max_dotclk_freq || hdisplay > 5120 ||
 	       connector->force_bigjoiner_enable;
 }
 
@@ -1220,7 +1227,7 @@ intel_dp_mode_valid(struct drm_connector *_connector,
 	const struct drm_display_mode *fixed_mode;
 	int target_clock = mode->clock;
 	int max_rate, mode_rate, max_lanes, max_link_clock;
-	int max_dotclk = dev_priv->max_dotclk_freq;
+	int max_dotclk = dev_priv->display.cdclk.max_dotclk_freq;
 	u16 dsc_max_compressed_bpp = 0;
 	u8 dsc_slice_count = 0;
 	enum drm_mode_status status;
@@ -1233,6 +1240,9 @@ intel_dp_mode_valid(struct drm_connector *_connector,
 	if (mode->flags & DRM_MODE_FLAG_DBLCLK)
 		return MODE_H_ILLEGAL;
 
+	if (mode->clock < 10000)
+		return MODE_CLOCK_LOW;
+
 	fixed_mode = intel_panel_fixed_mode(connector, mode);
 	if (intel_dp_is_edp(intel_dp) && fixed_mode) {
 		status = intel_panel_mode_valid(connector, mode);
@@ -1242,10 +1252,8 @@ intel_dp_mode_valid(struct drm_connector *_connector,
 		target_clock = fixed_mode->clock;
 	}
 
-	if (mode->clock < 10000)
-		return MODE_CLOCK_LOW;
-
-	if (intel_dp_need_bigjoiner(intel_dp, mode->hdisplay, target_clock)) {
+	if (intel_dp_need_bigjoiner(intel_dp, connector,
+				    mode->hdisplay, target_clock)) {
 		bigjoiner = true;
 		max_dotclk *= 2;
 	}
@@ -1306,11 +1314,7 @@ intel_dp_mode_valid(struct drm_connector *_connector,
 		dsc = dsc_max_compressed_bpp && dsc_slice_count;
 	}
 
-	/*
-	 * Big joiner configuration needs DSC for TGL which is not true for
-	 * XE_LPD where uncompressed joiner is supported.
-	 */
-	if (DISPLAY_VER(dev_priv) < 13 && bigjoiner && !dsc)
+	if (intel_dp_joiner_needs_dsc(dev_priv, bigjoiner) && !dsc)
 		return MODE_CLOCK_HIGH;
 
 	if (mode_rate > max_rate && !dsc)
@@ -1704,7 +1708,6 @@ static int intel_dp_dsc_compute_params(const struct intel_connector *connector,
 {
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config;
-	u8 line_buf_depth;
 	int ret;
 
 	/*
@@ -1733,20 +1736,14 @@ static int intel_dp_dsc_compute_params(const struct intel_connector *connector,
 			connector->dp.dsc_dpcd[DP_DSC_DEC_COLOR_FORMAT_CAP - DP_DSC_SUPPORT] &
 			DP_DSC_RGB;
 
-	line_buf_depth = drm_dp_dsc_sink_line_buf_depth(connector->dp.dsc_dpcd);
-	if (!line_buf_depth) {
+	vdsc_cfg->line_buf_depth = min(INTEL_DP_DSC_MAX_LINE_BUF_DEPTH,
+				       drm_dp_dsc_sink_line_buf_depth(connector->dp.dsc_dpcd));
+	if (!vdsc_cfg->line_buf_depth) {
 		drm_dbg_kms(&i915->drm,
 			    "DSC Sink Line Buffer Depth invalid\n");
 		return -EINVAL;
 	}
 
-	if (vdsc_cfg->dsc_version_minor == 2)
-		vdsc_cfg->line_buf_depth = (line_buf_depth == DSC_1_2_MAX_LINEBUF_DEPTH_BITS) ?
-			DSC_1_2_MAX_LINEBUF_DEPTH_VAL : line_buf_depth;
-	else
-		vdsc_cfg->line_buf_depth = (line_buf_depth > DSC_1_1_MAX_LINEBUF_DEPTH_BITS) ?
-			DSC_1_1_MAX_LINEBUF_DEPTH_BITS : line_buf_depth;
-
 	vdsc_cfg->block_pred_enable =
 		connector->dp.dsc_dpcd[DP_DSC_BLK_PREDICTION_SUPPORT - DP_DSC_SUPPORT] &
 		DP_DSC_BLK_PREDICTION_IS_SUPPORTED;
@@ -2401,6 +2398,16 @@ int intel_dp_config_required_rate(const struct intel_crtc_state *crtc_state)
 	return intel_dp_link_required(adjusted_mode->crtc_clock, bpp);
 }
 
+bool intel_dp_joiner_needs_dsc(struct drm_i915_private *i915, bool use_joiner)
+{
+	/*
+	 * Pipe joiner needs compression up to display 12 due to bandwidth
+	 * limitation. DG2 onwards pipe joiner can be enabled without
+	 * compression.
+	 */
+	return DISPLAY_VER(i915) < 13 && use_joiner;
+}
+
 static int
 intel_dp_compute_link_config(struct intel_encoder *encoder,
 			     struct intel_crtc_state *pipe_config,
@@ -2409,30 +2416,25 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
-	const struct intel_connector *connector =
+	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
 	const struct drm_display_mode *adjusted_mode =
 		&pipe_config->hw.adjusted_mode;
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 	struct link_config_limits limits;
-	bool joiner_needs_dsc = false;
-	bool dsc_needed;
+	bool dsc_needed, joiner_needs_dsc;
 	int ret = 0;
 
 	if (pipe_config->fec_enable &&
 	    !intel_dp_supports_fec(intel_dp, connector, pipe_config))
 		return -EINVAL;
 
-	if (intel_dp_need_bigjoiner(intel_dp, adjusted_mode->crtc_hdisplay,
+	if (intel_dp_need_bigjoiner(intel_dp, connector,
+				    adjusted_mode->crtc_hdisplay,
 				    adjusted_mode->crtc_clock))
 		pipe_config->bigjoiner_pipes = GENMASK(crtc->pipe + 1, crtc->pipe);
 
-	/*
-	 * Pipe joiner needs compression up to display 12 due to bandwidth
-	 * limitation. DG2 onwards pipe joiner can be enabled without
-	 * compression.
-	 */
-	joiner_needs_dsc = DISPLAY_VER(i915) < 13 && pipe_config->bigjoiner_pipes;
+	joiner_needs_dsc = intel_dp_joiner_needs_dsc(i915, pipe_config->bigjoiner_pipes);
 
 	dsc_needed = joiner_needs_dsc || intel_dp->force_dsc_en ||
 		     !intel_dp_compute_config_limits(intel_dp, pipe_config,
@@ -2615,6 +2617,29 @@ static void intel_dp_compute_vsc_colorimetry(const struct intel_crtc_state *crtc
 	vsc->content_type = DP_CONTENT_TYPE_NOT_DEFINED;
 }
 
+static void intel_dp_compute_as_sdp(struct intel_dp *intel_dp,
+				    struct intel_crtc_state *crtc_state)
+{
+	struct drm_dp_as_sdp *as_sdp = &crtc_state->infoframes.as_sdp;
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->hw.adjusted_mode;
+
+	if (!crtc_state->vrr.enable ||
+	    !intel_dp_as_sdp_supported(intel_dp))
+		return;
+
+	crtc_state->infoframes.enable |= intel_hdmi_infoframe_enable(DP_SDP_ADAPTIVE_SYNC);
+
+	/* Currently only DP_AS_SDP_AVT_FIXED_VTOTAL mode supported */
+	as_sdp->sdp_type = DP_SDP_ADAPTIVE_SYNC;
+	as_sdp->length = 0x9;
+	as_sdp->mode = DP_AS_SDP_AVT_FIXED_VTOTAL;
+	as_sdp->vtotal = adjusted_mode->vtotal;
+	as_sdp->target_rr = 0;
+	as_sdp->duration_incr_ms = 0;
+	as_sdp->duration_incr_ms = 0;
+}
+
 static void intel_dp_compute_vsc_sdp(struct intel_dp *intel_dp,
 				     struct intel_crtc_state *crtc_state,
 				     const struct drm_connector_state *conn_state)
@@ -2970,6 +2995,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 		g4x_dp_set_clock(encoder, pipe_config);
 
 	intel_vrr_compute_config(pipe_config, conn_state);
+	intel_dp_compute_as_sdp(intel_dp, pipe_config);
 	intel_psr_compute_config(intel_dp, pipe_config, conn_state);
 	intel_dp_drrs_compute_config(connector, pipe_config, link_bpp_x16);
 	intel_dp_compute_vsc_sdp(intel_dp, pipe_config, conn_state);
@@ -3362,6 +3388,14 @@ bool intel_dp_initial_fastset_check(struct intel_encoder *encoder,
 		fastset = false;
 	}
 
+	if (CAN_PANEL_REPLAY(intel_dp)) {
+		drm_dbg_kms(&i915->drm,
+			    "[ENCODER:%d:%s] Forcing full modeset to compute panel replay state\n",
+			    encoder->base.base.id, encoder->base.name);
+		crtc_state->uapi.mode_changed = true;
+		fastset = false;
+	}
+
 	return fastset;
 }
 
@@ -4045,39 +4079,84 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
 					   intel_dp->downstream_ports) == 0;
 }
 
-static bool
-intel_dp_can_mst(struct intel_dp *intel_dp)
+static const char *intel_dp_mst_mode_str(enum drm_dp_mst_mode mst_mode)
+{
+	if (mst_mode == DRM_DP_MST)
+		return "MST";
+	else if (mst_mode == DRM_DP_SST_SIDEBAND_MSG)
+		return "SST w/ sideband messaging";
+	else
+		return "SST";
+}
+
+static enum drm_dp_mst_mode
+intel_dp_mst_mode_choose(struct intel_dp *intel_dp,
+			 enum drm_dp_mst_mode sink_mst_mode)
 {
 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 
-	return i915->display.params.enable_dp_mst &&
-		intel_dp_mst_source_support(intel_dp) &&
-		drm_dp_read_mst_cap(&intel_dp->aux, intel_dp->dpcd);
+	if (!i915->display.params.enable_dp_mst)
+		return DRM_DP_SST;
+
+	if (!intel_dp_mst_source_support(intel_dp))
+		return DRM_DP_SST;
+
+	if (sink_mst_mode == DRM_DP_SST_SIDEBAND_MSG &&
+	    !(intel_dp->dpcd[DP_MAIN_LINK_CHANNEL_CODING] & DP_CAP_ANSI_128B132B))
+		return DRM_DP_SST;
+
+	return sink_mst_mode;
 }
 
-static void
-intel_dp_configure_mst(struct intel_dp *intel_dp)
+static enum drm_dp_mst_mode
+intel_dp_mst_detect(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
-	struct intel_encoder *encoder =
-		&dp_to_dig_port(intel_dp)->base;
-	bool sink_can_mst = drm_dp_read_mst_cap(&intel_dp->aux, intel_dp->dpcd);
+	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
+	enum drm_dp_mst_mode sink_mst_mode;
+	enum drm_dp_mst_mode mst_detect;
+
+	sink_mst_mode = drm_dp_read_mst_cap(&intel_dp->aux, intel_dp->dpcd);
+
+	mst_detect = intel_dp_mst_mode_choose(intel_dp, sink_mst_mode);
 
 	drm_dbg_kms(&i915->drm,
-		    "[ENCODER:%d:%s] MST support: port: %s, sink: %s, modparam: %s\n",
+		    "[ENCODER:%d:%s] MST support: port: %s, sink: %s, modparam: %s -> enable: %s\n",
 		    encoder->base.base.id, encoder->base.name,
 		    str_yes_no(intel_dp_mst_source_support(intel_dp)),
-		    str_yes_no(sink_can_mst),
-		    str_yes_no(i915->display.params.enable_dp_mst));
+		    intel_dp_mst_mode_str(sink_mst_mode),
+		    str_yes_no(i915->display.params.enable_dp_mst),
+		    intel_dp_mst_mode_str(mst_detect));
+
+	return mst_detect;
+}
 
+static void
+intel_dp_mst_configure(struct intel_dp *intel_dp)
+{
 	if (!intel_dp_mst_source_support(intel_dp))
 		return;
 
-	intel_dp->is_mst = sink_can_mst &&
-		i915->display.params.enable_dp_mst;
+	intel_dp->is_mst = intel_dp->mst_detect != DRM_DP_SST;
+
+	drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, intel_dp->is_mst);
+
+	/* Avoid stale info on the next detect cycle. */
+	intel_dp->mst_detect = DRM_DP_SST;
+}
+
+static void
+intel_dp_mst_disconnect(struct intel_dp *intel_dp)
+{
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
+
+	if (!intel_dp->is_mst)
+		return;
 
-	drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr,
-					intel_dp->is_mst);
+	drm_dbg_kms(&i915->drm, "MST device may have disappeared %d vs %d\n",
+		    intel_dp->is_mst, intel_dp->mst_mgr.mst_state);
+	intel_dp->is_mst = false;
+	drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, intel_dp->is_mst);
 }
 
 static bool
@@ -4125,6 +4204,32 @@ intel_dp_needs_vsc_sdp(const struct intel_crtc_state *crtc_state,
 	return false;
 }
 
+static ssize_t intel_dp_as_sdp_pack(const struct drm_dp_as_sdp *as_sdp,
+				    struct dp_sdp *sdp, size_t size)
+{
+	size_t length = sizeof(struct dp_sdp);
+
+	if (size < length)
+		return -ENOSPC;
+
+	memset(sdp, 0, size);
+
+	/* Prepare AS (Adaptive Sync) SDP Header */
+	sdp->sdp_header.HB0 = 0;
+	sdp->sdp_header.HB1 = as_sdp->sdp_type;
+	sdp->sdp_header.HB2 = 0x02;
+	sdp->sdp_header.HB3 = as_sdp->length;
+
+	/* Fill AS (Adaptive Sync) SDP Payload */
+	sdp->db[0] = as_sdp->mode;
+	sdp->db[1] = as_sdp->vtotal & 0xFF;
+	sdp->db[2] = (as_sdp->vtotal >> 8) & 0xFF;
+	sdp->db[3] = as_sdp->target_rr & 0xFF;
+	sdp->db[4] = (as_sdp->target_rr >> 8) & 0x3;
+
+	return length;
+}
+
 static ssize_t
 intel_dp_hdr_metadata_infoframe_sdp_pack(struct drm_i915_private *i915,
 					 const struct hdmi_drm_infoframe *drm_infoframe,
@@ -4224,6 +4329,10 @@ static void intel_write_dp_sdp(struct intel_encoder *encoder,
 							       &crtc_state->infoframes.drm.drm,
 							       &sdp, sizeof(sdp));
 		break;
+	case DP_SDP_ADAPTIVE_SYNC:
+		len = intel_dp_as_sdp_pack(&crtc_state->infoframes.as_sdp, &sdp,
+					   sizeof(sdp));
+		break;
 	default:
 		MISSING_CASE(type);
 		return;
@@ -4245,6 +4354,10 @@ void intel_dp_set_infoframes(struct intel_encoder *encoder,
 	u32 dip_enable = VIDEO_DIP_ENABLE_AVI_HSW | VIDEO_DIP_ENABLE_GCP_HSW |
 			 VIDEO_DIP_ENABLE_VS_HSW | VIDEO_DIP_ENABLE_GMP_HSW |
 			 VIDEO_DIP_ENABLE_SPD_HSW | VIDEO_DIP_ENABLE_DRM_GLK;
+
+	if (HAS_AS_SDP(dev_priv))
+		dip_enable |= VIDEO_DIP_ENABLE_AS_ADL;
+
 	u32 val = intel_de_read(dev_priv, reg) & ~dip_enable;
 
 	/* TODO: Sanitize DSC enabling wrt. intel_dsc_dp_pps_write(). */
@@ -4262,10 +4375,42 @@ void intel_dp_set_infoframes(struct intel_encoder *encoder,
 		return;
 
 	intel_write_dp_sdp(encoder, crtc_state, DP_SDP_VSC);
+	intel_write_dp_sdp(encoder, crtc_state, DP_SDP_ADAPTIVE_SYNC);
 
 	intel_write_dp_sdp(encoder, crtc_state, HDMI_PACKET_TYPE_GAMUT_METADATA);
 }
 
+static
+int intel_dp_as_sdp_unpack(struct drm_dp_as_sdp *as_sdp,
+			   const void *buffer, size_t size)
+{
+	const struct dp_sdp *sdp = buffer;
+
+	if (size < sizeof(struct dp_sdp))
+		return -EINVAL;
+
+	memset(as_sdp, 0, sizeof(*as_sdp));
+
+	if (sdp->sdp_header.HB0 != 0)
+		return -EINVAL;
+
+	if (sdp->sdp_header.HB1 != DP_SDP_ADAPTIVE_SYNC)
+		return -EINVAL;
+
+	if (sdp->sdp_header.HB2 != 0x02)
+		return -EINVAL;
+
+	if ((sdp->sdp_header.HB3 & 0x3F) != 9)
+		return -EINVAL;
+
+	as_sdp->length = sdp->sdp_header.HB3 & DP_ADAPTIVE_SYNC_SDP_LENGTH;
+	as_sdp->mode = sdp->db[0] & DP_ADAPTIVE_SYNC_SDP_OPERATION_MODE;
+	as_sdp->vtotal = (sdp->db[2] << 8) | sdp->db[1];
+	as_sdp->target_rr = (u64)sdp->db[3] | ((u64)sdp->db[4] & 0x3);
+
+	return 0;
+}
+
 static int intel_dp_vsc_sdp_unpack(struct drm_dp_vsc_sdp *vsc,
 				   const void *buffer, size_t size)
 {
@@ -4336,6 +4481,29 @@ static int intel_dp_vsc_sdp_unpack(struct drm_dp_vsc_sdp *vsc,
 	return 0;
 }
 
+static void
+intel_read_dp_as_sdp(struct intel_encoder *encoder,
+		     struct intel_crtc_state *crtc_state,
+		     struct drm_dp_as_sdp *as_sdp)
+{
+	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	unsigned int type = DP_SDP_ADAPTIVE_SYNC;
+	struct dp_sdp sdp = {};
+	int ret;
+
+	if ((crtc_state->infoframes.enable &
+	     intel_hdmi_infoframe_enable(type)) == 0)
+		return;
+
+	dig_port->read_infoframe(encoder, crtc_state, type, &sdp,
+				 sizeof(sdp));
+
+	ret = intel_dp_as_sdp_unpack(as_sdp, &sdp, sizeof(sdp));
+	if (ret)
+		drm_dbg_kms(&dev_priv->drm, "Failed to unpack DP AS SDP\n");
+}
+
 static int
 intel_dp_hdr_metadata_infoframe_sdp_unpack(struct hdmi_drm_infoframe *drm_infoframe,
 					   const void *buffer, size_t size)
@@ -4442,6 +4610,10 @@ void intel_read_dp_sdp(struct intel_encoder *encoder,
 		intel_read_dp_hdr_metadata_infoframe_sdp(encoder, crtc_state,
 							 &crtc_state->infoframes.drm.drm);
 		break;
+	case DP_SDP_ADAPTIVE_SYNC:
+		intel_read_dp_as_sdp(encoder, crtc_state,
+				     &crtc_state->infoframes.as_sdp);
+		break;
 	default:
 		MISSING_CASE(type);
 		break;
@@ -5369,6 +5541,8 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp)
 	if (!intel_dp_get_dpcd(intel_dp))
 		return connector_status_disconnected;
 
+	intel_dp->mst_detect = intel_dp_mst_detect(intel_dp);
+
 	/* if there's no downstream port, we're done */
 	if (!drm_dp_is_branch(dpcd))
 		return connector_status_connected;
@@ -5380,7 +5554,7 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp)
 		connector_status_connected : connector_status_disconnected;
 	}
 
-	if (intel_dp_can_mst(intel_dp))
+	if (intel_dp->mst_detect == DRM_DP_MST)
 		return connector_status_connected;
 
 	/* If no HPD, poke DDC gently */
@@ -5685,15 +5859,7 @@ intel_dp_detect(struct drm_connector *connector,
 		memset(intel_connector->dp.dsc_dpcd, 0, sizeof(intel_connector->dp.dsc_dpcd));
 		intel_dp->psr.sink_panel_replay_support = false;
 
-		if (intel_dp->is_mst) {
-			drm_dbg_kms(&dev_priv->drm,
-				    "MST device may have disappeared %d vs %d\n",
-				    intel_dp->is_mst,
-				    intel_dp->mst_mgr.mst_state);
-			intel_dp->is_mst = false;
-			drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr,
-							intel_dp->is_mst);
-		}
+		intel_dp_mst_disconnect(intel_dp);
 
 		intel_dp_tunnel_disconnect(intel_dp);
 
@@ -5712,7 +5878,7 @@ intel_dp_detect(struct drm_connector *connector,
 
 	intel_dp_detect_dsc_caps(intel_dp, intel_connector);
 
-	intel_dp_configure_mst(intel_dp);
+	intel_dp_mst_configure(intel_dp);
 
 	/*
 	 * TODO: Reset link params when switching to MST mode, until MST
@@ -6495,7 +6661,6 @@ intel_dp_init_connector(struct intel_digital_port *dig_port,
 	struct drm_device *dev = intel_encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum port port = intel_encoder->port;
-	enum phy phy = intel_port_to_phy(dev_priv, port);
 	int type;
 
 	/* Initialize the work for modeset in case of link train failure */
@@ -6520,7 +6685,7 @@ intel_dp_init_connector(struct intel_digital_port *dig_port,
 		 * Currently we don't support eDP on TypeC ports, although in
 		 * theory it could work on TypeC legacy ports.
 		 */
-		drm_WARN_ON(dev, intel_phy_is_tc(dev_priv, phy));
+		drm_WARN_ON(dev, intel_encoder_is_tc(intel_encoder));
 		type = DRM_MODE_CONNECTOR_eDP;
 		intel_encoder->type = INTEL_OUTPUT_EDP;
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp.h b/drivers/gpu/drm/i915/display/intel_dp.h
index c540d3a73fe7..106ecfde36d9 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.h
+++ b/drivers/gpu/drm/i915/display/intel_dp.h
@@ -88,6 +88,7 @@ void intel_dp_audio_compute_config(struct intel_encoder *encoder,
 				   struct drm_connector_state *conn_state);
 bool intel_dp_has_hdmi_sink(struct intel_dp *intel_dp);
 bool intel_dp_is_edp(struct intel_dp *intel_dp);
+bool intel_dp_as_sdp_supported(struct intel_dp *intel_dp);
 bool intel_dp_is_uhbr(const struct intel_crtc_state *crtc_state);
 int intel_dp_link_symbol_size(int rate);
 int intel_dp_link_symbol_clock(int rate);
@@ -119,7 +120,8 @@ int intel_dp_effective_data_rate(int pixel_clock, int bpp_x16,
 				 int bw_overhead);
 int intel_dp_max_link_data_rate(struct intel_dp *intel_dp,
 				int max_dprx_rate, int max_dprx_lanes);
-bool intel_dp_can_bigjoiner(struct intel_dp *intel_dp);
+bool intel_dp_joiner_needs_dsc(struct drm_i915_private *i915, bool use_joiner);
+bool intel_dp_has_bigjoiner(struct intel_dp *intel_dp);
 bool intel_dp_needs_vsc_sdp(const struct intel_crtc_state *crtc_state,
 			    const struct drm_connector_state *conn_state);
 void intel_dp_set_infoframes(struct intel_encoder *encoder, bool enable,
@@ -149,6 +151,7 @@ u8 intel_dp_dsc_get_slice_count(const struct intel_connector *connector,
 				int mode_clock, int mode_hdisplay,
 				bool bigjoiner);
 bool intel_dp_need_bigjoiner(struct intel_dp *intel_dp,
+			     struct intel_connector *connector,
 			     int hdisplay, int clock);
 
 static inline unsigned int intel_dp_unused_lane_mask(int lane_count)
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.c b/drivers/gpu/drm/i915/display/intel_dp_aux.c
index 4f4a0e3b3114..b8a53bb174da 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux.c
@@ -61,9 +61,8 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp)
 	u32 status;
 	int ret;
 
-	ret = __intel_de_wait_for_register(i915, ch_ctl,
-					   DP_AUX_CH_CTL_SEND_BUSY, 0,
-					   2, timeout_ms, &status);
+	ret = intel_de_wait_custom(i915, ch_ctl, DP_AUX_CH_CTL_SEND_BUSY, 0,
+				   2, timeout_ms, &status);
 
 	if (ret == -ETIMEDOUT)
 		drm_err(&i915->drm,
@@ -143,9 +142,15 @@ static int intel_dp_aux_sync_len(void)
 	return precharge + preamble;
 }
 
-static int intel_dp_aux_fw_sync_len(void)
+int intel_dp_aux_fw_sync_len(void)
 {
-	int precharge = 10; /* 10-16 */
+	/*
+	 * We faced some glitches on Dell Precision 5490 MTL laptop with panel:
+	 * "Manufacturer: AUO, Model: 63898" when using HW default 18. Using 20
+	 * is fixing these problems with the panel. It is still within range
+	 * mentioned in eDP specification.
+	 */
+	int precharge = 12; /* 10-16 */
 	int preamble = 8;
 
 	return precharge + preamble;
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.h b/drivers/gpu/drm/i915/display/intel_dp_aux.h
index 8447f3e601fe..76d1f2ed7c2f 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux.h
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux.h
@@ -20,5 +20,6 @@ enum aux_ch intel_dp_aux_ch(struct intel_encoder *encoder);
 
 void intel_dp_aux_irq_handler(struct drm_i915_private *i915);
 u32 intel_dp_aux_pack(const u8 *src, int src_bytes);
+int intel_dp_aux_fw_sync_len(void);
 
 #endif /* __INTEL_DP_AUX_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c
index 9db43bd81ce2..92b03073acdd 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c
@@ -769,11 +769,9 @@ intel_dp_mst_hdcp_stream_encryption(struct intel_connector *connector,
 		return -EINVAL;
 
 	/* Wait for encryption confirmation */
-	if (intel_de_wait_for_register(i915,
-				       HDCP_STATUS(i915, cpu_transcoder, port),
-				       stream_enc_status,
-				       enable ? stream_enc_status : 0,
-				       HDCP_ENCRYPT_STATUS_CHANGE_TIMEOUT_MS)) {
+	if (intel_de_wait(i915, HDCP_STATUS(i915, cpu_transcoder, port),
+			  stream_enc_status, enable ? stream_enc_status : 0,
+			  HDCP_ENCRYPT_STATUS_CHANGE_TIMEOUT_MS)) {
 		drm_err(&i915->drm, "Timed out waiting for transcoder: %s stream encryption %s\n",
 			transcoder_name(cpu_transcoder), enable ? "enabled" : "disabled");
 		return -ETIMEDOUT;
@@ -804,11 +802,10 @@ intel_dp_mst_hdcp2_stream_encryption(struct intel_connector *connector,
 		return ret;
 
 	/* Wait for encryption confirmation */
-	if (intel_de_wait_for_register(i915,
-				       HDCP2_STREAM_STATUS(i915, cpu_transcoder, pipe),
-				       STREAM_ENCRYPTION_STATUS,
-				       enable ? STREAM_ENCRYPTION_STATUS : 0,
-				       HDCP_ENCRYPT_STATUS_CHANGE_TIMEOUT_MS)) {
+	if (intel_de_wait(i915, HDCP2_STREAM_STATUS(i915, cpu_transcoder, pipe),
+			  STREAM_ENCRYPTION_STATUS,
+			  enable ? STREAM_ENCRYPTION_STATUS : 0,
+			  HDCP_ENCRYPT_STATUS_CHANGE_TIMEOUT_MS)) {
 		drm_err(&i915->drm, "Timed out waiting for transcoder: %s stream encryption %s\n",
 			transcoder_name(cpu_transcoder), enable ? "enabled" : "disabled");
 		return -ETIMEDOUT;
diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c
index fb84ca98bb7a..947575140059 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c
@@ -334,7 +334,7 @@ static bool has_per_lane_signal_levels(struct intel_dp *intel_dp,
 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 
 	return !intel_dp_phy_is_downstream_of_source(intel_dp, dp_phy) ||
-		DISPLAY_VER(i915) >= 11;
+		DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915);
 }
 
 /* 128b/132b */
diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c
index b651c990af85..715d2f59f565 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c
@@ -51,25 +51,39 @@
 #include "intel_vdsc.h"
 #include "skl_scaler.h"
 
-static int intel_dp_mst_check_constraints(struct drm_i915_private *i915, int bpp,
-					  const struct drm_display_mode *adjusted_mode,
-					  struct intel_crtc_state *crtc_state,
-					  bool dsc)
+static int intel_dp_mst_max_dpt_bpp(const struct intel_crtc_state *crtc_state,
+				    bool dsc)
 {
-	if (intel_dp_is_uhbr(crtc_state) && DISPLAY_VER(i915) < 14 && dsc) {
-		int output_bpp = bpp;
-		/* DisplayPort 2 128b/132b, bits per lane is always 32 */
-		int symbol_clock = crtc_state->port_clock / 32;
-
-		if (output_bpp * adjusted_mode->crtc_clock >=
-		    symbol_clock * 72) {
-			drm_dbg_kms(&i915->drm, "UHBR check failed(required bw %d available %d)\n",
-				    output_bpp * adjusted_mode->crtc_clock, symbol_clock * 72);
-			return -EINVAL;
-		}
-	}
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->hw.adjusted_mode;
 
-	return 0;
+	if (!intel_dp_is_uhbr(crtc_state) || DISPLAY_VER(i915) >= 20 || !dsc)
+		return INT_MAX;
+
+	/*
+	 * DSC->DPT interface width:
+	 *   ICL-MTL: 72 bits (each branch has 72 bits, only left branch is used)
+	 *   LNL+:    144 bits (not a bottleneck in any config)
+	 *
+	 * Bspec/49259 suggests that the FEC overhead needs to be
+	 * applied here, though HW people claim that neither this FEC
+	 * or any other overhead is applicable here (that is the actual
+	 * available_bw is just symbol_clock * 72). However based on
+	 * testing on MTL-P the
+	 * - DELL U3224KBA display
+	 * - Unigraf UCD-500 CTS test sink
+	 * devices the
+	 * - 5120x2880/995.59Mhz
+	 * - 6016x3384/1357.23Mhz
+	 * - 6144x3456/1413.39Mhz
+	 * modes (all the ones having a DPT limit on the above devices),
+	 * both the channel coding efficiency and an additional 3%
+	 * overhead needs to be accounted for.
+	 */
+	return div64_u64(mul_u32_u32(intel_dp_link_symbol_clock(crtc_state->port_clock) * 72,
+				     drm_dp_bw_channel_coding_efficiency(true)),
+			 mul_u32_u32(adjusted_mode->crtc_clock, 1030000));
 }
 
 static int intel_dp_mst_bw_overhead(const struct intel_crtc_state *crtc_state,
@@ -88,11 +102,10 @@ static int intel_dp_mst_bw_overhead(const struct intel_crtc_state *crtc_state,
 
 	if (dsc) {
 		flags |= DRM_DP_BW_OVERHEAD_DSC;
-		/* TODO: add support for bigjoiner */
 		dsc_slice_count = intel_dp_dsc_get_slice_count(connector,
 							       adjusted_mode->clock,
 							       adjusted_mode->hdisplay,
-							       false);
+							       crtc_state->bigjoiner_pipes);
 	}
 
 	overhead = drm_dp_bw_overhead(crtc_state->lane_count,
@@ -158,6 +171,7 @@ static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder,
 	const struct drm_display_mode *adjusted_mode =
 		&crtc_state->hw.adjusted_mode;
 	int bpp, slots = -EINVAL;
+	int max_dpt_bpp;
 	int ret = 0;
 
 	mst_state = drm_atomic_get_mst_topology_state(state, &intel_dp->mst_mgr);
@@ -178,6 +192,13 @@ static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder,
 						      crtc_state->port_clock,
 						      crtc_state->lane_count);
 
+	max_dpt_bpp = intel_dp_mst_max_dpt_bpp(crtc_state, dsc);
+	if (max_bpp > max_dpt_bpp) {
+		drm_dbg_kms(&i915->drm, "Limiting bpp to max DPT bpp (%d -> %d)\n",
+			    max_bpp, max_dpt_bpp);
+		max_bpp = max_dpt_bpp;
+	}
+
 	drm_dbg_kms(&i915->drm, "Looking for slots in range min bpp %d max bpp %d\n",
 		    min_bpp, max_bpp);
 
@@ -189,10 +210,6 @@ static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder,
 
 		drm_dbg_kms(&i915->drm, "Trying bpp %d\n", bpp);
 
-		ret = intel_dp_mst_check_constraints(i915, bpp, adjusted_mode, crtc_state, dsc);
-		if (ret)
-			continue;
-
 		link_bpp_x16 = to_bpp_x16(dsc ? bpp :
 					  intel_dp_output_bpp(crtc_state->output_format, bpp));
 
@@ -404,15 +421,22 @@ static int mode_hblank_period_ns(const struct drm_display_mode *mode)
 
 static bool
 hblank_expansion_quirk_needs_dsc(const struct intel_connector *connector,
-				 const struct intel_crtc_state *crtc_state)
+				 const struct intel_crtc_state *crtc_state,
+				 const struct link_config_limits *limits)
 {
 	const struct drm_display_mode *adjusted_mode =
 		&crtc_state->hw.adjusted_mode;
+	bool is_uhbr_sink = connector->mst_port &&
+			    drm_dp_128b132b_supported(connector->mst_port->dpcd);
+	int hblank_limit = is_uhbr_sink ? 500 : 300;
 
 	if (!connector->dp.dsc_hblank_expansion_quirk)
 		return false;
 
-	if (mode_hblank_period_ns(adjusted_mode) > 300)
+	if (is_uhbr_sink && !drm_dp_is_uhbr_rate(limits->max_rate))
+		return false;
+
+	if (mode_hblank_period_ns(adjusted_mode) > hblank_limit)
 		return false;
 
 	return true;
@@ -428,7 +452,7 @@ adjust_limits_for_dsc_hblank_expansion_quirk(const struct intel_connector *conne
 	const struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	int min_bpp_x16 = limits->link.min_bpp_x16;
 
-	if (!hblank_expansion_quirk_needs_dsc(connector, crtc_state))
+	if (!hblank_expansion_quirk_needs_dsc(connector, crtc_state, limits))
 		return true;
 
 	if (!dsc) {
@@ -525,14 +549,15 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state);
+	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder);
 	struct intel_dp *intel_dp = &intel_mst->primary->dp;
-	const struct intel_connector *connector =
+	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
 	const struct drm_display_mode *adjusted_mode =
 		&pipe_config->hw.adjusted_mode;
 	struct link_config_limits limits;
-	bool dsc_needed;
+	bool dsc_needed, joiner_needs_dsc;
 	int ret = 0;
 
 	if (pipe_config->fec_enable &&
@@ -542,11 +567,18 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder,
 	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)
 		return -EINVAL;
 
+	if (intel_dp_need_bigjoiner(intel_dp, connector,
+				    adjusted_mode->crtc_hdisplay,
+				    adjusted_mode->crtc_clock))
+		pipe_config->bigjoiner_pipes = GENMASK(crtc->pipe + 1, crtc->pipe);
+
 	pipe_config->sink_format = INTEL_OUTPUT_FORMAT_RGB;
 	pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
 	pipe_config->has_pch_encoder = false;
 
-	dsc_needed = intel_dp->force_dsc_en ||
+	joiner_needs_dsc = intel_dp_joiner_needs_dsc(dev_priv, pipe_config->bigjoiner_pipes);
+
+	dsc_needed = joiner_needs_dsc || intel_dp->force_dsc_en ||
 		     !intel_dp_mst_compute_config_limits(intel_dp,
 							 connector,
 							 pipe_config,
@@ -566,8 +598,8 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder,
 
 	/* enable compression if the mode doesn't fit available BW */
 	if (dsc_needed) {
-		drm_dbg_kms(&dev_priv->drm, "Try DSC (fallback=%s, force=%s)\n",
-			    str_yes_no(ret),
+		drm_dbg_kms(&dev_priv->drm, "Try DSC (fallback=%s, joiner=%s, force=%s)\n",
+			    str_yes_no(ret), str_yes_no(joiner_needs_dsc),
 			    str_yes_no(intel_dp->force_dsc_en));
 
 		if (!intel_dp_mst_dsc_source_support(pipe_config))
@@ -613,7 +645,7 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder,
 
 	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
 		pipe_config->lane_lat_optim_mask =
-			bxt_ddi_phy_calc_lane_lat_optim_mask(pipe_config->lane_count);
+			bxt_dpio_phy_calc_lane_lat_optim_mask(pipe_config->lane_count);
 
 	intel_dp_audio_compute_config(encoder, pipe_config, conn_state);
 
@@ -954,6 +986,7 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state,
 	struct drm_dp_mst_atomic_payload *new_payload =
 		drm_atomic_get_mst_payload_state(new_mst_state, connector->port);
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_crtc *pipe_crtc;
 	bool last_mst_stream;
 
 	intel_dp->active_mst_links--;
@@ -962,7 +995,13 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state,
 		    DISPLAY_VER(dev_priv) >= 12 && last_mst_stream &&
 		    !intel_dp_mst_is_master_trans(old_crtc_state));
 
-	intel_crtc_vblank_off(old_crtc_state);
+	for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, pipe_crtc,
+					 intel_crtc_joined_pipe_mask(old_crtc_state)) {
+		const struct intel_crtc_state *old_pipe_crtc_state =
+			intel_atomic_get_old_crtc_state(state, pipe_crtc);
+
+		intel_crtc_vblank_off(old_pipe_crtc_state);
+	}
 
 	intel_disable_transcoder(old_crtc_state);
 
@@ -980,12 +1019,18 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state,
 
 	intel_ddi_disable_transcoder_func(old_crtc_state);
 
-	intel_dsc_disable(old_crtc_state);
+	for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, pipe_crtc,
+					 intel_crtc_joined_pipe_mask(old_crtc_state)) {
+		const struct intel_crtc_state *old_pipe_crtc_state =
+			intel_atomic_get_old_crtc_state(state, pipe_crtc);
 
-	if (DISPLAY_VER(dev_priv) >= 9)
-		skl_scaler_disable(old_crtc_state);
-	else
-		ilk_pfit_disable(old_crtc_state);
+		intel_dsc_disable(old_pipe_crtc_state);
+
+		if (DISPLAY_VER(dev_priv) >= 9)
+			skl_scaler_disable(old_pipe_crtc_state);
+		else
+			ilk_pfit_disable(old_pipe_crtc_state);
+	}
 
 	/*
 	 * Power down mst path before disabling the port, otherwise we end
@@ -1117,6 +1162,39 @@ static void intel_mst_pre_enable_dp(struct intel_atomic_state *state,
 	intel_ddi_set_dp_msa(pipe_config, conn_state);
 }
 
+static void enable_bs_jitter_was(const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	u32 clear = 0;
+	u32 set = 0;
+
+	if (!IS_ALDERLAKE_P(i915))
+		return;
+
+	if (!IS_DISPLAY_STEP(i915, STEP_D0, STEP_FOREVER))
+		return;
+
+	/* Wa_14013163432:adlp */
+	if (crtc_state->fec_enable || intel_dp_is_uhbr(crtc_state))
+		set |= DP_MST_FEC_BS_JITTER_WA(crtc_state->cpu_transcoder);
+
+	/* Wa_14014143976:adlp */
+	if (IS_DISPLAY_STEP(i915, STEP_E0, STEP_FOREVER)) {
+		if (intel_dp_is_uhbr(crtc_state))
+			set |= DP_MST_SHORT_HBLANK_WA(crtc_state->cpu_transcoder);
+		else if (crtc_state->fec_enable)
+			clear |= DP_MST_SHORT_HBLANK_WA(crtc_state->cpu_transcoder);
+
+		if (crtc_state->fec_enable || intel_dp_is_uhbr(crtc_state))
+			set |= DP_MST_DPT_DPTP_ALIGN_WA(crtc_state->cpu_transcoder);
+	}
+
+	if (!clear && !set)
+		return;
+
+	intel_de_rmw(i915, CHICKEN_MISC_3, clear, set);
+}
+
 static void intel_mst_enable_dp(struct intel_atomic_state *state,
 				struct intel_encoder *encoder,
 				const struct intel_crtc_state *pipe_config,
@@ -1131,6 +1209,7 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state,
 		drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr);
 	enum transcoder trans = pipe_config->cpu_transcoder;
 	bool first_mst_stream = intel_dp->active_mst_links == 1;
+	struct intel_crtc *pipe_crtc;
 
 	drm_WARN_ON(&dev_priv->drm, pipe_config->has_pch_encoder);
 
@@ -1145,6 +1224,8 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state,
 			       TRANS_DP2_VFREQ_PIXEL_CLOCK(crtc_clock_hz & 0xffffff));
 	}
 
+	enable_bs_jitter_was(pipe_config);
+
 	intel_ddi_enable_transcoder_func(encoder, pipe_config);
 
 	clear_act_sent(encoder, pipe_config);
@@ -1160,7 +1241,7 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state,
 	if (first_mst_stream)
 		intel_ddi_wait_for_fec_status(encoder, pipe_config, true);
 
-	drm_dp_add_payload_part2(&intel_dp->mst_mgr, &state->base,
+	drm_dp_add_payload_part2(&intel_dp->mst_mgr,
 				 drm_atomic_get_mst_payload_state(mst_state, connector->port));
 
 	if (DISPLAY_VER(dev_priv) >= 12)
@@ -1172,7 +1253,13 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state,
 
 	intel_enable_transcoder(pipe_config);
 
-	intel_crtc_vblank_on(pipe_config);
+	for_each_intel_crtc_in_pipe_mask_reverse(&dev_priv->drm, pipe_crtc,
+						 intel_crtc_joined_pipe_mask(pipe_config)) {
+		const struct intel_crtc_state *pipe_crtc_state =
+			intel_atomic_get_new_crtc_state(state, pipe_crtc);
+
+		intel_crtc_vblank_on(pipe_crtc_state);
+	}
 
 	intel_hdcp_enable(state, encoder, pipe_config, conn_state);
 }
@@ -1285,7 +1372,7 @@ intel_dp_mst_mode_valid_ctx(struct drm_connector *connector,
 	struct drm_dp_mst_topology_mgr *mgr = &intel_dp->mst_mgr;
 	struct drm_dp_mst_port *port = intel_connector->port;
 	const int min_bpp = 18;
-	int max_dotclk = to_i915(connector->dev)->max_dotclk_freq;
+	int max_dotclk = to_i915(connector->dev)->display.cdclk.max_dotclk_freq;
 	int max_rate, mode_rate, max_lanes, max_link_clock;
 	int ret;
 	bool dsc = false, bigjoiner = false;
@@ -1302,8 +1389,13 @@ intel_dp_mst_mode_valid_ctx(struct drm_connector *connector,
 	if (*status != MODE_OK)
 		return 0;
 
-	if (mode->flags & DRM_MODE_FLAG_DBLSCAN) {
-		*status = MODE_NO_DBLESCAN;
+	if (mode->flags & DRM_MODE_FLAG_DBLCLK) {
+		*status = MODE_H_ILLEGAL;
+		return 0;
+	}
+
+	if (mode->clock < 10000) {
+		*status = MODE_CLOCK_LOW;
 		return 0;
 	}
 
@@ -1314,10 +1406,6 @@ intel_dp_mst_mode_valid_ctx(struct drm_connector *connector,
 					       max_link_clock, max_lanes);
 	mode_rate = intel_dp_link_required(mode->clock, min_bpp);
 
-	ret = drm_modeset_lock(&mgr->base.lock, ctx);
-	if (ret)
-		return ret;
-
 	/*
 	 * TODO:
 	 * - Also check if compression would allow for the mode
@@ -1330,27 +1418,18 @@ intel_dp_mst_mode_valid_ctx(struct drm_connector *connector,
 	 *   corresponding link capabilities of the sink) in case the
 	 *   stream is uncompressed for it by the last branch device.
 	 */
-	if (mode_rate > max_rate || mode->clock > max_dotclk ||
-	    drm_dp_calc_pbn_mode(mode->clock, min_bpp << 4) > port->full_pbn) {
-		*status = MODE_CLOCK_HIGH;
-		return 0;
-	}
-
-	if (mode->clock < 10000) {
-		*status = MODE_CLOCK_LOW;
-		return 0;
-	}
-
-	if (mode->flags & DRM_MODE_FLAG_DBLCLK) {
-		*status = MODE_H_ILLEGAL;
-		return 0;
-	}
-
-	if (intel_dp_need_bigjoiner(intel_dp, mode->hdisplay, target_clock)) {
+	if (intel_dp_need_bigjoiner(intel_dp, intel_connector,
+				    mode->hdisplay, target_clock)) {
 		bigjoiner = true;
 		max_dotclk *= 2;
+	}
+
+	ret = drm_modeset_lock(&mgr->base.lock, ctx);
+	if (ret)
+		return ret;
 
-		/* TODO: add support for bigjoiner */
+	if (mode_rate > max_rate || mode->clock > max_dotclk ||
+	    drm_dp_calc_pbn_mode(mode->clock, min_bpp << 4) > port->full_pbn) {
 		*status = MODE_CLOCK_HIGH;
 		return 0;
 	}
@@ -1383,11 +1462,7 @@ intel_dp_mst_mode_valid_ctx(struct drm_connector *connector,
 		dsc = dsc_max_compressed_bpp && dsc_slice_count;
 	}
 
-	/*
-	 * Big joiner configuration needs DSC for TGL which is not true for
-	 * XE_LPD where uncompressed joiner is supported.
-	 */
-	if (DISPLAY_VER(dev_priv) < 13 && bigjoiner && !dsc) {
+	if (intel_dp_joiner_needs_dsc(dev_priv, bigjoiner) && !dsc) {
 		*status = MODE_CLOCK_HIGH;
 		return 0;
 	}
@@ -1397,7 +1472,7 @@ intel_dp_mst_mode_valid_ctx(struct drm_connector *connector,
 		return 0;
 	}
 
-	*status = intel_mode_valid_max_plane_size(dev_priv, mode, false);
+	*status = intel_mode_valid_max_plane_size(dev_priv, mode, bigjoiner);
 	return 0;
 }
 
@@ -1509,24 +1584,41 @@ intel_dp_mst_read_decompression_port_dsc_caps(struct intel_dp *intel_dp,
 static bool detect_dsc_hblank_expansion_quirk(const struct intel_connector *connector)
 {
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
+	struct drm_dp_aux *aux = connector->dp.dsc_decompression_aux;
 	struct drm_dp_desc desc;
 	u8 dpcd[DP_RECEIVER_CAP_SIZE];
 
-	if (!connector->dp.dsc_decompression_aux)
+	if (!aux)
 		return false;
 
-	if (drm_dp_read_desc(connector->dp.dsc_decompression_aux,
-			     &desc, true) < 0)
+	/*
+	 * A logical port's OUI (at least for affected sinks) is all 0, so
+	 * instead of that the parent port's OUI is used for identification.
+	 */
+	if (drm_dp_mst_port_is_logical(connector->port)) {
+		aux = drm_dp_mst_aux_for_parent(connector->port);
+		if (!aux)
+			aux = &connector->mst_port->aux;
+	}
+
+	if (drm_dp_read_dpcd_caps(aux, dpcd) < 0)
 		return false;
 
-	if (!drm_dp_has_quirk(&desc,
-			      DP_DPCD_QUIRK_HBLANK_EXPANSION_REQUIRES_DSC))
+	if (drm_dp_read_desc(aux, &desc, drm_dp_is_branch(dpcd)) < 0)
 		return false;
 
-	if (drm_dp_read_dpcd_caps(connector->dp.dsc_decompression_aux, dpcd) < 0)
+	if (!drm_dp_has_quirk(&desc,
+			      DP_DPCD_QUIRK_HBLANK_EXPANSION_REQUIRES_DSC))
 		return false;
 
-	if (!(dpcd[DP_RECEIVE_PORT_0_CAP_0] & DP_HBLANK_EXPANSION_CAPABLE))
+	/*
+	 * UHBR (MST sink) devices requiring this quirk don't advertise the
+	 * HBLANK expansion support. Presuming that they perform HBLANK
+	 * expansion internally, or are affected by this issue on modes with a
+	 * short HBLANK for other reasons.
+	 */
+	if (!drm_dp_128b132b_supported(dpcd) &&
+	    !(dpcd[DP_RECEIVE_PORT_0_CAP_0] & DP_HBLANK_EXPANSION_CAPABLE))
 		return false;
 
 	drm_dbg_kms(&i915->drm,
diff --git a/drivers/gpu/drm/i915/display/intel_dp_tunnel.c b/drivers/gpu/drm/i915/display/intel_dp_tunnel.c
index 75d76f91ecbd..6503abdc2b98 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_tunnel.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_tunnel.c
@@ -348,7 +348,7 @@ void intel_dp_tunnel_resume(struct intel_dp *intel_dp,
 
 out_err:
 	drm_dbg_kms(&i915->drm,
-		    "[DPTUN %s][CONNECTOR:%d:%s][ENCODER:%d:%s] Tunnel can't be resumed, will drop and redect it (err %pe)\n",
+		    "[DPTUN %s][CONNECTOR:%d:%s][ENCODER:%d:%s] Tunnel can't be resumed, will drop and reject it (err %pe)\n",
 		    drm_dp_tunnel_name(intel_dp->tunnel),
 		    connector->base.base.id, connector->base.name,
 		    encoder->base.base.id, encoder->base.name,
diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.c b/drivers/gpu/drm/i915/display/intel_dpio_phy.c
index 4ca910874a4f..d20e4e9cf7f7 100644
--- a/drivers/gpu/drm/i915/display/intel_dpio_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.c
@@ -21,6 +21,7 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
+#include "bxt_dpio_phy_regs.h"
 #include "i915_reg.h"
 #include "intel_ddi.h"
 #include "intel_ddi_buf_trans.h"
@@ -29,6 +30,7 @@
 #include "intel_display_types.h"
 #include "intel_dp.h"
 #include "intel_dpio_phy.h"
+#include "vlv_dpio_phy_regs.h"
 #include "vlv_sideband.h"
 
 /**
@@ -123,9 +125,9 @@
  */
 
 /**
- * struct bxt_ddi_phy_info - Hold info for a broxton DDI phy
+ * struct bxt_dpio_phy_info - Hold info for a broxton DDI phy
  */
-struct bxt_ddi_phy_info {
+struct bxt_dpio_phy_info {
 	/**
 	 * @dual_channel: true if this phy has a second channel.
 	 */
@@ -161,7 +163,7 @@ struct bxt_ddi_phy_info {
 	} channel[2];
 };
 
-static const struct bxt_ddi_phy_info bxt_ddi_phy_info[] = {
+static const struct bxt_dpio_phy_info bxt_dpio_phy_info[] = {
 	[DPIO_PHY0] = {
 		.dual_channel = true,
 		.rcomp_phy = DPIO_PHY1,
@@ -183,7 +185,7 @@ static const struct bxt_ddi_phy_info bxt_ddi_phy_info[] = {
 	},
 };
 
-static const struct bxt_ddi_phy_info glk_ddi_phy_info[] = {
+static const struct bxt_dpio_phy_info glk_dpio_phy_info[] = {
 	[DPIO_PHY0] = {
 		.dual_channel = false,
 		.rcomp_phy = DPIO_PHY1,
@@ -216,23 +218,23 @@ static const struct bxt_ddi_phy_info glk_ddi_phy_info[] = {
 	},
 };
 
-static const struct bxt_ddi_phy_info *
+static const struct bxt_dpio_phy_info *
 bxt_get_phy_list(struct drm_i915_private *dev_priv, int *count)
 {
 	if (IS_GEMINILAKE(dev_priv)) {
-		*count =  ARRAY_SIZE(glk_ddi_phy_info);
-		return glk_ddi_phy_info;
+		*count =  ARRAY_SIZE(glk_dpio_phy_info);
+		return glk_dpio_phy_info;
 	} else {
-		*count =  ARRAY_SIZE(bxt_ddi_phy_info);
-		return bxt_ddi_phy_info;
+		*count =  ARRAY_SIZE(bxt_dpio_phy_info);
+		return bxt_dpio_phy_info;
 	}
 }
 
-static const struct bxt_ddi_phy_info *
+static const struct bxt_dpio_phy_info *
 bxt_get_phy_info(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 {
 	int count;
-	const struct bxt_ddi_phy_info *phy_list =
+	const struct bxt_dpio_phy_info *phy_list =
 		bxt_get_phy_list(dev_priv, &count);
 
 	return &phy_list[phy];
@@ -241,7 +243,7 @@ bxt_get_phy_info(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port,
 			     enum dpio_phy *phy, enum dpio_channel *ch)
 {
-	const struct bxt_ddi_phy_info *phy_info, *phys;
+	const struct bxt_dpio_phy_info *phy_info, *phys;
 	int i, count;
 
 	phys = bxt_get_phy_list(dev_priv, &count);
@@ -269,16 +271,32 @@ void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port,
 	*ch = DPIO_CH0;
 }
 
-void bxt_ddi_phy_set_signal_levels(struct intel_encoder *encoder,
-				   const struct intel_crtc_state *crtc_state)
+/*
+ * Like intel_de_rmw() but reads from a single per-lane register and
+ * writes to the group register to write the same value to all the lanes.
+ */
+static u32 bxt_dpio_phy_rmw_grp(struct drm_i915_private *i915,
+				i915_reg_t reg_single,
+				i915_reg_t reg_group,
+				u32 clear, u32 set)
+{
+	u32 old, val;
+
+	old = intel_de_read(i915, reg_single);
+	val = (old & ~clear) | set;
+	intel_de_write(i915, reg_group, val);
+
+	return old;
+}
+
+void bxt_dpio_phy_set_signal_levels(struct intel_encoder *encoder,
+				    const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	int level = intel_ddi_level(encoder, crtc_state, 0);
 	const struct intel_ddi_buf_trans *trans;
 	enum dpio_channel ch;
 	enum dpio_phy phy;
-	int n_entries;
-	u32 val;
+	int lane, n_entries;
 
 	trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !trans))
@@ -290,41 +308,51 @@ void bxt_ddi_phy_set_signal_levels(struct intel_encoder *encoder,
 	 * While we write to the group register to program all lanes at once we
 	 * can read only lane registers and we pick lanes 0/1 for that.
 	 */
-	val = intel_de_read(dev_priv, BXT_PORT_PCS_DW10_LN01(phy, ch));
-	val &= ~(TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT);
-	intel_de_write(dev_priv, BXT_PORT_PCS_DW10_GRP(phy, ch), val);
-
-	val = intel_de_read(dev_priv, BXT_PORT_TX_DW2_LN0(phy, ch));
-	val &= ~(MARGIN_000 | UNIQ_TRANS_SCALE);
-	val |= trans->entries[level].bxt.margin << MARGIN_000_SHIFT |
-		trans->entries[level].bxt.scale << UNIQ_TRANS_SCALE_SHIFT;
-	intel_de_write(dev_priv, BXT_PORT_TX_DW2_GRP(phy, ch), val);
-
-	val = intel_de_read(dev_priv, BXT_PORT_TX_DW3_LN0(phy, ch));
-	val &= ~SCALE_DCOMP_METHOD;
-	if (trans->entries[level].bxt.enable)
-		val |= SCALE_DCOMP_METHOD;
-
-	if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD))
-		drm_err(&dev_priv->drm,
-			"Disabled scaling while ouniqetrangenmethod was set");
-
-	intel_de_write(dev_priv, BXT_PORT_TX_DW3_GRP(phy, ch), val);
-
-	val = intel_de_read(dev_priv, BXT_PORT_TX_DW4_LN0(phy, ch));
-	val &= ~DE_EMPHASIS;
-	val |= trans->entries[level].bxt.deemphasis << DEEMPH_SHIFT;
-	intel_de_write(dev_priv, BXT_PORT_TX_DW4_GRP(phy, ch), val);
-
-	val = intel_de_read(dev_priv, BXT_PORT_PCS_DW10_LN01(phy, ch));
-	val |= TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT;
-	intel_de_write(dev_priv, BXT_PORT_PCS_DW10_GRP(phy, ch), val);
+	bxt_dpio_phy_rmw_grp(dev_priv, BXT_PORT_PCS_DW10_LN01(phy, ch),
+			     BXT_PORT_PCS_DW10_GRP(phy, ch),
+			     TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT, 0);
+
+	for (lane = 0; lane < crtc_state->lane_count; lane++) {
+		int level = intel_ddi_level(encoder, crtc_state, lane);
+
+		intel_de_rmw(dev_priv, BXT_PORT_TX_DW2_LN(phy, ch, lane),
+			     MARGIN_000_MASK | UNIQ_TRANS_SCALE_MASK,
+			     MARGIN_000(trans->entries[level].bxt.margin) |
+			     UNIQ_TRANS_SCALE(trans->entries[level].bxt.scale));
+	}
+
+	for (lane = 0; lane < crtc_state->lane_count; lane++) {
+		int level = intel_ddi_level(encoder, crtc_state, lane);
+		u32 val;
+
+		intel_de_rmw(dev_priv, BXT_PORT_TX_DW3_LN(phy, ch, lane),
+			     SCALE_DCOMP_METHOD,
+			     trans->entries[level].bxt.enable ?
+			     SCALE_DCOMP_METHOD : 0);
+
+		val = intel_de_read(dev_priv, BXT_PORT_TX_DW3_LN(phy, ch, lane));
+		if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD))
+			drm_err(&dev_priv->drm,
+				"Disabled scaling while ouniqetrangenmethod was set");
+	}
+
+	for (lane = 0; lane < crtc_state->lane_count; lane++) {
+		int level = intel_ddi_level(encoder, crtc_state, lane);
+
+		intel_de_rmw(dev_priv, BXT_PORT_TX_DW4_LN(phy, ch, lane),
+			     DE_EMPHASIS_MASK,
+			     DE_EMPHASIS(trans->entries[level].bxt.deemphasis));
+	}
+
+	bxt_dpio_phy_rmw_grp(dev_priv, BXT_PORT_PCS_DW10_LN01(phy, ch),
+			     BXT_PORT_PCS_DW10_GRP(phy, ch),
+			     0, TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT);
 }
 
-bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv,
-			    enum dpio_phy phy)
+bool bxt_dpio_phy_is_enabled(struct drm_i915_private *dev_priv,
+			     enum dpio_phy phy)
 {
-	const struct bxt_ddi_phy_info *phy_info;
+	const struct bxt_dpio_phy_info *phy_info;
 
 	phy_info = bxt_get_phy_info(dev_priv, phy);
 
@@ -353,7 +381,7 @@ static u32 bxt_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 {
 	u32 val = intel_de_read(dev_priv, BXT_PORT_REF_DW6(phy));
 
-	return (val & GRC_CODE_MASK) >> GRC_CODE_SHIFT;
+	return REG_FIELD_GET(GRC_CODE_MASK, val);
 }
 
 static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv,
@@ -365,20 +393,20 @@ static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv,
 			phy);
 }
 
-static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv,
-			      enum dpio_phy phy)
+static void _bxt_dpio_phy_init(struct drm_i915_private *dev_priv,
+			       enum dpio_phy phy)
 {
-	const struct bxt_ddi_phy_info *phy_info;
+	const struct bxt_dpio_phy_info *phy_info;
 	u32 val;
 
 	phy_info = bxt_get_phy_info(dev_priv, phy);
 
-	if (bxt_ddi_phy_is_enabled(dev_priv, phy)) {
+	if (bxt_dpio_phy_is_enabled(dev_priv, phy)) {
 		/* Still read out the GRC value for state verification */
 		if (phy_info->rcomp_phy != -1)
 			dev_priv->display.state.bxt_phy_grc = bxt_get_grc(dev_priv, phy);
 
-		if (bxt_ddi_phy_verify_state(dev_priv, phy)) {
+		if (bxt_dpio_phy_verify_state(dev_priv, phy)) {
 			drm_dbg(&dev_priv->drm, "DDI PHY %d already enabled, "
 				"won't reprogram it\n", phy);
 			return;
@@ -399,20 +427,17 @@ static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv,
 	 * The flag should get set in 100us according to the HW team, but
 	 * use 1ms due to occasional timeouts observed with that.
 	 */
-	if (intel_wait_for_register_fw(&dev_priv->uncore,
-				       BXT_PORT_CL1CM_DW0(phy),
-				       PHY_RESERVED | PHY_POWER_GOOD,
-				       PHY_POWER_GOOD,
-				       1))
+	if (intel_de_wait_fw(dev_priv, BXT_PORT_CL1CM_DW0(phy),
+			     PHY_RESERVED | PHY_POWER_GOOD, PHY_POWER_GOOD, 1))
 		drm_err(&dev_priv->drm, "timeout during PHY%d power on\n",
 			phy);
 
 	/* Program PLL Rcomp code offset */
-	intel_de_rmw(dev_priv, BXT_PORT_CL1CM_DW9(phy), IREF0RC_OFFSET_MASK,
-		     0xE4 << IREF0RC_OFFSET_SHIFT);
+	intel_de_rmw(dev_priv, BXT_PORT_CL1CM_DW9(phy),
+		     IREF0RC_OFFSET_MASK, IREF0RC_OFFSET(0xE4));
 
-	intel_de_rmw(dev_priv, BXT_PORT_CL1CM_DW10(phy), IREF1RC_OFFSET_MASK,
-		     0xE4 << IREF1RC_OFFSET_SHIFT);
+	intel_de_rmw(dev_priv, BXT_PORT_CL1CM_DW10(phy),
+		     IREF1RC_OFFSET_MASK, IREF1RC_OFFSET(0xE4));
 
 	/* Program power gating */
 	intel_de_rmw(dev_priv, BXT_PORT_CL1CM_DW28(phy), 0,
@@ -435,9 +460,9 @@ static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv,
 		val = bxt_get_grc(dev_priv, phy_info->rcomp_phy);
 		dev_priv->display.state.bxt_phy_grc = val;
 
-		grc_code = val << GRC_CODE_FAST_SHIFT |
-			   val << GRC_CODE_SLOW_SHIFT |
-			   val;
+		grc_code = GRC_CODE_FAST(val) |
+			GRC_CODE_SLOW(val) |
+			GRC_CODE_NOM(val);
 		intel_de_write(dev_priv, BXT_PORT_REF_DW6(phy), grc_code);
 		intel_de_rmw(dev_priv, BXT_PORT_REF_DW8(phy),
 			     0, GRC_DIS | GRC_RDY_OVRD);
@@ -449,9 +474,9 @@ static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv,
 	intel_de_rmw(dev_priv, BXT_PHY_CTL_FAMILY(phy), 0, COMMON_RESET_DIS);
 }
 
-void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy)
+void bxt_dpio_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 {
-	const struct bxt_ddi_phy_info *phy_info;
+	const struct bxt_dpio_phy_info *phy_info;
 
 	phy_info = bxt_get_phy_info(dev_priv, phy);
 
@@ -460,9 +485,9 @@ void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 	intel_de_rmw(dev_priv, BXT_P_CR_GT_DISP_PWRON, phy_info->pwron_mask, 0);
 }
 
-void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy)
+void bxt_dpio_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 {
-	const struct bxt_ddi_phy_info *phy_info =
+	const struct bxt_dpio_phy_info *phy_info =
 		bxt_get_phy_info(dev_priv, phy);
 	enum dpio_phy rcomp_phy = phy_info->rcomp_phy;
 	bool was_enabled;
@@ -471,19 +496,19 @@ void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 
 	was_enabled = true;
 	if (rcomp_phy != -1)
-		was_enabled = bxt_ddi_phy_is_enabled(dev_priv, rcomp_phy);
+		was_enabled = bxt_dpio_phy_is_enabled(dev_priv, rcomp_phy);
 
 	/*
 	 * We need to copy the GRC calibration value from rcomp_phy,
 	 * so make sure it's powered up.
 	 */
 	if (!was_enabled)
-		_bxt_ddi_phy_init(dev_priv, rcomp_phy);
+		_bxt_dpio_phy_init(dev_priv, rcomp_phy);
 
-	_bxt_ddi_phy_init(dev_priv, phy);
+	_bxt_dpio_phy_init(dev_priv, phy);
 
 	if (!was_enabled)
-		bxt_ddi_phy_uninit(dev_priv, rcomp_phy);
+		bxt_dpio_phy_uninit(dev_priv, rcomp_phy);
 }
 
 static bool __printf(6, 7)
@@ -513,10 +538,10 @@ __phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy,
 	return false;
 }
 
-bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
-			      enum dpio_phy phy)
+bool bxt_dpio_phy_verify_state(struct drm_i915_private *dev_priv,
+			       enum dpio_phy phy)
 {
-	const struct bxt_ddi_phy_info *phy_info;
+	const struct bxt_dpio_phy_info *phy_info;
 	u32 mask;
 	bool ok;
 
@@ -526,23 +551,23 @@ bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
 	__phy_reg_verify_state(dev_priv, phy, reg, mask, exp, fmt,	\
 			       ## __VA_ARGS__)
 
-	if (!bxt_ddi_phy_is_enabled(dev_priv, phy))
+	if (!bxt_dpio_phy_is_enabled(dev_priv, phy))
 		return false;
 
 	ok = true;
 
 	/* PLL Rcomp code offset */
 	ok &= _CHK(BXT_PORT_CL1CM_DW9(phy),
-		    IREF0RC_OFFSET_MASK, 0xe4 << IREF0RC_OFFSET_SHIFT,
-		    "BXT_PORT_CL1CM_DW9(%d)", phy);
+		   IREF0RC_OFFSET_MASK, IREF0RC_OFFSET(0xe4),
+		   "BXT_PORT_CL1CM_DW9(%d)", phy);
 	ok &= _CHK(BXT_PORT_CL1CM_DW10(phy),
-		    IREF1RC_OFFSET_MASK, 0xe4 << IREF1RC_OFFSET_SHIFT,
-		    "BXT_PORT_CL1CM_DW10(%d)", phy);
+		   IREF1RC_OFFSET_MASK, IREF1RC_OFFSET(0xe4),
+		   "BXT_PORT_CL1CM_DW10(%d)", phy);
 
 	/* Power gating */
 	mask = OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | SUS_CLK_CONFIG;
 	ok &= _CHK(BXT_PORT_CL1CM_DW28(phy), mask, mask,
-		    "BXT_PORT_CL1CM_DW28(%d)", phy);
+		   "BXT_PORT_CL1CM_DW28(%d)", phy);
 
 	if (phy_info->dual_channel)
 		ok &= _CHK(BXT_PORT_CL2CM_DW6(phy),
@@ -552,9 +577,9 @@ bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
 	if (phy_info->rcomp_phy != -1) {
 		u32 grc_code = dev_priv->display.state.bxt_phy_grc;
 
-		grc_code = grc_code << GRC_CODE_FAST_SHIFT |
-			   grc_code << GRC_CODE_SLOW_SHIFT |
-			   grc_code;
+		grc_code = GRC_CODE_FAST(grc_code) |
+			GRC_CODE_SLOW(grc_code) |
+			GRC_CODE_NOM(grc_code);
 		mask = GRC_CODE_FAST_MASK | GRC_CODE_SLOW_MASK |
 		       GRC_CODE_NOM_MASK;
 		ok &= _CHK(BXT_PORT_REF_DW6(phy), mask, grc_code,
@@ -562,7 +587,7 @@ bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
 
 		mask = GRC_DIS | GRC_RDY_OVRD;
 		ok &= _CHK(BXT_PORT_REF_DW8(phy), mask, mask,
-			    "BXT_PORT_REF_DW8(%d)", phy);
+			   "BXT_PORT_REF_DW8(%d)", phy);
 	}
 
 	return ok;
@@ -570,7 +595,7 @@ bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
 }
 
 u8
-bxt_ddi_phy_calc_lane_lat_optim_mask(u8 lane_count)
+bxt_dpio_phy_calc_lane_lat_optim_mask(u8 lane_count)
 {
 	switch (lane_count) {
 	case 1:
@@ -586,8 +611,8 @@ bxt_ddi_phy_calc_lane_lat_optim_mask(u8 lane_count)
 	}
 }
 
-void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder,
-				     u8 lane_lat_optim_mask)
+void bxt_dpio_phy_set_lane_optim_mask(struct intel_encoder *encoder,
+				      u8 lane_lat_optim_mask)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum port port = encoder->port;
@@ -598,24 +623,18 @@ void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder,
 	bxt_port_to_phy_channel(dev_priv, port, &phy, &ch);
 
 	for (lane = 0; lane < 4; lane++) {
-		u32 val = intel_de_read(dev_priv,
-					BXT_PORT_TX_DW14_LN(phy, ch, lane));
-
 		/*
 		 * Note that on CHV this flag is called UPAR, but has
 		 * the same function.
 		 */
-		val &= ~LATENCY_OPTIM;
-		if (lane_lat_optim_mask & BIT(lane))
-			val |= LATENCY_OPTIM;
-
-		intel_de_write(dev_priv, BXT_PORT_TX_DW14_LN(phy, ch, lane),
-			       val);
+		intel_de_rmw(dev_priv, BXT_PORT_TX_DW14_LN(phy, ch, lane),
+			     LATENCY_OPTIM,
+			     lane_lat_optim_mask & BIT(lane) ? LATENCY_OPTIM : 0);
 	}
 }
 
 u8
-bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder)
+bxt_dpio_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum port port = encoder->port;
@@ -701,9 +720,8 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
-	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
+	enum dpio_phy phy = vlv_dig_port_to_phy(dig_port);
 	u32 val;
 	int i;
 
@@ -740,7 +758,7 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 	for (i = 0; i < crtc_state->lane_count; i++) {
 		val = vlv_dpio_read(dev_priv, phy, CHV_TX_DW4(ch, i));
 		val &= ~DPIO_SWING_DEEMPH9P5_MASK;
-		val |= deemph_reg_value << DPIO_SWING_DEEMPH9P5_SHIFT;
+		val |= DPIO_SWING_DEEMPH9P5(deemph_reg_value);
 		vlv_dpio_write(dev_priv, phy, CHV_TX_DW4(ch, i), val);
 	}
 
@@ -749,15 +767,15 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 		val = vlv_dpio_read(dev_priv, phy, CHV_TX_DW2(ch, i));
 
 		val &= ~DPIO_SWING_MARGIN000_MASK;
-		val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT;
+		val |= DPIO_SWING_MARGIN000(margin_reg_value);
 
 		/*
 		 * Supposedly this value shouldn't matter when unique transition
 		 * scale is disabled, but in fact it does matter. Let's just
 		 * always program the same value and hope it's OK.
 		 */
-		val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT);
-		val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT;
+		val &= ~DPIO_UNIQ_TRANS_SCALE_MASK;
+		val |= DPIO_UNIQ_TRANS_SCALE(0x9a);
 
 		vlv_dpio_write(dev_priv, phy, CHV_TX_DW2(ch, i), val);
 	}
@@ -796,9 +814,9 @@ void chv_data_lane_soft_reset(struct intel_encoder *encoder,
 			      bool reset)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	enum dpio_channel ch = vlv_dig_port_to_channel(enc_to_dig_port(encoder));
-	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
+	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
+	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
+	enum dpio_phy phy = vlv_dig_port_to_phy(dig_port);
 	u32 val;
 
 	val = vlv_dpio_read(dev_priv, phy, VLV_PCS01_DW0(ch));
@@ -843,7 +861,7 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
-	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
+	enum dpio_phy phy = vlv_dig_port_to_phy(dig_port);
 	enum pipe pipe = crtc->pipe;
 	unsigned int lane_mask =
 		intel_dp_unused_lane_mask(crtc_state->lane_count);
@@ -866,39 +884,39 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
 
 	/* program left/right clock distribution */
 	if (pipe != PIPE_B) {
-		val = vlv_dpio_read(dev_priv, phy, _CHV_CMN_DW5_CH0);
+		val = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW5_CH0);
 		val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
 		if (ch == DPIO_CH0)
 			val |= CHV_BUFLEFTENA1_FORCE;
 		if (ch == DPIO_CH1)
 			val |= CHV_BUFRIGHTENA1_FORCE;
-		vlv_dpio_write(dev_priv, phy, _CHV_CMN_DW5_CH0, val);
+		vlv_dpio_write(dev_priv, phy, CHV_CMN_DW5_CH0, val);
 	} else {
-		val = vlv_dpio_read(dev_priv, phy, _CHV_CMN_DW1_CH1);
+		val = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW1_CH1);
 		val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
 		if (ch == DPIO_CH0)
 			val |= CHV_BUFLEFTENA2_FORCE;
 		if (ch == DPIO_CH1)
 			val |= CHV_BUFRIGHTENA2_FORCE;
-		vlv_dpio_write(dev_priv, phy, _CHV_CMN_DW1_CH1, val);
+		vlv_dpio_write(dev_priv, phy, CHV_CMN_DW1_CH1, val);
 	}
 
 	/* program clock channel usage */
 	val = vlv_dpio_read(dev_priv, phy, VLV_PCS01_DW8(ch));
-	val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
-	if (pipe != PIPE_B)
-		val &= ~CHV_PCS_USEDCLKCHANNEL;
+	val |= DPIO_PCS_USEDCLKCHANNEL_OVRRIDE;
+	if (pipe == PIPE_B)
+		val |= DPIO_PCS_USEDCLKCHANNEL;
 	else
-		val |= CHV_PCS_USEDCLKCHANNEL;
+		val &= ~DPIO_PCS_USEDCLKCHANNEL;
 	vlv_dpio_write(dev_priv, phy, VLV_PCS01_DW8(ch), val);
 
 	if (crtc_state->lane_count > 2) {
 		val = vlv_dpio_read(dev_priv, phy, VLV_PCS23_DW8(ch));
-		val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
-		if (pipe != PIPE_B)
-			val &= ~CHV_PCS_USEDCLKCHANNEL;
+		val |= DPIO_PCS_USEDCLKCHANNEL_OVRRIDE;
+		if (pipe == PIPE_B)
+			val |= DPIO_PCS_USEDCLKCHANNEL;
 		else
-			val |= CHV_PCS_USEDCLKCHANNEL;
+			val &= ~DPIO_PCS_USEDCLKCHANNEL;
 		vlv_dpio_write(dev_priv, phy, VLV_PCS23_DW8(ch), val);
 	}
 
@@ -908,10 +926,10 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
 	 * pick the CL based on the port.
 	 */
 	val = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW19(ch));
-	if (pipe != PIPE_B)
-		val &= ~CHV_CMN_USEDCLKCHANNEL;
-	else
+	if (pipe == PIPE_B)
 		val |= CHV_CMN_USEDCLKCHANNEL;
+	else
+		val &= ~CHV_CMN_USEDCLKCHANNEL;
 	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW19(ch), val);
 
 	vlv_dpio_put(dev_priv);
@@ -923,9 +941,8 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
-	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
+	enum dpio_phy phy = vlv_dig_port_to_phy(dig_port);
 	int data, i, stagger;
 	u32 val;
 
@@ -946,11 +963,10 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 	for (i = 0; i < crtc_state->lane_count; i++) {
 		/* Set the upar bit */
 		if (crtc_state->lane_count == 1)
-			data = 0x0;
+			data = 0;
 		else
-			data = (i == 1) ? 0x0 : 0x1;
-		vlv_dpio_write(dev_priv, phy, CHV_TX_DW14(ch, i),
-				data << DPIO_UPAR_SHIFT);
+			data = (i == 1) ? 0 : DPIO_UPAR;
+		vlv_dpio_write(dev_priv, phy, CHV_TX_DW14(ch, i), data);
 	}
 
 	/* Data lane stagger programming */
@@ -1012,21 +1028,21 @@ void chv_phy_post_pll_disable(struct intel_encoder *encoder,
 			      const struct intel_crtc_state *old_crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	enum dpio_phy phy = vlv_dig_port_to_phy(enc_to_dig_port(encoder));
 	enum pipe pipe = to_intel_crtc(old_crtc_state->uapi.crtc)->pipe;
-	enum dpio_phy phy = vlv_pipe_to_phy(pipe);
 	u32 val;
 
 	vlv_dpio_get(dev_priv);
 
 	/* disable left/right clock distribution */
 	if (pipe != PIPE_B) {
-		val = vlv_dpio_read(dev_priv, phy, _CHV_CMN_DW5_CH0);
+		val = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW5_CH0);
 		val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
-		vlv_dpio_write(dev_priv, phy, _CHV_CMN_DW5_CH0, val);
+		vlv_dpio_write(dev_priv, phy, CHV_CMN_DW5_CH0, val);
 	} else {
-		val = vlv_dpio_read(dev_priv, phy, _CHV_CMN_DW1_CH1);
+		val = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW1_CH1);
 		val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
-		vlv_dpio_write(dev_priv, phy, _CHV_CMN_DW1_CH1, val);
+		vlv_dpio_write(dev_priv, phy, CHV_CMN_DW1_CH1, val);
 	}
 
 	vlv_dpio_put(dev_priv);
@@ -1050,24 +1066,23 @@ void vlv_set_phy_signal_level(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	enum dpio_channel port = vlv_dig_port_to_channel(dig_port);
-	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
+	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
+	enum dpio_phy phy = vlv_dig_port_to_phy(dig_port);
 
 	vlv_dpio_get(dev_priv);
 
-	vlv_dpio_write(dev_priv, phy, VLV_TX_DW5(port), 0x00000000);
-	vlv_dpio_write(dev_priv, phy, VLV_TX_DW4(port), demph_reg_value);
-	vlv_dpio_write(dev_priv, phy, VLV_TX_DW2(port),
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW5_GRP(ch), 0x00000000);
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW4_GRP(ch), demph_reg_value);
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW2_GRP(ch),
 			 uniqtranscale_reg_value);
-	vlv_dpio_write(dev_priv, phy, VLV_TX_DW3(port), 0x0C782040);
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW3_GRP(ch), 0x0C782040);
 
 	if (tx3_demph)
-		vlv_dpio_write(dev_priv, phy, VLV_TX3_DW4(port), tx3_demph);
+		vlv_dpio_write(dev_priv, phy, VLV_TX_DW4(ch, 3), tx3_demph);
 
-	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW11(port), 0x00030000);
-	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW9(port), preemph_reg_value);
-	vlv_dpio_write(dev_priv, phy, VLV_TX_DW5(port), DPIO_TX_OCALINIT_EN);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW11_GRP(ch), 0x00030000);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW9_GRP(ch), preemph_reg_value);
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW5_GRP(ch), DPIO_TX_OCALINIT_EN);
 
 	vlv_dpio_put(dev_priv);
 }
@@ -1077,26 +1092,25 @@ void vlv_phy_pre_pll_enable(struct intel_encoder *encoder,
 {
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	enum dpio_channel port = vlv_dig_port_to_channel(dig_port);
-	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
+	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
+	enum dpio_phy phy = vlv_dig_port_to_phy(dig_port);
 
 	/* Program Tx lane resets to default */
 	vlv_dpio_get(dev_priv);
 
-	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW0(port),
-			 DPIO_PCS_TX_LANE2_RESET |
-			 DPIO_PCS_TX_LANE1_RESET);
-	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW1(port),
-			 DPIO_PCS_CLK_CRI_RXEB_EIOS_EN |
-			 DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN |
-			 (1<<DPIO_PCS_CLK_DATAWIDTH_SHIFT) |
-				 DPIO_PCS_CLK_SOFT_RESET);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW0_GRP(ch),
+		       DPIO_PCS_TX_LANE2_RESET |
+		       DPIO_PCS_TX_LANE1_RESET);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW1_GRP(ch),
+		       DPIO_PCS_CLK_CRI_RXEB_EIOS_EN |
+		       DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN |
+		       DPIO_PCS_CLK_DATAWIDTH_8_10 |
+		       DPIO_PCS_CLK_SOFT_RESET);
 
 	/* Fix up inter-pair skew failure */
-	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW12(port), 0x00750f00);
-	vlv_dpio_write(dev_priv, phy, VLV_TX_DW11(port), 0x00001500);
-	vlv_dpio_write(dev_priv, phy, VLV_TX_DW14(port), 0x40400000);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW12_GRP(ch), 0x00750f00);
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW11_GRP(ch), 0x00001500);
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW14_GRP(ch), 0x40400000);
 
 	vlv_dpio_put(dev_priv);
 }
@@ -1108,26 +1122,23 @@ void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	enum dpio_channel port = vlv_dig_port_to_channel(dig_port);
+	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
+	enum dpio_phy phy = vlv_dig_port_to_phy(dig_port);
 	enum pipe pipe = crtc->pipe;
-	enum dpio_phy phy = vlv_pipe_to_phy(pipe);
 	u32 val;
 
 	vlv_dpio_get(dev_priv);
 
 	/* Enable clock channels for this port */
-	val = vlv_dpio_read(dev_priv, phy, VLV_PCS01_DW8(port));
-	val = 0;
-	if (pipe)
-		val |= (1<<21);
-	else
-		val &= ~(1<<21);
-	val |= 0x001000c4;
-	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW8(port), val);
+	val = DPIO_PCS_USEDCLKCHANNEL_OVRRIDE;
+	if (pipe == PIPE_B)
+		val |= DPIO_PCS_USEDCLKCHANNEL;
+	val |= 0xc4;
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW8_GRP(ch), val);
 
 	/* Program lane clock */
-	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW14(port), 0x00760018);
-	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW23(port), 0x00400888);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW14_GRP(ch), 0x00760018);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW23_GRP(ch), 0x00400888);
 
 	vlv_dpio_put(dev_priv);
 }
@@ -1137,12 +1148,11 @@ void vlv_phy_reset_lanes(struct intel_encoder *encoder,
 {
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
-	enum dpio_channel port = vlv_dig_port_to_channel(dig_port);
-	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
+	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
+	enum dpio_phy phy = vlv_dig_port_to_phy(dig_port);
 
 	vlv_dpio_get(dev_priv);
-	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW0(port), 0x00000000);
-	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW1(port), 0x00e00060);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW0_GRP(ch), 0x00000000);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW1_GRP(ch), 0x00e00060);
 	vlv_dpio_put(dev_priv);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.h b/drivers/gpu/drm/i915/display/intel_dpio_phy.h
index 9adc4e8c1738..226994dcb89b 100644
--- a/drivers/gpu/drm/i915/display/intel_dpio_phy.h
+++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.h
@@ -29,18 +29,18 @@ enum dpio_phy {
 #ifdef I915
 void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port,
 			     enum dpio_phy *phy, enum dpio_channel *ch);
-void bxt_ddi_phy_set_signal_levels(struct intel_encoder *encoder,
-				   const struct intel_crtc_state *crtc_state);
-void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy);
-void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy);
-bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv,
-			    enum dpio_phy phy);
-bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
-			      enum dpio_phy phy);
-u8 bxt_ddi_phy_calc_lane_lat_optim_mask(u8 lane_count);
-void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder,
-				     u8 lane_lat_optim_mask);
-u8 bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder);
+void bxt_dpio_phy_set_signal_levels(struct intel_encoder *encoder,
+				    const struct intel_crtc_state *crtc_state);
+void bxt_dpio_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy);
+void bxt_dpio_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy);
+bool bxt_dpio_phy_is_enabled(struct drm_i915_private *dev_priv,
+			     enum dpio_phy phy);
+bool bxt_dpio_phy_verify_state(struct drm_i915_private *dev_priv,
+			       enum dpio_phy phy);
+u8 bxt_dpio_phy_calc_lane_lat_optim_mask(u8 lane_count);
+void bxt_dpio_phy_set_lane_optim_mask(struct intel_encoder *encoder,
+				      u8 lane_lat_optim_mask);
+u8 bxt_dpio_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder);
 
 enum dpio_channel vlv_dig_port_to_channel(struct intel_digital_port *dig_port);
 enum dpio_phy vlv_dig_port_to_phy(struct intel_digital_port *dig_port);
@@ -77,35 +77,35 @@ static inline void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, en
 					   enum dpio_phy *phy, enum dpio_channel *ch)
 {
 }
-static inline void bxt_ddi_phy_set_signal_levels(struct intel_encoder *encoder,
-						 const struct intel_crtc_state *crtc_state)
+static inline void bxt_dpio_phy_set_signal_levels(struct intel_encoder *encoder,
+						  const struct intel_crtc_state *crtc_state)
 {
 }
-static inline void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy)
+static inline void bxt_dpio_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 {
 }
-static inline void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy)
+static inline void bxt_dpio_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 {
 }
-static inline bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv,
-					  enum dpio_phy phy)
+static inline bool bxt_dpio_phy_is_enabled(struct drm_i915_private *dev_priv,
+					   enum dpio_phy phy)
 {
 	return false;
 }
-static inline bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
-					    enum dpio_phy phy)
+static inline bool bxt_dpio_phy_verify_state(struct drm_i915_private *dev_priv,
+					     enum dpio_phy phy)
 {
 	return true;
 }
-static inline u8 bxt_ddi_phy_calc_lane_lat_optim_mask(u8 lane_count)
+static inline u8 bxt_dpio_phy_calc_lane_lat_optim_mask(u8 lane_count)
 {
 	return 0;
 }
-static inline void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder,
-						   u8 lane_lat_optim_mask)
+static inline void bxt_dpio_phy_set_lane_optim_mask(struct intel_encoder *encoder,
+						    u8 lane_lat_optim_mask)
 {
 }
-static inline u8 bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder)
+static inline u8 bxt_dpio_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder)
 {
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c
index 3038655377ea..a981f45facb3 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll.c
@@ -20,6 +20,7 @@
 #include "intel_panel.h"
 #include "intel_pps.h"
 #include "intel_snps_phy.h"
+#include "vlv_dpio_phy_regs.h"
 #include "vlv_sideband.h"
 
 struct intel_dpll_funcs {
@@ -369,38 +370,68 @@ int chv_calc_dpll_params(int refclk, struct dpll *clock)
 	return clock->dot;
 }
 
-static int i9xx_pll_refclk(struct drm_device *dev,
-			   const struct intel_crtc_state *pipe_config)
+static int i9xx_pll_refclk(const struct intel_crtc_state *crtc_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	u32 dpll = pipe_config->dpll_hw_state.dpll;
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	const struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
 
-	if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
-		return dev_priv->display.vbt.lvds_ssc_freq;
-	else if (HAS_PCH_SPLIT(dev_priv))
+	if ((hw_state->dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
+		return i915->display.vbt.lvds_ssc_freq;
+	else if (HAS_PCH_SPLIT(i915))
 		return 120000;
-	else if (DISPLAY_VER(dev_priv) != 2)
+	else if (DISPLAY_VER(i915) != 2)
 		return 96000;
 	else
 		return 48000;
 }
 
+void i9xx_dpll_get_hw_state(struct intel_crtc *crtc,
+			    struct intel_dpll_hw_state *dpll_hw_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	struct i9xx_dpll_hw_state *hw_state = &dpll_hw_state->i9xx;
+
+	if (DISPLAY_VER(dev_priv) >= 4) {
+		u32 tmp;
+
+		/* No way to read it out on pipes B and C */
+		if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
+			tmp = dev_priv->display.state.chv_dpll_md[crtc->pipe];
+		else
+			tmp = intel_de_read(dev_priv, DPLL_MD(crtc->pipe));
+
+		hw_state->dpll_md = tmp;
+	}
+
+	hw_state->dpll = intel_de_read(dev_priv, DPLL(crtc->pipe));
+
+	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
+		hw_state->fp0 = intel_de_read(dev_priv, FP0(crtc->pipe));
+		hw_state->fp1 = intel_de_read(dev_priv, FP1(crtc->pipe));
+	} else {
+		/* Mask out read-only status bits. */
+		hw_state->dpll &= ~(DPLL_LOCK_VLV |
+				    DPLL_PORTC_READY_MASK |
+				    DPLL_PORTB_READY_MASK);
+	}
+}
+
 /* Returns the clock of the currently programmed mode of the given pipe. */
-void i9xx_crtc_clock_get(struct intel_crtc *crtc,
-			 struct intel_crtc_state *pipe_config)
+void i9xx_crtc_clock_get(struct intel_crtc_state *crtc_state)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	u32 dpll = pipe_config->dpll_hw_state.dpll;
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	const struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
+	u32 dpll = hw_state->dpll;
 	u32 fp;
 	struct dpll clock;
 	int port_clock;
-	int refclk = i9xx_pll_refclk(dev, pipe_config);
+	int refclk = i9xx_pll_refclk(crtc_state);
 
 	if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
-		fp = pipe_config->dpll_hw_state.fp0;
+		fp = hw_state->fp0;
 	else
-		fp = pipe_config->dpll_hw_state.fp1;
+		fp = hw_state->fp1;
 
 	clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
 	if (IS_PINEVIEW(dev_priv)) {
@@ -475,68 +506,69 @@ void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 	 * port_clock to compute adjusted_mode.crtc_clock in the
 	 * encoder's get_config() function.
 	 */
-	pipe_config->port_clock = port_clock;
+	crtc_state->port_clock = port_clock;
 }
 
-void vlv_crtc_clock_get(struct intel_crtc *crtc,
-			struct intel_crtc_state *pipe_config)
+void vlv_crtc_clock_get(struct intel_crtc_state *crtc_state)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	enum dpio_channel ch = vlv_pipe_to_channel(crtc->pipe);
 	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
-	struct dpll clock;
-	u32 mdiv;
+	const struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
 	int refclk = 100000;
+	struct dpll clock;
+	u32 tmp;
 
 	/* In case of DSI, DPLL will not be used */
-	if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
+	if ((hw_state->dpll & DPLL_VCO_ENABLE) == 0)
 		return;
 
 	vlv_dpio_get(dev_priv);
-	mdiv = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW3(crtc->pipe));
+	tmp = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW3(ch));
 	vlv_dpio_put(dev_priv);
 
-	clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
-	clock.m2 = mdiv & DPIO_M2DIV_MASK;
-	clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
-	clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
-	clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
+	clock.m1 = REG_FIELD_GET(DPIO_M1_DIV_MASK, tmp);
+	clock.m2 = REG_FIELD_GET(DPIO_M2_DIV_MASK, tmp);
+	clock.n = REG_FIELD_GET(DPIO_N_DIV_MASK, tmp);
+	clock.p1 = REG_FIELD_GET(DPIO_P1_DIV_MASK, tmp);
+	clock.p2 = REG_FIELD_GET(DPIO_P2_DIV_MASK, tmp);
 
-	pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
+	crtc_state->port_clock = vlv_calc_dpll_params(refclk, &clock);
 }
 
-void chv_crtc_clock_get(struct intel_crtc *crtc,
-			struct intel_crtc_state *pipe_config)
+void chv_crtc_clock_get(struct intel_crtc_state *crtc_state)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	enum dpio_channel port = vlv_pipe_to_channel(crtc->pipe);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	enum dpio_channel ch = vlv_pipe_to_channel(crtc->pipe);
 	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
+	const struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
 	struct dpll clock;
 	u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
 	int refclk = 100000;
 
 	/* In case of DSI, DPLL will not be used */
-	if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
+	if ((hw_state->dpll & DPLL_VCO_ENABLE) == 0)
 		return;
 
 	vlv_dpio_get(dev_priv);
-	cmn_dw13 = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW13(port));
-	pll_dw0 = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW0(port));
-	pll_dw1 = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW1(port));
-	pll_dw2 = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW2(port));
-	pll_dw3 = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW3(port));
+	cmn_dw13 = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW13(ch));
+	pll_dw0 = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW0(ch));
+	pll_dw1 = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW1(ch));
+	pll_dw2 = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW2(ch));
+	pll_dw3 = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW3(ch));
 	vlv_dpio_put(dev_priv);
 
-	clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
-	clock.m2 = (pll_dw0 & 0xff) << 22;
+	clock.m1 = REG_FIELD_GET(DPIO_CHV_M1_DIV_MASK, pll_dw1) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
+	clock.m2 = REG_FIELD_GET(DPIO_CHV_M2_DIV_MASK, pll_dw0) << 22;
 	if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
-		clock.m2 |= pll_dw2 & 0x3fffff;
-	clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
-	clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
-	clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
+		clock.m2 |= REG_FIELD_GET(DPIO_CHV_M2_FRAC_DIV_MASK, pll_dw2);
+	clock.n = REG_FIELD_GET(DPIO_CHV_N_DIV_MASK, pll_dw1);
+	clock.p1 = REG_FIELD_GET(DPIO_CHV_P1_DIV_MASK, cmn_dw13);
+	clock.p2 = REG_FIELD_GET(DPIO_CHV_P2_DIV_MASK, cmn_dw13);
 
-	pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
+	crtc_state->port_clock = chv_calc_dpll_params(refclk, &clock);
 }
 
 /*
@@ -958,37 +990,20 @@ static u32 pnv_dpll_compute_fp(const struct dpll *dpll)
 	return (1 << dpll->n) << 16 | dpll->m2;
 }
 
-static void i9xx_update_pll_dividers(struct intel_crtc_state *crtc_state,
-				     const struct dpll *clock,
-				     const struct dpll *reduced_clock)
+static u32 i965_dpll_md(const struct intel_crtc_state *crtc_state)
 {
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	u32 fp, fp2;
-
-	if (IS_PINEVIEW(dev_priv)) {
-		fp = pnv_dpll_compute_fp(clock);
-		fp2 = pnv_dpll_compute_fp(reduced_clock);
-	} else {
-		fp = i9xx_dpll_compute_fp(clock);
-		fp2 = i9xx_dpll_compute_fp(reduced_clock);
-	}
-
-	crtc_state->dpll_hw_state.fp0 = fp;
-	crtc_state->dpll_hw_state.fp1 = fp2;
+	return (crtc_state->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
 }
 
-static void i9xx_compute_dpll(struct intel_crtc_state *crtc_state,
-			      const struct dpll *clock,
-			      const struct dpll *reduced_clock)
+static u32 i9xx_dpll(const struct intel_crtc_state *crtc_state,
+		     const struct dpll *clock,
+		     const struct dpll *reduced_clock)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	u32 dpll;
 
-	i9xx_update_pll_dividers(crtc_state, clock, reduced_clock);
-
-	dpll = DPLL_VGA_MODE_DIS;
+	dpll = DPLL_VCO_ENABLE | DPLL_VGA_MODE_DIS;
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
 		dpll |= DPLLB_MODE_LVDS;
@@ -1047,27 +1062,40 @@ static void i9xx_compute_dpll(struct intel_crtc_state *crtc_state,
 	else
 		dpll |= PLL_REF_INPUT_DREFCLK;
 
-	dpll |= DPLL_VCO_ENABLE;
-	crtc_state->dpll_hw_state.dpll = dpll;
-
-	if (DISPLAY_VER(dev_priv) >= 4) {
-		u32 dpll_md = (crtc_state->pixel_multiplier - 1)
-			<< DPLL_MD_UDI_MULTIPLIER_SHIFT;
-		crtc_state->dpll_hw_state.dpll_md = dpll_md;
-	}
+	return dpll;
 }
 
-static void i8xx_compute_dpll(struct intel_crtc_state *crtc_state,
+static void i9xx_compute_dpll(struct intel_crtc_state *crtc_state,
 			      const struct dpll *clock,
 			      const struct dpll *reduced_clock)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	u32 dpll;
+	struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
 
-	i9xx_update_pll_dividers(crtc_state, clock, reduced_clock);
+	if (IS_PINEVIEW(dev_priv)) {
+		hw_state->fp0 = pnv_dpll_compute_fp(clock);
+		hw_state->fp1 = pnv_dpll_compute_fp(reduced_clock);
+	} else {
+		hw_state->fp0 = i9xx_dpll_compute_fp(clock);
+		hw_state->fp1 = i9xx_dpll_compute_fp(reduced_clock);
+	}
+
+	hw_state->dpll = i9xx_dpll(crtc_state, clock, reduced_clock);
+
+	if (DISPLAY_VER(dev_priv) >= 4)
+		hw_state->dpll_md = i965_dpll_md(crtc_state);
+}
 
-	dpll = DPLL_VGA_MODE_DIS;
+static u32 i8xx_dpll(const struct intel_crtc_state *crtc_state,
+		     const struct dpll *clock,
+		     const struct dpll *reduced_clock)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	u32 dpll;
+
+	dpll = DPLL_VCO_ENABLE | DPLL_VGA_MODE_DIS;
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 		dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
@@ -1104,8 +1132,19 @@ static void i8xx_compute_dpll(struct intel_crtc_state *crtc_state,
 	else
 		dpll |= PLL_REF_INPUT_DREFCLK;
 
-	dpll |= DPLL_VCO_ENABLE;
-	crtc_state->dpll_hw_state.dpll = dpll;
+	return dpll;
+}
+
+static void i8xx_compute_dpll(struct intel_crtc_state *crtc_state,
+			      const struct dpll *clock,
+			      const struct dpll *reduced_clock)
+{
+	struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
+
+	hw_state->fp0 = i9xx_dpll_compute_fp(clock);
+	hw_state->fp1 = i9xx_dpll_compute_fp(reduced_clock);
+
+	hw_state->dpll = i8xx_dpll(crtc_state, clock, reduced_clock);
 }
 
 static int hsw_crtc_compute_clock(struct intel_atomic_state *state,
@@ -1185,62 +1224,54 @@ static int mtl_crtc_compute_clock(struct intel_atomic_state *state,
 		return ret;
 
 	/* TODO: Do the readback via intel_compute_shared_dplls() */
-	crtc_state->port_clock = intel_cx0pll_calc_port_clock(encoder, &crtc_state->cx0pll_state);
+	crtc_state->port_clock = intel_cx0pll_calc_port_clock(encoder, &crtc_state->dpll_hw_state.cx0pll);
 
 	crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
 
 	return 0;
 }
 
+static int ilk_fb_cb_factor(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
+	    ((intel_panel_use_ssc(i915) && i915->display.vbt.lvds_ssc_freq == 100000) ||
+	     (HAS_PCH_IBX(i915) && intel_is_dual_link_lvds(i915))))
+		return 25;
+
+	if (crtc_state->sdvo_tv_clock)
+		return 20;
+
+	return 21;
+}
+
 static bool ilk_needs_fb_cb_tune(const struct dpll *dpll, int factor)
 {
 	return dpll->m < factor * dpll->n;
 }
 
-static void ilk_update_pll_dividers(struct intel_crtc_state *crtc_state,
-				    const struct dpll *clock,
-				    const struct dpll *reduced_clock)
+static u32 ilk_dpll_compute_fp(const struct dpll *clock, int factor)
 {
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	u32 fp, fp2;
-	int factor;
-
-	/* Enable autotuning of the PLL clock (if permissible) */
-	factor = 21;
-	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
-		if ((intel_panel_use_ssc(dev_priv) &&
-		     dev_priv->display.vbt.lvds_ssc_freq == 100000) ||
-		    (HAS_PCH_IBX(dev_priv) &&
-		     intel_is_dual_link_lvds(dev_priv)))
-			factor = 25;
-	} else if (crtc_state->sdvo_tv_clock) {
-		factor = 20;
-	}
+	u32 fp;
 
 	fp = i9xx_dpll_compute_fp(clock);
 	if (ilk_needs_fb_cb_tune(clock, factor))
 		fp |= FP_CB_TUNE;
 
-	fp2 = i9xx_dpll_compute_fp(reduced_clock);
-	if (ilk_needs_fb_cb_tune(reduced_clock, factor))
-		fp2 |= FP_CB_TUNE;
-
-	crtc_state->dpll_hw_state.fp0 = fp;
-	crtc_state->dpll_hw_state.fp1 = fp2;
+	return fp;
 }
 
-static void ilk_compute_dpll(struct intel_crtc_state *crtc_state,
-			     const struct dpll *clock,
-			     const struct dpll *reduced_clock)
+static u32 ilk_dpll(const struct intel_crtc_state *crtc_state,
+		    const struct dpll *clock,
+		    const struct dpll *reduced_clock)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	u32 dpll;
 
-	ilk_update_pll_dividers(crtc_state, clock, reduced_clock);
-
-	dpll = 0;
+	dpll = DPLL_VCO_ENABLE;
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
 		dpll |= DPLLB_MODE_LVDS;
@@ -1302,9 +1333,20 @@ static void ilk_compute_dpll(struct intel_crtc_state *crtc_state,
 	else
 		dpll |= PLL_REF_INPUT_DREFCLK;
 
-	dpll |= DPLL_VCO_ENABLE;
+	return dpll;
+}
+
+static void ilk_compute_dpll(struct intel_crtc_state *crtc_state,
+			     const struct dpll *clock,
+			     const struct dpll *reduced_clock)
+{
+	struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
+	int factor = ilk_fb_cb_factor(crtc_state);
+
+	hw_state->fp0 = ilk_dpll_compute_fp(clock, factor);
+	hw_state->fp1 = ilk_dpll_compute_fp(reduced_clock, factor);
 
-	crtc_state->dpll_hw_state.dpll = dpll;
+	hw_state->dpll = ilk_dpll(crtc_state, clock, reduced_clock);
 }
 
 static int ilk_crtc_compute_clock(struct intel_atomic_state *state,
@@ -1377,39 +1419,56 @@ static int ilk_crtc_get_shared_dpll(struct intel_atomic_state *state,
 	return intel_reserve_shared_dplls(state, crtc, NULL);
 }
 
-void vlv_compute_dpll(struct intel_crtc_state *crtc_state)
+static u32 vlv_dpll(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	u32 dpll;
 
-	crtc_state->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
+	dpll = DPLL_INTEGRATED_REF_CLK_VLV |
 		DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
+
 	if (crtc->pipe != PIPE_A)
-		crtc_state->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
+		dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
 
 	/* DPLL not used with DSI, but still need the rest set up */
 	if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI))
-		crtc_state->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
-			DPLL_EXT_BUFFER_ENABLE_VLV;
+		dpll |= DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV;
 
-	crtc_state->dpll_hw_state.dpll_md =
-		(crtc_state->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
+	return dpll;
 }
 
-void chv_compute_dpll(struct intel_crtc_state *crtc_state)
+void vlv_compute_dpll(struct intel_crtc_state *crtc_state)
+{
+	struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
+
+	hw_state->dpll = vlv_dpll(crtc_state);
+	hw_state->dpll_md = i965_dpll_md(crtc_state);
+}
+
+static u32 chv_dpll(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	u32 dpll;
 
-	crtc_state->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
+	dpll = DPLL_SSC_REF_CLK_CHV |
 		DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
+
 	if (crtc->pipe != PIPE_A)
-		crtc_state->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
+		dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
 
 	/* DPLL not used with DSI, but still need the rest set up */
 	if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI))
-		crtc_state->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
+		dpll |= DPLL_VCO_ENABLE;
 
-	crtc_state->dpll_hw_state.dpll_md =
-		(crtc_state->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
+	return dpll;
+}
+
+void chv_compute_dpll(struct intel_crtc_state *crtc_state)
+{
+	struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
+
+	hw_state->dpll = chv_dpll(crtc_state);
+	hw_state->dpll_md = i965_dpll_md(crtc_state);
 }
 
 static int chv_crtc_compute_clock(struct intel_atomic_state *state,
@@ -1765,7 +1824,7 @@ void i9xx_enable_pll(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	u32 dpll = crtc_state->dpll_hw_state.dpll;
+	const struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
 	enum pipe pipe = crtc->pipe;
 	int i;
 
@@ -1775,157 +1834,152 @@ void i9xx_enable_pll(const struct intel_crtc_state *crtc_state)
 	if (i9xx_has_pps(dev_priv))
 		assert_pps_unlocked(dev_priv, pipe);
 
-	intel_de_write(dev_priv, FP0(pipe), crtc_state->dpll_hw_state.fp0);
-	intel_de_write(dev_priv, FP1(pipe), crtc_state->dpll_hw_state.fp1);
+	intel_de_write(dev_priv, FP0(pipe), hw_state->fp0);
+	intel_de_write(dev_priv, FP1(pipe), hw_state->fp1);
 
 	/*
 	 * Apparently we need to have VGA mode enabled prior to changing
 	 * the P1/P2 dividers. Otherwise the DPLL will keep using the old
 	 * dividers, even though the register value does change.
 	 */
-	intel_de_write(dev_priv, DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
-	intel_de_write(dev_priv, DPLL(pipe), dpll);
+	intel_de_write(dev_priv, DPLL(pipe), hw_state->dpll & ~DPLL_VGA_MODE_DIS);
+	intel_de_write(dev_priv, DPLL(pipe), hw_state->dpll);
 
 	/* Wait for the clocks to stabilize. */
 	intel_de_posting_read(dev_priv, DPLL(pipe));
 	udelay(150);
 
 	if (DISPLAY_VER(dev_priv) >= 4) {
-		intel_de_write(dev_priv, DPLL_MD(pipe),
-			       crtc_state->dpll_hw_state.dpll_md);
+		intel_de_write(dev_priv, DPLL_MD(pipe), hw_state->dpll_md);
 	} else {
 		/* The pixel multiplier can only be updated once the
 		 * DPLL is enabled and the clocks are stable.
 		 *
 		 * So write it again.
 		 */
-		intel_de_write(dev_priv, DPLL(pipe), dpll);
+		intel_de_write(dev_priv, DPLL(pipe), hw_state->dpll);
 	}
 
 	/* We do this three times for luck */
 	for (i = 0; i < 3; i++) {
-		intel_de_write(dev_priv, DPLL(pipe), dpll);
+		intel_de_write(dev_priv, DPLL(pipe), hw_state->dpll);
 		intel_de_posting_read(dev_priv, DPLL(pipe));
 		udelay(150); /* wait for warmup */
 	}
 }
 
 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv,
-				 enum dpio_phy phy)
+				 enum dpio_phy phy, enum dpio_channel ch)
 {
-	u32 reg_val;
+	u32 tmp;
 
 	/*
 	 * PLLB opamp always calibrates to max value of 0x3f, force enable it
 	 * and set it to a reasonable value instead.
 	 */
-	reg_val = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW9(1));
-	reg_val &= 0xffffff00;
-	reg_val |= 0x00000030;
-	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW9(1), reg_val);
-
-	reg_val = vlv_dpio_read(dev_priv, phy, VLV_REF_DW13);
-	reg_val &= 0x00ffffff;
-	reg_val |= 0x8c000000;
-	vlv_dpio_write(dev_priv, phy, VLV_REF_DW13, reg_val);
-
-	reg_val = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW9(1));
-	reg_val &= 0xffffff00;
-	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW9(1), reg_val);
-
-	reg_val = vlv_dpio_read(dev_priv, phy, VLV_REF_DW13);
-	reg_val &= 0x00ffffff;
-	reg_val |= 0xb0000000;
-	vlv_dpio_write(dev_priv, phy, VLV_REF_DW13, reg_val);
+	tmp = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW17(ch));
+	tmp &= 0xffffff00;
+	tmp |= 0x00000030;
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW17(ch), tmp);
+
+	tmp = vlv_dpio_read(dev_priv, phy, VLV_REF_DW11);
+	tmp &= 0x00ffffff;
+	tmp |= 0x8c000000;
+	vlv_dpio_write(dev_priv, phy, VLV_REF_DW11, tmp);
+
+	tmp = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW17(ch));
+	tmp &= 0xffffff00;
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW17(ch), tmp);
+
+	tmp = vlv_dpio_read(dev_priv, phy, VLV_REF_DW11);
+	tmp &= 0x00ffffff;
+	tmp |= 0xb0000000;
+	vlv_dpio_write(dev_priv, phy, VLV_REF_DW11, tmp);
 }
 
 static void vlv_prepare_pll(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	const struct dpll *clock = &crtc_state->dpll;
+	enum dpio_channel ch = vlv_pipe_to_channel(crtc->pipe);
 	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
 	enum pipe pipe = crtc->pipe;
-	u32 mdiv;
-	u32 bestn, bestm1, bestm2, bestp1, bestp2;
-	u32 coreclk, reg_val;
+	u32 tmp, coreclk;
 
 	vlv_dpio_get(dev_priv);
 
-	bestn = crtc_state->dpll.n;
-	bestm1 = crtc_state->dpll.m1;
-	bestm2 = crtc_state->dpll.m2;
-	bestp1 = crtc_state->dpll.p1;
-	bestp2 = crtc_state->dpll.p2;
-
 	/* See eDP HDMI DPIO driver vbios notes doc */
 
 	/* PLL B needs special handling */
 	if (pipe == PIPE_B)
-		vlv_pllb_recal_opamp(dev_priv, phy);
+		vlv_pllb_recal_opamp(dev_priv, phy, ch);
 
 	/* Set up Tx target for periodic Rcomp update */
-	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW9_BCAST, 0x0100000f);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW17_BCAST, 0x0100000f);
 
 	/* Disable target IRef on PLL */
-	reg_val = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW8(pipe));
-	reg_val &= 0x00ffffff;
-	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW8(pipe), reg_val);
+	tmp = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW16(ch));
+	tmp &= 0x00ffffff;
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW16(ch), tmp);
 
 	/* Disable fast lock */
 	vlv_dpio_write(dev_priv, phy, VLV_CMN_DW0, 0x610);
 
 	/* Set idtafcrecal before PLL is enabled */
-	mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
-	mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
-	mdiv |= ((bestn << DPIO_N_SHIFT));
-	mdiv |= (1 << DPIO_K_SHIFT);
+	tmp = DPIO_M1_DIV(clock->m1) |
+		DPIO_M2_DIV(clock->m2) |
+		DPIO_P1_DIV(clock->p1) |
+		DPIO_P2_DIV(clock->p2) |
+		DPIO_N_DIV(clock->n) |
+		DPIO_K_DIV(1);
 
 	/*
 	 * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
 	 * but we don't support that).
 	 * Note: don't use the DAC post divider as it seems unstable.
 	 */
-	mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
-	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW3(pipe), mdiv);
+	tmp |= DPIO_S1_DIV(DPIO_S1_DIV_HDMIDP);
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW3(ch), tmp);
 
-	mdiv |= DPIO_ENABLE_CALIBRATION;
-	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW3(pipe), mdiv);
+	tmp |= DPIO_ENABLE_CALIBRATION;
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW3(ch), tmp);
 
 	/* Set HBR and RBR LPF coefficients */
 	if (crtc_state->port_clock == 162000 ||
 	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG) ||
 	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
-		vlv_dpio_write(dev_priv, phy, VLV_PLL_DW10(pipe),
+		vlv_dpio_write(dev_priv, phy, VLV_PLL_DW18(ch),
 				 0x009f0003);
 	else
-		vlv_dpio_write(dev_priv, phy, VLV_PLL_DW10(pipe),
+		vlv_dpio_write(dev_priv, phy, VLV_PLL_DW18(ch),
 				 0x00d0000f);
 
 	if (intel_crtc_has_dp_encoder(crtc_state)) {
 		/* Use SSC source */
 		if (pipe == PIPE_A)
-			vlv_dpio_write(dev_priv, phy, VLV_PLL_DW5(pipe),
+			vlv_dpio_write(dev_priv, phy, VLV_PLL_DW5(ch),
 					 0x0df40000);
 		else
-			vlv_dpio_write(dev_priv, phy, VLV_PLL_DW5(pipe),
+			vlv_dpio_write(dev_priv, phy, VLV_PLL_DW5(ch),
 					 0x0df70000);
 	} else { /* HDMI or VGA */
 		/* Use bend source */
 		if (pipe == PIPE_A)
-			vlv_dpio_write(dev_priv, phy, VLV_PLL_DW5(pipe),
+			vlv_dpio_write(dev_priv, phy, VLV_PLL_DW5(ch),
 					 0x0df70000);
 		else
-			vlv_dpio_write(dev_priv, phy, VLV_PLL_DW5(pipe),
+			vlv_dpio_write(dev_priv, phy, VLV_PLL_DW5(ch),
 					 0x0df40000);
 	}
 
-	coreclk = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW7(pipe));
+	coreclk = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW7(ch));
 	coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
 	if (intel_crtc_has_dp_encoder(crtc_state))
 		coreclk |= 0x01000000;
-	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW7(pipe), coreclk);
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW7(ch), coreclk);
 
-	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW11(pipe), 0x87871000);
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW19(ch), 0x87871000);
 
 	vlv_dpio_put(dev_priv);
 }
@@ -1934,9 +1988,10 @@ static void _vlv_enable_pll(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	const struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
 	enum pipe pipe = crtc->pipe;
 
-	intel_de_write(dev_priv, DPLL(pipe), crtc_state->dpll_hw_state.dpll);
+	intel_de_write(dev_priv, DPLL(pipe), hw_state->dpll);
 	intel_de_posting_read(dev_priv, DPLL(pipe));
 	udelay(150);
 
@@ -1948,6 +2003,7 @@ void vlv_enable_pll(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	const struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
 	enum pipe pipe = crtc->pipe;
 
 	assert_transcoder_disabled(dev_priv, crtc_state->cpu_transcoder);
@@ -1957,16 +2013,14 @@ void vlv_enable_pll(const struct intel_crtc_state *crtc_state)
 
 	/* Enable Refclk */
 	intel_de_write(dev_priv, DPLL(pipe),
-		       crtc_state->dpll_hw_state.dpll &
-		       ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
+		       hw_state->dpll & ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
 
-	if (crtc_state->dpll_hw_state.dpll & DPLL_VCO_ENABLE) {
+	if (hw_state->dpll & DPLL_VCO_ENABLE) {
 		vlv_prepare_pll(crtc_state);
 		_vlv_enable_pll(crtc_state);
 	}
 
-	intel_de_write(dev_priv, DPLL_MD(pipe),
-		       crtc_state->dpll_hw_state.dpll_md);
+	intel_de_write(dev_priv, DPLL_MD(pipe), hw_state->dpll_md);
 	intel_de_posting_read(dev_priv, DPLL_MD(pipe));
 }
 
@@ -1974,93 +2028,87 @@ static void chv_prepare_pll(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum pipe pipe = crtc->pipe;
-	enum dpio_channel port = vlv_pipe_to_channel(pipe);
+	const struct dpll *clock = &crtc_state->dpll;
+	enum dpio_channel ch = vlv_pipe_to_channel(crtc->pipe);
 	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
-	u32 loopfilter, tribuf_calcntr;
-	u32 bestm2, bestp1, bestp2, bestm2_frac;
-	u32 dpio_val;
-	int vco;
-
-	bestm2_frac = crtc_state->dpll.m2 & 0x3fffff;
-	bestm2 = crtc_state->dpll.m2 >> 22;
-	bestp1 = crtc_state->dpll.p1;
-	bestp2 = crtc_state->dpll.p2;
-	vco = crtc_state->dpll.vco;
-	dpio_val = 0;
-	loopfilter = 0;
+	u32 tmp, loopfilter, tribuf_calcntr;
+	u32 m2_frac;
+
+	m2_frac = clock->m2 & 0x3fffff;
 
 	vlv_dpio_get(dev_priv);
 
 	/* p1 and p2 divider */
-	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW13(port),
-			5 << DPIO_CHV_S1_DIV_SHIFT |
-			bestp1 << DPIO_CHV_P1_DIV_SHIFT |
-			bestp2 << DPIO_CHV_P2_DIV_SHIFT |
-			1 << DPIO_CHV_K_DIV_SHIFT);
+	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW13(ch),
+		       DPIO_CHV_S1_DIV(5) |
+		       DPIO_CHV_P1_DIV(clock->p1) |
+		       DPIO_CHV_P2_DIV(clock->p2) |
+		       DPIO_CHV_K_DIV(1));
 
 	/* Feedback post-divider - m2 */
-	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW0(port), bestm2);
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW0(ch),
+		       DPIO_CHV_M2_DIV(clock->m2 >> 22));
 
 	/* Feedback refclk divider - n and m1 */
-	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW1(port),
-			DPIO_CHV_M1_DIV_BY_2 |
-			1 << DPIO_CHV_N_DIV_SHIFT);
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW1(ch),
+		       DPIO_CHV_M1_DIV(DPIO_CHV_M1_DIV_BY_2) |
+		       DPIO_CHV_N_DIV(1));
 
 	/* M2 fraction division */
-	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW2(port), bestm2_frac);
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW2(ch),
+		       DPIO_CHV_M2_FRAC_DIV(m2_frac));
 
 	/* M2 fraction division enable */
-	dpio_val = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW3(port));
-	dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
-	dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
-	if (bestm2_frac)
-		dpio_val |= DPIO_CHV_FRAC_DIV_EN;
-	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW3(port), dpio_val);
+	tmp = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW3(ch));
+	tmp &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
+	tmp |= DPIO_CHV_FEEDFWD_GAIN(2);
+	if (m2_frac)
+		tmp |= DPIO_CHV_FRAC_DIV_EN;
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW3(ch), tmp);
 
 	/* Program digital lock detect threshold */
-	dpio_val = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW9(port));
-	dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
-					DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
-	dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
-	if (!bestm2_frac)
-		dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
-	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW9(port), dpio_val);
+	tmp = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW9(ch));
+	tmp &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
+		      DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
+	tmp |= DPIO_CHV_INT_LOCK_THRESHOLD(0x5);
+	if (!m2_frac)
+		tmp |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW9(ch), tmp);
 
 	/* Loop filter */
-	if (vco == 5400000) {
-		loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
-		loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
-		loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
+	if (clock->vco == 5400000) {
+		loopfilter = DPIO_CHV_PROP_COEFF(0x3) |
+			DPIO_CHV_INT_COEFF(0x8) |
+			DPIO_CHV_GAIN_CTRL(0x1);
 		tribuf_calcntr = 0x9;
-	} else if (vco <= 6200000) {
-		loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
-		loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
-		loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
+	} else if (clock->vco <= 6200000) {
+		loopfilter = DPIO_CHV_PROP_COEFF(0x5) |
+			DPIO_CHV_INT_COEFF(0xB) |
+			DPIO_CHV_GAIN_CTRL(0x3);
 		tribuf_calcntr = 0x9;
-	} else if (vco <= 6480000) {
-		loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
-		loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
-		loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
+	} else if (clock->vco <= 6480000) {
+		loopfilter = DPIO_CHV_PROP_COEFF(0x4) |
+			DPIO_CHV_INT_COEFF(0x9) |
+			DPIO_CHV_GAIN_CTRL(0x3);
 		tribuf_calcntr = 0x8;
 	} else {
 		/* Not supported. Apply the same limits as in the max case */
-		loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
-		loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
-		loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
+		loopfilter = DPIO_CHV_PROP_COEFF(0x4) |
+			DPIO_CHV_INT_COEFF(0x9) |
+			DPIO_CHV_GAIN_CTRL(0x3);
 		tribuf_calcntr = 0;
 	}
-	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW6(port), loopfilter);
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW6(ch), loopfilter);
 
-	dpio_val = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW8(port));
-	dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
-	dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
-	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW8(port), dpio_val);
+	tmp = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW8(ch));
+	tmp &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
+	tmp |= DPIO_CHV_TDC_TARGET_CNT(tribuf_calcntr);
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW8(ch), tmp);
 
 	/* AFC Recal */
-	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW14(port),
-			vlv_dpio_read(dev_priv, phy, CHV_CMN_DW14(port)) |
-			DPIO_AFC_RECAL);
+	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW14(ch),
+		       vlv_dpio_read(dev_priv, phy, CHV_CMN_DW14(ch)) |
+		       DPIO_AFC_RECAL);
 
 	vlv_dpio_put(dev_priv);
 }
@@ -2069,17 +2117,18 @@ static void _chv_enable_pll(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum pipe pipe = crtc->pipe;
-	enum dpio_channel port = vlv_pipe_to_channel(pipe);
+	const struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
+	enum dpio_channel ch = vlv_pipe_to_channel(crtc->pipe);
 	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
+	enum pipe pipe = crtc->pipe;
 	u32 tmp;
 
 	vlv_dpio_get(dev_priv);
 
 	/* Enable back the 10bit clock to display controller */
-	tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW14(port));
+	tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW14(ch));
 	tmp |= DPIO_DCLKP_EN;
-	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW14(port), tmp);
+	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW14(ch), tmp);
 
 	vlv_dpio_put(dev_priv);
 
@@ -2089,7 +2138,7 @@ static void _chv_enable_pll(const struct intel_crtc_state *crtc_state)
 	udelay(1);
 
 	/* Enable PLL */
-	intel_de_write(dev_priv, DPLL(pipe), crtc_state->dpll_hw_state.dpll);
+	intel_de_write(dev_priv, DPLL(pipe), hw_state->dpll);
 
 	/* Check PLL is locked */
 	if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
@@ -2100,6 +2149,7 @@ void chv_enable_pll(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	const struct i9xx_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.i9xx;
 	enum pipe pipe = crtc->pipe;
 
 	assert_transcoder_disabled(dev_priv, crtc_state->cpu_transcoder);
@@ -2109,9 +2159,9 @@ void chv_enable_pll(const struct intel_crtc_state *crtc_state)
 
 	/* Enable Refclk and SSC */
 	intel_de_write(dev_priv, DPLL(pipe),
-		       crtc_state->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
+		       hw_state->dpll & ~DPLL_VCO_ENABLE);
 
-	if (crtc_state->dpll_hw_state.dpll & DPLL_VCO_ENABLE) {
+	if (hw_state->dpll & DPLL_VCO_ENABLE) {
 		chv_prepare_pll(crtc_state);
 		_chv_enable_pll(crtc_state);
 	}
@@ -2124,10 +2174,9 @@ void chv_enable_pll(const struct intel_crtc_state *crtc_state)
 		 * the value from DPLLBMD to either pipe B or C.
 		 */
 		intel_de_write(dev_priv, CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
-		intel_de_write(dev_priv, DPLL_MD(PIPE_B),
-			       crtc_state->dpll_hw_state.dpll_md);
+		intel_de_write(dev_priv, DPLL_MD(PIPE_B), hw_state->dpll_md);
 		intel_de_write(dev_priv, CBR4_VLV, 0);
-		dev_priv->display.state.chv_dpll_md[pipe] = crtc_state->dpll_hw_state.dpll_md;
+		dev_priv->display.state.chv_dpll_md[pipe] = hw_state->dpll_md;
 
 		/*
 		 * DPLLB VGA mode also seems to cause problems.
@@ -2137,8 +2186,7 @@ void chv_enable_pll(const struct intel_crtc_state *crtc_state)
 			    (intel_de_read(dev_priv, DPLL(PIPE_B)) &
 			     DPLL_VGA_MODE_DIS) == 0);
 	} else {
-		intel_de_write(dev_priv, DPLL_MD(pipe),
-			       crtc_state->dpll_hw_state.dpll_md);
+		intel_de_write(dev_priv, DPLL_MD(pipe), hw_state->dpll_md);
 		intel_de_posting_read(dev_priv, DPLL_MD(pipe));
 	}
 }
@@ -2199,7 +2247,7 @@ void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
 
 void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
-	enum dpio_channel port = vlv_pipe_to_channel(pipe);
+	enum dpio_channel ch = vlv_pipe_to_channel(pipe);
 	enum dpio_phy phy = vlv_pipe_to_phy(pipe);
 	u32 val;
 
@@ -2217,9 +2265,9 @@ void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
 	vlv_dpio_get(dev_priv);
 
 	/* Disable 10bit clock to display controller */
-	val = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW14(port));
+	val = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW14(ch));
 	val &= ~DPIO_DCLKP_EN;
-	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW14(port), val);
+	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW14(ch), val);
 
 	vlv_dpio_put(dev_priv);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dpll.h b/drivers/gpu/drm/i915/display/intel_dpll.h
index ac01bb19cc6c..a86a79408af0 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll.h
+++ b/drivers/gpu/drm/i915/display/intel_dpll.h
@@ -13,6 +13,7 @@ struct drm_i915_private;
 struct intel_atomic_state;
 struct intel_crtc;
 struct intel_crtc_state;
+struct intel_dpll_hw_state;
 enum pipe;
 
 void intel_dpll_init_clock_hook(struct drm_i915_private *dev_priv);
@@ -22,6 +23,8 @@ int intel_dpll_crtc_get_shared_dpll(struct intel_atomic_state *state,
 				    struct intel_crtc *crtc);
 int i9xx_calc_dpll_params(int refclk, struct dpll *clock);
 u32 i9xx_dpll_compute_fp(const struct dpll *dpll);
+void i9xx_dpll_get_hw_state(struct intel_crtc *crtc,
+			    struct intel_dpll_hw_state *dpll_hw_state);
 void vlv_compute_dpll(struct intel_crtc_state *crtc_state);
 void chv_compute_dpll(struct intel_crtc_state *crtc_state);
 
@@ -39,12 +42,9 @@ bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
 			struct dpll *best_clock);
 int chv_calc_dpll_params(int refclk, struct dpll *pll_clock);
 
-void i9xx_crtc_clock_get(struct intel_crtc *crtc,
-			 struct intel_crtc_state *pipe_config);
-void vlv_crtc_clock_get(struct intel_crtc *crtc,
-			struct intel_crtc_state *pipe_config);
-void chv_crtc_clock_get(struct intel_crtc *crtc,
-			struct intel_crtc_state *pipe_config);
+void i9xx_crtc_clock_get(struct intel_crtc_state *crtc_state);
+void vlv_crtc_clock_get(struct intel_crtc_state *crtc_state);
+void chv_crtc_clock_get(struct intel_crtc_state *crtc_state);
 
 void assert_pll_enabled(struct drm_i915_private *i915, enum pipe pipe);
 void assert_pll_disabled(struct drm_i915_private *i915, enum pipe pipe);
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
index b6d24410740f..90998b037349 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
@@ -24,6 +24,7 @@
 #include <linux/math.h>
 #include <linux/string_helpers.h>
 
+#include "bxt_dpio_phy_regs.h"
 #include "i915_reg.h"
 #include "intel_de.h"
 #include "intel_display_types.h"
@@ -64,7 +65,8 @@ struct intel_shared_dpll_funcs {
 	 * the pll is not already enabled.
 	 */
 	void (*enable)(struct drm_i915_private *i915,
-		       struct intel_shared_dpll *pll);
+		       struct intel_shared_dpll *pll,
+		       const struct intel_dpll_hw_state *dpll_hw_state);
 
 	/*
 	 * Hook for disabling the pll, called from intel_disable_shared_dpll()
@@ -81,7 +83,7 @@ struct intel_shared_dpll_funcs {
 	 */
 	bool (*get_hw_state)(struct drm_i915_private *i915,
 			     struct intel_shared_dpll *pll,
-			     struct intel_dpll_hw_state *hw_state);
+			     struct intel_dpll_hw_state *dpll_hw_state);
 
 	/*
 	 * Hook for calculating the pll's output frequency based on its passed
@@ -89,7 +91,7 @@ struct intel_shared_dpll_funcs {
 	 */
 	int (*get_freq)(struct drm_i915_private *i915,
 			const struct intel_shared_dpll *pll,
-			const struct intel_dpll_hw_state *pll_state);
+			const struct intel_dpll_hw_state *dpll_hw_state);
 };
 
 struct intel_dpll_mgr {
@@ -107,8 +109,8 @@ struct intel_dpll_mgr {
 				   struct intel_crtc *crtc,
 				   struct intel_encoder *encoder);
 	void (*update_ref_clks)(struct drm_i915_private *i915);
-	void (*dump_hw_state)(struct drm_i915_private *i915,
-			      const struct intel_dpll_hw_state *hw_state);
+	void (*dump_hw_state)(struct drm_printer *p,
+			      const struct intel_dpll_hw_state *dpll_hw_state);
 	bool (*compare_hw_state)(const struct intel_dpll_hw_state *a,
 				 const struct intel_dpll_hw_state *b);
 };
@@ -227,7 +229,7 @@ static void _intel_enable_shared_dpll(struct drm_i915_private *i915,
 	if (pll->info->power_domain)
 		pll->wakeref = intel_display_power_get(i915, pll->info->power_domain);
 
-	pll->info->funcs->enable(i915, pll);
+	pll->info->funcs->enable(i915, pll, &pll->state.hw_state);
 	pll->on = true;
 }
 
@@ -352,7 +354,7 @@ intel_dpll_mask_all(struct drm_i915_private *i915)
 static struct intel_shared_dpll *
 intel_find_shared_dpll(struct intel_atomic_state *state,
 		       const struct intel_crtc *crtc,
-		       const struct intel_dpll_hw_state *pll_state,
+		       const struct intel_dpll_hw_state *dpll_hw_state,
 		       unsigned long dpll_mask)
 {
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
@@ -379,9 +381,9 @@ intel_find_shared_dpll(struct intel_atomic_state *state,
 			continue;
 		}
 
-		if (memcmp(pll_state,
+		if (memcmp(dpll_hw_state,
 			   &shared_dpll[pll->index].hw_state,
-			   sizeof(*pll_state)) == 0) {
+			   sizeof(*dpll_hw_state)) == 0) {
 			drm_dbg_kms(&i915->drm,
 				    "[CRTC:%d:%s] sharing existing %s (pipe mask 0x%x, active 0x%x)\n",
 				    crtc->base.base.id, crtc->base.name,
@@ -430,14 +432,14 @@ static void
 intel_reference_shared_dpll(struct intel_atomic_state *state,
 			    const struct intel_crtc *crtc,
 			    const struct intel_shared_dpll *pll,
-			    const struct intel_dpll_hw_state *pll_state)
+			    const struct intel_dpll_hw_state *dpll_hw_state)
 {
 	struct intel_shared_dpll_state *shared_dpll;
 
 	shared_dpll = intel_atomic_get_shared_dpll_state(&state->base);
 
 	if (shared_dpll[pll->index].pipe_mask == 0)
-		shared_dpll[pll->index].hw_state = *pll_state;
+		shared_dpll[pll->index].hw_state = *dpll_hw_state;
 
 	intel_reference_shared_dpll_crtc(crtc, pll, &shared_dpll[pll->index]);
 }
@@ -519,8 +521,9 @@ void intel_shared_dpll_swap_state(struct intel_atomic_state *state)
 
 static bool ibx_pch_dpll_get_hw_state(struct drm_i915_private *i915,
 				      struct intel_shared_dpll *pll,
-				      struct intel_dpll_hw_state *hw_state)
+				      struct intel_dpll_hw_state *dpll_hw_state)
 {
+	struct i9xx_dpll_hw_state *hw_state = &dpll_hw_state->i9xx;
 	const enum intel_dpll_id id = pll->info->id;
 	intel_wakeref_t wakeref;
 	u32 val;
@@ -553,17 +556,19 @@ static void ibx_assert_pch_refclk_enabled(struct drm_i915_private *i915)
 }
 
 static void ibx_pch_dpll_enable(struct drm_i915_private *i915,
-				struct intel_shared_dpll *pll)
+				struct intel_shared_dpll *pll,
+				const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct i9xx_dpll_hw_state *hw_state = &dpll_hw_state->i9xx;
 	const enum intel_dpll_id id = pll->info->id;
 
 	/* PCH refclock must be enabled first */
 	ibx_assert_pch_refclk_enabled(i915);
 
-	intel_de_write(i915, PCH_FP0(id), pll->state.hw_state.fp0);
-	intel_de_write(i915, PCH_FP1(id), pll->state.hw_state.fp1);
+	intel_de_write(i915, PCH_FP0(id), hw_state->fp0);
+	intel_de_write(i915, PCH_FP1(id), hw_state->fp1);
 
-	intel_de_write(i915, PCH_DPLL(id), pll->state.hw_state.dpll);
+	intel_de_write(i915, PCH_DPLL(id), hw_state->dpll);
 
 	/* Wait for the clocks to stabilize. */
 	intel_de_posting_read(i915, PCH_DPLL(id));
@@ -574,7 +579,7 @@ static void ibx_pch_dpll_enable(struct drm_i915_private *i915,
 	 *
 	 * So write it again.
 	 */
-	intel_de_write(i915, PCH_DPLL(id), pll->state.hw_state.dpll);
+	intel_de_write(i915, PCH_DPLL(id), hw_state->dpll);
 	intel_de_posting_read(i915, PCH_DPLL(id));
 	udelay(200);
 }
@@ -634,21 +639,25 @@ static int ibx_get_dpll(struct intel_atomic_state *state,
 	return 0;
 }
 
-static void ibx_dump_hw_state(struct drm_i915_private *i915,
-			      const struct intel_dpll_hw_state *hw_state)
+static void ibx_dump_hw_state(struct drm_printer *p,
+			      const struct intel_dpll_hw_state *dpll_hw_state)
 {
-	drm_dbg_kms(&i915->drm,
-		    "dpll_hw_state: dpll: 0x%x, dpll_md: 0x%x, "
-		    "fp0: 0x%x, fp1: 0x%x\n",
-		    hw_state->dpll,
-		    hw_state->dpll_md,
-		    hw_state->fp0,
-		    hw_state->fp1);
+	const struct i9xx_dpll_hw_state *hw_state = &dpll_hw_state->i9xx;
+
+	drm_printf(p, "dpll_hw_state: dpll: 0x%x, dpll_md: 0x%x, "
+		   "fp0: 0x%x, fp1: 0x%x\n",
+		   hw_state->dpll,
+		   hw_state->dpll_md,
+		   hw_state->fp0,
+		   hw_state->fp1);
 }
 
-static bool ibx_compare_hw_state(const struct intel_dpll_hw_state *a,
-				 const struct intel_dpll_hw_state *b)
+static bool ibx_compare_hw_state(const struct intel_dpll_hw_state *_a,
+				 const struct intel_dpll_hw_state *_b)
 {
+	const struct i9xx_dpll_hw_state *a = &_a->i9xx;
+	const struct i9xx_dpll_hw_state *b = &_b->i9xx;
+
 	return a->dpll == b->dpll &&
 		a->dpll_md == b->dpll_md &&
 		a->fp0 == b->fp0 &&
@@ -677,19 +686,24 @@ static const struct intel_dpll_mgr pch_pll_mgr = {
 };
 
 static void hsw_ddi_wrpll_enable(struct drm_i915_private *i915,
-				 struct intel_shared_dpll *pll)
+				 struct intel_shared_dpll *pll,
+				 const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct hsw_dpll_hw_state *hw_state = &dpll_hw_state->hsw;
 	const enum intel_dpll_id id = pll->info->id;
 
-	intel_de_write(i915, WRPLL_CTL(id), pll->state.hw_state.wrpll);
+	intel_de_write(i915, WRPLL_CTL(id), hw_state->wrpll);
 	intel_de_posting_read(i915, WRPLL_CTL(id));
 	udelay(20);
 }
 
 static void hsw_ddi_spll_enable(struct drm_i915_private *i915,
-				struct intel_shared_dpll *pll)
+				struct intel_shared_dpll *pll,
+				const struct intel_dpll_hw_state *dpll_hw_state)
 {
-	intel_de_write(i915, SPLL_CTL, pll->state.hw_state.spll);
+	const struct hsw_dpll_hw_state *hw_state = &dpll_hw_state->hsw;
+
+	intel_de_write(i915, SPLL_CTL, hw_state->spll);
 	intel_de_posting_read(i915, SPLL_CTL);
 	udelay(20);
 }
@@ -728,8 +742,9 @@ static void hsw_ddi_spll_disable(struct drm_i915_private *i915,
 
 static bool hsw_ddi_wrpll_get_hw_state(struct drm_i915_private *i915,
 				       struct intel_shared_dpll *pll,
-				       struct intel_dpll_hw_state *hw_state)
+				       struct intel_dpll_hw_state *dpll_hw_state)
 {
+	struct hsw_dpll_hw_state *hw_state = &dpll_hw_state->hsw;
 	const enum intel_dpll_id id = pll->info->id;
 	intel_wakeref_t wakeref;
 	u32 val;
@@ -749,8 +764,9 @@ static bool hsw_ddi_wrpll_get_hw_state(struct drm_i915_private *i915,
 
 static bool hsw_ddi_spll_get_hw_state(struct drm_i915_private *i915,
 				      struct intel_shared_dpll *pll,
-				      struct intel_dpll_hw_state *hw_state)
+				      struct intel_dpll_hw_state *dpll_hw_state)
 {
+	struct hsw_dpll_hw_state *hw_state = &dpll_hw_state->hsw;
 	intel_wakeref_t wakeref;
 	u32 val;
 
@@ -975,11 +991,12 @@ hsw_ddi_calculate_wrpll(int clock /* in Hz */,
 
 static int hsw_ddi_wrpll_get_freq(struct drm_i915_private *i915,
 				  const struct intel_shared_dpll *pll,
-				  const struct intel_dpll_hw_state *pll_state)
+				  const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct hsw_dpll_hw_state *hw_state = &dpll_hw_state->hsw;
 	int refclk;
 	int n, p, r;
-	u32 wrpll = pll_state->wrpll;
+	u32 wrpll = hw_state->wrpll;
 
 	switch (wrpll & WRPLL_REF_MASK) {
 	case WRPLL_REF_SPECIAL_HSW:
@@ -1020,11 +1037,12 @@ hsw_ddi_wrpll_compute_dpll(struct intel_atomic_state *state,
 	struct drm_i915_private *i915 = to_i915(state->base.dev);
 	struct intel_crtc_state *crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
+	struct hsw_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.hsw;
 	unsigned int p, n2, r2;
 
 	hsw_ddi_calculate_wrpll(crtc_state->port_clock * 1000, &r2, &n2, &p);
 
-	crtc_state->dpll_hw_state.wrpll =
+	hw_state->wrpll =
 		WRPLL_PLL_ENABLE | WRPLL_REF_LCPLL |
 		WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) |
 		WRPLL_DIVIDER_POST(p);
@@ -1099,7 +1117,7 @@ hsw_ddi_lcpll_get_dpll(struct intel_crtc_state *crtc_state)
 
 static int hsw_ddi_lcpll_get_freq(struct drm_i915_private *i915,
 				  const struct intel_shared_dpll *pll,
-				  const struct intel_dpll_hw_state *pll_state)
+				  const struct intel_dpll_hw_state *dpll_hw_state)
 {
 	int link_clock = 0;
 
@@ -1127,11 +1145,12 @@ hsw_ddi_spll_compute_dpll(struct intel_atomic_state *state,
 {
 	struct intel_crtc_state *crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
+	struct hsw_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.hsw;
 
 	if (drm_WARN_ON(crtc->base.dev, crtc_state->port_clock / 2 != 135000))
 		return -EINVAL;
 
-	crtc_state->dpll_hw_state.spll =
+	hw_state->spll =
 		SPLL_PLL_ENABLE | SPLL_FREQ_1350MHz | SPLL_REF_MUXED_SSC;
 
 	return 0;
@@ -1150,11 +1169,12 @@ hsw_ddi_spll_get_dpll(struct intel_atomic_state *state,
 
 static int hsw_ddi_spll_get_freq(struct drm_i915_private *i915,
 				 const struct intel_shared_dpll *pll,
-				 const struct intel_dpll_hw_state *pll_state)
+				 const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct hsw_dpll_hw_state *hw_state = &dpll_hw_state->hsw;
 	int link_clock = 0;
 
-	switch (pll_state->spll & SPLL_FREQ_MASK) {
+	switch (hw_state->spll & SPLL_FREQ_MASK) {
 	case SPLL_FREQ_810MHz:
 		link_clock = 81000;
 		break;
@@ -1225,16 +1245,21 @@ static void hsw_update_dpll_ref_clks(struct drm_i915_private *i915)
 		i915->display.dpll.ref_clks.nssc = 135000;
 }
 
-static void hsw_dump_hw_state(struct drm_i915_private *i915,
-			      const struct intel_dpll_hw_state *hw_state)
+static void hsw_dump_hw_state(struct drm_printer *p,
+			      const struct intel_dpll_hw_state *dpll_hw_state)
 {
-	drm_dbg_kms(&i915->drm, "dpll_hw_state: wrpll: 0x%x spll: 0x%x\n",
-		    hw_state->wrpll, hw_state->spll);
+	const struct hsw_dpll_hw_state *hw_state = &dpll_hw_state->hsw;
+
+	drm_printf(p, "dpll_hw_state: wrpll: 0x%x spll: 0x%x\n",
+		   hw_state->wrpll, hw_state->spll);
 }
 
-static bool hsw_compare_hw_state(const struct intel_dpll_hw_state *a,
-				 const struct intel_dpll_hw_state *b)
+static bool hsw_compare_hw_state(const struct intel_dpll_hw_state *_a,
+				 const struct intel_dpll_hw_state *_b)
 {
+	const struct hsw_dpll_hw_state *a = &_a->hsw;
+	const struct hsw_dpll_hw_state *b = &_b->hsw;
+
 	return a->wrpll == b->wrpll &&
 		a->spll == b->spll;
 }
@@ -1254,7 +1279,8 @@ static const struct intel_shared_dpll_funcs hsw_ddi_spll_funcs = {
 };
 
 static void hsw_ddi_lcpll_enable(struct drm_i915_private *i915,
-				 struct intel_shared_dpll *pll)
+				 struct intel_shared_dpll *pll,
+				 const struct intel_dpll_hw_state *hw_state)
 {
 }
 
@@ -1265,7 +1291,7 @@ static void hsw_ddi_lcpll_disable(struct drm_i915_private *i915,
 
 static bool hsw_ddi_lcpll_get_hw_state(struct drm_i915_private *i915,
 				       struct intel_shared_dpll *pll,
-				       struct intel_dpll_hw_state *hw_state)
+				       struct intel_dpll_hw_state *dpll_hw_state)
 {
 	return true;
 }
@@ -1332,26 +1358,31 @@ static const struct skl_dpll_regs skl_dpll_regs[4] = {
 };
 
 static void skl_ddi_pll_write_ctrl1(struct drm_i915_private *i915,
-				    struct intel_shared_dpll *pll)
+				    struct intel_shared_dpll *pll,
+				    const struct skl_dpll_hw_state *hw_state)
 {
 	const enum intel_dpll_id id = pll->info->id;
 
 	intel_de_rmw(i915, DPLL_CTRL1,
-		     DPLL_CTRL1_HDMI_MODE(id) | DPLL_CTRL1_SSC(id) | DPLL_CTRL1_LINK_RATE_MASK(id),
-		     pll->state.hw_state.ctrl1 << (id * 6));
+		     DPLL_CTRL1_HDMI_MODE(id) |
+		     DPLL_CTRL1_SSC(id) |
+		     DPLL_CTRL1_LINK_RATE_MASK(id),
+		     hw_state->ctrl1 << (id * 6));
 	intel_de_posting_read(i915, DPLL_CTRL1);
 }
 
 static void skl_ddi_pll_enable(struct drm_i915_private *i915,
-			       struct intel_shared_dpll *pll)
+			       struct intel_shared_dpll *pll,
+			       const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct skl_dpll_hw_state *hw_state = &dpll_hw_state->skl;
 	const struct skl_dpll_regs *regs = skl_dpll_regs;
 	const enum intel_dpll_id id = pll->info->id;
 
-	skl_ddi_pll_write_ctrl1(i915, pll);
+	skl_ddi_pll_write_ctrl1(i915, pll, hw_state);
 
-	intel_de_write(i915, regs[id].cfgcr1, pll->state.hw_state.cfgcr1);
-	intel_de_write(i915, regs[id].cfgcr2, pll->state.hw_state.cfgcr2);
+	intel_de_write(i915, regs[id].cfgcr1, hw_state->cfgcr1);
+	intel_de_write(i915, regs[id].cfgcr2, hw_state->cfgcr2);
 	intel_de_posting_read(i915, regs[id].cfgcr1);
 	intel_de_posting_read(i915, regs[id].cfgcr2);
 
@@ -1363,9 +1394,12 @@ static void skl_ddi_pll_enable(struct drm_i915_private *i915,
 }
 
 static void skl_ddi_dpll0_enable(struct drm_i915_private *i915,
-				 struct intel_shared_dpll *pll)
+				 struct intel_shared_dpll *pll,
+				 const struct intel_dpll_hw_state *dpll_hw_state)
 {
-	skl_ddi_pll_write_ctrl1(i915, pll);
+	const struct skl_dpll_hw_state *hw_state = &dpll_hw_state->skl;
+
+	skl_ddi_pll_write_ctrl1(i915, pll, hw_state);
 }
 
 static void skl_ddi_pll_disable(struct drm_i915_private *i915,
@@ -1386,13 +1420,14 @@ static void skl_ddi_dpll0_disable(struct drm_i915_private *i915,
 
 static bool skl_ddi_pll_get_hw_state(struct drm_i915_private *i915,
 				     struct intel_shared_dpll *pll,
-				     struct intel_dpll_hw_state *hw_state)
+				     struct intel_dpll_hw_state *dpll_hw_state)
 {
-	u32 val;
+	struct skl_dpll_hw_state *hw_state = &dpll_hw_state->skl;
 	const struct skl_dpll_regs *regs = skl_dpll_regs;
 	const enum intel_dpll_id id = pll->info->id;
 	intel_wakeref_t wakeref;
 	bool ret;
+	u32 val;
 
 	wakeref = intel_display_power_get_if_enabled(i915,
 						     POWER_DOMAIN_DISPLAY_CORE);
@@ -1423,8 +1458,9 @@ out:
 
 static bool skl_ddi_dpll0_get_hw_state(struct drm_i915_private *i915,
 				       struct intel_shared_dpll *pll,
-				       struct intel_dpll_hw_state *hw_state)
+				       struct intel_dpll_hw_state *dpll_hw_state)
 {
+	struct skl_dpll_hw_state *hw_state = &dpll_hw_state->skl;
 	const struct skl_dpll_regs *regs = skl_dpll_regs;
 	const enum intel_dpll_id id = pll->info->id;
 	intel_wakeref_t wakeref;
@@ -1695,16 +1731,17 @@ skip_remaining_dividers:
 
 static int skl_ddi_wrpll_get_freq(struct drm_i915_private *i915,
 				  const struct intel_shared_dpll *pll,
-				  const struct intel_dpll_hw_state *pll_state)
+				  const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct skl_dpll_hw_state *hw_state = &dpll_hw_state->skl;
 	int ref_clock = i915->display.dpll.ref_clks.nssc;
 	u32 p0, p1, p2, dco_freq;
 
-	p0 = pll_state->cfgcr2 & DPLL_CFGCR2_PDIV_MASK;
-	p2 = pll_state->cfgcr2 & DPLL_CFGCR2_KDIV_MASK;
+	p0 = hw_state->cfgcr2 & DPLL_CFGCR2_PDIV_MASK;
+	p2 = hw_state->cfgcr2 & DPLL_CFGCR2_KDIV_MASK;
 
-	if (pll_state->cfgcr2 &  DPLL_CFGCR2_QDIV_MODE(1))
-		p1 = (pll_state->cfgcr2 & DPLL_CFGCR2_QDIV_RATIO_MASK) >> 8;
+	if (hw_state->cfgcr2 &  DPLL_CFGCR2_QDIV_MODE(1))
+		p1 = (hw_state->cfgcr2 & DPLL_CFGCR2_QDIV_RATIO_MASK) >> 8;
 	else
 		p1 = 1;
 
@@ -1752,10 +1789,10 @@ static int skl_ddi_wrpll_get_freq(struct drm_i915_private *i915,
 		return 0;
 	}
 
-	dco_freq = (pll_state->cfgcr1 & DPLL_CFGCR1_DCO_INTEGER_MASK) *
+	dco_freq = (hw_state->cfgcr1 & DPLL_CFGCR1_DCO_INTEGER_MASK) *
 		   ref_clock;
 
-	dco_freq += ((pll_state->cfgcr1 & DPLL_CFGCR1_DCO_FRACTION_MASK) >> 9) *
+	dco_freq += ((hw_state->cfgcr1 & DPLL_CFGCR1_DCO_FRACTION_MASK) >> 9) *
 		    ref_clock / 0x8000;
 
 	if (drm_WARN_ON(&i915->drm, p0 == 0 || p1 == 0 || p2 == 0))
@@ -1767,37 +1804,35 @@ static int skl_ddi_wrpll_get_freq(struct drm_i915_private *i915,
 static int skl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	struct skl_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.skl;
 	struct skl_wrpll_params wrpll_params = {};
-	u32 ctrl1, cfgcr1, cfgcr2;
 	int ret;
 
-	/*
-	 * See comment in intel_dpll_hw_state to understand why we always use 0
-	 * as the DPLL id in this function.
-	 */
-	ctrl1 = DPLL_CTRL1_OVERRIDE(0);
-
-	ctrl1 |= DPLL_CTRL1_HDMI_MODE(0);
-
 	ret = skl_ddi_calculate_wrpll(crtc_state->port_clock * 1000,
 				      i915->display.dpll.ref_clks.nssc, &wrpll_params);
 	if (ret)
 		return ret;
 
-	cfgcr1 = DPLL_CFGCR1_FREQ_ENABLE |
+	/*
+	 * See comment in intel_dpll_hw_state to understand why we always use 0
+	 * as the DPLL id in this function.
+	 */
+	hw_state->ctrl1 =
+		DPLL_CTRL1_OVERRIDE(0) |
+		DPLL_CTRL1_HDMI_MODE(0);
+
+	hw_state->cfgcr1 =
+		DPLL_CFGCR1_FREQ_ENABLE |
 		DPLL_CFGCR1_DCO_FRACTION(wrpll_params.dco_fraction) |
 		wrpll_params.dco_integer;
 
-	cfgcr2 = DPLL_CFGCR2_QDIV_RATIO(wrpll_params.qdiv_ratio) |
+	hw_state->cfgcr2 =
+		DPLL_CFGCR2_QDIV_RATIO(wrpll_params.qdiv_ratio) |
 		DPLL_CFGCR2_QDIV_MODE(wrpll_params.qdiv_mode) |
 		DPLL_CFGCR2_KDIV(wrpll_params.kdiv) |
 		DPLL_CFGCR2_PDIV(wrpll_params.pdiv) |
 		wrpll_params.central_freq;
 
-	crtc_state->dpll_hw_state.ctrl1 = ctrl1;
-	crtc_state->dpll_hw_state.cfgcr1 = cfgcr1;
-	crtc_state->dpll_hw_state.cfgcr2 = cfgcr2;
-
 	crtc_state->port_clock = skl_ddi_wrpll_get_freq(i915, NULL,
 							&crtc_state->dpll_hw_state);
 
@@ -1807,6 +1842,7 @@ static int skl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state)
 static int
 skl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state)
 {
+	struct skl_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.skl;
 	u32 ctrl1;
 
 	/*
@@ -1836,18 +1872,19 @@ skl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state)
 		break;
 	}
 
-	crtc_state->dpll_hw_state.ctrl1 = ctrl1;
+	hw_state->ctrl1 = ctrl1;
 
 	return 0;
 }
 
 static int skl_ddi_lcpll_get_freq(struct drm_i915_private *i915,
 				  const struct intel_shared_dpll *pll,
-				  const struct intel_dpll_hw_state *pll_state)
+				  const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct skl_dpll_hw_state *hw_state = &dpll_hw_state->skl;
 	int link_clock = 0;
 
-	switch ((pll_state->ctrl1 & DPLL_CTRL1_LINK_RATE_MASK(0)) >>
+	switch ((hw_state->ctrl1 & DPLL_CTRL1_LINK_RATE_MASK(0)) >>
 		DPLL_CTRL1_LINK_RATE_SHIFT(0)) {
 	case DPLL_CTRL1_LINK_RATE_810:
 		link_clock = 81000;
@@ -1921,16 +1958,18 @@ static int skl_get_dpll(struct intel_atomic_state *state,
 
 static int skl_ddi_pll_get_freq(struct drm_i915_private *i915,
 				const struct intel_shared_dpll *pll,
-				const struct intel_dpll_hw_state *pll_state)
+				const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct skl_dpll_hw_state *hw_state = &dpll_hw_state->skl;
+
 	/*
 	 * ctrl1 register is already shifted for each pll, just use 0 to get
 	 * the internal shift for each field
 	 */
-	if (pll_state->ctrl1 & DPLL_CTRL1_HDMI_MODE(0))
-		return skl_ddi_wrpll_get_freq(i915, pll, pll_state);
+	if (hw_state->ctrl1 & DPLL_CTRL1_HDMI_MODE(0))
+		return skl_ddi_wrpll_get_freq(i915, pll, dpll_hw_state);
 	else
-		return skl_ddi_lcpll_get_freq(i915, pll, pll_state);
+		return skl_ddi_lcpll_get_freq(i915, pll, dpll_hw_state);
 }
 
 static void skl_update_dpll_ref_clks(struct drm_i915_private *i915)
@@ -1939,19 +1978,21 @@ static void skl_update_dpll_ref_clks(struct drm_i915_private *i915)
 	i915->display.dpll.ref_clks.nssc = i915->display.cdclk.hw.ref;
 }
 
-static void skl_dump_hw_state(struct drm_i915_private *i915,
-			      const struct intel_dpll_hw_state *hw_state)
+static void skl_dump_hw_state(struct drm_printer *p,
+			      const struct intel_dpll_hw_state *dpll_hw_state)
 {
-	drm_dbg_kms(&i915->drm, "dpll_hw_state: "
-		      "ctrl1: 0x%x, cfgcr1: 0x%x, cfgcr2: 0x%x\n",
-		      hw_state->ctrl1,
-		      hw_state->cfgcr1,
-		      hw_state->cfgcr2);
+	const struct skl_dpll_hw_state *hw_state = &dpll_hw_state->skl;
+
+	drm_printf(p, "dpll_hw_state: ctrl1: 0x%x, cfgcr1: 0x%x, cfgcr2: 0x%x\n",
+		   hw_state->ctrl1, hw_state->cfgcr1, hw_state->cfgcr2);
 }
 
-static bool skl_compare_hw_state(const struct intel_dpll_hw_state *a,
-				 const struct intel_dpll_hw_state *b)
+static bool skl_compare_hw_state(const struct intel_dpll_hw_state *_a,
+				 const struct intel_dpll_hw_state *_b)
 {
+	const struct skl_dpll_hw_state *a = &_a->skl;
+	const struct skl_dpll_hw_state *b = &_b->skl;
+
 	return a->ctrl1 == b->ctrl1 &&
 		a->cfgcr1 == b->cfgcr1 &&
 		a->cfgcr2 == b->cfgcr2;
@@ -1991,12 +2032,14 @@ static const struct intel_dpll_mgr skl_pll_mgr = {
 };
 
 static void bxt_ddi_pll_enable(struct drm_i915_private *i915,
-			       struct intel_shared_dpll *pll)
+			       struct intel_shared_dpll *pll,
+			       const struct intel_dpll_hw_state *dpll_hw_state)
 {
-	u32 temp;
+	const struct bxt_dpll_hw_state *hw_state = &dpll_hw_state->bxt;
 	enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */
 	enum dpio_phy phy;
 	enum dpio_channel ch;
+	u32 temp;
 
 	bxt_port_to_phy_channel(i915, port, &phy, &ch);
 
@@ -2019,43 +2062,43 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *i915,
 
 	/* Write P1 & P2 */
 	intel_de_rmw(i915, BXT_PORT_PLL_EBB_0(phy, ch),
-		     PORT_PLL_P1_MASK | PORT_PLL_P2_MASK, pll->state.hw_state.ebb0);
+		     PORT_PLL_P1_MASK | PORT_PLL_P2_MASK, hw_state->ebb0);
 
 	/* Write M2 integer */
 	intel_de_rmw(i915, BXT_PORT_PLL(phy, ch, 0),
-		     PORT_PLL_M2_INT_MASK, pll->state.hw_state.pll0);
+		     PORT_PLL_M2_INT_MASK, hw_state->pll0);
 
 	/* Write N */
 	intel_de_rmw(i915, BXT_PORT_PLL(phy, ch, 1),
-		     PORT_PLL_N_MASK, pll->state.hw_state.pll1);
+		     PORT_PLL_N_MASK, hw_state->pll1);
 
 	/* Write M2 fraction */
 	intel_de_rmw(i915, BXT_PORT_PLL(phy, ch, 2),
-		     PORT_PLL_M2_FRAC_MASK, pll->state.hw_state.pll2);
+		     PORT_PLL_M2_FRAC_MASK, hw_state->pll2);
 
 	/* Write M2 fraction enable */
 	intel_de_rmw(i915, BXT_PORT_PLL(phy, ch, 3),
-		     PORT_PLL_M2_FRAC_ENABLE, pll->state.hw_state.pll3);
+		     PORT_PLL_M2_FRAC_ENABLE, hw_state->pll3);
 
 	/* Write coeff */
 	temp = intel_de_read(i915, BXT_PORT_PLL(phy, ch, 6));
 	temp &= ~PORT_PLL_PROP_COEFF_MASK;
 	temp &= ~PORT_PLL_INT_COEFF_MASK;
 	temp &= ~PORT_PLL_GAIN_CTL_MASK;
-	temp |= pll->state.hw_state.pll6;
+	temp |= hw_state->pll6;
 	intel_de_write(i915, BXT_PORT_PLL(phy, ch, 6), temp);
 
 	/* Write calibration val */
 	intel_de_rmw(i915, BXT_PORT_PLL(phy, ch, 8),
-		     PORT_PLL_TARGET_CNT_MASK, pll->state.hw_state.pll8);
+		     PORT_PLL_TARGET_CNT_MASK, hw_state->pll8);
 
 	intel_de_rmw(i915, BXT_PORT_PLL(phy, ch, 9),
-		     PORT_PLL_LOCK_THRESHOLD_MASK, pll->state.hw_state.pll9);
+		     PORT_PLL_LOCK_THRESHOLD_MASK, hw_state->pll9);
 
 	temp = intel_de_read(i915, BXT_PORT_PLL(phy, ch, 10));
 	temp &= ~PORT_PLL_DCO_AMP_OVR_EN_H;
 	temp &= ~PORT_PLL_DCO_AMP_MASK;
-	temp |= pll->state.hw_state.pll10;
+	temp |= hw_state->pll10;
 	intel_de_write(i915, BXT_PORT_PLL(phy, ch, 10), temp);
 
 	/* Recalibrate with new settings */
@@ -2063,7 +2106,7 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *i915,
 	temp |= PORT_PLL_RECALIBRATE;
 	intel_de_write(i915, BXT_PORT_PLL_EBB_4(phy, ch), temp);
 	temp &= ~PORT_PLL_10BIT_CLK_ENABLE;
-	temp |= pll->state.hw_state.ebb4;
+	temp |= hw_state->ebb4;
 	intel_de_write(i915, BXT_PORT_PLL_EBB_4(phy, ch), temp);
 
 	/* Enable PLL */
@@ -2075,7 +2118,7 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *i915,
 		drm_err(&i915->drm, "PLL %d not locked\n", port);
 
 	if (IS_GEMINILAKE(i915)) {
-		temp = intel_de_read(i915, BXT_PORT_TX_DW5_LN0(phy, ch));
+		temp = intel_de_read(i915, BXT_PORT_TX_DW5_LN(phy, ch, 0));
 		temp |= DCC_DELAY_RANGE_2;
 		intel_de_write(i915, BXT_PORT_TX_DW5_GRP(phy, ch), temp);
 	}
@@ -2087,7 +2130,7 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *i915,
 	temp = intel_de_read(i915, BXT_PORT_PCS_DW12_LN01(phy, ch));
 	temp &= ~LANE_STAGGER_MASK;
 	temp &= ~LANESTAGGER_STRAP_OVRD;
-	temp |= pll->state.hw_state.pcsdw12;
+	temp |= hw_state->pcsdw12;
 	intel_de_write(i915, BXT_PORT_PCS_DW12_GRP(phy, ch), temp);
 }
 
@@ -2112,8 +2155,9 @@ static void bxt_ddi_pll_disable(struct drm_i915_private *i915,
 
 static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *i915,
 				     struct intel_shared_dpll *pll,
-				     struct intel_dpll_hw_state *hw_state)
+				     struct intel_dpll_hw_state *dpll_hw_state)
 {
+	struct bxt_dpll_hw_state *hw_state = &dpll_hw_state->bxt;
 	enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */
 	intel_wakeref_t wakeref;
 	enum dpio_phy phy;
@@ -2245,7 +2289,7 @@ static int bxt_ddi_set_dpll_hw_state(struct intel_crtc_state *crtc_state,
 				     const struct dpll *clk_div)
 {
 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
-	struct intel_dpll_hw_state *dpll_hw_state = &crtc_state->dpll_hw_state;
+	struct bxt_dpll_hw_state *hw_state = &crtc_state->dpll_hw_state.bxt;
 	int clock = crtc_state->port_clock;
 	int vco = clk_div->vco;
 	u32 prop_coef, int_coef, gain_ctl, targ_cnt;
@@ -2283,45 +2327,47 @@ static int bxt_ddi_set_dpll_hw_state(struct intel_crtc_state *crtc_state,
 	else
 		lanestagger = 0x02;
 
-	dpll_hw_state->ebb0 = PORT_PLL_P1(clk_div->p1) | PORT_PLL_P2(clk_div->p2);
-	dpll_hw_state->pll0 = PORT_PLL_M2_INT(clk_div->m2 >> 22);
-	dpll_hw_state->pll1 = PORT_PLL_N(clk_div->n);
-	dpll_hw_state->pll2 = PORT_PLL_M2_FRAC(clk_div->m2 & 0x3fffff);
+	hw_state->ebb0 = PORT_PLL_P1(clk_div->p1) | PORT_PLL_P2(clk_div->p2);
+	hw_state->pll0 = PORT_PLL_M2_INT(clk_div->m2 >> 22);
+	hw_state->pll1 = PORT_PLL_N(clk_div->n);
+	hw_state->pll2 = PORT_PLL_M2_FRAC(clk_div->m2 & 0x3fffff);
 
 	if (clk_div->m2 & 0x3fffff)
-		dpll_hw_state->pll3 = PORT_PLL_M2_FRAC_ENABLE;
+		hw_state->pll3 = PORT_PLL_M2_FRAC_ENABLE;
 
-	dpll_hw_state->pll6 = PORT_PLL_PROP_COEFF(prop_coef) |
+	hw_state->pll6 = PORT_PLL_PROP_COEFF(prop_coef) |
 		PORT_PLL_INT_COEFF(int_coef) |
 		PORT_PLL_GAIN_CTL(gain_ctl);
 
-	dpll_hw_state->pll8 = PORT_PLL_TARGET_CNT(targ_cnt);
+	hw_state->pll8 = PORT_PLL_TARGET_CNT(targ_cnt);
 
-	dpll_hw_state->pll9 = PORT_PLL_LOCK_THRESHOLD(5);
+	hw_state->pll9 = PORT_PLL_LOCK_THRESHOLD(5);
 
-	dpll_hw_state->pll10 = PORT_PLL_DCO_AMP(15) |
+	hw_state->pll10 = PORT_PLL_DCO_AMP(15) |
 		PORT_PLL_DCO_AMP_OVR_EN_H;
 
-	dpll_hw_state->ebb4 = PORT_PLL_10BIT_CLK_ENABLE;
+	hw_state->ebb4 = PORT_PLL_10BIT_CLK_ENABLE;
 
-	dpll_hw_state->pcsdw12 = LANESTAGGER_STRAP_OVRD | lanestagger;
+	hw_state->pcsdw12 = LANESTAGGER_STRAP_OVRD | lanestagger;
 
 	return 0;
 }
 
 static int bxt_ddi_pll_get_freq(struct drm_i915_private *i915,
 				const struct intel_shared_dpll *pll,
-				const struct intel_dpll_hw_state *pll_state)
+				const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct bxt_dpll_hw_state *hw_state = &dpll_hw_state->bxt;
 	struct dpll clock;
 
 	clock.m1 = 2;
-	clock.m2 = REG_FIELD_GET(PORT_PLL_M2_INT_MASK, pll_state->pll0) << 22;
-	if (pll_state->pll3 & PORT_PLL_M2_FRAC_ENABLE)
-		clock.m2 |= REG_FIELD_GET(PORT_PLL_M2_FRAC_MASK, pll_state->pll2);
-	clock.n = REG_FIELD_GET(PORT_PLL_N_MASK, pll_state->pll1);
-	clock.p1 = REG_FIELD_GET(PORT_PLL_P1_MASK, pll_state->ebb0);
-	clock.p2 = REG_FIELD_GET(PORT_PLL_P2_MASK, pll_state->ebb0);
+	clock.m2 = REG_FIELD_GET(PORT_PLL_M2_INT_MASK, hw_state->pll0) << 22;
+	if (hw_state->pll3 & PORT_PLL_M2_FRAC_ENABLE)
+		clock.m2 |= REG_FIELD_GET(PORT_PLL_M2_FRAC_MASK,
+					  hw_state->pll2);
+	clock.n = REG_FIELD_GET(PORT_PLL_N_MASK, hw_state->pll1);
+	clock.p1 = REG_FIELD_GET(PORT_PLL_P1_MASK, hw_state->ebb0);
+	clock.p2 = REG_FIELD_GET(PORT_PLL_P2_MASK, hw_state->ebb0);
 
 	return chv_calc_dpll_params(i915->display.dpll.ref_clks.nssc, &clock);
 }
@@ -2402,28 +2448,26 @@ static void bxt_update_dpll_ref_clks(struct drm_i915_private *i915)
 	/* DSI non-SSC ref 19.2MHz */
 }
 
-static void bxt_dump_hw_state(struct drm_i915_private *i915,
-			      const struct intel_dpll_hw_state *hw_state)
+static void bxt_dump_hw_state(struct drm_printer *p,
+			      const struct intel_dpll_hw_state *dpll_hw_state)
 {
-	drm_dbg_kms(&i915->drm, "dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x,"
-		    "pll0: 0x%x, pll1: 0x%x, pll2: 0x%x, pll3: 0x%x, "
-		    "pll6: 0x%x, pll8: 0x%x, pll9: 0x%x, pll10: 0x%x, pcsdw12: 0x%x\n",
-		    hw_state->ebb0,
-		    hw_state->ebb4,
-		    hw_state->pll0,
-		    hw_state->pll1,
-		    hw_state->pll2,
-		    hw_state->pll3,
-		    hw_state->pll6,
-		    hw_state->pll8,
-		    hw_state->pll9,
-		    hw_state->pll10,
-		    hw_state->pcsdw12);
+	const struct bxt_dpll_hw_state *hw_state = &dpll_hw_state->bxt;
+
+	drm_printf(p, "dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x,"
+		   "pll0: 0x%x, pll1: 0x%x, pll2: 0x%x, pll3: 0x%x, "
+		   "pll6: 0x%x, pll8: 0x%x, pll9: 0x%x, pll10: 0x%x, pcsdw12: 0x%x\n",
+		   hw_state->ebb0, hw_state->ebb4,
+		   hw_state->pll0, hw_state->pll1, hw_state->pll2, hw_state->pll3,
+		   hw_state->pll6, hw_state->pll8, hw_state->pll9, hw_state->pll10,
+		   hw_state->pcsdw12);
 }
 
-static bool bxt_compare_hw_state(const struct intel_dpll_hw_state *a,
-				 const struct intel_dpll_hw_state *b)
+static bool bxt_compare_hw_state(const struct intel_dpll_hw_state *_a,
+				 const struct intel_dpll_hw_state *_b)
 {
+	const struct bxt_dpll_hw_state *a = &_a->bxt;
+	const struct bxt_dpll_hw_state *b = &_b->bxt;
+
 	return a->ebb0 == b->ebb0 &&
 		a->ebb4 == b->ebb4 &&
 		a->pll0 == b->pll0 &&
@@ -2706,7 +2750,7 @@ static int icl_calc_tbt_pll(struct intel_crtc_state *crtc_state,
 
 static int icl_ddi_tbt_pll_get_freq(struct drm_i915_private *i915,
 				    const struct intel_shared_dpll *pll,
-				    const struct intel_dpll_hw_state *pll_state)
+				    const struct intel_dpll_hw_state *dpll_hw_state)
 {
 	/*
 	 * The PLL outputs multiple frequencies at the same time, selection is
@@ -2777,17 +2821,18 @@ icl_calc_wrpll(struct intel_crtc_state *crtc_state,
 
 static int icl_ddi_combo_pll_get_freq(struct drm_i915_private *i915,
 				      const struct intel_shared_dpll *pll,
-				      const struct intel_dpll_hw_state *pll_state)
+				      const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	int ref_clock = icl_wrpll_ref_clock(i915);
 	u32 dco_fraction;
 	u32 p0, p1, p2, dco_freq;
 
-	p0 = pll_state->cfgcr1 & DPLL_CFGCR1_PDIV_MASK;
-	p2 = pll_state->cfgcr1 & DPLL_CFGCR1_KDIV_MASK;
+	p0 = hw_state->cfgcr1 & DPLL_CFGCR1_PDIV_MASK;
+	p2 = hw_state->cfgcr1 & DPLL_CFGCR1_KDIV_MASK;
 
-	if (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_MODE(1))
-		p1 = (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_RATIO_MASK) >>
+	if (hw_state->cfgcr1 & DPLL_CFGCR1_QDIV_MODE(1))
+		p1 = (hw_state->cfgcr1 & DPLL_CFGCR1_QDIV_RATIO_MASK) >>
 			DPLL_CFGCR1_QDIV_RATIO_SHIFT;
 	else
 		p1 = 1;
@@ -2819,10 +2864,10 @@ static int icl_ddi_combo_pll_get_freq(struct drm_i915_private *i915,
 		break;
 	}
 
-	dco_freq = (pll_state->cfgcr0 & DPLL_CFGCR0_DCO_INTEGER_MASK) *
+	dco_freq = (hw_state->cfgcr0 & DPLL_CFGCR0_DCO_INTEGER_MASK) *
 		   ref_clock;
 
-	dco_fraction = (pll_state->cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >>
+	dco_fraction = (hw_state->cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >>
 		       DPLL_CFGCR0_DCO_FRACTION_SHIFT;
 
 	if (ehl_combo_pll_div_frac_wa_needed(i915))
@@ -2838,33 +2883,34 @@ static int icl_ddi_combo_pll_get_freq(struct drm_i915_private *i915,
 
 static void icl_calc_dpll_state(struct drm_i915_private *i915,
 				const struct skl_wrpll_params *pll_params,
-				struct intel_dpll_hw_state *pll_state)
+				struct intel_dpll_hw_state *dpll_hw_state)
 {
+	struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	u32 dco_fraction = pll_params->dco_fraction;
 
 	if (ehl_combo_pll_div_frac_wa_needed(i915))
 		dco_fraction = DIV_ROUND_CLOSEST(dco_fraction, 2);
 
-	pll_state->cfgcr0 = DPLL_CFGCR0_DCO_FRACTION(dco_fraction) |
+	hw_state->cfgcr0 = DPLL_CFGCR0_DCO_FRACTION(dco_fraction) |
 			    pll_params->dco_integer;
 
-	pll_state->cfgcr1 = DPLL_CFGCR1_QDIV_RATIO(pll_params->qdiv_ratio) |
+	hw_state->cfgcr1 = DPLL_CFGCR1_QDIV_RATIO(pll_params->qdiv_ratio) |
 			    DPLL_CFGCR1_QDIV_MODE(pll_params->qdiv_mode) |
 			    DPLL_CFGCR1_KDIV(pll_params->kdiv) |
 			    DPLL_CFGCR1_PDIV(pll_params->pdiv);
 
 	if (DISPLAY_VER(i915) >= 12)
-		pll_state->cfgcr1 |= TGL_DPLL_CFGCR1_CFSELOVRD_NORMAL_XTAL;
+		hw_state->cfgcr1 |= TGL_DPLL_CFGCR1_CFSELOVRD_NORMAL_XTAL;
 	else
-		pll_state->cfgcr1 |= DPLL_CFGCR1_CENTRAL_FREQ_8400;
+		hw_state->cfgcr1 |= DPLL_CFGCR1_CENTRAL_FREQ_8400;
 
 	if (i915->display.vbt.override_afc_startup)
-		pll_state->div0 = TGL_DPLL0_DIV0_AFC_STARTUP(i915->display.vbt.override_afc_startup_val);
+		hw_state->div0 = TGL_DPLL0_DIV0_AFC_STARTUP(i915->display.vbt.override_afc_startup_val);
 }
 
 static int icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc,
 				    u32 *target_dco_khz,
-				    struct intel_dpll_hw_state *state,
+				    struct icl_dpll_hw_state *hw_state,
 				    bool is_dkl)
 {
 	static const u8 div1_vals[] = { 7, 5, 3, 2 };
@@ -2920,12 +2966,12 @@ static int icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc,
 
 			*target_dco_khz = dco;
 
-			state->mg_refclkin_ctl = MG_REFCLKIN_CTL_OD_2_MUX(1);
+			hw_state->mg_refclkin_ctl = MG_REFCLKIN_CTL_OD_2_MUX(1);
 
-			state->mg_clktop2_coreclkctl1 =
+			hw_state->mg_clktop2_coreclkctl1 =
 				MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO(a_divratio);
 
-			state->mg_clktop2_hsclkctl =
+			hw_state->mg_clktop2_hsclkctl =
 				MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL(tlinedrv) |
 				MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL(inputsel) |
 				hsdiv |
@@ -2943,9 +2989,10 @@ static int icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc,
  * adapted to integer-only calculation, that's why it looks so different.
  */
 static int icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
-				 struct intel_dpll_hw_state *pll_state)
+				 struct intel_dpll_hw_state *dpll_hw_state)
 {
 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	int refclk_khz = i915->display.dpll.ref_clks.nssc;
 	int clock = crtc_state->port_clock;
 	u32 dco_khz, m1div, m2div_int, m2div_rem, m2div_frac;
@@ -2960,7 +3007,7 @@ static int icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
 	int ret;
 
 	ret = icl_mg_pll_find_divisors(clock, is_dp, use_ssc, &dco_khz,
-				       pll_state, is_dkl);
+				       hw_state, is_dkl);
 	if (ret)
 		return ret;
 
@@ -3050,61 +3097,61 @@ static int icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
 
 	/* write pll_state calculations */
 	if (is_dkl) {
-		pll_state->mg_pll_div0 = DKL_PLL_DIV0_INTEG_COEFF(int_coeff) |
+		hw_state->mg_pll_div0 = DKL_PLL_DIV0_INTEG_COEFF(int_coeff) |
 					 DKL_PLL_DIV0_PROP_COEFF(prop_coeff) |
 					 DKL_PLL_DIV0_FBPREDIV(m1div) |
 					 DKL_PLL_DIV0_FBDIV_INT(m2div_int);
 		if (i915->display.vbt.override_afc_startup) {
 			u8 val = i915->display.vbt.override_afc_startup_val;
 
-			pll_state->mg_pll_div0 |= DKL_PLL_DIV0_AFC_STARTUP(val);
+			hw_state->mg_pll_div0 |= DKL_PLL_DIV0_AFC_STARTUP(val);
 		}
 
-		pll_state->mg_pll_div1 = DKL_PLL_DIV1_IREF_TRIM(iref_trim) |
+		hw_state->mg_pll_div1 = DKL_PLL_DIV1_IREF_TRIM(iref_trim) |
 					 DKL_PLL_DIV1_TDC_TARGET_CNT(tdc_targetcnt);
 
-		pll_state->mg_pll_ssc = DKL_PLL_SSC_IREF_NDIV_RATIO(iref_ndiv) |
+		hw_state->mg_pll_ssc = DKL_PLL_SSC_IREF_NDIV_RATIO(iref_ndiv) |
 					DKL_PLL_SSC_STEP_LEN(ssc_steplen) |
 					DKL_PLL_SSC_STEP_NUM(ssc_steplog) |
 					(use_ssc ? DKL_PLL_SSC_EN : 0);
 
-		pll_state->mg_pll_bias = (m2div_frac ? DKL_PLL_BIAS_FRAC_EN_H : 0) |
+		hw_state->mg_pll_bias = (m2div_frac ? DKL_PLL_BIAS_FRAC_EN_H : 0) |
 					  DKL_PLL_BIAS_FBDIV_FRAC(m2div_frac);
 
-		pll_state->mg_pll_tdc_coldst_bias =
+		hw_state->mg_pll_tdc_coldst_bias =
 				DKL_PLL_TDC_SSC_STEP_SIZE(ssc_stepsize) |
 				DKL_PLL_TDC_FEED_FWD_GAIN(feedfwgain);
 
 	} else {
-		pll_state->mg_pll_div0 =
+		hw_state->mg_pll_div0 =
 			(m2div_rem > 0 ? MG_PLL_DIV0_FRACNEN_H : 0) |
 			MG_PLL_DIV0_FBDIV_FRAC(m2div_frac) |
 			MG_PLL_DIV0_FBDIV_INT(m2div_int);
 
-		pll_state->mg_pll_div1 =
+		hw_state->mg_pll_div1 =
 			MG_PLL_DIV1_IREF_NDIVRATIO(iref_ndiv) |
 			MG_PLL_DIV1_DITHER_DIV_2 |
 			MG_PLL_DIV1_NDIVRATIO(1) |
 			MG_PLL_DIV1_FBPREDIV(m1div);
 
-		pll_state->mg_pll_lf =
+		hw_state->mg_pll_lf =
 			MG_PLL_LF_TDCTARGETCNT(tdc_targetcnt) |
 			MG_PLL_LF_AFCCNTSEL_512 |
 			MG_PLL_LF_GAINCTRL(1) |
 			MG_PLL_LF_INT_COEFF(int_coeff) |
 			MG_PLL_LF_PROP_COEFF(prop_coeff);
 
-		pll_state->mg_pll_frac_lock =
+		hw_state->mg_pll_frac_lock =
 			MG_PLL_FRAC_LOCK_TRUELOCK_CRIT_32 |
 			MG_PLL_FRAC_LOCK_EARLYLOCK_CRIT_32 |
 			MG_PLL_FRAC_LOCK_LOCKTHRESH(10) |
 			MG_PLL_FRAC_LOCK_DCODITHEREN |
 			MG_PLL_FRAC_LOCK_FEEDFWRDGAIN(feedfwgain);
 		if (use_ssc || m2div_rem > 0)
-			pll_state->mg_pll_frac_lock |=
+			hw_state->mg_pll_frac_lock |=
 				MG_PLL_FRAC_LOCK_FEEDFWRDCAL_EN;
 
-		pll_state->mg_pll_ssc =
+		hw_state->mg_pll_ssc =
 			(use_ssc ? MG_PLL_SSC_EN : 0) |
 			MG_PLL_SSC_TYPE(2) |
 			MG_PLL_SSC_STEPLENGTH(ssc_steplen) |
@@ -3112,14 +3159,14 @@ static int icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
 			MG_PLL_SSC_FLLEN |
 			MG_PLL_SSC_STEPSIZE(ssc_stepsize);
 
-		pll_state->mg_pll_tdc_coldst_bias =
+		hw_state->mg_pll_tdc_coldst_bias =
 			MG_PLL_TDC_COLDST_COLDSTART |
 			MG_PLL_TDC_COLDST_IREFINT_EN |
 			MG_PLL_TDC_COLDST_REFBIAS_START_PULSE_W(iref_pulse_w) |
 			MG_PLL_TDC_TDCOVCCORR_EN |
 			MG_PLL_TDC_TDCSEL(3);
 
-		pll_state->mg_pll_bias =
+		hw_state->mg_pll_bias =
 			MG_PLL_BIAS_BIAS_GB_SEL(3) |
 			MG_PLL_BIAS_INIT_DCOAMP(0x3F) |
 			MG_PLL_BIAS_BIAS_BONUS(10) |
@@ -3129,17 +3176,17 @@ static int icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
 			MG_PLL_BIAS_IREFTRIM(iref_trim);
 
 		if (refclk_khz == 38400) {
-			pll_state->mg_pll_tdc_coldst_bias_mask =
+			hw_state->mg_pll_tdc_coldst_bias_mask =
 				MG_PLL_TDC_COLDST_COLDSTART;
-			pll_state->mg_pll_bias_mask = 0;
+			hw_state->mg_pll_bias_mask = 0;
 		} else {
-			pll_state->mg_pll_tdc_coldst_bias_mask = -1U;
-			pll_state->mg_pll_bias_mask = -1U;
+			hw_state->mg_pll_tdc_coldst_bias_mask = -1U;
+			hw_state->mg_pll_bias_mask = -1U;
 		}
 
-		pll_state->mg_pll_tdc_coldst_bias &=
-			pll_state->mg_pll_tdc_coldst_bias_mask;
-		pll_state->mg_pll_bias &= pll_state->mg_pll_bias_mask;
+		hw_state->mg_pll_tdc_coldst_bias &=
+			hw_state->mg_pll_tdc_coldst_bias_mask;
+		hw_state->mg_pll_bias &= hw_state->mg_pll_bias_mask;
 	}
 
 	return 0;
@@ -3147,31 +3194,32 @@ static int icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
 
 static int icl_ddi_mg_pll_get_freq(struct drm_i915_private *i915,
 				   const struct intel_shared_dpll *pll,
-				   const struct intel_dpll_hw_state *pll_state)
+				   const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	u32 m1, m2_int, m2_frac, div1, div2, ref_clock;
 	u64 tmp;
 
 	ref_clock = i915->display.dpll.ref_clks.nssc;
 
 	if (DISPLAY_VER(i915) >= 12) {
-		m1 = pll_state->mg_pll_div0 & DKL_PLL_DIV0_FBPREDIV_MASK;
+		m1 = hw_state->mg_pll_div0 & DKL_PLL_DIV0_FBPREDIV_MASK;
 		m1 = m1 >> DKL_PLL_DIV0_FBPREDIV_SHIFT;
-		m2_int = pll_state->mg_pll_div0 & DKL_PLL_DIV0_FBDIV_INT_MASK;
+		m2_int = hw_state->mg_pll_div0 & DKL_PLL_DIV0_FBDIV_INT_MASK;
 
-		if (pll_state->mg_pll_bias & DKL_PLL_BIAS_FRAC_EN_H) {
-			m2_frac = pll_state->mg_pll_bias &
+		if (hw_state->mg_pll_bias & DKL_PLL_BIAS_FRAC_EN_H) {
+			m2_frac = hw_state->mg_pll_bias &
 				  DKL_PLL_BIAS_FBDIV_FRAC_MASK;
 			m2_frac = m2_frac >> DKL_PLL_BIAS_FBDIV_SHIFT;
 		} else {
 			m2_frac = 0;
 		}
 	} else {
-		m1 = pll_state->mg_pll_div1 & MG_PLL_DIV1_FBPREDIV_MASK;
-		m2_int = pll_state->mg_pll_div0 & MG_PLL_DIV0_FBDIV_INT_MASK;
+		m1 = hw_state->mg_pll_div1 & MG_PLL_DIV1_FBPREDIV_MASK;
+		m2_int = hw_state->mg_pll_div0 & MG_PLL_DIV0_FBDIV_INT_MASK;
 
-		if (pll_state->mg_pll_div0 & MG_PLL_DIV0_FRACNEN_H) {
-			m2_frac = pll_state->mg_pll_div0 &
+		if (hw_state->mg_pll_div0 & MG_PLL_DIV0_FRACNEN_H) {
+			m2_frac = hw_state->mg_pll_div0 &
 				  MG_PLL_DIV0_FBDIV_FRAC_MASK;
 			m2_frac = m2_frac >> MG_PLL_DIV0_FBDIV_FRAC_SHIFT;
 		} else {
@@ -3179,7 +3227,7 @@ static int icl_ddi_mg_pll_get_freq(struct drm_i915_private *i915,
 		}
 	}
 
-	switch (pll_state->mg_clktop2_hsclkctl &
+	switch (hw_state->mg_clktop2_hsclkctl &
 		MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK) {
 	case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_2:
 		div1 = 2;
@@ -3194,11 +3242,11 @@ static int icl_ddi_mg_pll_get_freq(struct drm_i915_private *i915,
 		div1 = 7;
 		break;
 	default:
-		MISSING_CASE(pll_state->mg_clktop2_hsclkctl);
+		MISSING_CASE(hw_state->mg_clktop2_hsclkctl);
 		return 0;
 	}
 
-	div2 = (pll_state->mg_clktop2_hsclkctl &
+	div2 = (hw_state->mg_clktop2_hsclkctl &
 		MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK) >>
 		MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_SHIFT;
 
@@ -3389,7 +3437,6 @@ static int icl_get_tc_phy_dplls(struct intel_atomic_state *state,
 				struct intel_crtc *crtc,
 				struct intel_encoder *encoder)
 {
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
 	struct intel_crtc_state *crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct icl_port_dpll *port_dpll =
@@ -3408,8 +3455,7 @@ static int icl_get_tc_phy_dplls(struct intel_atomic_state *state,
 
 
 	port_dpll = &crtc_state->icl_port_dplls[ICL_PORT_DPLL_MG_PHY];
-	dpll_id = icl_tc_port_to_pll_id(intel_port_to_tc(i915,
-							 encoder->port));
+	dpll_id = icl_tc_port_to_pll_id(intel_encoder_to_tc(encoder));
 	port_dpll->pll = intel_find_shared_dpll(state, crtc,
 						&port_dpll->hw_state,
 						BIT(dpll_id));
@@ -3435,15 +3481,12 @@ static int icl_compute_dplls(struct intel_atomic_state *state,
 			     struct intel_crtc *crtc,
 			     struct intel_encoder *encoder)
 {
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
-
-	if (intel_phy_is_combo(i915, phy))
+	if (intel_encoder_is_combo(encoder))
 		return icl_compute_combo_phy_dpll(state, crtc);
-	else if (intel_phy_is_tc(i915, phy))
+	else if (intel_encoder_is_tc(encoder))
 		return icl_compute_tc_phy_dplls(state, crtc);
 
-	MISSING_CASE(phy);
+	MISSING_CASE(encoder->port);
 
 	return 0;
 }
@@ -3452,15 +3495,12 @@ static int icl_get_dplls(struct intel_atomic_state *state,
 			 struct intel_crtc *crtc,
 			 struct intel_encoder *encoder)
 {
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
-
-	if (intel_phy_is_combo(i915, phy))
+	if (intel_encoder_is_combo(encoder))
 		return icl_get_combo_phy_dpll(state, crtc, encoder);
-	else if (intel_phy_is_tc(i915, phy))
+	else if (intel_encoder_is_tc(encoder))
 		return icl_get_tc_phy_dplls(state, crtc, encoder);
 
-	MISSING_CASE(phy);
+	MISSING_CASE(encoder->port);
 
 	return -EINVAL;
 }
@@ -3493,8 +3533,9 @@ static void icl_put_dplls(struct intel_atomic_state *state,
 
 static bool mg_pll_get_hw_state(struct drm_i915_private *i915,
 				struct intel_shared_dpll *pll,
-				struct intel_dpll_hw_state *hw_state)
+				struct intel_dpll_hw_state *dpll_hw_state)
 {
+	struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	const enum intel_dpll_id id = pll->info->id;
 	enum tc_port tc_port = icl_pll_id_to_tc_port(id);
 	intel_wakeref_t wakeref;
@@ -3559,8 +3600,9 @@ out:
 
 static bool dkl_pll_get_hw_state(struct drm_i915_private *i915,
 				 struct intel_shared_dpll *pll,
-				 struct intel_dpll_hw_state *hw_state)
+				 struct intel_dpll_hw_state *dpll_hw_state)
 {
+	struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	const enum intel_dpll_id id = pll->info->id;
 	enum tc_port tc_port = icl_pll_id_to_tc_port(id);
 	intel_wakeref_t wakeref;
@@ -3630,9 +3672,10 @@ out:
 
 static bool icl_pll_get_hw_state(struct drm_i915_private *i915,
 				 struct intel_shared_dpll *pll,
-				 struct intel_dpll_hw_state *hw_state,
+				 struct intel_dpll_hw_state *dpll_hw_state,
 				 i915_reg_t enable_reg)
 {
+	struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	const enum intel_dpll_id id = pll->info->id;
 	intel_wakeref_t wakeref;
 	bool ret = false;
@@ -3690,24 +3733,24 @@ out:
 
 static bool combo_pll_get_hw_state(struct drm_i915_private *i915,
 				   struct intel_shared_dpll *pll,
-				   struct intel_dpll_hw_state *hw_state)
+				   struct intel_dpll_hw_state *dpll_hw_state)
 {
 	i915_reg_t enable_reg = intel_combo_pll_enable_reg(i915, pll);
 
-	return icl_pll_get_hw_state(i915, pll, hw_state, enable_reg);
+	return icl_pll_get_hw_state(i915, pll, dpll_hw_state, enable_reg);
 }
 
 static bool tbt_pll_get_hw_state(struct drm_i915_private *i915,
 				 struct intel_shared_dpll *pll,
-				 struct intel_dpll_hw_state *hw_state)
+				 struct intel_dpll_hw_state *dpll_hw_state)
 {
-	return icl_pll_get_hw_state(i915, pll, hw_state, TBT_PLL_ENABLE);
+	return icl_pll_get_hw_state(i915, pll, dpll_hw_state, TBT_PLL_ENABLE);
 }
 
 static void icl_dpll_write(struct drm_i915_private *i915,
-			   struct intel_shared_dpll *pll)
+			   struct intel_shared_dpll *pll,
+			   const struct icl_dpll_hw_state *hw_state)
 {
-	struct intel_dpll_hw_state *hw_state = &pll->state.hw_state;
 	const enum intel_dpll_id id = pll->info->id;
 	i915_reg_t cfgcr0_reg, cfgcr1_reg, div0_reg = INVALID_MMIO_REG;
 
@@ -3747,9 +3790,9 @@ static void icl_dpll_write(struct drm_i915_private *i915,
 }
 
 static void icl_mg_pll_write(struct drm_i915_private *i915,
-			     struct intel_shared_dpll *pll)
+			     struct intel_shared_dpll *pll,
+			     const struct icl_dpll_hw_state *hw_state)
 {
-	struct intel_dpll_hw_state *hw_state = &pll->state.hw_state;
 	enum tc_port tc_port = icl_pll_id_to_tc_port(pll->info->id);
 
 	/*
@@ -3790,9 +3833,9 @@ static void icl_mg_pll_write(struct drm_i915_private *i915,
 }
 
 static void dkl_pll_write(struct drm_i915_private *i915,
-			  struct intel_shared_dpll *pll)
+			  struct intel_shared_dpll *pll,
+			  const struct icl_dpll_hw_state *hw_state)
 {
-	struct intel_dpll_hw_state *hw_state = &pll->state.hw_state;
 	enum tc_port tc_port = icl_pll_id_to_tc_port(pll->info->id);
 	u32 val;
 
@@ -3905,13 +3948,15 @@ static void adlp_cmtg_clock_gating_wa(struct drm_i915_private *i915, struct inte
 }
 
 static void combo_pll_enable(struct drm_i915_private *i915,
-			     struct intel_shared_dpll *pll)
+			     struct intel_shared_dpll *pll,
+			     const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	i915_reg_t enable_reg = intel_combo_pll_enable_reg(i915, pll);
 
 	icl_pll_power_enable(i915, pll, enable_reg);
 
-	icl_dpll_write(i915, pll);
+	icl_dpll_write(i915, pll, hw_state);
 
 	/*
 	 * DVFS pre sequence would be here, but in our driver the cdclk code
@@ -3927,11 +3972,14 @@ static void combo_pll_enable(struct drm_i915_private *i915,
 }
 
 static void tbt_pll_enable(struct drm_i915_private *i915,
-			   struct intel_shared_dpll *pll)
+			   struct intel_shared_dpll *pll,
+			   const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
+
 	icl_pll_power_enable(i915, pll, TBT_PLL_ENABLE);
 
-	icl_dpll_write(i915, pll);
+	icl_dpll_write(i915, pll, hw_state);
 
 	/*
 	 * DVFS pre sequence would be here, but in our driver the cdclk code
@@ -3945,16 +3993,18 @@ static void tbt_pll_enable(struct drm_i915_private *i915,
 }
 
 static void mg_pll_enable(struct drm_i915_private *i915,
-			  struct intel_shared_dpll *pll)
+			  struct intel_shared_dpll *pll,
+			  const struct intel_dpll_hw_state *dpll_hw_state)
 {
+	const struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	i915_reg_t enable_reg = intel_tc_pll_enable_reg(i915, pll);
 
 	icl_pll_power_enable(i915, pll, enable_reg);
 
 	if (DISPLAY_VER(i915) >= 12)
-		dkl_pll_write(i915, pll);
+		dkl_pll_write(i915, pll, hw_state);
 	else
-		icl_mg_pll_write(i915, pll);
+		icl_mg_pll_write(i915, pll, hw_state);
 
 	/*
 	 * DVFS pre sequence would be here, but in our driver the cdclk code
@@ -4026,33 +4076,36 @@ static void icl_update_dpll_ref_clks(struct drm_i915_private *i915)
 	i915->display.dpll.ref_clks.nssc = i915->display.cdclk.hw.ref;
 }
 
-static void icl_dump_hw_state(struct drm_i915_private *i915,
-			      const struct intel_dpll_hw_state *hw_state)
+static void icl_dump_hw_state(struct drm_printer *p,
+			      const struct intel_dpll_hw_state *dpll_hw_state)
 {
-	drm_dbg_kms(&i915->drm,
-		    "dpll_hw_state: cfgcr0: 0x%x, cfgcr1: 0x%x, div0: 0x%x, "
-		    "mg_refclkin_ctl: 0x%x, hg_clktop2_coreclkctl1: 0x%x, "
-		    "mg_clktop2_hsclkctl: 0x%x, mg_pll_div0: 0x%x, "
-		    "mg_pll_div2: 0x%x, mg_pll_lf: 0x%x, "
-		    "mg_pll_frac_lock: 0x%x, mg_pll_ssc: 0x%x, "
-		    "mg_pll_bias: 0x%x, mg_pll_tdc_coldst_bias: 0x%x\n",
-		    hw_state->cfgcr0, hw_state->cfgcr1,
-		    hw_state->div0,
-		    hw_state->mg_refclkin_ctl,
-		    hw_state->mg_clktop2_coreclkctl1,
-		    hw_state->mg_clktop2_hsclkctl,
-		    hw_state->mg_pll_div0,
-		    hw_state->mg_pll_div1,
-		    hw_state->mg_pll_lf,
-		    hw_state->mg_pll_frac_lock,
-		    hw_state->mg_pll_ssc,
-		    hw_state->mg_pll_bias,
-		    hw_state->mg_pll_tdc_coldst_bias);
-}
-
-static bool icl_compare_hw_state(const struct intel_dpll_hw_state *a,
-				 const struct intel_dpll_hw_state *b)
+	const struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
+
+	drm_printf(p, "dpll_hw_state: cfgcr0: 0x%x, cfgcr1: 0x%x, div0: 0x%x, "
+		   "mg_refclkin_ctl: 0x%x, hg_clktop2_coreclkctl1: 0x%x, "
+		   "mg_clktop2_hsclkctl: 0x%x, mg_pll_div0: 0x%x, "
+		   "mg_pll_div2: 0x%x, mg_pll_lf: 0x%x, "
+		   "mg_pll_frac_lock: 0x%x, mg_pll_ssc: 0x%x, "
+		   "mg_pll_bias: 0x%x, mg_pll_tdc_coldst_bias: 0x%x\n",
+		   hw_state->cfgcr0, hw_state->cfgcr1, hw_state->div0,
+		   hw_state->mg_refclkin_ctl,
+		   hw_state->mg_clktop2_coreclkctl1,
+		   hw_state->mg_clktop2_hsclkctl,
+		   hw_state->mg_pll_div0,
+		   hw_state->mg_pll_div1,
+		   hw_state->mg_pll_lf,
+		   hw_state->mg_pll_frac_lock,
+		   hw_state->mg_pll_ssc,
+		   hw_state->mg_pll_bias,
+		   hw_state->mg_pll_tdc_coldst_bias);
+}
+
+static bool icl_compare_hw_state(const struct intel_dpll_hw_state *_a,
+				 const struct intel_dpll_hw_state *_b)
 {
+	const struct icl_dpll_hw_state *a = &_a->icl;
+	const struct icl_dpll_hw_state *b = &_b->icl;
+
 	/* FIXME split combo vs. mg more thoroughly */
 	return a->cfgcr0 == b->cfgcr0 &&
 		a->cfgcr1 == b->cfgcr1 &&
@@ -4417,33 +4470,33 @@ void intel_update_active_dpll(struct intel_atomic_state *state,
  * intel_dpll_get_freq - calculate the DPLL's output frequency
  * @i915: i915 device
  * @pll: DPLL for which to calculate the output frequency
- * @pll_state: DPLL state from which to calculate the output frequency
+ * @dpll_hw_state: DPLL state from which to calculate the output frequency
  *
- * Return the output frequency corresponding to @pll's passed in @pll_state.
+ * Return the output frequency corresponding to @pll's passed in @dpll_hw_state.
  */
 int intel_dpll_get_freq(struct drm_i915_private *i915,
 			const struct intel_shared_dpll *pll,
-			const struct intel_dpll_hw_state *pll_state)
+			const struct intel_dpll_hw_state *dpll_hw_state)
 {
 	if (drm_WARN_ON(&i915->drm, !pll->info->funcs->get_freq))
 		return 0;
 
-	return pll->info->funcs->get_freq(i915, pll, pll_state);
+	return pll->info->funcs->get_freq(i915, pll, dpll_hw_state);
 }
 
 /**
  * intel_dpll_get_hw_state - readout the DPLL's hardware state
  * @i915: i915 device
  * @pll: DPLL for which to calculate the output frequency
- * @hw_state: DPLL's hardware state
+ * @dpll_hw_state: DPLL's hardware state
  *
- * Read out @pll's hardware state into @hw_state.
+ * Read out @pll's hardware state into @dpll_hw_state.
  */
 bool intel_dpll_get_hw_state(struct drm_i915_private *i915,
 			     struct intel_shared_dpll *pll,
-			     struct intel_dpll_hw_state *hw_state)
+			     struct intel_dpll_hw_state *dpll_hw_state)
 {
-	return pll->info->funcs->get_hw_state(i915, pll, hw_state);
+	return pll->info->funcs->get_hw_state(i915, pll, dpll_hw_state);
 }
 
 static void readout_dpll_hw_state(struct drm_i915_private *i915,
@@ -4514,22 +4567,24 @@ void intel_dpll_sanitize_state(struct drm_i915_private *i915)
 }
 
 /**
- * intel_dpll_dump_hw_state - write hw_state to dmesg
+ * intel_dpll_dump_hw_state - dump hw_state
  * @i915: i915 drm device
- * @hw_state: hw state to be written to the log
+ * @p: where to print the state to
+ * @dpll_hw_state: hw state to be dumped
  *
- * Write the relevant values in @hw_state to dmesg using drm_dbg_kms.
+ * Dumo out the relevant values in @dpll_hw_state.
  */
 void intel_dpll_dump_hw_state(struct drm_i915_private *i915,
-			      const struct intel_dpll_hw_state *hw_state)
+			      struct drm_printer *p,
+			      const struct intel_dpll_hw_state *dpll_hw_state)
 {
 	if (i915->display.dpll.mgr) {
-		i915->display.dpll.mgr->dump_hw_state(i915, hw_state);
+		i915->display.dpll.mgr->dump_hw_state(p, dpll_hw_state);
 	} else {
 		/* fallback for platforms that don't use the shared dpll
 		 * infrastructure
 		 */
-		ibx_dump_hw_state(i915, hw_state);
+		ibx_dump_hw_state(p, dpll_hw_state);
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
index cc0e1386309d..f09e513ce05b 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
@@ -36,6 +36,7 @@
 
 enum tc_port;
 struct drm_i915_private;
+struct drm_printer;
 struct intel_atomic_state;
 struct intel_crtc;
 struct intel_crtc_state;
@@ -180,18 +181,19 @@ enum icl_port_dpll_id {
 	ICL_PORT_DPLL_COUNT,
 };
 
-struct intel_dpll_hw_state {
-	/* i9xx, pch plls */
+struct i9xx_dpll_hw_state {
 	u32 dpll;
 	u32 dpll_md;
 	u32 fp0;
 	u32 fp1;
+};
 
-	/* hsw, bdw */
+struct hsw_dpll_hw_state {
 	u32 wrpll;
 	u32 spll;
+};
 
-	/* skl */
+struct skl_dpll_hw_state {
 	/*
 	 * DPLL_CTRL1 has 6 bits for each each this DPLL. We store those in
 	 * lower part of ctrl1 and they get shifted into position when writing
@@ -201,20 +203,18 @@ struct intel_dpll_hw_state {
 	u32 ctrl1;
 	/* HDMI only, 0 when used for DP */
 	u32 cfgcr1, cfgcr2;
+};
 
-	/* icl */
-	u32 cfgcr0;
+struct bxt_dpll_hw_state {
+	u32 ebb0, ebb4, pll0, pll1, pll2, pll3, pll6, pll8, pll9, pll10, pcsdw12;
+};
+
+struct icl_dpll_hw_state {
+	u32 cfgcr0, cfgcr1;
 
 	/* tgl */
 	u32 div0;
 
-	/* bxt */
-	u32 ebb0, ebb4, pll0, pll1, pll2, pll3, pll6, pll8, pll9, pll10, pcsdw12;
-
-	/*
-	 * ICL uses the following, already defined:
-	 * u32 cfgcr0, cfgcr1;
-	 */
 	u32 mg_refclkin_ctl;
 	u32 mg_clktop2_coreclkctl1;
 	u32 mg_clktop2_hsclkctl;
@@ -229,6 +229,55 @@ struct intel_dpll_hw_state {
 	u32 mg_pll_tdc_coldst_bias_mask;
 };
 
+struct intel_mpllb_state {
+	u32 clock; /* in KHz */
+	u32 ref_control;
+	u32 mpllb_cp;
+	u32 mpllb_div;
+	u32 mpllb_div2;
+	u32 mpllb_fracn1;
+	u32 mpllb_fracn2;
+	u32 mpllb_sscen;
+	u32 mpllb_sscstep;
+};
+
+struct intel_c10pll_state {
+	u32 clock; /* in KHz */
+	u8 tx;
+	u8 cmn;
+	u8 pll[20];
+};
+
+struct intel_c20pll_state {
+	u32 clock; /* in kHz */
+	u16 tx[3];
+	u16 cmn[4];
+	union {
+		u16 mplla[10];
+		u16 mpllb[11];
+	};
+};
+
+struct intel_cx0pll_state {
+	union {
+		struct intel_c10pll_state c10;
+		struct intel_c20pll_state c20;
+	};
+	bool ssc_enabled;
+};
+
+struct intel_dpll_hw_state {
+	union {
+		struct i9xx_dpll_hw_state i9xx;
+		struct hsw_dpll_hw_state hsw;
+		struct skl_dpll_hw_state skl;
+		struct bxt_dpll_hw_state bxt;
+		struct icl_dpll_hw_state icl;
+		struct intel_mpllb_state mpllb;
+		struct intel_cx0pll_state cx0pll;
+	};
+};
+
 /**
  * struct intel_shared_dpll_state - hold the DPLL atomic state
  *
@@ -364,10 +413,10 @@ void intel_update_active_dpll(struct intel_atomic_state *state,
 			      struct intel_encoder *encoder);
 int intel_dpll_get_freq(struct drm_i915_private *i915,
 			const struct intel_shared_dpll *pll,
-			const struct intel_dpll_hw_state *pll_state);
+			const struct intel_dpll_hw_state *dpll_hw_state);
 bool intel_dpll_get_hw_state(struct drm_i915_private *i915,
 			     struct intel_shared_dpll *pll,
-			     struct intel_dpll_hw_state *hw_state);
+			     struct intel_dpll_hw_state *dpll_hw_state);
 void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state);
 void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state);
 void intel_shared_dpll_swap_state(struct intel_atomic_state *state);
@@ -377,7 +426,8 @@ void intel_dpll_readout_hw_state(struct drm_i915_private *i915);
 void intel_dpll_sanitize_state(struct drm_i915_private *i915);
 
 void intel_dpll_dump_hw_state(struct drm_i915_private *i915,
-			      const struct intel_dpll_hw_state *hw_state);
+			      struct drm_printer *p,
+			      const struct intel_dpll_hw_state *dpll_hw_state);
 bool intel_dpll_compare_hw_state(struct drm_i915_private *i915,
 				 const struct intel_dpll_hw_state *a,
 				 const struct intel_dpll_hw_state *b);
diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c
index e4515bf92038..4baaa92ceaec 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.c
+++ b/drivers/gpu/drm/i915/display/intel_dsb.c
@@ -343,12 +343,13 @@ static int intel_dsb_dewake_scanline(const struct intel_crtc_state *crtc_state)
 static u32 dsb_chicken(struct intel_crtc *crtc)
 {
 	if (crtc->mode_flags & I915_MODE_FLAG_VRR)
-		return DSB_CTRL_WAIT_SAFE_WINDOW |
+		return DSB_SKIP_WAITS_EN |
+			DSB_CTRL_WAIT_SAFE_WINDOW |
 			DSB_CTRL_NO_WAIT_VBLANK |
 			DSB_INST_WAIT_SAFE_WINDOW |
 			DSB_INST_NO_WAIT_VBLANK;
 	else
-		return 0;
+		return DSB_SKIP_WAITS_EN;
 }
 
 static void _intel_dsb_commit(struct intel_dsb *dsb, u32 ctrl,
diff --git a/drivers/gpu/drm/i915/display/intel_dsi.c b/drivers/gpu/drm/i915/display/intel_dsi.c
index d3cf6a652221..bd5888ce4852 100644
--- a/drivers/gpu/drm/i915/display/intel_dsi.c
+++ b/drivers/gpu/drm/i915/display/intel_dsi.c
@@ -64,14 +64,11 @@ enum drm_mode_status intel_dsi_mode_valid(struct drm_connector *connector,
 	struct intel_connector *intel_connector = to_intel_connector(connector);
 	const struct drm_display_mode *fixed_mode =
 		intel_panel_fixed_mode(intel_connector, mode);
-	int max_dotclk = to_i915(connector->dev)->max_dotclk_freq;
+	int max_dotclk = to_i915(connector->dev)->display.cdclk.max_dotclk_freq;
 	enum drm_mode_status status;
 
 	drm_dbg_kms(&dev_priv->drm, "\n");
 
-	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
-		return MODE_NO_DBLESCAN;
-
 	status = intel_panel_mode_valid(intel_connector, mode);
 	if (status != MODE_OK)
 		return status;
diff --git a/drivers/gpu/drm/i915/display/intel_dvo.c b/drivers/gpu/drm/i915/display/intel_dvo.c
index c076da75b066..1840f5b59229 100644
--- a/drivers/gpu/drm/i915/display/intel_dvo.c
+++ b/drivers/gpu/drm/i915/display/intel_dvo.c
@@ -223,7 +223,7 @@ intel_dvo_mode_valid(struct drm_connector *_connector,
 	struct intel_dvo *intel_dvo = intel_attached_dvo(connector);
 	const struct drm_display_mode *fixed_mode =
 		intel_panel_fixed_mode(connector, mode);
-	int max_dotclk = to_i915(connector->base.dev)->max_dotclk_freq;
+	int max_dotclk = to_i915(connector->base.dev)->display.cdclk.max_dotclk_freq;
 	int target_clock = mode->clock;
 	enum drm_mode_status status;
 
@@ -231,9 +231,6 @@ intel_dvo_mode_valid(struct drm_connector *_connector,
 	if (status != MODE_OK)
 		return status;
 
-	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
-		return MODE_NO_DBLESCAN;
-
 	/* XXX: Validate clock range */
 
 	if (fixed_mode) {
diff --git a/drivers/gpu/drm/i915/display/intel_fb.c b/drivers/gpu/drm/i915/display/intel_fb.c
index 3ea6470d6d92..86b443433e8b 100644
--- a/drivers/gpu/drm/i915/display/intel_fb.c
+++ b/drivers/gpu/drm/i915/display/intel_fb.c
@@ -1106,7 +1106,7 @@ static int intel_fb_offset_to_xy(int *x, int *y,
 {
 	struct drm_i915_private *i915 = to_i915(fb->dev);
 	unsigned int height;
-	u32 alignment;
+	u32 alignment, unused;
 
 	if (DISPLAY_VER(i915) >= 12 &&
 	    !intel_fb_needs_pot_stride_remap(to_intel_framebuffer(fb)) &&
@@ -1128,8 +1128,8 @@ static int intel_fb_offset_to_xy(int *x, int *y,
 	height = ALIGN(height, intel_tile_height(fb, color_plane));
 
 	/* Catch potential overflows early */
-	if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
-			    fb->offsets[color_plane])) {
+	if (check_add_overflow(mul_u32_u32(height, fb->pitches[color_plane]),
+			       fb->offsets[color_plane], &unused)) {
 		drm_dbg_kms(&i915->drm,
 			    "Bad offset 0x%08x or pitch %d for color plane %d\n",
 			    fb->offsets[color_plane], fb->pitches[color_plane],
diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c
index b453fcbd67da..151dcd0c45b6 100644
--- a/drivers/gpu/drm/i915/display/intel_fbc.c
+++ b/drivers/gpu/drm/i915/display/intel_fbc.c
@@ -54,6 +54,7 @@
 #include "intel_display_trace.h"
 #include "intel_display_types.h"
 #include "intel_fbc.h"
+#include "intel_fbc_regs.h"
 #include "intel_frontbuffer.h"
 
 #define for_each_fbc_id(__dev_priv, __fbc_id) \
@@ -826,10 +827,36 @@ static void intel_fbc_program_cfb(struct intel_fbc *fbc)
 
 static void intel_fbc_program_workarounds(struct intel_fbc *fbc)
 {
+	struct drm_i915_private *i915 = fbc->i915;
+
+	if (IS_SKYLAKE(i915) || IS_BROXTON(i915)) {
+		/*
+		 * WaFbcHighMemBwCorruptionAvoidance:skl,bxt
+		 * Display WA #0883: skl,bxt
+		 */
+		intel_de_rmw(i915, ILK_DPFC_CHICKEN(fbc->id),
+			     0, DPFC_DISABLE_DUMMY0);
+	}
+
+	if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) ||
+	    IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) {
+		/*
+		 * WaFbcNukeOnHostModify:skl,kbl,cfl
+		 * Display WA #0873: skl,kbl,cfl
+		 */
+		intel_de_rmw(i915, ILK_DPFC_CHICKEN(fbc->id),
+			     0, DPFC_NUKE_ON_ANY_MODIFICATION);
+	}
+
+	/* Wa_1409120013:icl,jsl,tgl,dg1 */
+	if (IS_DISPLAY_VER(i915, 11, 12))
+		intel_de_rmw(i915, ILK_DPFC_CHICKEN(fbc->id),
+			     0, DPFC_CHICKEN_COMP_DUMMY_PIXEL);
+
 	/* Wa_22014263786:icl,jsl,tgl,dg1,rkl,adls,adlp,mtl */
-	if (DISPLAY_VER(fbc->i915) >= 11 && !IS_DG2(fbc->i915))
-		intel_de_rmw(fbc->i915, ILK_DPFC_CHICKEN(fbc->id), 0,
-			     DPFC_CHICKEN_FORCE_SLB_INVALIDATION);
+	if (DISPLAY_VER(i915) >= 11 && !IS_DG2(i915))
+		intel_de_rmw(i915, ILK_DPFC_CHICKEN(fbc->id),
+			     0, DPFC_CHICKEN_FORCE_SLB_INVALIDATION);
 }
 
 static void __intel_fbc_cleanup_cfb(struct intel_fbc *fbc)
diff --git a/drivers/gpu/drm/i915/display/intel_fbc_regs.h b/drivers/gpu/drm/i915/display/intel_fbc_regs.h
new file mode 100644
index 000000000000..ae0699c3c2fe
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_fbc_regs.h
@@ -0,0 +1,120 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2024 Intel Corporation */
+
+#ifndef __INTEL_FBC_REGS__
+#define __INTEL_FBC_REGS__
+
+#include "intel_display_reg_defs.h"
+
+#define FBC_CFB_BASE		_MMIO(0x3200) /* 4k page aligned */
+#define FBC_LL_BASE		_MMIO(0x3204) /* 4k page aligned */
+#define FBC_CONTROL		_MMIO(0x3208)
+#define   FBC_CTL_EN			REG_BIT(31)
+#define   FBC_CTL_PERIODIC		REG_BIT(30)
+#define   FBC_CTL_INTERVAL_MASK		REG_GENMASK(29, 16)
+#define   FBC_CTL_INTERVAL(x)		REG_FIELD_PREP(FBC_CTL_INTERVAL_MASK, (x))
+#define   FBC_CTL_STOP_ON_MOD		REG_BIT(15)
+#define   FBC_CTL_UNCOMPRESSIBLE	REG_BIT(14) /* i915+ */
+#define   FBC_CTL_C3_IDLE		REG_BIT(13) /* i945gm only */
+#define   FBC_CTL_STRIDE_MASK		REG_GENMASK(12, 5)
+#define   FBC_CTL_STRIDE(x)		REG_FIELD_PREP(FBC_CTL_STRIDE_MASK, (x))
+#define   FBC_CTL_FENCENO_MASK		REG_GENMASK(3, 0)
+#define   FBC_CTL_FENCENO(x)		REG_FIELD_PREP(FBC_CTL_FENCENO_MASK, (x))
+#define FBC_COMMAND		_MMIO(0x320c)
+#define   FBC_CMD_COMPRESS		REG_BIT(0)
+#define FBC_STATUS		_MMIO(0x3210)
+#define   FBC_STAT_COMPRESSING		REG_BIT(31)
+#define   FBC_STAT_COMPRESSED		REG_BIT(30)
+#define   FBC_STAT_MODIFIED		REG_BIT(29)
+#define   FBC_STAT_CURRENT_LINE_MASK	REG_GENMASK(10, 0)
+#define FBC_CONTROL2		_MMIO(0x3214) /* i965gm only */
+#define   FBC_CTL_FENCE_DBL		REG_BIT(4)
+#define   FBC_CTL_IDLE_MASK		REG_GENMASK(3, 2)
+#define   FBC_CTL_IDLE_IMM		REG_FIELD_PREP(FBC_CTL_IDLE_MASK, 0)
+#define   FBC_CTL_IDLE_FULL		REG_FIELD_PREP(FBC_CTL_IDLE_MASK, 1)
+#define   FBC_CTL_IDLE_LINE		REG_FIELD_PREP(FBC_CTL_IDLE_MASK, 2)
+#define   FBC_CTL_IDLE_DEBUG		REG_FIELD_PREP(FBC_CTL_IDLE_MASK, 3)
+#define   FBC_CTL_CPU_FENCE_EN		REG_BIT(1)
+#define   FBC_CTL_PLANE_MASK		REG_GENMASK(1, 0)
+#define   FBC_CTL_PLANE(i9xx_plane)	REG_FIELD_PREP(FBC_CTL_PLANE_MASK, (i9xx_plane))
+#define FBC_FENCE_OFF		_MMIO(0x3218)  /* i965gm only, BSpec typo has 321Bh */
+#define FBC_MOD_NUM		_MMIO(0x3220)  /* i965gm only */
+#define   FBC_MOD_NUM_MASK		REG_GENMASK(31, 1)
+#define   FBC_MOD_NUM_VALID		REG_BIT(0)
+#define FBC_TAG(i)		_MMIO(0x3300 + (i) * 4) /* 49 reisters */
+#define   FBC_TAG_MASK			REG_GENMASK(1, 0) /* 16 tags per register */
+#define   FBC_TAG_MODIFIED		REG_FIELD_PREP(FBC_TAG_MASK, 0)
+#define   FBC_TAG_UNCOMPRESSED		REG_FIELD_PREP(FBC_TAG_MASK, 1)
+#define   FBC_TAG_UNCOMPRESSIBLE	REG_FIELD_PREP(FBC_TAG_MASK, 2)
+#define   FBC_TAG_COMPRESSED		REG_FIELD_PREP(FBC_TAG_MASK, 3)
+
+#define FBC_LL_SIZE		(1536)
+
+#define DPFC_CB_BASE			_MMIO(0x3200)
+#define ILK_DPFC_CB_BASE(fbc_id)	_MMIO_PIPE((fbc_id), 0x43200, 0x43240)
+#define DPFC_CONTROL			_MMIO(0x3208)
+#define ILK_DPFC_CONTROL(fbc_id)	_MMIO_PIPE((fbc_id), 0x43208, 0x43248)
+#define   DPFC_CTL_EN				REG_BIT(31)
+#define   DPFC_CTL_PLANE_MASK_G4X		REG_BIT(30) /* g4x-snb */
+#define   DPFC_CTL_PLANE_G4X(i9xx_plane)	REG_FIELD_PREP(DPFC_CTL_PLANE_MASK_G4X, (i9xx_plane))
+#define   DPFC_CTL_FENCE_EN_G4X			REG_BIT(29) /* g4x-snb */
+#define   DPFC_CTL_PLANE_MASK_IVB		REG_GENMASK(30, 29) /* ivb only */
+#define   DPFC_CTL_PLANE_IVB(i9xx_plane)	REG_FIELD_PREP(DPFC_CTL_PLANE_MASK_IVB, (i9xx_plane))
+#define   DPFC_CTL_FENCE_EN_IVB			REG_BIT(28) /* ivb+ */
+#define   DPFC_CTL_PERSISTENT_MODE		REG_BIT(25) /* g4x-snb */
+#define   DPFC_CTL_PLANE_BINDING_MASK		REG_GENMASK(12, 11) /* lnl+ */
+#define   DPFC_CTL_PLANE_BINDING(plane_id)	REG_FIELD_PREP(DPFC_CTL_PLANE_BINDING_MASK, (plane_id))
+#define   DPFC_CTL_FALSE_COLOR			REG_BIT(10) /* ivb+ */
+#define   DPFC_CTL_SR_EN			REG_BIT(10) /* g4x only */
+#define   DPFC_CTL_SR_EXIT_DIS			REG_BIT(9) /* g4x only */
+#define   DPFC_CTL_LIMIT_MASK			REG_GENMASK(7, 6)
+#define   DPFC_CTL_LIMIT_1X			REG_FIELD_PREP(DPFC_CTL_LIMIT_MASK, 0)
+#define   DPFC_CTL_LIMIT_2X			REG_FIELD_PREP(DPFC_CTL_LIMIT_MASK, 1)
+#define   DPFC_CTL_LIMIT_4X			REG_FIELD_PREP(DPFC_CTL_LIMIT_MASK, 2)
+#define   DPFC_CTL_FENCENO_MASK			REG_GENMASK(3, 0)
+#define   DPFC_CTL_FENCENO(fence)		REG_FIELD_PREP(DPFC_CTL_FENCENO_MASK, (fence))
+#define DPFC_RECOMP_CTL			_MMIO(0x320c)
+#define ILK_DPFC_RECOMP_CTL(fbc_id)	_MMIO_PIPE((fbc_id), 0x4320c, 0x4324c)
+#define   DPFC_RECOMP_STALL_EN			REG_BIT(27)
+#define   DPFC_RECOMP_STALL_WM_MASK		REG_GENMASK(26, 16)
+#define   DPFC_RECOMP_TIMER_COUNT_MASK		REG_GENMASK(5, 0)
+#define DPFC_STATUS			_MMIO(0x3210)
+#define ILK_DPFC_STATUS(fbc_id)		_MMIO_PIPE((fbc_id), 0x43210, 0x43250)
+#define   DPFC_INVAL_SEG_MASK			REG_GENMASK(26, 16)
+#define   DPFC_COMP_SEG_MASK			REG_GENMASK(10, 0)
+#define DPFC_STATUS2			_MMIO(0x3214)
+#define ILK_DPFC_STATUS2(fbc_id)	_MMIO_PIPE((fbc_id), 0x43214, 0x43254)
+#define   DPFC_COMP_SEG_MASK_IVB		REG_GENMASK(11, 0)
+#define DPFC_FENCE_YOFF			_MMIO(0x3218)
+#define ILK_DPFC_FENCE_YOFF(fbc_id)	_MMIO_PIPE((fbc_id), 0x43218, 0x43258)
+#define DPFC_CHICKEN			_MMIO(0x3224)
+#define ILK_DPFC_CHICKEN(fbc_id)	_MMIO_PIPE((fbc_id), 0x43224, 0x43264)
+#define   DPFC_HT_MODIFY			REG_BIT(31) /* pre-ivb */
+#define   DPFC_NUKE_ON_ANY_MODIFICATION		REG_BIT(23) /* bdw+ */
+#define   DPFC_CHICKEN_COMP_DUMMY_PIXEL		REG_BIT(14) /* glk+ */
+#define   DPFC_CHICKEN_FORCE_SLB_INVALIDATION	REG_BIT(13) /* icl+ */
+#define   DPFC_DISABLE_DUMMY0			REG_BIT(8) /* ivb+ */
+
+#define GLK_FBC_STRIDE(fbc_id)	_MMIO_PIPE((fbc_id), 0x43228, 0x43268)
+#define   FBC_STRIDE_OVERRIDE	REG_BIT(15)
+#define   FBC_STRIDE_MASK	REG_GENMASK(14, 0)
+#define   FBC_STRIDE(x)		REG_FIELD_PREP(FBC_STRIDE_MASK, (x))
+
+#define ILK_FBC_RT_BASE		_MMIO(0x2128)
+#define   ILK_FBC_RT_VALID	REG_BIT(0)
+#define   SNB_FBC_FRONT_BUFFER	REG_BIT(1)
+
+#define SNB_DPFC_CTL_SA		_MMIO(0x100100)
+#define   SNB_DPFC_FENCE_EN		REG_BIT(29)
+#define   SNB_DPFC_FENCENO_MASK		REG_GENMASK(4, 0)
+#define   SNB_DPFC_FENCENO(fence)	REG_FIELD_PREP(SNB_DPFC_FENCENO_MASK, (fence))
+#define SNB_DPFC_CPU_FENCE_OFFSET	_MMIO(0x100104)
+
+#define IVB_FBC_RT_BASE			_MMIO(0x7020)
+#define IVB_FBC_RT_BASE_UPPER		_MMIO(0x7024)
+
+#define MSG_FBC_REND_STATE(fbc_id)	_MMIO_PIPE((fbc_id), 0x50380, 0x50384)
+#define   FBC_REND_NUKE			REG_BIT(2)
+#define   FBC_REND_CACHE_CLEAN		REG_BIT(1)
+
+#endif /* __INTEL_FBC_REGS__ */
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c
index 99894a855ef0..bda702c2cab8 100644
--- a/drivers/gpu/drm/i915/display/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/display/intel_fbdev.c
@@ -24,7 +24,6 @@
  *     David Airlie
  */
 
-#include <linux/async.h>
 #include <linux/console.h>
 #include <linux/delay.h>
 #include <linux/errno.h>
@@ -39,6 +38,7 @@
 #include <linux/vga_switcheroo.h>
 
 #include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -58,7 +58,6 @@ struct intel_fbdev {
 	struct intel_framebuffer *fb;
 	struct i915_vma *vma;
 	unsigned long vma_flags;
-	async_cookie_t cookie;
 	int preferred_bpp;
 
 	/* Whether or not fbdev hpd processing is temporarily suspended */
@@ -135,6 +134,29 @@ static int intel_fbdev_mmap(struct fb_info *info, struct vm_area_struct *vma)
 	return i915_gem_fb_mmap(obj, vma);
 }
 
+static void intel_fbdev_fb_destroy(struct fb_info *info)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+	struct intel_fbdev *ifbdev = container_of(fb_helper, struct intel_fbdev, helper);
+
+	drm_fb_helper_fini(&ifbdev->helper);
+
+	/*
+	 * We rely on the object-free to release the VMA pinning for
+	 * the info->screen_base mmaping. Leaking the VMA is simpler than
+	 * trying to rectify all the possible error paths leading here.
+	 */
+	intel_unpin_fb_vma(ifbdev->vma, ifbdev->vma_flags);
+	drm_framebuffer_remove(&ifbdev->fb->base);
+
+	drm_client_release(&fb_helper->client);
+	drm_fb_helper_unprepare(&ifbdev->helper);
+	kfree(ifbdev);
+}
+
+__diag_push();
+__diag_ignore_all("-Woverride-init", "Allow field initialization overrides for fb ops");
+
 static const struct fb_ops intelfb_ops = {
 	.owner = THIS_MODULE,
 	__FB_DEFAULT_DEFERRED_OPS_RDWR(intel_fbdev),
@@ -144,8 +166,11 @@ static const struct fb_ops intelfb_ops = {
 	.fb_pan_display = intel_fbdev_pan_display,
 	__FB_DEFAULT_DEFERRED_OPS_DRAW(intel_fbdev),
 	.fb_mmap = intel_fbdev_mmap,
+	.fb_destroy = intel_fbdev_fb_destroy,
 };
 
+__diag_pop();
+
 static int intelfb_create(struct drm_fb_helper *helper,
 			  struct drm_fb_helper_surface_size *sizes)
 {
@@ -153,7 +178,6 @@ static int intelfb_create(struct drm_fb_helper *helper,
 	struct intel_framebuffer *intel_fb = ifbdev->fb;
 	struct drm_device *dev = helper->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 	const struct i915_gtt_view view = {
 		.type = I915_GTT_VIEW_NORMAL,
 	};
@@ -245,7 +269,7 @@ static int intelfb_create(struct drm_fb_helper *helper,
 	ifbdev->vma_flags = flags;
 
 	intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
-	vga_switcheroo_client_fb_set(pdev, info);
+
 	return 0;
 
 out_unpin:
@@ -271,25 +295,6 @@ static const struct drm_fb_helper_funcs intel_fb_helper_funcs = {
 	.fb_dirty = intelfb_dirty,
 };
 
-static void intel_fbdev_destroy(struct intel_fbdev *ifbdev)
-{
-	/* We rely on the object-free to release the VMA pinning for
-	 * the info->screen_base mmaping. Leaking the VMA is simpler than
-	 * trying to rectify all the possible error paths leading here.
-	 */
-
-	drm_fb_helper_fini(&ifbdev->helper);
-
-	if (ifbdev->vma)
-		intel_unpin_fb_vma(ifbdev->vma, ifbdev->vma_flags);
-
-	if (ifbdev->fb)
-		drm_framebuffer_remove(&ifbdev->fb->base);
-
-	drm_fb_helper_unprepare(&ifbdev->helper);
-	kfree(ifbdev);
-}
-
 /*
  * Build an intel_fbdev struct using a BIOS allocated framebuffer, if possible.
  * The core display code will have read out the current plane configuration,
@@ -453,93 +458,6 @@ static void intel_fbdev_suspend_worker(struct work_struct *work)
 				true);
 }
 
-int intel_fbdev_init(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_fbdev *ifbdev;
-	int ret;
-
-	if (drm_WARN_ON(dev, !HAS_DISPLAY(dev_priv)))
-		return -ENODEV;
-
-	ifbdev = kzalloc(sizeof(struct intel_fbdev), GFP_KERNEL);
-	if (ifbdev == NULL)
-		return -ENOMEM;
-
-	mutex_init(&ifbdev->hpd_lock);
-	drm_fb_helper_prepare(dev, &ifbdev->helper, 32, &intel_fb_helper_funcs);
-
-	if (intel_fbdev_init_bios(dev, ifbdev))
-		ifbdev->helper.preferred_bpp = ifbdev->preferred_bpp;
-	else
-		ifbdev->preferred_bpp = ifbdev->helper.preferred_bpp;
-
-	ret = drm_fb_helper_init(dev, &ifbdev->helper);
-	if (ret) {
-		kfree(ifbdev);
-		return ret;
-	}
-
-	dev_priv->display.fbdev.fbdev = ifbdev;
-	INIT_WORK(&dev_priv->display.fbdev.suspend_work, intel_fbdev_suspend_worker);
-
-	return 0;
-}
-
-static void intel_fbdev_initial_config(void *data, async_cookie_t cookie)
-{
-	struct intel_fbdev *ifbdev = data;
-
-	/* Due to peculiar init order wrt to hpd handling this is separate. */
-	if (drm_fb_helper_initial_config(&ifbdev->helper))
-		intel_fbdev_unregister(to_i915(ifbdev->helper.dev));
-}
-
-void intel_fbdev_initial_config_async(struct drm_i915_private *dev_priv)
-{
-	struct intel_fbdev *ifbdev = dev_priv->display.fbdev.fbdev;
-
-	if (!ifbdev)
-		return;
-
-	ifbdev->cookie = async_schedule(intel_fbdev_initial_config, ifbdev);
-}
-
-static void intel_fbdev_sync(struct intel_fbdev *ifbdev)
-{
-	if (!ifbdev->cookie)
-		return;
-
-	/* Only serialises with all preceding async calls, hence +1 */
-	async_synchronize_cookie(ifbdev->cookie + 1);
-	ifbdev->cookie = 0;
-}
-
-void intel_fbdev_unregister(struct drm_i915_private *dev_priv)
-{
-	struct intel_fbdev *ifbdev = dev_priv->display.fbdev.fbdev;
-
-	if (!ifbdev)
-		return;
-
-	intel_fbdev_set_suspend(&dev_priv->drm, FBINFO_STATE_SUSPENDED, true);
-
-	if (!current_is_async())
-		intel_fbdev_sync(ifbdev);
-
-	drm_fb_helper_unregister_info(&ifbdev->helper);
-}
-
-void intel_fbdev_fini(struct drm_i915_private *dev_priv)
-{
-	struct intel_fbdev *ifbdev = fetch_and_zero(&dev_priv->display.fbdev.fbdev);
-
-	if (!ifbdev)
-		return;
-
-	intel_fbdev_destroy(ifbdev);
-}
-
 /* Suspends/resumes fbdev processing of incoming HPD events. When resuming HPD
  * processing, fbdev will perform a full connector reprobe if a hotplug event
  * was received while HPD was suspended.
@@ -622,15 +540,13 @@ set_suspend:
 	intel_fbdev_hpd_set_suspend(dev_priv, state);
 }
 
-void intel_fbdev_output_poll_changed(struct drm_device *dev)
+static int intel_fbdev_output_poll_changed(struct drm_device *dev)
 {
 	struct intel_fbdev *ifbdev = to_i915(dev)->display.fbdev.fbdev;
 	bool send_hpd;
 
 	if (!ifbdev)
-		return;
-
-	intel_fbdev_sync(ifbdev);
+		return -EINVAL;
 
 	mutex_lock(&ifbdev->hpd_lock);
 	send_hpd = !ifbdev->hpd_suspended;
@@ -639,21 +555,137 @@ void intel_fbdev_output_poll_changed(struct drm_device *dev)
 
 	if (send_hpd && (ifbdev->vma || ifbdev->helper.deferred_setup))
 		drm_fb_helper_hotplug_event(&ifbdev->helper);
+
+	return 0;
 }
 
-void intel_fbdev_restore_mode(struct drm_i915_private *dev_priv)
+static int intel_fbdev_restore_mode(struct drm_i915_private *dev_priv)
 {
 	struct intel_fbdev *ifbdev = dev_priv->display.fbdev.fbdev;
+	int ret;
 
 	if (!ifbdev)
-		return;
+		return -EINVAL;
 
-	intel_fbdev_sync(ifbdev);
 	if (!ifbdev->vma)
+		return -ENOMEM;
+
+	ret = drm_fb_helper_restore_fbdev_mode_unlocked(&ifbdev->helper);
+	if (ret)
+		return ret;
+
+	intel_fbdev_invalidate(ifbdev);
+
+	return 0;
+}
+
+/*
+ * Fbdev client and struct drm_client_funcs
+ */
+
+static void intel_fbdev_client_unregister(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+	struct drm_device *dev = fb_helper->dev;
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
+
+	if (fb_helper->info) {
+		vga_switcheroo_client_fb_set(pdev, NULL);
+		drm_fb_helper_unregister_info(fb_helper);
+	} else {
+		drm_fb_helper_unprepare(fb_helper);
+		drm_client_release(&fb_helper->client);
+		kfree(fb_helper);
+	}
+}
+
+static int intel_fbdev_client_restore(struct drm_client_dev *client)
+{
+	struct drm_i915_private *dev_priv = to_i915(client->dev);
+	int ret;
+
+	ret = intel_fbdev_restore_mode(dev_priv);
+	if (ret)
+		return ret;
+
+	vga_switcheroo_process_delayed_switch();
+
+	return 0;
+}
+
+static int intel_fbdev_client_hotplug(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+	struct drm_device *dev = client->dev;
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
+	int ret;
+
+	if (dev->fb_helper)
+		return intel_fbdev_output_poll_changed(dev);
+
+	ret = drm_fb_helper_init(dev, fb_helper);
+	if (ret)
+		goto err_drm_err;
+
+	ret = drm_fb_helper_initial_config(fb_helper);
+	if (ret)
+		goto err_drm_fb_helper_fini;
+
+	vga_switcheroo_client_fb_set(pdev, fb_helper->info);
+
+	return 0;
+
+err_drm_fb_helper_fini:
+	drm_fb_helper_fini(fb_helper);
+err_drm_err:
+	drm_err(dev, "Failed to setup i915 fbdev emulation (ret=%d)\n", ret);
+	return ret;
+}
+
+static const struct drm_client_funcs intel_fbdev_client_funcs = {
+	.owner		= THIS_MODULE,
+	.unregister	= intel_fbdev_client_unregister,
+	.restore	= intel_fbdev_client_restore,
+	.hotplug	= intel_fbdev_client_hotplug,
+};
+
+void intel_fbdev_setup(struct drm_i915_private *i915)
+{
+	struct drm_device *dev = &i915->drm;
+	struct intel_fbdev *ifbdev;
+	int ret;
+
+	if (!HAS_DISPLAY(i915))
 		return;
 
-	if (drm_fb_helper_restore_fbdev_mode_unlocked(&ifbdev->helper) == 0)
-		intel_fbdev_invalidate(ifbdev);
+	ifbdev = kzalloc(sizeof(*ifbdev), GFP_KERNEL);
+	if (!ifbdev)
+		return;
+	drm_fb_helper_prepare(dev, &ifbdev->helper, 32, &intel_fb_helper_funcs);
+
+	i915->display.fbdev.fbdev = ifbdev;
+	INIT_WORK(&i915->display.fbdev.suspend_work, intel_fbdev_suspend_worker);
+	mutex_init(&ifbdev->hpd_lock);
+	if (intel_fbdev_init_bios(dev, ifbdev))
+		ifbdev->helper.preferred_bpp = ifbdev->preferred_bpp;
+	else
+		ifbdev->preferred_bpp = ifbdev->helper.preferred_bpp;
+
+	ret = drm_client_init(dev, &ifbdev->helper.client, "intel-fbdev",
+			      &intel_fbdev_client_funcs);
+	if (ret) {
+		drm_err(dev, "Failed to register client: %d\n", ret);
+		goto err_drm_fb_helper_unprepare;
+	}
+
+	drm_client_register(&ifbdev->helper.client);
+
+	return;
+
+err_drm_fb_helper_unprepare:
+	drm_fb_helper_unprepare(&ifbdev->helper);
+	mutex_destroy(&ifbdev->hpd_lock);
+	kfree(ifbdev);
 }
 
 struct intel_framebuffer *intel_fbdev_framebuffer(struct intel_fbdev *fbdev)
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.h b/drivers/gpu/drm/i915/display/intel_fbdev.h
index 04fd523a5023..08de2d5b3433 100644
--- a/drivers/gpu/drm/i915/display/intel_fbdev.h
+++ b/drivers/gpu/drm/i915/display/intel_fbdev.h
@@ -14,29 +14,11 @@ struct intel_fbdev;
 struct intel_framebuffer;
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
-int intel_fbdev_init(struct drm_device *dev);
-void intel_fbdev_initial_config_async(struct drm_i915_private *dev_priv);
-void intel_fbdev_unregister(struct drm_i915_private *dev_priv);
-void intel_fbdev_fini(struct drm_i915_private *dev_priv);
+void intel_fbdev_setup(struct drm_i915_private *dev_priv);
 void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous);
-void intel_fbdev_output_poll_changed(struct drm_device *dev);
-void intel_fbdev_restore_mode(struct drm_i915_private *dev_priv);
 struct intel_framebuffer *intel_fbdev_framebuffer(struct intel_fbdev *fbdev);
 #else
-static inline int intel_fbdev_init(struct drm_device *dev)
-{
-	return 0;
-}
-
-static inline void intel_fbdev_initial_config_async(struct drm_i915_private *dev_priv)
-{
-}
-
-static inline void intel_fbdev_unregister(struct drm_i915_private *dev_priv)
-{
-}
-
-static inline void intel_fbdev_fini(struct drm_i915_private *dev_priv)
+static inline void intel_fbdev_setup(struct drm_i915_private *dev_priv)
 {
 }
 
@@ -44,13 +26,6 @@ static inline void intel_fbdev_set_suspend(struct drm_device *dev, int state, bo
 {
 }
 
-static inline void intel_fbdev_output_poll_changed(struct drm_device *dev)
-{
-}
-
-static inline void intel_fbdev_restore_mode(struct drm_i915_private *i915)
-{
-}
 static inline struct intel_framebuffer *intel_fbdev_framebuffer(struct intel_fbdev *fbdev)
 {
 	return NULL;
diff --git a/drivers/gpu/drm/i915/i915_fixed.h b/drivers/gpu/drm/i915/display/intel_fixed.h
index a327094de2bd..a327094de2bd 100644
--- a/drivers/gpu/drm/i915/i915_fixed.h
+++ b/drivers/gpu/drm/i915/display/intel_fixed.h
diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c
index d3e03ed5b79c..9c8e1e91ff1c 100644
--- a/drivers/gpu/drm/i915/display/intel_gmbus.c
+++ b/drivers/gpu/drm/i915/display/intel_gmbus.c
@@ -411,7 +411,7 @@ gmbus_wait_idle(struct drm_i915_private *i915)
 	add_wait_queue(&i915->display.gmbus.wait_queue, &wait);
 	intel_de_write_fw(i915, GMBUS4(i915), irq_enable);
 
-	ret = intel_de_wait_for_register_fw(i915, GMBUS2(i915), GMBUS_ACTIVE, 0, 10);
+	ret = intel_de_wait_fw(i915, GMBUS2(i915), GMBUS_ACTIVE, 0, 10);
 
 	intel_de_write_fw(i915, GMBUS4(i915), 0);
 	remove_wait_queue(&i915->display.gmbus.wait_queue, &wait);
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c
index 9edac27bab26..d5ed4c7dfbc0 100644
--- a/drivers/gpu/drm/i915/display/intel_hdcp.c
+++ b/drivers/gpu/drm/i915/display/intel_hdcp.c
@@ -369,9 +369,9 @@ static int intel_hdcp_load_keys(struct drm_i915_private *i915)
 	}
 
 	/* Wait for the keys to load (500us) */
-	ret = __intel_wait_for_register(&i915->uncore, HDCP_KEY_STATUS,
-					HDCP_KEY_LOAD_DONE, HDCP_KEY_LOAD_DONE,
-					10, 1, &val);
+	ret = intel_de_wait_custom(i915, HDCP_KEY_STATUS,
+				   HDCP_KEY_LOAD_DONE, HDCP_KEY_LOAD_DONE,
+				   10, 1, &val);
 	if (ret)
 		return ret;
 	else if (!(val & HDCP_KEY_LOAD_STATUS))
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp_gsc.c b/drivers/gpu/drm/i915/display/intel_hdcp_gsc.c
index 302bff75b06c..35823e1f65d6 100644
--- a/drivers/gpu/drm/i915/display/intel_hdcp_gsc.c
+++ b/drivers/gpu/drm/i915/display/intel_hdcp_gsc.c
@@ -13,6 +13,12 @@
 #include "intel_hdcp_gsc.h"
 #include "intel_hdcp_gsc_message.h"
 
+struct intel_hdcp_gsc_message {
+	struct i915_vma *vma;
+	void *hdcp_cmd_in;
+	void *hdcp_cmd_out;
+};
+
 bool intel_hdcp_gsc_cs_required(struct drm_i915_private *i915)
 {
 	return DISPLAY_VER(i915) >= 14;
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp_gsc.h b/drivers/gpu/drm/i915/display/intel_hdcp_gsc.h
index eba2057c5a9e..5f610df61cc9 100644
--- a/drivers/gpu/drm/i915/display/intel_hdcp_gsc.h
+++ b/drivers/gpu/drm/i915/display/intel_hdcp_gsc.h
@@ -10,12 +10,7 @@
 #include <linux/types.h>
 
 struct drm_i915_private;
-
-struct intel_hdcp_gsc_message {
-	struct i915_vma *vma;
-	void *hdcp_cmd_in;
-	void *hdcp_cmd_out;
-};
+struct intel_hdcp_gsc_message;
 
 bool intel_hdcp_gsc_cs_required(struct drm_i915_private *i915);
 ssize_t intel_hdcp_gsc_msg_send(struct drm_i915_private *i915, u8 *msg_in,
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c
index 90d2236fede3..5f6deceaf8ba 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.c
@@ -114,6 +114,8 @@ static u32 g4x_infoframe_enable(unsigned int type)
 		return VIDEO_DIP_ENABLE_GAMUT;
 	case DP_SDP_VSC:
 		return 0;
+	case DP_SDP_ADAPTIVE_SYNC:
+		return 0;
 	case HDMI_INFOFRAME_TYPE_AVI:
 		return VIDEO_DIP_ENABLE_AVI;
 	case HDMI_INFOFRAME_TYPE_SPD:
@@ -137,6 +139,8 @@ static u32 hsw_infoframe_enable(unsigned int type)
 		return VIDEO_DIP_ENABLE_GMP_HSW;
 	case DP_SDP_VSC:
 		return VIDEO_DIP_ENABLE_VSC_HSW;
+	case DP_SDP_ADAPTIVE_SYNC:
+		return VIDEO_DIP_ENABLE_AS_ADL;
 	case DP_SDP_PPS:
 		return VDIP_ENABLE_PPS;
 	case HDMI_INFOFRAME_TYPE_AVI:
@@ -164,6 +168,8 @@ hsw_dip_data_reg(struct drm_i915_private *dev_priv,
 		return HSW_TVIDEO_DIP_GMP_DATA(cpu_transcoder, i);
 	case DP_SDP_VSC:
 		return HSW_TVIDEO_DIP_VSC_DATA(cpu_transcoder, i);
+	case DP_SDP_ADAPTIVE_SYNC:
+		return ADL_TVIDEO_DIP_AS_SDP_DATA(cpu_transcoder, i);
 	case DP_SDP_PPS:
 		return ICL_VIDEO_DIP_PPS_DATA(cpu_transcoder, i);
 	case HDMI_INFOFRAME_TYPE_AVI:
@@ -186,6 +192,8 @@ static int hsw_dip_data_size(struct drm_i915_private *dev_priv,
 	switch (type) {
 	case DP_SDP_VSC:
 		return VIDEO_DIP_VSC_DATA_SIZE;
+	case DP_SDP_ADAPTIVE_SYNC:
+		return VIDEO_DIP_ASYNC_DATA_SIZE;
 	case DP_SDP_PPS:
 		return VIDEO_DIP_PPS_DATA_SIZE;
 	case HDMI_PACKET_TYPE_GAMUT_METADATA:
@@ -563,6 +571,9 @@ static u32 hsw_infoframes_enabled(struct intel_encoder *encoder,
 	if (DISPLAY_VER(dev_priv) >= 10)
 		mask |= VIDEO_DIP_ENABLE_DRM_GLK;
 
+	if (HAS_AS_SDP(dev_priv))
+		mask |= VIDEO_DIP_ENABLE_AS_ADL;
+
 	return val & mask;
 }
 
@@ -570,6 +581,7 @@ static const u8 infoframe_type_to_idx[] = {
 	HDMI_PACKET_TYPE_GENERAL_CONTROL,
 	HDMI_PACKET_TYPE_GAMUT_METADATA,
 	DP_SDP_VSC,
+	DP_SDP_ADAPTIVE_SYNC,
 	HDMI_INFOFRAME_TYPE_AVI,
 	HDMI_INFOFRAME_TYPE_SPD,
 	HDMI_INFOFRAME_TYPE_VENDOR,
@@ -1212,7 +1224,7 @@ static void hsw_set_infoframes(struct intel_encoder *encoder,
 	val &= ~(VIDEO_DIP_ENABLE_VSC_HSW | VIDEO_DIP_ENABLE_AVI_HSW |
 		 VIDEO_DIP_ENABLE_GCP_HSW | VIDEO_DIP_ENABLE_VS_HSW |
 		 VIDEO_DIP_ENABLE_GMP_HSW | VIDEO_DIP_ENABLE_SPD_HSW |
-		 VIDEO_DIP_ENABLE_DRM_GLK);
+		 VIDEO_DIP_ENABLE_DRM_GLK | VIDEO_DIP_ENABLE_AS_ADL);
 
 	if (!enable) {
 		intel_de_write(dev_priv, reg, val);
@@ -1832,7 +1844,7 @@ hdmi_port_clock_valid(struct intel_hdmi *hdmi,
 		      bool has_hdmi_sink)
 {
 	struct drm_i915_private *dev_priv = intel_hdmi_to_i915(hdmi);
-	enum phy phy = intel_port_to_phy(dev_priv, hdmi_to_dig_port(hdmi)->base.port);
+	struct intel_encoder *encoder = &hdmi_to_dig_port(hdmi)->base;
 
 	if (clock < 25000)
 		return MODE_CLOCK_LOW;
@@ -1854,11 +1866,11 @@ hdmi_port_clock_valid(struct intel_hdmi *hdmi,
 		return MODE_CLOCK_RANGE;
 
 	/* ICL+ combo PHY PLL can't generate 500-533.2 MHz */
-	if (intel_phy_is_combo(dev_priv, phy) && clock > 500000 && clock < 533200)
+	if (intel_encoder_is_combo(encoder) && clock > 500000 && clock < 533200)
 		return MODE_CLOCK_RANGE;
 
 	/* ICL+ TC PHY PLL can't generate 500-532.8 MHz */
-	if (intel_phy_is_tc(dev_priv, phy) && clock > 500000 && clock < 532800)
+	if (intel_encoder_is_tc(encoder) && clock > 500000 && clock < 532800)
 		return MODE_CLOCK_RANGE;
 
 	/*
@@ -1981,7 +1993,7 @@ intel_hdmi_mode_valid(struct drm_connector *connector,
 	struct drm_i915_private *dev_priv = intel_hdmi_to_i915(hdmi);
 	enum drm_mode_status status;
 	int clock = mode->clock;
-	int max_dotclk = to_i915(connector->dev)->max_dotclk_freq;
+	int max_dotclk = to_i915(connector->dev)->display.cdclk.max_dotclk_freq;
 	bool has_hdmi_sink = intel_has_hdmi_sink(hdmi, connector->state);
 	bool ycbcr_420_only;
 	enum intel_output_format sink_format;
@@ -2664,8 +2676,9 @@ bool intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder,
 		drm_scdc_set_scrambling(connector, scrambling);
 }
 
-static u8 chv_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
+static u8 chv_encoder_to_ddc_pin(struct intel_encoder *encoder)
 {
+	enum port port = encoder->port;
 	u8 ddc_pin;
 
 	switch (port) {
@@ -2686,8 +2699,9 @@ static u8 chv_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
 	return ddc_pin;
 }
 
-static u8 bxt_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
+static u8 bxt_encoder_to_ddc_pin(struct intel_encoder *encoder)
 {
+	enum port port = encoder->port;
 	u8 ddc_pin;
 
 	switch (port) {
@@ -2705,9 +2719,9 @@ static u8 bxt_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
 	return ddc_pin;
 }
 
-static u8 cnp_port_to_ddc_pin(struct drm_i915_private *dev_priv,
-			      enum port port)
+static u8 cnp_encoder_to_ddc_pin(struct intel_encoder *encoder)
 {
+	enum port port = encoder->port;
 	u8 ddc_pin;
 
 	switch (port) {
@@ -2731,22 +2745,23 @@ static u8 cnp_port_to_ddc_pin(struct drm_i915_private *dev_priv,
 	return ddc_pin;
 }
 
-static u8 icl_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
+static u8 icl_encoder_to_ddc_pin(struct intel_encoder *encoder)
 {
-	enum phy phy = intel_port_to_phy(dev_priv, port);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
 
-	if (intel_phy_is_combo(dev_priv, phy))
+	if (intel_encoder_is_combo(encoder))
 		return GMBUS_PIN_1_BXT + port;
-	else if (intel_phy_is_tc(dev_priv, phy))
-		return GMBUS_PIN_9_TC1_ICP + intel_port_to_tc(dev_priv, port);
+	else if (intel_encoder_is_tc(encoder))
+		return GMBUS_PIN_9_TC1_ICP + intel_encoder_to_tc(encoder);
 
 	drm_WARN(&dev_priv->drm, 1, "Unknown port:%c\n", port_name(port));
 	return GMBUS_PIN_2_BXT;
 }
 
-static u8 mcc_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
+static u8 mcc_encoder_to_ddc_pin(struct intel_encoder *encoder)
 {
-	enum phy phy = intel_port_to_phy(dev_priv, port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	u8 ddc_pin;
 
 	switch (phy) {
@@ -2767,11 +2782,12 @@ static u8 mcc_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
 	return ddc_pin;
 }
 
-static u8 rkl_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
+static u8 rkl_encoder_to_ddc_pin(struct intel_encoder *encoder)
 {
-	enum phy phy = intel_port_to_phy(dev_priv, port);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
-	WARN_ON(port == PORT_C);
+	WARN_ON(encoder->port == PORT_C);
 
 	/*
 	 * Pin mapping for RKL depends on which PCH is present.  With TGP, the
@@ -2785,11 +2801,12 @@ static u8 rkl_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
 	return GMBUS_PIN_1_BXT + phy;
 }
 
-static u8 gen9bc_tgp_port_to_ddc_pin(struct drm_i915_private *i915, enum port port)
+static u8 gen9bc_tgp_encoder_to_ddc_pin(struct intel_encoder *encoder)
 {
-	enum phy phy = intel_port_to_phy(i915, port);
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
-	drm_WARN_ON(&i915->drm, port == PORT_A);
+	drm_WARN_ON(&i915->drm, encoder->port == PORT_A);
 
 	/*
 	 * Pin mapping for GEN9 BC depends on which PCH is present.  With TGP,
@@ -2803,16 +2820,16 @@ static u8 gen9bc_tgp_port_to_ddc_pin(struct drm_i915_private *i915, enum port po
 	return GMBUS_PIN_1_BXT + phy;
 }
 
-static u8 dg1_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
+static u8 dg1_encoder_to_ddc_pin(struct intel_encoder *encoder)
 {
-	return intel_port_to_phy(dev_priv, port) + 1;
+	return intel_encoder_to_phy(encoder) + 1;
 }
 
-static u8 adls_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
+static u8 adls_encoder_to_ddc_pin(struct intel_encoder *encoder)
 {
-	enum phy phy = intel_port_to_phy(dev_priv, port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
-	WARN_ON(port == PORT_B || port == PORT_C);
+	WARN_ON(encoder->port == PORT_B || encoder->port == PORT_C);
 
 	/*
 	 * Pin mapping for ADL-S requires TC pins for all combo phy outputs
@@ -2824,9 +2841,9 @@ static u8 adls_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port
 	return GMBUS_PIN_9_TC1_ICP + phy - PHY_B;
 }
 
-static u8 g4x_port_to_ddc_pin(struct drm_i915_private *dev_priv,
-			      enum port port)
+static u8 g4x_encoder_to_ddc_pin(struct intel_encoder *encoder)
 {
+	enum port port = encoder->port;
 	u8 ddc_pin;
 
 	switch (port) {
@@ -2850,30 +2867,29 @@ static u8 g4x_port_to_ddc_pin(struct drm_i915_private *dev_priv,
 static u8 intel_hdmi_default_ddc_pin(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum port port = encoder->port;
 	u8 ddc_pin;
 
 	if (IS_ALDERLAKE_S(dev_priv))
-		ddc_pin = adls_port_to_ddc_pin(dev_priv, port);
+		ddc_pin = adls_encoder_to_ddc_pin(encoder);
 	else if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1)
-		ddc_pin = dg1_port_to_ddc_pin(dev_priv, port);
+		ddc_pin = dg1_encoder_to_ddc_pin(encoder);
 	else if (IS_ROCKETLAKE(dev_priv))
-		ddc_pin = rkl_port_to_ddc_pin(dev_priv, port);
+		ddc_pin = rkl_encoder_to_ddc_pin(encoder);
 	else if (DISPLAY_VER(dev_priv) == 9 && HAS_PCH_TGP(dev_priv))
-		ddc_pin = gen9bc_tgp_port_to_ddc_pin(dev_priv, port);
+		ddc_pin = gen9bc_tgp_encoder_to_ddc_pin(encoder);
 	else if ((IS_JASPERLAKE(dev_priv) || IS_ELKHARTLAKE(dev_priv)) &&
 		 HAS_PCH_TGP(dev_priv))
-		ddc_pin = mcc_port_to_ddc_pin(dev_priv, port);
+		ddc_pin = mcc_encoder_to_ddc_pin(encoder);
 	else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
-		ddc_pin = icl_port_to_ddc_pin(dev_priv, port);
+		ddc_pin = icl_encoder_to_ddc_pin(encoder);
 	else if (HAS_PCH_CNP(dev_priv))
-		ddc_pin = cnp_port_to_ddc_pin(dev_priv, port);
+		ddc_pin = cnp_encoder_to_ddc_pin(encoder);
 	else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
-		ddc_pin = bxt_port_to_ddc_pin(dev_priv, port);
+		ddc_pin = bxt_encoder_to_ddc_pin(encoder);
 	else if (IS_CHERRYVIEW(dev_priv))
-		ddc_pin = chv_port_to_ddc_pin(dev_priv, port);
+		ddc_pin = chv_encoder_to_ddc_pin(encoder);
 	else
-		ddc_pin = g4x_port_to_ddc_pin(dev_priv, port);
+		ddc_pin = g4x_encoder_to_ddc_pin(encoder);
 
 	return ddc_pin;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_hotplug_irq.c b/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
index 76076509f771..d270bb7b9462 100644
--- a/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
+++ b/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
@@ -1444,7 +1444,7 @@ void intel_hpd_enable_detection(struct intel_encoder *encoder)
 
 void intel_hpd_irq_setup(struct drm_i915_private *i915)
 {
-	if (i915->display_irqs_enabled && i915->display.funcs.hotplug)
+	if (i915->display.irq.display_irqs_enabled && i915->display.funcs.hotplug)
 		i915->display.funcs.hotplug->hpd_irq_setup(i915);
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_lpe_audio.c b/drivers/gpu/drm/i915/display/intel_lpe_audio.c
index 5863763de530..93e6cac9a4ed 100644
--- a/drivers/gpu/drm/i915/display/intel_lpe_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_lpe_audio.c
@@ -72,7 +72,7 @@
 
 #include "i915_drv.h"
 #include "i915_irq.h"
-#include "i915_reg.h"
+#include "intel_audio_regs.h"
 #include "intel_de.h"
 #include "intel_lpe_audio.h"
 #include "intel_pci_config.h"
diff --git a/drivers/gpu/drm/i915/display/intel_lvds.c b/drivers/gpu/drm/i915/display/intel_lvds.c
index 221f5c6c871b..8b8959073466 100644
--- a/drivers/gpu/drm/i915/display/intel_lvds.c
+++ b/drivers/gpu/drm/i915/display/intel_lvds.c
@@ -392,16 +392,13 @@ intel_lvds_mode_valid(struct drm_connector *_connector,
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	const struct drm_display_mode *fixed_mode =
 		intel_panel_fixed_mode(connector, mode);
-	int max_pixclk = to_i915(connector->base.dev)->max_dotclk_freq;
+	int max_pixclk = to_i915(connector->base.dev)->display.cdclk.max_dotclk_freq;
 	enum drm_mode_status status;
 
 	status = intel_cpu_transcoder_mode_valid(i915, mode);
 	if (status != MODE_OK)
 		return status;
 
-	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
-		return MODE_NO_DBLESCAN;
-
 	status = intel_panel_mode_valid(connector, mode);
 	if (status != MODE_OK)
 		return status;
diff --git a/drivers/gpu/drm/i915/display/intel_opregion.c b/drivers/gpu/drm/i915/display/intel_opregion.c
index fcbb083318a7..68bd5101ec89 100644
--- a/drivers/gpu/drm/i915/display/intel_opregion.c
+++ b/drivers/gpu/drm/i915/display/intel_opregion.c
@@ -27,7 +27,6 @@
 
 #include <linux/acpi.h>
 #include <linux/dmi.h>
-#include <linux/firmware.h>
 #include <acpi/video.h>
 
 #include <drm/drm_edid.h>
@@ -263,7 +262,6 @@ struct intel_opregion {
 	struct opregion_asle *asle;
 	struct opregion_asle_ext *asle_ext;
 	void *rvda;
-	void *vbt_firmware;
 	const void *vbt;
 	u32 vbt_size;
 	struct work_struct asle_work;
@@ -869,46 +867,6 @@ static const struct dmi_system_id intel_no_opregion_vbt[] = {
 	{ }
 };
 
-static int intel_load_vbt_firmware(struct drm_i915_private *dev_priv)
-{
-	struct intel_opregion *opregion = dev_priv->display.opregion;
-	const struct firmware *fw = NULL;
-	const char *name = dev_priv->display.params.vbt_firmware;
-	int ret;
-
-	if (!name || !*name)
-		return -ENOENT;
-
-	ret = request_firmware(&fw, name, dev_priv->drm.dev);
-	if (ret) {
-		drm_err(&dev_priv->drm,
-			"Requesting VBT firmware \"%s\" failed (%d)\n",
-			name, ret);
-		return ret;
-	}
-
-	if (intel_bios_is_valid_vbt(dev_priv, fw->data, fw->size)) {
-		opregion->vbt_firmware = kmemdup(fw->data, fw->size, GFP_KERNEL);
-		if (opregion->vbt_firmware) {
-			drm_dbg_kms(&dev_priv->drm,
-				    "Found valid VBT firmware \"%s\"\n", name);
-			opregion->vbt = opregion->vbt_firmware;
-			opregion->vbt_size = fw->size;
-			ret = 0;
-		} else {
-			ret = -ENOMEM;
-		}
-	} else {
-		drm_dbg_kms(&dev_priv->drm, "Invalid VBT firmware \"%s\"\n",
-			    name);
-		ret = -EINVAL;
-	}
-
-	release_firmware(fw);
-
-	return ret;
-}
-
 int intel_opregion_setup(struct drm_i915_private *dev_priv)
 {
 	struct intel_opregion *opregion;
@@ -1006,9 +964,6 @@ int intel_opregion_setup(struct drm_i915_private *dev_priv)
 		drm_dbg(&dev_priv->drm, "Mailbox #2 for backlight present\n");
 	}
 
-	if (intel_load_vbt_firmware(dev_priv) == 0)
-		goto out;
-
 	if (dmi_check_system(intel_no_opregion_vbt))
 		goto out;
 
@@ -1176,6 +1131,16 @@ const struct drm_edid *intel_opregion_get_edid(struct intel_connector *intel_con
 	return drm_edid;
 }
 
+bool intel_opregion_vbt_present(struct drm_i915_private *i915)
+{
+	struct intel_opregion *opregion = i915->display.opregion;
+
+	if (!opregion || !opregion->vbt)
+		return false;
+
+	return true;
+}
+
 const void *intel_opregion_get_vbt(struct drm_i915_private *i915, size_t *size)
 {
 	struct intel_opregion *opregion = i915->display.opregion;
@@ -1186,7 +1151,7 @@ const void *intel_opregion_get_vbt(struct drm_i915_private *i915, size_t *size)
 	if (size)
 		*size = opregion->vbt_size;
 
-	return opregion->vbt;
+	return kmemdup(opregion->vbt, opregion->vbt_size, GFP_KERNEL);
 }
 
 bool intel_opregion_headless_sku(struct drm_i915_private *i915)
@@ -1312,7 +1277,6 @@ void intel_opregion_cleanup(struct drm_i915_private *i915)
 	memunmap(opregion->header);
 	if (opregion->rvda)
 		memunmap(opregion->rvda);
-	kfree(opregion->vbt_firmware);
 	kfree(opregion);
 	i915->display.opregion = NULL;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_opregion.h b/drivers/gpu/drm/i915/display/intel_opregion.h
index 0bec224f711f..4b2b8e752632 100644
--- a/drivers/gpu/drm/i915/display/intel_opregion.h
+++ b/drivers/gpu/drm/i915/display/intel_opregion.h
@@ -53,6 +53,7 @@ int intel_opregion_notify_adapter(struct drm_i915_private *dev_priv,
 int intel_opregion_get_panel_type(struct drm_i915_private *dev_priv);
 const struct drm_edid *intel_opregion_get_edid(struct intel_connector *connector);
 
+bool intel_opregion_vbt_present(struct drm_i915_private *i915);
 const void *intel_opregion_get_vbt(struct drm_i915_private *i915, size_t *size);
 
 bool intel_opregion_headless_sku(struct drm_i915_private *i915);
@@ -119,6 +120,11 @@ intel_opregion_get_edid(struct intel_connector *connector)
 	return NULL;
 }
 
+static inline bool intel_opregion_vbt_present(struct drm_i915_private *i915)
+{
+	return false;
+}
+
 static inline const void *
 intel_opregion_get_vbt(struct drm_i915_private *i915, size_t *size)
 {
diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c
index 2b1392d5a902..1c2099ed5514 100644
--- a/drivers/gpu/drm/i915/display/intel_overlay.c
+++ b/drivers/gpu/drm/i915/display/intel_overlay.c
@@ -972,10 +972,11 @@ static int check_overlay_dst(struct intel_overlay *overlay,
 		      rec->dst_width, rec->dst_height);
 
 	clipped = req;
-	drm_rect_intersect(&clipped, &crtc_state->pipe_src);
 
-	if (!drm_rect_visible(&clipped) ||
-	    !drm_rect_equals(&clipped, &req))
+	if (!drm_rect_intersect(&clipped, &crtc_state->pipe_src))
+		return -EINVAL;
+
+	if (!drm_rect_equals(&clipped, &req))
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/gpu/drm/i915/display/intel_panel.c b/drivers/gpu/drm/i915/display/intel_panel.c
index 073ea3166c36..6f4ff6a89c32 100644
--- a/drivers/gpu/drm/i915/display/intel_panel.c
+++ b/drivers/gpu/drm/i915/display/intel_panel.c
@@ -47,10 +47,12 @@
 
 bool intel_panel_use_ssc(struct drm_i915_private *i915)
 {
-	if (i915->display.params.panel_use_ssc >= 0)
-		return i915->display.params.panel_use_ssc != 0;
-	return i915->display.vbt.lvds_use_ssc &&
-		!intel_has_quirk(i915, QUIRK_LVDS_SSC_DISABLE);
+	struct intel_display *display = &i915->display;
+
+	if (display->params.panel_use_ssc >= 0)
+		return display->params.panel_use_ssc != 0;
+	return display->vbt.lvds_use_ssc &&
+		!intel_has_quirk(display, QUIRK_LVDS_SSC_DISABLE);
 }
 
 const struct drm_display_mode *
diff --git a/drivers/gpu/drm/i915/display/intel_pch_display.c b/drivers/gpu/drm/i915/display/intel_pch_display.c
index baf679759e00..826e38a9e6a4 100644
--- a/drivers/gpu/drm/i915/display/intel_pch_display.c
+++ b/drivers/gpu/drm/i915/display/intel_pch_display.c
@@ -474,7 +474,7 @@ static void ilk_pch_clock_get(struct intel_crtc_state *crtc_state)
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
 	/* read out port_clock from the DPLL */
-	i9xx_crtc_clock_get(crtc, crtc_state);
+	i9xx_crtc_clock_get(crtc_state);
 
 	/*
 	 * In case there is an active pipe without active ports,
@@ -529,7 +529,7 @@ void ilk_pch_get_config(struct intel_crtc_state *crtc_state)
 					     &crtc_state->dpll_hw_state);
 	drm_WARN_ON(&dev_priv->drm, !pll_active);
 
-	tmp = crtc_state->dpll_hw_state.dpll;
+	tmp = crtc_state->dpll_hw_state.i9xx.dpll;
 	crtc_state->pixel_multiplier =
 		((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
 		 >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
diff --git a/drivers/gpu/drm/i915/display/intel_pmdemand.c b/drivers/gpu/drm/i915/display/intel_pmdemand.c
index 744e332fa2af..9ca981b7a12c 100644
--- a/drivers/gpu/drm/i915/display/intel_pmdemand.c
+++ b/drivers/gpu/drm/i915/display/intel_pmdemand.c
@@ -119,10 +119,11 @@ intel_pmdemand_update_phys_mask(struct drm_i915_private *i915,
 	if (!encoder)
 		return;
 
-	phy = intel_port_to_phy(i915, encoder->port);
-	if (intel_phy_is_tc(i915, phy))
+	if (intel_encoder_is_tc(encoder))
 		return;
 
+	phy = intel_encoder_to_phy(encoder);
+
 	if (set_bit)
 		pmdemand_state->active_combo_phys_mask |= BIT(phy);
 	else
@@ -222,14 +223,7 @@ static bool
 intel_pmdemand_encoder_has_tc_phy(struct drm_i915_private *i915,
 				  struct intel_encoder *encoder)
 {
-	enum phy phy;
-
-	if (!encoder)
-		return false;
-
-	phy = intel_port_to_phy(i915, encoder->port);
-
-	return intel_phy_is_tc(i915, phy);
+	return encoder && intel_encoder_is_tc(encoder);
 }
 
 static bool
diff --git a/drivers/gpu/drm/i915/display/intel_pmdemand.h b/drivers/gpu/drm/i915/display/intel_pmdemand.h
index 2941a1a18b72..128fd61f8f14 100644
--- a/drivers/gpu/drm/i915/display/intel_pmdemand.h
+++ b/drivers/gpu/drm/i915/display/intel_pmdemand.h
@@ -43,9 +43,8 @@ struct intel_pmdemand_state {
 	struct pmdemand_params params;
 };
 
-#define to_intel_pmdemand_state(x) container_of((x), \
-						struct intel_pmdemand_state, \
-						base)
+#define to_intel_pmdemand_state(global_state) \
+	container_of_const((global_state), struct intel_pmdemand_state, base)
 
 void intel_pmdemand_init_early(struct drm_i915_private *i915);
 int intel_pmdemand_init(struct drm_i915_private *i915);
diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c
index 2d65a538f83e..0ccbf9a85914 100644
--- a/drivers/gpu/drm/i915/display/intel_pps.c
+++ b/drivers/gpu/drm/i915/display/intel_pps.c
@@ -605,8 +605,7 @@ static void wait_panel_status(struct intel_dp *intel_dp,
 		    intel_de_read(dev_priv, pp_stat_reg),
 		    intel_de_read(dev_priv, pp_ctrl_reg));
 
-	if (intel_de_wait_for_register(dev_priv, pp_stat_reg,
-				       mask, value, 5000))
+	if (intel_de_wait(dev_priv, pp_stat_reg, mask, value, 5000))
 		drm_err(&dev_priv->drm,
 			"[ENCODER:%d:%s] %s panel status timeout: PP_STATUS: 0x%08x PP_CONTROL: 0x%08x\n",
 			dig_port->base.base.base.id, dig_port->base.base.name,
@@ -1351,7 +1350,7 @@ static void pps_init_delays_bios(struct intel_dp *intel_dp,
 static void pps_init_delays_vbt(struct intel_dp *intel_dp,
 				struct edp_power_seq *vbt)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_connector *connector = intel_dp->attached_connector;
 
 	*vbt = connector->panel.vbt.edp.pps;
@@ -1364,9 +1363,9 @@ static void pps_init_delays_vbt(struct intel_dp *intel_dp,
 	 * just fails to power back on. Increasing the delay to 800ms
 	 * seems sufficient to avoid this problem.
 	 */
-	if (intel_has_quirk(dev_priv, QUIRK_INCREASE_T12_DELAY)) {
+	if (intel_has_quirk(display, QUIRK_INCREASE_T12_DELAY)) {
 		vbt->t11_t12 = max_t(u16, vbt->t11_t12, 1300 * 10);
-		drm_dbg_kms(&dev_priv->drm,
+		drm_dbg_kms(display->drm,
 			    "Increasing T12 panel delay as per the quirk to %d\n",
 			    vbt->t11_t12);
 	}
@@ -1671,6 +1670,37 @@ void intel_pps_setup(struct drm_i915_private *i915)
 		i915->display.pps.mmio_base = PPS_BASE;
 }
 
+static int intel_pps_show(struct seq_file *m, void *data)
+{
+	struct intel_connector *connector = m->private;
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+
+	if (connector->base.status != connector_status_connected)
+		return -ENODEV;
+
+	seq_printf(m, "Panel power up delay: %d\n",
+		   intel_dp->pps.panel_power_up_delay);
+	seq_printf(m, "Panel power down delay: %d\n",
+		   intel_dp->pps.panel_power_down_delay);
+	seq_printf(m, "Backlight on delay: %d\n",
+		   intel_dp->pps.backlight_on_delay);
+	seq_printf(m, "Backlight off delay: %d\n",
+		   intel_dp->pps.backlight_off_delay);
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(intel_pps);
+
+void intel_pps_connector_debugfs_add(struct intel_connector *connector)
+{
+	struct dentry *root = connector->base.debugfs_entry;
+	int connector_type = connector->base.connector_type;
+
+	if (connector_type == DRM_MODE_CONNECTOR_eDP)
+		debugfs_create_file("i915_panel_timings", 0444, root,
+				    connector, &intel_pps_fops);
+}
+
 void assert_pps_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
 	i915_reg_t pp_reg;
diff --git a/drivers/gpu/drm/i915/display/intel_pps.h b/drivers/gpu/drm/i915/display/intel_pps.h
index a2c2467e3c22..07ef96ca8da2 100644
--- a/drivers/gpu/drm/i915/display/intel_pps.h
+++ b/drivers/gpu/drm/i915/display/intel_pps.h
@@ -51,6 +51,8 @@ void vlv_pps_init(struct intel_encoder *encoder,
 void intel_pps_unlock_regs_wa(struct drm_i915_private *i915);
 void intel_pps_setup(struct drm_i915_private *i915);
 
+void intel_pps_connector_debugfs_add(struct intel_connector *connector);
+
 void assert_pps_unlocked(struct drm_i915_private *i915, enum pipe pipe);
 
 #endif /* __INTEL_PPS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c
index aabd018bd737..f5b33335a9ae 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.c
+++ b/drivers/gpu/drm/i915/display/intel_psr.c
@@ -171,14 +171,27 @@
  *
  * The rest of the bits are more self-explanatory and/or
  * irrelevant for normal operation.
+ *
+ * Description of intel_crtc_state variables. has_psr, has_panel_replay and
+ * has_sel_update:
+ *
+ *  has_psr (alone):					PSR1
+ *  has_psr + has_sel_update:				PSR2
+ *  has_psr + has_panel_replay:				Panel Replay
+ *  has_psr + has_panel_replay + has_sel_update:	Panel Replay Selective Update
+ *
+ * Description of some intel_psr varibles. enabled, panel_replay_enabled,
+ * sel_update_enabled
+ *
+ *  enabled (alone):						PSR1
+ *  enabled + sel_update_enabled:				PSR2
+ *  enabled + panel_replay_enabled:				Panel Replay
+ *  enabled + panel_replay_enabled + sel_update_enabled:	Panel Replay SU
  */
 
 #define CAN_PSR(intel_dp) ((intel_dp)->psr.sink_support && \
 			   (intel_dp)->psr.source_support)
 
-#define CAN_PANEL_REPLAY(intel_dp) ((intel_dp)->psr.sink_panel_replay_support && \
-				    (intel_dp)->psr.source_panel_replay_support)
-
 bool intel_encoder_can_psr(struct intel_encoder *encoder)
 {
 	if (intel_encoder_is_dp(encoder) || encoder->type == INTEL_OUTPUT_DP_MST)
@@ -330,6 +343,9 @@ static void psr_irq_control(struct intel_dp *intel_dp)
 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
 	u32 mask;
 
+	if (intel_dp->psr.panel_replay_enabled)
+		return;
+
 	mask = psr_irq_psr_error_bit_get(intel_dp);
 	if (intel_dp->psr.debug & I915_PSR_DEBUG_IRQ)
 		mask |= psr_irq_post_exit_bit_get(intel_dp) |
@@ -619,40 +635,59 @@ static bool psr2_su_region_et_valid(struct intel_dp *intel_dp)
 	return false;
 }
 
-static void intel_psr_enable_sink(struct intel_dp *intel_dp)
+static unsigned int intel_psr_get_enable_sink_offset(struct intel_dp *intel_dp)
+{
+	return intel_dp->psr.panel_replay_enabled ?
+		PANEL_REPLAY_CONFIG : DP_PSR_EN_CFG;
+}
+
+/*
+ * Note: Most of the bits are same in PANEL_REPLAY_CONFIG and DP_PSR_EN_CFG. We
+ * are relying on PSR definitions on these "common" bits.
+ */
+void intel_psr_enable_sink(struct intel_dp *intel_dp,
+			   const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	u8 dpcd_val = DP_PSR_ENABLE;
 
-	if (intel_dp->psr.panel_replay_enabled)
-		return;
-
-	if (intel_dp->psr.psr2_enabled) {
+	if (crtc_state->has_psr2) {
 		/* Enable ALPM at sink for psr2 */
-		drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG,
-				   DP_ALPM_ENABLE |
-				   DP_ALPM_LOCK_ERROR_IRQ_HPD_ENABLE);
+		if (!crtc_state->has_panel_replay) {
+			drm_dp_dpcd_writeb(&intel_dp->aux,
+					   DP_RECEIVER_ALPM_CONFIG,
+					   DP_ALPM_ENABLE |
+					   DP_ALPM_LOCK_ERROR_IRQ_HPD_ENABLE);
+
+			if (psr2_su_region_et_valid(intel_dp))
+				dpcd_val |= DP_PSR_ENABLE_SU_REGION_ET;
+		}
 
 		dpcd_val |= DP_PSR_ENABLE_PSR2 | DP_PSR_IRQ_HPD_WITH_CRC_ERRORS;
-		if (psr2_su_region_et_valid(intel_dp))
-			dpcd_val |= DP_PSR_ENABLE_SU_REGION_ET;
 	} else {
 		if (intel_dp->psr.link_standby)
 			dpcd_val |= DP_PSR_MAIN_LINK_ACTIVE;
 
-		if (DISPLAY_VER(dev_priv) >= 8)
+		if (!crtc_state->has_panel_replay && DISPLAY_VER(dev_priv) >= 8)
 			dpcd_val |= DP_PSR_CRC_VERIFICATION;
 	}
 
-	if (intel_dp->psr.req_psr2_sdp_prior_scanline)
+	if (crtc_state->has_panel_replay)
+		dpcd_val |= DP_PANEL_REPLAY_UNRECOVERABLE_ERROR_EN |
+			DP_PANEL_REPLAY_RFB_STORAGE_ERROR_EN;
+
+	if (crtc_state->req_psr2_sdp_prior_scanline)
 		dpcd_val |= DP_PSR_SU_REGION_SCANLINE_CAPTURE;
 
 	if (intel_dp->psr.entry_setup_frames > 0)
 		dpcd_val |= DP_PSR_FRAME_CAPTURE;
 
-	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, dpcd_val);
+	drm_dp_dpcd_writeb(&intel_dp->aux,
+			   intel_psr_get_enable_sink_offset(intel_dp),
+			   dpcd_val);
 
-	drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
+	if (intel_dp_is_edp(intel_dp))
+		drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
 }
 
 static u32 intel_psr1_get_tp_time(struct intel_dp *intel_dp)
@@ -1126,6 +1161,141 @@ static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_dp *intel_d
 	return true;
 }
 
+/*
+ * See Bspec: 71632 for the table
+ *
+ * Silence_period = tSilence,Min + ((tSilence,Max - tSilence,Min) / 2)
+ *
+ * Half cycle duration:
+ *
+ * Link rates 1.62 - 4.32 and tLFPS_Cycle = 70 ns
+ * FLOOR( (Link Rate * tLFPS_Cycle) / (2 * 10) )
+ *
+ * Link rates 5.4 - 8.1
+ * PORT_ALPM_LFPS_CTL[ LFPS Cycle Count ] = 10
+ * LFPS Period chosen is the mid-point of the min:max values from the table
+ * FLOOR( LFPS Period in Symbol clocks /
+ * (2 * PORT_ALPM_LFPS_CTL[ LFPS Cycle Count ]) )
+ */
+static bool _lnl_get_silence_period_and_lfps_half_cycle(int link_rate,
+							int *silence_period,
+							int *lfps_half_cycle)
+{
+	switch (link_rate) {
+	case 162000:
+		*silence_period = 20;
+		*lfps_half_cycle = 5;
+		break;
+	case 216000:
+		*silence_period = 27;
+		*lfps_half_cycle = 7;
+		break;
+	case 243000:
+		*silence_period = 31;
+		*lfps_half_cycle = 8;
+		break;
+	case 270000:
+		*silence_period = 34;
+		*lfps_half_cycle = 9;
+		break;
+	case 324000:
+		*silence_period = 41;
+		*lfps_half_cycle = 11;
+		break;
+	case 432000:
+		*silence_period = 56;
+		*lfps_half_cycle = 15;
+		break;
+	case 540000:
+		*silence_period = 69;
+		*lfps_half_cycle = 12;
+		break;
+	case 648000:
+		*silence_period = 84;
+		*lfps_half_cycle = 15;
+		break;
+	case 675000:
+		*silence_period = 87;
+		*lfps_half_cycle = 15;
+		break;
+	case 810000:
+		*silence_period = 104;
+		*lfps_half_cycle = 19;
+		break;
+	default:
+		*silence_period = *lfps_half_cycle = -1;
+		return false;
+	}
+	return true;
+}
+
+/*
+ * AUX-Less Wake Time = CEILING( ((PHY P2 to P0) + tLFPS_Period, Max+
+ * tSilence, Max+ tPHY Establishment + tCDS) / tline)
+ * For the "PHY P2 to P0" latency see the PHY Power Control page
+ * (PHY P2 to P0) : https://gfxspecs.intel.com/Predator/Home/Index/68965
+ * : 12 us
+ * The tLFPS_Period, Max term is 800ns
+ * The tSilence, Max term is 180ns
+ * The tPHY Establishment (a.k.a. t1) term is 50us
+ * The tCDS term is 1 or 2 times t2
+ * t2 = Number ML_PHY_LOCK * tML_PHY_LOCK
+ * Number ML_PHY_LOCK = ( 7 + CEILING( 6.5us / tML_PHY_LOCK ) + 1)
+ * Rounding up the 6.5us padding to the next ML_PHY_LOCK boundary and
+ * adding the "+ 1" term ensures all ML_PHY_LOCK sequences that start
+ * within the CDS period complete within the CDS period regardless of
+ * entry into the period
+ * tML_PHY_LOCK = TPS4 Length * ( 10 / (Link Rate in MHz) )
+ * TPS4 Length = 252 Symbols
+ */
+static int _lnl_compute_aux_less_wake_time(int port_clock)
+{
+	int tphy2_p2_to_p0 = 12 * 1000;
+	int tlfps_period_max = 800;
+	int tsilence_max = 180;
+	int t1 = 50 * 1000;
+	int tps4 = 252;
+	int tml_phy_lock = 1000 * 1000 * tps4 * 10 / port_clock;
+	int num_ml_phy_lock = 7 + DIV_ROUND_UP(6500, tml_phy_lock) + 1;
+	int t2 = num_ml_phy_lock * tml_phy_lock;
+	int tcds = 1 * t2;
+
+	return DIV_ROUND_UP(tphy2_p2_to_p0 + tlfps_period_max + tsilence_max +
+			    t1 + tcds, 1000);
+}
+
+static int _lnl_compute_aux_less_alpm_params(struct intel_dp *intel_dp,
+					     struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
+	int aux_less_wake_time, aux_less_wake_lines, silence_period,
+		lfps_half_cycle;
+
+	aux_less_wake_time =
+		_lnl_compute_aux_less_wake_time(crtc_state->port_clock);
+	aux_less_wake_lines = intel_usecs_to_scanlines(&crtc_state->hw.adjusted_mode,
+						       aux_less_wake_time);
+
+	if (!_lnl_get_silence_period_and_lfps_half_cycle(crtc_state->port_clock,
+							 &silence_period,
+							 &lfps_half_cycle))
+		return false;
+
+	if (aux_less_wake_lines > ALPM_CTL_AUX_LESS_WAKE_TIME_MASK ||
+	    silence_period > PORT_ALPM_CTL_SILENCE_PERIOD_MASK ||
+	    lfps_half_cycle > PORT_ALPM_LFPS_CTL_LAST_LFPS_HALF_CYCLE_DURATION_MASK)
+		return false;
+
+	if (i915->display.params.psr_safest_params)
+		aux_less_wake_lines = ALPM_CTL_AUX_LESS_WAKE_TIME_MASK;
+
+	intel_dp->psr.alpm_parameters.fast_wake_lines = aux_less_wake_lines;
+	intel_dp->psr.alpm_parameters.silence_period_sym_clocks = silence_period;
+	intel_dp->psr.alpm_parameters.lfps_half_cycle_num_of_syms = lfps_half_cycle;
+
+	return true;
+}
+
 static bool _lnl_compute_alpm_params(struct intel_dp *intel_dp,
 				     struct intel_crtc_state *crtc_state)
 {
@@ -1142,6 +1312,9 @@ static bool _lnl_compute_alpm_params(struct intel_dp *intel_dp,
 	if (check_entry_lines > 15)
 		return false;
 
+	if (!_lnl_compute_aux_less_alpm_params(intel_dp, crtc_state))
+		return false;
+
 	if (i915->display.params.psr_safest_params)
 		check_entry_lines = 15;
 
@@ -1150,28 +1323,52 @@ static bool _lnl_compute_alpm_params(struct intel_dp *intel_dp,
 	return true;
 }
 
+/*
+ * IO wake time for DISPLAY_VER < 12 is not directly mentioned in Bspec. There
+ * are 50 us io wake time and 32 us fast wake time. Clearly preharge pulses are
+ * not (improperly) included in 32 us fast wake time. 50 us - 32 us = 18 us.
+ */
+static int skl_io_buffer_wake_time(void)
+{
+	return 18;
+}
+
+static int tgl_io_buffer_wake_time(void)
+{
+	return 10;
+}
+
+static int io_buffer_wake_time(const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+
+	if (DISPLAY_VER(i915) >= 12)
+		return tgl_io_buffer_wake_time();
+	else
+		return skl_io_buffer_wake_time();
+}
+
 static bool _compute_alpm_params(struct intel_dp *intel_dp,
 				 struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 	int io_wake_lines, io_wake_time, fast_wake_lines, fast_wake_time;
+	int tfw_exit_latency = 20; /* eDP spec */
+	int phy_wake = 4;	   /* eDP spec */
+	int preamble = 8;	   /* eDP spec */
+	int precharge = intel_dp_aux_fw_sync_len() - preamble;
 	u8 max_wake_lines;
 
-	if (DISPLAY_VER(i915) >= 12) {
-		io_wake_time = 42;
-		/*
-		 * According to Bspec it's 42us, but based on testing
-		 * it is not enough -> use 45 us.
-		 */
-		fast_wake_time = 45;
+	io_wake_time = max(precharge, io_buffer_wake_time(crtc_state)) +
+		preamble + phy_wake + tfw_exit_latency;
+	fast_wake_time = precharge + preamble + phy_wake +
+		tfw_exit_latency;
 
+	if (DISPLAY_VER(i915) >= 12)
 		/* TODO: Check how we can use ALPM_CTL fast wake extended field */
 		max_wake_lines = 12;
-	} else {
-		io_wake_time = 50;
-		fast_wake_time = 32;
+	else
 		max_wake_lines = 8;
-	}
 
 	io_wake_lines = intel_usecs_to_scanlines(
 		&crtc_state->hw.adjusted_mode, io_wake_time);
@@ -1435,10 +1632,11 @@ void intel_psr_compute_config(struct intel_dp *intel_dp,
 
 	if (CAN_PANEL_REPLAY(intel_dp))
 		crtc_state->has_panel_replay = true;
-	else
-		crtc_state->has_psr = _psr_compute_config(intel_dp, crtc_state);
 
-	if (!(crtc_state->has_panel_replay || crtc_state->has_psr))
+	crtc_state->has_psr = crtc_state->has_panel_replay ? true :
+		_psr_compute_config(intel_dp, crtc_state);
+
+	if (!crtc_state->has_psr)
 		return;
 
 	crtc_state->has_psr2 = intel_psr2_config_valid(intel_dp, crtc_state);
@@ -1465,7 +1663,7 @@ void intel_psr_get_config(struct intel_encoder *encoder,
 		goto unlock;
 
 	if (intel_dp->psr.panel_replay_enabled) {
-		pipe_config->has_panel_replay = true;
+		pipe_config->has_psr = pipe_config->has_panel_replay = true;
 	} else {
 		/*
 		 * Not possible to read EDP_PSR/PSR2_CTL registers as it is
@@ -1570,14 +1768,44 @@ static void lnl_alpm_configure(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
 	struct intel_psr *psr = &intel_dp->psr;
+	u32 alpm_ctl;
 
-	if (DISPLAY_VER(dev_priv) < 20)
+	if (DISPLAY_VER(dev_priv) < 20 || (!intel_dp->psr.psr2_enabled &&
+					   !intel_dp_is_edp(intel_dp)))
 		return;
 
-	intel_de_write(dev_priv, ALPM_CTL(cpu_transcoder),
-		       ALPM_CTL_EXTENDED_FAST_WAKE_ENABLE |
-		       ALPM_CTL_ALPM_ENTRY_CHECK(psr->alpm_parameters.check_entry_lines) |
-		       ALPM_CTL_EXTENDED_FAST_WAKE_TIME(psr->alpm_parameters.fast_wake_lines));
+	/*
+	 * Panel Replay on eDP is always using ALPM aux less. I.e. no need to
+	 * check panel support at this point.
+	 */
+	if (intel_dp->psr.panel_replay_enabled && intel_dp_is_edp(intel_dp)) {
+		alpm_ctl = ALPM_CTL_ALPM_ENABLE |
+			ALPM_CTL_ALPM_AUX_LESS_ENABLE |
+			ALPM_CTL_AUX_LESS_SLEEP_HOLD_TIME_50_SYMBOLS;
+
+		intel_de_write(dev_priv, PORT_ALPM_CTL(cpu_transcoder),
+			       PORT_ALPM_CTL_ALPM_AUX_LESS_ENABLE |
+			       PORT_ALPM_CTL_MAX_PHY_SWING_SETUP(15) |
+			       PORT_ALPM_CTL_MAX_PHY_SWING_HOLD(0) |
+			       PORT_ALPM_CTL_SILENCE_PERIOD(
+				       psr->alpm_parameters.silence_period_sym_clocks));
+
+		intel_de_write(dev_priv, PORT_ALPM_LFPS_CTL(cpu_transcoder),
+			       PORT_ALPM_LFPS_CTL_LFPS_CYCLE_COUNT(10) |
+			       PORT_ALPM_LFPS_CTL_LFPS_HALF_CYCLE_DURATION(
+				       psr->alpm_parameters.lfps_half_cycle_num_of_syms) |
+			       PORT_ALPM_LFPS_CTL_FIRST_LFPS_HALF_CYCLE_DURATION(
+				       psr->alpm_parameters.lfps_half_cycle_num_of_syms) |
+			       PORT_ALPM_LFPS_CTL_LAST_LFPS_HALF_CYCLE_DURATION(
+				       psr->alpm_parameters.lfps_half_cycle_num_of_syms));
+	} else {
+		alpm_ctl = ALPM_CTL_EXTENDED_FAST_WAKE_ENABLE |
+			ALPM_CTL_EXTENDED_FAST_WAKE_TIME(psr->alpm_parameters.fast_wake_lines);
+	}
+
+	alpm_ctl |= ALPM_CTL_ALPM_ENTRY_CHECK(psr->alpm_parameters.check_entry_lines);
+
+	intel_de_write(dev_priv, ALPM_CTL(cpu_transcoder), alpm_ctl);
 }
 
 static void intel_psr_enable_source(struct intel_dp *intel_dp,
@@ -1585,7 +1813,7 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp,
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
-	u32 mask;
+	u32 mask = 0;
 
 	/*
 	 * Only HSW and BDW have PSR AUX registers that need to be setup.
@@ -1599,34 +1827,46 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp,
 	 * mask LPSP to avoid dependency on other drivers that might block
 	 * runtime_pm besides preventing  other hw tracking issues now we
 	 * can rely on frontbuffer tracking.
+	 *
+	 * From bspec prior LunarLake:
+	 * Only PSR_MASK[Mask FBC modify] and PSR_MASK[Mask Hotplug] are used in
+	 * panel replay mode.
+	 *
+	 * From bspec beyod LunarLake:
+	 * Panel Replay on DP: No bits are applicable
+	 * Panel Replay on eDP: All bits are applicable
 	 */
-	mask = EDP_PSR_DEBUG_MASK_MEMUP |
-	       EDP_PSR_DEBUG_MASK_HPD;
+	if (DISPLAY_VER(dev_priv) < 20 || intel_dp_is_edp(intel_dp))
+		mask = EDP_PSR_DEBUG_MASK_HPD;
 
-	/*
-	 * For some unknown reason on HSW non-ULT (or at least on
-	 * Dell Latitude E6540) external displays start to flicker
-	 * when PSR is enabled on the eDP. SR/PC6 residency is much
-	 * higher than should be possible with an external display.
-	 * As a workaround leave LPSP unmasked to prevent PSR entry
-	 * when external displays are active.
-	 */
-	if (DISPLAY_VER(dev_priv) >= 8 || IS_HASWELL_ULT(dev_priv))
-		mask |= EDP_PSR_DEBUG_MASK_LPSP;
+	if (intel_dp_is_edp(intel_dp)) {
+		mask |= EDP_PSR_DEBUG_MASK_MEMUP;
 
-	if (DISPLAY_VER(dev_priv) < 20)
-		mask |= EDP_PSR_DEBUG_MASK_MAX_SLEEP;
+		/*
+		 * For some unknown reason on HSW non-ULT (or at least on
+		 * Dell Latitude E6540) external displays start to flicker
+		 * when PSR is enabled on the eDP. SR/PC6 residency is much
+		 * higher than should be possible with an external display.
+		 * As a workaround leave LPSP unmasked to prevent PSR entry
+		 * when external displays are active.
+		 */
+		if (DISPLAY_VER(dev_priv) >= 8 || IS_HASWELL_ULT(dev_priv))
+			mask |= EDP_PSR_DEBUG_MASK_LPSP;
 
-	/*
-	 * No separate pipe reg write mask on hsw/bdw, so have to unmask all
-	 * registers in order to keep the CURSURFLIVE tricks working :(
-	 */
-	if (IS_DISPLAY_VER(dev_priv, 9, 10))
-		mask |= EDP_PSR_DEBUG_MASK_DISP_REG_WRITE;
+		if (DISPLAY_VER(dev_priv) < 20)
+			mask |= EDP_PSR_DEBUG_MASK_MAX_SLEEP;
 
-	/* allow PSR with sprite enabled */
-	if (IS_HASWELL(dev_priv))
-		mask |= EDP_PSR_DEBUG_MASK_SPRITE_ENABLE;
+		/*
+		 * No separate pipe reg write mask on hsw/bdw, so have to unmask all
+		 * registers in order to keep the CURSURFLIVE tricks working :(
+		 */
+		if (IS_DISPLAY_VER(dev_priv, 9, 10))
+			mask |= EDP_PSR_DEBUG_MASK_DISP_REG_WRITE;
+
+		/* allow PSR with sprite enabled */
+		if (IS_HASWELL(dev_priv))
+			mask |= EDP_PSR_DEBUG_MASK_SPRITE_ENABLE;
+	}
 
 	intel_de_write(dev_priv, psr_debug_reg(dev_priv, cpu_transcoder), mask);
 
@@ -1645,7 +1885,8 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp,
 			     intel_dp->psr.psr2_sel_fetch_enabled ?
 			     IGNORE_PSR2_HW_TRACKING : 0);
 
-	lnl_alpm_configure(intel_dp);
+	if (intel_dp_is_edp(intel_dp))
+		lnl_alpm_configure(intel_dp);
 
 	/*
 	 * Wa_16013835468
@@ -1686,6 +1927,9 @@ static bool psr_interrupt_error_check(struct intel_dp *intel_dp)
 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
 	u32 val;
 
+	if (intel_dp->psr.panel_replay_enabled)
+		goto no_err;
+
 	/*
 	 * If a PSR error happened and the driver is reloaded, the EDP_PSR_IIR
 	 * will still keep the error set even after the reset done in the
@@ -1703,6 +1947,7 @@ static bool psr_interrupt_error_check(struct intel_dp *intel_dp)
 		return false;
 	}
 
+no_err:
 	return true;
 }
 
@@ -1711,7 +1956,6 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp,
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
-	enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
 	u32 val;
 
 	drm_WARN_ON(&dev_priv->drm, intel_dp->psr.enabled);
@@ -1733,14 +1977,22 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp,
 	if (!psr_interrupt_error_check(intel_dp))
 		return;
 
-	if (intel_dp->psr.panel_replay_enabled)
+	if (intel_dp->psr.panel_replay_enabled) {
 		drm_dbg_kms(&dev_priv->drm, "Enabling Panel Replay\n");
-	else
+	} else {
 		drm_dbg_kms(&dev_priv->drm, "Enabling PSR%s\n",
 			    intel_dp->psr.psr2_enabled ? "2" : "1");
 
-	intel_snps_phy_update_psr_power_state(dev_priv, phy, true);
-	intel_psr_enable_sink(intel_dp);
+		/*
+		 * Panel replay has to be enabled before link training: doing it
+		 * only for PSR here.
+		 */
+		intel_psr_enable_sink(intel_dp, crtc_state);
+	}
+
+	if (intel_dp_is_edp(intel_dp))
+		intel_snps_phy_update_psr_power_state(&dig_port->base, true);
+
 	intel_psr_enable_source(intel_dp, crtc_state);
 	intel_dp->psr.enabled = true;
 	intel_dp->psr.paused = false;
@@ -1810,8 +2062,6 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
-	enum phy phy = intel_port_to_phy(dev_priv,
-					 dp_to_dig_port(intel_dp)->base.port);
 
 	lockdep_assert_held(&intel_dp->psr.lock);
 
@@ -1846,12 +2096,25 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp)
 				     CLKGATE_DIS_MISC_DMASC_GATING_DIS, 0);
 	}
 
-	intel_snps_phy_update_psr_power_state(dev_priv, phy, false);
+	if (intel_dp_is_edp(intel_dp))
+		intel_snps_phy_update_psr_power_state(&dp_to_dig_port(intel_dp)->base, false);
+
+	/* Panel Replay on eDP is always using ALPM aux less. */
+	if (intel_dp->psr.panel_replay_enabled && intel_dp_is_edp(intel_dp)) {
+		intel_de_rmw(dev_priv, ALPM_CTL(cpu_transcoder),
+			     ALPM_CTL_ALPM_ENABLE |
+			     ALPM_CTL_ALPM_AUX_LESS_ENABLE, 0);
+
+		intel_de_rmw(dev_priv, PORT_ALPM_CTL(cpu_transcoder),
+			     PORT_ALPM_CTL_ALPM_AUX_LESS_ENABLE, 0);
+	}
 
 	/* Disable PSR on Sink */
-	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);
+	drm_dp_dpcd_writeb(&intel_dp->aux,
+			   intel_psr_get_enable_sink_offset(intel_dp), 0);
 
-	if (intel_dp->psr.psr2_enabled)
+	if (!intel_dp->psr.panel_replay_enabled &&
+	    intel_dp->psr.psr2_enabled)
 		drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG, 0);
 
 	intel_dp->psr.enabled = false;
@@ -1899,7 +2162,7 @@ void intel_psr_pause(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	struct intel_psr *psr = &intel_dp->psr;
 
-	if (!CAN_PSR(intel_dp))
+	if (!CAN_PSR(intel_dp) && !CAN_PANEL_REPLAY(intel_dp))
 		return;
 
 	mutex_lock(&psr->lock);
@@ -1932,7 +2195,7 @@ void intel_psr_resume(struct intel_dp *intel_dp)
 {
 	struct intel_psr *psr = &intel_dp->psr;
 
-	if (!CAN_PSR(intel_dp))
+	if (!CAN_PSR(intel_dp) && !CAN_PANEL_REPLAY(intel_dp))
 		return;
 
 	mutex_lock(&psr->lock);
@@ -2069,14 +2332,19 @@ exit:
 	crtc_state->psr2_man_track_ctl = val;
 }
 
-static u32 psr2_pipe_srcsz_early_tpt_calc(struct intel_crtc_state *crtc_state,
-					  bool full_update)
+static u32
+psr2_pipe_srcsz_early_tpt_calc(struct intel_crtc_state *crtc_state,
+			       bool full_update, bool cursor_in_su_area)
 {
 	int width, height;
 
 	if (!crtc_state->enable_psr2_su_region_et || full_update)
 		return 0;
 
+	if (!cursor_in_su_area)
+		return PIPESRC_WIDTH(0) |
+			PIPESRC_HEIGHT(drm_rect_height(&crtc_state->pipe_src));
+
 	width = drm_rect_width(&crtc_state->psr2_su_area);
 	height = drm_rect_height(&crtc_state->psr2_su_area);
 
@@ -2128,7 +2396,8 @@ static void intel_psr2_sel_fetch_pipe_alignment(struct intel_crtc_state *crtc_st
  */
 static void
 intel_psr2_sel_fetch_et_alignment(struct intel_atomic_state *state,
-				  struct intel_crtc *crtc)
+				  struct intel_crtc *crtc,
+				  bool *cursor_in_su_area)
 {
 	struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_plane_state *new_plane_state;
@@ -2156,6 +2425,7 @@ intel_psr2_sel_fetch_et_alignment(struct intel_atomic_state *state,
 
 		clip_area_update(&crtc_state->psr2_su_area, &new_plane_state->uapi.dst,
 				 &crtc_state->pipe_src);
+		*cursor_in_su_area = true;
 	}
 }
 
@@ -2201,7 +2471,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
 	struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_plane_state *new_plane_state, *old_plane_state;
 	struct intel_plane *plane;
-	bool full_update = false;
+	bool full_update = false, cursor_in_su_area = false;
 	int i, ret;
 
 	if (!crtc_state->enable_psr2_sel_fetch)
@@ -2318,7 +2588,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
 	 * drm_atomic_add_affected_planes to ensure visible cursor is added into
 	 * affected planes even when cursor is not updated by itself.
 	 */
-	intel_psr2_sel_fetch_et_alignment(state, crtc);
+	intel_psr2_sel_fetch_et_alignment(state, crtc, &cursor_in_su_area);
 
 	intel_psr2_sel_fetch_pipe_alignment(crtc_state);
 
@@ -2382,7 +2652,8 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
 skip_sel_fetch_set_loop:
 	psr2_man_trk_ctl_calc(crtc_state, full_update);
 	crtc_state->pipe_srcsz_early_tpt =
-		psr2_pipe_srcsz_early_tpt_calc(crtc_state, full_update);
+		psr2_pipe_srcsz_early_tpt_calc(crtc_state, full_update,
+					       cursor_in_su_area);
 	return 0;
 }
 
@@ -2439,7 +2710,7 @@ void intel_psr_post_plane_update(struct intel_atomic_state *state,
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_encoder *encoder;
 
-	if (!(crtc_state->has_psr || crtc_state->has_panel_replay))
+	if (!crtc_state->has_psr)
 		return;
 
 	for_each_intel_encoder_mask_with_psr(state->base.dev, encoder,
@@ -3039,6 +3310,13 @@ static void psr_capability_changed_check(struct intel_dp *intel_dp)
 	}
 }
 
+/*
+ * On common bits:
+ * DP_PSR_RFB_STORAGE_ERROR == DP_PANEL_REPLAY_RFB_STORAGE_ERROR
+ * DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR == DP_PANEL_REPLAY_VSC_SDP_UNCORRECTABLE_ERROR
+ * DP_PSR_LINK_CRC_ERROR == DP_PANEL_REPLAY_LINK_CRC_ERROR
+ * this function is relying on PSR definitions
+ */
 void intel_psr_short_pulse(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
@@ -3048,7 +3326,7 @@ void intel_psr_short_pulse(struct intel_dp *intel_dp)
 			  DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR |
 			  DP_PSR_LINK_CRC_ERROR;
 
-	if (!CAN_PSR(intel_dp))
+	if (!CAN_PSR(intel_dp) && !CAN_PANEL_REPLAY(intel_dp))
 		return;
 
 	mutex_lock(&psr->lock);
@@ -3062,12 +3340,14 @@ void intel_psr_short_pulse(struct intel_dp *intel_dp)
 		goto exit;
 	}
 
-	if (status == DP_PSR_SINK_INTERNAL_ERROR || (error_status & errors)) {
+	if ((!psr->panel_replay_enabled && status == DP_PSR_SINK_INTERNAL_ERROR) ||
+	    (error_status & errors)) {
 		intel_psr_disable_locked(intel_dp);
 		psr->sink_not_reliable = true;
 	}
 
-	if (status == DP_PSR_SINK_INTERNAL_ERROR && !error_status)
+	if (!psr->panel_replay_enabled && status == DP_PSR_SINK_INTERNAL_ERROR &&
+	    !error_status)
 		drm_dbg_kms(&dev_priv->drm,
 			    "PSR sink internal error, disabling PSR\n");
 	if (error_status & DP_PSR_RFB_STORAGE_ERROR)
@@ -3087,8 +3367,10 @@ void intel_psr_short_pulse(struct intel_dp *intel_dp)
 	/* clear status register */
 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ERROR_STATUS, error_status);
 
-	psr_alpm_check(intel_dp);
-	psr_capability_changed_check(intel_dp);
+	if (!psr->panel_replay_enabled) {
+		psr_alpm_check(intel_dp);
+		psr_capability_changed_check(intel_dp);
+	}
 
 exit:
 	mutex_unlock(&psr->lock);
diff --git a/drivers/gpu/drm/i915/display/intel_psr.h b/drivers/gpu/drm/i915/display/intel_psr.h
index cde781df84d5..d483c85870e1 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.h
+++ b/drivers/gpu/drm/i915/display/intel_psr.h
@@ -21,8 +21,13 @@ struct intel_encoder;
 struct intel_plane;
 struct intel_plane_state;
 
+#define CAN_PANEL_REPLAY(intel_dp) ((intel_dp)->psr.sink_panel_replay_support && \
+				    (intel_dp)->psr.source_panel_replay_support)
+
 bool intel_encoder_can_psr(struct intel_encoder *encoder);
 void intel_psr_init_dpcd(struct intel_dp *intel_dp);
+void intel_psr_enable_sink(struct intel_dp *intel_dp,
+			   const struct intel_crtc_state *crtc_state);
 void intel_psr_pre_plane_update(struct intel_atomic_state *state,
 				struct intel_crtc *crtc);
 void intel_psr_post_plane_update(struct intel_atomic_state *state,
diff --git a/drivers/gpu/drm/i915/display/intel_psr_regs.h b/drivers/gpu/drm/i915/display/intel_psr_regs.h
index 8427a736f639..ebc22999572c 100644
--- a/drivers/gpu/drm/i915/display/intel_psr_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_psr_regs.h
@@ -9,7 +9,7 @@
 #include "intel_display_reg_defs.h"
 #include "intel_dp_aux_regs.h"
 
-#define TRANS_EXITLINE(trans)	_MMIO_TRANS2((trans), _TRANS_EXITLINE_A)
+#define TRANS_EXITLINE(trans)	_MMIO_TRANS2(dev_priv, (trans), _TRANS_EXITLINE_A)
 #define   EXITLINE_ENABLE	REG_BIT(31)
 #define   EXITLINE_MASK		REG_GENMASK(12, 0)
 #define   EXITLINE_SHIFT	0
@@ -23,7 +23,7 @@
 #define HSW_SRD_CTL				_MMIO(0x64800)
 #define _SRD_CTL_A				0x60800
 #define _SRD_CTL_EDP				0x6f800
-#define EDP_PSR_CTL(tran)			_MMIO_TRANS2(tran, _SRD_CTL_A)
+#define EDP_PSR_CTL(tran)			_MMIO_TRANS2(dev_priv, tran, _SRD_CTL_A)
 #define   EDP_PSR_ENABLE			REG_BIT(31)
 #define   BDW_PSR_SINGLE_FRAME			REG_BIT(30)
 #define   EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK	REG_BIT(29) /* SW can't modify */
@@ -66,8 +66,8 @@
 #define EDP_PSR_IIR				_MMIO(0x64838)
 #define _PSR_IMR_A				0x60814
 #define _PSR_IIR_A				0x60818
-#define TRANS_PSR_IMR(tran)			_MMIO_TRANS2(tran, _PSR_IMR_A)
-#define TRANS_PSR_IIR(tran)			_MMIO_TRANS2(tran, _PSR_IIR_A)
+#define TRANS_PSR_IMR(tran)			_MMIO_TRANS2(dev_priv, tran, _PSR_IMR_A)
+#define TRANS_PSR_IIR(tran)			_MMIO_TRANS2(dev_priv, tran, _PSR_IIR_A)
 #define   _EDP_PSR_TRANS_SHIFT(trans)		((trans) == TRANSCODER_EDP ? \
 						 0 : ((trans) - TRANSCODER_A + 1) * 8)
 #define   TGL_PSR_MASK			REG_GENMASK(2, 0)
@@ -86,7 +86,7 @@
 #define HSW_SRD_AUX_CTL				_MMIO(0x64810)
 #define _SRD_AUX_CTL_A				0x60810
 #define _SRD_AUX_CTL_EDP			0x6f810
-#define EDP_PSR_AUX_CTL(tran)			_MMIO_TRANS2(tran, _SRD_AUX_CTL_A)
+#define EDP_PSR_AUX_CTL(tran)			_MMIO_TRANS2(dev_priv, tran, _SRD_AUX_CTL_A)
 #define   EDP_PSR_AUX_CTL_TIME_OUT_MASK		DP_AUX_CH_CTL_TIME_OUT_MASK
 #define   EDP_PSR_AUX_CTL_MESSAGE_SIZE_MASK	DP_AUX_CH_CTL_MESSAGE_SIZE_MASK
 #define   EDP_PSR_AUX_CTL_PRECHARGE_2US_MASK	DP_AUX_CH_CTL_PRECHARGE_2US_MASK
@@ -96,12 +96,12 @@
 #define HSW_SRD_AUX_DATA(i)			_MMIO(0x64814 + (i) * 4) /* 5 registers */
 #define _SRD_AUX_DATA_A				0x60814
 #define _SRD_AUX_DATA_EDP			0x6f814
-#define EDP_PSR_AUX_DATA(tran, i)		_MMIO_TRANS2(tran, _SRD_AUX_DATA_A + (i) * 4) /* 5 registers */
+#define EDP_PSR_AUX_DATA(tran, i)		_MMIO_TRANS2(dev_priv, tran, _SRD_AUX_DATA_A + (i) * 4) /* 5 registers */
 
 #define HSW_SRD_STATUS				_MMIO(0x64840)
 #define _SRD_STATUS_A				0x60840
 #define _SRD_STATUS_EDP				0x6f840
-#define EDP_PSR_STATUS(tran)			_MMIO_TRANS2(tran, _SRD_STATUS_A)
+#define EDP_PSR_STATUS(tran)			_MMIO_TRANS2(dev_priv, tran, _SRD_STATUS_A)
 #define   EDP_PSR_STATUS_STATE_MASK		REG_GENMASK(31, 29)
 #define   EDP_PSR_STATUS_STATE_IDLE		REG_FIELD_PREP(EDP_PSR_STATUS_STATE_MASK, 0)
 #define   EDP_PSR_STATUS_STATE_SRDONACK		REG_FIELD_PREP(EDP_PSR_STATUS_STATE_MASK, 1)
@@ -126,14 +126,14 @@
 #define HSW_SRD_PERF_CNT		_MMIO(0x64844)
 #define _SRD_PERF_CNT_A			0x60844
 #define _SRD_PERF_CNT_EDP		0x6f844
-#define EDP_PSR_PERF_CNT(tran)		_MMIO_TRANS2(tran, _SRD_PERF_CNT_A)
+#define EDP_PSR_PERF_CNT(tran)		_MMIO_TRANS2(dev_priv, tran, _SRD_PERF_CNT_A)
 #define   EDP_PSR_PERF_CNT_MASK		REG_GENMASK(23, 0)
 
 /* PSR_MASK on SKL+ */
 #define HSW_SRD_DEBUG				_MMIO(0x64860)
 #define _SRD_DEBUG_A				0x60860
 #define _SRD_DEBUG_EDP				0x6f860
-#define EDP_PSR_DEBUG(tran)			_MMIO_TRANS2(tran, _SRD_DEBUG_A)
+#define EDP_PSR_DEBUG(tran)			_MMIO_TRANS2(dev_priv, tran, _SRD_DEBUG_A)
 #define   EDP_PSR_DEBUG_MASK_MAX_SLEEP		REG_BIT(28)
 #define   EDP_PSR_DEBUG_MASK_LPSP		REG_BIT(27)
 #define   EDP_PSR_DEBUG_MASK_MEMUP		REG_BIT(26)
@@ -153,7 +153,7 @@
 
 #define _PSR2_CTL_A				0x60900
 #define _PSR2_CTL_EDP				0x6f900
-#define EDP_PSR2_CTL(tran)			_MMIO_TRANS2(tran, _PSR2_CTL_A)
+#define EDP_PSR2_CTL(tran)			_MMIO_TRANS2(dev_priv, tran, _PSR2_CTL_A)
 #define   EDP_PSR2_ENABLE			REG_BIT(31)
 #define   EDP_SU_TRACK_ENABLE			REG_BIT(30) /* up to adl-p */
 #define   TGL_EDP_PSR2_BLOCK_COUNT_MASK		REG_BIT(28)
@@ -195,7 +195,7 @@
 #define _PSR_EVENT_TRANS_C			0x62848
 #define _PSR_EVENT_TRANS_D			0x63848
 #define _PSR_EVENT_TRANS_EDP			0x6f848
-#define PSR_EVENT(tran)				_MMIO_TRANS2(tran, _PSR_EVENT_TRANS_A)
+#define PSR_EVENT(tran)				_MMIO_TRANS2(dev_priv, tran, _PSR_EVENT_TRANS_A)
 #define  PSR_EVENT_PSR2_WD_TIMER_EXPIRE		REG_BIT(17)
 #define  PSR_EVENT_PSR2_DISABLED		REG_BIT(16)
 #define  PSR_EVENT_SU_DIRTY_FIFO_UNDERRUN	REG_BIT(15)
@@ -215,13 +215,13 @@
 
 #define _PSR2_STATUS_A				0x60940
 #define _PSR2_STATUS_EDP			0x6f940
-#define EDP_PSR2_STATUS(tran)			_MMIO_TRANS2(tran, _PSR2_STATUS_A)
+#define EDP_PSR2_STATUS(tran)			_MMIO_TRANS2(dev_priv, tran, _PSR2_STATUS_A)
 #define EDP_PSR2_STATUS_STATE_MASK		REG_GENMASK(31, 28)
 #define EDP_PSR2_STATUS_STATE_DEEP_SLEEP	REG_FIELD_PREP(EDP_PSR2_STATUS_STATE_MASK, 0x8)
 
 #define _PSR2_SU_STATUS_A		0x60914
 #define _PSR2_SU_STATUS_EDP		0x6f914
-#define _PSR2_SU_STATUS(tran, index)	_MMIO_TRANS2(tran, _PSR2_SU_STATUS_A + (index) * 4)
+#define _PSR2_SU_STATUS(tran, index)	_MMIO_TRANS2(dev_priv, tran, _PSR2_SU_STATUS_A + (index) * 4)
 #define PSR2_SU_STATUS(tran, frame)	(_PSR2_SU_STATUS(tran, (frame) / 3))
 #define PSR2_SU_STATUS_SHIFT(frame)	(((frame) % 3) * 10)
 #define PSR2_SU_STATUS_MASK(frame)	(0x3ff << PSR2_SU_STATUS_SHIFT(frame))
@@ -229,7 +229,7 @@
 
 #define _PSR2_MAN_TRK_CTL_A					0x60910
 #define _PSR2_MAN_TRK_CTL_EDP					0x6f910
-#define PSR2_MAN_TRK_CTL(tran)					_MMIO_TRANS2(tran, _PSR2_MAN_TRK_CTL_A)
+#define PSR2_MAN_TRK_CTL(tran)					_MMIO_TRANS2(dev_priv, tran, _PSR2_MAN_TRK_CTL_A)
 #define  PSR2_MAN_TRK_CTL_ENABLE				REG_BIT(31)
 #define  PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK		REG_GENMASK(30, 21)
 #define  PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(val)		REG_FIELD_PREP(PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR_MASK, val)
@@ -249,7 +249,7 @@
 /* PSR2 Early transport */
 #define _PIPE_SRCSZ_ERLY_TPT_A	0x70074
 
-#define PIPE_SRCSZ_ERLY_TPT(trans)	_MMIO_TRANS2(trans, _PIPE_SRCSZ_ERLY_TPT_A)
+#define PIPE_SRCSZ_ERLY_TPT(trans)	_MMIO_TRANS2(dev_priv, trans, _PIPE_SRCSZ_ERLY_TPT_A)
 
 #define _SEL_FETCH_PLANE_BASE_1_A		0x70890
 #define _SEL_FETCH_PLANE_BASE_2_A		0x708B0
@@ -297,7 +297,7 @@
 						  _SEL_FETCH_PLANE_BASE_1_A)
 
 #define _ALPM_CTL_A	0x60950
-#define ALPM_CTL(tran)	_MMIO_TRANS2(tran, _ALPM_CTL_A)
+#define ALPM_CTL(tran)	_MMIO_TRANS2(dev_priv, tran, _ALPM_CTL_A)
 #define  ALPM_CTL_ALPM_ENABLE				REG_BIT(31)
 #define  ALPM_CTL_ALPM_AUX_LESS_ENABLE			REG_BIT(30)
 #define  ALPM_CTL_LOBF_ENABLE				REG_BIT(29)
@@ -321,7 +321,7 @@
 #define  ALPM_CTL_AUX_LESS_WAKE_TIME(val)		REG_FIELD_PREP(ALPM_CTL_AUX_LESS_WAKE_TIME_MASK, val)
 
 #define _ALPM_CTL2_A	0x60954
-#define ALPM_CTL2(tran)	_MMIO_TRANS2(tran, _ALPM_CTL2_A)
+#define ALPM_CTL2(tran)	_MMIO_TRANS2(dev_priv, tran, _ALPM_CTL2_A)
 #define  ALPM_CTL2_SWITCH_TO_ACTIVE_LATENCY_MASK		REG_GENMASK(28, 24)
 #define  ALPM_CTL2_SWITCH_TO_ACTIVE_LATENCY(val)		REG_FIELD_PREP(ALPM_CTL2_SWITCH_TO_ACTIVE_LATENCY_MASK, val)
 #define  ALPM_CTL2_AUX_LESS_WAKE_TIME_EXTENSION_MASK		REG_GENMASK(19, 16)
@@ -335,7 +335,7 @@
 #define  ALPM_CTL2_NUMBER_AUX_LESS_ML_PHY_SLEEP_SEQUENCES(val)	REG_FIELD_PREP(ALPM_CTL2_NUMBER_AUX_LESS_ML_PHY_SLEEP_SEQUENCES_MASK, val)
 
 #define _PORT_ALPM_CTL_A			0x16fa2c
-#define PORT_ALPM_CTL(tran)			_MMIO_TRANS2(tran, _PORT_ALPM_CTL_A)
+#define PORT_ALPM_CTL(tran)			_MMIO_TRANS2(dev_priv, tran, _PORT_ALPM_CTL_A)
 #define  PORT_ALPM_CTL_ALPM_AUX_LESS_ENABLE	REG_BIT(31)
 #define  PORT_ALPM_CTL_MAX_PHY_SWING_SETUP_MASK	REG_GENMASK(23, 20)
 #define  PORT_ALPM_CTL_MAX_PHY_SWING_SETUP(val)	REG_FIELD_PREP(PORT_ALPM_CTL_MAX_PHY_SWING_SETUP_MASK, val)
@@ -345,12 +345,16 @@
 #define  PORT_ALPM_CTL_SILENCE_PERIOD(val)	REG_FIELD_PREP(PORT_ALPM_CTL_SILENCE_PERIOD_MASK, val)
 
 #define _PORT_ALPM_LFPS_CTL_A					0x16fa30
-#define PORT_ALPM_LFPS_CTL(tran)				_MMIO_TRANS2(tran, _PORT_ALPM_LFPS_CTL_A)
+#define PORT_ALPM_LFPS_CTL(tran)				_MMIO_TRANS2(dev_priv, tran, _PORT_ALPM_LFPS_CTL_A)
 #define  PORT_ALPM_LFPS_CTL_LFPS_START_POLARITY			REG_BIT(31)
 #define  PORT_ALPM_LFPS_CTL_LFPS_CYCLE_COUNT_MASK		REG_GENMASK(27, 24)
-#define  ALPM_CTL_EXTENDED_FAST_WAKE_MIN_LINES		5
-#define  ALPM_CTL_EXTENDED_FAST_WAKE_TIME(lines)	REG_FIELD_PREP(ALPM_CTL_EXTENDED_FAST_WAKE_TIME_MASK, (lines) - ALPM_CTL_EXTENDED_FAST_WAKE_MIN_LINES)
-#define  ALPM_CTL_AUX_LESS_WAKE_TIME_MASK		REG_GENMASK(5, 0)
-#define  ALPM_CTL_AUX_LESS_WAKE_TIME(val)		REG_FIELD_PREP(ALPM_CTL_AUX_LESS_WAKE_TIME_MASK, val)
+#define  PORT_ALPM_LFPS_CTL_LFPS_CYCLE_COUNT_MIN		7
+#define  PORT_ALPM_LFPS_CTL_LFPS_CYCLE_COUNT(val)		REG_FIELD_PREP(PORT_ALPM_LFPS_CTL_LFPS_CYCLE_COUNT_MASK, (val) - PORT_ALPM_LFPS_CTL_LFPS_CYCLE_COUNT_MIN)
+#define  PORT_ALPM_LFPS_CTL_LFPS_HALF_CYCLE_DURATION_MASK	REG_GENMASK(20, 16)
+#define  PORT_ALPM_LFPS_CTL_LFPS_HALF_CYCLE_DURATION(val)	REG_FIELD_PREP(PORT_ALPM_LFPS_CTL_LFPS_HALF_CYCLE_DURATION_MASK, val)
+#define  PORT_ALPM_LFPS_CTL_FIRST_LFPS_HALF_CYCLE_DURATION_MASK	REG_GENMASK(12, 8)
+#define  PORT_ALPM_LFPS_CTL_FIRST_LFPS_HALF_CYCLE_DURATION(val)	REG_FIELD_PREP(PORT_ALPM_LFPS_CTL_LFPS_HALF_CYCLE_DURATION_MASK, val)
+#define  PORT_ALPM_LFPS_CTL_LAST_LFPS_HALF_CYCLE_DURATION_MASK	REG_GENMASK(4, 0)
+#define  PORT_ALPM_LFPS_CTL_LAST_LFPS_HALF_CYCLE_DURATION(val)	REG_FIELD_PREP(PORT_ALPM_LFPS_CTL_LFPS_HALF_CYCLE_DURATION_MASK, val)
 
 #endif /* __INTEL_PSR_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c
index a280448df771..14d5fefc9c5b 100644
--- a/drivers/gpu/drm/i915/display/intel_quirks.c
+++ b/drivers/gpu/drm/i915/display/intel_quirks.c
@@ -9,72 +9,72 @@
 #include "intel_display_types.h"
 #include "intel_quirks.h"
 
-static void intel_set_quirk(struct drm_i915_private *i915, enum intel_quirk_id quirk)
+static void intel_set_quirk(struct intel_display *display, enum intel_quirk_id quirk)
 {
-	i915->display.quirks.mask |= BIT(quirk);
+	display->quirks.mask |= BIT(quirk);
 }
 
 /*
  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
  */
-static void quirk_ssc_force_disable(struct drm_i915_private *i915)
+static void quirk_ssc_force_disable(struct intel_display *display)
 {
-	intel_set_quirk(i915, QUIRK_LVDS_SSC_DISABLE);
-	drm_info(&i915->drm, "applying lvds SSC disable quirk\n");
+	intel_set_quirk(display, QUIRK_LVDS_SSC_DISABLE);
+	drm_info(display->drm, "applying lvds SSC disable quirk\n");
 }
 
 /*
  * A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight
  * brightness value
  */
-static void quirk_invert_brightness(struct drm_i915_private *i915)
+static void quirk_invert_brightness(struct intel_display *display)
 {
-	intel_set_quirk(i915, QUIRK_INVERT_BRIGHTNESS);
-	drm_info(&i915->drm, "applying inverted panel brightness quirk\n");
+	intel_set_quirk(display, QUIRK_INVERT_BRIGHTNESS);
+	drm_info(display->drm, "applying inverted panel brightness quirk\n");
 }
 
 /* Some VBT's incorrectly indicate no backlight is present */
-static void quirk_backlight_present(struct drm_i915_private *i915)
+static void quirk_backlight_present(struct intel_display *display)
 {
-	intel_set_quirk(i915, QUIRK_BACKLIGHT_PRESENT);
-	drm_info(&i915->drm, "applying backlight present quirk\n");
+	intel_set_quirk(display, QUIRK_BACKLIGHT_PRESENT);
+	drm_info(display->drm, "applying backlight present quirk\n");
 }
 
 /* Toshiba Satellite P50-C-18C requires T12 delay to be min 800ms
  * which is 300 ms greater than eDP spec T12 min.
  */
-static void quirk_increase_t12_delay(struct drm_i915_private *i915)
+static void quirk_increase_t12_delay(struct intel_display *display)
 {
-	intel_set_quirk(i915, QUIRK_INCREASE_T12_DELAY);
-	drm_info(&i915->drm, "Applying T12 delay quirk\n");
+	intel_set_quirk(display, QUIRK_INCREASE_T12_DELAY);
+	drm_info(display->drm, "Applying T12 delay quirk\n");
 }
 
 /*
  * GeminiLake NUC HDMI outputs require additional off time
  * this allows the onboard retimer to correctly sync to signal
  */
-static void quirk_increase_ddi_disabled_time(struct drm_i915_private *i915)
+static void quirk_increase_ddi_disabled_time(struct intel_display *display)
 {
-	intel_set_quirk(i915, QUIRK_INCREASE_DDI_DISABLED_TIME);
-	drm_info(&i915->drm, "Applying Increase DDI Disabled quirk\n");
+	intel_set_quirk(display, QUIRK_INCREASE_DDI_DISABLED_TIME);
+	drm_info(display->drm, "Applying Increase DDI Disabled quirk\n");
 }
 
-static void quirk_no_pps_backlight_power_hook(struct drm_i915_private *i915)
+static void quirk_no_pps_backlight_power_hook(struct intel_display *display)
 {
-	intel_set_quirk(i915, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK);
-	drm_info(&i915->drm, "Applying no pps backlight power quirk\n");
+	intel_set_quirk(display, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK);
+	drm_info(display->drm, "Applying no pps backlight power quirk\n");
 }
 
 struct intel_quirk {
 	int device;
 	int subsystem_vendor;
 	int subsystem_device;
-	void (*hook)(struct drm_i915_private *i915);
+	void (*hook)(struct intel_display *display);
 };
 
 /* For systems that don't have a meaningful PCI subdevice/subvendor ID */
 struct intel_dmi_quirk {
-	void (*hook)(struct drm_i915_private *i915);
+	void (*hook)(struct intel_display *display);
 	const struct dmi_system_id (*dmi_id_list)[];
 };
 
@@ -203,9 +203,9 @@ static struct intel_quirk intel_quirks[] = {
 	{ 0x0f31, 0x103c, 0x220f, quirk_invert_brightness },
 };
 
-void intel_init_quirks(struct drm_i915_private *i915)
+void intel_init_quirks(struct intel_display *display)
 {
-	struct pci_dev *d = to_pci_dev(i915->drm.dev);
+	struct pci_dev *d = to_pci_dev(display->drm->dev);
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(intel_quirks); i++) {
@@ -216,15 +216,15 @@ void intel_init_quirks(struct drm_i915_private *i915)
 		     q->subsystem_vendor == PCI_ANY_ID) &&
 		    (d->subsystem_device == q->subsystem_device ||
 		     q->subsystem_device == PCI_ANY_ID))
-			q->hook(i915);
+			q->hook(display);
 	}
 	for (i = 0; i < ARRAY_SIZE(intel_dmi_quirks); i++) {
 		if (dmi_check_system(*intel_dmi_quirks[i].dmi_id_list) != 0)
-			intel_dmi_quirks[i].hook(i915);
+			intel_dmi_quirks[i].hook(display);
 	}
 }
 
-bool intel_has_quirk(struct drm_i915_private *i915, enum intel_quirk_id quirk)
+bool intel_has_quirk(struct intel_display *display, enum intel_quirk_id quirk)
 {
-	return i915->display.quirks.mask & BIT(quirk);
+	return display->quirks.mask & BIT(quirk);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_quirks.h b/drivers/gpu/drm/i915/display/intel_quirks.h
index 10a4d163149f..151c8f4ae576 100644
--- a/drivers/gpu/drm/i915/display/intel_quirks.h
+++ b/drivers/gpu/drm/i915/display/intel_quirks.h
@@ -8,7 +8,7 @@
 
 #include <linux/types.h>
 
-struct drm_i915_private;
+struct intel_display;
 
 enum intel_quirk_id {
 	QUIRK_BACKLIGHT_PRESENT,
@@ -19,7 +19,7 @@ enum intel_quirk_id {
 	QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK,
 };
 
-void intel_init_quirks(struct drm_i915_private *i915);
-bool intel_has_quirk(struct drm_i915_private *i915, enum intel_quirk_id quirk);
+void intel_init_quirks(struct intel_display *display);
+bool intel_has_quirk(struct intel_display *display, enum intel_quirk_id quirk);
 
 #endif /* __INTEL_QUIRKS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c
index 0cd9c183f621..d0d712405129 100644
--- a/drivers/gpu/drm/i915/display/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/display/intel_sdvo.c
@@ -193,7 +193,7 @@ to_intel_sdvo_connector(struct drm_connector *connector)
 }
 
 #define to_intel_sdvo_connector_state(conn_state) \
-	container_of((conn_state), struct intel_sdvo_connector_state, base.base)
+	container_of_const((conn_state), struct intel_sdvo_connector_state, base.base)
 
 static bool
 intel_sdvo_output_setup(struct intel_sdvo *intel_sdvo);
@@ -1944,7 +1944,7 @@ intel_sdvo_mode_valid(struct drm_connector *connector,
 	struct intel_sdvo_connector *intel_sdvo_connector =
 		to_intel_sdvo_connector(connector);
 	bool has_hdmi_sink = intel_has_hdmi_sink(intel_sdvo_connector, connector->state);
-	int max_dotclk = i915->max_dotclk_freq;
+	int max_dotclk = i915->display.cdclk.max_dotclk_freq;
 	enum drm_mode_status status;
 	int clock = mode->clock;
 
@@ -1952,9 +1952,6 @@ intel_sdvo_mode_valid(struct drm_connector *connector,
 	if (status != MODE_OK)
 		return status;
 
-	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
-		return MODE_NO_DBLESCAN;
-
 	if (clock > max_dotclk)
 		return MODE_CLOCK_HIGH;
 
@@ -2378,7 +2375,7 @@ intel_sdvo_connector_atomic_get_property(struct drm_connector *connector,
 					 u64 *val)
 {
 	struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector);
-	const struct intel_sdvo_connector_state *sdvo_state = to_intel_sdvo_connector_state((void *)state);
+	const struct intel_sdvo_connector_state *sdvo_state = to_intel_sdvo_connector_state(state);
 
 	if (property == intel_sdvo_connector->tv_format) {
 		int i;
diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.c b/drivers/gpu/drm/i915/display/intel_snps_phy.c
index bc61e736f9b3..e6df1f92def5 100644
--- a/drivers/gpu/drm/i915/display/intel_snps_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_snps_phy.c
@@ -44,12 +44,14 @@ void intel_snps_phy_wait_for_calibration(struct drm_i915_private *i915)
 	}
 }
 
-void intel_snps_phy_update_psr_power_state(struct drm_i915_private *i915,
-					   enum phy phy, bool enable)
+void intel_snps_phy_update_psr_power_state(struct intel_encoder *encoder,
+					   bool enable)
 {
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	u32 val;
 
-	if (!intel_phy_is_snps(i915, phy))
+	if (!intel_encoder_is_snps(encoder))
 		return;
 
 	val = REG_FIELD_PREP(SNPS_PHY_TX_REQ_LN_DIS_PWR_STATE_PSR,
@@ -63,7 +65,7 @@ void intel_snps_phy_set_signal_levels(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	const struct intel_ddi_buf_trans *trans;
-	enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	int n_entries, ln;
 
 	trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
@@ -1809,7 +1811,7 @@ int intel_mpllb_calc_state(struct intel_crtc_state *crtc_state,
 
 	for (i = 0; tables[i]; i++) {
 		if (crtc_state->port_clock == tables[i]->clock) {
-			crtc_state->mpllb_state = *tables[i];
+			crtc_state->dpll_hw_state.mpllb = *tables[i];
 			return 0;
 		}
 	}
@@ -1821,8 +1823,8 @@ void intel_mpllb_enable(struct intel_encoder *encoder,
 			const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	const struct intel_mpllb_state *pll_state = &crtc_state->mpllb_state;
-	enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
+	const struct intel_mpllb_state *pll_state = &crtc_state->dpll_hw_state.mpllb;
+	enum phy phy = intel_encoder_to_phy(encoder);
 	i915_reg_t enable_reg = (phy <= PHY_D ?
 				 DG2_PLL_ENABLE(phy) : MG_PLL_ENABLE(0));
 
@@ -1879,7 +1881,7 @@ void intel_mpllb_enable(struct intel_encoder *encoder,
 void intel_mpllb_disable(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	i915_reg_t enable_reg = (phy <= PHY_D ?
 				 DG2_PLL_ENABLE(phy) : MG_PLL_ENABLE(0));
 
@@ -1951,7 +1953,7 @@ void intel_mpllb_readout_hw_state(struct intel_encoder *encoder,
 				  struct intel_mpllb_state *pll_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
+	enum phy phy = intel_encoder_to_phy(encoder);
 
 	pll_state->mpllb_cp = intel_de_read(dev_priv, SNPS_PHY_MPLLB_CP(phy));
 	pll_state->mpllb_div = intel_de_read(dev_priv, SNPS_PHY_MPLLB_DIV(phy));
@@ -1999,7 +2001,7 @@ void intel_mpllb_state_verify(struct intel_atomic_state *state,
 	const struct intel_crtc_state *new_crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_mpllb_state mpllb_hw_state = {};
-	const struct intel_mpllb_state *mpllb_sw_state = &new_crtc_state->mpllb_state;
+	const struct intel_mpllb_state *mpllb_sw_state = &new_crtc_state->dpll_hw_state.mpllb;
 	struct intel_encoder *encoder;
 
 	if (!IS_DG2(i915))
diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.h b/drivers/gpu/drm/i915/display/intel_snps_phy.h
index 515abf7c5902..bc08b92a7cd9 100644
--- a/drivers/gpu/drm/i915/display/intel_snps_phy.h
+++ b/drivers/gpu/drm/i915/display/intel_snps_phy.h
@@ -17,8 +17,8 @@ struct intel_mpllb_state;
 enum phy;
 
 void intel_snps_phy_wait_for_calibration(struct drm_i915_private *dev_priv);
-void intel_snps_phy_update_psr_power_state(struct drm_i915_private *dev_priv,
-					   enum phy phy, bool enable);
+void intel_snps_phy_update_psr_power_state(struct intel_encoder *encoder,
+					   bool enable);
 
 int intel_mpllb_calc_state(struct intel_crtc_state *crtc_state,
 			   struct intel_encoder *encoder);
diff --git a/drivers/gpu/drm/i915/display/intel_sprite.c b/drivers/gpu/drm/i915/display/intel_sprite.c
index d7b440c8caef..36a253a19c74 100644
--- a/drivers/gpu/drm/i915/display/intel_sprite.c
+++ b/drivers/gpu/drm/i915/display/intel_sprite.c
@@ -47,6 +47,7 @@
 #include "intel_fb.h"
 #include "intel_frontbuffer.h"
 #include "intel_sprite.h"
+#include "intel_sprite_regs.h"
 
 static char sprite_name(struct drm_i915_private *i915, enum pipe pipe, int sprite)
 {
diff --git a/drivers/gpu/drm/i915/display/intel_sprite_regs.h b/drivers/gpu/drm/i915/display/intel_sprite_regs.h
new file mode 100644
index 000000000000..bb67705652b2
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_sprite_regs.h
@@ -0,0 +1,348 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2024 Intel Corporation */
+
+#ifndef __INTEL_SPRITE_REGS__
+#define __INTEL_SPRITE_REGS__
+
+#include "intel_display_reg_defs.h"
+
+#define _DVSACNTR		0x72180
+#define   DVS_ENABLE			REG_BIT(31)
+#define   DVS_PIPE_GAMMA_ENABLE		REG_BIT(30)
+#define   DVS_YUV_RANGE_CORRECTION_DISABLE	REG_BIT(27)
+#define   DVS_FORMAT_MASK		REG_GENMASK(26, 25)
+#define   DVS_FORMAT_YUV422		REG_FIELD_PREP(DVS_FORMAT_MASK, 0)
+#define   DVS_FORMAT_RGBX101010		REG_FIELD_PREP(DVS_FORMAT_MASK, 1)
+#define   DVS_FORMAT_RGBX888		REG_FIELD_PREP(DVS_FORMAT_MASK, 2)
+#define   DVS_FORMAT_RGBX161616		REG_FIELD_PREP(DVS_FORMAT_MASK, 3)
+#define   DVS_PIPE_CSC_ENABLE		REG_BIT(24)
+#define   DVS_SOURCE_KEY		REG_BIT(22)
+#define   DVS_RGB_ORDER_XBGR		REG_BIT(20)
+#define   DVS_YUV_FORMAT_BT709		REG_BIT(18)
+#define   DVS_YUV_ORDER_MASK		REG_GENMASK(17, 16)
+#define   DVS_YUV_ORDER_YUYV		REG_FIELD_PREP(DVS_YUV_ORDER_MASK, 0)
+#define   DVS_YUV_ORDER_UYVY		REG_FIELD_PREP(DVS_YUV_ORDER_MASK, 1)
+#define   DVS_YUV_ORDER_YVYU		REG_FIELD_PREP(DVS_YUV_ORDER_MASK, 2)
+#define   DVS_YUV_ORDER_VYUY		REG_FIELD_PREP(DVS_YUV_ORDER_MASK, 3)
+#define   DVS_ROTATE_180		REG_BIT(15)
+#define   DVS_TRICKLE_FEED_DISABLE	REG_BIT(14)
+#define   DVS_TILED			REG_BIT(10)
+#define   DVS_DEST_KEY			REG_BIT(2)
+#define _DVSALINOFF		0x72184
+#define _DVSASTRIDE		0x72188
+#define _DVSAPOS		0x7218c
+#define   DVS_POS_Y_MASK		REG_GENMASK(31, 16)
+#define   DVS_POS_Y(y)			REG_FIELD_PREP(DVS_POS_Y_MASK, (y))
+#define   DVS_POS_X_MASK		REG_GENMASK(15, 0)
+#define   DVS_POS_X(x)			REG_FIELD_PREP(DVS_POS_X_MASK, (x))
+#define _DVSASIZE		0x72190
+#define   DVS_HEIGHT_MASK		REG_GENMASK(31, 16)
+#define   DVS_HEIGHT(h)			REG_FIELD_PREP(DVS_HEIGHT_MASK, (h))
+#define   DVS_WIDTH_MASK		REG_GENMASK(15, 0)
+#define   DVS_WIDTH(w)			REG_FIELD_PREP(DVS_WIDTH_MASK, (w))
+#define _DVSAKEYVAL		0x72194
+#define _DVSAKEYMSK		0x72198
+#define _DVSASURF		0x7219c
+#define   DVS_ADDR_MASK			REG_GENMASK(31, 12)
+#define _DVSAKEYMAXVAL		0x721a0
+#define _DVSATILEOFF		0x721a4
+#define   DVS_OFFSET_Y_MASK		REG_GENMASK(31, 16)
+#define   DVS_OFFSET_Y(y)		REG_FIELD_PREP(DVS_OFFSET_Y_MASK, (y))
+#define   DVS_OFFSET_X_MASK		REG_GENMASK(15, 0)
+#define   DVS_OFFSET_X(x)		REG_FIELD_PREP(DVS_OFFSET_X_MASK, (x))
+#define _DVSASURFLIVE		0x721ac
+#define _DVSAGAMC_G4X		0x721e0 /* g4x */
+#define _DVSASCALE		0x72204
+#define   DVS_SCALE_ENABLE		REG_BIT(31)
+#define   DVS_FILTER_MASK		REG_GENMASK(30, 29)
+#define   DVS_FILTER_MEDIUM		REG_FIELD_PREP(DVS_FILTER_MASK, 0)
+#define   DVS_FILTER_ENHANCING		REG_FIELD_PREP(DVS_FILTER_MASK, 1)
+#define   DVS_FILTER_SOFTENING		REG_FIELD_PREP(DVS_FILTER_MASK, 2)
+#define   DVS_VERTICAL_OFFSET_HALF	REG_BIT(28) /* must be enabled below */
+#define   DVS_VERTICAL_OFFSET_ENABLE	REG_BIT(27)
+#define   DVS_SRC_WIDTH_MASK		REG_GENMASK(26, 16)
+#define   DVS_SRC_WIDTH(w)		REG_FIELD_PREP(DVS_SRC_WIDTH_MASK, (w))
+#define   DVS_SRC_HEIGHT_MASK		REG_GENMASK(10, 0)
+#define   DVS_SRC_HEIGHT(h)		REG_FIELD_PREP(DVS_SRC_HEIGHT_MASK, (h))
+#define _DVSAGAMC_ILK		0x72300 /* ilk/snb */
+#define _DVSAGAMCMAX_ILK	0x72340 /* ilk/snb */
+
+#define _DVSBCNTR		0x73180
+#define _DVSBLINOFF		0x73184
+#define _DVSBSTRIDE		0x73188
+#define _DVSBPOS		0x7318c
+#define _DVSBSIZE		0x73190
+#define _DVSBKEYVAL		0x73194
+#define _DVSBKEYMSK		0x73198
+#define _DVSBSURF		0x7319c
+#define _DVSBKEYMAXVAL		0x731a0
+#define _DVSBTILEOFF		0x731a4
+#define _DVSBSURFLIVE		0x731ac
+#define _DVSBGAMC_G4X		0x731e0 /* g4x */
+#define _DVSBSCALE		0x73204
+#define _DVSBGAMC_ILK		0x73300 /* ilk/snb */
+#define _DVSBGAMCMAX_ILK	0x73340 /* ilk/snb */
+
+#define DVSCNTR(pipe) _MMIO_PIPE(pipe, _DVSACNTR, _DVSBCNTR)
+#define DVSLINOFF(pipe) _MMIO_PIPE(pipe, _DVSALINOFF, _DVSBLINOFF)
+#define DVSSTRIDE(pipe) _MMIO_PIPE(pipe, _DVSASTRIDE, _DVSBSTRIDE)
+#define DVSPOS(pipe) _MMIO_PIPE(pipe, _DVSAPOS, _DVSBPOS)
+#define DVSSURF(pipe) _MMIO_PIPE(pipe, _DVSASURF, _DVSBSURF)
+#define DVSKEYMAX(pipe) _MMIO_PIPE(pipe, _DVSAKEYMAXVAL, _DVSBKEYMAXVAL)
+#define DVSSIZE(pipe) _MMIO_PIPE(pipe, _DVSASIZE, _DVSBSIZE)
+#define DVSSCALE(pipe) _MMIO_PIPE(pipe, _DVSASCALE, _DVSBSCALE)
+#define DVSTILEOFF(pipe) _MMIO_PIPE(pipe, _DVSATILEOFF, _DVSBTILEOFF)
+#define DVSKEYVAL(pipe) _MMIO_PIPE(pipe, _DVSAKEYVAL, _DVSBKEYVAL)
+#define DVSKEYMSK(pipe) _MMIO_PIPE(pipe, _DVSAKEYMSK, _DVSBKEYMSK)
+#define DVSSURFLIVE(pipe) _MMIO_PIPE(pipe, _DVSASURFLIVE, _DVSBSURFLIVE)
+#define DVSGAMC_G4X(pipe, i) _MMIO(_PIPE(pipe, _DVSAGAMC_G4X, _DVSBGAMC_G4X) + (5 - (i)) * 4) /* 6 x u0.8 */
+#define DVSGAMC_ILK(pipe, i) _MMIO(_PIPE(pipe, _DVSAGAMC_ILK, _DVSBGAMC_ILK) + (i) * 4) /* 16 x u0.10 */
+#define DVSGAMCMAX_ILK(pipe, i) _MMIO(_PIPE(pipe, _DVSAGAMCMAX_ILK, _DVSBGAMCMAX_ILK) + (i) * 4) /* 3 x u1.10 */
+
+#define _SPRA_CTL		0x70280
+#define   SPRITE_ENABLE				REG_BIT(31)
+#define   SPRITE_PIPE_GAMMA_ENABLE		REG_BIT(30)
+#define   SPRITE_YUV_RANGE_CORRECTION_DISABLE	REG_BIT(28)
+#define   SPRITE_FORMAT_MASK			REG_GENMASK(27, 25)
+#define   SPRITE_FORMAT_YUV422			REG_FIELD_PREP(SPRITE_FORMAT_MASK, 0)
+#define   SPRITE_FORMAT_RGBX101010		REG_FIELD_PREP(SPRITE_FORMAT_MASK, 1)
+#define   SPRITE_FORMAT_RGBX888			REG_FIELD_PREP(SPRITE_FORMAT_MASK, 2)
+#define   SPRITE_FORMAT_RGBX161616		REG_FIELD_PREP(SPRITE_FORMAT_MASK, 3)
+#define   SPRITE_FORMAT_YUV444			REG_FIELD_PREP(SPRITE_FORMAT_MASK, 4)
+#define   SPRITE_FORMAT_XR_BGR101010		REG_FIELD_PREP(SPRITE_FORMAT_MASK, 5) /* Extended range */
+#define   SPRITE_PIPE_CSC_ENABLE		REG_BIT(24)
+#define   SPRITE_SOURCE_KEY			REG_BIT(22)
+#define   SPRITE_RGB_ORDER_RGBX			REG_BIT(20) /* only for 888 and 161616 */
+#define   SPRITE_YUV_TO_RGB_CSC_DISABLE		REG_BIT(19)
+#define   SPRITE_YUV_TO_RGB_CSC_FORMAT_BT709	REG_BIT(18) /* 0 is BT601 */
+#define   SPRITE_YUV_ORDER_MASK			REG_GENMASK(17, 16)
+#define   SPRITE_YUV_ORDER_YUYV			REG_FIELD_PREP(SPRITE_YUV_ORDER_MASK, 0)
+#define   SPRITE_YUV_ORDER_UYVY			REG_FIELD_PREP(SPRITE_YUV_ORDER_MASK, 1)
+#define   SPRITE_YUV_ORDER_YVYU			REG_FIELD_PREP(SPRITE_YUV_ORDER_MASK, 2)
+#define   SPRITE_YUV_ORDER_VYUY			REG_FIELD_PREP(SPRITE_YUV_ORDER_MASK, 3)
+#define   SPRITE_ROTATE_180			REG_BIT(15)
+#define   SPRITE_TRICKLE_FEED_DISABLE		REG_BIT(14)
+#define   SPRITE_PLANE_GAMMA_DISABLE		REG_BIT(13)
+#define   SPRITE_TILED				REG_BIT(10)
+#define   SPRITE_DEST_KEY			REG_BIT(2)
+#define _SPRA_LINOFF		0x70284
+#define _SPRA_STRIDE		0x70288
+#define _SPRA_POS		0x7028c
+#define   SPRITE_POS_Y_MASK	REG_GENMASK(31, 16)
+#define   SPRITE_POS_Y(y)	REG_FIELD_PREP(SPRITE_POS_Y_MASK, (y))
+#define   SPRITE_POS_X_MASK	REG_GENMASK(15, 0)
+#define   SPRITE_POS_X(x)	REG_FIELD_PREP(SPRITE_POS_X_MASK, (x))
+#define _SPRA_SIZE		0x70290
+#define   SPRITE_HEIGHT_MASK	REG_GENMASK(31, 16)
+#define   SPRITE_HEIGHT(h)	REG_FIELD_PREP(SPRITE_HEIGHT_MASK, (h))
+#define   SPRITE_WIDTH_MASK	REG_GENMASK(15, 0)
+#define   SPRITE_WIDTH(w)	REG_FIELD_PREP(SPRITE_WIDTH_MASK, (w))
+#define _SPRA_KEYVAL		0x70294
+#define _SPRA_KEYMSK		0x70298
+#define _SPRA_SURF		0x7029c
+#define   SPRITE_ADDR_MASK	REG_GENMASK(31, 12)
+#define _SPRA_KEYMAX		0x702a0
+#define _SPRA_TILEOFF		0x702a4
+#define   SPRITE_OFFSET_Y_MASK	REG_GENMASK(31, 16)
+#define   SPRITE_OFFSET_Y(y)	REG_FIELD_PREP(SPRITE_OFFSET_Y_MASK, (y))
+#define   SPRITE_OFFSET_X_MASK	REG_GENMASK(15, 0)
+#define   SPRITE_OFFSET_X(x)	REG_FIELD_PREP(SPRITE_OFFSET_X_MASK, (x))
+#define _SPRA_OFFSET		0x702a4
+#define _SPRA_SURFLIVE		0x702ac
+#define _SPRA_SCALE		0x70304
+#define   SPRITE_SCALE_ENABLE			REG_BIT(31)
+#define   SPRITE_FILTER_MASK			REG_GENMASK(30, 29)
+#define   SPRITE_FILTER_MEDIUM			REG_FIELD_PREP(SPRITE_FILTER_MASK, 0)
+#define   SPRITE_FILTER_ENHANCING		REG_FIELD_PREP(SPRITE_FILTER_MASK, 1)
+#define   SPRITE_FILTER_SOFTENING		REG_FIELD_PREP(SPRITE_FILTER_MASK, 2)
+#define   SPRITE_VERTICAL_OFFSET_HALF		REG_BIT(28) /* must be enabled below */
+#define   SPRITE_VERTICAL_OFFSET_ENABLE		REG_BIT(27)
+#define   SPRITE_SRC_WIDTH_MASK			REG_GENMASK(26, 16)
+#define   SPRITE_SRC_WIDTH(w)			REG_FIELD_PREP(SPRITE_SRC_WIDTH_MASK, (w))
+#define   SPRITE_SRC_HEIGHT_MASK		REG_GENMASK(10, 0)
+#define   SPRITE_SRC_HEIGHT(h)			REG_FIELD_PREP(SPRITE_SRC_HEIGHT_MASK, (h))
+#define _SPRA_GAMC		0x70400
+#define _SPRA_GAMC16		0x70440
+#define _SPRA_GAMC17		0x7044c
+
+#define _SPRB_CTL		0x71280
+#define _SPRB_LINOFF		0x71284
+#define _SPRB_STRIDE		0x71288
+#define _SPRB_POS		0x7128c
+#define _SPRB_SIZE		0x71290
+#define _SPRB_KEYVAL		0x71294
+#define _SPRB_KEYMSK		0x71298
+#define _SPRB_SURF		0x7129c
+#define _SPRB_KEYMAX		0x712a0
+#define _SPRB_TILEOFF		0x712a4
+#define _SPRB_OFFSET		0x712a4
+#define _SPRB_SURFLIVE		0x712ac
+#define _SPRB_SCALE		0x71304
+#define _SPRB_GAMC		0x71400
+#define _SPRB_GAMC16		0x71440
+#define _SPRB_GAMC17		0x7144c
+
+#define SPRCTL(pipe) _MMIO_PIPE(pipe, _SPRA_CTL, _SPRB_CTL)
+#define SPRLINOFF(pipe) _MMIO_PIPE(pipe, _SPRA_LINOFF, _SPRB_LINOFF)
+#define SPRSTRIDE(pipe) _MMIO_PIPE(pipe, _SPRA_STRIDE, _SPRB_STRIDE)
+#define SPRPOS(pipe) _MMIO_PIPE(pipe, _SPRA_POS, _SPRB_POS)
+#define SPRSIZE(pipe) _MMIO_PIPE(pipe, _SPRA_SIZE, _SPRB_SIZE)
+#define SPRKEYVAL(pipe) _MMIO_PIPE(pipe, _SPRA_KEYVAL, _SPRB_KEYVAL)
+#define SPRKEYMSK(pipe) _MMIO_PIPE(pipe, _SPRA_KEYMSK, _SPRB_KEYMSK)
+#define SPRSURF(pipe) _MMIO_PIPE(pipe, _SPRA_SURF, _SPRB_SURF)
+#define SPRKEYMAX(pipe) _MMIO_PIPE(pipe, _SPRA_KEYMAX, _SPRB_KEYMAX)
+#define SPRTILEOFF(pipe) _MMIO_PIPE(pipe, _SPRA_TILEOFF, _SPRB_TILEOFF)
+#define SPROFFSET(pipe) _MMIO_PIPE(pipe, _SPRA_OFFSET, _SPRB_OFFSET)
+#define SPRSCALE(pipe) _MMIO_PIPE(pipe, _SPRA_SCALE, _SPRB_SCALE)
+#define SPRGAMC(pipe, i) _MMIO(_PIPE(pipe, _SPRA_GAMC, _SPRB_GAMC) + (i) * 4) /* 16 x u0.10 */
+#define SPRGAMC16(pipe, i) _MMIO(_PIPE(pipe, _SPRA_GAMC16, _SPRB_GAMC16) + (i) * 4) /* 3 x u1.10 */
+#define SPRGAMC17(pipe, i) _MMIO(_PIPE(pipe, _SPRA_GAMC17, _SPRB_GAMC17) + (i) * 4) /* 3 x u2.10 */
+#define SPRSURFLIVE(pipe) _MMIO_PIPE(pipe, _SPRA_SURFLIVE, _SPRB_SURFLIVE)
+
+#define _SPACNTR		(VLV_DISPLAY_BASE + 0x72180)
+#define   SP_ENABLE			REG_BIT(31)
+#define   SP_PIPE_GAMMA_ENABLE		REG_BIT(30)
+#define   SP_FORMAT_MASK		REG_GENMASK(29, 26)
+#define   SP_FORMAT_YUV422		REG_FIELD_PREP(SP_FORMAT_MASK, 0)
+#define   SP_FORMAT_8BPP		REG_FIELD_PREP(SP_FORMAT_MASK, 2)
+#define   SP_FORMAT_BGR565		REG_FIELD_PREP(SP_FORMAT_MASK, 5)
+#define   SP_FORMAT_BGRX8888		REG_FIELD_PREP(SP_FORMAT_MASK, 6)
+#define   SP_FORMAT_BGRA8888		REG_FIELD_PREP(SP_FORMAT_MASK, 7)
+#define   SP_FORMAT_RGBX1010102		REG_FIELD_PREP(SP_FORMAT_MASK, 8)
+#define   SP_FORMAT_RGBA1010102		REG_FIELD_PREP(SP_FORMAT_MASK, 9)
+#define   SP_FORMAT_BGRX1010102		REG_FIELD_PREP(SP_FORMAT_MASK, 10) /* CHV pipe B */
+#define   SP_FORMAT_BGRA1010102		REG_FIELD_PREP(SP_FORMAT_MASK, 11) /* CHV pipe B */
+#define   SP_FORMAT_RGBX8888		REG_FIELD_PREP(SP_FORMAT_MASK, 14)
+#define   SP_FORMAT_RGBA8888		REG_FIELD_PREP(SP_FORMAT_MASK, 15)
+#define   SP_ALPHA_PREMULTIPLY		REG_BIT(23) /* CHV pipe B */
+#define   SP_SOURCE_KEY			REG_BIT(22)
+#define   SP_YUV_FORMAT_BT709		REG_BIT(18)
+#define   SP_YUV_ORDER_MASK		REG_GENMASK(17, 16)
+#define   SP_YUV_ORDER_YUYV		REG_FIELD_PREP(SP_YUV_ORDER_MASK, 0)
+#define   SP_YUV_ORDER_UYVY		REG_FIELD_PREP(SP_YUV_ORDER_MASK, 1)
+#define   SP_YUV_ORDER_YVYU		REG_FIELD_PREP(SP_YUV_ORDER_MASK, 2)
+#define   SP_YUV_ORDER_VYUY		REG_FIELD_PREP(SP_YUV_ORDER_MASK, 3)
+#define   SP_ROTATE_180			REG_BIT(15)
+#define   SP_TILED			REG_BIT(10)
+#define   SP_MIRROR			REG_BIT(8) /* CHV pipe B */
+#define _SPALINOFF		(VLV_DISPLAY_BASE + 0x72184)
+#define _SPASTRIDE		(VLV_DISPLAY_BASE + 0x72188)
+#define _SPAPOS			(VLV_DISPLAY_BASE + 0x7218c)
+#define   SP_POS_Y_MASK			REG_GENMASK(31, 16)
+#define   SP_POS_Y(y)			REG_FIELD_PREP(SP_POS_Y_MASK, (y))
+#define   SP_POS_X_MASK			REG_GENMASK(15, 0)
+#define   SP_POS_X(x)			REG_FIELD_PREP(SP_POS_X_MASK, (x))
+#define _SPASIZE		(VLV_DISPLAY_BASE + 0x72190)
+#define   SP_HEIGHT_MASK		REG_GENMASK(31, 16)
+#define   SP_HEIGHT(h)			REG_FIELD_PREP(SP_HEIGHT_MASK, (h))
+#define   SP_WIDTH_MASK			REG_GENMASK(15, 0)
+#define   SP_WIDTH(w)			REG_FIELD_PREP(SP_WIDTH_MASK, (w))
+#define _SPAKEYMINVAL		(VLV_DISPLAY_BASE + 0x72194)
+#define _SPAKEYMSK		(VLV_DISPLAY_BASE + 0x72198)
+#define _SPASURF		(VLV_DISPLAY_BASE + 0x7219c)
+#define   SP_ADDR_MASK			REG_GENMASK(31, 12)
+#define _SPAKEYMAXVAL		(VLV_DISPLAY_BASE + 0x721a0)
+#define _SPATILEOFF		(VLV_DISPLAY_BASE + 0x721a4)
+#define   SP_OFFSET_Y_MASK		REG_GENMASK(31, 16)
+#define   SP_OFFSET_Y(y)		REG_FIELD_PREP(SP_OFFSET_Y_MASK, (y))
+#define   SP_OFFSET_X_MASK		REG_GENMASK(15, 0)
+#define   SP_OFFSET_X(x)		REG_FIELD_PREP(SP_OFFSET_X_MASK, (x))
+#define _SPACONSTALPHA		(VLV_DISPLAY_BASE + 0x721a8)
+#define   SP_CONST_ALPHA_ENABLE		REG_BIT(31)
+#define   SP_CONST_ALPHA_MASK		REG_GENMASK(7, 0)
+#define   SP_CONST_ALPHA(alpha)		REG_FIELD_PREP(SP_CONST_ALPHA_MASK, (alpha))
+#define _SPASURFLIVE		(VLV_DISPLAY_BASE + 0x721ac)
+#define _SPACLRC0		(VLV_DISPLAY_BASE + 0x721d0)
+#define   SP_CONTRAST_MASK		REG_GENMASK(26, 18)
+#define   SP_CONTRAST(x)		REG_FIELD_PREP(SP_CONTRAST_MASK, (x)) /* u3.6 */
+#define   SP_BRIGHTNESS_MASK		REG_GENMASK(7, 0)
+#define   SP_BRIGHTNESS(x)		REG_FIELD_PREP(SP_BRIGHTNESS_MASK, (x)) /* s8 */
+#define _SPACLRC1		(VLV_DISPLAY_BASE + 0x721d4)
+#define   SP_SH_SIN_MASK		REG_GENMASK(26, 16)
+#define   SP_SH_SIN(x)			REG_FIELD_PREP(SP_SH_SIN_MASK, (x)) /* s4.7 */
+#define   SP_SH_COS_MASK		REG_GENMASK(9, 0)
+#define   SP_SH_COS(x)			REG_FIELD_PREP(SP_SH_COS_MASK, (x)) /* u3.7 */
+#define _SPAGAMC		(VLV_DISPLAY_BASE + 0x721e0)
+
+#define _SPBCNTR		(VLV_DISPLAY_BASE + 0x72280)
+#define _SPBLINOFF		(VLV_DISPLAY_BASE + 0x72284)
+#define _SPBSTRIDE		(VLV_DISPLAY_BASE + 0x72288)
+#define _SPBPOS			(VLV_DISPLAY_BASE + 0x7228c)
+#define _SPBSIZE		(VLV_DISPLAY_BASE + 0x72290)
+#define _SPBKEYMINVAL		(VLV_DISPLAY_BASE + 0x72294)
+#define _SPBKEYMSK		(VLV_DISPLAY_BASE + 0x72298)
+#define _SPBSURF		(VLV_DISPLAY_BASE + 0x7229c)
+#define _SPBKEYMAXVAL		(VLV_DISPLAY_BASE + 0x722a0)
+#define _SPBTILEOFF		(VLV_DISPLAY_BASE + 0x722a4)
+#define _SPBCONSTALPHA		(VLV_DISPLAY_BASE + 0x722a8)
+#define _SPBSURFLIVE		(VLV_DISPLAY_BASE + 0x722ac)
+#define _SPBCLRC0		(VLV_DISPLAY_BASE + 0x722d0)
+#define _SPBCLRC1		(VLV_DISPLAY_BASE + 0x722d4)
+#define _SPBGAMC		(VLV_DISPLAY_BASE + 0x722e0)
+
+#define _VLV_SPR(pipe, plane_id, reg_a, reg_b) \
+	_PIPE((pipe) * 2 + (plane_id) - PLANE_SPRITE0, (reg_a), (reg_b))
+#define _MMIO_VLV_SPR(pipe, plane_id, reg_a, reg_b) \
+	_MMIO(_VLV_SPR((pipe), (plane_id), (reg_a), (reg_b)))
+
+#define SPCNTR(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPACNTR, _SPBCNTR)
+#define SPLINOFF(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPALINOFF, _SPBLINOFF)
+#define SPSTRIDE(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPASTRIDE, _SPBSTRIDE)
+#define SPPOS(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPAPOS, _SPBPOS)
+#define SPSIZE(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPASIZE, _SPBSIZE)
+#define SPKEYMINVAL(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPAKEYMINVAL, _SPBKEYMINVAL)
+#define SPKEYMSK(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPAKEYMSK, _SPBKEYMSK)
+#define SPSURF(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPASURF, _SPBSURF)
+#define SPKEYMAXVAL(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPAKEYMAXVAL, _SPBKEYMAXVAL)
+#define SPTILEOFF(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPATILEOFF, _SPBTILEOFF)
+#define SPCONSTALPHA(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPACONSTALPHA, _SPBCONSTALPHA)
+#define SPSURFLIVE(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPASURFLIVE, _SPBSURFLIVE)
+#define SPCLRC0(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPACLRC0, _SPBCLRC0)
+#define SPCLRC1(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPACLRC1, _SPBCLRC1)
+#define SPGAMC(pipe, plane_id, i)	_MMIO(_VLV_SPR((pipe), (plane_id), _SPAGAMC, _SPBGAMC) + (5 - (i)) * 4) /* 6 x u0.10 */
+
+/*
+ * CHV pipe B sprite CSC
+ *
+ * |cr|   |c0 c1 c2|   |cr + cr_ioff|   |cr_ooff|
+ * |yg| = |c3 c4 c5| x |yg + yg_ioff| + |yg_ooff|
+ * |cb|   |c6 c7 c8|   |cb + cr_ioff|   |cb_ooff|
+ */
+#define _MMIO_CHV_SPCSC(plane_id, reg) \
+	_MMIO(VLV_DISPLAY_BASE + ((plane_id) - PLANE_SPRITE0) * 0x1000 + (reg))
+
+#define SPCSCYGOFF(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d900)
+#define SPCSCCBOFF(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d904)
+#define SPCSCCROFF(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d908)
+#define  SPCSC_OOFF_MASK	REG_GENMASK(26, 16)
+#define  SPCSC_OOFF(x)		REG_FIELD_PREP(SPCSC_OOFF_MASK, (x) & 0x7ff) /* s11 */
+#define  SPCSC_IOFF_MASK	REG_GENMASK(10, 0)
+#define  SPCSC_IOFF(x)		REG_FIELD_PREP(SPCSC_IOFF_MASK, (x) & 0x7ff) /* s11 */
+
+#define SPCSCC01(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d90c)
+#define SPCSCC23(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d910)
+#define SPCSCC45(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d914)
+#define SPCSCC67(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d918)
+#define SPCSCC8(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d91c)
+#define  SPCSC_C1_MASK		REG_GENMASK(30, 16)
+#define  SPCSC_C1(x)		REG_FIELD_PREP(SPCSC_C1_MASK, (x) & 0x7fff) /* s3.12 */
+#define  SPCSC_C0_MASK		REG_GENMASK(14, 0)
+#define  SPCSC_C0(x)		REG_FIELD_PREP(SPCSC_C0_MASK, (x) & 0x7fff) /* s3.12 */
+
+#define SPCSCYGICLAMP(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d920)
+#define SPCSCCBICLAMP(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d924)
+#define SPCSCCRICLAMP(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d928)
+#define  SPCSC_IMAX_MASK	REG_GENMASK(26, 16)
+#define  SPCSC_IMAX(x)		REG_FIELD_PREP(SPCSC_IMAX_MASK, (x) & 0x7ff) /* s11 */
+#define  SPCSC_IMIN_MASK	REG_GENMASK(10, 0)
+#define  SPCSC_IMIN(x)		REG_FIELD_PREP(SPCSC_IMIN_MASK, (x) & 0x7ff) /* s11 */
+
+#define SPCSCYGOCLAMP(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d92c)
+#define SPCSCCBOCLAMP(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d930)
+#define SPCSCCROCLAMP(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d934)
+#define  SPCSC_OMAX_MASK	REG_GENMASK(25, 16)
+#define  SPCSC_OMAX(x)		REG_FIELD_PREP(SPCSC_OMAX_MASK, (x)) /* u10 */
+#define  SPCSC_OMIN_MASK	REG_GENMASK(9, 0)
+#define  SPCSC_OMIN(x)		REG_FIELD_PREP(SPCSC_OMIN_MASK, (x)) /* u10 */
+
+#endif /* __INTEL_SPRITE_REGS__ */
diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c
index 6b374d481cd9..9887967b2ca5 100644
--- a/drivers/gpu/drm/i915/display/intel_tc.c
+++ b/drivers/gpu/drm/i915/display/intel_tc.c
@@ -100,11 +100,9 @@ static struct drm_i915_private *tc_to_i915(struct intel_tc_port *tc)
 static bool intel_tc_port_in_mode(struct intel_digital_port *dig_port,
 				  enum tc_port_mode mode)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
 	struct intel_tc_port *tc = to_tc_port(dig_port);
 
-	return intel_phy_is_tc(i915, phy) && tc->mode == mode;
+	return intel_encoder_is_tc(&dig_port->base) && tc->mode == mode;
 }
 
 bool intel_tc_port_in_tbt_alt_mode(struct intel_digital_port *dig_port)
@@ -124,11 +122,9 @@ bool intel_tc_port_in_legacy_mode(struct intel_digital_port *dig_port)
 
 bool intel_tc_port_handles_hpd_glitches(struct intel_digital_port *dig_port)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
 	struct intel_tc_port *tc = to_tc_port(dig_port);
 
-	return intel_phy_is_tc(i915, phy) && !tc->legacy_port;
+	return intel_encoder_is_tc(&dig_port->base) && !tc->legacy_port;
 }
 
 /*
@@ -254,8 +250,7 @@ assert_tc_cold_blocked(struct intel_tc_port *tc)
 static enum intel_display_power_domain
 tc_port_power_domain(struct intel_tc_port *tc)
 {
-	struct drm_i915_private *i915 = tc_to_i915(tc);
-	enum tc_port tc_port = intel_port_to_tc(i915, tc->dig_port->base.port);
+	enum tc_port tc_port = intel_encoder_to_tc(&tc->dig_port->base);
 
 	return POWER_DOMAIN_PORT_DDI_LANES_TC1 + tc_port - TC_PORT_1;
 }
@@ -302,7 +297,7 @@ u32 intel_tc_port_get_pin_assignment_mask(struct intel_digital_port *dig_port)
 static int lnl_tc_port_get_max_lane_count(struct intel_digital_port *dig_port)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum tc_port tc_port = intel_port_to_tc(i915, dig_port->base.port);
+	enum tc_port tc_port = intel_encoder_to_tc(&dig_port->base);
 	intel_wakeref_t wakeref;
 	u32 val, pin_assignment;
 
@@ -375,9 +370,8 @@ int intel_tc_port_max_lane_count(struct intel_digital_port *dig_port)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
 	struct intel_tc_port *tc = to_tc_port(dig_port);
-	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
 
-	if (!intel_phy_is_tc(i915, phy) || tc->mode != TC_PORT_DP_ALT)
+	if (!intel_encoder_is_tc(&dig_port->base) || tc->mode != TC_PORT_DP_ALT)
 		return 4;
 
 	assert_tc_cold_blocked(tc);
@@ -458,9 +452,7 @@ static void tc_port_fixup_legacy_flag(struct intel_tc_port *tc,
 
 static void tc_phy_load_fia_params(struct intel_tc_port *tc, bool modular_fia)
 {
-	struct drm_i915_private *i915 = tc_to_i915(tc);
-	enum port port = tc->dig_port->base.port;
-	enum tc_port tc_port = intel_port_to_tc(i915, port);
+	enum tc_port tc_port = intel_encoder_to_tc(&tc->dig_port->base);
 
 	/*
 	 * Each Modular FIA instance houses 2 TC ports. In SOC that has more
@@ -812,7 +804,7 @@ static u32 adlp_tc_phy_hpd_live_status(struct intel_tc_port *tc)
 static bool adlp_tc_phy_is_ready(struct intel_tc_port *tc)
 {
 	struct drm_i915_private *i915 = tc_to_i915(tc);
-	enum tc_port tc_port = intel_port_to_tc(i915, tc->dig_port->base.port);
+	enum tc_port tc_port = intel_encoder_to_tc(&tc->dig_port->base);
 	u32 val;
 
 	assert_display_core_power_enabled(tc);
@@ -1635,10 +1627,7 @@ static bool __intel_tc_port_link_needs_reset(struct intel_tc_port *tc)
 
 bool intel_tc_port_link_needs_reset(struct intel_digital_port *dig_port)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
-
-	if (!intel_phy_is_tc(i915, phy))
+	if (!intel_encoder_is_tc(&dig_port->base))
 		return false;
 
 	return __intel_tc_port_link_needs_reset(to_tc_port(dig_port));
@@ -1740,11 +1729,9 @@ bool intel_tc_port_link_reset(struct intel_digital_port *dig_port)
 
 void intel_tc_port_link_cancel_reset_work(struct intel_digital_port *dig_port)
 {
-	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
-	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
 	struct intel_tc_port *tc = to_tc_port(dig_port);
 
-	if (!intel_phy_is_tc(i915, phy))
+	if (!intel_encoder_is_tc(&dig_port->base))
 		return;
 
 	cancel_delayed_work(&tc->link_reset_work);
@@ -1861,7 +1848,7 @@ int intel_tc_port_init(struct intel_digital_port *dig_port, bool is_legacy)
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
 	struct intel_tc_port *tc;
 	enum port port = dig_port->base.port;
-	enum tc_port tc_port = intel_port_to_tc(i915, port);
+	enum tc_port tc_port = intel_encoder_to_tc(&dig_port->base);
 
 	if (drm_WARN_ON(&i915->drm, tc_port == TC_PORT_NONE))
 		return -EINVAL;
diff --git a/drivers/gpu/drm/i915/display/intel_tv.c b/drivers/gpu/drm/i915/display/intel_tv.c
index 2b77d399f1a1..9df0f1263913 100644
--- a/drivers/gpu/drm/i915/display/intel_tv.c
+++ b/drivers/gpu/drm/i915/display/intel_tv.c
@@ -885,7 +885,8 @@ struct intel_tv_connector_state {
 	bool bypass_vfilter;
 };
 
-#define to_intel_tv_connector_state(x) container_of(x, struct intel_tv_connector_state, base)
+#define to_intel_tv_connector_state(conn_state) \
+	container_of_const((conn_state), struct intel_tv_connector_state, base)
 
 static struct drm_connector_state *
 intel_tv_connector_duplicate_state(struct drm_connector *connector)
@@ -961,16 +962,13 @@ intel_tv_mode_valid(struct drm_connector *connector,
 {
 	struct drm_i915_private *i915 = to_i915(connector->dev);
 	const struct tv_mode *tv_mode = intel_tv_mode_find(connector->state);
-	int max_dotclk = i915->max_dotclk_freq;
+	int max_dotclk = i915->display.cdclk.max_dotclk_freq;
 	enum drm_mode_status status;
 
 	status = intel_cpu_transcoder_mode_valid(i915, mode);
 	if (status != MODE_OK)
 		return status;
 
-	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
-		return MODE_NO_DBLESCAN;
-
 	if (mode->clock > max_dotclk)
 		return MODE_CLOCK_HIGH;
 
diff --git a/drivers/gpu/drm/i915/display/intel_vbt_defs.h b/drivers/gpu/drm/i915/display/intel_vbt_defs.h
index a9f44abfc9fc..228702c0e492 100644
--- a/drivers/gpu/drm/i915/display/intel_vbt_defs.h
+++ b/drivers/gpu/drm/i915/display/intel_vbt_defs.h
@@ -485,6 +485,7 @@ struct child_device_config {
 	u8 hdmi_iboost_level:4;					/* 196+ */
 	u8 dp_max_link_rate:3;					/* 216+ */
 	u8 dp_max_link_rate_reserved:5;				/* 216+ */
+	u8 efp_index;						/* 256+ */
 } __packed;
 
 struct bdb_general_definitions {
@@ -602,22 +603,22 @@ struct bdb_driver_features {
 	u8 custom_vbt_version;					/* 155+ */
 
 	/* Driver Feature Flags */
-	u16 rmpm_enabled:1;					/* 165+ */
-	u16 s2ddt_enabled:1;					/* 165+ */
-	u16 dpst_enabled:1;					/* 165-227 */
-	u16 bltclt_enabled:1;					/* 165+ */
-	u16 adb_enabled:1;					/* 165-227 */
-	u16 drrs_enabled:1;					/* 165-227 */
-	u16 grs_enabled:1;					/* 165+ */
-	u16 gpmt_enabled:1;					/* 165+ */
-	u16 tbt_enabled:1;					/* 165+ */
+	u16 rmpm_enabled:1;					/* 159+ */
+	u16 s2ddt_enabled:1;					/* 159+ */
+	u16 dpst_enabled:1;					/* 159-227 */
+	u16 bltclt_enabled:1;					/* 159+ */
+	u16 adb_enabled:1;					/* 159-227 */
+	u16 drrs_enabled:1;					/* 159-227 */
+	u16 grs_enabled:1;					/* 159+ */
+	u16 gpmt_enabled:1;					/* 159+ */
+	u16 tbt_enabled:1;					/* 159+ */
 	u16 psr_enabled:1;					/* 165-227 */
 	u16 ips_enabled:1;					/* 165+ */
-	u16 dpfs_enabled:1;					/* 165+ */
+	u16 dfps_enabled:1;					/* 165+ */
 	u16 dmrrs_enabled:1;					/* 174-227 */
 	u16 adt_enabled:1;					/* ???-228 */
 	u16 hpd_wake:1;						/* 201-240 */
-	u16 pc_feature_valid:1;
+	u16 pc_feature_valid:1;					/* 159+ */
 } __packed;
 
 /*
@@ -880,11 +881,12 @@ struct bdb_lvds_lfp_data_tail {
 struct lfp_backlight_data_entry {
 	u8 type:2;
 	u8 active_low_pwm:1;
-	u8 obsolete1:5;
+	u8 i2c_pin:3;						/* obsolete since ? */
+	u8 i2c_speed:2;						/* obsolete since ? */
 	u16 pwm_freq_hz;
 	u8 min_brightness;					/* ???-233 */
-	u8 obsolete2;
-	u8 obsolete3;
+	u8 i2c_address;						/* obsolete since ? */
+	u8 i2c_command;						/* obsolete since ? */
 } __packed;
 
 struct lfp_backlight_control_method {
@@ -897,16 +899,11 @@ struct lfp_brightness_level {
 	u16 reserved;
 } __packed;
 
-#define EXP_BDB_LFP_BL_DATA_SIZE_REV_191 \
-	offsetof(struct bdb_lfp_backlight_data, brightness_level)
-#define EXP_BDB_LFP_BL_DATA_SIZE_REV_234 \
-	offsetof(struct bdb_lfp_backlight_data, brightness_precision_bits)
-
 struct bdb_lfp_backlight_data {
 	u8 entry_size;
 	struct lfp_backlight_data_entry data[16];
-	u8 level[16];							/* ???-233 */
-	struct lfp_backlight_control_method backlight_control[16];
+	u8 level[16];							/* 162-233 */
+	struct lfp_backlight_control_method backlight_control[16];	/* 191+ */
 	struct lfp_brightness_level brightness_level[16];		/* 234+ */
 	struct lfp_brightness_level brightness_min_level[16];		/* 234+ */
 	u8 brightness_precision_bits[16];				/* 236+ */
@@ -917,7 +914,7 @@ struct bdb_lfp_backlight_data {
  * Block 44 - LFP Power Conservation Features Block
  */
 struct lfp_power_features {
-	u8 reserved1:1;
+	u8 dpst_support:1;						/* ???-159 */
 	u8 power_conservation_pref:3;
 	u8 reserved2:1;
 	u8 lace_enabled_status:1;					/* 210+ */
diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c
index f542ee1db1d9..894ee97b3e1b 100644
--- a/drivers/gpu/drm/i915/display/intel_vrr.c
+++ b/drivers/gpu/drm/i915/display/intel_vrr.c
@@ -9,6 +9,7 @@
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_vrr.h"
+#include "intel_dp.h"
 
 bool intel_vrr_is_capable(struct intel_connector *connector)
 {
@@ -113,6 +114,7 @@ intel_vrr_compute_config(struct intel_crtc_state *crtc_state,
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
 	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
 	const struct drm_display_info *info = &connector->base.display_info;
 	int vmin, vmax;
@@ -172,6 +174,14 @@ intel_vrr_compute_config(struct intel_crtc_state *crtc_state,
 	if (crtc_state->uapi.vrr_enabled) {
 		crtc_state->vrr.enable = true;
 		crtc_state->mode_flags |= I915_MODE_FLAG_VRR;
+		if (intel_dp_as_sdp_supported(intel_dp)) {
+			crtc_state->vrr.vsync_start =
+				(crtc_state->hw.adjusted_mode.crtc_vtotal -
+					crtc_state->hw.adjusted_mode.vsync_start);
+			crtc_state->vrr.vsync_end =
+				(crtc_state->hw.adjusted_mode.crtc_vtotal -
+					crtc_state->hw.adjusted_mode.vsync_end);
+		}
 	}
 }
 
@@ -247,6 +257,12 @@ void intel_vrr_enable(const struct intel_crtc_state *crtc_state)
 		return;
 
 	intel_de_write(dev_priv, TRANS_PUSH(cpu_transcoder), TRANS_PUSH_EN);
+
+	if (HAS_AS_SDP(dev_priv))
+		intel_de_write(dev_priv, TRANS_VRR_VSYNC(cpu_transcoder),
+			       VRR_VSYNC_END(crtc_state->vrr.vsync_end) |
+			       VRR_VSYNC_START(crtc_state->vrr.vsync_start));
+
 	intel_de_write(dev_priv, TRANS_VRR_CTL(cpu_transcoder),
 		       VRR_CTL_VRR_ENABLE | trans_vrr_ctl(crtc_state));
 }
@@ -265,13 +281,16 @@ void intel_vrr_disable(const struct intel_crtc_state *old_crtc_state)
 	intel_de_wait_for_clear(dev_priv, TRANS_VRR_STATUS(cpu_transcoder),
 				VRR_STATUS_VRR_EN_LIVE, 1000);
 	intel_de_write(dev_priv, TRANS_PUSH(cpu_transcoder), 0);
+
+	if (HAS_AS_SDP(dev_priv))
+		intel_de_write(dev_priv, TRANS_VRR_VSYNC(cpu_transcoder), 0);
 }
 
 void intel_vrr_get_config(struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
-	u32 trans_vrr_ctl;
+	u32 trans_vrr_ctl, trans_vrr_vsync;
 
 	trans_vrr_ctl = intel_de_read(dev_priv, TRANS_VRR_CTL(cpu_transcoder));
 
@@ -291,6 +310,16 @@ void intel_vrr_get_config(struct intel_crtc_state *crtc_state)
 		crtc_state->vrr.vmin = intel_de_read(dev_priv, TRANS_VRR_VMIN(cpu_transcoder)) + 1;
 	}
 
-	if (crtc_state->vrr.enable)
+	if (crtc_state->vrr.enable) {
 		crtc_state->mode_flags |= I915_MODE_FLAG_VRR;
+
+		if (HAS_AS_SDP(dev_priv)) {
+			trans_vrr_vsync =
+				intel_de_read(dev_priv, TRANS_VRR_VSYNC(cpu_transcoder));
+			crtc_state->vrr.vsync_start =
+				REG_FIELD_GET(VRR_VSYNC_START_MASK, trans_vrr_vsync);
+			crtc_state->vrr.vsync_end =
+				REG_FIELD_GET(VRR_VSYNC_END_MASK, trans_vrr_vsync);
+		}
+	}
 }
diff --git a/drivers/gpu/drm/i915/display/skl_scaler.c b/drivers/gpu/drm/i915/display/skl_scaler.c
index 8a934bada624..baa601d27815 100644
--- a/drivers/gpu/drm/i915/display/skl_scaler.c
+++ b/drivers/gpu/drm/i915/display/skl_scaler.c
@@ -213,10 +213,11 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
 	 * The pipe scaler does not use all the bits of PIPESRC, at least
 	 * on the earlier platforms. So even when we're scaling a plane
 	 * the *pipe* source size must not be too large. For simplicity
-	 * we assume the limits match the scaler source size limits. Might
-	 * not be 100% accurate on all platforms, but good enough for now.
+	 * we assume the limits match the scaler destination size limits.
+	 * Might not be 100% accurate on all platforms, but good enough for
+	 * now.
 	 */
-	if (pipe_src_w > max_src_w || pipe_src_h > max_src_h) {
+	if (pipe_src_w > max_dst_w || pipe_src_h > max_dst_h) {
 		drm_dbg_kms(&dev_priv->drm,
 			    "scaler_user index %u.%u: pipe src size %ux%u "
 			    "is out of scaler range\n",
diff --git a/drivers/gpu/drm/i915/display/skl_watermark.c b/drivers/gpu/drm/i915/display/skl_watermark.c
index c6b9be80d83c..7c6187b4479f 100644
--- a/drivers/gpu/drm/i915/display/skl_watermark.c
+++ b/drivers/gpu/drm/i915/display/skl_watermark.c
@@ -6,18 +6,19 @@
 #include <drm/drm_blend.h>
 
 #include "i915_drv.h"
-#include "i915_fixed.h"
 #include "i915_reg.h"
 #include "i9xx_wm.h"
 #include "intel_atomic.h"
 #include "intel_atomic_plane.h"
 #include "intel_bw.h"
+#include "intel_cdclk.h"
 #include "intel_crtc.h"
 #include "intel_de.h"
 #include "intel_display.h"
 #include "intel_display_power.h"
 #include "intel_display_types.h"
 #include "intel_fb.h"
+#include "intel_fixed.h"
 #include "intel_pcode.h"
 #include "intel_wm.h"
 #include "skl_watermark.h"
@@ -69,7 +70,7 @@ static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
 	return DISPLAY_VER(i915) == 9;
 }
 
-static bool
+bool
 intel_has_sagv(struct drm_i915_private *i915)
 {
 	return HAS_SAGV(i915) &&
@@ -2601,10 +2602,17 @@ skl_compute_ddb(struct intel_atomic_state *state)
 			return ret;
 	}
 
-	if (HAS_MBUS_JOINING(i915))
+	if (HAS_MBUS_JOINING(i915)) {
 		new_dbuf_state->joined_mbus =
 			adlp_check_mbus_joined(new_dbuf_state->active_pipes);
 
+		if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
+			ret = intel_cdclk_state_set_joined_mbus(state, new_dbuf_state->joined_mbus);
+			if (ret)
+				return ret;
+		}
+	}
+
 	for_each_intel_crtc(&i915->drm, crtc) {
 		enum pipe pipe = crtc->pipe;
 
@@ -2628,13 +2636,6 @@ skl_compute_ddb(struct intel_atomic_state *state)
 		if (ret)
 			return ret;
 
-		if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
-			/* TODO: Implement vblank synchronized MBUS joining changes */
-			ret = intel_modeset_all_pipes_late(state, "MBUS joining change");
-			if (ret)
-				return ret;
-		}
-
 		drm_dbg_kms(&i915->drm,
 			    "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
 			    old_dbuf_state->enabled_slices,
@@ -3057,6 +3058,8 @@ static void skl_wm_get_hw_state(struct drm_i915_private *i915)
 	if (HAS_MBUS_JOINING(i915))
 		dbuf_state->joined_mbus = intel_de_read(i915, MBUS_CTL) & MBUS_JOIN;
 
+	dbuf_state->mdclk_cdclk_ratio = intel_mdclk_cdclk_ratio(i915, &i915->display.cdclk.hw);
+
 	for_each_intel_crtc(&i915->drm, crtc) {
 		struct intel_crtc_state *crtc_state =
 			to_intel_crtc_state(crtc->base.state);
@@ -3530,85 +3533,6 @@ int intel_dbuf_init(struct drm_i915_private *i915)
 	return 0;
 }
 
-/*
- * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
- * update the request state of all DBUS slices.
- */
-static void update_mbus_pre_enable(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
-	u32 mbus_ctl, dbuf_min_tracker_val;
-	enum dbuf_slice slice;
-	const struct intel_dbuf_state *dbuf_state =
-		intel_atomic_get_new_dbuf_state(state);
-
-	if (!HAS_MBUS_JOINING(i915))
-		return;
-
-	/*
-	 * TODO: Implement vblank synchronized MBUS joining changes.
-	 * Must be properly coordinated with dbuf reprogramming.
-	 */
-	if (dbuf_state->joined_mbus) {
-		mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
-			MBUS_JOIN_PIPE_SELECT_NONE;
-		dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
-	} else {
-		mbus_ctl = MBUS_HASHING_MODE_2x2 |
-			MBUS_JOIN_PIPE_SELECT_NONE;
-		dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
-	}
-
-	intel_de_rmw(i915, MBUS_CTL,
-		     MBUS_HASHING_MODE_MASK | MBUS_JOIN |
-		     MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
-
-	for_each_dbuf_slice(i915, slice)
-		intel_de_rmw(i915, DBUF_CTL_S(slice),
-			     DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
-			     dbuf_min_tracker_val);
-}
-
-void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
-	const struct intel_dbuf_state *new_dbuf_state =
-		intel_atomic_get_new_dbuf_state(state);
-	const struct intel_dbuf_state *old_dbuf_state =
-		intel_atomic_get_old_dbuf_state(state);
-
-	if (!new_dbuf_state ||
-	    (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
-	     new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
-		return;
-
-	WARN_ON(!new_dbuf_state->base.changed);
-
-	update_mbus_pre_enable(state);
-	gen9_dbuf_slices_update(i915,
-				old_dbuf_state->enabled_slices |
-				new_dbuf_state->enabled_slices);
-}
-
-void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
-	const struct intel_dbuf_state *new_dbuf_state =
-		intel_atomic_get_new_dbuf_state(state);
-	const struct intel_dbuf_state *old_dbuf_state =
-		intel_atomic_get_old_dbuf_state(state);
-
-	if (!new_dbuf_state ||
-	    (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
-	     new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
-		return;
-
-	WARN_ON(!new_dbuf_state->base.changed);
-
-	gen9_dbuf_slices_update(i915,
-				new_dbuf_state->enabled_slices);
-}
-
 static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
 {
 	switch (pipe) {
@@ -3628,14 +3552,12 @@ static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
 	return false;
 }
 
-void intel_mbus_dbox_update(struct intel_atomic_state *state)
+static void intel_mbus_dbox_update(struct intel_atomic_state *state)
 {
 	struct drm_i915_private *i915 = to_i915(state->base.dev);
 	const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
-	const struct intel_crtc_state *new_crtc_state;
 	const struct intel_crtc *crtc;
 	u32 val = 0;
-	int i;
 
 	if (DISPLAY_VER(i915) < 11)
 		return;
@@ -3679,12 +3601,9 @@ void intel_mbus_dbox_update(struct intel_atomic_state *state)
 		val |= MBUS_DBOX_B_CREDIT(8);
 	}
 
-	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
+	for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, new_dbuf_state->active_pipes) {
 		u32 pipe_val = val;
 
-		if (!new_crtc_state->hw.active)
-			continue;
-
 		if (DISPLAY_VER(i915) >= 14) {
 			if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe,
 							      new_dbuf_state->active_pipes))
@@ -3697,6 +3616,217 @@ void intel_mbus_dbox_update(struct intel_atomic_state *state)
 	}
 }
 
+int intel_dbuf_state_set_mdclk_cdclk_ratio(struct intel_atomic_state *state,
+					   int ratio)
+{
+	struct intel_dbuf_state *dbuf_state;
+
+	dbuf_state = intel_atomic_get_dbuf_state(state);
+	if (IS_ERR(dbuf_state))
+		return PTR_ERR(dbuf_state);
+
+	dbuf_state->mdclk_cdclk_ratio = ratio;
+
+	return intel_atomic_lock_global_state(&dbuf_state->base);
+}
+
+void intel_dbuf_mdclk_cdclk_ratio_update(struct drm_i915_private *i915,
+					 int ratio, bool joined_mbus)
+{
+	enum dbuf_slice slice;
+
+	if (!HAS_MBUS_JOINING(i915))
+		return;
+
+	if (DISPLAY_VER(i915) >= 20)
+		intel_de_rmw(i915, MBUS_CTL, MBUS_TRANSLATION_THROTTLE_MIN_MASK,
+			     MBUS_TRANSLATION_THROTTLE_MIN(ratio - 1));
+
+	if (joined_mbus)
+		ratio *= 2;
+
+	drm_dbg_kms(&i915->drm, "Updating dbuf ratio to %d (mbus joined: %s)\n",
+		    ratio, str_yes_no(joined_mbus));
+
+	for_each_dbuf_slice(i915, slice)
+		intel_de_rmw(i915, DBUF_CTL_S(slice),
+			     DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
+			     DBUF_MIN_TRACKER_STATE_SERVICE(ratio - 1));
+}
+
+static void intel_dbuf_mdclk_min_tracker_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_dbuf_state *old_dbuf_state =
+		intel_atomic_get_old_dbuf_state(state);
+	const struct intel_dbuf_state *new_dbuf_state =
+		intel_atomic_get_new_dbuf_state(state);
+	int mdclk_cdclk_ratio;
+
+	if (intel_cdclk_is_decreasing_later(state)) {
+		/* cdclk/mdclk will be changed later by intel_set_cdclk_post_plane_update() */
+		mdclk_cdclk_ratio = old_dbuf_state->mdclk_cdclk_ratio;
+	} else {
+		/* cdclk/mdclk already changed by intel_set_cdclk_pre_plane_update() */
+		mdclk_cdclk_ratio = new_dbuf_state->mdclk_cdclk_ratio;
+	}
+
+	intel_dbuf_mdclk_cdclk_ratio_update(i915, mdclk_cdclk_ratio,
+					    new_dbuf_state->joined_mbus);
+}
+
+static enum pipe intel_mbus_joined_pipe(struct intel_atomic_state *state,
+					const struct intel_dbuf_state *dbuf_state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	enum pipe pipe = ffs(dbuf_state->active_pipes) - 1;
+	const struct intel_crtc_state *new_crtc_state;
+	struct intel_crtc *crtc;
+
+	drm_WARN_ON(&i915->drm, !dbuf_state->joined_mbus);
+	drm_WARN_ON(&i915->drm, !is_power_of_2(dbuf_state->active_pipes));
+
+	crtc = intel_crtc_for_pipe(i915, pipe);
+	new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
+
+	if (new_crtc_state && !intel_crtc_needs_modeset(new_crtc_state))
+		return pipe;
+	else
+		return INVALID_PIPE;
+}
+
+static void intel_dbuf_mbus_join_update(struct intel_atomic_state *state,
+					enum pipe pipe)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_dbuf_state *old_dbuf_state =
+		intel_atomic_get_old_dbuf_state(state);
+	const struct intel_dbuf_state *new_dbuf_state =
+		intel_atomic_get_new_dbuf_state(state);
+	u32 mbus_ctl;
+
+	drm_dbg_kms(&i915->drm, "Changing mbus joined: %s -> %s (pipe: %c)\n",
+		    str_yes_no(old_dbuf_state->joined_mbus),
+		    str_yes_no(new_dbuf_state->joined_mbus),
+		    pipe != INVALID_PIPE ? pipe_name(pipe) : '*');
+
+	if (new_dbuf_state->joined_mbus)
+		mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN;
+	else
+		mbus_ctl = MBUS_HASHING_MODE_2x2;
+
+	if (pipe != INVALID_PIPE)
+		mbus_ctl |= MBUS_JOIN_PIPE_SELECT(pipe);
+	else
+		mbus_ctl |= MBUS_JOIN_PIPE_SELECT_NONE;
+
+	intel_de_rmw(i915, MBUS_CTL,
+		     MBUS_HASHING_MODE_MASK | MBUS_JOIN |
+		     MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
+}
+
+void intel_dbuf_mbus_pre_ddb_update(struct intel_atomic_state *state)
+{
+	const struct intel_dbuf_state *new_dbuf_state =
+		intel_atomic_get_new_dbuf_state(state);
+	const struct intel_dbuf_state *old_dbuf_state =
+		intel_atomic_get_old_dbuf_state(state);
+
+	if (!new_dbuf_state)
+		return;
+
+	if (!old_dbuf_state->joined_mbus && new_dbuf_state->joined_mbus) {
+		enum pipe pipe = intel_mbus_joined_pipe(state, new_dbuf_state);
+
+		WARN_ON(!new_dbuf_state->base.changed);
+
+		intel_dbuf_mbus_join_update(state, pipe);
+		intel_mbus_dbox_update(state);
+		intel_dbuf_mdclk_min_tracker_update(state);
+	}
+}
+
+void intel_dbuf_mbus_post_ddb_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_dbuf_state *new_dbuf_state =
+		intel_atomic_get_new_dbuf_state(state);
+	const struct intel_dbuf_state *old_dbuf_state =
+		intel_atomic_get_old_dbuf_state(state);
+
+	if (!new_dbuf_state)
+		return;
+
+	if (old_dbuf_state->joined_mbus && !new_dbuf_state->joined_mbus) {
+		enum pipe pipe = intel_mbus_joined_pipe(state, old_dbuf_state);
+
+		WARN_ON(!new_dbuf_state->base.changed);
+
+		intel_dbuf_mdclk_min_tracker_update(state);
+		intel_mbus_dbox_update(state);
+		intel_dbuf_mbus_join_update(state, pipe);
+
+		if (pipe != INVALID_PIPE) {
+			struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe);
+
+			intel_crtc_wait_for_next_vblank(crtc);
+		}
+	} else if (old_dbuf_state->joined_mbus == new_dbuf_state->joined_mbus &&
+		   old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
+		WARN_ON(!new_dbuf_state->base.changed);
+
+		intel_dbuf_mdclk_min_tracker_update(state);
+		intel_mbus_dbox_update(state);
+	}
+
+}
+
+void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_dbuf_state *new_dbuf_state =
+		intel_atomic_get_new_dbuf_state(state);
+	const struct intel_dbuf_state *old_dbuf_state =
+		intel_atomic_get_old_dbuf_state(state);
+	u8 old_slices, new_slices;
+
+	if (!new_dbuf_state)
+		return;
+
+	old_slices = old_dbuf_state->enabled_slices;
+	new_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
+
+	if (old_slices == new_slices)
+		return;
+
+	WARN_ON(!new_dbuf_state->base.changed);
+
+	gen9_dbuf_slices_update(i915, new_slices);
+}
+
+void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_dbuf_state *new_dbuf_state =
+		intel_atomic_get_new_dbuf_state(state);
+	const struct intel_dbuf_state *old_dbuf_state =
+		intel_atomic_get_old_dbuf_state(state);
+	u8 old_slices, new_slices;
+
+	if (!new_dbuf_state)
+		return;
+
+	old_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
+	new_slices = new_dbuf_state->enabled_slices;
+
+	if (old_slices == new_slices)
+		return;
+
+	WARN_ON(!new_dbuf_state->base.changed);
+
+	gen9_dbuf_slices_update(i915, new_slices);
+}
+
 static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *i915 = m->private;
diff --git a/drivers/gpu/drm/i915/display/skl_watermark.h b/drivers/gpu/drm/i915/display/skl_watermark.h
index e3d1d74a7b17..91f92c0e706e 100644
--- a/drivers/gpu/drm/i915/display/skl_watermark.h
+++ b/drivers/gpu/drm/i915/display/skl_watermark.h
@@ -25,6 +25,7 @@ void intel_sagv_pre_plane_update(struct intel_atomic_state *state);
 void intel_sagv_post_plane_update(struct intel_atomic_state *state);
 bool intel_can_enable_sagv(struct drm_i915_private *i915,
 			   const struct intel_bw_state *bw_state);
+bool intel_has_sagv(struct drm_i915_private *i915);
 
 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
 			    const struct skl_ddb_entry *entry);
@@ -58,22 +59,31 @@ struct intel_dbuf_state {
 	u8 slices[I915_MAX_PIPES];
 	u8 enabled_slices;
 	u8 active_pipes;
+	u8 mdclk_cdclk_ratio;
 	bool joined_mbus;
 };
 
 struct intel_dbuf_state *
 intel_atomic_get_dbuf_state(struct intel_atomic_state *state);
 
-#define to_intel_dbuf_state(x) container_of((x), struct intel_dbuf_state, base)
+#define to_intel_dbuf_state(global_state) \
+	container_of_const((global_state), struct intel_dbuf_state, base)
+
 #define intel_atomic_get_old_dbuf_state(state) \
 	to_intel_dbuf_state(intel_atomic_get_old_global_obj_state(state, &to_i915(state->base.dev)->display.dbuf.obj))
 #define intel_atomic_get_new_dbuf_state(state) \
 	to_intel_dbuf_state(intel_atomic_get_new_global_obj_state(state, &to_i915(state->base.dev)->display.dbuf.obj))
 
 int intel_dbuf_init(struct drm_i915_private *i915);
+int intel_dbuf_state_set_mdclk_cdclk_ratio(struct intel_atomic_state *state,
+					   int ratio);
+
 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state);
 void intel_dbuf_post_plane_update(struct intel_atomic_state *state);
-void intel_mbus_dbox_update(struct intel_atomic_state *state);
+void intel_dbuf_mdclk_cdclk_ratio_update(struct drm_i915_private *i915,
+					 int ratio, bool joined_mbus);
+void intel_dbuf_mbus_pre_ddb_update(struct intel_atomic_state *state);
+void intel_dbuf_mbus_post_ddb_update(struct intel_atomic_state *state);
 
 #endif /* __SKL_WATERMARK_H__ */
 
diff --git a/drivers/gpu/drm/i915/display/skl_watermark_regs.h b/drivers/gpu/drm/i915/display/skl_watermark_regs.h
index 20b30c9a6613..269163fa3350 100644
--- a/drivers/gpu/drm/i915/display/skl_watermark_regs.h
+++ b/drivers/gpu/drm/i915/display/skl_watermark_regs.h
@@ -32,14 +32,16 @@
 #define MBUS_BBOX_CTL_S1		_MMIO(0x45040)
 #define MBUS_BBOX_CTL_S2		_MMIO(0x45044)
 
-#define MBUS_CTL			_MMIO(0x4438C)
-#define MBUS_JOIN			REG_BIT(31)
-#define MBUS_HASHING_MODE_MASK		REG_BIT(30)
-#define MBUS_HASHING_MODE_2x2		REG_FIELD_PREP(MBUS_HASHING_MODE_MASK, 0)
-#define MBUS_HASHING_MODE_1x4		REG_FIELD_PREP(MBUS_HASHING_MODE_MASK, 1)
-#define MBUS_JOIN_PIPE_SELECT_MASK	REG_GENMASK(28, 26)
-#define MBUS_JOIN_PIPE_SELECT(pipe)	REG_FIELD_PREP(MBUS_JOIN_PIPE_SELECT_MASK, pipe)
-#define MBUS_JOIN_PIPE_SELECT_NONE	MBUS_JOIN_PIPE_SELECT(7)
+#define MBUS_CTL				_MMIO(0x4438C)
+#define   MBUS_JOIN				REG_BIT(31)
+#define   MBUS_HASHING_MODE_MASK		REG_BIT(30)
+#define   MBUS_HASHING_MODE_2x2			REG_FIELD_PREP(MBUS_HASHING_MODE_MASK, 0)
+#define   MBUS_HASHING_MODE_1x4			REG_FIELD_PREP(MBUS_HASHING_MODE_MASK, 1)
+#define   MBUS_JOIN_PIPE_SELECT_MASK		REG_GENMASK(28, 26)
+#define   MBUS_JOIN_PIPE_SELECT(pipe)		REG_FIELD_PREP(MBUS_JOIN_PIPE_SELECT_MASK, pipe)
+#define   MBUS_JOIN_PIPE_SELECT_NONE		MBUS_JOIN_PIPE_SELECT(7)
+#define   MBUS_TRANSLATION_THROTTLE_MIN_MASK	REG_GENMASK(15, 13)
+#define   MBUS_TRANSLATION_THROTTLE_MIN(val)	REG_FIELD_PREP(MBUS_TRANSLATION_THROTTLE_MIN_MASK, val)
 
 /* Watermark register definitions for SKL */
 #define _CUR_WM_A_0		0x70140
diff --git a/drivers/gpu/drm/i915/display/vlv_dpio_phy_regs.h b/drivers/gpu/drm/i915/display/vlv_dpio_phy_regs.h
new file mode 100644
index 000000000000..2b83f334b1ff
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/vlv_dpio_phy_regs.h
@@ -0,0 +1,309 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __VLV_DPIO_PHY_REGS_H__
+#define __VLV_DPIO_PHY_REGS_H__
+
+#include "intel_display_reg_defs.h"
+
+#define _VLV_CMN(dw) (0x8100 + (dw) * 4)
+#define _CHV_CMN(cl, dw) (0x8100 - (cl) * 0x80 + (dw) * 4)
+#define _VLV_PLL(ch, dw) (0x8000 + (ch) * 0x20 + (dw) * 4) /* dw 0-7,16-23 */
+#define _CHV_PLL(ch, dw) (0x8000 + (ch) * 0x180 + (dw) * 4)
+#define _VLV_REF(dw) (0x80a0 + ((dw) - 8) * 4) /* dw 8-15 */
+#define _VLV_PCS(ch, spline, dw) (0x200 + (ch) * 0x2400 + (spline) * 0x200 + (dw) * 4)
+#define _VLV_PCS_GRP(ch, dw) (0x8200 + (ch) * 0x200 + (dw) * 4)
+#define _VLV_PCS_BCAST(dw) (0xc000 + (dw) * 4)
+#define _VLV_TX(ch, lane, dw) (0x80 + (ch) * 0x2400 + (lane) * 0x200 + (dw) * 4)
+#define _VLV_TX_GRP(ch, dw) (0x8280 + (ch) * 0x200 + (dw) * 4)
+#define _VLV_TX_BCAST(dw) (0xc080 + (dw) * 4)
+
+/*
+ * Per pipe/PLL DPIO regs
+ */
+#define VLV_PLL_DW3(ch)			_VLV_PLL((ch), 3)
+#define   DPIO_S1_DIV_MASK		REG_GENMASK(30, 28)
+#define   DPIO_S1_DIV(s1)		REG_FIELD_PREP(DPIO_S1_DIV_MASK, (s1))
+#define   DPIO_S1_DIV_DAC		0 /* 10, DAC 25-225M rate */
+#define   DPIO_S1_DIV_HDMIDP		1 /* 5, DAC 225-400M rate */
+#define   DPIO_S1_DIV_LVDS1		2 /* 14 */
+#define   DPIO_S1_DIV_LVDS2		3 /* 7 */
+#define   DPIO_K_DIV_MASK		REG_GENMASK(27, 24)
+#define   DPIO_K_DIV(k)			REG_FIELD_PREP(DPIO_K_DIV_MASK, (k))
+#define   DPIO_P1_DIV_MASK		REG_GENMASK(23, 21)
+#define   DPIO_P1_DIV(p1)		REG_FIELD_PREP(DPIO_P1_DIV_MASK, (p1))
+#define   DPIO_P2_DIV_MASK		REG_GENMASK(20, 16)
+#define   DPIO_P2_DIV(p2)		REG_FIELD_PREP(DPIO_P2_DIV_MASK, (p2))
+#define   DPIO_N_DIV_MASK		REG_GENMASK(15, 12)
+#define   DPIO_N_DIV(n)			REG_FIELD_PREP(DPIO_N_DIV_MASK, (n))
+#define   DPIO_ENABLE_CALIBRATION	REG_BIT(11)
+#define   DPIO_M1_DIV_MASK		REG_GENMASK(10, 8)
+#define   DPIO_M1_DIV(m1)		REG_FIELD_PREP(DPIO_M1_DIV_MASK, (m1))
+#define   DPIO_M2_DIV_MASK		REG_GENMASK(7, 0)
+#define   DPIO_M2_DIV(m2)		REG_FIELD_PREP(DPIO_M2_DIV_MASK, (m2))
+
+#define VLV_PLL_DW5(ch)			_VLV_PLL((ch), 5)
+#define   DPIO_REFSEL_OVERRIDE		REG_BIT(27)
+#define   DPIO_PLL_MODESEL_MASK		REG_GENMASK(26, 24)
+#define   DPIO_BIAS_CURRENT_CTL_MASK	REG_GENMASK(22, 20) /* always 0x7 */
+#define   DPIO_PLL_REFCLK_SEL_MASK	REG_GENMASK(17, 16)
+#define   DPIO_DRIVER_CTL_MASK		REG_GENMASK(15, 12) /* always set to 0x8 */
+#define   DPIO_CLK_BIAS_CTL_MASK	REG_GENMASK(11, 8) /* always set to 0x5 */
+
+#define VLV_PLL_DW7(ch)			_VLV_PLL((ch), 7)
+
+#define VLV_PLL_DW16(ch)		_VLV_PLL((ch), 16)
+
+#define VLV_PLL_DW17(ch)		_VLV_PLL((ch), 17)
+
+#define VLV_PLL_DW18(ch)		_VLV_PLL((ch), 18)
+
+#define VLV_PLL_DW19(ch)		_VLV_PLL((ch), 19)
+
+#define VLV_REF_DW11			_VLV_REF(11)
+
+#define VLV_CMN_DW0			_VLV_CMN(0)
+
+/*
+ * Per DDI channel DPIO regs
+ */
+#define VLV_PCS_DW0_GRP(ch)		_VLV_PCS_GRP((ch), 0)
+#define VLV_PCS01_DW0(ch)		_VLV_PCS((ch), 0, 0)
+#define VLV_PCS23_DW0(ch)		_VLV_PCS((ch), 1, 0)
+#define   DPIO_PCS_TX_LANE2_RESET	REG_BIT(16)
+#define   DPIO_PCS_TX_LANE1_RESET	REG_BIT(7)
+#define   DPIO_LEFT_TXFIFO_RST_MASTER2	REG_BIT(4)
+#define   DPIO_RIGHT_TXFIFO_RST_MASTER2	REG_BIT(3)
+
+#define VLV_PCS_DW1_GRP(ch)		 _VLV_PCS_GRP((ch), 1)
+#define VLV_PCS01_DW1(ch) _VLV_PCS((ch), 0, 1)
+#define VLV_PCS23_DW1(ch) _VLV_PCS((ch), 1, 1)
+#define   CHV_PCS_REQ_SOFTRESET_EN		REG_BIT(23)
+#define   DPIO_PCS_CLK_CRI_RXEB_EIOS_EN		REG_BIT(22)
+#define   DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN	REG_BIT(21)
+#define   DPIO_PCS_CLK_DATAWIDTH_MASK		REG_GENMASK(7, 6)
+#define   DPIO_PCS_CLK_DATAWIDTH_8_10		REG_FIELD_PREP(DPIO_PCS_CLK_DATAWIDTH_MASK, 1)
+#define   DPIO_PCS_CLK_DATAWIDTH_16_20		REG_FIELD_PREP(DPIO_PCS_CLK_DATAWIDTH_MASK, 2)
+#define   DPIO_PCS_CLK_DATAWIDTH_32_40		REG_FIELD_PREP(DPIO_PCS_CLK_DATAWIDTH_MASK, 3)
+#define   DPIO_PCS_CLK_SOFT_RESET		REG_BIT(5)
+
+#define VLV_PCS_DW8_GRP(ch)		_VLV_PCS_GRP((ch), 8)
+#define VLV_PCS01_DW8(ch)		_VLV_PCS((ch), 0, 8)
+#define VLV_PCS23_DW8(ch)		_VLV_PCS((ch), 1, 8)
+#define   DPIO_PCS_USEDCLKCHANNEL		REG_BIT(21)
+#define   DPIO_PCS_USEDCLKCHANNEL_OVRRIDE	REG_BIT(20)
+
+#define VLV_PCS_DW9_GRP(ch)		_VLV_PCS_GRP((ch), 9)
+#define VLV_PCS01_DW9(ch)		_VLV_PCS((ch), 0, 9)
+#define VLV_PCS23_DW9(ch)		_VLV_PCS((ch), 1, 9)
+#define   DPIO_PCS_TX2MARGIN_MASK	REG_GENMASK(15, 13)
+#define   DPIO_PCS_TX2MARGIN_000	REG_FIELD_PREP(DPIO_PCS_TX2MARGIN_MASK, 0)
+#define   DPIO_PCS_TX2MARGIN_101	REG_FIELD_PREP(DPIO_PCS_TX2MARGIN_MASK, 1)
+#define   DPIO_PCS_TX1MARGIN_MASK	REG_GENMASK(12, 10)
+#define   DPIO_PCS_TX1MARGIN_000	REG_FIELD_PREP(DPIO_PCS_TX1MARGIN_MASK, 0)
+#define   DPIO_PCS_TX1MARGIN_101	REG_FIELD_PREP(DPIO_PCS_TX1MARGIN_MASK, 1)
+
+#define VLV_PCS_DW10_GRP(ch)		_VLV_PCS_GRP((ch), 10)
+#define VLV_PCS01_DW10(ch)		_VLV_PCS((ch), 0, 10)
+#define VLV_PCS23_DW10(ch)		_VLV_PCS((ch), 1, 10)
+#define   DPIO_PCS_SWING_CALC_TX1_TX3	REG_BIT(31)
+#define   DPIO_PCS_SWING_CALC_TX0_TX2	REG_BIT(30)
+#define   DPIO_PCS_TX2DEEMP_MASK	REG_GENMASK(27, 24)
+#define   DPIO_PCS_TX2DEEMP_9P5		REG_FIELD_PREP(DPIO_PCS_TX2DEEMP_MASK, 0)
+#define   DPIO_PCS_TX2DEEMP_6P0		REG_FIELD_PREP(DPIO_PCS_TX2DEEMP_MASK, 2)
+#define   DPIO_PCS_TX1DEEMP_MASK	REG_GENMASK(19, 16)
+#define   DPIO_PCS_TX1DEEMP_9P5		REG_FIELD_PREP(DPIO_PCS_TX1DEEMP_MASK, 0)
+#define   DPIO_PCS_TX1DEEMP_6P0		REG_FIELD_PREP(DPIO_PCS_TX1DEEMP_MASK, 2)
+
+#define VLV_PCS_DW11_GRP(ch)		_VLV_PCS_GRP((ch), 11)
+#define VLV_PCS01_DW11(ch)		_VLV_PCS((ch), 0, 11)
+#define VLV_PCS23_DW11(ch)		_VLV_PCS((ch), 1, 11)
+#define   DPIO_TX2_STAGGER_MASK_MASK	REG_GENMASK(28, 24)
+#define   DPIO_TX2_STAGGER_MASK(x)	REG_FIELD_PREP(DPIO_TX2_STAGGER_MASK_MASK, (x))
+#define   DPIO_LANEDESKEW_STRAP_OVRD	REG_BIT(3)
+#define   DPIO_LEFT_TXFIFO_RST_MASTER	REG_BIT(1)
+#define   DPIO_RIGHT_TXFIFO_RST_MASTER	REG_BIT(0)
+
+#define VLV_PCS_DW12_GRP(ch)		_VLV_PCS_GRP((ch), 12)
+#define VLV_PCS01_DW12(ch)		_VLV_PCS((ch), 0, 12)
+#define VLV_PCS23_DW12(ch)		_VLV_PCS((ch), 1, 12)
+#define   DPIO_TX2_STAGGER_MULT_MASK	REG_GENMASK(22, 20)
+#define   DPIO_TX2_STAGGER_MULT(x)	REG_FIELD_PREP(DPIO_TX2_STAGGER_MULT_MASK, (x))
+#define   DPIO_TX1_STAGGER_MULT_MASK	REG_GENMASK(20, 16)
+#define   DPIO_TX1_STAGGER_MULT(x)	REG_FIELD_PREP(DPIO_TX1_STAGGER_MULT_MASK, (x))
+#define   DPIO_TX1_STAGGER_MASK_MASK	REG_GENMASK(12, 8)
+#define   DPIO_TX1_STAGGER_MASK(x)	REG_FIELD_PREP(DPIO_TX1_STAGGER_MASK_MASK, (x))
+#define   DPIO_LANESTAGGER_STRAP_OVRD	REG_BIT(6)
+#define   DPIO_LANESTAGGER_STRAP_MASK	REG_GENMASK(4, 0)
+#define   DPIO_LANESTAGGER_STRAP(x)	REG_FIELD_PREP(DPIO_LANESTAGGER_STRAP_MASK, (x))
+
+#define VLV_PCS_DW14_GRP(ch)		_VLV_PCS_GRP((ch), 14)
+#define VLV_PCS01_DW14(ch)		_VLV_PCS((ch), 0, 14)
+#define VLV_PCS23_DW14(ch)		_VLV_PCS((ch), 1, 14)
+
+#define VLV_PCS_DW17_BCAST		_VLV_PCS_BCAST(17)
+#define VLV_PCS_DW17_GRP(ch)		_VLV_PCS_GRP((ch), 17)
+#define VLV_PCS01_DW17(ch)		_VLV_PCS((ch), 0, 17)
+#define VLV_PCS23_DW17(ch)		_VLV_PCS((ch), 1, 17)
+
+#define VLV_PCS_DW23_GRP(ch)		_VLV_PCS_GRP((ch), 23)
+#define VLV_PCS01_DW23(ch)		_VLV_PCS((ch), 0, 23)
+#define VLV_PCS23_DW23(ch)		_VLV_PCS((ch), 1, 23)
+
+#define VLV_TX_DW2_GRP(ch)		_VLV_TX_GRP((ch), 2)
+#define VLV_TX_DW2(ch, lane)		_VLV_TX((ch), (lane), 2)
+#define   DPIO_SWING_MARGIN000_MASK	REG_GENMASK(23, 16)
+#define   DPIO_SWING_MARGIN000(x)	REG_FIELD_PREP(DPIO_SWING_MARGIN000_MASK, (x))
+#define   DPIO_UNIQ_TRANS_SCALE_MASK	REG_GENMASK(15, 8)
+#define   DPIO_UNIQ_TRANS_SCALE(x)	REG_FIELD_PREP(DPIO_UNIQ_TRANS_SCALE_MASK, (x))
+
+#define VLV_TX_DW3_GRP(ch)		_VLV_TX_GRP((ch), 3)
+#define VLV_TX_DW3(ch, lane)		_VLV_TX((ch), (lane), 3)
+/* The following bit for CHV phy */
+#define   DPIO_TX_UNIQ_TRANS_SCALE_EN	REG_BIT(27)
+#define   DPIO_SWING_MARGIN101_MASK	REG_GENMASK(23, 16)
+#define   DPIO_SWING_MARGIN101(x)	REG_FIELD_PREP(DPIO_SWING_MARGIN101_MASK, (x))
+
+#define VLV_TX_DW4_GRP(ch)		_VLV_TX_GRP((ch), 4)
+#define VLV_TX_DW4(ch, lane)		_VLV_TX((ch), (lane), 4)
+#define   DPIO_SWING_DEEMPH9P5_MASK	REG_GENMASK(31, 24)
+#define   DPIO_SWING_DEEMPH9P5(x)	REG_FIELD_PREP(DPIO_SWING_DEEMPH9P5_MASK, (x))
+#define   DPIO_SWING_DEEMPH6P0_MASK	REG_GENMASK(23, 16)
+#define   DPIO_SWING_DEEMPH6P0_SHIFT	REG_FIELD_PREP(DPIO_SWING_DEEMPH6P0_MASK, (x))
+
+#define VLV_TX_DW5_GRP(ch)		_VLV_TX_GRP((ch), 5)
+#define VLV_TX_DW5(ch, lane)		_VLV_TX((ch), (lane), 5)
+#define   DPIO_TX_OCALINIT_EN		REG_BIT(31)
+
+#define VLV_TX_DW11_GRP(ch)		_VLV_TX_GRP((ch), 11)
+#define VLV_TX_DW11(ch, lane)		_VLV_TX((ch), (lane), 11)
+
+#define VLV_TX_DW14_GRP(ch)		_VLV_TX_GRP((ch), 14)
+#define VLV_TX_DW14(ch, lane)		_VLV_TX((ch), (lane), 14)
+
+/* CHV dpPhy registers */
+#define CHV_PLL_DW0(ch)			_CHV_PLL((ch), 0)
+#define   DPIO_CHV_M2_DIV_MASK		REG_GENMASK(7, 0)
+#define   DPIO_CHV_M2_DIV(m2)		REG_FIELD_PREP(DPIO_CHV_M2_DIV_MASK, (m2))
+
+#define CHV_PLL_DW1(ch)			_CHV_PLL((ch), 1)
+#define   DPIO_CHV_N_DIV_MASK		REG_GENMASK(11, 8)
+#define   DPIO_CHV_N_DIV(n)		REG_FIELD_PREP(DPIO_CHV_N_DIV_MASK, (n))
+#define   DPIO_CHV_M1_DIV_MASK		REG_GENMASK(2, 0)
+#define   DPIO_CHV_M1_DIV(m1)		REG_FIELD_PREP(DPIO_CHV_M1_DIV_MASK, (m1))
+#define   DPIO_CHV_M1_DIV_BY_2		0
+
+#define CHV_PLL_DW2(ch)			_CHV_PLL((ch), 2)
+#define   DPIO_CHV_M2_FRAC_DIV_MASK	REG_GENMASK(21, 0)
+#define   DPIO_CHV_M2_FRAC_DIV(m2_frac)	REG_FIELD_PREP(DPIO_CHV_M2_FRAC_DIV_MASK, (m2_frac))
+
+#define CHV_PLL_DW3(ch)			_CHV_PLL((ch), 3)
+#define   DPIO_CHV_FRAC_DIV_EN		REG_BIT(16)
+#define   DPIO_CHV_SECOND_MOD		REG_BIT(8)
+#define   DPIO_CHV_FEEDFWD_GAIN_MASK	REG_GENMASK(3, 0)
+#define   DPIO_CHV_FEEDFWD_GAIN(x)	REG_FIELD_PREP(DPIO_CHV_FEEDFWD_GAIN_MASK, (x))
+
+#define CHV_PLL_DW6(ch)			_CHV_PLL((ch), 6)
+#define   DPIO_CHV_GAIN_CTRL_MASK	REG_GENMASK(18, 16)
+#define   DPIO_CHV_GAIN_CTRL(x)		REG_FIELD_PREP(DPIO_CHV_GAIN_CTRL_MASK, (x))
+#define   DPIO_CHV_INT_COEFF_MASK	REG_GENMASK(12, 8)
+#define   DPIO_CHV_INT_COEFF(x)		REG_FIELD_PREP(DPIO_CHV_INT_COEFF_MASK, (x))
+#define   DPIO_CHV_PROP_COEFF_MASK	REG_GENMASK(3, 0)
+#define   DPIO_CHV_PROP_COEFF(x)	REG_FIELD_PREP(DPIO_CHV_PROP_COEFF_MASK, (x))
+
+#define CHV_PLL_DW8(ch)			_CHV_PLL((ch), 8)
+#define   DPIO_CHV_TDC_TARGET_CNT_MASK	REG_GENMASK(9, 0)
+#define   DPIO_CHV_TDC_TARGET_CNT(x)	REG_FIELD_PREP(DPIO_CHV_TDC_TARGET_CNT_MASK, (x))
+
+#define CHV_PLL_DW9(ch)			_CHV_PLL((ch), 9)
+#define   DPIO_CHV_INT_LOCK_THRESHOLD_MASK		REG_GENMASK(3, 1)
+#define   DPIO_CHV_INT_LOCK_THRESHOLD(x)		REG_FIELD_PREP(DPIO_CHV_INT_LOCK_THRESHOLD_MASK, (x))
+#define   DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE	REG_BIT(0) /* 1: coarse & 0 : fine  */
+
+#define CHV_CMN_DW0_CH0			_CHV_CMN(0, 0)
+#define   DPIO_ALLDL_POWERDOWN_CH0	REG_BIT(19)
+#define   DPIO_ANYDL_POWERDOWN_CH0	REG_BIT(18)
+#define   DPIO_ALLDL_POWERDOWN		BIT(1)
+#define   DPIO_ANYDL_POWERDOWN		BIT(0)
+
+#define CHV_CMN_DW5_CH0			_CHV_CMN(0, 5)
+#define   CHV_BUFRIGHTENA1_MASK		REG_GENMASK(21, 20)
+#define   CHV_BUFRIGHTENA1_DISABLE	REG_FIELD_PREP(CHV_BUFRIGHTENA1_MASK, 0)
+#define   CHV_BUFRIGHTENA1_NORMAL	REG_FIELD_PREP(CHV_BUFRIGHTENA1_MASK, 1)
+#define   CHV_BUFRIGHTENA1_FORCE	REG_FIELD_PREP(CHV_BUFRIGHTENA1_MASK, 3)
+#define   CHV_BUFLEFTENA1_MASK		REG_GENMASK(23, 22)
+#define   CHV_BUFLEFTENA1_DISABLE	REG_FIELD_PREP(CHV_BUFLEFTENA1_MASK, 0)
+#define   CHV_BUFLEFTENA1_NORMAL	REG_FIELD_PREP(CHV_BUFLEFTENA1_MASK, 1)
+#define   CHV_BUFLEFTENA1_FORCE		REG_FIELD_PREP(CHV_BUFLEFTENA1_MASK, 3)
+
+#define CHV_CMN_DW13_CH0		_CHV_CMN(0, 13)
+#define CHV_CMN_DW0_CH1			_CHV_CMN(1, 0)
+#define   DPIO_CHV_S1_DIV_MASK		REG_GENMASK(23, 21)
+#define   DPIO_CHV_S1_DIV(s1)		REG_FIELD_PREP(DPIO_CHV_S1_DIV_MASK, (s1))
+#define   DPIO_CHV_P1_DIV_MASK		REG_GENMASK(15, 13)
+#define   DPIO_CHV_P1_DIV(p1)		REG_FIELD_PREP(DPIO_CHV_P1_DIV_MASK, (p1))
+#define   DPIO_CHV_P2_DIV_MASK		REG_GENMASK(12, 8)
+#define   DPIO_CHV_P2_DIV(p2)		REG_FIELD_PREP(DPIO_CHV_P2_DIV_MASK, (p2))
+#define   DPIO_CHV_K_DIV_MASK		REG_GENMASK(7, 4)
+#define   DPIO_CHV_K_DIV(k)		REG_FIELD_PREP(DPIO_CHV_K_DIV_MASK, (k))
+#define   DPIO_PLL_FREQLOCK		REG_BIT(1)
+#define   DPIO_PLL_LOCK			REG_BIT(0)
+#define CHV_CMN_DW13(ch)		_PIPE(ch, CHV_CMN_DW13_CH0, CHV_CMN_DW0_CH1)
+
+#define CHV_CMN_DW14_CH0		_CHV_CMN(0, 14)
+#define CHV_CMN_DW1_CH1			_CHV_CMN(1, 1)
+#define   DPIO_AFC_RECAL		REG_BIT(14)
+#define   DPIO_DCLKP_EN			REG_BIT(13)
+#define   CHV_BUFLEFTENA2_MASK		REG_GENMASK(18, 17) /* CL2 DW1 only */
+#define   CHV_BUFLEFTENA2_DISABLE	REG_FIELD_PREP(CHV_BUFLEFTENA2_MASK, 0)
+#define   CHV_BUFLEFTENA2_NORMAL	REG_FIELD_PREP(CHV_BUFLEFTENA2_MASK, 1)
+#define   CHV_BUFLEFTENA2_FORCE		REG_FIELD_PREP(CHV_BUFLEFTENA2_MASK, 3)
+#define   CHV_BUFRIGHTENA2_MASK		REG_GENMASK(20, 19) /* CL2 DW1 only */
+#define   CHV_BUFRIGHTENA2_DISABLE	REG_FIELD_PREP(CHV_BUFRIGHTENA2_MASK, 0)
+#define   CHV_BUFRIGHTENA2_NORMAL	REG_FIELD_PREP(CHV_BUFRIGHTENA2_MASK, 1)
+#define   CHV_BUFRIGHTENA2_FORCE	REG_FIELD_PREP(CHV_BUFRIGHTENA2_MASK, 3)
+#define CHV_CMN_DW14(ch)		_PIPE(ch, CHV_CMN_DW14_CH0, CHV_CMN_DW1_CH1)
+
+#define CHV_CMN_DW19_CH0		_CHV_CMN(0, 19)
+#define CHV_CMN_DW6_CH1			_CHV_CMN(1, 6)
+#define   DPIO_ALLDL_POWERDOWN_CH1	REG_BIT(30) /* CL2 DW6 only */
+#define   DPIO_ANYDL_POWERDOWN_CH1	REG_BIT(29) /* CL2 DW6 only */
+#define   DPIO_DYNPWRDOWNEN_CH1		REG_BIT(28) /* CL2 DW6 only */
+#define   CHV_CMN_USEDCLKCHANNEL	REG_BIT(13)
+#define CHV_CMN_DW19(ch)		_PIPE(ch, CHV_CMN_DW19_CH0, CHV_CMN_DW6_CH1)
+
+#define CHV_CMN_DW28			_CHV_CMN(0, 28)
+#define   DPIO_CL1POWERDOWNEN			REG_BIT(23)
+#define   DPIO_DYNPWRDOWNEN_CH0			REG_BIT(22)
+#define   DPIO_SUS_CLK_CONFIG_MASK		REG_GENMASK(1, 0)
+#define   DPIO_SUS_CLK_CONFIG_ON		REG_FIELD_PREP(DPIO_SUS_CLK_CONFIG_MASK, 0)
+#define   DPIO_SUS_CLK_CONFIG_CLKREQ		REG_FIELD_PREP(DPIO_SUS_CLK_CONFIG_MASK, 1)
+#define   DPIO_SUS_CLK_CONFIG_GATE		REG_FIELD_PREP(DPIO_SUS_CLK_CONFIG_MASK, 2)
+#define   DPIO_SUS_CLK_CONFIG_GATE_CLKREQ	REG_FIELD_PREP(DPIO_SUS_CLK_CONFIG_MASK, 3)
+
+#define CHV_CMN_DW30			_CHV_CMN(0, 30)
+#define   DPIO_CL2_LDOFUSE_PWRENB	REG_BIT(6)
+#define   DPIO_LRC_BYPASS		REG_BIT(3)
+
+#define CHV_TX_DW0(ch, lane)		_VLV_TX((ch), (lane), 0)
+#define CHV_TX_DW1(ch, lane)		_VLV_TX((ch), (lane), 1)
+#define CHV_TX_DW2(ch, lane)		_VLV_TX((ch), (lane), 2)
+#define CHV_TX_DW3(ch, lane)		_VLV_TX((ch), (lane), 3)
+#define CHV_TX_DW4(ch, lane)		_VLV_TX((ch), (lane), 4)
+#define CHV_TX_DW5(ch, lane)		_VLV_TX((ch), (lane), 5)
+#define CHV_TX_DW6(ch, lane)		_VLV_TX((ch), (lane), 6)
+#define CHV_TX_DW7(ch, lane)		_VLV_TX((ch), (lane), 7)
+#define CHV_TX_DW8(ch, lane)		_VLV_TX((ch), (lane), 8)
+#define CHV_TX_DW9(ch, lane)		_VLV_TX((ch), (lane), 9)
+#define CHV_TX_DW10(ch, lane)		_VLV_TX((ch), (lane), 10)
+
+#define CHV_TX_DW11(ch, lane)		_VLV_TX((ch), (lane), 11)
+#define   DPIO_FRC_LATENCY_MASK		REG_GENMASK(10, 8)
+#define   DPIO_FRC_LATENCY(x)		REG_FIELD_PREP(DPIO_FRC_LATENCY_MASK, (x))
+
+#define CHV_TX_DW14(ch, lane)		_VLV_TX((ch), (lane), 14)
+#define   DPIO_UPAR			REG_BIT(30)
+
+#endif /* __VLV_DPIO_PHY_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi.c b/drivers/gpu/drm/i915/display/vlv_dsi.c
index 9b33b8a74d64..ee9923c7b115 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi.c
+++ b/drivers/gpu/drm/i915/display/vlv_dsi.c
@@ -85,20 +85,18 @@ enum mipi_dsi_pixel_format pixel_format_from_register_bits(u32 fmt)
 
 void vlv_dsi_wait_for_fifo_empty(struct intel_dsi *intel_dsi, enum port port)
 {
-	struct drm_encoder *encoder = &intel_dsi->base.base;
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_display *display = to_intel_display(&intel_dsi->base);
 	u32 mask;
 
 	mask = LP_CTRL_FIFO_EMPTY | HS_CTRL_FIFO_EMPTY |
 		LP_DATA_FIFO_EMPTY | HS_DATA_FIFO_EMPTY;
 
-	if (intel_de_wait_for_set(dev_priv, MIPI_GEN_FIFO_STAT(port),
+	if (intel_de_wait_for_set(display, MIPI_GEN_FIFO_STAT(display, port),
 				  mask, 100))
-		drm_err(&dev_priv->drm, "DPI FIFOs are not empty\n");
+		drm_err(display->drm, "DPI FIFOs are not empty\n");
 }
 
-static void write_data(struct drm_i915_private *dev_priv,
+static void write_data(struct intel_display *display,
 		       i915_reg_t reg,
 		       const u8 *data, u32 len)
 {
@@ -110,18 +108,18 @@ static void write_data(struct drm_i915_private *dev_priv,
 		for (j = 0; j < min_t(u32, len - i, 4); j++)
 			val |= *data++ << 8 * j;
 
-		intel_de_write(dev_priv, reg, val);
+		intel_de_write(display, reg, val);
 	}
 }
 
-static void read_data(struct drm_i915_private *dev_priv,
+static void read_data(struct intel_display *display,
 		      i915_reg_t reg,
 		      u8 *data, u32 len)
 {
 	u32 i, j;
 
 	for (i = 0; i < len; i += 4) {
-		u32 val = intel_de_read(dev_priv, reg);
+		u32 val = intel_de_read(display, reg);
 
 		for (j = 0; j < min_t(u32, len - i, 4); j++)
 			*data++ = val >> 8 * j;
@@ -132,8 +130,8 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host,
 				       const struct mipi_dsi_msg *msg)
 {
 	struct intel_dsi_host *intel_dsi_host = to_intel_dsi_host(host);
-	struct drm_device *dev = intel_dsi_host->intel_dsi->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_dsi *intel_dsi = intel_dsi_host->intel_dsi;
+	struct intel_display *display = to_intel_display(&intel_dsi->base);
 	enum port port = intel_dsi_host->port;
 	struct mipi_dsi_packet packet;
 	ssize_t ret;
@@ -148,51 +146,51 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host,
 	header = packet.header;
 
 	if (msg->flags & MIPI_DSI_MSG_USE_LPM) {
-		data_reg = MIPI_LP_GEN_DATA(port);
+		data_reg = MIPI_LP_GEN_DATA(display, port);
 		data_mask = LP_DATA_FIFO_FULL;
-		ctrl_reg = MIPI_LP_GEN_CTRL(port);
+		ctrl_reg = MIPI_LP_GEN_CTRL(display, port);
 		ctrl_mask = LP_CTRL_FIFO_FULL;
 	} else {
-		data_reg = MIPI_HS_GEN_DATA(port);
+		data_reg = MIPI_HS_GEN_DATA(display, port);
 		data_mask = HS_DATA_FIFO_FULL;
-		ctrl_reg = MIPI_HS_GEN_CTRL(port);
+		ctrl_reg = MIPI_HS_GEN_CTRL(display, port);
 		ctrl_mask = HS_CTRL_FIFO_FULL;
 	}
 
 	/* note: this is never true for reads */
 	if (packet.payload_length) {
-		if (intel_de_wait_for_clear(dev_priv, MIPI_GEN_FIFO_STAT(port),
+		if (intel_de_wait_for_clear(display, MIPI_GEN_FIFO_STAT(display, port),
 					    data_mask, 50))
-			drm_err(&dev_priv->drm,
+			drm_err(display->drm,
 				"Timeout waiting for HS/LP DATA FIFO !full\n");
 
-		write_data(dev_priv, data_reg, packet.payload,
+		write_data(display, data_reg, packet.payload,
 			   packet.payload_length);
 	}
 
 	if (msg->rx_len) {
-		intel_de_write(dev_priv, MIPI_INTR_STAT(port),
+		intel_de_write(display, MIPI_INTR_STAT(display, port),
 			       GEN_READ_DATA_AVAIL);
 	}
 
-	if (intel_de_wait_for_clear(dev_priv, MIPI_GEN_FIFO_STAT(port),
+	if (intel_de_wait_for_clear(display, MIPI_GEN_FIFO_STAT(display, port),
 				    ctrl_mask, 50)) {
-		drm_err(&dev_priv->drm,
+		drm_err(display->drm,
 			"Timeout waiting for HS/LP CTRL FIFO !full\n");
 	}
 
-	intel_de_write(dev_priv, ctrl_reg,
+	intel_de_write(display, ctrl_reg,
 		       header[2] << 16 | header[1] << 8 | header[0]);
 
 	/* ->rx_len is set only for reads */
 	if (msg->rx_len) {
 		data_mask = GEN_READ_DATA_AVAIL;
-		if (intel_de_wait_for_set(dev_priv, MIPI_INTR_STAT(port),
+		if (intel_de_wait_for_set(display, MIPI_INTR_STAT(display, port),
 					  data_mask, 50))
-			drm_err(&dev_priv->drm,
+			drm_err(display->drm,
 				"Timeout waiting for read data.\n");
 
-		read_data(dev_priv, data_reg, msg->rx_buf, msg->rx_len);
+		read_data(display, data_reg, msg->rx_buf, msg->rx_len);
 	}
 
 	/* XXX: fix for reads and writes */
@@ -225,9 +223,7 @@ static const struct mipi_dsi_host_ops intel_dsi_host_ops = {
 static int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd, bool hs,
 			enum port port)
 {
-	struct drm_encoder *encoder = &intel_dsi->base.base;
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_display *display = to_intel_display(&intel_dsi->base);
 	u32 mask;
 
 	/* XXX: pipe, hs */
@@ -237,18 +233,18 @@ static int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd, bool hs,
 		cmd |= DPI_LP_MODE;
 
 	/* clear bit */
-	intel_de_write(dev_priv, MIPI_INTR_STAT(port), SPL_PKT_SENT_INTERRUPT);
+	intel_de_write(display, MIPI_INTR_STAT(display, port), SPL_PKT_SENT_INTERRUPT);
 
 	/* XXX: old code skips write if control unchanged */
-	if (cmd == intel_de_read(dev_priv, MIPI_DPI_CONTROL(port)))
-		drm_dbg_kms(&dev_priv->drm,
+	if (cmd == intel_de_read(display, MIPI_DPI_CONTROL(display, port)))
+		drm_dbg_kms(display->drm,
 			    "Same special packet %02x twice in a row.\n", cmd);
 
-	intel_de_write(dev_priv, MIPI_DPI_CONTROL(port), cmd);
+	intel_de_write(display, MIPI_DPI_CONTROL(display, port), cmd);
 
 	mask = SPL_PKT_SENT_INTERRUPT;
-	if (intel_de_wait_for_set(dev_priv, MIPI_INTR_STAT(port), mask, 100))
-		drm_err(&dev_priv->drm,
+	if (intel_de_wait_for_set(display, MIPI_INTR_STAT(display, port), mask, 100))
+		drm_err(display->drm,
 			"Video mode command 0x%08x send failed.\n", cmd);
 
 	return 0;
@@ -273,8 +269,7 @@ static int intel_dsi_compute_config(struct intel_encoder *encoder,
 				    struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi,
-						   base);
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	struct intel_connector *intel_connector = intel_dsi->attached_connector;
 	struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
 	int ret;
@@ -329,7 +324,7 @@ static int intel_dsi_compute_config(struct intel_encoder *encoder,
 
 static bool glk_dsi_enable_io(struct intel_encoder *encoder)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 	bool cold_boot = false;
@@ -339,29 +334,30 @@ static bool glk_dsi_enable_io(struct intel_encoder *encoder)
 	 * Power ON MIPI IO first and then write into IO reset and LP wake bits
 	 */
 	for_each_dsi_port(port, intel_dsi->ports)
-		intel_de_rmw(dev_priv, MIPI_CTRL(port), 0, GLK_MIPIIO_ENABLE);
+		intel_de_rmw(display, MIPI_CTRL(display, port), 0, GLK_MIPIIO_ENABLE);
 
 	/* Put the IO into reset */
-	intel_de_rmw(dev_priv, MIPI_CTRL(PORT_A), GLK_MIPIIO_RESET_RELEASED, 0);
+	intel_de_rmw(display, MIPI_CTRL(display, PORT_A), GLK_MIPIIO_RESET_RELEASED, 0);
 
 	/* Program LP Wake */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		u32 tmp = intel_de_read(dev_priv, MIPI_DEVICE_READY(port));
-		intel_de_rmw(dev_priv, MIPI_CTRL(port),
+		u32 tmp = intel_de_read(display, MIPI_DEVICE_READY(display, port));
+
+		intel_de_rmw(display, MIPI_CTRL(display, port),
 			     GLK_LP_WAKE, (tmp & DEVICE_READY) ? GLK_LP_WAKE : 0);
 	}
 
 	/* Wait for Pwr ACK */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_de_wait_for_set(dev_priv, MIPI_CTRL(port),
+		if (intel_de_wait_for_set(display, MIPI_CTRL(display, port),
 					  GLK_MIPIIO_PORT_POWERED, 20))
-			drm_err(&dev_priv->drm, "MIPIO port is powergated\n");
+			drm_err(display->drm, "MIPIO port is powergated\n");
 	}
 
 	/* Check for cold boot scenario */
 	for_each_dsi_port(port, intel_dsi->ports) {
 		cold_boot |=
-			!(intel_de_read(dev_priv, MIPI_DEVICE_READY(port)) & DEVICE_READY);
+			!(intel_de_read(display, MIPI_DEVICE_READY(display, port)) & DEVICE_READY);
 	}
 
 	return cold_boot;
@@ -369,99 +365,100 @@ static bool glk_dsi_enable_io(struct intel_encoder *encoder)
 
 static void glk_dsi_device_ready(struct intel_encoder *encoder)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 
 	/* Wait for MIPI PHY status bit to set */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_de_wait_for_set(dev_priv, MIPI_CTRL(port),
+		if (intel_de_wait_for_set(display, MIPI_CTRL(display, port),
 					  GLK_PHY_STATUS_PORT_READY, 20))
-			drm_err(&dev_priv->drm, "PHY is not ON\n");
+			drm_err(display->drm, "PHY is not ON\n");
 	}
 
 	/* Get IO out of reset */
-	intel_de_rmw(dev_priv, MIPI_CTRL(PORT_A), 0, GLK_MIPIIO_RESET_RELEASED);
+	intel_de_rmw(display, MIPI_CTRL(display, PORT_A), 0, GLK_MIPIIO_RESET_RELEASED);
 
 	/* Get IO out of Low power state*/
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (!(intel_de_read(dev_priv, MIPI_DEVICE_READY(port)) & DEVICE_READY)) {
-			intel_de_rmw(dev_priv, MIPI_DEVICE_READY(port),
+		if (!(intel_de_read(display, MIPI_DEVICE_READY(display, port)) & DEVICE_READY)) {
+			intel_de_rmw(display, MIPI_DEVICE_READY(display, port),
 				     ULPS_STATE_MASK, DEVICE_READY);
 			usleep_range(10, 15);
 		} else {
 			/* Enter ULPS */
-			intel_de_rmw(dev_priv, MIPI_DEVICE_READY(port),
+			intel_de_rmw(display, MIPI_DEVICE_READY(display, port),
 				     ULPS_STATE_MASK, ULPS_STATE_ENTER | DEVICE_READY);
 
 			/* Wait for ULPS active */
-			if (intel_de_wait_for_clear(dev_priv, MIPI_CTRL(port),
+			if (intel_de_wait_for_clear(display, MIPI_CTRL(display, port),
 						    GLK_ULPS_NOT_ACTIVE, 20))
-				drm_err(&dev_priv->drm, "ULPS not active\n");
+				drm_err(display->drm, "ULPS not active\n");
 
 			/* Exit ULPS */
-			intel_de_rmw(dev_priv, MIPI_DEVICE_READY(port),
+			intel_de_rmw(display, MIPI_DEVICE_READY(display, port),
 				     ULPS_STATE_MASK, ULPS_STATE_EXIT | DEVICE_READY);
 
 			/* Enter Normal Mode */
-			intel_de_rmw(dev_priv, MIPI_DEVICE_READY(port),
+			intel_de_rmw(display, MIPI_DEVICE_READY(display, port),
 				     ULPS_STATE_MASK,
 				     ULPS_STATE_NORMAL_OPERATION | DEVICE_READY);
 
-			intel_de_rmw(dev_priv, MIPI_CTRL(port), GLK_LP_WAKE, 0);
+			intel_de_rmw(display, MIPI_CTRL(display, port), GLK_LP_WAKE, 0);
 		}
 	}
 
 	/* Wait for Stop state */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_de_wait_for_set(dev_priv, MIPI_CTRL(port),
+		if (intel_de_wait_for_set(display, MIPI_CTRL(display, port),
 					  GLK_DATA_LANE_STOP_STATE, 20))
-			drm_err(&dev_priv->drm,
+			drm_err(display->drm,
 				"Date lane not in STOP state\n");
 	}
 
 	/* Wait for AFE LATCH */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_de_wait_for_set(dev_priv, BXT_MIPI_PORT_CTRL(port),
+		if (intel_de_wait_for_set(display, BXT_MIPI_PORT_CTRL(port),
 					  AFE_LATCHOUT, 20))
-			drm_err(&dev_priv->drm,
+			drm_err(display->drm,
 				"D-PHY not entering LP-11 state\n");
 	}
 }
 
 static void bxt_dsi_device_ready(struct intel_encoder *encoder)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 	u32 val;
 
-	drm_dbg_kms(&dev_priv->drm, "\n");
+	drm_dbg_kms(display->drm, "\n");
 
 	/* Enable MIPI PHY transparent latch */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		intel_de_rmw(dev_priv, BXT_MIPI_PORT_CTRL(port), 0, LP_OUTPUT_HOLD);
+		intel_de_rmw(display, BXT_MIPI_PORT_CTRL(port), 0, LP_OUTPUT_HOLD);
 		usleep_range(2000, 2500);
 	}
 
 	/* Clear ULPS and set device ready */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		val = intel_de_read(dev_priv, MIPI_DEVICE_READY(port));
+		val = intel_de_read(display, MIPI_DEVICE_READY(display, port));
 		val &= ~ULPS_STATE_MASK;
-		intel_de_write(dev_priv, MIPI_DEVICE_READY(port), val);
+		intel_de_write(display, MIPI_DEVICE_READY(display, port), val);
 		usleep_range(2000, 2500);
 		val |= DEVICE_READY;
-		intel_de_write(dev_priv, MIPI_DEVICE_READY(port), val);
+		intel_de_write(display, MIPI_DEVICE_READY(display, port), val);
 	}
 }
 
 static void vlv_dsi_device_ready(struct intel_encoder *encoder)
 {
+	struct intel_display *display = to_intel_display(encoder);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 
-	drm_dbg_kms(&dev_priv->drm, "\n");
+	drm_dbg_kms(display->drm, "\n");
 
 	vlv_flisdsi_get(dev_priv);
 	/* program rcomp for compliance, reduce from 50 ohms to 45 ohms
@@ -474,7 +471,7 @@ static void vlv_dsi_device_ready(struct intel_encoder *encoder)
 
 	for_each_dsi_port(port, intel_dsi->ports) {
 
-		intel_de_write(dev_priv, MIPI_DEVICE_READY(port),
+		intel_de_write(display, MIPI_DEVICE_READY(display, port),
 			       ULPS_STATE_ENTER);
 		usleep_range(2500, 3000);
 
@@ -482,14 +479,14 @@ static void vlv_dsi_device_ready(struct intel_encoder *encoder)
 		 * Common bit for both MIPI Port A & MIPI Port C
 		 * No similar bit in MIPI Port C reg
 		 */
-		intel_de_rmw(dev_priv, MIPI_PORT_CTRL(PORT_A), 0, LP_OUTPUT_HOLD);
+		intel_de_rmw(display, VLV_MIPI_PORT_CTRL(PORT_A), 0, LP_OUTPUT_HOLD);
 		usleep_range(1000, 1500);
 
-		intel_de_write(dev_priv, MIPI_DEVICE_READY(port),
+		intel_de_write(display, MIPI_DEVICE_READY(display, port),
 			       ULPS_STATE_EXIT);
 		usleep_range(2500, 3000);
 
-		intel_de_write(dev_priv, MIPI_DEVICE_READY(port),
+		intel_de_write(display, MIPI_DEVICE_READY(display, port),
 			       DEVICE_READY);
 		usleep_range(2500, 3000);
 	}
@@ -509,50 +506,50 @@ static void intel_dsi_device_ready(struct intel_encoder *encoder)
 
 static void glk_dsi_enter_low_power_mode(struct intel_encoder *encoder)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 
 	/* Enter ULPS */
 	for_each_dsi_port(port, intel_dsi->ports)
-		intel_de_rmw(dev_priv, MIPI_DEVICE_READY(port),
+		intel_de_rmw(display, MIPI_DEVICE_READY(display, port),
 			     ULPS_STATE_MASK, ULPS_STATE_ENTER | DEVICE_READY);
 
 	/* Wait for MIPI PHY status bit to unset */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_de_wait_for_clear(dev_priv, MIPI_CTRL(port),
+		if (intel_de_wait_for_clear(display, MIPI_CTRL(display, port),
 					    GLK_PHY_STATUS_PORT_READY, 20))
-			drm_err(&dev_priv->drm, "PHY is not turning OFF\n");
+			drm_err(display->drm, "PHY is not turning OFF\n");
 	}
 
 	/* Wait for Pwr ACK bit to unset */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_de_wait_for_clear(dev_priv, MIPI_CTRL(port),
+		if (intel_de_wait_for_clear(display, MIPI_CTRL(display, port),
 					    GLK_MIPIIO_PORT_POWERED, 20))
-			drm_err(&dev_priv->drm,
+			drm_err(display->drm,
 				"MIPI IO Port is not powergated\n");
 	}
 }
 
 static void glk_dsi_disable_mipi_io(struct intel_encoder *encoder)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 
 	/* Put the IO into reset */
-	intel_de_rmw(dev_priv, MIPI_CTRL(PORT_A), GLK_MIPIIO_RESET_RELEASED, 0);
+	intel_de_rmw(display, MIPI_CTRL(display, PORT_A), GLK_MIPIIO_RESET_RELEASED, 0);
 
 	/* Wait for MIPI PHY status bit to unset */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_de_wait_for_clear(dev_priv, MIPI_CTRL(port),
+		if (intel_de_wait_for_clear(display, MIPI_CTRL(display, port),
 					    GLK_PHY_STATUS_PORT_READY, 20))
-			drm_err(&dev_priv->drm, "PHY is not turning OFF\n");
+			drm_err(display->drm, "PHY is not turning OFF\n");
 	}
 
 	/* Clear MIPI mode */
 	for_each_dsi_port(port, intel_dsi->ports)
-		intel_de_rmw(dev_priv, MIPI_CTRL(port), GLK_MIPIIO_ENABLE, 0);
+		intel_de_rmw(display, MIPI_CTRL(display, port), GLK_MIPIIO_ENABLE, 0);
 }
 
 static void glk_dsi_clear_device_ready(struct intel_encoder *encoder)
@@ -564,30 +561,31 @@ static void glk_dsi_clear_device_ready(struct intel_encoder *encoder)
 static i915_reg_t port_ctrl_reg(struct drm_i915_private *i915, enum port port)
 {
 	return IS_GEMINILAKE(i915) || IS_BROXTON(i915) ?
-		BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
+		BXT_MIPI_PORT_CTRL(port) : VLV_MIPI_PORT_CTRL(port);
 }
 
 static void vlv_dsi_clear_device_ready(struct intel_encoder *encoder)
 {
+	struct intel_display *display = to_intel_display(encoder);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 
-	drm_dbg_kms(&dev_priv->drm, "\n");
+	drm_dbg_kms(display->drm, "\n");
 	for_each_dsi_port(port, intel_dsi->ports) {
 		/* Common bit for both MIPI Port A & MIPI Port C on VLV/CHV */
 		i915_reg_t port_ctrl = IS_BROXTON(dev_priv) ?
-			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(PORT_A);
+			BXT_MIPI_PORT_CTRL(port) : VLV_MIPI_PORT_CTRL(PORT_A);
 
-		intel_de_write(dev_priv, MIPI_DEVICE_READY(port),
+		intel_de_write(display, MIPI_DEVICE_READY(display, port),
 			       DEVICE_READY | ULPS_STATE_ENTER);
 		usleep_range(2000, 2500);
 
-		intel_de_write(dev_priv, MIPI_DEVICE_READY(port),
+		intel_de_write(display, MIPI_DEVICE_READY(display, port),
 			       DEVICE_READY | ULPS_STATE_EXIT);
 		usleep_range(2000, 2500);
 
-		intel_de_write(dev_priv, MIPI_DEVICE_READY(port),
+		intel_de_write(display, MIPI_DEVICE_READY(display, port),
 			       DEVICE_READY | ULPS_STATE_ENTER);
 		usleep_range(2000, 2500);
 
@@ -596,15 +594,15 @@ static void vlv_dsi_clear_device_ready(struct intel_encoder *encoder)
 		 * Port A only. MIPI Port C has no similar bit for checking.
 		 */
 		if ((IS_BROXTON(dev_priv) || port == PORT_A) &&
-		    intel_de_wait_for_clear(dev_priv, port_ctrl,
+		    intel_de_wait_for_clear(display, port_ctrl,
 					    AFE_LATCHOUT, 30))
-			drm_err(&dev_priv->drm, "DSI LP not going Low\n");
+			drm_err(display->drm, "DSI LP not going Low\n");
 
 		/* Disable MIPI PHY transparent latch */
-		intel_de_rmw(dev_priv, port_ctrl, LP_OUTPUT_HOLD, 0);
+		intel_de_rmw(display, port_ctrl, LP_OUTPUT_HOLD, 0);
 		usleep_range(1000, 1500);
 
-		intel_de_write(dev_priv, MIPI_DEVICE_READY(port), 0x00);
+		intel_de_write(display, MIPI_DEVICE_READY(display, port), 0x00);
 		usleep_range(2000, 2500);
 	}
 }
@@ -612,6 +610,7 @@ static void vlv_dsi_clear_device_ready(struct intel_encoder *encoder)
 static void intel_dsi_port_enable(struct intel_encoder *encoder,
 				  const struct intel_crtc_state *crtc_state)
 {
+	struct intel_display *display = to_intel_display(encoder);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
@@ -622,11 +621,11 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder,
 
 		if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
 			for_each_dsi_port(port, intel_dsi->ports)
-				intel_de_rmw(dev_priv, MIPI_CTRL(port),
+				intel_de_rmw(display, MIPI_CTRL(display, port),
 					     BXT_PIXEL_OVERLAP_CNT_MASK,
 					     temp << BXT_PIXEL_OVERLAP_CNT_SHIFT);
 		} else {
-			intel_de_rmw(dev_priv, VLV_CHICKEN_3,
+			intel_de_rmw(display, VLV_CHICKEN_3,
 				     PIXEL_OVERLAP_CNT_MASK,
 				     temp << PIXEL_OVERLAP_CNT_SHIFT);
 		}
@@ -636,7 +635,7 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder,
 		i915_reg_t port_ctrl = port_ctrl_reg(dev_priv, port);
 		u32 temp;
 
-		temp = intel_de_read(dev_priv, port_ctrl);
+		temp = intel_de_read(display, port_ctrl);
 
 		temp &= ~LANE_CONFIGURATION_MASK;
 		temp &= ~DUAL_LINK_MODE_MASK;
@@ -656,15 +655,15 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder,
 			temp |= DITHERING_ENABLE;
 
 		/* assert ip_tg_enable signal */
-		intel_de_write(dev_priv, port_ctrl, temp | DPI_ENABLE);
-		intel_de_posting_read(dev_priv, port_ctrl);
+		intel_de_write(display, port_ctrl, temp | DPI_ENABLE);
+		intel_de_posting_read(display, port_ctrl);
 	}
 }
 
 static void intel_dsi_port_disable(struct intel_encoder *encoder)
 {
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_display *display = to_intel_display(encoder);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 
@@ -672,11 +671,12 @@ static void intel_dsi_port_disable(struct intel_encoder *encoder)
 		i915_reg_t port_ctrl = port_ctrl_reg(dev_priv, port);
 
 		/* de-assert ip_tg_enable signal */
-		intel_de_rmw(dev_priv, port_ctrl, DPI_ENABLE, 0);
-		intel_de_posting_read(dev_priv, port_ctrl);
+		intel_de_rmw(display, port_ctrl, DPI_ENABLE, 0);
+		intel_de_posting_read(display, port_ctrl);
 	}
 }
-static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
+
+static void intel_dsi_prepare(struct intel_encoder *encoder,
 			      const struct intel_crtc_state *pipe_config);
 static void intel_dsi_unprepare(struct intel_encoder *encoder);
 
@@ -726,6 +726,7 @@ static void intel_dsi_pre_enable(struct intel_atomic_state *state,
 				 const struct intel_crtc_state *pipe_config,
 				 const struct drm_connector_state *conn_state)
 {
+	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
@@ -733,7 +734,7 @@ static void intel_dsi_pre_enable(struct intel_atomic_state *state,
 	enum port port;
 	bool glk_cold_boot = false;
 
-	drm_dbg_kms(&dev_priv->drm, "\n");
+	drm_dbg_kms(display->drm, "\n");
 
 	intel_dsi_wait_panel_power_cycle(intel_dsi);
 
@@ -753,16 +754,16 @@ static void intel_dsi_pre_enable(struct intel_atomic_state *state,
 
 	if (IS_BROXTON(dev_priv)) {
 		/* Add MIPI IO reset programming for modeset */
-		intel_de_rmw(dev_priv, BXT_P_CR_GT_DISP_PWRON, 0, MIPIO_RST_CTRL);
+		intel_de_rmw(display, BXT_P_CR_GT_DISP_PWRON, 0, MIPIO_RST_CTRL);
 
 		/* Power up DSI regulator */
-		intel_de_write(dev_priv, BXT_P_DSI_REGULATOR_CFG, STAP_SELECT);
-		intel_de_write(dev_priv, BXT_P_DSI_REGULATOR_TX_CTRL, 0);
+		intel_de_write(display, BXT_P_DSI_REGULATOR_CFG, STAP_SELECT);
+		intel_de_write(display, BXT_P_DSI_REGULATOR_TX_CTRL, 0);
 	}
 
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		/* Disable DPOunit clock gating, can stall pipe */
-		intel_de_rmw(dev_priv, DSPCLK_GATE_D(dev_priv),
+		intel_de_rmw(display, DSPCLK_GATE_D(dev_priv),
 			     0, DPOUNIT_CLOCK_GATE_DISABLE);
 	}
 
@@ -798,8 +799,8 @@ static void intel_dsi_pre_enable(struct intel_atomic_state *state,
 	 */
 	if (is_cmd_mode(intel_dsi)) {
 		for_each_dsi_port(port, intel_dsi->ports)
-			intel_de_write(dev_priv,
-				       MIPI_MAX_RETURN_PKT_SIZE(port), 8 * 4);
+			intel_de_write(display,
+				       MIPI_MAX_RETURN_PKT_SIZE(display, port), 8 * 4);
 		intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_TEAR_ON);
 		intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_DISPLAY_ON);
 	} else {
@@ -871,11 +872,12 @@ static void intel_dsi_post_disable(struct intel_atomic_state *state,
 				   const struct intel_crtc_state *old_crtc_state,
 				   const struct drm_connector_state *old_conn_state)
 {
+	struct intel_display *display = to_intel_display(encoder);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 
-	drm_dbg_kms(&dev_priv->drm, "\n");
+	drm_dbg_kms(display->drm, "\n");
 
 	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
 		intel_crtc_vblank_off(old_crtc_state);
@@ -906,12 +908,12 @@ static void intel_dsi_post_disable(struct intel_atomic_state *state,
 
 	if (IS_BROXTON(dev_priv)) {
 		/* Power down DSI regulator to save power */
-		intel_de_write(dev_priv, BXT_P_DSI_REGULATOR_CFG, STAP_SELECT);
-		intel_de_write(dev_priv, BXT_P_DSI_REGULATOR_TX_CTRL,
+		intel_de_write(display, BXT_P_DSI_REGULATOR_CFG, STAP_SELECT);
+		intel_de_write(display, BXT_P_DSI_REGULATOR_TX_CTRL,
 			       HS_IO_CTRL_SELECT);
 
 		/* Add MIPI IO reset programming for modeset */
-		intel_de_rmw(dev_priv, BXT_P_CR_GT_DISP_PWRON, MIPIO_RST_CTRL, 0);
+		intel_de_rmw(display, BXT_P_CR_GT_DISP_PWRON, MIPIO_RST_CTRL, 0);
 	}
 
 	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
@@ -919,7 +921,7 @@ static void intel_dsi_post_disable(struct intel_atomic_state *state,
 	} else {
 		vlv_dsi_pll_disable(encoder);
 
-		intel_de_rmw(dev_priv, DSPCLK_GATE_D(dev_priv),
+		intel_de_rmw(display, DSPCLK_GATE_D(dev_priv),
 			     DPOUNIT_CLOCK_GATE_DISABLE, 0);
 	}
 
@@ -935,13 +937,14 @@ static void intel_dsi_post_disable(struct intel_atomic_state *state,
 static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 				   enum pipe *pipe)
 {
+	struct intel_display *display = to_intel_display(encoder);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	intel_wakeref_t wakeref;
 	enum port port;
 	bool active = false;
 
-	drm_dbg_kms(&dev_priv->drm, "\n");
+	drm_dbg_kms(display->drm, "\n");
 
 	wakeref = intel_display_power_get_if_enabled(dev_priv,
 						     encoder->power_domain);
@@ -960,7 +963,7 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 	/* XXX: this only works for one DSI output */
 	for_each_dsi_port(port, intel_dsi->ports) {
 		i915_reg_t port_ctrl = port_ctrl_reg(dev_priv, port);
-		bool enabled = intel_de_read(dev_priv, port_ctrl) & DPI_ENABLE;
+		bool enabled = intel_de_read(display, port_ctrl) & DPI_ENABLE;
 
 		/*
 		 * Due to some hardware limitations on VLV/CHV, the DPI enable
@@ -969,27 +972,27 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 		 */
 		if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 		    port == PORT_C)
-			enabled = intel_de_read(dev_priv, TRANSCONF(PIPE_B)) & TRANSCONF_ENABLE;
+			enabled = intel_de_read(display, TRANSCONF(PIPE_B)) & TRANSCONF_ENABLE;
 
 		/* Try command mode if video mode not enabled */
 		if (!enabled) {
-			u32 tmp = intel_de_read(dev_priv,
-						MIPI_DSI_FUNC_PRG(port));
+			u32 tmp = intel_de_read(display,
+						MIPI_DSI_FUNC_PRG(display, port));
 			enabled = tmp & CMD_MODE_DATA_WIDTH_MASK;
 		}
 
 		if (!enabled)
 			continue;
 
-		if (!(intel_de_read(dev_priv, MIPI_DEVICE_READY(port)) & DEVICE_READY))
+		if (!(intel_de_read(display, MIPI_DEVICE_READY(display, port)) & DEVICE_READY))
 			continue;
 
 		if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
-			u32 tmp = intel_de_read(dev_priv, MIPI_CTRL(port));
+			u32 tmp = intel_de_read(display, MIPI_CTRL(display, port));
 			tmp &= BXT_PIPE_SELECT_MASK;
 			tmp >>= BXT_PIPE_SELECT_SHIFT;
 
-			if (drm_WARN_ON(&dev_priv->drm, tmp > PIPE_C))
+			if (drm_WARN_ON(display->drm, tmp > PIPE_C))
 				continue;
 
 			*pipe = tmp;
@@ -1010,8 +1013,7 @@ out_put_power:
 static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
 				    struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_display *display = to_intel_display(encoder);
 	struct drm_display_mode *adjusted_mode =
 					&pipe_config->hw.adjusted_mode;
 	struct drm_display_mode *adjusted_mode_sw;
@@ -1033,11 +1035,11 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
 	 * encoder->get_hw_state() returns true.
 	 */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_de_read(dev_priv, BXT_MIPI_PORT_CTRL(port)) & DPI_ENABLE)
+		if (intel_de_read(display, BXT_MIPI_PORT_CTRL(port)) & DPI_ENABLE)
 			break;
 	}
 
-	fmt = intel_de_read(dev_priv, MIPI_DSI_FUNC_PRG(port)) & VID_MODE_FORMAT_MASK;
+	fmt = intel_de_read(display, MIPI_DSI_FUNC_PRG(display, port)) & VID_MODE_FORMAT_MASK;
 	bpp = mipi_dsi_pixel_format_to_bpp(
 			pixel_format_from_register_bits(fmt));
 
@@ -1049,24 +1051,24 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
 
 	/* In terms of pixels */
 	adjusted_mode->crtc_hdisplay =
-				intel_de_read(dev_priv,
+				intel_de_read(display,
 				              BXT_MIPI_TRANS_HACTIVE(port));
 	adjusted_mode->crtc_vdisplay =
-				intel_de_read(dev_priv,
+				intel_de_read(display,
 				              BXT_MIPI_TRANS_VACTIVE(port));
 	adjusted_mode->crtc_vtotal =
-				intel_de_read(dev_priv,
+				intel_de_read(display,
 				              BXT_MIPI_TRANS_VTOTAL(port));
 
 	hactive = adjusted_mode->crtc_hdisplay;
-	hfp = intel_de_read(dev_priv, MIPI_HFP_COUNT(port));
+	hfp = intel_de_read(display, MIPI_HFP_COUNT(display, port));
 
 	/*
 	 * Meaningful for video mode non-burst sync pulse mode only,
 	 * can be zero for non-burst sync events and burst modes
 	 */
-	hsync = intel_de_read(dev_priv, MIPI_HSYNC_PADDING_COUNT(port));
-	hbp = intel_de_read(dev_priv, MIPI_HBP_COUNT(port));
+	hsync = intel_de_read(display, MIPI_HSYNC_PADDING_COUNT(display, port));
+	hbp = intel_de_read(display, MIPI_HBP_COUNT(display, port));
 
 	/* harizontal values are in terms of high speed byte clock */
 	hfp = pixels_from_txbyteclkhs(hfp, bpp, lane_count,
@@ -1083,8 +1085,8 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
 	}
 
 	/* vertical values are in terms of lines */
-	vfp = intel_de_read(dev_priv, MIPI_VFP_COUNT(port));
-	vsync = intel_de_read(dev_priv, MIPI_VSYNC_PADDING_COUNT(port));
+	vfp = intel_de_read(display, MIPI_VFP_COUNT(display, port));
+	vsync = intel_de_read(display, MIPI_VSYNC_PADDING_COUNT(display, port));
 
 	adjusted_mode->crtc_htotal = hactive + hfp + hsync + hbp;
 	adjusted_mode->crtc_hsync_start = hfp + adjusted_mode->crtc_hdisplay;
@@ -1210,12 +1212,12 @@ static u16 txclkesc(u32 divider, unsigned int us)
 	}
 }
 
-static void set_dsi_timings(struct drm_encoder *encoder,
+static void set_dsi_timings(struct intel_encoder *encoder,
 			    const struct drm_display_mode *adjusted_mode)
 {
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_dsi *intel_dsi = enc_to_intel_dsi(to_intel_encoder(encoder));
+	struct intel_display *display = to_intel_display(encoder);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
 	unsigned int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
 	unsigned int lane_count = intel_dsi->lane_count;
@@ -1256,29 +1258,29 @@ static void set_dsi_timings(struct drm_encoder *encoder,
 			 * vactive, as they are calculated per channel basis,
 			 * whereas these values should be based on resolution.
 			 */
-			intel_de_write(dev_priv, BXT_MIPI_TRANS_HACTIVE(port),
+			intel_de_write(display, BXT_MIPI_TRANS_HACTIVE(port),
 				       adjusted_mode->crtc_hdisplay);
-			intel_de_write(dev_priv, BXT_MIPI_TRANS_VACTIVE(port),
+			intel_de_write(display, BXT_MIPI_TRANS_VACTIVE(port),
 				       adjusted_mode->crtc_vdisplay);
-			intel_de_write(dev_priv, BXT_MIPI_TRANS_VTOTAL(port),
+			intel_de_write(display, BXT_MIPI_TRANS_VTOTAL(port),
 				       adjusted_mode->crtc_vtotal);
 		}
 
-		intel_de_write(dev_priv, MIPI_HACTIVE_AREA_COUNT(port),
+		intel_de_write(display, MIPI_HACTIVE_AREA_COUNT(display, port),
 			       hactive);
-		intel_de_write(dev_priv, MIPI_HFP_COUNT(port), hfp);
+		intel_de_write(display, MIPI_HFP_COUNT(display, port), hfp);
 
 		/* meaningful for video mode non-burst sync pulse mode only,
 		 * can be zero for non-burst sync events and burst modes */
-		intel_de_write(dev_priv, MIPI_HSYNC_PADDING_COUNT(port),
+		intel_de_write(display, MIPI_HSYNC_PADDING_COUNT(display, port),
 			       hsync);
-		intel_de_write(dev_priv, MIPI_HBP_COUNT(port), hbp);
+		intel_de_write(display, MIPI_HBP_COUNT(display, port), hbp);
 
 		/* vertical values are in terms of lines */
-		intel_de_write(dev_priv, MIPI_VFP_COUNT(port), vfp);
-		intel_de_write(dev_priv, MIPI_VSYNC_PADDING_COUNT(port),
+		intel_de_write(display, MIPI_VFP_COUNT(display, port), vfp);
+		intel_de_write(display, MIPI_VSYNC_PADDING_COUNT(display, port),
 			       vsync);
-		intel_de_write(dev_priv, MIPI_VBP_COUNT(port), vbp);
+		intel_de_write(display, MIPI_VBP_COUNT(display, port), vbp);
 	}
 }
 
@@ -1299,21 +1301,20 @@ static u32 pixel_format_to_reg(enum mipi_dsi_pixel_format fmt)
 	}
 }
 
-static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
+static void intel_dsi_prepare(struct intel_encoder *encoder,
 			      const struct intel_crtc_state *pipe_config)
 {
-	struct drm_encoder *encoder = &intel_encoder->base;
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_display *display = to_intel_display(encoder);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
-	struct intel_dsi *intel_dsi = enc_to_intel_dsi(to_intel_encoder(encoder));
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
 	enum port port;
 	unsigned int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
 	u32 val, tmp;
 	u16 mode_hdisplay;
 
-	drm_dbg_kms(&dev_priv->drm, "pipe %c\n", pipe_name(crtc->pipe));
+	drm_dbg_kms(display->drm, "pipe %c\n", pipe_name(crtc->pipe));
 
 	mode_hdisplay = adjusted_mode->crtc_hdisplay;
 
@@ -1329,31 +1330,31 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 			 * escape clock divider, 20MHz, shared for A and C.
 			 * device ready must be off when doing this! txclkesc?
 			 */
-			tmp = intel_de_read(dev_priv, MIPI_CTRL(PORT_A));
+			tmp = intel_de_read(display, MIPI_CTRL(display, PORT_A));
 			tmp &= ~ESCAPE_CLOCK_DIVIDER_MASK;
-			intel_de_write(dev_priv, MIPI_CTRL(PORT_A),
+			intel_de_write(display, MIPI_CTRL(display, PORT_A),
 				       tmp | ESCAPE_CLOCK_DIVIDER_1);
 
 			/* read request priority is per pipe */
-			tmp = intel_de_read(dev_priv, MIPI_CTRL(port));
+			tmp = intel_de_read(display, MIPI_CTRL(display, port));
 			tmp &= ~READ_REQUEST_PRIORITY_MASK;
-			intel_de_write(dev_priv, MIPI_CTRL(port),
+			intel_de_write(display, MIPI_CTRL(display, port),
 				       tmp | READ_REQUEST_PRIORITY_HIGH);
 		} else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
 			enum pipe pipe = crtc->pipe;
 
-			intel_de_rmw(dev_priv, MIPI_CTRL(port),
+			intel_de_rmw(display, MIPI_CTRL(display, port),
 				     BXT_PIPE_SELECT_MASK, BXT_PIPE_SELECT(pipe));
 		}
 
 		/* XXX: why here, why like this? handling in irq handler?! */
-		intel_de_write(dev_priv, MIPI_INTR_STAT(port), 0xffffffff);
-		intel_de_write(dev_priv, MIPI_INTR_EN(port), 0xffffffff);
+		intel_de_write(display, MIPI_INTR_STAT(display, port), 0xffffffff);
+		intel_de_write(display, MIPI_INTR_EN(display, port), 0xffffffff);
 
-		intel_de_write(dev_priv, MIPI_DPHY_PARAM(port),
+		intel_de_write(display, MIPI_DPHY_PARAM(display, port),
 			       intel_dsi->dphy_reg);
 
-		intel_de_write(dev_priv, MIPI_DPI_RESOLUTION(port),
+		intel_de_write(display, MIPI_DPI_RESOLUTION(display, port),
 			       adjusted_mode->crtc_vdisplay << VERTICAL_ADDRESS_SHIFT | mode_hdisplay << HORIZONTAL_ADDRESS_SHIFT);
 	}
 
@@ -1381,7 +1382,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 	}
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		intel_de_write(dev_priv, MIPI_DSI_FUNC_PRG(port), val);
+		intel_de_write(display, MIPI_DSI_FUNC_PRG(display, port), val);
 
 		/* timeouts for recovery. one frame IIUC. if counter expires,
 		 * EOT and stop state. */
@@ -1402,23 +1403,23 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 
 		if (is_vid_mode(intel_dsi) &&
 			intel_dsi->video_mode == BURST_MODE) {
-			intel_de_write(dev_priv, MIPI_HS_TX_TIMEOUT(port),
+			intel_de_write(display, MIPI_HS_TX_TIMEOUT(display, port),
 				       txbyteclkhs(adjusted_mode->crtc_htotal, bpp, intel_dsi->lane_count, intel_dsi->burst_mode_ratio) + 1);
 		} else {
-			intel_de_write(dev_priv, MIPI_HS_TX_TIMEOUT(port),
+			intel_de_write(display, MIPI_HS_TX_TIMEOUT(display, port),
 				       txbyteclkhs(adjusted_mode->crtc_vtotal * adjusted_mode->crtc_htotal, bpp, intel_dsi->lane_count, intel_dsi->burst_mode_ratio) + 1);
 		}
-		intel_de_write(dev_priv, MIPI_LP_RX_TIMEOUT(port),
+		intel_de_write(display, MIPI_LP_RX_TIMEOUT(display, port),
 			       intel_dsi->lp_rx_timeout);
-		intel_de_write(dev_priv, MIPI_TURN_AROUND_TIMEOUT(port),
+		intel_de_write(display, MIPI_TURN_AROUND_TIMEOUT(display, port),
 			       intel_dsi->turn_arnd_val);
-		intel_de_write(dev_priv, MIPI_DEVICE_RESET_TIMER(port),
+		intel_de_write(display, MIPI_DEVICE_RESET_TIMER(display, port),
 			       intel_dsi->rst_timer_val);
 
 		/* dphy stuff */
 
 		/* in terms of low power clock */
-		intel_de_write(dev_priv, MIPI_INIT_COUNT(port),
+		intel_de_write(display, MIPI_INIT_COUNT(display, port),
 			       txclkesc(intel_dsi->escape_clk_div, 100));
 
 		if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
@@ -1429,16 +1430,16 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 			 * getting used. So write the other port
 			 * if not in dual link mode.
 			 */
-			intel_de_write(dev_priv,
-				       MIPI_INIT_COUNT(port == PORT_A ? PORT_C : PORT_A),
+			intel_de_write(display,
+				       MIPI_INIT_COUNT(display, port == PORT_A ? PORT_C : PORT_A),
 				       intel_dsi->init_count);
 		}
 
 		/* recovery disables */
-		intel_de_write(dev_priv, MIPI_EOT_DISABLE(port), tmp);
+		intel_de_write(display, MIPI_EOT_DISABLE(display, port), tmp);
 
 		/* in terms of low power clock */
-		intel_de_write(dev_priv, MIPI_INIT_COUNT(port),
+		intel_de_write(display, MIPI_INIT_COUNT(display, port),
 			       intel_dsi->init_count);
 
 		/* in terms of txbyteclkhs. actual high to low switch +
@@ -1446,7 +1447,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 		 *
 		 * XXX: write MIPI_STOP_STATE_STALL?
 		 */
-		intel_de_write(dev_priv, MIPI_HIGH_LOW_SWITCH_COUNT(port),
+		intel_de_write(display, MIPI_HIGH_LOW_SWITCH_COUNT(display, port),
 			       intel_dsi->hs_to_lp_count);
 
 		/* XXX: low power clock equivalence in terms of byte clock.
@@ -1455,14 +1456,14 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 		 * txclkesc time / txbyteclk time * (105 + MIPI_STOP_STATE_STALL
 		 * ) / 105.???
 		 */
-		intel_de_write(dev_priv, MIPI_LP_BYTECLK(port),
+		intel_de_write(display, MIPI_LP_BYTECLK(display, port),
 			       intel_dsi->lp_byte_clk);
 
 		if (IS_GEMINILAKE(dev_priv)) {
-			intel_de_write(dev_priv, MIPI_TLPX_TIME_COUNT(port),
+			intel_de_write(display, MIPI_TLPX_TIME_COUNT(display, port),
 				       intel_dsi->lp_byte_clk);
 			/* Shadow of DPHY reg */
-			intel_de_write(dev_priv, MIPI_CLK_LANE_TIMING(port),
+			intel_de_write(display, MIPI_CLK_LANE_TIMING(display, port),
 				       intel_dsi->dphy_reg);
 		}
 
@@ -1471,10 +1472,10 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 		 * this register in terms of byte clocks. based on dsi transfer
 		 * rate and the number of lanes configured the time taken to
 		 * transmit 16 long packets in a dsi stream varies. */
-		intel_de_write(dev_priv, MIPI_DBI_BW_CTRL(port),
+		intel_de_write(display, MIPI_DBI_BW_CTRL(display, port),
 			       intel_dsi->bw_timer);
 
-		intel_de_write(dev_priv, MIPI_CLK_LANE_SWITCH_TIME_CNT(port),
+		intel_de_write(display, MIPI_CLK_LANE_SWITCH_TIME_CNT(display, port),
 			       intel_dsi->clk_lp_to_hs_count << LP_HS_SSW_CNT_SHIFT | intel_dsi->clk_hs_to_lp_count << HS_LP_PWR_SW_CNT_SHIFT);
 
 		if (is_vid_mode(intel_dsi)) {
@@ -1502,13 +1503,14 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 				break;
 			}
 
-			intel_de_write(dev_priv, MIPI_VIDEO_MODE_FORMAT(port), fmt);
+			intel_de_write(display, MIPI_VIDEO_MODE_FORMAT(display, port), fmt);
 		}
 	}
 }
 
 static void intel_dsi_unprepare(struct intel_encoder *encoder)
 {
+	struct intel_display *display = to_intel_display(encoder);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum port port;
@@ -1518,17 +1520,17 @@ static void intel_dsi_unprepare(struct intel_encoder *encoder)
 
 	for_each_dsi_port(port, intel_dsi->ports) {
 		/* Panel commands can be sent when clock is in LP11 */
-		intel_de_write(dev_priv, MIPI_DEVICE_READY(port), 0x0);
+		intel_de_write(display, MIPI_DEVICE_READY(display, port), 0x0);
 
 		if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
 			bxt_dsi_reset_clocks(encoder, port);
 		else
 			vlv_dsi_reset_clocks(encoder, port);
-		intel_de_write(dev_priv, MIPI_EOT_DISABLE(port), CLOCKSTOP);
+		intel_de_write(display, MIPI_EOT_DISABLE(display, port), CLOCKSTOP);
 
-		intel_de_rmw(dev_priv, MIPI_DSI_FUNC_PRG(port), VID_MODE_FORMAT_MASK, 0);
+		intel_de_rmw(display, MIPI_DSI_FUNC_PRG(display, port), VID_MODE_FORMAT_MASK, 0);
 
-		intel_de_write(dev_priv, MIPI_DEVICE_READY(port), 0x1);
+		intel_de_write(display, MIPI_DEVICE_READY(display, port), 0x1);
 	}
 }
 
@@ -1592,8 +1594,7 @@ static void vlv_dsi_add_properties(struct intel_connector *connector)
 
 static void vlv_dphy_param_init(struct intel_dsi *intel_dsi)
 {
-	struct drm_device *dev = intel_dsi->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dsi->base.base.dev);
 	struct intel_connector *connector = intel_dsi->attached_connector;
 	struct mipi_config *mipi_config = connector->panel.vbt.dsi.config;
 	u32 tlpx_ns, extra_byte_count, tlpx_ui;
@@ -1879,10 +1880,8 @@ static const struct dmi_system_id vlv_dsi_dmi_quirk_table[] = {
 void vlv_dsi_init(struct drm_i915_private *dev_priv)
 {
 	struct intel_dsi *intel_dsi;
-	struct intel_encoder *intel_encoder;
-	struct drm_encoder *encoder;
-	struct intel_connector *intel_connector;
-	struct drm_connector *connector;
+	struct intel_encoder *encoder;
+	struct intel_connector *connector;
 	struct drm_display_mode *current_mode;
 	const struct dmi_system_id *dmi_id;
 	enum port port;
@@ -1903,64 +1902,61 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv)
 	if (!intel_dsi)
 		return;
 
-	intel_connector = intel_connector_alloc();
-	if (!intel_connector) {
+	connector = intel_connector_alloc();
+	if (!connector) {
 		kfree(intel_dsi);
 		return;
 	}
 
-	intel_encoder = &intel_dsi->base;
-	encoder = &intel_encoder->base;
-	intel_dsi->attached_connector = intel_connector;
-
-	connector = &intel_connector->base;
+	encoder = &intel_dsi->base;
+	intel_dsi->attached_connector = connector;
 
-	drm_encoder_init(&dev_priv->drm, encoder, &intel_dsi_funcs, DRM_MODE_ENCODER_DSI,
-			 "DSI %c", port_name(port));
+	drm_encoder_init(&dev_priv->drm, &encoder->base, &intel_dsi_funcs,
+			 DRM_MODE_ENCODER_DSI, "DSI %c", port_name(port));
 
-	intel_encoder->compute_config = intel_dsi_compute_config;
-	intel_encoder->pre_enable = intel_dsi_pre_enable;
+	encoder->compute_config = intel_dsi_compute_config;
+	encoder->pre_enable = intel_dsi_pre_enable;
 	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
-		intel_encoder->enable = bxt_dsi_enable;
-	intel_encoder->disable = intel_dsi_disable;
-	intel_encoder->post_disable = intel_dsi_post_disable;
-	intel_encoder->get_hw_state = intel_dsi_get_hw_state;
-	intel_encoder->get_config = intel_dsi_get_config;
-	intel_encoder->update_pipe = intel_backlight_update;
-	intel_encoder->shutdown = intel_dsi_shutdown;
+		encoder->enable = bxt_dsi_enable;
+	encoder->disable = intel_dsi_disable;
+	encoder->post_disable = intel_dsi_post_disable;
+	encoder->get_hw_state = intel_dsi_get_hw_state;
+	encoder->get_config = intel_dsi_get_config;
+	encoder->update_pipe = intel_backlight_update;
+	encoder->shutdown = intel_dsi_shutdown;
 
-	intel_connector->get_hw_state = intel_connector_get_hw_state;
+	connector->get_hw_state = intel_connector_get_hw_state;
 
-	intel_encoder->port = port;
-	intel_encoder->type = INTEL_OUTPUT_DSI;
-	intel_encoder->power_domain = POWER_DOMAIN_PORT_DSI;
-	intel_encoder->cloneable = 0;
+	encoder->port = port;
+	encoder->type = INTEL_OUTPUT_DSI;
+	encoder->power_domain = POWER_DOMAIN_PORT_DSI;
+	encoder->cloneable = 0;
 
 	/*
 	 * On BYT/CHV, pipe A maps to MIPI DSI port A, pipe B maps to MIPI DSI
 	 * port C. BXT isn't limited like this.
 	 */
 	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
-		intel_encoder->pipe_mask = ~0;
+		encoder->pipe_mask = ~0;
 	else if (port == PORT_A)
-		intel_encoder->pipe_mask = BIT(PIPE_A);
+		encoder->pipe_mask = BIT(PIPE_A);
 	else
-		intel_encoder->pipe_mask = BIT(PIPE_B);
+		encoder->pipe_mask = BIT(PIPE_B);
 
 	intel_dsi->panel_power_off_time = ktime_get_boottime();
 
-	intel_bios_init_panel_late(dev_priv, &intel_connector->panel, NULL, NULL);
+	intel_bios_init_panel_late(dev_priv, &connector->panel, NULL, NULL);
 
-	if (intel_connector->panel.vbt.dsi.config->dual_link)
+	if (connector->panel.vbt.dsi.config->dual_link)
 		intel_dsi->ports = BIT(PORT_A) | BIT(PORT_C);
 	else
 		intel_dsi->ports = BIT(port);
 
-	if (drm_WARN_ON(&dev_priv->drm, intel_connector->panel.vbt.dsi.bl_ports & ~intel_dsi->ports))
-		intel_connector->panel.vbt.dsi.bl_ports &= intel_dsi->ports;
+	if (drm_WARN_ON(&dev_priv->drm, connector->panel.vbt.dsi.bl_ports & ~intel_dsi->ports))
+		connector->panel.vbt.dsi.bl_ports &= intel_dsi->ports;
 
-	if (drm_WARN_ON(&dev_priv->drm, intel_connector->panel.vbt.dsi.cabc_ports & ~intel_dsi->ports))
-		intel_connector->panel.vbt.dsi.cabc_ports &= intel_dsi->ports;
+	if (drm_WARN_ON(&dev_priv->drm, connector->panel.vbt.dsi.cabc_ports & ~intel_dsi->ports))
+		connector->panel.vbt.dsi.cabc_ports &= intel_dsi->ports;
 
 	/* Create a DSI host (and a device) for each port. */
 	for_each_dsi_port(port, intel_dsi->ports) {
@@ -1980,7 +1976,7 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv)
 	}
 
 	/* Use clock read-back from current hw-state for fastboot */
-	current_mode = intel_encoder_current_mode(intel_encoder);
+	current_mode = intel_encoder_current_mode(encoder);
 	if (current_mode) {
 		drm_dbg_kms(&dev_priv->drm, "Calculated pclk %d GOP %d\n",
 			    intel_dsi->pclk, current_mode->clock);
@@ -1996,22 +1992,22 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv)
 	vlv_dphy_param_init(intel_dsi);
 
 	intel_dsi_vbt_gpio_init(intel_dsi,
-				intel_dsi_get_hw_state(intel_encoder, &pipe));
+				intel_dsi_get_hw_state(encoder, &pipe));
 
-	drm_connector_init(&dev_priv->drm, connector, &intel_dsi_connector_funcs,
+	drm_connector_init(&dev_priv->drm, &connector->base, &intel_dsi_connector_funcs,
 			   DRM_MODE_CONNECTOR_DSI);
 
-	drm_connector_helper_add(connector, &intel_dsi_connector_helper_funcs);
+	drm_connector_helper_add(&connector->base, &intel_dsi_connector_helper_funcs);
 
-	connector->display_info.subpixel_order = SubPixelHorizontalRGB; /*XXX*/
+	connector->base.display_info.subpixel_order = SubPixelHorizontalRGB; /*XXX*/
 
-	intel_connector_attach_encoder(intel_connector, intel_encoder);
+	intel_connector_attach_encoder(connector, encoder);
 
 	mutex_lock(&dev_priv->drm.mode_config.mutex);
-	intel_panel_add_vbt_lfp_fixed_mode(intel_connector);
+	intel_panel_add_vbt_lfp_fixed_mode(connector);
 	mutex_unlock(&dev_priv->drm.mode_config.mutex);
 
-	if (!intel_panel_preferred_fixed_mode(intel_connector)) {
+	if (!intel_panel_preferred_fixed_mode(connector)) {
 		drm_dbg_kms(&dev_priv->drm, "no fixed mode\n");
 		goto err_cleanup_connector;
 	}
@@ -2024,18 +2020,18 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv)
 		quirk_func(intel_dsi);
 	}
 
-	intel_panel_init(intel_connector, NULL);
+	intel_panel_init(connector, NULL);
 
-	intel_backlight_setup(intel_connector, INVALID_PIPE);
+	intel_backlight_setup(connector, INVALID_PIPE);
 
-	vlv_dsi_add_properties(intel_connector);
+	vlv_dsi_add_properties(connector);
 
 	return;
 
 err_cleanup_connector:
-	drm_connector_cleanup(&intel_connector->base);
+	drm_connector_cleanup(&connector->base);
 err:
-	drm_encoder_cleanup(&intel_encoder->base);
+	drm_encoder_cleanup(&encoder->base);
 	kfree(intel_dsi);
-	kfree(intel_connector);
+	kfree(connector);
 }
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c
index ae0a0b11bae3..70c5a13a3c75 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c
+++ b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c
@@ -365,13 +365,13 @@ u32 bxt_dsi_get_pclk(struct intel_encoder *encoder,
 
 void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 {
-	u32 temp;
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
+	u32 temp;
 
-	temp = intel_de_read(dev_priv, MIPI_CTRL(port));
+	temp = intel_de_read(display, MIPI_CTRL(display, port));
 	temp &= ~ESCAPE_CLOCK_DIVIDER_MASK;
-	intel_de_write(dev_priv, MIPI_CTRL(port),
+	intel_de_write(display, MIPI_CTRL(display, port),
 		       temp | intel_dsi->escape_clk_div << ESCAPE_CLOCK_DIVIDER_SHIFT);
 }
 
@@ -570,24 +570,24 @@ void bxt_dsi_pll_enable(struct intel_encoder *encoder,
 
 void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 {
+	struct intel_display *display = to_intel_display(encoder);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	u32 tmp;
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
 
 	/* Clear old configurations */
 	if (IS_BROXTON(dev_priv)) {
-		tmp = intel_de_read(dev_priv, BXT_MIPI_CLOCK_CTL);
+		tmp = intel_de_read(display, BXT_MIPI_CLOCK_CTL);
 		tmp &= ~(BXT_MIPI_TX_ESCLK_FIXDIV_MASK(port));
 		tmp &= ~(BXT_MIPI_RX_ESCLK_UPPER_FIXDIV_MASK(port));
 		tmp &= ~(BXT_MIPI_8X_BY3_DIVIDER_MASK(port));
 		tmp &= ~(BXT_MIPI_RX_ESCLK_LOWER_FIXDIV_MASK(port));
-		intel_de_write(dev_priv, BXT_MIPI_CLOCK_CTL, tmp);
+		intel_de_write(display, BXT_MIPI_CLOCK_CTL, tmp);
 	} else {
-		intel_de_rmw(dev_priv, MIPIO_TXESC_CLK_DIV1, GLK_TX_ESC_CLK_DIV1_MASK, 0);
+		intel_de_rmw(display, MIPIO_TXESC_CLK_DIV1, GLK_TX_ESC_CLK_DIV1_MASK, 0);
 
-		intel_de_rmw(dev_priv, MIPIO_TXESC_CLK_DIV2, GLK_TX_ESC_CLK_DIV2_MASK, 0);
+		intel_de_rmw(display, MIPIO_TXESC_CLK_DIV2, GLK_TX_ESC_CLK_DIV2_MASK, 0);
 	}
-	intel_de_write(dev_priv, MIPI_EOT_DISABLE(port), CLOCKSTOP);
+	intel_de_write(display, MIPI_EOT_DISABLE(display, port), CLOCKSTOP);
 }
 
 static void assert_dsi_pll(struct drm_i915_private *i915, bool state)
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_regs.h b/drivers/gpu/drm/i915/display/vlv_dsi_regs.h
index abbe427e462e..c1126d170ec6 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi_regs.h
+++ b/drivers/gpu/drm/i915/display/vlv_dsi_regs.h
@@ -11,26 +11,23 @@
 #define VLV_MIPI_BASE			VLV_DISPLAY_BASE
 #define BXT_MIPI_BASE			0x60000
 
-#define _MIPI_MMIO_BASE(__i915) ((__i915)->display.dsi.mmio_base)
+#define _MIPI_MMIO_BASE(display)	((display)->dsi.mmio_base)
 
 #define _MIPI_PORT(port, a, c)	(((port) == PORT_A) ? a : c)	/* ports A and C only */
-#define _MMIO_MIPI(port, a, c)	_MMIO(_MIPI_PORT(port, a, c))
+#define _MMIO_MIPI(base, port, a, c)	_MMIO((base) + _MIPI_PORT(port, a, c))
 
 /* BXT MIPI mode configure */
-#define  _BXT_MIPIA_TRANS_HACTIVE			0x6B0F8
-#define  _BXT_MIPIC_TRANS_HACTIVE			0x6B8F8
-#define  BXT_MIPI_TRANS_HACTIVE(tc)	_MMIO_MIPI(tc, \
-		_BXT_MIPIA_TRANS_HACTIVE, _BXT_MIPIC_TRANS_HACTIVE)
+#define  _BXT_MIPIA_TRANS_HACTIVE		0xb0f8
+#define  _BXT_MIPIC_TRANS_HACTIVE		0xb8f8
+#define  BXT_MIPI_TRANS_HACTIVE(tc)		_MMIO_MIPI(BXT_MIPI_BASE, tc, _BXT_MIPIA_TRANS_HACTIVE, _BXT_MIPIC_TRANS_HACTIVE)
 
-#define  _BXT_MIPIA_TRANS_VACTIVE			0x6B0FC
-#define  _BXT_MIPIC_TRANS_VACTIVE			0x6B8FC
-#define  BXT_MIPI_TRANS_VACTIVE(tc)	_MMIO_MIPI(tc, \
-		_BXT_MIPIA_TRANS_VACTIVE, _BXT_MIPIC_TRANS_VACTIVE)
+#define  _BXT_MIPIA_TRANS_VACTIVE		0xb0fc
+#define  _BXT_MIPIC_TRANS_VACTIVE		0xb8fc
+#define  BXT_MIPI_TRANS_VACTIVE(tc)		_MMIO_MIPI(BXT_MIPI_BASE, tc, _BXT_MIPIA_TRANS_VACTIVE, _BXT_MIPIC_TRANS_VACTIVE)
 
-#define  _BXT_MIPIA_TRANS_VTOTAL			0x6B100
-#define  _BXT_MIPIC_TRANS_VTOTAL			0x6B900
-#define  BXT_MIPI_TRANS_VTOTAL(tc)	_MMIO_MIPI(tc, \
-		_BXT_MIPIA_TRANS_VTOTAL, _BXT_MIPIC_TRANS_VTOTAL)
+#define  _BXT_MIPIA_TRANS_VTOTAL		0xb100
+#define  _BXT_MIPIC_TRANS_VTOTAL		0xb900
+#define  BXT_MIPI_TRANS_VTOTAL(tc)		_MMIO_MIPI(BXT_MIPI_BASE, tc, _BXT_MIPIA_TRANS_VTOTAL, _BXT_MIPIC_TRANS_VTOTAL)
 
 #define BXT_P_DSI_REGULATOR_CFG			_MMIO(0x160020)
 #define  STAP_SELECT					(1 << 0)
@@ -38,14 +35,14 @@
 #define BXT_P_DSI_REGULATOR_TX_CTRL		_MMIO(0x160054)
 #define  HS_IO_CTRL_SELECT				(1 << 0)
 
-#define _MIPIA_PORT_CTRL			(VLV_DISPLAY_BASE + 0x61190)
-#define _MIPIC_PORT_CTRL			(VLV_DISPLAY_BASE + 0x61700)
-#define MIPI_PORT_CTRL(port)	_MMIO_MIPI(port, _MIPIA_PORT_CTRL, _MIPIC_PORT_CTRL)
+#define _MIPIA_PORT_CTRL			0x61190
+#define _MIPIC_PORT_CTRL			0x61700
+#define VLV_MIPI_PORT_CTRL(port)		_MMIO_MIPI(VLV_MIPI_BASE, port, _MIPIA_PORT_CTRL, _MIPIC_PORT_CTRL)
 
  /* BXT port control */
-#define _BXT_MIPIA_PORT_CTRL				0x6B0C0
-#define _BXT_MIPIC_PORT_CTRL				0x6B8C0
-#define BXT_MIPI_PORT_CTRL(tc)	_MMIO_MIPI(tc, _BXT_MIPIA_PORT_CTRL, _BXT_MIPIC_PORT_CTRL)
+#define _BXT_MIPIA_PORT_CTRL			0xb0c0
+#define _BXT_MIPIC_PORT_CTRL			0xb8c0
+#define BXT_MIPI_PORT_CTRL(tc)			_MMIO_MIPI(BXT_MIPI_BASE, tc, _BXT_MIPIA_PORT_CTRL, _BXT_MIPIC_PORT_CTRL)
 
 #define  DPI_ENABLE					(1 << 31) /* A + C */
 #define  MIPIA_MIPI4DPHY_DELAY_COUNT_SHIFT		27
@@ -87,20 +84,17 @@
 #define  LANE_CONFIGURATION_DUAL_LINK_A			(1 << 0)
 #define  LANE_CONFIGURATION_DUAL_LINK_B			(2 << 0)
 
-#define _MIPIA_TEARING_CTRL			(VLV_DISPLAY_BASE + 0x61194)
-#define _MIPIC_TEARING_CTRL			(VLV_DISPLAY_BASE + 0x61704)
-#define MIPI_TEARING_CTRL(port)			_MMIO_MIPI(port, _MIPIA_TEARING_CTRL, _MIPIC_TEARING_CTRL)
+#define _MIPIA_TEARING_CTRL			0x61194
+#define _MIPIC_TEARING_CTRL			0x61704
+#define VLV_MIPI_TEARING_CTRL(port)			_MMIO_MIPI(VLV_MIPI_BASE, port, _MIPIA_TEARING_CTRL, _MIPIC_TEARING_CTRL)
 #define  TEARING_EFFECT_DELAY_SHIFT			0
 #define  TEARING_EFFECT_DELAY_MASK			(0xffff << 0)
 
-/* XXX: all bits reserved */
-#define _MIPIA_AUTOPWG			(VLV_DISPLAY_BASE + 0x611a0)
-
 /* MIPI DSI Controller and D-PHY registers */
 
-#define _MIPIA_DEVICE_READY		(_MIPI_MMIO_BASE(dev_priv) + 0xb000)
-#define _MIPIC_DEVICE_READY		(_MIPI_MMIO_BASE(dev_priv) + 0xb800)
-#define MIPI_DEVICE_READY(port)		_MMIO_MIPI(port, _MIPIA_DEVICE_READY, _MIPIC_DEVICE_READY)
+#define _MIPIA_DEVICE_READY			0xb000
+#define _MIPIC_DEVICE_READY			0xb800
+#define MIPI_DEVICE_READY(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_DEVICE_READY, _MIPIC_DEVICE_READY)
 #define  BUS_POSSESSION					(1 << 3) /* set to give bus to receiver */
 #define  ULPS_STATE_MASK				(3 << 1)
 #define  ULPS_STATE_ENTER				(2 << 1)
@@ -108,12 +102,12 @@
 #define  ULPS_STATE_NORMAL_OPERATION			(0 << 1)
 #define  DEVICE_READY					(1 << 0)
 
-#define _MIPIA_INTR_STAT		(_MIPI_MMIO_BASE(dev_priv) + 0xb004)
-#define _MIPIC_INTR_STAT		(_MIPI_MMIO_BASE(dev_priv) + 0xb804)
-#define MIPI_INTR_STAT(port)		_MMIO_MIPI(port, _MIPIA_INTR_STAT, _MIPIC_INTR_STAT)
-#define _MIPIA_INTR_EN			(_MIPI_MMIO_BASE(dev_priv) + 0xb008)
-#define _MIPIC_INTR_EN			(_MIPI_MMIO_BASE(dev_priv) + 0xb808)
-#define MIPI_INTR_EN(port)		_MMIO_MIPI(port, _MIPIA_INTR_EN, _MIPIC_INTR_EN)
+#define _MIPIA_INTR_STAT			0xb004
+#define _MIPIC_INTR_STAT			0xb804
+#define MIPI_INTR_STAT(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_INTR_STAT, _MIPIC_INTR_STAT)
+#define _MIPIA_INTR_EN				0xb008
+#define _MIPIC_INTR_EN				0xb808
+#define MIPI_INTR_EN(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_INTR_EN, _MIPIC_INTR_EN)
 #define  TEARING_EFFECT					(1 << 31)
 #define  SPL_PKT_SENT_INTERRUPT				(1 << 30)
 #define  GEN_READ_DATA_AVAIL				(1 << 29)
@@ -147,9 +141,9 @@
 #define  RXSOT_SYNC_ERROR				(1 << 1)
 #define  RXSOT_ERROR					(1 << 0)
 
-#define _MIPIA_DSI_FUNC_PRG		(_MIPI_MMIO_BASE(dev_priv) + 0xb00c)
-#define _MIPIC_DSI_FUNC_PRG		(_MIPI_MMIO_BASE(dev_priv) + 0xb80c)
-#define MIPI_DSI_FUNC_PRG(port)		_MMIO_MIPI(port, _MIPIA_DSI_FUNC_PRG, _MIPIC_DSI_FUNC_PRG)
+#define _MIPIA_DSI_FUNC_PRG			0xb00c
+#define _MIPIC_DSI_FUNC_PRG			0xb80c
+#define MIPI_DSI_FUNC_PRG(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_DSI_FUNC_PRG, _MIPIC_DSI_FUNC_PRG)
 #define  CMD_MODE_DATA_WIDTH_MASK			(7 << 13)
 #define  CMD_MODE_NOT_SUPPORTED				(0 << 13)
 #define  CMD_MODE_DATA_WIDTH_16_BIT			(1 << 13)
@@ -170,77 +164,77 @@
 #define  DATA_LANES_PRG_REG_SHIFT			0
 #define  DATA_LANES_PRG_REG_MASK			(7 << 0)
 
-#define _MIPIA_HS_TX_TIMEOUT		(_MIPI_MMIO_BASE(dev_priv) + 0xb010)
-#define _MIPIC_HS_TX_TIMEOUT		(_MIPI_MMIO_BASE(dev_priv) + 0xb810)
-#define MIPI_HS_TX_TIMEOUT(port)	_MMIO_MIPI(port, _MIPIA_HS_TX_TIMEOUT, _MIPIC_HS_TX_TIMEOUT)
+#define _MIPIA_HS_TX_TIMEOUT			0xb010
+#define _MIPIC_HS_TX_TIMEOUT			0xb810
+#define MIPI_HS_TX_TIMEOUT(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_HS_TX_TIMEOUT, _MIPIC_HS_TX_TIMEOUT)
 #define  HIGH_SPEED_TX_TIMEOUT_COUNTER_MASK		0xffffff
 
-#define _MIPIA_LP_RX_TIMEOUT		(_MIPI_MMIO_BASE(dev_priv) + 0xb014)
-#define _MIPIC_LP_RX_TIMEOUT		(_MIPI_MMIO_BASE(dev_priv) + 0xb814)
-#define MIPI_LP_RX_TIMEOUT(port)	_MMIO_MIPI(port, _MIPIA_LP_RX_TIMEOUT, _MIPIC_LP_RX_TIMEOUT)
+#define _MIPIA_LP_RX_TIMEOUT			0xb014
+#define _MIPIC_LP_RX_TIMEOUT			0xb814
+#define MIPI_LP_RX_TIMEOUT(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_LP_RX_TIMEOUT, _MIPIC_LP_RX_TIMEOUT)
 #define  LOW_POWER_RX_TIMEOUT_COUNTER_MASK		0xffffff
 
-#define _MIPIA_TURN_AROUND_TIMEOUT	(_MIPI_MMIO_BASE(dev_priv) + 0xb018)
-#define _MIPIC_TURN_AROUND_TIMEOUT	(_MIPI_MMIO_BASE(dev_priv) + 0xb818)
-#define MIPI_TURN_AROUND_TIMEOUT(port)	_MMIO_MIPI(port, _MIPIA_TURN_AROUND_TIMEOUT, _MIPIC_TURN_AROUND_TIMEOUT)
+#define _MIPIA_TURN_AROUND_TIMEOUT		0xb018
+#define _MIPIC_TURN_AROUND_TIMEOUT		0xb818
+#define MIPI_TURN_AROUND_TIMEOUT(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_TURN_AROUND_TIMEOUT, _MIPIC_TURN_AROUND_TIMEOUT)
 #define  TURN_AROUND_TIMEOUT_MASK			0x3f
 
-#define _MIPIA_DEVICE_RESET_TIMER	(_MIPI_MMIO_BASE(dev_priv) + 0xb01c)
-#define _MIPIC_DEVICE_RESET_TIMER	(_MIPI_MMIO_BASE(dev_priv) + 0xb81c)
-#define MIPI_DEVICE_RESET_TIMER(port)	_MMIO_MIPI(port, _MIPIA_DEVICE_RESET_TIMER, _MIPIC_DEVICE_RESET_TIMER)
+#define _MIPIA_DEVICE_RESET_TIMER		0xb01c
+#define _MIPIC_DEVICE_RESET_TIMER		0xb81c
+#define MIPI_DEVICE_RESET_TIMER(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_DEVICE_RESET_TIMER, _MIPIC_DEVICE_RESET_TIMER)
 #define  DEVICE_RESET_TIMER_MASK			0xffff
 
-#define _MIPIA_DPI_RESOLUTION		(_MIPI_MMIO_BASE(dev_priv) + 0xb020)
-#define _MIPIC_DPI_RESOLUTION		(_MIPI_MMIO_BASE(dev_priv) + 0xb820)
-#define MIPI_DPI_RESOLUTION(port)	_MMIO_MIPI(port, _MIPIA_DPI_RESOLUTION, _MIPIC_DPI_RESOLUTION)
+#define _MIPIA_DPI_RESOLUTION			0xb020
+#define _MIPIC_DPI_RESOLUTION			0xb820
+#define MIPI_DPI_RESOLUTION(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_DPI_RESOLUTION, _MIPIC_DPI_RESOLUTION)
 #define  VERTICAL_ADDRESS_SHIFT				16
 #define  VERTICAL_ADDRESS_MASK				(0xffff << 16)
 #define  HORIZONTAL_ADDRESS_SHIFT			0
 #define  HORIZONTAL_ADDRESS_MASK			0xffff
 
-#define _MIPIA_DBI_FIFO_THROTTLE	(_MIPI_MMIO_BASE(dev_priv) + 0xb024)
-#define _MIPIC_DBI_FIFO_THROTTLE	(_MIPI_MMIO_BASE(dev_priv) + 0xb824)
-#define MIPI_DBI_FIFO_THROTTLE(port)	_MMIO_MIPI(port, _MIPIA_DBI_FIFO_THROTTLE, _MIPIC_DBI_FIFO_THROTTLE)
+#define _MIPIA_DBI_FIFO_THROTTLE		0xb024
+#define _MIPIC_DBI_FIFO_THROTTLE		0xb824
+#define MIPI_DBI_FIFO_THROTTLE(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_DBI_FIFO_THROTTLE, _MIPIC_DBI_FIFO_THROTTLE)
 #define  DBI_FIFO_EMPTY_HALF				(0 << 0)
 #define  DBI_FIFO_EMPTY_QUARTER				(1 << 0)
 #define  DBI_FIFO_EMPTY_7_LOCATIONS			(2 << 0)
 
 /* regs below are bits 15:0 */
-#define _MIPIA_HSYNC_PADDING_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb028)
-#define _MIPIC_HSYNC_PADDING_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb828)
-#define MIPI_HSYNC_PADDING_COUNT(port)	_MMIO_MIPI(port, _MIPIA_HSYNC_PADDING_COUNT, _MIPIC_HSYNC_PADDING_COUNT)
+#define _MIPIA_HSYNC_PADDING_COUNT		0xb028
+#define _MIPIC_HSYNC_PADDING_COUNT		0xb828
+#define MIPI_HSYNC_PADDING_COUNT(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_HSYNC_PADDING_COUNT, _MIPIC_HSYNC_PADDING_COUNT)
 
-#define _MIPIA_HBP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb02c)
-#define _MIPIC_HBP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb82c)
-#define MIPI_HBP_COUNT(port)		_MMIO_MIPI(port, _MIPIA_HBP_COUNT, _MIPIC_HBP_COUNT)
+#define _MIPIA_HBP_COUNT			0xb02c
+#define _MIPIC_HBP_COUNT			0xb82c
+#define MIPI_HBP_COUNT(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_HBP_COUNT, _MIPIC_HBP_COUNT)
 
-#define _MIPIA_HFP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb030)
-#define _MIPIC_HFP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb830)
-#define MIPI_HFP_COUNT(port)		_MMIO_MIPI(port, _MIPIA_HFP_COUNT, _MIPIC_HFP_COUNT)
+#define _MIPIA_HFP_COUNT			0xb030
+#define _MIPIC_HFP_COUNT			0xb830
+#define MIPI_HFP_COUNT(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_HFP_COUNT, _MIPIC_HFP_COUNT)
 
-#define _MIPIA_HACTIVE_AREA_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb034)
-#define _MIPIC_HACTIVE_AREA_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb834)
-#define MIPI_HACTIVE_AREA_COUNT(port)	_MMIO_MIPI(port, _MIPIA_HACTIVE_AREA_COUNT, _MIPIC_HACTIVE_AREA_COUNT)
+#define _MIPIA_HACTIVE_AREA_COUNT		0xb034
+#define _MIPIC_HACTIVE_AREA_COUNT		0xb834
+#define MIPI_HACTIVE_AREA_COUNT(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_HACTIVE_AREA_COUNT, _MIPIC_HACTIVE_AREA_COUNT)
 
-#define _MIPIA_VSYNC_PADDING_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb038)
-#define _MIPIC_VSYNC_PADDING_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb838)
-#define MIPI_VSYNC_PADDING_COUNT(port)	_MMIO_MIPI(port, _MIPIA_VSYNC_PADDING_COUNT, _MIPIC_VSYNC_PADDING_COUNT)
+#define _MIPIA_VSYNC_PADDING_COUNT		0xb038
+#define _MIPIC_VSYNC_PADDING_COUNT		0xb838
+#define MIPI_VSYNC_PADDING_COUNT(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_VSYNC_PADDING_COUNT, _MIPIC_VSYNC_PADDING_COUNT)
 
-#define _MIPIA_VBP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb03c)
-#define _MIPIC_VBP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb83c)
-#define MIPI_VBP_COUNT(port)		_MMIO_MIPI(port, _MIPIA_VBP_COUNT, _MIPIC_VBP_COUNT)
+#define _MIPIA_VBP_COUNT			0xb03c
+#define _MIPIC_VBP_COUNT			0xb83c
+#define MIPI_VBP_COUNT(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_VBP_COUNT, _MIPIC_VBP_COUNT)
 
-#define _MIPIA_VFP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb040)
-#define _MIPIC_VFP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb840)
-#define MIPI_VFP_COUNT(port)		_MMIO_MIPI(port, _MIPIA_VFP_COUNT, _MIPIC_VFP_COUNT)
+#define _MIPIA_VFP_COUNT			0xb040
+#define _MIPIC_VFP_COUNT			0xb840
+#define MIPI_VFP_COUNT(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_VFP_COUNT, _MIPIC_VFP_COUNT)
 
-#define _MIPIA_HIGH_LOW_SWITCH_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb044)
-#define _MIPIC_HIGH_LOW_SWITCH_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb844)
-#define MIPI_HIGH_LOW_SWITCH_COUNT(port)	_MMIO_MIPI(port,	_MIPIA_HIGH_LOW_SWITCH_COUNT, _MIPIC_HIGH_LOW_SWITCH_COUNT)
+#define _MIPIA_HIGH_LOW_SWITCH_COUNT		0xb044
+#define _MIPIC_HIGH_LOW_SWITCH_COUNT		0xb844
+#define MIPI_HIGH_LOW_SWITCH_COUNT(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port,	_MIPIA_HIGH_LOW_SWITCH_COUNT, _MIPIC_HIGH_LOW_SWITCH_COUNT)
 
-#define _MIPIA_DPI_CONTROL		(_MIPI_MMIO_BASE(dev_priv) + 0xb048)
-#define _MIPIC_DPI_CONTROL		(_MIPI_MMIO_BASE(dev_priv) + 0xb848)
-#define MIPI_DPI_CONTROL(port)		_MMIO_MIPI(port, _MIPIA_DPI_CONTROL, _MIPIC_DPI_CONTROL)
+#define _MIPIA_DPI_CONTROL			0xb048
+#define _MIPIC_DPI_CONTROL			0xb848
+#define MIPI_DPI_CONTROL(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_DPI_CONTROL, _MIPIC_DPI_CONTROL)
 #define  DPI_LP_MODE					(1 << 6)
 #define  BACKLIGHT_OFF					(1 << 5)
 #define  BACKLIGHT_ON					(1 << 4)
@@ -249,28 +243,27 @@
 #define  TURN_ON					(1 << 1)
 #define  SHUTDOWN					(1 << 0)
 
-#define _MIPIA_DPI_DATA			(_MIPI_MMIO_BASE(dev_priv) + 0xb04c)
-#define _MIPIC_DPI_DATA			(_MIPI_MMIO_BASE(dev_priv) + 0xb84c)
-#define MIPI_DPI_DATA(port)		_MMIO_MIPI(port, _MIPIA_DPI_DATA, _MIPIC_DPI_DATA)
+#define _MIPIA_DPI_DATA				0xb04c
+#define _MIPIC_DPI_DATA				0xb84c
+#define MIPI_DPI_DATA(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_DPI_DATA, _MIPIC_DPI_DATA)
 #define  COMMAND_BYTE_SHIFT				0
 #define  COMMAND_BYTE_MASK				(0x3f << 0)
 
-#define _MIPIA_INIT_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb050)
-#define _MIPIC_INIT_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb850)
-#define MIPI_INIT_COUNT(port)		_MMIO_MIPI(port, _MIPIA_INIT_COUNT, _MIPIC_INIT_COUNT)
+#define _MIPIA_INIT_COUNT			0xb050
+#define _MIPIC_INIT_COUNT			0xb850
+#define MIPI_INIT_COUNT(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_INIT_COUNT, _MIPIC_INIT_COUNT)
 #define  MASTER_INIT_TIMER_SHIFT			0
 #define  MASTER_INIT_TIMER_MASK				(0xffff << 0)
 
-#define _MIPIA_MAX_RETURN_PKT_SIZE	(_MIPI_MMIO_BASE(dev_priv) + 0xb054)
-#define _MIPIC_MAX_RETURN_PKT_SIZE	(_MIPI_MMIO_BASE(dev_priv) + 0xb854)
-#define MIPI_MAX_RETURN_PKT_SIZE(port)	_MMIO_MIPI(port, \
-			_MIPIA_MAX_RETURN_PKT_SIZE, _MIPIC_MAX_RETURN_PKT_SIZE)
+#define _MIPIA_MAX_RETURN_PKT_SIZE		0xb054
+#define _MIPIC_MAX_RETURN_PKT_SIZE		0xb854
+#define MIPI_MAX_RETURN_PKT_SIZE(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_MAX_RETURN_PKT_SIZE, _MIPIC_MAX_RETURN_PKT_SIZE)
 #define  MAX_RETURN_PKT_SIZE_SHIFT			0
 #define  MAX_RETURN_PKT_SIZE_MASK			(0x3ff << 0)
 
-#define _MIPIA_VIDEO_MODE_FORMAT	(_MIPI_MMIO_BASE(dev_priv) + 0xb058)
-#define _MIPIC_VIDEO_MODE_FORMAT	(_MIPI_MMIO_BASE(dev_priv) + 0xb858)
-#define MIPI_VIDEO_MODE_FORMAT(port)	_MMIO_MIPI(port, _MIPIA_VIDEO_MODE_FORMAT, _MIPIC_VIDEO_MODE_FORMAT)
+#define _MIPIA_VIDEO_MODE_FORMAT		0xb058
+#define _MIPIC_VIDEO_MODE_FORMAT		0xb858
+#define MIPI_VIDEO_MODE_FORMAT(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_VIDEO_MODE_FORMAT, _MIPIC_VIDEO_MODE_FORMAT)
 #define  RANDOM_DPI_DISPLAY_RESOLUTION			(1 << 4)
 #define  DISABLE_VIDEO_BTA				(1 << 3)
 #define  IP_TG_CONFIG					(1 << 2)
@@ -278,9 +271,9 @@
 #define  VIDEO_MODE_NON_BURST_WITH_SYNC_EVENTS		(2 << 0)
 #define  VIDEO_MODE_BURST				(3 << 0)
 
-#define _MIPIA_EOT_DISABLE		(_MIPI_MMIO_BASE(dev_priv) + 0xb05c)
-#define _MIPIC_EOT_DISABLE		(_MIPI_MMIO_BASE(dev_priv) + 0xb85c)
-#define MIPI_EOT_DISABLE(port)		_MMIO_MIPI(port, _MIPIA_EOT_DISABLE, _MIPIC_EOT_DISABLE)
+#define _MIPIA_EOT_DISABLE			0xb05c
+#define _MIPIC_EOT_DISABLE			0xb85c
+#define MIPI_EOT_DISABLE(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_EOT_DISABLE, _MIPIC_EOT_DISABLE)
 #define  BXT_DEFEATURE_DPI_FIFO_CTR			(1 << 9)
 #define  BXT_DPHY_DEFEATURE_EN				(1 << 8)
 #define  LP_RX_TIMEOUT_ERROR_RECOVERY_DISABLE		(1 << 7)
@@ -292,36 +285,36 @@
 #define  CLOCKSTOP					(1 << 1)
 #define  EOT_DISABLE					(1 << 0)
 
-#define _MIPIA_LP_BYTECLK		(_MIPI_MMIO_BASE(dev_priv) + 0xb060)
-#define _MIPIC_LP_BYTECLK		(_MIPI_MMIO_BASE(dev_priv) + 0xb860)
-#define MIPI_LP_BYTECLK(port)		_MMIO_MIPI(port, _MIPIA_LP_BYTECLK, _MIPIC_LP_BYTECLK)
+#define _MIPIA_LP_BYTECLK			0xb060
+#define _MIPIC_LP_BYTECLK			0xb860
+#define MIPI_LP_BYTECLK(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_LP_BYTECLK, _MIPIC_LP_BYTECLK)
 #define  LP_BYTECLK_SHIFT				0
 #define  LP_BYTECLK_MASK				(0xffff << 0)
 
-#define _MIPIA_TLPX_TIME_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb0a4)
-#define _MIPIC_TLPX_TIME_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb8a4)
-#define MIPI_TLPX_TIME_COUNT(port)	 _MMIO_MIPI(port, _MIPIA_TLPX_TIME_COUNT, _MIPIC_TLPX_TIME_COUNT)
+#define _MIPIA_TLPX_TIME_COUNT			0xb0a4
+#define _MIPIC_TLPX_TIME_COUNT			0xb8a4
+#define MIPI_TLPX_TIME_COUNT(display, port)	 _MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_TLPX_TIME_COUNT, _MIPIC_TLPX_TIME_COUNT)
 
-#define _MIPIA_CLK_LANE_TIMING		(_MIPI_MMIO_BASE(dev_priv) + 0xb098)
-#define _MIPIC_CLK_LANE_TIMING		(_MIPI_MMIO_BASE(dev_priv) + 0xb898)
-#define MIPI_CLK_LANE_TIMING(port)	 _MMIO_MIPI(port, _MIPIA_CLK_LANE_TIMING, _MIPIC_CLK_LANE_TIMING)
+#define _MIPIA_CLK_LANE_TIMING			0xb098
+#define _MIPIC_CLK_LANE_TIMING			0xb898
+#define MIPI_CLK_LANE_TIMING(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_CLK_LANE_TIMING, _MIPIC_CLK_LANE_TIMING)
 
 /* bits 31:0 */
-#define _MIPIA_LP_GEN_DATA		(_MIPI_MMIO_BASE(dev_priv) + 0xb064)
-#define _MIPIC_LP_GEN_DATA		(_MIPI_MMIO_BASE(dev_priv) + 0xb864)
-#define MIPI_LP_GEN_DATA(port)		_MMIO_MIPI(port, _MIPIA_LP_GEN_DATA, _MIPIC_LP_GEN_DATA)
+#define _MIPIA_LP_GEN_DATA			0xb064
+#define _MIPIC_LP_GEN_DATA			0xb864
+#define MIPI_LP_GEN_DATA(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_LP_GEN_DATA, _MIPIC_LP_GEN_DATA)
 
 /* bits 31:0 */
-#define _MIPIA_HS_GEN_DATA		(_MIPI_MMIO_BASE(dev_priv) + 0xb068)
-#define _MIPIC_HS_GEN_DATA		(_MIPI_MMIO_BASE(dev_priv) + 0xb868)
-#define MIPI_HS_GEN_DATA(port)		_MMIO_MIPI(port, _MIPIA_HS_GEN_DATA, _MIPIC_HS_GEN_DATA)
-
-#define _MIPIA_LP_GEN_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb06c)
-#define _MIPIC_LP_GEN_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb86c)
-#define MIPI_LP_GEN_CTRL(port)		_MMIO_MIPI(port, _MIPIA_LP_GEN_CTRL, _MIPIC_LP_GEN_CTRL)
-#define _MIPIA_HS_GEN_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb070)
-#define _MIPIC_HS_GEN_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb870)
-#define MIPI_HS_GEN_CTRL(port)		_MMIO_MIPI(port, _MIPIA_HS_GEN_CTRL, _MIPIC_HS_GEN_CTRL)
+#define _MIPIA_HS_GEN_DATA			0xb068
+#define _MIPIC_HS_GEN_DATA			0xb868
+#define MIPI_HS_GEN_DATA(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_HS_GEN_DATA, _MIPIC_HS_GEN_DATA)
+
+#define _MIPIA_LP_GEN_CTRL			0xb06c
+#define _MIPIC_LP_GEN_CTRL			0xb86c
+#define MIPI_LP_GEN_CTRL(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_LP_GEN_CTRL, _MIPIC_LP_GEN_CTRL)
+#define _MIPIA_HS_GEN_CTRL			0xb070
+#define _MIPIC_HS_GEN_CTRL			0xb870
+#define MIPI_HS_GEN_CTRL(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_HS_GEN_CTRL, _MIPIC_HS_GEN_CTRL)
 #define  LONG_PACKET_WORD_COUNT_SHIFT			8
 #define  LONG_PACKET_WORD_COUNT_MASK			(0xffff << 8)
 #define  SHORT_PACKET_PARAM_SHIFT			8
@@ -332,9 +325,9 @@
 #define  DATA_TYPE_MASK					(0x3f << 0)
 /* data type values, see include/video/mipi_display.h */
 
-#define _MIPIA_GEN_FIFO_STAT		(_MIPI_MMIO_BASE(dev_priv) + 0xb074)
-#define _MIPIC_GEN_FIFO_STAT		(_MIPI_MMIO_BASE(dev_priv) + 0xb874)
-#define MIPI_GEN_FIFO_STAT(port)	_MMIO_MIPI(port, _MIPIA_GEN_FIFO_STAT, _MIPIC_GEN_FIFO_STAT)
+#define _MIPIA_GEN_FIFO_STAT			0xb074
+#define _MIPIC_GEN_FIFO_STAT			0xb874
+#define MIPI_GEN_FIFO_STAT(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_GEN_FIFO_STAT, _MIPIC_GEN_FIFO_STAT)
 #define  DPI_FIFO_EMPTY					(1 << 28)
 #define  DBI_FIFO_EMPTY					(1 << 27)
 #define  LP_CTRL_FIFO_EMPTY				(1 << 26)
@@ -350,16 +343,16 @@
 #define  HS_DATA_FIFO_HALF_EMPTY			(1 << 1)
 #define  HS_DATA_FIFO_FULL				(1 << 0)
 
-#define _MIPIA_HS_LS_DBI_ENABLE		(_MIPI_MMIO_BASE(dev_priv) + 0xb078)
-#define _MIPIC_HS_LS_DBI_ENABLE		(_MIPI_MMIO_BASE(dev_priv) + 0xb878)
-#define MIPI_HS_LP_DBI_ENABLE(port)	_MMIO_MIPI(port, _MIPIA_HS_LS_DBI_ENABLE, _MIPIC_HS_LS_DBI_ENABLE)
+#define _MIPIA_HS_LS_DBI_ENABLE			0xb078
+#define _MIPIC_HS_LS_DBI_ENABLE			0xb878
+#define MIPI_HS_LP_DBI_ENABLE(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_HS_LS_DBI_ENABLE, _MIPIC_HS_LS_DBI_ENABLE)
 #define  DBI_HS_LP_MODE_MASK				(1 << 0)
 #define  DBI_LP_MODE					(1 << 0)
 #define  DBI_HS_MODE					(0 << 0)
 
-#define _MIPIA_DPHY_PARAM		(_MIPI_MMIO_BASE(dev_priv) + 0xb080)
-#define _MIPIC_DPHY_PARAM		(_MIPI_MMIO_BASE(dev_priv) + 0xb880)
-#define MIPI_DPHY_PARAM(port)		_MMIO_MIPI(port, _MIPIA_DPHY_PARAM, _MIPIC_DPHY_PARAM)
+#define _MIPIA_DPHY_PARAM			0xb080
+#define _MIPIC_DPHY_PARAM			0xb880
+#define MIPI_DPHY_PARAM(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_DPHY_PARAM, _MIPIC_DPHY_PARAM)
 #define  EXIT_ZERO_COUNT_SHIFT				24
 #define  EXIT_ZERO_COUNT_MASK				(0x3f << 24)
 #define  TRAIL_COUNT_SHIFT				16
@@ -369,34 +362,34 @@
 #define  PREPARE_COUNT_SHIFT				0
 #define  PREPARE_COUNT_MASK				(0x3f << 0)
 
-#define _MIPIA_DBI_BW_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb084)
-#define _MIPIC_DBI_BW_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb884)
-#define MIPI_DBI_BW_CTRL(port)		_MMIO_MIPI(port, _MIPIA_DBI_BW_CTRL, _MIPIC_DBI_BW_CTRL)
+#define _MIPIA_DBI_BW_CTRL			0xb084
+#define _MIPIC_DBI_BW_CTRL			0xb884
+#define MIPI_DBI_BW_CTRL(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_DBI_BW_CTRL, _MIPIC_DBI_BW_CTRL)
 
-#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb088)
-#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb888)
-#define MIPI_CLK_LANE_SWITCH_TIME_CNT(port)	_MMIO_MIPI(port, _MIPIA_CLK_LANE_SWITCH_TIME_CNT, _MIPIC_CLK_LANE_SWITCH_TIME_CNT)
+#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT		0xb088
+#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT		0xb888
+#define MIPI_CLK_LANE_SWITCH_TIME_CNT(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_CLK_LANE_SWITCH_TIME_CNT, _MIPIC_CLK_LANE_SWITCH_TIME_CNT)
 #define  LP_HS_SSW_CNT_SHIFT				16
 #define  LP_HS_SSW_CNT_MASK				(0xffff << 16)
 #define  HS_LP_PWR_SW_CNT_SHIFT				0
 #define  HS_LP_PWR_SW_CNT_MASK				(0xffff << 0)
 
-#define _MIPIA_STOP_STATE_STALL		(_MIPI_MMIO_BASE(dev_priv) + 0xb08c)
-#define _MIPIC_STOP_STATE_STALL		(_MIPI_MMIO_BASE(dev_priv) + 0xb88c)
-#define MIPI_STOP_STATE_STALL(port)	_MMIO_MIPI(port, _MIPIA_STOP_STATE_STALL, _MIPIC_STOP_STATE_STALL)
+#define _MIPIA_STOP_STATE_STALL			0xb08c
+#define _MIPIC_STOP_STATE_STALL			0xb88c
+#define MIPI_STOP_STATE_STALL(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_STOP_STATE_STALL, _MIPIC_STOP_STATE_STALL)
 #define  STOP_STATE_STALL_COUNTER_SHIFT			0
 #define  STOP_STATE_STALL_COUNTER_MASK			(0xff << 0)
 
-#define _MIPIA_INTR_STAT_REG_1		(_MIPI_MMIO_BASE(dev_priv) + 0xb090)
-#define _MIPIC_INTR_STAT_REG_1		(_MIPI_MMIO_BASE(dev_priv) + 0xb890)
-#define MIPI_INTR_STAT_REG_1(port)	_MMIO_MIPI(port, _MIPIA_INTR_STAT_REG_1, _MIPIC_INTR_STAT_REG_1)
-#define _MIPIA_INTR_EN_REG_1		(_MIPI_MMIO_BASE(dev_priv) + 0xb094)
-#define _MIPIC_INTR_EN_REG_1		(_MIPI_MMIO_BASE(dev_priv) + 0xb894)
-#define MIPI_INTR_EN_REG_1(port)	_MMIO_MIPI(port, _MIPIA_INTR_EN_REG_1, _MIPIC_INTR_EN_REG_1)
+#define _MIPIA_INTR_STAT_REG_1			0xb090
+#define _MIPIC_INTR_STAT_REG_1			0xb890
+#define MIPI_INTR_STAT_REG_1(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_INTR_STAT_REG_1, _MIPIC_INTR_STAT_REG_1)
+#define _MIPIA_INTR_EN_REG_1			0xb094
+#define _MIPIC_INTR_EN_REG_1			0xb894
+#define MIPI_INTR_EN_REG_1(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_INTR_EN_REG_1, _MIPIC_INTR_EN_REG_1)
 #define  RX_CONTENTION_DETECTED				(1 << 0)
 
 /* XXX: only pipe A ?!? */
-#define MIPIA_DBI_TYPEC_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb100)
+#define MIPIA_DBI_TYPEC_CTRL(display)		(_MIPI_MMIO_BASE(display) + 0xb100)
 #define  DBI_TYPEC_ENABLE				(1 << 31)
 #define  DBI_TYPEC_WIP					(1 << 30)
 #define  DBI_TYPEC_OPTION_SHIFT				28
@@ -409,9 +402,9 @@
 
 /* MIPI adapter registers */
 
-#define _MIPIA_CTRL			(_MIPI_MMIO_BASE(dev_priv) + 0xb104)
-#define _MIPIC_CTRL			(_MIPI_MMIO_BASE(dev_priv) + 0xb904)
-#define MIPI_CTRL(port)			_MMIO_MIPI(port, _MIPIA_CTRL, _MIPIC_CTRL)
+#define _MIPIA_CTRL				0xb104
+#define _MIPIC_CTRL				0xb904
+#define MIPI_CTRL(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_CTRL, _MIPIC_CTRL)
 #define  ESCAPE_CLOCK_DIVIDER_SHIFT			5 /* A only */
 #define  ESCAPE_CLOCK_DIVIDER_MASK			(3 << 5)
 #define  ESCAPE_CLOCK_DIVIDER_1				(0 << 5)
@@ -442,41 +435,41 @@
 #define  GLK_MIPIIO_PORT_POWERED			(1 << 1) /* RO */
 #define  GLK_MIPIIO_ENABLE				(1 << 0)
 
-#define _MIPIA_DATA_ADDRESS		(_MIPI_MMIO_BASE(dev_priv) + 0xb108)
-#define _MIPIC_DATA_ADDRESS		(_MIPI_MMIO_BASE(dev_priv) + 0xb908)
-#define MIPI_DATA_ADDRESS(port)		_MMIO_MIPI(port, _MIPIA_DATA_ADDRESS, _MIPIC_DATA_ADDRESS)
+#define _MIPIA_DATA_ADDRESS			0xb108
+#define _MIPIC_DATA_ADDRESS			0xb908
+#define MIPI_DATA_ADDRESS(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_DATA_ADDRESS, _MIPIC_DATA_ADDRESS)
 #define  DATA_MEM_ADDRESS_SHIFT				5
 #define  DATA_MEM_ADDRESS_MASK				(0x7ffffff << 5)
 #define  DATA_VALID					(1 << 0)
 
-#define _MIPIA_DATA_LENGTH		(_MIPI_MMIO_BASE(dev_priv) + 0xb10c)
-#define _MIPIC_DATA_LENGTH		(_MIPI_MMIO_BASE(dev_priv) + 0xb90c)
-#define MIPI_DATA_LENGTH(port)		_MMIO_MIPI(port, _MIPIA_DATA_LENGTH, _MIPIC_DATA_LENGTH)
+#define _MIPIA_DATA_LENGTH			0xb10c
+#define _MIPIC_DATA_LENGTH			0xb90c
+#define MIPI_DATA_LENGTH(display, port)		_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_DATA_LENGTH, _MIPIC_DATA_LENGTH)
 #define  DATA_LENGTH_SHIFT				0
 #define  DATA_LENGTH_MASK				(0xfffff << 0)
 
-#define _MIPIA_COMMAND_ADDRESS		(_MIPI_MMIO_BASE(dev_priv) + 0xb110)
-#define _MIPIC_COMMAND_ADDRESS		(_MIPI_MMIO_BASE(dev_priv) + 0xb910)
-#define MIPI_COMMAND_ADDRESS(port)	_MMIO_MIPI(port, _MIPIA_COMMAND_ADDRESS, _MIPIC_COMMAND_ADDRESS)
+#define _MIPIA_COMMAND_ADDRESS			0xb110
+#define _MIPIC_COMMAND_ADDRESS			0xb910
+#define MIPI_COMMAND_ADDRESS(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_COMMAND_ADDRESS, _MIPIC_COMMAND_ADDRESS)
 #define  COMMAND_MEM_ADDRESS_SHIFT			5
 #define  COMMAND_MEM_ADDRESS_MASK			(0x7ffffff << 5)
 #define  AUTO_PWG_ENABLE				(1 << 2)
 #define  MEMORY_WRITE_DATA_FROM_PIPE_RENDERING		(1 << 1)
 #define  COMMAND_VALID					(1 << 0)
 
-#define _MIPIA_COMMAND_LENGTH		(_MIPI_MMIO_BASE(dev_priv) + 0xb114)
-#define _MIPIC_COMMAND_LENGTH		(_MIPI_MMIO_BASE(dev_priv) + 0xb914)
-#define MIPI_COMMAND_LENGTH(port)	_MMIO_MIPI(port, _MIPIA_COMMAND_LENGTH, _MIPIC_COMMAND_LENGTH)
+#define _MIPIA_COMMAND_LENGTH			0xb114
+#define _MIPIC_COMMAND_LENGTH			0xb914
+#define MIPI_COMMAND_LENGTH(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_COMMAND_LENGTH, _MIPIC_COMMAND_LENGTH)
 #define  COMMAND_LENGTH_SHIFT(n)			(8 * (n)) /* n: 0...3 */
 #define  COMMAND_LENGTH_MASK(n)				(0xff << (8 * (n)))
 
-#define _MIPIA_READ_DATA_RETURN0	(_MIPI_MMIO_BASE(dev_priv) + 0xb118)
-#define _MIPIC_READ_DATA_RETURN0	(_MIPI_MMIO_BASE(dev_priv) + 0xb918)
-#define MIPI_READ_DATA_RETURN(port, n) _MMIO(_MIPI(port, _MIPIA_READ_DATA_RETURN0, _MIPIC_READ_DATA_RETURN0) + 4 * (n)) /* n: 0...7 */
+#define _MIPIA_READ_DATA_RETURN0		0xb118
+#define _MIPIC_READ_DATA_RETURN0		0xb918
+#define MIPI_READ_DATA_RETURN(display, port, n)	_MMIO_MIPI(_MIPI_MMIO_BASE(display) + 4 * (n), port, _MIPIA_READ_DATA_RETURN0, _MIPIC_READ_DATA_RETURN0) /* n: 0...7 */
 
-#define _MIPIA_READ_DATA_VALID		(_MIPI_MMIO_BASE(dev_priv) + 0xb138)
-#define _MIPIC_READ_DATA_VALID		(_MIPI_MMIO_BASE(dev_priv) + 0xb938)
-#define MIPI_READ_DATA_VALID(port)	_MMIO_MIPI(port, _MIPIA_READ_DATA_VALID, _MIPIC_READ_DATA_VALID)
+#define _MIPIA_READ_DATA_VALID			0xb138
+#define _MIPIC_READ_DATA_VALID			0xb938
+#define MIPI_READ_DATA_VALID(display, port)	_MMIO_MIPI(_MIPI_MMIO_BASE(display), port, _MIPIA_READ_DATA_VALID, _MIPIC_READ_DATA_VALID)
 #define  READ_DATA_VALID(n)				(1 << (n))
 
 #endif /* __VLV_DSI_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index dcbfe32fd30c..81f65cab1330 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -879,6 +879,7 @@ static int set_proto_ctx_param(struct drm_i915_file_private *fpriv,
 			       struct i915_gem_proto_context *pc,
 			       struct drm_i915_gem_context_param *args)
 {
+	struct drm_i915_private *i915 = fpriv->i915;
 	int ret = 0;
 
 	switch (args->param) {
@@ -904,6 +905,13 @@ static int set_proto_ctx_param(struct drm_i915_file_private *fpriv,
 			pc->user_flags &= ~BIT(UCONTEXT_BANNABLE);
 		break;
 
+	case I915_CONTEXT_PARAM_LOW_LATENCY:
+		if (intel_uc_uses_guc_submission(&to_gt(i915)->uc))
+			pc->user_flags |= BIT(UCONTEXT_LOW_LATENCY);
+		else
+			ret = -EINVAL;
+		break;
+
 	case I915_CONTEXT_PARAM_RECOVERABLE:
 		if (args->size)
 			ret = -EINVAL;
@@ -992,6 +1000,9 @@ static int intel_context_set_gem(struct intel_context *ce,
 	if (sseu.slice_mask && !WARN_ON(ce->engine->class != RENDER_CLASS))
 		ret = intel_context_reconfigure_sseu(ce, sseu);
 
+	if (test_bit(UCONTEXT_LOW_LATENCY, &ctx->user_flags))
+		__set_bit(CONTEXT_LOW_LATENCY, &ce->flags);
+
 	return ret;
 }
 
@@ -1630,6 +1641,9 @@ i915_gem_create_context(struct drm_i915_private *i915,
 	if (vm)
 		ctx->vm = vm;
 
+	/* Assign early so intel_context_set_gem can access these flags */
+	ctx->user_flags = pc->user_flags;
+
 	mutex_init(&ctx->engines_mutex);
 	if (pc->num_user_engines >= 0) {
 		i915_gem_context_set_user_engines(ctx);
@@ -1652,8 +1666,6 @@ i915_gem_create_context(struct drm_i915_private *i915,
 	 * is no remap info, it will be a NOP. */
 	ctx->remap_slice = ALL_L3_SLICES(i915);
 
-	ctx->user_flags = pc->user_flags;
-
 	for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++)
 		ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES;
 
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h
index 03bc7f9d191b..b6d97da63d1f 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h
@@ -338,6 +338,7 @@ struct i915_gem_context {
 #define UCONTEXT_BANNABLE		2
 #define UCONTEXT_RECOVERABLE		3
 #define UCONTEXT_PERSISTENCE		4
+#define UCONTEXT_LOW_LATENCY		5
 
 	/**
 	 * @flags: small set of booleans
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index d3a771afb083..090724fa766c 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -2457,7 +2457,7 @@ static int eb_submit(struct i915_execbuffer *eb)
  * The engine index is returned.
  */
 static unsigned int
-gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
+gen8_dispatch_bsd_engine(struct drm_i915_private *i915,
 			 struct drm_file *file)
 {
 	struct drm_i915_file_private *file_priv = file->driver_priv;
@@ -2465,7 +2465,7 @@ gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
 	/* Check whether the file_priv has already selected one ring. */
 	if ((int)file_priv->bsd_engine < 0)
 		file_priv->bsd_engine =
-			get_random_u32_below(dev_priv->engine_uabi_class_count[I915_ENGINE_CLASS_VIDEO]);
+			get_random_u32_below(i915->engine_uabi_class_count[I915_ENGINE_CLASS_VIDEO]);
 
 	return file_priv->bsd_engine;
 }
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h
index 3560a062d287..5d7446a48ae7 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h
@@ -284,7 +284,9 @@ bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj);
 static inline bool
 i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj)
 {
-	return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_SHRINKABLE);
+	/* TODO: make DPT shrinkable when it has no bound vmas */
+	return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_SHRINKABLE) &&
+		!obj->is_dpt;
 }
 
 static inline bool
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
index 0c5cdab278b6..1495b6074492 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
@@ -386,7 +386,7 @@ struct drm_i915_gem_object {
 	 * and kernel mode driver for caching policy control after GEN12.
 	 * In the meantime platform specific tables are created to translate
 	 * i915_cache_level into pat index, for more details check the macros
-	 * defined i915/i915_pci.c, e.g. PVC_CACHELEVEL.
+	 * defined i915/i915_pci.c, e.g. TGL_CACHELEVEL.
 	 * For backward compatibility, this field contains values exactly match
 	 * the entries of enum i915_cache_level for pre-GEN12 platforms (See
 	 * LEGACY_CACHELEVEL), so that the PTE encode functions for these
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
index 0ba955611dfb..8780aa243105 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
@@ -5,6 +5,7 @@
  */
 
 #include <drm/drm_cache.h>
+#include <linux/vmalloc.h>
 
 #include "gt/intel_gt.h"
 #include "gt/intel_tlb.h"
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
index 38b72d86560f..c5e1c718a6d2 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
@@ -654,7 +654,7 @@ i915_gem_object_create_shmem(struct drm_i915_private *i915,
 
 /* Allocate a new GEM object and fill it with the supplied data */
 struct drm_i915_gem_object *
-i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv,
+i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915,
 				       const void *data, resource_size_t size)
 {
 	struct drm_i915_gem_object *obj;
@@ -663,8 +663,8 @@ i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv,
 	resource_size_t offset;
 	int err;
 
-	GEM_WARN_ON(IS_DGFX(dev_priv));
-	obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE));
+	GEM_WARN_ON(IS_DGFX(i915));
+	obj = i915_gem_object_create_shmem(i915, round_up(size, PAGE_SIZE));
 	if (IS_ERR(obj))
 		return obj;
 
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.h b/drivers/gpu/drm/i915/gem/i915_gem_stolen.h
index 258381d1c054..dfe0db8bb1b9 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.h
@@ -14,14 +14,14 @@ struct drm_i915_gem_object;
 
 #define i915_stolen_fb drm_mm_node
 
-int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv,
+int i915_gem_stolen_insert_node(struct drm_i915_private *i915,
 				struct drm_mm_node *node, u64 size,
 				unsigned alignment);
-int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv,
+int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *i915,
 					 struct drm_mm_node *node, u64 size,
 					 unsigned alignment, u64 start,
 					 u64 end);
-void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
+void i915_gem_stolen_remove_node(struct drm_i915_private *i915,
 				 struct drm_mm_node *node);
 struct intel_memory_region *
 i915_gem_stolen_smem_setup(struct drm_i915_private *i915, u16 type,
@@ -31,7 +31,7 @@ i915_gem_stolen_lmem_setup(struct drm_i915_private *i915, u16 type,
 			   u16 instance);
 
 struct drm_i915_gem_object *
-i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
+i915_gem_object_create_stolen(struct drm_i915_private *i915,
 			      resource_size_t size);
 
 bool i915_gem_object_is_stolen(const struct drm_i915_gem_object *obj);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c
index a049ca0b7980..d9eb84c1d2f1 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c
@@ -343,12 +343,12 @@ int
 i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
 			  struct drm_file *file)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *i915 = to_i915(dev);
 	struct drm_i915_gem_set_tiling *args = data;
 	struct drm_i915_gem_object *obj;
 	int err;
 
-	if (!to_gt(dev_priv)->ggtt->num_fences)
+	if (!to_gt(i915)->ggtt->num_fences)
 		return -EOPNOTSUPP;
 
 	obj = i915_gem_object_lookup(file, args->handle);
@@ -374,9 +374,9 @@ i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
 		args->stride = 0;
 	} else {
 		if (args->tiling_mode == I915_TILING_X)
-			args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_x;
+			args->swizzle_mode = to_gt(i915)->ggtt->bit_6_swizzle_x;
 		else
-			args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_y;
+			args->swizzle_mode = to_gt(i915)->ggtt->bit_6_swizzle_y;
 
 		/* Hide bit 17 swizzling from the user.  This prevents old Mesa
 		 * from aborting the application on sw fallbacks to bit 17,
@@ -427,11 +427,11 @@ i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
 			  struct drm_file *file)
 {
 	struct drm_i915_gem_get_tiling *args = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *i915 = to_i915(dev);
 	struct drm_i915_gem_object *obj;
 	int err = -ENOENT;
 
-	if (!to_gt(dev_priv)->ggtt->num_fences)
+	if (!to_gt(i915)->ggtt->num_fences)
 		return -EOPNOTSUPP;
 
 	rcu_read_lock();
@@ -447,10 +447,10 @@ i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
 
 	switch (args->tiling_mode) {
 	case I915_TILING_X:
-		args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_x;
+		args->swizzle_mode = to_gt(i915)->ggtt->bit_6_swizzle_x;
 		break;
 	case I915_TILING_Y:
-		args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_y;
+		args->swizzle_mode = to_gt(i915)->ggtt->bit_6_swizzle_y;
 		break;
 	default:
 	case I915_TILING_NONE:
@@ -459,7 +459,7 @@ i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
 	}
 
 	/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
-	if (dev_priv->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)
+	if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)
 		args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN;
 	else
 		args->phys_swizzle_mode = args->swizzle_mode;
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
index 61abfb505766..09b68713ab32 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
@@ -463,13 +463,13 @@ i915_gem_userptr_ioctl(struct drm_device *dev,
 		       struct drm_file *file)
 {
 	static struct lock_class_key __maybe_unused lock_class;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *i915 = to_i915(dev);
 	struct drm_i915_gem_userptr *args = data;
 	struct drm_i915_gem_object __maybe_unused *obj;
 	int __maybe_unused ret;
 	u32 __maybe_unused handle;
 
-	if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) {
+	if (!HAS_LLC(i915) && !HAS_SNOOP(i915)) {
 		/* We cannot support coherent userptr objects on hw without
 		 * LLC and broken snooping.
 		 */
@@ -501,7 +501,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev,
 		 * On almost all of the older hw, we cannot tell the GPU that
 		 * a page is readonly.
 		 */
-		if (!to_gt(dev_priv)->vm->has_read_only)
+		if (!to_gt(i915)->vm->has_read_only)
 			return -ENODEV;
 	}
 
diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
index 3ff3d8889c6c..84d41e6ccf05 100644
--- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
@@ -713,7 +713,7 @@ static int igt_ppgtt_huge_fill(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
 	unsigned int supported = RUNTIME_INFO(i915)->page_sizes;
-	bool has_pte64 = GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50);
+	bool has_pte64 = GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55);
 	struct i915_address_space *vm;
 	struct i915_gem_context *ctx;
 	unsigned long max_pages;
@@ -857,7 +857,7 @@ out:
 static int igt_ppgtt_64K(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
-	bool has_pte64 = GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50);
+	bool has_pte64 = GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55);
 	struct drm_i915_gem_object *obj;
 	struct i915_address_space *vm;
 	struct i915_gem_context *ctx;
@@ -1969,19 +1969,19 @@ int i915_gem_huge_page_mock_selftests(void)
 		SUBTEST(igt_mock_memory_region_huge_pages),
 		SUBTEST(igt_mock_ppgtt_misaligned_dma),
 	};
-	struct drm_i915_private *dev_priv;
+	struct drm_i915_private *i915;
 	struct i915_ppgtt *ppgtt;
 	int err;
 
-	dev_priv = mock_gem_device();
-	if (!dev_priv)
+	i915 = mock_gem_device();
+	if (!i915)
 		return -ENOMEM;
 
 	/* Pretend to be a device which supports the 48b PPGTT */
-	RUNTIME_INFO(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL;
-	RUNTIME_INFO(dev_priv)->ppgtt_size = 48;
+	RUNTIME_INFO(i915)->ppgtt_type = INTEL_PPGTT_FULL;
+	RUNTIME_INFO(i915)->ppgtt_size = 48;
 
-	ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0);
+	ppgtt = i915_ppgtt_create(to_gt(i915), 0);
 	if (IS_ERR(ppgtt)) {
 		err = PTR_ERR(ppgtt);
 		goto out_unlock;
@@ -2005,7 +2005,7 @@ int i915_gem_huge_page_mock_selftests(void)
 out_put:
 	i915_vm_put(&ppgtt->vm);
 out_unlock:
-	mock_destroy_device(dev_priv);
+	mock_destroy_device(i915);
 	return err;
 }
 
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c
index 10a7847f1b04..bac15196b4d2 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c
@@ -117,7 +117,7 @@ static bool fastblit_supports_x_tiling(const struct drm_i915_private *i915)
 	if (gen < 12)
 		return true;
 
-	if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
+	if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 55))
 		return false;
 
 	return HAS_DISPLAY(i915);
@@ -166,7 +166,7 @@ static int prepare_blit(const struct tiled_blits *t,
 		src_pitch = t->width; /* in dwords */
 		if (src->tiling == CLIENT_TILING_Y) {
 			src_tiles = XY_FAST_COPY_BLT_D0_SRC_TILE_MODE(YMAJOR);
-			if (GRAPHICS_VER_FULL(to_i915(batch->base.dev)) >= IP_VER(12, 50))
+			if (GRAPHICS_VER_FULL(to_i915(batch->base.dev)) >= IP_VER(12, 55))
 				src_4t = XY_FAST_COPY_BLT_D1_SRC_TILE4;
 		} else if (src->tiling == CLIENT_TILING_X) {
 			src_tiles = XY_FAST_COPY_BLT_D0_SRC_TILE_MODE(TILE_X);
@@ -177,7 +177,7 @@ static int prepare_blit(const struct tiled_blits *t,
 		dst_pitch = t->width; /* in dwords */
 		if (dst->tiling == CLIENT_TILING_Y) {
 			dst_tiles = XY_FAST_COPY_BLT_D0_DST_TILE_MODE(YMAJOR);
-			if (GRAPHICS_VER_FULL(to_i915(batch->base.dev)) >= IP_VER(12, 50))
+			if (GRAPHICS_VER_FULL(to_i915(batch->base.dev)) >= IP_VER(12, 55))
 				dst_4t = XY_FAST_COPY_BLT_D1_DST_TILE4;
 		} else if (dst->tiling == CLIENT_TILING_X) {
 			dst_tiles = XY_FAST_COPY_BLT_D0_DST_TILE_MODE(TILE_X);
@@ -365,7 +365,7 @@ static u64 tiled_offset(const struct intel_gt *gt,
 		v += x;
 
 		swizzle = gt->ggtt->bit_6_swizzle_x;
-	} else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) {
+	} else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 55)) {
 		/* Y-major tiling layout is Tile4 for Xe_HP and beyond */
 		v = linear_x_y_to_ftiled_pos(x_pos, y_pos, stride, 32);
 
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c
index d684a70f2c04..3527b8f446fe 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c
@@ -7,6 +7,7 @@
 #include "i915_drv.h"
 #include "i915_selftest.h"
 #include "gem/i915_gem_context.h"
+#include "gt/intel_gt.h"
 
 #include "mock_context.h"
 #include "mock_dmabuf.h"
@@ -155,6 +156,7 @@ static int verify_access(struct drm_i915_private *i915,
 	struct file *file;
 	u32 *vaddr;
 	int err = 0, i;
+	unsigned int mode;
 
 	file = mock_file(i915);
 	if (IS_ERR(file))
@@ -194,7 +196,8 @@ static int verify_access(struct drm_i915_private *i915,
 	if (err)
 		goto out_file;
 
-	vaddr = i915_gem_object_pin_map_unlocked(native_obj, I915_MAP_WB);
+	mode = intel_gt_coherent_map_type(to_gt(i915), native_obj, false);
+	vaddr = i915_gem_object_pin_map_unlocked(native_obj, mode);
 	if (IS_ERR(vaddr)) {
 		err = PTR_ERR(vaddr);
 		goto out_file;
diff --git a/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c b/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c
index b2a5882b8f81..075657018739 100644
--- a/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c
+++ b/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c
@@ -4,6 +4,7 @@
  * Copyright © 2016 Intel Corporation
  */
 
+#include <linux/vmalloc.h>
 #include "mock_dmabuf.h"
 
 static struct sg_table *mock_map_dma_buf(struct dma_buf_attachment *attachment,
diff --git a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c
index e1bf13e3d307..e9f65f27b53f 100644
--- a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c
@@ -189,9 +189,6 @@ static bool gen12_needs_ccs_aux_inv(struct intel_engine_cs *engine)
 {
 	i915_reg_t reg = gen12_get_aux_inv_reg(engine);
 
-	if (IS_PONTEVECCHIO(engine->i915))
-		return false;
-
 	/*
 	 * So far platforms supported by i915 having flat ccs do not require
 	 * AUX invalidation. Check also whether the engine requires it.
@@ -743,21 +740,25 @@ static u32 *gen12_emit_preempt_busywait(struct i915_request *rq, u32 *cs)
 }
 
 /* Wa_14014475959:dg2 */
-#define CCS_SEMAPHORE_PPHWSP_OFFSET	0x540
-static u32 ccs_semaphore_offset(struct i915_request *rq)
+/* Wa_16019325821 */
+/* Wa_14019159160 */
+#define HOLD_SWITCHOUT_SEMAPHORE_PPHWSP_OFFSET	0x540
+static u32 hold_switchout_semaphore_offset(struct i915_request *rq)
 {
 	return i915_ggtt_offset(rq->context->state) +
-		(LRC_PPHWSP_PN * PAGE_SIZE) + CCS_SEMAPHORE_PPHWSP_OFFSET;
+		(LRC_PPHWSP_PN * PAGE_SIZE) + HOLD_SWITCHOUT_SEMAPHORE_PPHWSP_OFFSET;
 }
 
 /* Wa_14014475959:dg2 */
-static u32 *ccs_emit_wa_busywait(struct i915_request *rq, u32 *cs)
+/* Wa_16019325821 */
+/* Wa_14019159160 */
+static u32 *hold_switchout_emit_wa_busywait(struct i915_request *rq, u32 *cs)
 {
 	int i;
 
 	*cs++ = MI_ATOMIC_INLINE | MI_ATOMIC_GLOBAL_GTT | MI_ATOMIC_CS_STALL |
 		MI_ATOMIC_MOVE;
-	*cs++ = ccs_semaphore_offset(rq);
+	*cs++ = hold_switchout_semaphore_offset(rq);
 	*cs++ = 0;
 	*cs++ = 1;
 
@@ -773,7 +774,7 @@ static u32 *ccs_emit_wa_busywait(struct i915_request *rq, u32 *cs)
 		MI_SEMAPHORE_POLL |
 		MI_SEMAPHORE_SAD_EQ_SDD;
 	*cs++ = 0;
-	*cs++ = ccs_semaphore_offset(rq);
+	*cs++ = hold_switchout_semaphore_offset(rq);
 	*cs++ = 0;
 
 	return cs;
@@ -790,8 +791,10 @@ gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)
 		cs = gen12_emit_preempt_busywait(rq, cs);
 
 	/* Wa_14014475959:dg2 */
-	if (intel_engine_uses_wa_hold_ccs_switchout(rq->engine))
-		cs = ccs_emit_wa_busywait(rq, cs);
+	/* Wa_16019325821 */
+	/* Wa_14019159160 */
+	if (intel_engine_uses_wa_hold_switchout(rq->engine))
+		cs = hold_switchout_emit_wa_busywait(rq, cs);
 
 	rq->tail = intel_ring_offset(rq, cs);
 	assert_ring_tail_valid(rq->ring, rq->tail);
@@ -827,7 +830,7 @@ u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)
 		cs = gen12_emit_pipe_control(cs, 0,
 					     PIPE_CONTROL_DEPTH_CACHE_FLUSH, 0);
 
-	if (GRAPHICS_VER(i915) == 12 && GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
+	if (GRAPHICS_VER(i915) == 12 && GRAPHICS_VER_FULL(i915) < IP_VER(12, 55))
 		/* Wa_1409600907 */
 		flags |= PIPE_CONTROL_DEPTH_STALL;
 
diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c
index 81bf2216371b..398d60a66410 100644
--- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c
+++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c
@@ -500,11 +500,11 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt,
 }
 
 static void
-xehpsdv_ppgtt_insert_huge(struct i915_address_space *vm,
-			  struct i915_vma_resource *vma_res,
-			  struct sgt_dma *iter,
-			  unsigned int pat_index,
-			  u32 flags)
+xehp_ppgtt_insert_huge(struct i915_address_space *vm,
+		       struct i915_vma_resource *vma_res,
+		       struct sgt_dma *iter,
+		       unsigned int pat_index,
+		       u32 flags)
 {
 	const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags);
 	unsigned int rem = sg_dma_len(iter->sg);
@@ -741,8 +741,8 @@ static void gen8_ppgtt_insert(struct i915_address_space *vm,
 	struct sgt_dma iter = sgt_dma(vma_res);
 
 	if (vma_res->bi.page_sizes.sg > I915_GTT_PAGE_SIZE) {
-		if (GRAPHICS_VER_FULL(vm->i915) >= IP_VER(12, 50))
-			xehpsdv_ppgtt_insert_huge(vm, vma_res, &iter, pat_index, flags);
+		if (GRAPHICS_VER_FULL(vm->i915) >= IP_VER(12, 55))
+			xehp_ppgtt_insert_huge(vm, vma_res, &iter, pat_index, flags);
 		else
 			gen8_ppgtt_insert_huge(vm, vma_res, &iter, pat_index, flags);
 	} else  {
@@ -781,11 +781,11 @@ static void gen8_ppgtt_insert_entry(struct i915_address_space *vm,
 	drm_clflush_virt_range(&vaddr[gen8_pd_index(idx, 0)], sizeof(*vaddr));
 }
 
-static void __xehpsdv_ppgtt_insert_entry_lm(struct i915_address_space *vm,
-					    dma_addr_t addr,
-					    u64 offset,
-					    unsigned int pat_index,
-					    u32 flags)
+static void xehp_ppgtt_insert_entry_lm(struct i915_address_space *vm,
+				       dma_addr_t addr,
+				       u64 offset,
+				       unsigned int pat_index,
+				       u32 flags)
 {
 	u64 idx = offset >> GEN8_PTE_SHIFT;
 	struct i915_page_directory * const pdp =
@@ -810,15 +810,15 @@ static void __xehpsdv_ppgtt_insert_entry_lm(struct i915_address_space *vm,
 	vaddr[gen8_pd_index(idx, 0) / 16] = vm->pte_encode(addr, pat_index, flags);
 }
 
-static void xehpsdv_ppgtt_insert_entry(struct i915_address_space *vm,
-				       dma_addr_t addr,
-				       u64 offset,
-				       unsigned int pat_index,
-				       u32 flags)
+static void xehp_ppgtt_insert_entry(struct i915_address_space *vm,
+				    dma_addr_t addr,
+				    u64 offset,
+				    unsigned int pat_index,
+				    u32 flags)
 {
 	if (flags & PTE_LM)
-		return __xehpsdv_ppgtt_insert_entry_lm(vm, addr, offset,
-						       pat_index, flags);
+		return xehp_ppgtt_insert_entry_lm(vm, addr, offset,
+						  pat_index, flags);
 
 	return gen8_ppgtt_insert_entry(vm, addr, offset, pat_index, flags);
 }
@@ -1045,7 +1045,7 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt,
 	ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND;
 	ppgtt->vm.insert_entries = gen8_ppgtt_insert;
 	if (HAS_64K_PAGES(gt->i915))
-		ppgtt->vm.insert_page = xehpsdv_ppgtt_insert_entry;
+		ppgtt->vm.insert_page = xehp_ppgtt_insert_entry;
 	else
 		ppgtt->vm.insert_page = gen8_ppgtt_insert_entry;
 	ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc;
diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c
index d650beb8ed22..20b9b04ec1e0 100644
--- a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c
@@ -263,8 +263,13 @@ static void signal_irq_work(struct irq_work *work)
 		i915_request_put(rq);
 	}
 
+	/* Lazy irq enabling after HW submission */
 	if (!READ_ONCE(b->irq_armed) && !list_empty(&b->signalers))
 		intel_breadcrumbs_arm_irq(b);
+
+	/* And confirm that we still want irqs enabled before we yield */
+	if (READ_ONCE(b->irq_armed) && !atomic_read(&b->active))
+		intel_breadcrumbs_disarm_irq(b);
 }
 
 struct intel_breadcrumbs *
@@ -315,13 +320,7 @@ void __intel_breadcrumbs_park(struct intel_breadcrumbs *b)
 		return;
 
 	/* Kick the work once more to drain the signalers, and disarm the irq */
-	irq_work_sync(&b->irq_work);
-	while (READ_ONCE(b->irq_armed) && !atomic_read(&b->active)) {
-		local_irq_disable();
-		signal_irq_work(&b->irq_work);
-		local_irq_enable();
-		cond_resched();
-	}
+	irq_work_queue(&b->irq_work);
 }
 
 void intel_breadcrumbs_free(struct kref *kref)
@@ -404,7 +403,7 @@ static void insert_breadcrumb(struct i915_request *rq)
 	 * the request as it may have completed and raised the interrupt as
 	 * we were attaching it into the lists.
 	 */
-	if (!b->irq_armed || __i915_request_is_complete(rq))
+	if (!READ_ONCE(b->irq_armed) || __i915_request_is_complete(rq))
 		irq_work_queue(&b->irq_work);
 }
 
diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h
index 7eccbd70d89f..ed95a7b57cbb 100644
--- a/drivers/gpu/drm/i915/gt/intel_context_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_context_types.h
@@ -130,6 +130,7 @@ struct intel_context {
 #define CONTEXT_PERMA_PIN		11
 #define CONTEXT_IS_PARKING		12
 #define CONTEXT_EXITING			13
+#define CONTEXT_LOW_LATENCY		14
 
 	struct {
 		u64 timeout_us;
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index 7a6dc371c384..3b740ca25000 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -497,9 +497,8 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id,
 	engine->logical_mask = BIT(logical_instance);
 	__sprint_engine_name(engine);
 
-	if ((engine->class == COMPUTE_CLASS && !RCS_MASK(engine->gt) &&
-	     __ffs(CCS_MASK(engine->gt)) == engine->instance) ||
-	     engine->class == RENDER_CLASS)
+	if ((engine->class == COMPUTE_CLASS || engine->class == RENDER_CLASS) &&
+	    __ffs(CCS_MASK(engine->gt) | RCS_MASK(engine->gt)) == engine->instance)
 		engine->flags |= I915_ENGINE_FIRST_RENDER_COMPUTE;
 
 	/* features common between engines sharing EUs */
@@ -589,7 +588,7 @@ u64 intel_clamp_preempt_timeout_ms(struct intel_engine_cs *engine, u64 value)
 	 * NB: The GuC API only supports 32bit values. However, the limit is further
 	 * reduced due to internal calculations which would otherwise overflow.
 	 */
-	if (intel_guc_submission_is_wanted(&engine->gt->uc.guc))
+	if (intel_guc_submission_is_wanted(gt_to_guc(engine->gt)))
 		value = min_t(u64, value, guc_policy_max_preempt_timeout_ms());
 
 	value = min_t(u64, value, jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT));
@@ -610,7 +609,7 @@ u64 intel_clamp_timeslice_duration_ms(struct intel_engine_cs *engine, u64 value)
 	 * NB: The GuC API only supports 32bit values. However, the limit is further
 	 * reduced due to internal calculations which would otherwise overflow.
 	 */
-	if (intel_guc_submission_is_wanted(&engine->gt->uc.guc))
+	if (intel_guc_submission_is_wanted(gt_to_guc(engine->gt)))
 		value = min_t(u64, value, guc_policy_max_exec_quantum_ms());
 
 	value = min_t(u64, value, jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT));
@@ -679,7 +678,7 @@ void intel_engines_release(struct intel_gt *gt)
 	 */
 	GEM_BUG_ON(intel_gt_pm_is_awake(gt));
 	if (!INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
-		__intel_gt_reset(gt, ALL_ENGINES);
+		intel_gt_reset_all_engines(gt);
 
 	/* Decouple the backend; but keep the layout for late GPU resets */
 	for_each_engine(engine, gt, id) {
@@ -765,14 +764,14 @@ static void engine_mask_apply_media_fuses(struct intel_gt *gt)
 	 * and bits have disable semantices.
 	 */
 	media_fuse = intel_uncore_read(gt->uncore, GEN11_GT_VEBOX_VDBOX_DISABLE);
-	if (MEDIA_VER_FULL(i915) < IP_VER(12, 50))
+	if (MEDIA_VER_FULL(i915) < IP_VER(12, 55))
 		media_fuse = ~media_fuse;
 
 	vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK;
 	vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >>
 		      GEN11_GT_VEBOX_DISABLE_SHIFT;
 
-	if (MEDIA_VER_FULL(i915) >= IP_VER(12, 50)) {
+	if (MEDIA_VER_FULL(i915) >= IP_VER(12, 55)) {
 		fuse1 = intel_uncore_read(gt->uncore, HSW_PAVP_FUSE1);
 		gt->info.sfc_mask = REG_FIELD_GET(XEHP_SFC_ENABLE_MASK, fuse1);
 	} else {
@@ -839,38 +838,6 @@ static void engine_mask_apply_compute_fuses(struct intel_gt *gt)
 	}
 }
 
-static void engine_mask_apply_copy_fuses(struct intel_gt *gt)
-{
-	struct drm_i915_private *i915 = gt->i915;
-	struct intel_gt_info *info = &gt->info;
-	unsigned long meml3_mask;
-	unsigned long quad;
-
-	if (!(GRAPHICS_VER_FULL(i915) >= IP_VER(12, 60) &&
-	      GRAPHICS_VER_FULL(i915) < IP_VER(12, 70)))
-		return;
-
-	meml3_mask = intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3);
-	meml3_mask = REG_FIELD_GET(GEN12_MEML3_EN_MASK, meml3_mask);
-
-	/*
-	 * Link Copy engines may be fused off according to meml3_mask. Each
-	 * bit is a quad that houses 2 Link Copy and two Sub Copy engines.
-	 */
-	for_each_clear_bit(quad, &meml3_mask, GEN12_MAX_MSLICES) {
-		unsigned int instance = quad * 2 + 1;
-		intel_engine_mask_t mask = GENMASK(_BCS(instance + 1),
-						   _BCS(instance));
-
-		if (mask & info->engine_mask) {
-			gt_dbg(gt, "bcs%u fused off\n", instance);
-			gt_dbg(gt, "bcs%u fused off\n", instance + 1);
-
-			info->engine_mask &= ~mask;
-		}
-	}
-}
-
 /*
  * Determine which engines are fused off in our particular hardware.
  * Note that we have a catch-22 situation where we need to be able to access
@@ -889,7 +856,6 @@ static intel_engine_mask_t init_engine_mask(struct intel_gt *gt)
 
 	engine_mask_apply_media_fuses(gt);
 	engine_mask_apply_compute_fuses(gt);
-	engine_mask_apply_copy_fuses(gt);
 
 	/*
 	 * The only use of the GSC CS is to load and communicate with the GSC
@@ -919,6 +885,12 @@ static intel_engine_mask_t init_engine_mask(struct intel_gt *gt)
 	if (IS_DG2(gt->i915)) {
 		u8 first_ccs = __ffs(CCS_MASK(gt));
 
+		/*
+		 * Store the number of active cslices before
+		 * changing the CCS engine configuration
+		 */
+		gt->ccs.cslices = CCS_MASK(gt);
+
 		/* Mask off all the CCS engine */
 		info->engine_mask &= ~GENMASK(CCS3, CCS0);
 		/* Put back in the first CCS engine */
@@ -1210,7 +1182,6 @@ static int intel_engine_init_tlb_invalidation(struct intel_engine_cs *engine)
 		if (GRAPHICS_VER_FULL(i915) == IP_VER(12, 74) ||
 		    GRAPHICS_VER_FULL(i915) == IP_VER(12, 71) ||
 		    GRAPHICS_VER_FULL(i915) == IP_VER(12, 70) ||
-		    GRAPHICS_VER_FULL(i915) == IP_VER(12, 50) ||
 		    GRAPHICS_VER_FULL(i915) == IP_VER(12, 55)) {
 			regs = xehp_regs;
 			num = ARRAY_SIZE(xehp_regs);
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 960e6be2042f..ba55c059063d 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -586,7 +586,7 @@ struct intel_engine_cs {
 #define I915_ENGINE_HAS_RCS_REG_STATE  BIT(9)
 #define I915_ENGINE_HAS_EU_PRIORITY    BIT(10)
 #define I915_ENGINE_FIRST_RENDER_COMPUTE BIT(11)
-#define I915_ENGINE_USES_WA_HOLD_CCS_SWITCHOUT BIT(12)
+#define I915_ENGINE_USES_WA_HOLD_SWITCHOUT BIT(12)
 	unsigned int flags;
 
 	/*
@@ -696,10 +696,12 @@ intel_engine_has_relative_mmio(const struct intel_engine_cs * const engine)
 }
 
 /* Wa_14014475959:dg2 */
+/* Wa_16019325821 */
+/* Wa_14019159160 */
 static inline bool
-intel_engine_uses_wa_hold_ccs_switchout(struct intel_engine_cs *engine)
+intel_engine_uses_wa_hold_switchout(struct intel_engine_cs *engine)
 {
-	return engine->flags & I915_ENGINE_USES_WA_HOLD_CCS_SWITCHOUT;
+	return engine->flags & I915_ENGINE_USES_WA_HOLD_SWITCHOUT;
 }
 
 #endif /* __INTEL_ENGINE_TYPES_H__ */
diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
index b061a0a0d6b0..21829439e686 100644
--- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
@@ -493,7 +493,7 @@ __execlists_schedule_in(struct i915_request *rq)
 		/* Use a fixed tag for OA and friends */
 		GEM_BUG_ON(ce->tag <= BITS_PER_LONG);
 		ce->lrc.ccid = ce->tag;
-	} else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) {
+	} else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) {
 		/* We don't need a strict matching tag, just different values */
 		unsigned int tag = ffs(READ_ONCE(engine->context_tag));
 
@@ -613,7 +613,7 @@ static void __execlists_schedule_out(struct i915_request * const rq,
 		intel_engine_add_retire(engine, ce->timeline);
 
 	ccid = ce->lrc.ccid;
-	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) {
+	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) {
 		ccid >>= XEHP_SW_CTX_ID_SHIFT - 32;
 		ccid &= XEHP_MAX_CONTEXT_HW_ID;
 	} else {
@@ -1907,7 +1907,7 @@ process_csb(struct intel_engine_cs *engine, struct i915_request **inactive)
 		ENGINE_TRACE(engine, "csb[%d]: status=0x%08x:0x%08x\n",
 			     head, upper_32_bits(csb), lower_32_bits(csb));
 
-		if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
+		if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
 			promote = xehp_csb_parse(csb);
 		else if (GRAPHICS_VER(engine->i915) >= 12)
 			promote = gen12_csb_parse(csb);
@@ -2898,7 +2898,7 @@ static void enable_error_interrupt(struct intel_engine_cs *engine)
 		drm_err(&engine->i915->drm,
 			"engine '%s' resumed still in error: %08x\n",
 			engine->name, status);
-		__intel_gt_reset(engine->gt, engine->mask);
+		intel_gt_reset_engine(engine);
 	}
 
 	/*
@@ -3482,7 +3482,7 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine)
 		}
 	}
 
-	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) {
+	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) {
 		if (intel_engine_has_preemption(engine))
 			engine->emit_bb_start = xehp_emit_bb_start;
 		else
@@ -3585,7 +3585,7 @@ int intel_execlists_submission_setup(struct intel_engine_cs *engine)
 
 	engine->context_tag = GENMASK(BITS_PER_LONG - 2, 0);
 	if (GRAPHICS_VER(engine->i915) >= 11 &&
-	    GRAPHICS_VER_FULL(engine->i915) < IP_VER(12, 50)) {
+	    GRAPHICS_VER_FULL(engine->i915) < IP_VER(12, 55)) {
 		execlists->ccid |= engine->instance << (GEN11_ENGINE_INSTANCE_SHIFT - 32);
 		execlists->ccid |= engine->class << (GEN11_ENGINE_CLASS_SHIFT - 32);
 	}
diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c
index ec1cbe229f0e..0d0a0dc9f610 100644
--- a/drivers/gpu/drm/i915/gt/intel_ggtt.c
+++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c
@@ -231,11 +231,8 @@ static void guc_ggtt_ct_invalidate(struct intel_gt *gt)
 	struct intel_uncore *uncore = gt->uncore;
 	intel_wakeref_t wakeref;
 
-	with_intel_runtime_pm_if_active(uncore->rpm, wakeref) {
-		struct intel_guc *guc = &gt->uc.guc;
-
-		intel_guc_invalidate_tlb_guc(guc);
-	}
+	with_intel_runtime_pm_if_active(uncore->rpm, wakeref)
+		intel_guc_invalidate_tlb_guc(gt_to_guc(gt));
 }
 
 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
@@ -246,7 +243,7 @@ static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
 	gen8_ggtt_invalidate(ggtt);
 
 	list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) {
-		if (intel_guc_tlb_invalidation_is_available(&gt->uc.guc))
+		if (intel_guc_tlb_invalidation_is_available(gt_to_guc(gt)))
 			guc_ggtt_ct_invalidate(gt);
 		else if (GRAPHICS_VER(i915) >= 12)
 			intel_uncore_write_fw(gt->uncore,
diff --git a/drivers/gpu/drm/i915/gt/intel_gsc.c b/drivers/gpu/drm/i915/gt/intel_gsc.c
index 6d440de8ba01..1e925c75fb08 100644
--- a/drivers/gpu/drm/i915/gt/intel_gsc.c
+++ b/drivers/gpu/drm/i915/gt/intel_gsc.c
@@ -103,19 +103,6 @@ static const struct gsc_def gsc_def_dg1[] = {
 	}
 };
 
-static const struct gsc_def gsc_def_xehpsdv[] = {
-	{
-		/* HECI1 not enabled on the device. */
-	},
-	{
-		.name = "mei-gscfi",
-		.bar = DG1_GSC_HECI2_BASE,
-		.bar_size = GSC_BAR_LENGTH,
-		.use_polling = true,
-		.slow_firmware = true,
-	}
-};
-
 static const struct gsc_def gsc_def_dg2[] = {
 	{
 		.name = "mei-gsc",
@@ -188,8 +175,6 @@ static void gsc_init_one(struct drm_i915_private *i915, struct intel_gsc *gsc,
 
 	if (IS_DG1(i915)) {
 		def = &gsc_def_dg1[intf_id];
-	} else if (IS_XEHPSDV(i915)) {
-		def = &gsc_def_xehpsdv[intf_id];
 	} else if (IS_DG2(i915)) {
 		def = &gsc_def_dg2[intf_id];
 	} else {
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
index 6a2c2718bcc3..626b166e67ef 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt.c
@@ -278,7 +278,7 @@ intel_gt_clear_error_registers(struct intel_gt *gt,
 		intel_uncore_posting_read(uncore,
 					  XELPMP_RING_FAULT_REG);
 
-	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
+	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
 		intel_gt_mcr_multicast_rmw(gt, XEHP_RING_FAULT_REG,
 					   RING_FAULT_VALID, 0);
 		intel_gt_mcr_read_any(gt, XEHP_RING_FAULT_REG);
@@ -403,7 +403,7 @@ void intel_gt_check_and_clear_faults(struct intel_gt *gt)
 	struct drm_i915_private *i915 = gt->i915;
 
 	/* From GEN8 onwards we only have one 'All Engine Fault Register' */
-	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
+	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
 		xehp_check_faults(gt);
 	else if (GRAPHICS_VER(i915) >= 8)
 		gen8_check_faults(gt);
@@ -832,7 +832,7 @@ void intel_gt_driver_unregister(struct intel_gt *gt)
 
 	/* Scrub all HW state upon release */
 	with_intel_runtime_pm(gt->uncore->rpm, wakeref)
-		__intel_gt_reset(gt, ALL_ENGINES);
+		intel_gt_reset_all_engines(gt);
 }
 
 void intel_gt_driver_release(struct intel_gt *gt)
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.h b/drivers/gpu/drm/i915/gt/intel_gt.h
index 003eb93b826f..b5e114d284ad 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt.h
@@ -124,6 +124,11 @@ static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc)
 	return guc_to_gt(guc)->i915;
 }
 
+static inline struct intel_guc *gt_to_guc(struct intel_gt *gt)
+{
+	return &gt->uc.guc;
+}
+
 void intel_gt_common_init_early(struct intel_gt *gt);
 int intel_root_gt_init_early(struct drm_i915_private *i915);
 int intel_gt_assign_ggtt(struct intel_gt *gt);
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.c b/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.c
index 044219c5960a..3c62a44e9106 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.c
@@ -8,18 +8,18 @@
 #include "intel_gt_ccs_mode.h"
 #include "intel_gt_regs.h"
 
-void intel_gt_apply_ccs_mode(struct intel_gt *gt)
+unsigned int intel_gt_apply_ccs_mode(struct intel_gt *gt)
 {
 	int cslice;
 	u32 mode = 0;
 	int first_ccs = __ffs(CCS_MASK(gt));
 
 	if (!IS_DG2(gt->i915))
-		return;
+		return 0;
 
 	/* Build the value for the fixed CCS load balancing */
 	for (cslice = 0; cslice < I915_MAX_CCS; cslice++) {
-		if (CCS_MASK(gt) & BIT(cslice))
+		if (gt->ccs.cslices & BIT(cslice))
 			/*
 			 * If available, assign the cslice
 			 * to the first available engine...
@@ -35,5 +35,5 @@ void intel_gt_apply_ccs_mode(struct intel_gt *gt)
 						     XEHP_CCS_MODE_CSLICE_MASK);
 	}
 
-	intel_uncore_write(gt->uncore, XEHP_CCS_MODE, mode);
+	return mode;
 }
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.h b/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.h
index 9e5549caeb26..55547f2ff426 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_ccs_mode.h
@@ -8,6 +8,6 @@
 
 struct intel_gt;
 
-void intel_gt_apply_ccs_mode(struct intel_gt *gt);
+unsigned int intel_gt_apply_ccs_mode(struct intel_gt *gt);
 
 #endif /* __INTEL_GT_CCS_MODE_H__ */
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c
index 77fb57223465..ad4c51f18d3a 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c
@@ -68,9 +68,9 @@ gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
 	struct intel_gt *media_gt = gt->i915->media_gt;
 
 	if (instance == OTHER_GUC_INSTANCE)
-		return guc_irq_handler(&gt->uc.guc, iir);
+		return guc_irq_handler(gt_to_guc(gt), iir);
 	if (instance == OTHER_MEDIA_GUC_INSTANCE && media_gt)
-		return guc_irq_handler(&media_gt->uc.guc, iir);
+		return guc_irq_handler(gt_to_guc(media_gt), iir);
 
 	if (instance == OTHER_GTPM_INSTANCE)
 		return gen11_rps_irq_handler(&gt->rps, iir);
@@ -442,7 +442,7 @@ void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl)
 		iir = raw_reg_read(regs, GEN8_GT_IIR(2));
 		if (likely(iir)) {
 			gen6_rps_irq_handler(&gt->rps, iir);
-			guc_irq_handler(&gt->uc.guc, iir >> 16);
+			guc_irq_handler(gt_to_guc(gt), iir >> 16);
 			raw_reg_write(regs, GEN8_GT_IIR(2), iir);
 		}
 	}
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_mcr.c b/drivers/gpu/drm/i915/gt/intel_gt_mcr.c
index e253750a51c5..b8912bd6c08e 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_mcr.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_mcr.c
@@ -57,51 +57,18 @@ static const struct intel_mmio_range icl_l3bank_steering_table[] = {
  * are of a "GAM" subclass that has special rules.  Thus we use a separate
  * GAM table farther down for those.
  */
-static const struct intel_mmio_range xehpsdv_mslice_steering_table[] = {
+static const struct intel_mmio_range dg2_mslice_steering_table[] = {
 	{ 0x00DD00, 0x00DDFF },
 	{ 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
 	{},
 };
 
-static const struct intel_mmio_range xehpsdv_gam_steering_table[] = {
-	{ 0x004000, 0x004AFF },
-	{ 0x00C800, 0x00CFFF },
-	{},
-};
-
-static const struct intel_mmio_range xehpsdv_lncf_steering_table[] = {
-	{ 0x00B000, 0x00B0FF },
-	{ 0x00D800, 0x00D8FF },
-	{},
-};
-
 static const struct intel_mmio_range dg2_lncf_steering_table[] = {
 	{ 0x00B000, 0x00B0FF },
 	{ 0x00D880, 0x00D8FF },
 	{},
 };
 
-/*
- * We have several types of MCR registers on PVC where steering to (0,0)
- * will always provide us with a non-terminated value.  We'll stick them
- * all in the same table for simplicity.
- */
-static const struct intel_mmio_range pvc_instance0_steering_table[] = {
-	{ 0x004000, 0x004AFF },		/* HALF-BSLICE */
-	{ 0x008800, 0x00887F },		/* CC */
-	{ 0x008A80, 0x008AFF },		/* TILEPSMI */
-	{ 0x00B000, 0x00B0FF },		/* HALF-BSLICE */
-	{ 0x00B100, 0x00B3FF },		/* L3BANK */
-	{ 0x00C800, 0x00CFFF },		/* HALF-BSLICE */
-	{ 0x00D800, 0x00D8FF },		/* HALF-BSLICE */
-	{ 0x00DD00, 0x00DDFF },		/* BSLICE */
-	{ 0x00E900, 0x00E9FF },		/* HALF-BSLICE */
-	{ 0x00EC00, 0x00EEFF },		/* HALF-BSLICE */
-	{ 0x00F000, 0x00FFFF },		/* HALF-BSLICE */
-	{ 0x024180, 0x0241FF },		/* HALF-BSLICE */
-	{},
-};
-
 static const struct intel_mmio_range xelpg_instance0_steering_table[] = {
 	{ 0x000B00, 0x000BFF },         /* SQIDI */
 	{ 0x001000, 0x001FFF },         /* SQIDI */
@@ -185,22 +152,16 @@ void intel_gt_mcr_init(struct intel_gt *gt)
 		gt->steering_table[INSTANCE0] = xelpg_instance0_steering_table;
 		gt->steering_table[L3BANK] = xelpg_l3bank_steering_table;
 		gt->steering_table[DSS] = xelpg_dss_steering_table;
-	} else if (IS_PONTEVECCHIO(i915)) {
-		gt->steering_table[INSTANCE0] = pvc_instance0_steering_table;
 	} else if (IS_DG2(i915)) {
-		gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table;
+		gt->steering_table[MSLICE] = dg2_mslice_steering_table;
 		gt->steering_table[LNCF] = dg2_lncf_steering_table;
 		/*
 		 * No need to hook up the GAM table since it has a dedicated
 		 * steering control register on DG2 and can use implicit
 		 * steering.
 		 */
-	} else if (IS_XEHPSDV(i915)) {
-		gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table;
-		gt->steering_table[LNCF] = xehpsdv_lncf_steering_table;
-		gt->steering_table[GAM] = xehpsdv_gam_steering_table;
 	} else if (GRAPHICS_VER(i915) >= 11 &&
-		   GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) {
+		   GRAPHICS_VER_FULL(i915) < IP_VER(12, 55)) {
 		gt->steering_table[L3BANK] = icl_l3bank_steering_table;
 		gt->info.l3bank_mask =
 			~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &
@@ -821,8 +782,6 @@ void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt,
 		for (int i = 0; i < NUM_STEERING_TYPES; i++)
 			if (gt->steering_table[i])
 				report_steering_type(p, gt, i, dump_table);
-	} else if (IS_PONTEVECCHIO(gt->i915)) {
-		report_steering_type(p, gt, INSTANCE0, dump_table);
 	} else if (HAS_MSLICE_STEERING(gt->i915)) {
 		report_steering_type(p, gt, MSLICE, dump_table);
 		report_steering_type(p, gt, LNCF, dump_table);
@@ -842,10 +801,7 @@ void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt,
 void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss,
 				   unsigned int *group, unsigned int *instance)
 {
-	if (IS_PONTEVECCHIO(gt->i915)) {
-		*group = dss / GEN_DSS_PER_CSLICE;
-		*instance = dss % GEN_DSS_PER_CSLICE;
-	} else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) {
+	if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 55)) {
 		*group = dss / GEN_DSS_PER_GSLICE;
 		*instance = dss % GEN_DSS_PER_GSLICE;
 	} else {
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_mcr.h b/drivers/gpu/drm/i915/gt/intel_gt_mcr.h
index 01ac565a56a4..a67a4c35a4fa 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_mcr.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_mcr.h
@@ -54,7 +54,7 @@ int intel_gt_mcr_wait_for_reg(struct intel_gt *gt,
  * the topology, so we lookup the DSS ID directly in "slice 0."
  */
 #define _HAS_SS(ss_, gt_, group_, instance_) ( \
-	GRAPHICS_VER_FULL(gt_->i915) >= IP_VER(12, 50) ? \
+	GRAPHICS_VER_FULL(gt_->i915) >= IP_VER(12, 55) ? \
 		intel_sseu_has_subslice(&(gt_)->info.sseu, 0, ss_) : \
 		intel_sseu_has_subslice(&(gt_)->info.sseu, group_, instance_))
 
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
index 220ac4f92edf..c08fdb65cc69 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
@@ -159,7 +159,7 @@ static bool reset_engines(struct intel_gt *gt)
 	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
 		return false;
 
-	return __intel_gt_reset(gt, ALL_ENGINES) == 0;
+	return intel_gt_reset_all_engines(gt) == 0;
 }
 
 static void gt_sanitize(struct intel_gt *gt, bool force)
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c
index 7114c116e928..4fcba42cfe34 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c
@@ -392,10 +392,6 @@ void intel_gt_pm_frequency_dump(struct intel_gt *gt, struct drm_printer *p)
 		drm_puts(p, "no P-state info available\n");
 	}
 
-	drm_printf(p, "Current CD clock frequency: %d kHz\n", i915->display.cdclk.hw.cdclk);
-	drm_printf(p, "Max CD clock frequency: %d kHz\n", i915->display.cdclk.max_cdclk_freq);
-	drm_printf(p, "Max pixel clock frequency: %d kHz\n", i915->max_dotclk_freq);
-
 	intel_runtime_pm_put(uncore->rpm, wakeref);
 }
 
@@ -538,7 +534,7 @@ static bool rps_eval(void *data)
 {
 	struct intel_gt *gt = data;
 
-	if (intel_guc_slpc_is_used(&gt->uc.guc))
+	if (intel_guc_slpc_is_used(gt_to_guc(gt)))
 		return false;
 	else
 		return HAS_RPS(gt->i915);
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_regs.h b/drivers/gpu/drm/i915/gt/intel_gt_regs.h
index 743fe3566722..e42b3a5d4e63 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_regs.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_regs.h
@@ -718,44 +718,11 @@
 
 #define UNSLICE_UNIT_LEVEL_CLKGATE		_MMIO(0x9434)
 #define   VFUNIT_CLKGATE_DIS			REG_BIT(20)
-#define   TSGUNIT_CLKGATE_DIS			REG_BIT(17) /* XEHPSDV */
 #define   CG3DDISCFEG_CLKGATE_DIS		REG_BIT(17) /* DG2 */
 #define   GAMEDIA_CLKGATE_DIS			REG_BIT(11)
 #define   HSUNIT_CLKGATE_DIS			REG_BIT(8)
 #define   VSUNIT_CLKGATE_DIS			REG_BIT(3)
 
-#define UNSLCGCTL9440				_MMIO(0x9440)
-#define   GAMTLBOACS_CLKGATE_DIS		REG_BIT(28)
-#define   GAMTLBVDBOX5_CLKGATE_DIS		REG_BIT(27)
-#define   GAMTLBVDBOX6_CLKGATE_DIS		REG_BIT(26)
-#define   GAMTLBVDBOX3_CLKGATE_DIS		REG_BIT(24)
-#define   GAMTLBVDBOX4_CLKGATE_DIS		REG_BIT(23)
-#define   GAMTLBVDBOX7_CLKGATE_DIS		REG_BIT(22)
-#define   GAMTLBVDBOX2_CLKGATE_DIS		REG_BIT(21)
-#define   GAMTLBVDBOX0_CLKGATE_DIS		REG_BIT(17)
-#define   GAMTLBKCR_CLKGATE_DIS			REG_BIT(16)
-#define   GAMTLBGUC_CLKGATE_DIS			REG_BIT(15)
-#define   GAMTLBBLT_CLKGATE_DIS			REG_BIT(14)
-#define   GAMTLBVDBOX1_CLKGATE_DIS		REG_BIT(6)
-
-#define UNSLCGCTL9444				_MMIO(0x9444)
-#define   GAMTLBGFXA0_CLKGATE_DIS		REG_BIT(30)
-#define   GAMTLBGFXA1_CLKGATE_DIS		REG_BIT(29)
-#define   GAMTLBCOMPA0_CLKGATE_DIS		REG_BIT(28)
-#define   GAMTLBCOMPA1_CLKGATE_DIS		REG_BIT(27)
-#define   GAMTLBCOMPB0_CLKGATE_DIS		REG_BIT(26)
-#define   GAMTLBCOMPB1_CLKGATE_DIS		REG_BIT(25)
-#define   GAMTLBCOMPC0_CLKGATE_DIS		REG_BIT(24)
-#define   GAMTLBCOMPC1_CLKGATE_DIS		REG_BIT(23)
-#define   GAMTLBCOMPD0_CLKGATE_DIS		REG_BIT(22)
-#define   GAMTLBCOMPD1_CLKGATE_DIS		REG_BIT(21)
-#define   GAMTLBMERT_CLKGATE_DIS		REG_BIT(20)
-#define   GAMTLBVEBOX3_CLKGATE_DIS		REG_BIT(19)
-#define   GAMTLBVEBOX2_CLKGATE_DIS		REG_BIT(18)
-#define   GAMTLBVEBOX1_CLKGATE_DIS		REG_BIT(17)
-#define   GAMTLBVEBOX0_CLKGATE_DIS		REG_BIT(16)
-#define   LTCDD_CLKGATE_DIS			REG_BIT(10)
-
 #define GEN11_SLICE_UNIT_LEVEL_CLKGATE		_MMIO(0x94d4)
 #define XEHP_SLICE_UNIT_LEVEL_CLKGATE		MCR_REG(0x94d4)
 #define   SARBUNIT_CLKGATE_DIS			(1 << 5)
@@ -765,9 +732,6 @@
 #define   L3_CLKGATE_DIS			REG_BIT(16)
 #define   L3_CR2X_CLKGATE_DIS			REG_BIT(17)
 
-#define SCCGCTL94DC				MCR_REG(0x94dc)
-#define   CG3DDISURB				REG_BIT(14)
-
 #define UNSLICE_UNIT_LEVEL_CLKGATE2		_MMIO(0x94e4)
 #define   VSUNIT_CLKGATE_DIS_TGL		REG_BIT(19)
 #define   PSDUNIT_CLKGATE_DIS			REG_BIT(5)
@@ -989,10 +953,6 @@
 #define   GEN7_WA_FOR_GEN7_L3_CONTROL		0x3C47FF8C
 #define   GEN7_L3AGDIS				(1 << 19)
 
-#define XEHPC_LNCFMISCCFGREG0			MCR_REG(0xb01c)
-#define   XEHPC_HOSTCACHEEN			REG_BIT(1)
-#define   XEHPC_OVRLSCCC			REG_BIT(0)
-
 #define GEN7_L3CNTLREG2				_MMIO(0xb020)
 
 /* MOCS (Memory Object Control State) registers */
@@ -1046,20 +1006,9 @@
 #define XEHP_L3SQCREG5				MCR_REG(0xb158)
 #define   L3_PWM_TIMER_INIT_VAL_MASK		REG_GENMASK(9, 0)
 
-#define MLTICTXCTL				MCR_REG(0xb170)
-#define   TDONRENDER				REG_BIT(2)
-
 #define XEHP_L3SCQREG7				MCR_REG(0xb188)
 #define   BLEND_FILL_CACHING_OPT_DIS		REG_BIT(3)
 
-#define XEHPC_L3SCRUB				MCR_REG(0xb18c)
-#define   SCRUB_CL_DWNGRADE_SHARED		REG_BIT(12)
-#define   SCRUB_RATE_PER_BANK_MASK		REG_GENMASK(2, 0)
-#define   SCRUB_RATE_4B_PER_CLK			REG_FIELD_PREP(SCRUB_RATE_PER_BANK_MASK, 0x6)
-
-#define L3SQCREG1_CCS0				MCR_REG(0xb200)
-#define   FLUSHALLNONCOH			REG_BIT(5)
-
 #define GEN11_GLBLINVL				_MMIO(0xb404)
 #define   GEN11_BANK_HASH_ADDR_EXCL_MASK	(0x7f << 5)
 #define   GEN11_BANK_HASH_ADDR_EXCL_BIT0	(1 << 5)
@@ -1109,7 +1058,6 @@
 #define XEHP_COMPCTX_TLB_INV_CR			MCR_REG(0xcf04)
 #define XELPMP_GSC_TLB_INV_CR			_MMIO(0xcf04)   /* media GT only */
 
-#define XEHP_MERT_MOD_CTRL			MCR_REG(0xcf28)
 #define RENDER_MOD_CTRL				MCR_REG(0xcf2c)
 #define COMP_MOD_CTRL				MCR_REG(0xcf30)
 #define XELPMP_GSC_MOD_CTRL			_MMIO(0xcf30)	/* media GT only */
@@ -1185,7 +1133,6 @@
 #define EU_PERF_CNTL4				PERF_REG(0xe45c)
 
 #define GEN9_ROW_CHICKEN4			MCR_REG(0xe48c)
-#define   GEN12_DISABLE_GRF_CLEAR		REG_BIT(13)
 #define   XEHP_DIS_BBL_SYSPIPE			REG_BIT(11)
 #define   GEN12_DISABLE_TDL_PUSH		REG_BIT(9)
 #define   GEN11_DIS_PICK_2ND_EU			REG_BIT(7)
@@ -1202,7 +1149,6 @@
 #define   FLOW_CONTROL_ENABLE			REG_BIT(15)
 #define   UGM_BACKUP_MODE			REG_BIT(13)
 #define   MDQ_ARBITRATION_MODE			REG_BIT(12)
-#define   SYSTOLIC_DOP_CLOCK_GATING_DIS		REG_BIT(10)
 #define   PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE	REG_BIT(8)
 #define   STALL_DOP_GATING_DISABLE		REG_BIT(5)
 #define   THROTTLE_12_5				REG_GENMASK(4, 2)
@@ -1215,6 +1161,7 @@
 #define   GEN12_DISABLE_EARLY_READ		REG_BIT(14)
 #define   GEN12_ENABLE_LARGE_GRF_MODE		REG_BIT(12)
 #define   GEN12_PUSH_CONST_DEREF_HOLD_DIS	REG_BIT(8)
+#define   XELPG_DISABLE_TDL_SVHS_GATING		REG_BIT(1)
 #define   GEN12_DISABLE_DOP_GATING              REG_BIT(0)
 
 #define RT_CTRL					MCR_REG(0xe530)
@@ -1685,11 +1632,6 @@
 
 #define GEN12_SFC_DONE(n)			_MMIO(0x1cc000 + (n) * 0x1000)
 
-#define GT0_PACKAGE_ENERGY_STATUS		_MMIO(0x250004)
-#define GT0_PACKAGE_RAPL_LIMIT			_MMIO(0x250008)
-#define GT0_PACKAGE_POWER_SKU_UNIT		_MMIO(0x250068)
-#define GT0_PLATFORM_ENERGY_STATUS		_MMIO(0x25006c)
-
 /*
  * Standalone Media's non-engine GT registers are located at their regular GT
  * offsets plus 0x380000.  This extra offset is stored inside the intel_uncore
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c
index c0b202223940..d7784650e4d9 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c
@@ -442,7 +442,7 @@ static ssize_t slpc_ignore_eff_freq_show(struct kobject *kobj,
 					 char *buff)
 {
 	struct intel_gt *gt = intel_gt_sysfs_get_drvdata(kobj, attr->attr.name);
-	struct intel_guc_slpc *slpc = &gt->uc.guc.slpc;
+	struct intel_guc_slpc *slpc = &gt_to_guc(gt)->slpc;
 
 	return sysfs_emit(buff, "%u\n", slpc->ignore_eff_freq);
 }
@@ -452,7 +452,7 @@ static ssize_t slpc_ignore_eff_freq_store(struct kobject *kobj,
 					  const char *buff, size_t count)
 {
 	struct intel_gt *gt = intel_gt_sysfs_get_drvdata(kobj, attr->attr.name);
-	struct intel_guc_slpc *slpc = &gt->uc.guc.slpc;
+	struct intel_guc_slpc *slpc = &gt_to_guc(gt)->slpc;
 	int err;
 	u32 val;
 
@@ -573,7 +573,6 @@ static ssize_t media_freq_factor_show(struct kobject *kobj,
 				      char *buff)
 {
 	struct intel_gt *gt = intel_gt_sysfs_get_drvdata(kobj, attr->attr.name);
-	struct intel_guc_slpc *slpc = &gt->uc.guc.slpc;
 	intel_wakeref_t wakeref;
 	u32 mode;
 
@@ -581,20 +580,12 @@ static ssize_t media_freq_factor_show(struct kobject *kobj,
 	 * Retrieve media_ratio_mode from GEN6_RPNSWREQ bit 13 set by
 	 * GuC. GEN6_RPNSWREQ:13 value 0 represents 1:2 and 1 represents 1:1
 	 */
-	if (IS_XEHPSDV(gt->i915) &&
-	    slpc->media_ratio_mode == SLPC_MEDIA_RATIO_MODE_DYNAMIC_CONTROL) {
-		/*
-		 * For XEHPSDV dynamic mode GEN6_RPNSWREQ:13 does not contain
-		 * the media_ratio_mode, just return the cached media ratio
-		 */
-		mode = slpc->media_ratio_mode;
-	} else {
-		with_intel_runtime_pm(gt->uncore->rpm, wakeref)
-			mode = intel_uncore_read(gt->uncore, GEN6_RPNSWREQ);
-		mode = REG_FIELD_GET(GEN12_MEDIA_FREQ_RATIO, mode) ?
-			SLPC_MEDIA_RATIO_MODE_FIXED_ONE_TO_ONE :
-			SLPC_MEDIA_RATIO_MODE_FIXED_ONE_TO_TWO;
-	}
+	with_intel_runtime_pm(gt->uncore->rpm, wakeref)
+		mode = intel_uncore_read(gt->uncore, GEN6_RPNSWREQ);
+
+	mode = REG_FIELD_GET(GEN12_MEDIA_FREQ_RATIO, mode) ?
+		SLPC_MEDIA_RATIO_MODE_FIXED_ONE_TO_ONE :
+		SLPC_MEDIA_RATIO_MODE_FIXED_ONE_TO_TWO;
 
 	return sysfs_emit(buff, "%u\n", media_ratio_mode_to_factor(mode));
 }
@@ -604,7 +595,7 @@ static ssize_t media_freq_factor_store(struct kobject *kobj,
 				       const char *buff, size_t count)
 {
 	struct intel_gt *gt = intel_gt_sysfs_get_drvdata(kobj, attr->attr.name);
-	struct intel_guc_slpc *slpc = &gt->uc.guc.slpc;
+	struct intel_guc_slpc *slpc = &gt_to_guc(gt)->slpc;
 	u32 factor, mode;
 	int err;
 
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h
index def7dd0eb6f1..cfdd2ad5e954 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h
@@ -207,6 +207,14 @@ struct intel_gt {
 					    [MAX_ENGINE_INSTANCE + 1];
 	enum intel_submission_method submission_method;
 
+	struct {
+		/*
+		 * Mask of the non fused CCS slices
+		 * to be used for the load balancing
+		 */
+		intel_engine_mask_t cslices;
+	} ccs;
+
 	/*
 	 * Default address space (either GGTT or ppGTT depending on arch).
 	 *
diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c
index 7811a8c9da06..30b128b1fde7 100644
--- a/drivers/gpu/drm/i915/gt/intel_gtt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gtt.c
@@ -680,7 +680,7 @@ void setup_private_pat(struct intel_gt *gt)
 
 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
 		xelpg_setup_private_ppat(gt);
-	else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
+	else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
 		xehp_setup_private_ppat(gt);
 	else if (GRAPHICS_VER(i915) >= 12)
 		tgl_setup_private_ppat(uncore);
diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index 7c367ba8d9dc..b387146ede98 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -546,47 +546,6 @@ static const u8 gen12_rcs_offsets[] = {
 	END
 };
 
-static const u8 xehp_rcs_offsets[] = {
-	NOP(1),
-	LRI(13, POSTED),
-	REG16(0x244),
-	REG(0x034),
-	REG(0x030),
-	REG(0x038),
-	REG(0x03c),
-	REG(0x168),
-	REG(0x140),
-	REG(0x110),
-	REG(0x1c0),
-	REG(0x1c4),
-	REG(0x1c8),
-	REG(0x180),
-	REG16(0x2b4),
-
-	NOP(5),
-	LRI(9, POSTED),
-	REG16(0x3a8),
-	REG16(0x28c),
-	REG16(0x288),
-	REG16(0x284),
-	REG16(0x280),
-	REG16(0x27c),
-	REG16(0x278),
-	REG16(0x274),
-	REG16(0x270),
-
-	LRI(3, POSTED),
-	REG(0x1b0),
-	REG16(0x5a8),
-	REG16(0x5ac),
-
-	NOP(6),
-	LRI(1, 0),
-	REG(0x0c8),
-
-	END
-};
-
 static const u8 dg2_rcs_offsets[] = {
 	NOP(1),
 	LRI(15, POSTED),
@@ -695,8 +654,6 @@ static const u8 *reg_offsets(const struct intel_engine_cs *engine)
 			return mtl_rcs_offsets;
 		else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
 			return dg2_rcs_offsets;
-		else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
-			return xehp_rcs_offsets;
 		else if (GRAPHICS_VER(engine->i915) >= 12)
 			return gen12_rcs_offsets;
 		else if (GRAPHICS_VER(engine->i915) >= 11)
@@ -719,7 +676,7 @@ static const u8 *reg_offsets(const struct intel_engine_cs *engine)
 
 static int lrc_ring_mi_mode(const struct intel_engine_cs *engine)
 {
-	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
+	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
 		return 0x70;
 	else if (GRAPHICS_VER(engine->i915) >= 12)
 		return 0x60;
@@ -733,7 +690,7 @@ static int lrc_ring_mi_mode(const struct intel_engine_cs *engine)
 
 static int lrc_ring_bb_offset(const struct intel_engine_cs *engine)
 {
-	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
+	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
 		return 0x80;
 	else if (GRAPHICS_VER(engine->i915) >= 12)
 		return 0x70;
@@ -748,7 +705,7 @@ static int lrc_ring_bb_offset(const struct intel_engine_cs *engine)
 
 static int lrc_ring_gpr0(const struct intel_engine_cs *engine)
 {
-	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
+	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
 		return 0x84;
 	else if (GRAPHICS_VER(engine->i915) >= 12)
 		return 0x74;
@@ -795,7 +752,7 @@ static int lrc_ring_indirect_offset(const struct intel_engine_cs *engine)
 static int lrc_ring_cmd_buf_cctl(const struct intel_engine_cs *engine)
 {
 
-	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
+	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
 		/*
 		 * Note that the CSFE context has a dummy slot for CMD_BUF_CCTL
 		 * simply to match the RCS context image layout.
diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.c b/drivers/gpu/drm/i915/gt/intel_migrate.c
index 576e5ef0289b..6f7af4077135 100644
--- a/drivers/gpu/drm/i915/gt/intel_migrate.c
+++ b/drivers/gpu/drm/i915/gt/intel_migrate.c
@@ -35,9 +35,9 @@ static bool engine_supports_migration(struct intel_engine_cs *engine)
 	return true;
 }
 
-static void xehpsdv_toggle_pdes(struct i915_address_space *vm,
-				struct i915_page_table *pt,
-				void *data)
+static void xehp_toggle_pdes(struct i915_address_space *vm,
+			     struct i915_page_table *pt,
+			     void *data)
 {
 	struct insert_pte_data *d = data;
 
@@ -52,9 +52,9 @@ static void xehpsdv_toggle_pdes(struct i915_address_space *vm,
 	d->offset += SZ_2M;
 }
 
-static void xehpsdv_insert_pte(struct i915_address_space *vm,
-			       struct i915_page_table *pt,
-			       void *data)
+static void xehp_insert_pte(struct i915_address_space *vm,
+			    struct i915_page_table *pt,
+			    void *data)
 {
 	struct insert_pte_data *d = data;
 
@@ -120,7 +120,7 @@ static struct i915_address_space *migrate_vm(struct intel_gt *gt)
 	 * 512 entry layout using 4K GTT pages. The other two windows just map
 	 * lmem pages and must use the new compact 32 entry layout using 64K GTT
 	 * pages, which ensures we can address any lmem object that the user
-	 * throws at us. We then also use the xehpsdv_toggle_pdes as a way of
+	 * throws at us. We then also use the xehp_toggle_pdes as a way of
 	 * just toggling the PDE bit(GEN12_PDE_64K) for us, to enable the
 	 * compact layout for each of these page-tables, that fall within the
 	 * [CHUNK_SIZE, 3 * CHUNK_SIZE) range.
@@ -209,12 +209,12 @@ static struct i915_address_space *migrate_vm(struct intel_gt *gt)
 		/* Now allow the GPU to rewrite the PTE via its own ppGTT */
 		if (HAS_64K_PAGES(gt->i915)) {
 			vm->vm.foreach(&vm->vm, base, d.offset - base,
-				       xehpsdv_insert_pte, &d);
+				       xehp_insert_pte, &d);
 			d.offset = base + CHUNK_SZ;
 			vm->vm.foreach(&vm->vm,
 				       d.offset,
 				       2 * CHUNK_SZ,
-				       xehpsdv_toggle_pdes, &d);
+				       xehp_toggle_pdes, &d);
 		} else {
 			vm->vm.foreach(&vm->vm, base, d.offset - base,
 				       insert_pte, &d);
@@ -925,7 +925,7 @@ static int emit_clear(struct i915_request *rq, u32 offset, int size,
 
 	GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX);
 
-	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
+	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
 		ring_sz = XY_FAST_COLOR_BLT_DW;
 	else if (ver >= 8)
 		ring_sz = 8;
@@ -936,7 +936,7 @@ static int emit_clear(struct i915_request *rq, u32 offset, int size,
 	if (IS_ERR(cs))
 		return PTR_ERR(cs);
 
-	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
+	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
 		*cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |
 			(XY_FAST_COLOR_BLT_DW - 2);
 		*cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) |
diff --git a/drivers/gpu/drm/i915/gt/intel_mocs.c b/drivers/gpu/drm/i915/gt/intel_mocs.c
index 25c1023eb5f9..d791d63d49b4 100644
--- a/drivers/gpu/drm/i915/gt/intel_mocs.c
+++ b/drivers/gpu/drm/i915/gt/intel_mocs.c
@@ -53,7 +53,6 @@ struct drm_i915_mocs_table {
 
 /* Helper defines */
 #define GEN9_NUM_MOCS_ENTRIES	64  /* 63-64 are reserved, but configured. */
-#define PVC_NUM_MOCS_ENTRIES	3
 #define MTL_NUM_MOCS_ENTRIES	16
 
 /* (e)LLC caching options */
@@ -367,31 +366,6 @@ static const struct drm_i915_mocs_entry gen12_mocs_table[] = {
 		   L3_3_WB),
 };
 
-static const struct drm_i915_mocs_entry xehpsdv_mocs_table[] = {
-	/* wa_1608975824 */
-	MOCS_ENTRY(0, 0, L3_3_WB | L3_LKUP(1)),
-
-	/* UC - Coherent; GO:L3 */
-	MOCS_ENTRY(1, 0, L3_1_UC | L3_LKUP(1)),
-	/* UC - Coherent; GO:Memory */
-	MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)),
-	/* UC - Non-Coherent; GO:Memory */
-	MOCS_ENTRY(3, 0, L3_1_UC | L3_GLBGO(1)),
-	/* UC - Non-Coherent; GO:L3 */
-	MOCS_ENTRY(4, 0, L3_1_UC),
-
-	/* WB */
-	MOCS_ENTRY(5, 0, L3_3_WB | L3_LKUP(1)),
-
-	/* HW Reserved - SW program but never use. */
-	MOCS_ENTRY(48, 0, L3_3_WB | L3_LKUP(1)),
-	MOCS_ENTRY(49, 0, L3_1_UC | L3_LKUP(1)),
-	MOCS_ENTRY(60, 0, L3_1_UC),
-	MOCS_ENTRY(61, 0, L3_1_UC),
-	MOCS_ENTRY(62, 0, L3_1_UC),
-	MOCS_ENTRY(63, 0, L3_1_UC),
-};
-
 static const struct drm_i915_mocs_entry dg2_mocs_table[] = {
 	/* UC - Coherent; GO:L3 */
 	MOCS_ENTRY(0, 0, L3_1_UC | L3_LKUP(1)),
@@ -404,17 +378,6 @@ static const struct drm_i915_mocs_entry dg2_mocs_table[] = {
 	MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)),
 };
 
-static const struct drm_i915_mocs_entry pvc_mocs_table[] = {
-	/* Error */
-	MOCS_ENTRY(0, 0, L3_3_WB),
-
-	/* UC */
-	MOCS_ENTRY(1, 0, L3_1_UC),
-
-	/* WB */
-	MOCS_ENTRY(2, 0, L3_3_WB),
-};
-
 static const struct drm_i915_mocs_entry mtl_mocs_table[] = {
 	/* Error - Reserved for Non-Use */
 	MOCS_ENTRY(0,
@@ -501,25 +464,12 @@ static unsigned int get_mocs_settings(struct drm_i915_private *i915,
 		table->n_entries = MTL_NUM_MOCS_ENTRIES;
 		table->uc_index = 9;
 		table->unused_entries_index = 1;
-	} else if (IS_PONTEVECCHIO(i915)) {
-		table->size = ARRAY_SIZE(pvc_mocs_table);
-		table->table = pvc_mocs_table;
-		table->n_entries = PVC_NUM_MOCS_ENTRIES;
-		table->uc_index = 1;
-		table->wb_index = 2;
-		table->unused_entries_index = 2;
 	} else if (IS_DG2(i915)) {
 		table->size = ARRAY_SIZE(dg2_mocs_table);
 		table->table = dg2_mocs_table;
 		table->uc_index = 1;
 		table->n_entries = GEN9_NUM_MOCS_ENTRIES;
 		table->unused_entries_index = 3;
-	} else if (IS_XEHPSDV(i915)) {
-		table->size = ARRAY_SIZE(xehpsdv_mocs_table);
-		table->table = xehpsdv_mocs_table;
-		table->uc_index = 2;
-		table->n_entries = GEN9_NUM_MOCS_ENTRIES;
-		table->unused_entries_index = 5;
 	} else if (IS_DG1(i915)) {
 		table->size = ARRAY_SIZE(dg1_mocs_table);
 		table->table = dg1_mocs_table;
@@ -670,7 +620,7 @@ static void init_l3cc_table(struct intel_gt *gt,
 
 	intel_gt_mcr_lock(gt, &flags);
 	for_each_l3cc(l3cc, table, i)
-		if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50))
+		if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 55))
 			intel_gt_mcr_multicast_write_fw(gt, XEHP_LNCFCMOCS(i), l3cc);
 		else
 			intel_uncore_write_fw(gt->uncore, GEN9_LNCFCMOCS(i), l3cc);
diff --git a/drivers/gpu/drm/i915/gt/intel_rc6.c b/drivers/gpu/drm/i915/gt/intel_rc6.c
index 8f4b3c8af09c..c864d101faf9 100644
--- a/drivers/gpu/drm/i915/gt/intel_rc6.c
+++ b/drivers/gpu/drm/i915/gt/intel_rc6.c
@@ -109,7 +109,7 @@ static void gen11_rc6_enable(struct intel_rc6 *rc6)
 	 * thus allowing GuC to control RC6 entry/exit fully instead.
 	 * We will not set the HW ENABLE and EI bits
 	 */
-	if (!intel_guc_rc_enable(&gt->uc.guc))
+	if (!intel_guc_rc_enable(gt_to_guc(gt)))
 		rc6->ctl_enable = GEN6_RC_CTL_RC6_ENABLE;
 	else
 		rc6->ctl_enable =
@@ -569,7 +569,7 @@ static void __intel_rc6_disable(struct intel_rc6 *rc6)
 	struct intel_gt *gt = rc6_to_gt(rc6);
 
 	/* Take control of RC6 back from GuC */
-	intel_guc_rc_disable(&gt->uc.guc);
+	intel_guc_rc_disable(gt_to_guc(gt));
 
 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
 	if (GRAPHICS_VER(i915) >= 9)
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index c8e9aa41fdea..6161f7a3ff70 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -764,7 +764,7 @@ wa_14015076503_end(struct intel_gt *gt, intel_engine_mask_t engine_mask)
 			 HECI_H_GS1_ER_PREP, 0);
 }
 
-int __intel_gt_reset(struct intel_gt *gt, intel_engine_mask_t engine_mask)
+static int __intel_gt_reset(struct intel_gt *gt, intel_engine_mask_t engine_mask)
 {
 	const int retries = engine_mask == ALL_ENGINES ? RESET_MAX_RETRIES : 1;
 	reset_func reset;
@@ -879,8 +879,17 @@ static intel_engine_mask_t reset_prepare(struct intel_gt *gt)
 	intel_engine_mask_t awake = 0;
 	enum intel_engine_id id;
 
-	/* For GuC mode, ensure submission is disabled before stopping ring */
-	intel_uc_reset_prepare(&gt->uc);
+	/**
+	 * For GuC mode with submission enabled, ensure submission
+	 * is disabled before stopping ring.
+	 *
+	 * For GuC mode with submission disabled, ensure that GuC is not
+	 * sanitized, do that after engine reset. reset_prepare()
+	 * is followed by engine reset which in this mode requires GuC to
+	 * process any CSB FIFO entries generated by the resets.
+	 */
+	if (intel_uc_uses_guc_submission(&gt->uc))
+		intel_uc_reset_prepare(&gt->uc);
 
 	for_each_engine(engine, gt, id) {
 		if (intel_engine_pm_get_if_awake(engine))
@@ -978,7 +987,7 @@ static void __intel_gt_set_wedged(struct intel_gt *gt)
 
 	/* Even if the GPU reset fails, it should still stop the engines */
 	if (!INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
-		__intel_gt_reset(gt, ALL_ENGINES);
+		intel_gt_reset_all_engines(gt);
 
 	for_each_engine(engine, gt, id)
 		engine->submit_request = nop_submit_request;
@@ -1089,7 +1098,7 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt)
 	/* We must reset pending GPU events before restoring our submission */
 	ok = !HAS_EXECLISTS(gt->i915); /* XXX better agnosticism desired */
 	if (!INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
-		ok = __intel_gt_reset(gt, ALL_ENGINES) == 0;
+		ok = intel_gt_reset_all_engines(gt) == 0;
 	if (!ok) {
 		/*
 		 * Warn CI about the unrecoverable wedged condition.
@@ -1133,10 +1142,10 @@ static int do_reset(struct intel_gt *gt, intel_engine_mask_t stalled_mask)
 {
 	int err, i;
 
-	err = __intel_gt_reset(gt, ALL_ENGINES);
+	err = intel_gt_reset_all_engines(gt);
 	for (i = 0; err && i < RESET_MAX_RETRIES; i++) {
 		msleep(10 * (i + 1));
-		err = __intel_gt_reset(gt, ALL_ENGINES);
+		err = intel_gt_reset_all_engines(gt);
 	}
 	if (err)
 		return err;
@@ -1227,6 +1236,9 @@ void intel_gt_reset(struct intel_gt *gt,
 
 	intel_overlay_reset(gt->i915);
 
+	/* sanitize uC after engine reset */
+	if (!intel_uc_uses_guc_submission(&gt->uc))
+		intel_uc_reset_prepare(&gt->uc);
 	/*
 	 * Next we need to restore the context, but we don't use those
 	 * yet either...
@@ -1270,7 +1282,30 @@ error:
 	goto finish;
 }
 
-static int intel_gt_reset_engine(struct intel_engine_cs *engine)
+/**
+ * intel_gt_reset_all_engines() - Reset all engines in the given gt.
+ * @gt: the GT to reset all engines for.
+ *
+ * This function resets all engines within the given gt.
+ *
+ * Returns:
+ * Zero on success, negative error code on failure.
+ */
+int intel_gt_reset_all_engines(struct intel_gt *gt)
+{
+	return __intel_gt_reset(gt, ALL_ENGINES);
+}
+
+/**
+ * intel_gt_reset_engine() - Reset a specific engine within a gt.
+ * @engine: engine to be reset.
+ *
+ * This function resets the specified engine within a gt.
+ *
+ * Returns:
+ * Zero on success, negative error code on failure.
+ */
+int intel_gt_reset_engine(struct intel_engine_cs *engine)
 {
 	return __intel_gt_reset(engine->gt, engine->mask);
 }
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.h b/drivers/gpu/drm/i915/gt/intel_reset.h
index f615b30b81c5..c00de353075c 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.h
+++ b/drivers/gpu/drm/i915/gt/intel_reset.h
@@ -54,7 +54,8 @@ int intel_gt_terminally_wedged(struct intel_gt *gt);
 void intel_gt_set_wedged_on_init(struct intel_gt *gt);
 void intel_gt_set_wedged_on_fini(struct intel_gt *gt);
 
-int __intel_gt_reset(struct intel_gt *gt, intel_engine_mask_t engine_mask);
+int intel_gt_reset_engine(struct intel_engine_cs *engine);
+int intel_gt_reset_all_engines(struct intel_gt *gt);
 
 int intel_reset_guc(struct intel_gt *gt);
 
diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c
index 4feef874e6d6..c9cb2a391942 100644
--- a/drivers/gpu/drm/i915/gt/intel_rps.c
+++ b/drivers/gpu/drm/i915/gt/intel_rps.c
@@ -52,7 +52,7 @@ static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
 {
 	struct intel_gt *gt = rps_to_gt(rps);
 
-	return &gt->uc.guc.slpc;
+	return &gt_to_guc(gt)->slpc;
 }
 
 static bool rps_uses_slpc(struct intel_rps *rps)
@@ -1013,6 +1013,10 @@ void intel_rps_boost(struct i915_request *rq)
 	if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
 		return;
 
+	/* Waitboost is not needed for contexts marked with a Freq hint */
+	if (test_bit(CONTEXT_LOW_LATENCY, &rq->context->flags))
+		return;
+
 	/* Serializes with i915_request_retire() */
 	if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {
 		struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
@@ -1086,11 +1090,7 @@ static u32 intel_rps_read_state_cap(struct intel_rps *rps)
 	struct drm_i915_private *i915 = rps_to_i915(rps);
 	struct intel_uncore *uncore = rps_to_uncore(rps);
 
-	if (IS_PONTEVECCHIO(i915))
-		return intel_uncore_read(uncore, PVC_RP_STATE_CAP);
-	else if (IS_XEHPSDV(i915))
-		return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP);
-	else if (IS_GEN9_LP(i915))
+	if (IS_GEN9_LP(i915))
 		return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
 	else
 		return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
diff --git a/drivers/gpu/drm/i915/gt/intel_sseu.c b/drivers/gpu/drm/i915/gt/intel_sseu.c
index 6a3246240e81..c8fadf58d836 100644
--- a/drivers/gpu/drm/i915/gt/intel_sseu.c
+++ b/drivers/gpu/drm/i915/gt/intel_sseu.c
@@ -214,13 +214,8 @@ static void xehp_sseu_info_init(struct intel_gt *gt)
 	int num_compute_regs, num_geometry_regs;
 	int eu;
 
-	if (IS_PONTEVECCHIO(gt->i915)) {
-		num_geometry_regs = 0;
-		num_compute_regs = 2;
-	} else {
-		num_geometry_regs = 1;
-		num_compute_regs = 1;
-	}
+	num_geometry_regs = 1;
+	num_compute_regs = 1;
 
 	/*
 	 * The concept of slice has been removed in Xe_HP.  To be compatible
@@ -642,7 +637,7 @@ void intel_sseu_info_init(struct intel_gt *gt)
 {
 	struct drm_i915_private *i915 = gt->i915;
 
-	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
+	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
 		xehp_sseu_info_init(gt);
 	else if (GRAPHICS_VER(i915) >= 12)
 		gen12_sseu_info_init(gt);
@@ -851,7 +846,7 @@ void intel_sseu_print_topology(struct drm_i915_private *i915,
 {
 	if (sseu->max_slices == 0)
 		drm_printf(p, "Unavailable\n");
-	else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
+	else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
 		sseu_print_xehp_topology(sseu, p);
 	else
 		sseu_print_hsw_topology(sseu, p);
diff --git a/drivers/gpu/drm/i915/gt/intel_tlb.c b/drivers/gpu/drm/i915/gt/intel_tlb.c
index 4bb13d1890e3..756e9ebbc725 100644
--- a/drivers/gpu/drm/i915/gt/intel_tlb.c
+++ b/drivers/gpu/drm/i915/gt/intel_tlb.c
@@ -132,7 +132,7 @@ void intel_gt_invalidate_tlb_full(struct intel_gt *gt, u32 seqno)
 		return;
 
 	with_intel_gt_pm_if_awake(gt, wakeref) {
-		struct intel_guc *guc = &gt->uc.guc;
+		struct intel_guc *guc = gt_to_guc(gt);
 
 		mutex_lock(&gt->tlb.invalidate_lock);
 		if (tlb_seqno_passed(gt, seqno))
diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c
index 6ec3582c9735..5a0f1b279a80 100644
--- a/drivers/gpu/drm/i915/gt/intel_workarounds.c
+++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c
@@ -17,6 +17,8 @@
 #include "intel_ring.h"
 #include "intel_workarounds.h"
 
+#include "display/intel_fbc_regs.h"
+
 /**
  * DOC: Hardware workarounds
  *
@@ -260,12 +262,6 @@ wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 }
 
 static void
-wa_mcr_write(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 set)
-{
-	wa_mcr_write_clr_set(wal, reg, ~0, set);
-}
-
-static void
 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 {
 	wa_write_clr_set(wal, reg, set, set);
@@ -920,12 +916,8 @@ __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
 
 	if (IS_GFX_GT_IP_RANGE(engine->gt, IP_VER(12, 70), IP_VER(12, 74)))
 		xelpg_ctx_workarounds_init(engine, wal);
-	else if (IS_PONTEVECCHIO(i915))
-		; /* noop; none at this time */
 	else if (IS_DG2(i915))
 		dg2_ctx_workarounds_init(engine, wal);
-	else if (IS_XEHPSDV(i915))
-		; /* noop; none at this time */
 	else if (IS_DG1(i915))
 		dg1_ctx_workarounds_init(engine, wal);
 	else if (GRAPHICS_VER(i915) == 12)
@@ -1352,9 +1344,6 @@ xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
 		gt->steering_table[MSLICE] = NULL;
 	}
 
-	if (IS_XEHPSDV(gt->i915) && slice_mask & BIT(0))
-		gt->steering_table[GAM] = NULL;
-
 	slice = __ffs(slice_mask);
 	subslice = intel_sseu_find_first_xehp_dss(sseu, GEN_DSS_PER_GSLICE, slice) %
 		GEN_DSS_PER_GSLICE;
@@ -1382,20 +1371,6 @@ xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
 }
 
 static void
-pvc_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
-{
-	unsigned int dss;
-
-	/*
-	 * Setup implicit steering for COMPUTE and DSS ranges to the first
-	 * non-fused-off DSS.  All other types of MCR registers will be
-	 * explicitly steered.
-	 */
-	dss = intel_sseu_find_first_xehp_dss(&gt->info.sseu, 0, 0);
-	__add_mcr_wa(gt, wal, dss / GEN_DSS_PER_CSLICE, dss % GEN_DSS_PER_CSLICE);
-}
-
-static void
 icl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 {
 	struct drm_i915_private *i915 = gt->i915;
@@ -1522,76 +1497,6 @@ dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 }
 
 static void
-xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
-{
-	struct drm_i915_private *i915 = gt->i915;
-
-	xehp_init_mcr(gt, wal);
-
-	/* Wa_1409757795:xehpsdv */
-	wa_mcr_write_or(wal, SCCGCTL94DC, CG3DDISURB);
-
-	/* Wa_18011725039:xehpsdv */
-	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_B0)) {
-		wa_mcr_masked_dis(wal, MLTICTXCTL, TDONRENDER);
-		wa_mcr_write_or(wal, L3SQCREG1_CCS0, FLUSHALLNONCOH);
-	}
-
-	/* Wa_16011155590:xehpsdv */
-	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
-		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
-			    TSGUNIT_CLKGATE_DIS);
-
-	/* Wa_14011780169:xehpsdv */
-	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_B0, STEP_FOREVER)) {
-		wa_write_or(wal, UNSLCGCTL9440, GAMTLBOACS_CLKGATE_DIS |
-			    GAMTLBVDBOX7_CLKGATE_DIS |
-			    GAMTLBVDBOX6_CLKGATE_DIS |
-			    GAMTLBVDBOX5_CLKGATE_DIS |
-			    GAMTLBVDBOX4_CLKGATE_DIS |
-			    GAMTLBVDBOX3_CLKGATE_DIS |
-			    GAMTLBVDBOX2_CLKGATE_DIS |
-			    GAMTLBVDBOX1_CLKGATE_DIS |
-			    GAMTLBVDBOX0_CLKGATE_DIS |
-			    GAMTLBKCR_CLKGATE_DIS |
-			    GAMTLBGUC_CLKGATE_DIS |
-			    GAMTLBBLT_CLKGATE_DIS);
-		wa_write_or(wal, UNSLCGCTL9444, GAMTLBGFXA0_CLKGATE_DIS |
-			    GAMTLBGFXA1_CLKGATE_DIS |
-			    GAMTLBCOMPA0_CLKGATE_DIS |
-			    GAMTLBCOMPA1_CLKGATE_DIS |
-			    GAMTLBCOMPB0_CLKGATE_DIS |
-			    GAMTLBCOMPB1_CLKGATE_DIS |
-			    GAMTLBCOMPC0_CLKGATE_DIS |
-			    GAMTLBCOMPC1_CLKGATE_DIS |
-			    GAMTLBCOMPD0_CLKGATE_DIS |
-			    GAMTLBCOMPD1_CLKGATE_DIS |
-			    GAMTLBMERT_CLKGATE_DIS   |
-			    GAMTLBVEBOX3_CLKGATE_DIS |
-			    GAMTLBVEBOX2_CLKGATE_DIS |
-			    GAMTLBVEBOX1_CLKGATE_DIS |
-			    GAMTLBVEBOX0_CLKGATE_DIS);
-	}
-
-	/* Wa_16012725990:xehpsdv */
-	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_FOREVER))
-		wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, VFUNIT_CLKGATE_DIS);
-
-	/* Wa_14011060649:xehpsdv */
-	wa_14011060649(gt, wal);
-
-	/* Wa_14012362059:xehpsdv */
-	wa_mcr_write_or(wal, XEHP_MERT_MOD_CTRL, FORCE_MISS_FTLB);
-
-	/* Wa_14014368820:xehpsdv */
-	wa_mcr_write_or(wal, XEHP_GAMCNTRL_CTRL,
-			INVALIDATION_BROADCAST_MODE_DIS | GLOBAL_INVALIDATION_MODE);
-
-	/* Wa_14010670810:xehpsdv */
-	wa_mcr_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
-}
-
-static void
 dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 {
 	xehp_init_mcr(gt, wal);
@@ -1634,24 +1539,6 @@ dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 }
 
 static void
-pvc_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
-{
-	pvc_init_mcr(gt, wal);
-
-	/* Wa_14015795083 */
-	wa_write_clr(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE);
-
-	/* Wa_18018781329 */
-	wa_mcr_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB);
-	wa_mcr_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB);
-	wa_mcr_write_or(wal, XEHP_VDBX_MOD_CTRL, FORCE_MISS_FTLB);
-	wa_mcr_write_or(wal, XEHP_VEBX_MOD_CTRL, FORCE_MISS_FTLB);
-
-	/* Wa_16016694945 */
-	wa_mcr_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_OVRLSCCC);
-}
-
-static void
 xelpg_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 {
 	/* Wa_14018575942 / Wa_18018781329 */
@@ -1727,12 +1614,6 @@ static void gt_tuning_settings(struct intel_gt *gt, struct i915_wa_list *wal)
 		wa_mcr_write_or(wal, XEHP_SQCM, EN_32B_ACCESS);
 	}
 
-	if (IS_PONTEVECCHIO(gt->i915)) {
-		wa_mcr_write(wal, XEHPC_L3SCRUB,
-			     SCRUB_CL_DWNGRADE_SHARED | SCRUB_RATE_4B_PER_CLK);
-		wa_mcr_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_HOSTCACHEEN);
-	}
-
 	if (IS_DG2(gt->i915)) {
 		wa_mcr_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS);
 		wa_mcr_write_or(wal, XEHP_SQCM, EN_32B_ACCESS);
@@ -1757,12 +1638,8 @@ gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal)
 
 	if (IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 74)))
 		xelpg_gt_workarounds_init(gt, wal);
-	else if (IS_PONTEVECCHIO(i915))
-		pvc_gt_workarounds_init(gt, wal);
 	else if (IS_DG2(i915))
 		dg2_gt_workarounds_init(gt, wal);
-	else if (IS_XEHPSDV(i915))
-		xehpsdv_gt_workarounds_init(gt, wal);
 	else if (IS_DG1(i915))
 		dg1_gt_workarounds_init(gt, wal);
 	else if (GRAPHICS_VER(i915) == 12)
@@ -2180,30 +2057,6 @@ static void dg2_whitelist_build(struct intel_engine_cs *engine)
 	}
 }
 
-static void blacklist_trtt(struct intel_engine_cs *engine)
-{
-	struct i915_wa_list *w = &engine->whitelist;
-
-	/*
-	 * Prevent read/write access to [0x4400, 0x4600) which covers
-	 * the TRTT range across all engines. Note that normally userspace
-	 * cannot access the other engines' trtt control, but for simplicity
-	 * we cover the entire range on each engine.
-	 */
-	whitelist_reg_ext(w, _MMIO(0x4400),
-			  RING_FORCE_TO_NONPRIV_DENY |
-			  RING_FORCE_TO_NONPRIV_RANGE_64);
-	whitelist_reg_ext(w, _MMIO(0x4500),
-			  RING_FORCE_TO_NONPRIV_DENY |
-			  RING_FORCE_TO_NONPRIV_RANGE_64);
-}
-
-static void pvc_whitelist_build(struct intel_engine_cs *engine)
-{
-	/* Wa_16014440446:pvc */
-	blacklist_trtt(engine);
-}
-
 static void xelpg_whitelist_build(struct intel_engine_cs *engine)
 {
 	struct i915_wa_list *w = &engine->whitelist;
@@ -2230,12 +2083,8 @@ void intel_engine_init_whitelist(struct intel_engine_cs *engine)
 		; /* none yet */
 	else if (IS_GFX_GT_IP_RANGE(engine->gt, IP_VER(12, 70), IP_VER(12, 74)))
 		xelpg_whitelist_build(engine);
-	else if (IS_PONTEVECCHIO(i915))
-		pvc_whitelist_build(engine);
 	else if (IS_DG2(i915))
 		dg2_whitelist_build(engine);
-	else if (IS_XEHPSDV(i915))
-		; /* none needed */
 	else if (GRAPHICS_VER(i915) == 12)
 		tgl_whitelist_build(engine);
 	else if (GRAPHICS_VER(i915) == 11)
@@ -2816,10 +2665,7 @@ xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 static void
 ccs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 {
-	if (IS_PVC_CT_STEP(engine->i915, STEP_A0, STEP_C0)) {
-		/* Wa_14014999345:pvc */
-		wa_mcr_masked_en(wal, GEN10_CACHE_MODE_SS, DISABLE_ECC);
-	}
+	/* boilerplate for any CCS engine workaround */
 }
 
 /*
@@ -2852,13 +2698,14 @@ add_render_compute_tuning_settings(struct intel_gt *gt,
 		wa_mcr_masked_field_set(wal, GEN9_ROW_CHICKEN4, THREAD_EX_ARB_MODE,
 					THREAD_EX_ARB_MODE_RR_AFTER_DEP);
 
-	if (GRAPHICS_VER(i915) == 12 && GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
+	if (GRAPHICS_VER(i915) == 12 && GRAPHICS_VER_FULL(i915) < IP_VER(12, 55))
 		wa_write_clr(wal, GEN8_GARBCNTL, GEN12_BUS_HASH_CTL_BIT_EXC);
 }
 
 static void ccs_engine_wa_mode(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 {
 	struct intel_gt *gt = engine->gt;
+	u32 mode;
 
 	if (!IS_DG2(gt->i915))
 		return;
@@ -2875,7 +2722,8 @@ static void ccs_engine_wa_mode(struct intel_engine_cs *engine, struct i915_wa_li
 	 * After having disabled automatic load balancing we need to
 	 * assign all slices to a single CCS. We will call it CCS mode 1
 	 */
-	intel_gt_apply_ccs_mode(gt);
+	mode = intel_gt_apply_ccs_mode(gt);
+	wa_masked_en(wal, XEHP_CCS_MODE, mode);
 }
 
 /*
@@ -2916,10 +2764,14 @@ general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_li
 
 	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_B0, STEP_FOREVER) ||
 	    IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_B0, STEP_FOREVER) ||
-	    IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 74), IP_VER(12, 74)))
+	    IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 74), IP_VER(12, 74))) {
 		/* Wa_14017856879 */
 		wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN3, MTL_DISABLE_FIX_FOR_EOT_FLUSH);
 
+		/* Wa_14020495402 */
+		wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2, XELPG_DISABLE_TDL_SVHS_GATING);
+	}
+
 	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
 	    IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0))
 		/*
@@ -2947,21 +2799,15 @@ general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_li
 
 	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
 	    IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0) ||
-	    IS_PONTEVECCHIO(i915) ||
 	    IS_DG2(i915)) {
 		/* Wa_22014226127 */
 		wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0, DISABLE_D8_D16_COASLESCE);
 	}
 
-	if (IS_PONTEVECCHIO(i915) || IS_DG2(i915)) {
+	if (IS_DG2(i915)) {
 		/* Wa_14015227452:dg2,pvc */
 		wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE);
 
-		/* Wa_16015675438:dg2,pvc */
-		wa_masked_en(wal, FF_SLICE_CS_CHICKEN2, GEN12_PERF_FIX_BALANCING_CFE_DISABLE);
-	}
-
-	if (IS_DG2(i915)) {
 		/*
 		 * Wa_16011620976:dg2_g11
 		 * Wa_22015475538:dg2
@@ -2997,22 +2843,6 @@ general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_li
 			   0 /* write-only, so skip validation */,
 			   true);
 	}
-
-	if (IS_XEHPSDV(i915)) {
-		/* Wa_1409954639 */
-		wa_mcr_masked_en(wal,
-				 GEN8_ROW_CHICKEN,
-				 SYSTOLIC_DOP_CLOCK_GATING_DIS);
-
-		/* Wa_1607196519 */
-		wa_mcr_masked_en(wal,
-				 GEN9_ROW_CHICKEN4,
-				 GEN12_DISABLE_GRF_CLEAR);
-
-		/* Wa_14010449647:xehpsdv */
-		wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN1,
-				 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
-	}
 }
 
 static void
@@ -3095,7 +2925,7 @@ static bool mcr_range(struct drm_i915_private *i915, u32 offset)
 	const struct i915_range *mcr_ranges;
 	int i;
 
-	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
+	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
 		mcr_ranges = mcr_ranges_xehp;
 	else if (GRAPHICS_VER(i915) >= 12)
 		mcr_ranges = mcr_ranges_gen12;
diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
index 0dd4d00ee894..9ce8ff1c04fe 100644
--- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
+++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
@@ -319,7 +319,7 @@ static int igt_hang_sanitycheck(void *arg)
 		i915_request_add(rq);
 
 		timeout = 0;
-		intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */)
+		intel_wedge_on_timeout(&w, gt, HZ / 5 /* 200ms */)
 			timeout = i915_request_wait(rq, 0,
 						    MAX_SCHEDULE_TIMEOUT);
 		if (intel_gt_is_wedged(gt))
diff --git a/drivers/gpu/drm/i915/gt/selftest_reset.c b/drivers/gpu/drm/i915/gt/selftest_reset.c
index f40de408cd3a..2cfc23c58e90 100644
--- a/drivers/gpu/drm/i915/gt/selftest_reset.c
+++ b/drivers/gpu/drm/i915/gt/selftest_reset.c
@@ -281,7 +281,7 @@ static int igt_atomic_reset(void *arg)
 		awake = reset_prepare(gt);
 		p->critical_section_begin();
 
-		err = __intel_gt_reset(gt, ALL_ENGINES);
+		err = intel_gt_reset_all_engines(gt);
 
 		p->critical_section_end();
 		reset_finish(gt, awake);
diff --git a/drivers/gpu/drm/i915/gt/selftest_slpc.c b/drivers/gpu/drm/i915/gt/selftest_slpc.c
index 302d0540295d..4ecc4ae74a54 100644
--- a/drivers/gpu/drm/i915/gt/selftest_slpc.c
+++ b/drivers/gpu/drm/i915/gt/selftest_slpc.c
@@ -53,7 +53,7 @@ static int slpc_set_max_freq(struct intel_guc_slpc *slpc, u32 freq)
 static int slpc_set_freq(struct intel_gt *gt, u32 freq)
 {
 	int err;
-	struct intel_guc_slpc *slpc = &gt->uc.guc.slpc;
+	struct intel_guc_slpc *slpc = &gt_to_guc(gt)->slpc;
 
 	err = slpc_set_max_freq(slpc, freq);
 	if (err) {
@@ -182,7 +182,7 @@ static int vary_min_freq(struct intel_guc_slpc *slpc, struct intel_rps *rps,
 
 static int slpc_power(struct intel_gt *gt, struct intel_engine_cs *engine)
 {
-	struct intel_guc_slpc *slpc = &gt->uc.guc.slpc;
+	struct intel_guc_slpc *slpc = &gt_to_guc(gt)->slpc;
 	struct {
 		u64 power;
 		int freq;
@@ -262,7 +262,7 @@ static int max_granted_freq(struct intel_guc_slpc *slpc, struct intel_rps *rps,
 
 static int run_test(struct intel_gt *gt, int test_type)
 {
-	struct intel_guc_slpc *slpc = &gt->uc.guc.slpc;
+	struct intel_guc_slpc *slpc = &gt_to_guc(gt)->slpc;
 	struct intel_rps *rps = &gt->rps;
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.c b/drivers/gpu/drm/i915/gt/shmem_utils.c
index bccc3a1200bc..1fb6ff77fd89 100644
--- a/drivers/gpu/drm/i915/gt/shmem_utils.c
+++ b/drivers/gpu/drm/i915/gt/shmem_utils.c
@@ -7,6 +7,7 @@
 #include <linux/mm.h>
 #include <linux/pagemap.h>
 #include <linux/shmem_fs.h>
+#include <linux/vmalloc.h>
 
 #include "i915_drv.h"
 #include "gem/i915_gem_object.h"
diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_slpc_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_slpc_abi.h
index 811add10c30d..c34674e797c6 100644
--- a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_slpc_abi.h
+++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_slpc_abi.h
@@ -207,6 +207,27 @@ struct slpc_shared_data {
 	u8 reserved_mode_definition[4096];
 } __packed;
 
+struct slpc_context_frequency_request {
+	u32 frequency_request:16;
+	u32 reserved:12;
+	u32 is_compute:1;
+	u32 ignore_busyness:1;
+	u32 is_minimum:1;
+	u32 is_predefined:1;
+} __packed;
+
+#define SLPC_CTX_FREQ_REQ_IS_COMPUTE		REG_BIT(28)
+
+struct slpc_optimized_strategies {
+	u32 compute:1;
+	u32 async_flip:1;
+	u32 media:1;
+	u32 vsync_flip:1;
+	u32 reserved:28;
+} __packed;
+
+#define SLPC_OPTIMIZED_STRATEGY_COMPUTE		REG_BIT(0)
+
 /**
  * DOC: SLPC H2G MESSAGE FORMAT
  *
diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h
index dabeaf4f245f..00d6402333f8 100644
--- a/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h
+++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h
@@ -36,6 +36,7 @@ enum intel_guc_load_status {
 	INTEL_GUC_LOAD_STATUS_INVALID_INIT_DATA_RANGE_START,
 	INTEL_GUC_LOAD_STATUS_MPU_DATA_INVALID                 = 0x73,
 	INTEL_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID   = 0x74,
+	INTEL_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR        = 0x75,
 	INTEL_GUC_LOAD_STATUS_INVALID_INIT_DATA_RANGE_END,
 
 	INTEL_GUC_LOAD_STATUS_READY                            = 0xF0,
diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_klvs_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_klvs_abi.h
index 58012edd4eb0..525587cfe1af 100644
--- a/drivers/gpu/drm/i915/gt/uc/abi/guc_klvs_abi.h
+++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_klvs_abi.h
@@ -29,9 +29,9 @@
  */
 
 #define GUC_KLV_LEN_MIN				1u
-#define GUC_KLV_0_KEY				(0xffff << 16)
-#define GUC_KLV_0_LEN				(0xffff << 0)
-#define GUC_KLV_n_VALUE				(0xffffffff << 0)
+#define GUC_KLV_0_KEY				(0xffffu << 16)
+#define GUC_KLV_0_LEN				(0xffffu << 0)
+#define GUC_KLV_n_VALUE				(0xffffffffu << 0)
 
 /**
  * DOC: GuC Self Config KLVs
@@ -101,4 +101,11 @@ enum {
 	GUC_CONTEXT_POLICIES_KLV_NUM_IDS = 5,
 };
 
+/*
+ * Workaround keys:
+ */
+enum {
+	GUC_WORKAROUND_KLV_SERIALIZED_RA_MODE				= 0x9001,
+};
+
 #endif /* _ABI_GUC_KLVS_ABI_H */
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.c
index e2e42b3e0d5d..3b69bc6616bd 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_fw.c
@@ -298,7 +298,7 @@ static int gsc_fw_load_prepare(struct intel_gsc_uc *gsc)
 	memcpy_toio(gsc->local_vaddr, src, gsc->fw.size);
 	memset_io(gsc->local_vaddr + gsc->fw.size, 0, gsc->local->size - gsc->fw.size);
 
-	intel_guc_write_barrier(&gt->uc.guc);
+	intel_guc_write_barrier(gt_to_guc(gt));
 
 	i915_gem_object_unpin_map(gsc->fw.obj);
 
@@ -351,7 +351,7 @@ static int gsc_fw_query_compatibility_version(struct intel_gsc_uc *gsc)
 	void *vaddr;
 	int err;
 
-	err = intel_guc_allocate_and_map_vma(&gt->uc.guc, GSC_VER_PKT_SZ * 2,
+	err = intel_guc_allocate_and_map_vma(gt_to_guc(gt), GSC_VER_PKT_SZ * 2,
 					     &vma, &vaddr);
 	if (err) {
 		gt_err(gt, "failed to allocate vma for GSC version query\n");
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_gsc_proxy.c b/drivers/gpu/drm/i915/gt/uc/intel_gsc_proxy.c
index 40817ebcca71..a7d5465655f9 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_gsc_proxy.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_gsc_proxy.c
@@ -358,7 +358,8 @@ static int proxy_channel_alloc(struct intel_gsc_uc *gsc)
 	void *vaddr;
 	int err;
 
-	err = intel_guc_allocate_and_map_vma(&gt->uc.guc, GSC_PROXY_CHANNEL_SIZE,
+	err = intel_guc_allocate_and_map_vma(gt_to_guc(gt),
+					     GSC_PROXY_CHANNEL_SIZE,
 					     &vma, &vaddr);
 	if (err)
 		return err;
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c
index 2b450c43bbd7..5e60a34692af 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c
@@ -286,7 +286,7 @@ static u32 guc_ctl_wa_flags(struct intel_guc *guc)
 
 	/* Wa_22012773006:gen11,gen12 < XeHP */
 	if (GRAPHICS_VER(gt->i915) >= 11 &&
-	    GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 50))
+	    GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 55))
 		flags |= GUC_WA_POLLCS;
 
 	/* Wa_14014475959 */
@@ -294,6 +294,11 @@ static u32 guc_ctl_wa_flags(struct intel_guc *guc)
 	    IS_DG2(gt->i915))
 		flags |= GUC_WA_HOLD_CCS_SWITCHOUT;
 
+	/* Wa_16019325821 */
+	/* Wa_14019159160 */
+	if (IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 71)))
+		flags |= GUC_WA_RCS_CCS_SWITCHOUT;
+
 	/*
 	 * Wa_14012197797
 	 * Wa_22011391025
@@ -315,15 +320,12 @@ static u32 guc_ctl_wa_flags(struct intel_guc *guc)
 	if (IS_DG2_G11(gt->i915))
 		flags |= GUC_WA_CONTEXT_ISOLATION;
 
-	/* Wa_16015675438 */
-	if (!RCS_MASK(gt))
-		flags |= GUC_WA_RCS_REGS_IN_CCS_REGS_LIST;
-
-	/* Wa_14018913170 */
-	if (GUC_FIRMWARE_VER(guc) >= MAKE_GUC_VER(70, 7, 0)) {
-		if (IS_DG2(gt->i915) || IS_METEORLAKE(gt->i915) || IS_PONTEVECCHIO(gt->i915))
-			flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6;
-	}
+	/*
+	 * Wa_14018913170: Applicable to all platforms supported by i915 so
+	 * don't bother testing for all X/Y/Z platforms explicitly.
+	 */
+	if (GUC_FIRMWARE_VER(guc) >= MAKE_GUC_VER(70, 7, 0))
+		flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6;
 
 	return flags;
 }
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h
index be70c46604b4..57b903132776 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h
@@ -204,6 +204,8 @@ struct intel_guc {
 	struct guc_mmio_reg *ads_regset;
 	/** @ads_golden_ctxt_size: size of the golden contexts in the ADS */
 	u32 ads_golden_ctxt_size;
+	/** @ads_waklv_size: size of workaround KLVs */
+	u32 ads_waklv_size;
 	/** @ads_capture_size: size of register lists in the ADS used for error capture */
 	u32 ads_capture_size;
 
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c
index f7372f736a77..c606bb5e3b7b 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c
@@ -46,6 +46,10 @@
  *      +---------------------------------------+
  *      | padding                               |
  *      +---------------------------------------+ <== 4K aligned
+ *      | w/a KLVs                              |
+ *      +---------------------------------------+
+ *      | padding                               |
+ *      +---------------------------------------+ <== 4K aligned
  *      | capture lists                         |
  *      +---------------------------------------+
  *      | padding                               |
@@ -88,6 +92,11 @@ static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc)
 	return PAGE_ALIGN(guc->ads_golden_ctxt_size);
 }
 
+static u32 guc_ads_waklv_size(struct intel_guc *guc)
+{
+	return PAGE_ALIGN(guc->ads_waklv_size);
+}
+
 static u32 guc_ads_capture_size(struct intel_guc *guc)
 {
 	return PAGE_ALIGN(guc->ads_capture_size);
@@ -113,7 +122,7 @@ static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc)
 	return PAGE_ALIGN(offset);
 }
 
-static u32 guc_ads_capture_offset(struct intel_guc *guc)
+static u32 guc_ads_waklv_offset(struct intel_guc *guc)
 {
 	u32 offset;
 
@@ -123,6 +132,16 @@ static u32 guc_ads_capture_offset(struct intel_guc *guc)
 	return PAGE_ALIGN(offset);
 }
 
+static u32 guc_ads_capture_offset(struct intel_guc *guc)
+{
+	u32 offset;
+
+	offset = guc_ads_waklv_offset(guc) +
+		 guc_ads_waklv_size(guc);
+
+	return PAGE_ALIGN(offset);
+}
+
 static u32 guc_ads_private_data_offset(struct intel_guc *guc)
 {
 	u32 offset;
@@ -393,7 +412,7 @@ static int guc_mmio_regset_init(struct temp_regset *regset,
 
 	/* add in local MOCS registers */
 	for (i = 0; i < LNCFCMOCS_REG_COUNT; i++)
-		if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
+		if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
 			ret |= GUC_MCR_REG_ADD(gt, regset, XEHP_LNCFCMOCS(i), false);
 		else
 			ret |= GUC_MMIO_REG_ADD(gt, regset, GEN9_LNCFCMOCS(i), false);
@@ -503,7 +522,7 @@ static void fill_engine_enable_masks(struct intel_gt *gt,
 
 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32))
 #define XEHP_LR_HW_CONTEXT_SIZE (96 * sizeof(u32))
-#define LR_HW_CONTEXT_SZ(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50) ? \
+#define LR_HW_CONTEXT_SZ(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55) ? \
 				    XEHP_LR_HW_CONTEXT_SIZE : \
 				    LR_HW_CONTEXT_SIZE)
 #define LRC_SKIP_SIZE(i915) (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SZ(i915))
@@ -796,6 +815,65 @@ engine_instance_list:
 	return PAGE_ALIGN(total_size);
 }
 
+/* Wa_14019159160 */
+static u32 guc_waklv_ra_mode(struct intel_guc *guc, u32 offset, u32 remain)
+{
+	u32 size;
+	u32 klv_entry[] = {
+		/* 16:16 key/length */
+		FIELD_PREP(GUC_KLV_0_KEY, GUC_WORKAROUND_KLV_SERIALIZED_RA_MODE) |
+		FIELD_PREP(GUC_KLV_0_LEN, 0),
+		/* 0 dwords data */
+	};
+
+	size = sizeof(klv_entry);
+	GEM_BUG_ON(remain < size);
+
+	iosys_map_memcpy_to(&guc->ads_map, offset, klv_entry, size);
+
+	return size;
+}
+
+static void guc_waklv_init(struct intel_guc *guc)
+{
+	struct intel_gt *gt = guc_to_gt(guc);
+	u32 offset, addr_ggtt, remain, size;
+
+	if (!intel_uc_uses_guc_submission(&gt->uc))
+		return;
+
+	if (GUC_FIRMWARE_VER(guc) < MAKE_GUC_VER(70, 10, 0))
+		return;
+
+	GEM_BUG_ON(iosys_map_is_null(&guc->ads_map));
+	offset = guc_ads_waklv_offset(guc);
+	remain = guc_ads_waklv_size(guc);
+
+	/* Wa_14019159160 */
+	if (IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 71))) {
+		size = guc_waklv_ra_mode(guc, offset, remain);
+		offset += size;
+		remain -= size;
+	}
+
+	size = guc_ads_waklv_size(guc) - remain;
+	if (!size)
+		return;
+
+	offset = guc_ads_waklv_offset(guc);
+	addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
+
+	ads_blob_write(guc, ads.wa_klv_addr_lo, addr_ggtt);
+	ads_blob_write(guc, ads.wa_klv_addr_hi, 0);
+	ads_blob_write(guc, ads.wa_klv_size, size);
+}
+
+static int guc_prep_waklv(struct intel_guc *guc)
+{
+	/* Fudge something chunky for now: */
+	return PAGE_SIZE;
+}
+
 static void __guc_ads_init(struct intel_guc *guc)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
@@ -843,6 +921,9 @@ static void __guc_ads_init(struct intel_guc *guc)
 	/* MMIO save/restore list */
 	guc_mmio_reg_state_init(guc);
 
+	/* Workaround KLV list */
+	guc_waklv_init(guc);
+
 	/* Private Data */
 	ads_blob_write(guc, ads.private_data, base +
 		       guc_ads_private_data_offset(guc));
@@ -886,6 +967,12 @@ int intel_guc_ads_create(struct intel_guc *guc)
 		return ret;
 	guc->ads_capture_size = ret;
 
+	/* And don't forget the workaround KLVs: */
+	ret = guc_prep_waklv(guc);
+	if (ret < 0)
+		return ret;
+	guc->ads_waklv_size = ret;
+
 	/* Now the total size can be determined: */
 	size = guc_ads_blob_size(guc);
 
@@ -961,7 +1048,7 @@ u32 intel_guc_engine_usage_offset(struct intel_guc *guc)
 
 struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine)
 {
-	struct intel_guc *guc = &engine->gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(engine->gt);
 	u8 guc_class = engine_class_to_guc_class(engine->class);
 	size_t offset = offsetof(struct __guc_ads_blob,
 				 engine_usage.engines[guc_class][ilog2(engine->logical_mask)]);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c
index a1cd40d80517..9547fff672bd 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c
@@ -51,6 +51,7 @@
 	{ RING_ESR(0),              0,      0, "ESR" }, \
 	{ RING_DMA_FADD(0),         0,      0, "RING_DMA_FADD_LDW" }, \
 	{ RING_DMA_FADD_UDW(0),     0,      0, "RING_DMA_FADD_UDW" }, \
+	{ RING_EIR(0),              0,      0, "EIR" }, \
 	{ RING_IPEIR(0),            0,      0, "IPEIR" }, \
 	{ RING_IPEHR(0),            0,      0, "IPEHR" }, \
 	{ RING_INSTPS(0),           0,      0, "INSTPS" }, \
@@ -80,9 +81,6 @@
 	{ GEN8_RING_PDP_LDW(0, 3),  0,      0, "PDP3_LDW" }, \
 	{ GEN8_RING_PDP_UDW(0, 3),  0,      0, "PDP3_UDW" }
 
-#define COMMON_BASE_HAS_EU \
-	{ EIR,                      0,      0, "EIR" }
-
 #define COMMON_BASE_RENDER \
 	{ GEN7_SC_INSTDONE,         0,      0, "GEN7_SC_INSTDONE" }
 
@@ -105,7 +103,6 @@ static const struct __guc_mmio_reg_descr xe_lp_global_regs[] = {
 
 /* XE_LP Render / Compute Per-Class */
 static const struct __guc_mmio_reg_descr xe_lp_rc_class_regs[] = {
-	COMMON_BASE_HAS_EU,
 	COMMON_BASE_RENDER,
 	COMMON_GEN12BASE_RENDER,
 };
@@ -148,7 +145,6 @@ static const struct __guc_mmio_reg_descr gen8_global_regs[] = {
 };
 
 static const struct __guc_mmio_reg_descr gen8_rc_class_regs[] = {
-	COMMON_BASE_HAS_EU,
 	COMMON_BASE_RENDER,
 };
 
@@ -1441,7 +1437,7 @@ int intel_guc_capture_print_engine_node(struct drm_i915_error_state_buf *ebuf,
 	if (!cap || !ee->engine)
 		return -ENODEV;
 
-	guc = &ee->engine->gt->uc.guc;
+	guc = gt_to_guc(ee->engine->gt);
 
 	i915_error_printf(ebuf, "global --- GuC Error Capture on %s command stream:\n",
 			  ee->engine->name);
@@ -1543,7 +1539,7 @@ bool intel_guc_capture_is_matching_engine(struct intel_gt *gt,
 	if (!gt || !ce || !engine)
 		return false;
 
-	guc = &gt->uc.guc;
+	guc = gt_to_guc(gt);
 	if (!guc->capture)
 		return false;
 
@@ -1573,7 +1569,7 @@ void intel_guc_capture_get_matching_node(struct intel_gt *gt,
 	if (!gt || !ee || !ce)
 		return;
 
-	guc = &gt->uc.guc;
+	guc = gt_to_guc(gt);
 	if (!guc->capture)
 		return;
 
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c
index 52332bb14339..23f54c84cbab 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c
@@ -26,7 +26,7 @@ static void guc_prepare_xfer(struct intel_gt *gt)
 			 GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA |
 			 GUC_ENABLE_MIA_CLOCK_GATING;
 
-	if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 50))
+	if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 55))
 		shim_flags |= GUC_DISABLE_SRAM_INIT_TO_ZEROES |
 			      GUC_ENABLE_MIA_CACHING;
 
@@ -115,6 +115,7 @@ static inline bool guc_load_done(struct intel_uncore *uncore, u32 *status, bool
 	case INTEL_GUC_LOAD_STATUS_INIT_DATA_INVALID:
 	case INTEL_GUC_LOAD_STATUS_MPU_DATA_INVALID:
 	case INTEL_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
+	case INTEL_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR:
 		*success = false;
 		return true;
 	}
@@ -241,6 +242,11 @@ static int guc_wait_ucode(struct intel_guc *guc)
 			ret = -EPERM;
 			break;
 
+		case INTEL_GUC_LOAD_STATUS_KLV_WORKAROUND_INIT_ERROR:
+			guc_info(guc, "invalid w/a KLV entry\n");
+			ret = -EINVAL;
+			break;
+
 		case INTEL_GUC_LOAD_STATUS_HWCONFIG_START:
 			guc_info(guc, "still extracting hwconfig table.\n");
 			ret = -ETIMEDOUT;
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h
index 8ae1846431da..14797e80bc92 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h
@@ -96,8 +96,9 @@
 #define   GUC_WA_GAM_CREDITS		BIT(10)
 #define   GUC_WA_DUAL_QUEUE		BIT(11)
 #define   GUC_WA_RCS_RESET_BEFORE_RC6	BIT(13)
-#define   GUC_WA_CONTEXT_ISOLATION	BIT(15)
 #define   GUC_WA_PRE_PARSER		BIT(14)
+#define   GUC_WA_CONTEXT_ISOLATION	BIT(15)
+#define   GUC_WA_RCS_CCS_SWITCHOUT	BIT(16)
 #define   GUC_WA_HOLD_CCS_SWITCHOUT	BIT(17)
 #define   GUC_WA_POLLCS			BIT(18)
 #define   GUC_WA_RCS_REGS_IN_CCS_REGS_LIST	BIT(21)
@@ -430,7 +431,10 @@ struct guc_ads {
 	u32 capture_instance[GUC_CAPTURE_LIST_INDEX_MAX][GUC_MAX_ENGINE_CLASSES];
 	u32 capture_class[GUC_CAPTURE_LIST_INDEX_MAX][GUC_MAX_ENGINE_CLASSES];
 	u32 capture_global[GUC_CAPTURE_LIST_INDEX_MAX];
-	u32 reserved[14];
+	u32 wa_klv_addr_lo;
+	u32 wa_klv_addr_hi;
+	u32 wa_klv_size;
+	u32 reserved[11];
 } __packed;
 
 /* Engine usage stats */
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_hwconfig.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_hwconfig.c
index cc9569af7f0c..b67a15f74276 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_hwconfig.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_hwconfig.c
@@ -111,7 +111,7 @@ static bool has_table(struct drm_i915_private *i915)
 static int guc_hwconfig_init(struct intel_gt *gt)
 {
 	struct intel_hwconfig *hwconfig = &gt->info.hwconfig;
-	struct intel_guc *guc = &gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(gt);
 	int ret;
 
 	if (!has_table(gt->i915))
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
index 3e681ab6fbf9..706fffca698b 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
@@ -537,6 +537,20 @@ int intel_guc_slpc_get_min_freq(struct intel_guc_slpc *slpc, u32 *val)
 	return ret;
 }
 
+int intel_guc_slpc_set_strategy(struct intel_guc_slpc *slpc, u32 val)
+{
+	struct drm_i915_private *i915 = slpc_to_i915(slpc);
+	intel_wakeref_t wakeref;
+	int ret = 0;
+
+	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
+		ret = slpc_set_param(slpc,
+				     SLPC_PARAM_STRATEGIES,
+				     val);
+
+	return ret;
+}
+
 int intel_guc_slpc_set_media_ratio_mode(struct intel_guc_slpc *slpc, u32 val)
 {
 	struct drm_i915_private *i915 = slpc_to_i915(slpc);
@@ -711,6 +725,9 @@ int intel_guc_slpc_enable(struct intel_guc_slpc *slpc)
 	/* Set cached media freq ratio mode */
 	intel_guc_slpc_set_media_ratio_mode(slpc, slpc->media_ratio_mode);
 
+	/* Enable SLPC Optimized Strategy for compute */
+	intel_guc_slpc_set_strategy(slpc, SLPC_OPTIMIZED_STRATEGY_COMPUTE);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.h
index 6ac6503c39d4..1cb5fd44f05c 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.h
@@ -45,5 +45,6 @@ void intel_guc_pm_intrmsk_enable(struct intel_gt *gt);
 void intel_guc_slpc_boost(struct intel_guc_slpc *slpc);
 void intel_guc_slpc_dec_waiters(struct intel_guc_slpc *slpc);
 int intel_guc_slpc_set_ignore_eff_freq(struct intel_guc_slpc *slpc, bool val);
+int intel_guc_slpc_set_strategy(struct intel_guc_slpc *slpc, u32 val);
 
 #endif
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
index 0f83c6d4376f..0eaa1064242c 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
@@ -398,7 +398,7 @@ static inline void set_context_guc_id_invalid(struct intel_context *ce)
 
 static inline struct intel_guc *ce_to_guc(struct intel_context *ce)
 {
-	return &ce->engine->gt->uc.guc;
+	return gt_to_guc(ce->engine->gt);
 }
 
 static inline struct i915_priolist *to_priolist(struct rb_node *rb)
@@ -1246,7 +1246,7 @@ static void __get_engine_usage_record(struct intel_engine_cs *engine,
 static void guc_update_engine_gt_clks(struct intel_engine_cs *engine)
 {
 	struct intel_engine_guc_stats *stats = &engine->stats.guc;
-	struct intel_guc *guc = &engine->gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(engine->gt);
 	u32 last_switch, ctx_id, total;
 
 	lockdep_assert_held(&guc->timestamp.lock);
@@ -1311,7 +1311,7 @@ static ktime_t guc_engine_busyness(struct intel_engine_cs *engine, ktime_t *now)
 	struct intel_engine_guc_stats stats_saved, *stats = &engine->stats.guc;
 	struct i915_gpu_error *gpu_error = &engine->i915->gpu_error;
 	struct intel_gt *gt = engine->gt;
-	struct intel_guc *guc = &gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(gt);
 	u64 total, gt_stamp_saved;
 	unsigned long flags;
 	u32 reset_count;
@@ -1577,7 +1577,7 @@ static void guc_fini_engine_stats(struct intel_guc *guc)
 
 void intel_guc_busyness_park(struct intel_gt *gt)
 {
-	struct intel_guc *guc = &gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(gt);
 
 	if (!guc_submission_initialized(guc))
 		return;
@@ -1604,7 +1604,7 @@ void intel_guc_busyness_park(struct intel_gt *gt)
 
 void intel_guc_busyness_unpark(struct intel_gt *gt)
 {
-	struct intel_guc *guc = &gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(gt);
 	unsigned long flags;
 	ktime_t unused;
 
@@ -2189,7 +2189,7 @@ static bool need_tasklet(struct intel_guc *guc, struct i915_request *rq)
 static void guc_submit_request(struct i915_request *rq)
 {
 	struct i915_sched_engine *sched_engine = rq->engine->sched_engine;
-	struct intel_guc *guc = &rq->engine->gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(rq->engine->gt);
 	unsigned long flags;
 
 	/* Will be called from irq-context when using foreign fences. */
@@ -2215,11 +2215,10 @@ static int new_guc_id(struct intel_guc *guc, struct intel_context *ce)
 					      order_base_2(ce->parallel.number_children
 							   + 1));
 	else
-		ret = ida_simple_get(&guc->submission_state.guc_ids,
-				     NUMBER_MULTI_LRC_GUC_ID(guc),
-				     guc->submission_state.num_guc_ids,
-				     GFP_KERNEL | __GFP_RETRY_MAYFAIL |
-				     __GFP_NOWARN);
+		ret = ida_alloc_range(&guc->submission_state.guc_ids,
+				      NUMBER_MULTI_LRC_GUC_ID(guc),
+				      guc->submission_state.num_guc_ids - 1,
+				      GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 	if (unlikely(ret < 0))
 		return ret;
 
@@ -2242,8 +2241,8 @@ static void __release_guc_id(struct intel_guc *guc, struct intel_context *ce)
 							   + 1));
 		} else {
 			--guc->submission_state.guc_ids_in_use;
-			ida_simple_remove(&guc->submission_state.guc_ids,
-					  ce->guc_id.id);
+			ida_free(&guc->submission_state.guc_ids,
+				 ce->guc_id.id);
 		}
 		clr_ctx_id_mapping(guc, ce->guc_id.id);
 		set_context_guc_id_invalid(ce);
@@ -2640,6 +2639,7 @@ MAKE_CONTEXT_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM)
 MAKE_CONTEXT_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT)
 MAKE_CONTEXT_POLICY_ADD(priority, SCHEDULING_PRIORITY)
 MAKE_CONTEXT_POLICY_ADD(preempt_to_idle, PREEMPT_TO_IDLE_ON_QUANTUM_EXPIRY)
+MAKE_CONTEXT_POLICY_ADD(slpc_ctx_freq_req, SLPM_GT_FREQUENCY)
 
 #undef MAKE_CONTEXT_POLICY_ADD
 
@@ -2655,10 +2655,11 @@ static int __guc_context_set_context_policies(struct intel_guc *guc,
 static int guc_context_policy_init_v70(struct intel_context *ce, bool loop)
 {
 	struct intel_engine_cs *engine = ce->engine;
-	struct intel_guc *guc = &engine->gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(engine->gt);
 	struct context_policy policy;
 	u32 execution_quantum;
 	u32 preemption_timeout;
+	u32 slpc_ctx_freq_req = 0;
 	unsigned long flags;
 	int ret;
 
@@ -2670,11 +2671,15 @@ static int guc_context_policy_init_v70(struct intel_context *ce, bool loop)
 	execution_quantum = engine->props.timeslice_duration_ms * 1000;
 	preemption_timeout = engine->props.preempt_timeout_ms * 1000;
 
+	if (ce->flags & BIT(CONTEXT_LOW_LATENCY))
+		slpc_ctx_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE;
+
 	__guc_context_policy_start_klv(&policy, ce->guc_id.id);
 
 	__guc_context_policy_add_priority(&policy, ce->guc_state.prio);
 	__guc_context_policy_add_execution_quantum(&policy, execution_quantum);
 	__guc_context_policy_add_preemption_timeout(&policy, preemption_timeout);
+	__guc_context_policy_add_slpc_ctx_freq_req(&policy, slpc_ctx_freq_req);
 
 	if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION)
 		__guc_context_policy_add_preempt_to_idle(&policy, 1);
@@ -2731,7 +2736,7 @@ static u32 map_guc_prio_to_lrc_desc_prio(u8 prio)
 static void prepare_context_registration_info_v69(struct intel_context *ce)
 {
 	struct intel_engine_cs *engine = ce->engine;
-	struct intel_guc *guc = &engine->gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(engine->gt);
 	u32 ctx_id = ce->guc_id.id;
 	struct guc_lrc_desc_v69 *desc;
 	struct intel_context *child;
@@ -2800,7 +2805,7 @@ static void prepare_context_registration_info_v70(struct intel_context *ce,
 						  struct guc_ctxt_registration_info *info)
 {
 	struct intel_engine_cs *engine = ce->engine;
-	struct intel_guc *guc = &engine->gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(engine->gt);
 	u32 ctx_id = ce->guc_id.id;
 
 	GEM_BUG_ON(!engine->mask);
@@ -2863,7 +2868,7 @@ static int try_context_registration(struct intel_context *ce, bool loop)
 {
 	struct intel_engine_cs *engine = ce->engine;
 	struct intel_runtime_pm *runtime_pm = engine->uncore->rpm;
-	struct intel_guc *guc = &engine->gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(engine->gt);
 	intel_wakeref_t wakeref;
 	u32 ctx_id = ce->guc_id.id;
 	bool context_registered;
@@ -4491,7 +4496,13 @@ static void guc_default_vfuncs(struct intel_engine_cs *engine)
 	if (engine->class == COMPUTE_CLASS)
 		if (IS_GFX_GT_IP_STEP(engine->gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
 		    IS_DG2(engine->i915))
-			engine->flags |= I915_ENGINE_USES_WA_HOLD_CCS_SWITCHOUT;
+			engine->flags |= I915_ENGINE_USES_WA_HOLD_SWITCHOUT;
+
+	/* Wa_16019325821 */
+	/* Wa_14019159160 */
+	if ((engine->class == COMPUTE_CLASS || engine->class == RENDER_CLASS) &&
+	    IS_GFX_GT_IP_RANGE(engine->gt, IP_VER(12, 70), IP_VER(12, 71)))
+		engine->flags |= I915_ENGINE_USES_WA_HOLD_SWITCHOUT;
 
 	/*
 	 * TODO: GuC supports timeslicing and semaphores as well, but they're
@@ -4502,7 +4513,7 @@ static void guc_default_vfuncs(struct intel_engine_cs *engine)
 	 */
 
 	engine->emit_bb_start = gen8_emit_bb_start;
-	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
+	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
 		engine->emit_bb_start = xehp_emit_bb_start;
 }
 
@@ -4544,7 +4555,7 @@ static void guc_sched_engine_destroy(struct kref *kref)
 int intel_guc_submission_setup(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *i915 = engine->i915;
-	struct intel_guc *guc = &engine->gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(engine->gt);
 
 	/*
 	 * The setup relies on several assumptions (e.g. irqs always enabled)
@@ -5303,7 +5314,7 @@ int intel_guc_engine_failure_process_msg(struct intel_guc *guc,
 
 void intel_guc_find_hung_context(struct intel_engine_cs *engine)
 {
-	struct intel_guc *guc = &engine->gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(engine->gt);
 	struct intel_context *ce;
 	struct i915_request *rq;
 	unsigned long index;
@@ -5365,7 +5376,7 @@ void intel_guc_dump_active_requests(struct intel_engine_cs *engine,
 				    struct i915_request *hung_rq,
 				    struct drm_printer *m)
 {
-	struct intel_guc *guc = &engine->gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(engine->gt);
 	struct intel_context *ce;
 	unsigned long index;
 	unsigned long flags;
@@ -5817,7 +5828,7 @@ guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
 	if (!ve)
 		return ERR_PTR(-ENOMEM);
 
-	guc = &siblings[0]->gt->uc.guc;
+	guc = gt_to_guc(siblings[0]->gt);
 
 	ve->base.i915 = siblings[0]->i915;
 	ve->base.gt = siblings[0]->gt;
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.c b/drivers/gpu/drm/i915/gt/uc/intel_huc.c
index 0945b177d5f9..2d9152eb7282 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_huc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.c
@@ -385,7 +385,7 @@ int intel_huc_init(struct intel_huc *huc)
 	if (HAS_ENGINE(gt, GSC0)) {
 		struct i915_vma *vma;
 
-		vma = intel_guc_allocate_vma(&gt->uc.guc, PXP43_HUC_AUTH_INOUT_SIZE * 2);
+		vma = intel_guc_allocate_vma(gt_to_guc(gt), PXP43_HUC_AUTH_INOUT_SIZE * 2);
 		if (IS_ERR(vma)) {
 			err = PTR_ERR(vma);
 			huc_info(huc, "Failed to allocate heci pkt\n");
@@ -540,7 +540,7 @@ int intel_huc_wait_for_auth_complete(struct intel_huc *huc,
 int intel_huc_auth(struct intel_huc *huc, enum intel_huc_authentication_type type)
 {
 	struct intel_gt *gt = huc_to_gt(huc);
-	struct intel_guc *guc = &gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(gt);
 	int ret;
 
 	if (!intel_uc_fw_is_loaded(&huc->fw))
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_uc.c
index 399bc319180b..7a63abf8f644 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_uc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.c
@@ -50,10 +50,6 @@ static void uc_expand_default_options(struct intel_uc *uc)
 
 	/* Default: enable HuC authentication and GuC submission */
 	i915->params.enable_guc = ENABLE_GUC_LOAD_HUC | ENABLE_GUC_SUBMISSION;
-
-	/* XEHPSDV and PVC do not use HuC */
-	if (IS_XEHPSDV(i915) || IS_PONTEVECCHIO(i915))
-		i915->params.enable_guc &= ~ENABLE_GUC_LOAD_HUC;
 }
 
 /* Reset GuC providing us with fresh state for both GuC and HuC.
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
index 756093eaf2ad..d80278eb45d7 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
@@ -807,7 +807,7 @@ static int try_firmware_load(struct intel_uc_fw *uc_fw, const struct firmware **
 static int check_mtl_huc_guc_compatibility(struct intel_gt *gt,
 					   struct intel_uc_fw_file *huc_selected)
 {
-	struct intel_uc_fw_file *guc_selected = &gt->uc.guc.fw.file_selected;
+	struct intel_uc_fw_file *guc_selected = &gt_to_guc(gt)->fw.file_selected;
 	struct intel_uc_fw_ver *huc_ver = &huc_selected->ver;
 	struct intel_uc_fw_ver *guc_ver = &guc_selected->ver;
 	bool new_huc, new_guc;
@@ -1209,7 +1209,7 @@ static int uc_fw_rsa_data_create(struct intel_uc_fw *uc_fw)
 	 * since its GGTT offset will be GuC accessible.
 	 */
 	GEM_BUG_ON(uc_fw->rsa_size > PAGE_SIZE);
-	vma = intel_guc_allocate_vma(&gt->uc.guc, PAGE_SIZE);
+	vma = intel_guc_allocate_vma(gt_to_guc(gt), PAGE_SIZE);
 	if (IS_ERR(vma))
 		return PTR_ERR(vma);
 
diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc.c
index c900aac85adb..68feb55654f7 100644
--- a/drivers/gpu/drm/i915/gt/uc/selftest_guc.c
+++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc.c
@@ -144,7 +144,7 @@ err:
 static int intel_guc_steal_guc_ids(void *arg)
 {
 	struct intel_gt *gt = arg;
-	struct intel_guc *guc = &gt->uc.guc;
+	struct intel_guc *guc = gt_to_guc(gt);
 	int ret, sv, context_index = 0;
 	intel_wakeref_t wakeref;
 	struct intel_engine_cs *engine;
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
index d4a3f3e093b0..4be8cb65fb7e 100644
--- a/drivers/gpu/drm/i915/gvt/cmd_parser.c
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -50,6 +50,7 @@
 #include "trace.h"
 
 #include "display/intel_display.h"
+#include "display/intel_sprite_regs.h"
 #include "gem/i915_gem_context.h"
 #include "gem/i915_gem_pm.h"
 #include "gt/intel_context.h"
diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c
index e0c5dfb788eb..2b7df7fcf369 100644
--- a/drivers/gpu/drm/i915/gvt/display.c
+++ b/drivers/gpu/drm/i915/gvt/display.c
@@ -36,8 +36,10 @@
 #include "i915_reg.h"
 #include "gvt.h"
 
+#include "display/bxt_dpio_phy_regs.h"
 #include "display/intel_display.h"
 #include "display/intel_dpio_phy.h"
+#include "display/intel_sprite_regs.h"
 
 static int get_edp_pipe(struct intel_vgpu *vgpu)
 {
diff --git a/drivers/gpu/drm/i915/gvt/fb_decoder.c b/drivers/gpu/drm/i915/gvt/fb_decoder.c
index 313efdabee57..4140da68aabb 100644
--- a/drivers/gpu/drm/i915/gvt/fb_decoder.c
+++ b/drivers/gpu/drm/i915/gvt/fb_decoder.c
@@ -34,11 +34,14 @@
  */
 
 #include <uapi/drm/drm_fourcc.h>
-#include "i915_drv.h"
+
 #include "gvt.h"
+#include "i915_drv.h"
 #include "i915_pvinfo.h"
 #include "i915_reg.h"
 
+#include "display/intel_sprite_regs.h"
+
 #define PRIMARY_FORMAT_NUM	16
 struct pixel_format {
 	int drm_format;	/* Pixel format in DRM definition */
diff --git a/drivers/gpu/drm/i915/gvt/firmware.c b/drivers/gpu/drm/i915/gvt/firmware.c
index 4dd52ac2043e..221a3ae81baf 100644
--- a/drivers/gpu/drm/i915/gvt/firmware.c
+++ b/drivers/gpu/drm/i915/gvt/firmware.c
@@ -30,6 +30,7 @@
 
 #include <linux/firmware.h>
 #include <linux/crc32.h>
+#include <linux/vmalloc.h>
 
 #include "i915_drv.h"
 #include "gvt.h"
@@ -50,21 +51,7 @@ struct gvt_firmware_header {
 
 #define dev_to_drm_minor(d) dev_get_drvdata((d))
 
-static ssize_t
-gvt_firmware_read(struct file *filp, struct kobject *kobj,
-	     struct bin_attribute *attr, char *buf,
-	     loff_t offset, size_t count)
-{
-	memcpy(buf, attr->private + offset, count);
-	return count;
-}
-
-static struct bin_attribute firmware_attr = {
-	.attr = {.name = "gvt_firmware", .mode = (S_IRUSR)},
-	.read = gvt_firmware_read,
-	.write = NULL,
-	.mmap = NULL,
-};
+static BIN_ATTR_SIMPLE_ADMIN_RO(gvt_firmware);
 
 static int expose_firmware_sysfs(struct intel_gvt *gvt)
 {
@@ -107,10 +94,10 @@ static int expose_firmware_sysfs(struct intel_gvt *gvt)
 	crc32_start = offsetof(struct gvt_firmware_header, version);
 	h->crc32 = crc32_le(0, firmware + crc32_start, size - crc32_start);
 
-	firmware_attr.size = size;
-	firmware_attr.private = firmware;
+	bin_attr_gvt_firmware.size = size;
+	bin_attr_gvt_firmware.private = firmware;
 
-	ret = device_create_bin_file(&pdev->dev, &firmware_attr);
+	ret = device_create_bin_file(&pdev->dev, &bin_attr_gvt_firmware);
 	if (ret) {
 		vfree(firmware);
 		return ret;
@@ -122,8 +109,8 @@ static void clean_firmware_sysfs(struct intel_gvt *gvt)
 {
 	struct pci_dev *pdev = to_pci_dev(gvt->gt->i915->drm.dev);
 
-	device_remove_bin_file(&pdev->dev, &firmware_attr);
-	vfree(firmware_attr.private);
+	device_remove_bin_file(&pdev->dev, &bin_attr_gvt_firmware);
+	vfree(bin_attr_gvt_firmware.private);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c
index 094fca9b0e73..58cca4906f41 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.c
+++ b/drivers/gpu/drm/i915/gvt/gtt.c
@@ -39,6 +39,7 @@
 #include "trace.h"
 
 #include "gt/intel_gt_regs.h"
+#include <linux/vmalloc.h>
 
 #if defined(VERBOSE_DEBUG)
 #define gvt_vdbg_mm(fmt, args...) gvt_dbg_mm(fmt, ##args)
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index efcb00472be2..22fbddbe3e23 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -41,6 +41,7 @@
 #include "gvt.h"
 #include "i915_pvinfo.h"
 #include "intel_mchbar_regs.h"
+#include "display/bxt_dpio_phy_regs.h"
 #include "display/intel_display_types.h"
 #include "display/intel_dmc_regs.h"
 #include "display/intel_dp_aux_regs.h"
@@ -49,9 +50,11 @@
 #include "display/intel_fdi_regs.h"
 #include "display/intel_pps_regs.h"
 #include "display/intel_psr_regs.h"
+#include "display/intel_sprite_regs.h"
 #include "display/skl_watermark_regs.h"
 #include "display/vlv_dsi_pll_regs.h"
 #include "gt/intel_gt_regs.h"
+#include <linux/vmalloc.h>
 
 /* XXX FIXME i915 has changed PP_XXX definition */
 #define PCH_PP_STATUS  _MMIO(0xc7200)
@@ -2763,15 +2766,15 @@ static int init_bxt_mmio_info(struct intel_gvt *gvt)
 
 	MMIO_DH(BXT_PORT_PCS_DW12_GRP(DPIO_PHY0, DPIO_CH0), D_BXT,
 		NULL, bxt_pcs_dw12_grp_write);
-	MMIO_DH(BXT_PORT_TX_DW3_LN0(DPIO_PHY0, DPIO_CH0), D_BXT,
+	MMIO_DH(BXT_PORT_TX_DW3_LN(DPIO_PHY0, DPIO_CH0, 0), D_BXT,
 		bxt_port_tx_dw3_read, NULL);
 	MMIO_DH(BXT_PORT_PCS_DW12_GRP(DPIO_PHY0, DPIO_CH1), D_BXT,
 		NULL, bxt_pcs_dw12_grp_write);
-	MMIO_DH(BXT_PORT_TX_DW3_LN0(DPIO_PHY0, DPIO_CH1), D_BXT,
+	MMIO_DH(BXT_PORT_TX_DW3_LN(DPIO_PHY0, DPIO_CH1, 0), D_BXT,
 		bxt_port_tx_dw3_read, NULL);
 	MMIO_DH(BXT_PORT_PCS_DW12_GRP(DPIO_PHY1, DPIO_CH0), D_BXT,
 		NULL, bxt_pcs_dw12_grp_write);
-	MMIO_DH(BXT_PORT_TX_DW3_LN0(DPIO_PHY1, DPIO_CH0), D_BXT,
+	MMIO_DH(BXT_PORT_TX_DW3_LN(DPIO_PHY1, DPIO_CH0, 0), D_BXT,
 		bxt_port_tx_dw3_read, NULL);
 	MMIO_DH(BXT_DE_PLL_ENABLE, D_BXT, NULL, bxt_de_pll_enable_write);
 	MMIO_DFH(GEN8_L3SQCREG1, D_BXT, F_CMD_ACCESS, NULL, NULL);
diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
index 5b5def6ddef7..e16e0d4c9534 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.c
+++ b/drivers/gpu/drm/i915/gvt/mmio.c
@@ -33,10 +33,12 @@
  *
  */
 
+#include <linux/vmalloc.h>
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "gvt.h"
 
+#include "display/bxt_dpio_phy_regs.h"
 #include "display/intel_dpio_phy.h"
 #include "gt/intel_gt_regs.h"
 
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
index 08ad1bd651f1..63c751ca4119 100644
--- a/drivers/gpu/drm/i915/gvt/vgpu.c
+++ b/drivers/gpu/drm/i915/gvt/vgpu.c
@@ -34,6 +34,7 @@
 #include "i915_drv.h"
 #include "gvt.h"
 #include "i915_pvinfo.h"
+#include <linux/vmalloc.h>
 
 void populate_pvinfo_page(struct intel_vgpu *vgpu)
 {
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 990eaa029d9c..bc717cf544e4 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -30,6 +30,7 @@
 #include <linux/sort.h>
 #include <linux/string_helpers.h>
 
+#include <linux/debugfs.h>
 #include <drm/drm_debugfs.h>
 
 #include "display/intel_display_params.h"
@@ -156,18 +157,6 @@ static const char *i915_cache_level_str(struct drm_i915_gem_object *obj)
 		case 4: return " WB (2-Way Coh)";
 		default: return " not defined";
 		}
-	} else if (IS_PONTEVECCHIO(i915)) {
-		switch (obj->pat_index) {
-		case 0: return " UC";
-		case 1: return " WC";
-		case 2: return " WT";
-		case 3: return " WB";
-		case 4: return " WT (CLOS1)";
-		case 5: return " WB (CLOS1)";
-		case 6: return " WT (CLOS2)";
-		case 7: return " WT (CLOS2)";
-		default: return " not defined";
-		}
 	} else if (GRAPHICS_VER(i915) >= 12) {
 		switch (obj->pat_index) {
 		case 0: return " WB";
diff --git a/drivers/gpu/drm/i915/i915_debugfs_params.c b/drivers/gpu/drm/i915/i915_debugfs_params.c
index 8bca02025e09..33d2dcb0de65 100644
--- a/drivers/gpu/drm/i915/i915_debugfs_params.c
+++ b/drivers/gpu/drm/i915/i915_debugfs_params.c
@@ -4,6 +4,7 @@
  */
 
 #include <linux/kernel.h>
+#include <linux/debugfs.h>
 
 #include "i915_debugfs_params.h"
 #include "gt/intel_gt.h"
diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c
index 4b9233c07a22..161b21eff694 100644
--- a/drivers/gpu/drm/i915/i915_driver.c
+++ b/drivers/gpu/drm/i915/i915_driver.c
@@ -202,7 +202,7 @@ static void sanitize_gpu(struct drm_i915_private *i915)
 		unsigned int i;
 
 		for_each_gt(gt, i915, i)
-			__intel_gt_reset(gt, ALL_ENGINES);
+			intel_gt_reset_all_engines(gt);
 	}
 }
 
@@ -920,27 +920,6 @@ static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
 	return 0;
 }
 
-/**
- * i915_driver_lastclose - clean up after all DRM clients have exited
- * @dev: DRM device
- *
- * Take care of cleaning up after all DRM clients have exited.  In the
- * mode setting case, we want to restore the kernel's initial mode (just
- * in case the last client left us in a bad state).
- *
- * Additionally, in the non-mode setting case, we'll tear down the GTT
- * and DMA structures, since the kernel won't be using them, and clea
- * up any GEM state.
- */
-static void i915_driver_lastclose(struct drm_device *dev)
-{
-	struct drm_i915_private *i915 = to_i915(dev);
-
-	intel_fbdev_restore_mode(i915);
-
-	vga_switcheroo_process_delayed_switch();
-}
-
 static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
 {
 	struct drm_i915_file_private *file_priv = file->driver_priv;
@@ -1831,7 +1810,6 @@ static const struct drm_driver i915_drm_driver = {
 	    DRIVER_SYNCOBJ_TIMELINE,
 	.release = i915_driver_release,
 	.open = i915_driver_open,
-	.lastclose = i915_driver_lastclose,
 	.postclose = i915_driver_postclose,
 	.show_fdinfo = PTR_IF(IS_ENABLED(CONFIG_PROC_FS), i915_drm_client_fdinfo),
 
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index e81b3b2858ac..ee0d7d5f135d 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -235,25 +235,17 @@ struct drm_i915_private {
 	/* protects the irq masks */
 	spinlock_t irq_lock;
 
-	bool display_irqs_enabled;
-
 	/* Sideband mailbox protection */
 	struct mutex sb_lock;
 	struct pm_qos_request sb_qos;
 
 	/** Cached value of IMR to avoid reads in updating the bitfield */
-	union {
-		u32 irq_mask;
-		u32 de_irq_mask[I915_MAX_PIPES];
-	};
-	u32 pipestat_irq_mask[I915_MAX_PIPES];
+	u32 irq_mask;
 
 	bool preserve_bios_swizzle;
 
 	unsigned int fsb_freq, mem_freq, is_ddr3;
-	unsigned int skl_preferred_vco_freq;
 
-	unsigned int max_dotclk_freq;
 	unsigned int hpll_freq;
 	unsigned int czclk_freq;
 
@@ -350,9 +342,6 @@ struct drm_i915_private {
 
 	struct intel_pxp *pxp;
 
-	/* For i915gm/i945gm vblank irq workaround */
-	u8 vblank_enabled;
-
 	bool irq_enabled;
 
 	struct i915_pmu pmu;
@@ -544,9 +533,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 #define IS_DG1(i915)        IS_PLATFORM(i915, INTEL_DG1)
 #define IS_ALDERLAKE_S(i915) IS_PLATFORM(i915, INTEL_ALDERLAKE_S)
 #define IS_ALDERLAKE_P(i915) IS_PLATFORM(i915, INTEL_ALDERLAKE_P)
-#define IS_XEHPSDV(i915) IS_PLATFORM(i915, INTEL_XEHPSDV)
 #define IS_DG2(i915)	IS_PLATFORM(i915, INTEL_DG2)
-#define IS_PONTEVECCHIO(i915) IS_PLATFORM(i915, INTEL_PONTEVECCHIO)
 #define IS_METEORLAKE(i915) IS_PLATFORM(i915, INTEL_METEORLAKE)
 #define IS_LUNARLAKE(i915) 0
 
@@ -621,17 +608,6 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 #define IS_TIGERLAKE_UY(i915) \
 	IS_SUBPLATFORM(i915, INTEL_TIGERLAKE, INTEL_SUBPLATFORM_UY)
 
-#define IS_XEHPSDV_GRAPHICS_STEP(__i915, since, until) \
-	(IS_XEHPSDV(__i915) && IS_GRAPHICS_STEP(__i915, since, until))
-
-#define IS_PVC_BD_STEP(__i915, since, until) \
-	(IS_PONTEVECCHIO(__i915) && \
-	 IS_BASEDIE_STEP(__i915, since, until))
-
-#define IS_PVC_CT_STEP(__i915, since, until) \
-	(IS_PONTEVECCHIO(__i915) && \
-	 IS_GRAPHICS_STEP(__i915, since, until))
-
 #define IS_LP(i915)		(INTEL_INFO(i915)->is_lp)
 #define IS_GEN9_LP(i915)	(GRAPHICS_VER(i915) == 9 && IS_LP(i915))
 #define IS_GEN9_BC(i915)	(GRAPHICS_VER(i915) == 9 && !IS_LP(i915))
diff --git a/drivers/gpu/drm/i915/i915_getparam.c b/drivers/gpu/drm/i915/i915_getparam.c
index 5c3fec63cb4c..a62405787e77 100644
--- a/drivers/gpu/drm/i915/i915_getparam.c
+++ b/drivers/gpu/drm/i915/i915_getparam.c
@@ -155,12 +155,18 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data,
 		 */
 		value = 1;
 		break;
+	case I915_PARAM_HAS_CONTEXT_FREQ_HINT:
+		if (intel_uc_uses_guc_submission(&to_gt(i915)->uc))
+			value = 1;
+		else
+			value = -EINVAL;
+		break;
 	case I915_PARAM_HAS_CONTEXT_ISOLATION:
 		value = intel_engines_has_context_isolation(i915);
 		break;
 	case I915_PARAM_SLICE_MASK:
 		/* Not supported from Xe_HP onward; use topology queries */
-		if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
+		if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
 			return -EINVAL;
 
 		value = sseu->slice_mask;
@@ -169,7 +175,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data,
 		break;
 	case I915_PARAM_SUBSLICE_MASK:
 		/* Not supported from Xe_HP onward; use topology queries */
-		if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
+		if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
 			return -EINVAL;
 
 		/* Only copy bits from the first slice */
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index a0b784ebaddd..625b3c024540 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -28,6 +28,7 @@
  */
 
 #include <linux/ascii85.h>
+#include <linux/debugfs.h>
 #include <linux/highmem.h>
 #include <linux/nmi.h>
 #include <linux/pagevec.h>
@@ -1245,8 +1246,7 @@ static void engine_record_registers(struct intel_engine_coredump *ee)
 		if (MEDIA_VER(i915) >= 13 && engine->gt->type == GT_MEDIA)
 			ee->fault_reg = intel_uncore_read(engine->uncore,
 							  XELPMP_RING_FAULT_REG);
-
-		else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
+		else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
 			ee->fault_reg = intel_gt_mcr_read_any(engine->gt,
 							      XEHP_RING_FAULT_REG);
 		else if (GRAPHICS_VER(i915) >= 12)
@@ -1852,7 +1852,7 @@ static void gt_record_global_regs(struct intel_gt_coredump *gt)
 	if (GRAPHICS_VER(i915) == 7)
 		gt->err_int = intel_uncore_read(uncore, GEN7_ERR_INT);
 
-	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
+	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
 		gt->fault_data0 = intel_gt_mcr_read_any((struct intel_gt *)gt->_gt,
 							XEHP_FAULT_TLB_DATA0);
 		gt->fault_data1 = intel_gt_mcr_read_any((struct intel_gt *)gt->_gt,
diff --git a/drivers/gpu/drm/i915/i915_hwmon.c b/drivers/gpu/drm/i915/i915_hwmon.c
index b758fd110c20..49db3e09826c 100644
--- a/drivers/gpu/drm/i915/i915_hwmon.c
+++ b/drivers/gpu/drm/i915/i915_hwmon.c
@@ -739,12 +739,6 @@ hwm_get_preregistration_info(struct drm_i915_private *i915)
 		hwmon->rg.pkg_rapl_limit = PCU_PACKAGE_RAPL_LIMIT;
 		hwmon->rg.energy_status_all = PCU_PACKAGE_ENERGY_STATUS;
 		hwmon->rg.energy_status_tile = INVALID_MMIO_REG;
-	} else if (IS_XEHPSDV(i915)) {
-		hwmon->rg.pkg_power_sku_unit = GT0_PACKAGE_POWER_SKU_UNIT;
-		hwmon->rg.pkg_power_sku = INVALID_MMIO_REG;
-		hwmon->rg.pkg_rapl_limit = GT0_PACKAGE_RAPL_LIMIT;
-		hwmon->rg.energy_status_all = GT0_PLATFORM_ENERGY_STATUS;
-		hwmon->rg.energy_status_tile = GT0_PACKAGE_ENERGY_STATUS;
 	} else {
 		hwmon->rg.pkg_power_sku_unit = INVALID_MMIO_REG;
 		hwmon->rg.pkg_power_sku = INVALID_MMIO_REG;
@@ -793,7 +787,7 @@ void i915_hwmon_register(struct drm_i915_private *i915)
 	if (!IS_DGFX(i915))
 		return;
 
-	hwmon = devm_kzalloc(dev, sizeof(*hwmon), GFP_KERNEL);
+	hwmon = kzalloc(sizeof(*hwmon), GFP_KERNEL);
 	if (!hwmon)
 		return;
 
@@ -819,14 +813,12 @@ void i915_hwmon_register(struct drm_i915_private *i915)
 	hwm_get_preregistration_info(i915);
 
 	/*  hwmon_dev points to device hwmon<i> */
-	hwmon_dev = devm_hwmon_device_register_with_info(dev, ddat->name,
-							 ddat,
-							 &hwm_chip_info,
-							 hwm_groups);
-	if (IS_ERR(hwmon_dev)) {
-		i915->hwmon = NULL;
-		return;
-	}
+	hwmon_dev = hwmon_device_register_with_info(dev, ddat->name,
+						    ddat,
+						    &hwm_chip_info,
+						    hwm_groups);
+	if (IS_ERR(hwmon_dev))
+		goto err;
 
 	ddat->hwmon_dev = hwmon_dev;
 
@@ -839,16 +831,36 @@ void i915_hwmon_register(struct drm_i915_private *i915)
 		if (!hwm_gt_is_visible(ddat_gt, hwmon_energy, hwmon_energy_input, 0))
 			continue;
 
-		hwmon_dev = devm_hwmon_device_register_with_info(dev, ddat_gt->name,
-								 ddat_gt,
-								 &hwm_gt_chip_info,
-								 NULL);
+		hwmon_dev = hwmon_device_register_with_info(dev, ddat_gt->name,
+							    ddat_gt,
+							    &hwm_gt_chip_info,
+							    NULL);
 		if (!IS_ERR(hwmon_dev))
 			ddat_gt->hwmon_dev = hwmon_dev;
 	}
+	return;
+err:
+	i915_hwmon_unregister(i915);
 }
 
 void i915_hwmon_unregister(struct drm_i915_private *i915)
 {
-	fetch_and_zero(&i915->hwmon);
+	struct i915_hwmon *hwmon = i915->hwmon;
+	struct intel_gt *gt;
+	int i;
+
+	if (!hwmon)
+		return;
+
+	for_each_gt(gt, i915, i)
+		if (hwmon->ddat_gt[i].hwmon_dev)
+			hwmon_device_unregister(hwmon->ddat_gt[i].hwmon_dev);
+
+	if (hwmon->ddat.hwmon_dev)
+		hwmon_device_unregister(hwmon->ddat.hwmon_dev);
+
+	mutex_destroy(&hwmon->hwmon_lock);
+
+	kfree(i915->hwmon);
+	i915->hwmon = NULL;
 }
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 8130f043693b..678d632ed043 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -702,7 +702,7 @@ static void valleyview_irq_reset(struct drm_i915_private *dev_priv)
 	gen5_gt_irq_reset(to_gt(dev_priv));
 
 	spin_lock_irq(&dev_priv->irq_lock);
-	if (dev_priv->display_irqs_enabled)
+	if (dev_priv->display.irq.display_irqs_enabled)
 		vlv_display_irq_reset(dev_priv);
 	spin_unlock_irq(&dev_priv->irq_lock);
 }
@@ -767,7 +767,7 @@ static void cherryview_irq_reset(struct drm_i915_private *dev_priv)
 	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
 
 	spin_lock_irq(&dev_priv->irq_lock);
-	if (dev_priv->display_irqs_enabled)
+	if (dev_priv->display.irq.display_irqs_enabled)
 		vlv_display_irq_reset(dev_priv);
 	spin_unlock_irq(&dev_priv->irq_lock);
 }
@@ -784,7 +784,7 @@ static void valleyview_irq_postinstall(struct drm_i915_private *dev_priv)
 	gen5_gt_irq_postinstall(to_gt(dev_priv));
 
 	spin_lock_irq(&dev_priv->irq_lock);
-	if (dev_priv->display_irqs_enabled)
+	if (dev_priv->display.irq.display_irqs_enabled)
 		vlv_display_irq_postinstall(dev_priv);
 	spin_unlock_irq(&dev_priv->irq_lock);
 
@@ -838,7 +838,7 @@ static void cherryview_irq_postinstall(struct drm_i915_private *dev_priv)
 	gen8_gt_irq_postinstall(to_gt(dev_priv));
 
 	spin_lock_irq(&dev_priv->irq_lock);
-	if (dev_priv->display_irqs_enabled)
+	if (dev_priv->display.irq.display_irqs_enabled)
 		vlv_display_irq_postinstall(dev_priv);
 	spin_unlock_irq(&dev_priv->irq_lock);
 
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index de43048543e8..8c00169e3ab7 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -108,9 +108,6 @@ i915_param_named_unsafe(guc_firmware_path, charp, 0400,
 i915_param_named_unsafe(huc_firmware_path, charp, 0400,
 	"HuC firmware path to use instead of the default one");
 
-i915_param_named_unsafe(dmc_firmware_path, charp, 0400,
-	"DMC firmware path to use instead of the default one");
-
 i915_param_named_unsafe(gsc_firmware_path, charp, 0400,
 	"GSC firmware path to use instead of the default one");
 
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index 1315d7fac850..2eb3f2115ff2 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -51,7 +51,6 @@ struct drm_printer;
 	param(int, guc_log_level, -1, 0400) \
 	param(char *, guc_firmware_path, NULL, 0400) \
 	param(char *, huc_firmware_path, NULL, 0400) \
-	param(char *, dmc_firmware_path, NULL, 0400) \
 	param(char *, gsc_firmware_path, NULL, 0400) \
 	param(bool, memtest, false, 0400) \
 	param(int, mmio_debug, -IS_ENABLED(CONFIG_DRM_I915_DEBUG_MMIO), 0600) \
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 8b4fdeabb12a..405ca17a990b 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -38,6 +38,9 @@
 #include "i915_reg.h"
 #include "intel_pci_config.h"
 
+__diag_push();
+__diag_ignore_all("-Woverride-init", "Allow field initialization overrides for device info");
+
 #define PLATFORM(x) .platform = (x)
 #define GEN(x) \
 	.__runtime.graphics.ip.ver = (x), \
@@ -59,14 +62,6 @@
 		[I915_CACHE_WT]     = 2, \
 	}
 
-#define PVC_CACHELEVEL \
-	.cachelevel_to_pat = { \
-		[I915_CACHE_NONE]   = 0, \
-		[I915_CACHE_LLC]    = 3, \
-		[I915_CACHE_L3_LLC] = 3, \
-		[I915_CACHE_WT]     = 2, \
-	}
-
 #define MTL_CACHELEVEL \
 	.cachelevel_to_pat = { \
 		[I915_CACHE_NONE]   = 2, \
@@ -705,8 +700,6 @@ static const struct intel_device_info adl_p_info = {
 		I915_GTT_PAGE_SIZE_2M
 
 #define XE_HP_FEATURES \
-	.__runtime.graphics.ip.ver = 12, \
-	.__runtime.graphics.ip.rel = 50, \
 	XE_HP_PAGE_SIZES, \
 	TGL_CACHELEVEL, \
 	.dma_mask_size = 46, \
@@ -730,32 +723,12 @@ static const struct intel_device_info adl_p_info = {
 	.__runtime.ppgtt_size = 48, \
 	.__runtime.ppgtt_type = INTEL_PPGTT_FULL
 
-#define XE_HPM_FEATURES \
-	.__runtime.media.ip.ver = 12, \
-	.__runtime.media.ip.rel = 50
-
-__maybe_unused
-static const struct intel_device_info xehpsdv_info = {
-	XE_HP_FEATURES,
-	XE_HPM_FEATURES,
-	DGFX_FEATURES,
-	PLATFORM(INTEL_XEHPSDV),
-	.has_64k_pages = 1,
-	.has_media_ratio_mode = 1,
-	.platform_engine_mask =
-		BIT(RCS0) | BIT(BCS0) |
-		BIT(VECS0) | BIT(VECS1) | BIT(VECS2) | BIT(VECS3) |
-		BIT(VCS0) | BIT(VCS1) | BIT(VCS2) | BIT(VCS3) |
-		BIT(VCS4) | BIT(VCS5) | BIT(VCS6) | BIT(VCS7) |
-		BIT(CCS0) | BIT(CCS1) | BIT(CCS2) | BIT(CCS3),
-	.require_force_probe = 1,
-};
-
 #define DG2_FEATURES \
 	XE_HP_FEATURES, \
-	XE_HPM_FEATURES, \
 	DGFX_FEATURES, \
+	.__runtime.graphics.ip.ver = 12, \
 	.__runtime.graphics.ip.rel = 55, \
+	.__runtime.media.ip.ver = 12, \
 	.__runtime.media.ip.rel = 55, \
 	PLATFORM(INTEL_DG2), \
 	.has_64k_pages = 1, \
@@ -778,33 +751,6 @@ static const struct intel_device_info ats_m_info = {
 	.tuning_thread_rr_after_dep = 1,
 };
 
-#define XE_HPC_FEATURES \
-	XE_HP_FEATURES, \
-	.dma_mask_size = 52, \
-	.has_3d_pipeline = 0, \
-	.has_guc_deprivilege = 1, \
-	.has_l3_ccs_read = 1, \
-	.has_mslice_steering = 0, \
-	.has_one_eu_per_fuse_bit = 1
-
-__maybe_unused
-static const struct intel_device_info pvc_info = {
-	XE_HPC_FEATURES,
-	XE_HPM_FEATURES,
-	DGFX_FEATURES,
-	.__runtime.graphics.ip.rel = 60,
-	.__runtime.media.ip.rel = 60,
-	PLATFORM(INTEL_PONTEVECCHIO),
-	.has_flat_ccs = 0,
-	.max_pat_index = 7,
-	.platform_engine_mask =
-		BIT(BCS0) |
-		BIT(VCS0) |
-		BIT(CCS0) | BIT(CCS1) | BIT(CCS2) | BIT(CCS3),
-	.require_force_probe = 1,
-	PVC_CACHELEVEL,
-};
-
 static const struct intel_gt_definition xelpmp_extra_gt[] = {
 	{
 		.type = GT_MEDIA,
@@ -842,6 +788,8 @@ static const struct intel_device_info mtl_info = {
 
 #undef PLATFORM
 
+__diag_pop();
+
 /*
  * Make sure any device matches here are from most specific to most
  * general.  For example, since the Quanta match is based on the subsystem
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index bd9d812b1afa..0b1cd4c7a525 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -292,7 +292,7 @@ static u32 i915_perf_stream_paranoid = true;
 #define OAREPORT_REASON_CTX_SWITCH     (1<<3)
 #define OAREPORT_REASON_CLK_RATIO      (1<<5)
 
-#define HAS_MI_SET_PREDICATE(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
+#define HAS_MI_SET_PREDICATE(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
 
 /* For sysctl proc_dointvec_minmax of i915_oa_max_sample_rate
  *
@@ -817,7 +817,7 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream,
 		 */
 
 		if (oa_report_ctx_invalid(stream, report) &&
-		    GRAPHICS_VER_FULL(stream->engine->i915) < IP_VER(12, 50)) {
+		    GRAPHICS_VER_FULL(stream->engine->i915) < IP_VER(12, 55)) {
 			ctx_id = INVALID_CTX_ID;
 			oa_context_id_squash(stream, report32);
 		}
@@ -1419,7 +1419,7 @@ static int gen12_get_render_context_id(struct i915_perf_stream *stream)
 
 		mask = ((1U << GEN12_GUC_SW_CTX_ID_WIDTH) - 1) <<
 			(GEN12_GUC_SW_CTX_ID_SHIFT - 32);
-	} else if (GRAPHICS_VER_FULL(stream->engine->i915) >= IP_VER(12, 50)) {
+	} else if (GRAPHICS_VER_FULL(stream->engine->i915) >= IP_VER(12, 55)) {
 		ctx_id = (XEHP_MAX_CONTEXT_HW_ID - 1) <<
 			(XEHP_SW_CTX_ID_SHIFT - 32);
 
@@ -2881,11 +2881,11 @@ gen12_enable_metric_set(struct i915_perf_stream *stream,
 	int ret;
 
 	/*
-	 * Wa_1508761755:xehpsdv, dg2
+	 * Wa_1508761755
 	 * EU NOA signals behave incorrectly if EU clock gating is enabled.
 	 * Disable thread stall DOP gating and EU DOP gating.
 	 */
-	if (IS_XEHPSDV(i915) || IS_DG2(i915)) {
+	if (IS_DG2(i915)) {
 		intel_gt_mcr_multicast_write(uncore->gt, GEN8_ROW_CHICKEN,
 					     _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
 		intel_uncore_write(uncore, GEN7_ROW_CHICKEN2,
@@ -2911,7 +2911,7 @@ gen12_enable_metric_set(struct i915_perf_stream *stream,
 	/*
 	 * Initialize Super Queue Internal Cnt Register
 	 * Set PMON Enable in order to collect valid metrics.
-	 * Enable byets per clock reporting in OA for XEHPSDV onward.
+	 * Enable bytes per clock reporting in OA.
 	 */
 	sqcnt1 = GEN12_SQCNT1_PMON_ENABLE |
 		 (HAS_OA_BPC_REPORTING(i915) ? GEN12_SQCNT1_OABPC : 0);
@@ -2971,10 +2971,9 @@ static void gen12_disable_metric_set(struct i915_perf_stream *stream)
 	u32 sqcnt1;
 
 	/*
-	 * Wa_1508761755:xehpsdv, dg2
-	 * Enable thread stall DOP gating and EU DOP gating.
+	 * Wa_1508761755: Enable thread stall DOP gating and EU DOP gating.
 	 */
-	if (IS_XEHPSDV(i915) || IS_DG2(i915)) {
+	if (IS_DG2(i915)) {
 		intel_gt_mcr_multicast_write(uncore->gt, GEN8_ROW_CHICKEN,
 					     _MASKED_BIT_DISABLE(STALL_DOP_GATING_DISABLE));
 		intel_uncore_write(uncore, GEN7_ROW_CHICKEN2,
@@ -4123,7 +4122,7 @@ static int read_properties_unlocked(struct i915_perf *perf,
 			props->hold_preemption = !!value;
 			break;
 		case DRM_I915_PERF_PROP_GLOBAL_SSEU: {
-			if (GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 50)) {
+			if (GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 55)) {
 				drm_dbg(&perf->i915->drm,
 					"SSEU config not supported on gfx %x\n",
 					GRAPHICS_VER_FULL(perf->i915));
diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c
index 3baa2f54a86e..14d9ec0ed777 100644
--- a/drivers/gpu/drm/i915/i915_query.c
+++ b/drivers/gpu/drm/i915/i915_query.c
@@ -105,7 +105,7 @@ static int query_geometry_subslices(struct drm_i915_private *i915,
 	struct intel_engine_cs *engine;
 	struct i915_engine_class_instance classinstance;
 
-	if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
+	if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 55))
 		return -ENODEV;
 
 	classinstance = *((struct i915_engine_class_instance *)&query_item->flags);
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 3b2e49ce29ba..e22a82a5ddd7 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -195,367 +195,6 @@
 #define  DPIO_SFR_BYPASS		(1 << 1)
 #define  DPIO_CMNRST			(1 << 0)
 
-/*
- * Per pipe/PLL DPIO regs
- */
-#define _VLV_PLL_DW3_CH0		0x800c
-#define   DPIO_POST_DIV_SHIFT		(28) /* 3 bits */
-#define   DPIO_POST_DIV_DAC		0
-#define   DPIO_POST_DIV_HDMIDP		1 /* DAC 225-400M rate */
-#define   DPIO_POST_DIV_LVDS1		2
-#define   DPIO_POST_DIV_LVDS2		3
-#define   DPIO_K_SHIFT			(24) /* 4 bits */
-#define   DPIO_P1_SHIFT			(21) /* 3 bits */
-#define   DPIO_P2_SHIFT			(16) /* 5 bits */
-#define   DPIO_N_SHIFT			(12) /* 4 bits */
-#define   DPIO_ENABLE_CALIBRATION	(1 << 11)
-#define   DPIO_M1DIV_SHIFT		(8) /* 3 bits */
-#define   DPIO_M2DIV_MASK		0xff
-#define _VLV_PLL_DW3_CH1		0x802c
-#define VLV_PLL_DW3(ch) _PIPE(ch, _VLV_PLL_DW3_CH0, _VLV_PLL_DW3_CH1)
-
-#define _VLV_PLL_DW5_CH0		0x8014
-#define   DPIO_REFSEL_OVERRIDE		27
-#define   DPIO_PLL_MODESEL_SHIFT	24 /* 3 bits */
-#define   DPIO_BIAS_CURRENT_CTL_SHIFT	21 /* 3 bits, always 0x7 */
-#define   DPIO_PLL_REFCLK_SEL_SHIFT	16 /* 2 bits */
-#define   DPIO_PLL_REFCLK_SEL_MASK	3
-#define   DPIO_DRIVER_CTL_SHIFT		12 /* always set to 0x8 */
-#define   DPIO_CLK_BIAS_CTL_SHIFT	8 /* always set to 0x5 */
-#define _VLV_PLL_DW5_CH1		0x8034
-#define VLV_PLL_DW5(ch) _PIPE(ch, _VLV_PLL_DW5_CH0, _VLV_PLL_DW5_CH1)
-
-#define _VLV_PLL_DW7_CH0		0x801c
-#define _VLV_PLL_DW7_CH1		0x803c
-#define VLV_PLL_DW7(ch) _PIPE(ch, _VLV_PLL_DW7_CH0, _VLV_PLL_DW7_CH1)
-
-#define _VLV_PLL_DW8_CH0		0x8040
-#define _VLV_PLL_DW8_CH1		0x8060
-#define VLV_PLL_DW8(ch) _PIPE(ch, _VLV_PLL_DW8_CH0, _VLV_PLL_DW8_CH1)
-
-#define VLV_PLL_DW9_BCAST		0xc044
-#define _VLV_PLL_DW9_CH0		0x8044
-#define _VLV_PLL_DW9_CH1		0x8064
-#define VLV_PLL_DW9(ch) _PIPE(ch, _VLV_PLL_DW9_CH0, _VLV_PLL_DW9_CH1)
-
-#define _VLV_PLL_DW10_CH0		0x8048
-#define _VLV_PLL_DW10_CH1		0x8068
-#define VLV_PLL_DW10(ch) _PIPE(ch, _VLV_PLL_DW10_CH0, _VLV_PLL_DW10_CH1)
-
-#define _VLV_PLL_DW11_CH0		0x804c
-#define _VLV_PLL_DW11_CH1		0x806c
-#define VLV_PLL_DW11(ch) _PIPE(ch, _VLV_PLL_DW11_CH0, _VLV_PLL_DW11_CH1)
-
-/* Spec for ref block start counts at DW10 */
-#define VLV_REF_DW13			0x80ac
-
-#define VLV_CMN_DW0			0x8100
-
-/*
- * Per DDI channel DPIO regs
- */
-
-#define _VLV_PCS_DW0_CH0		0x8200
-#define _VLV_PCS_DW0_CH1		0x8400
-#define   DPIO_PCS_TX_LANE2_RESET	(1 << 16)
-#define   DPIO_PCS_TX_LANE1_RESET	(1 << 7)
-#define   DPIO_LEFT_TXFIFO_RST_MASTER2	(1 << 4)
-#define   DPIO_RIGHT_TXFIFO_RST_MASTER2	(1 << 3)
-#define VLV_PCS_DW0(ch) _PORT(ch, _VLV_PCS_DW0_CH0, _VLV_PCS_DW0_CH1)
-
-#define _VLV_PCS01_DW0_CH0		0x200
-#define _VLV_PCS23_DW0_CH0		0x400
-#define _VLV_PCS01_DW0_CH1		0x2600
-#define _VLV_PCS23_DW0_CH1		0x2800
-#define VLV_PCS01_DW0(ch) _PORT(ch, _VLV_PCS01_DW0_CH0, _VLV_PCS01_DW0_CH1)
-#define VLV_PCS23_DW0(ch) _PORT(ch, _VLV_PCS23_DW0_CH0, _VLV_PCS23_DW0_CH1)
-
-#define _VLV_PCS_DW1_CH0		0x8204
-#define _VLV_PCS_DW1_CH1		0x8404
-#define   CHV_PCS_REQ_SOFTRESET_EN	(1 << 23)
-#define   DPIO_PCS_CLK_CRI_RXEB_EIOS_EN	(1 << 22)
-#define   DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN (1 << 21)
-#define   DPIO_PCS_CLK_DATAWIDTH_SHIFT	(6)
-#define   DPIO_PCS_CLK_SOFT_RESET	(1 << 5)
-#define VLV_PCS_DW1(ch) _PORT(ch, _VLV_PCS_DW1_CH0, _VLV_PCS_DW1_CH1)
-
-#define _VLV_PCS01_DW1_CH0		0x204
-#define _VLV_PCS23_DW1_CH0		0x404
-#define _VLV_PCS01_DW1_CH1		0x2604
-#define _VLV_PCS23_DW1_CH1		0x2804
-#define VLV_PCS01_DW1(ch) _PORT(ch, _VLV_PCS01_DW1_CH0, _VLV_PCS01_DW1_CH1)
-#define VLV_PCS23_DW1(ch) _PORT(ch, _VLV_PCS23_DW1_CH0, _VLV_PCS23_DW1_CH1)
-
-#define _VLV_PCS_DW8_CH0		0x8220
-#define _VLV_PCS_DW8_CH1		0x8420
-#define   CHV_PCS_USEDCLKCHANNEL_OVRRIDE	(1 << 20)
-#define   CHV_PCS_USEDCLKCHANNEL		(1 << 21)
-#define VLV_PCS_DW8(ch) _PORT(ch, _VLV_PCS_DW8_CH0, _VLV_PCS_DW8_CH1)
-
-#define _VLV_PCS01_DW8_CH0		0x0220
-#define _VLV_PCS23_DW8_CH0		0x0420
-#define _VLV_PCS01_DW8_CH1		0x2620
-#define _VLV_PCS23_DW8_CH1		0x2820
-#define VLV_PCS01_DW8(port) _PORT(port, _VLV_PCS01_DW8_CH0, _VLV_PCS01_DW8_CH1)
-#define VLV_PCS23_DW8(port) _PORT(port, _VLV_PCS23_DW8_CH0, _VLV_PCS23_DW8_CH1)
-
-#define _VLV_PCS_DW9_CH0		0x8224
-#define _VLV_PCS_DW9_CH1		0x8424
-#define   DPIO_PCS_TX2MARGIN_MASK	(0x7 << 13)
-#define   DPIO_PCS_TX2MARGIN_000	(0 << 13)
-#define   DPIO_PCS_TX2MARGIN_101	(1 << 13)
-#define   DPIO_PCS_TX1MARGIN_MASK	(0x7 << 10)
-#define   DPIO_PCS_TX1MARGIN_000	(0 << 10)
-#define   DPIO_PCS_TX1MARGIN_101	(1 << 10)
-#define	VLV_PCS_DW9(ch) _PORT(ch, _VLV_PCS_DW9_CH0, _VLV_PCS_DW9_CH1)
-
-#define _VLV_PCS01_DW9_CH0		0x224
-#define _VLV_PCS23_DW9_CH0		0x424
-#define _VLV_PCS01_DW9_CH1		0x2624
-#define _VLV_PCS23_DW9_CH1		0x2824
-#define VLV_PCS01_DW9(ch) _PORT(ch, _VLV_PCS01_DW9_CH0, _VLV_PCS01_DW9_CH1)
-#define VLV_PCS23_DW9(ch) _PORT(ch, _VLV_PCS23_DW9_CH0, _VLV_PCS23_DW9_CH1)
-
-#define _CHV_PCS_DW10_CH0		0x8228
-#define _CHV_PCS_DW10_CH1		0x8428
-#define   DPIO_PCS_SWING_CALC_TX0_TX2	(1 << 30)
-#define   DPIO_PCS_SWING_CALC_TX1_TX3	(1 << 31)
-#define   DPIO_PCS_TX2DEEMP_MASK	(0xf << 24)
-#define   DPIO_PCS_TX2DEEMP_9P5		(0 << 24)
-#define   DPIO_PCS_TX2DEEMP_6P0		(2 << 24)
-#define   DPIO_PCS_TX1DEEMP_MASK	(0xf << 16)
-#define   DPIO_PCS_TX1DEEMP_9P5		(0 << 16)
-#define   DPIO_PCS_TX1DEEMP_6P0		(2 << 16)
-#define CHV_PCS_DW10(ch) _PORT(ch, _CHV_PCS_DW10_CH0, _CHV_PCS_DW10_CH1)
-
-#define _VLV_PCS01_DW10_CH0		0x0228
-#define _VLV_PCS23_DW10_CH0		0x0428
-#define _VLV_PCS01_DW10_CH1		0x2628
-#define _VLV_PCS23_DW10_CH1		0x2828
-#define VLV_PCS01_DW10(port) _PORT(port, _VLV_PCS01_DW10_CH0, _VLV_PCS01_DW10_CH1)
-#define VLV_PCS23_DW10(port) _PORT(port, _VLV_PCS23_DW10_CH0, _VLV_PCS23_DW10_CH1)
-
-#define _VLV_PCS_DW11_CH0		0x822c
-#define _VLV_PCS_DW11_CH1		0x842c
-#define   DPIO_TX2_STAGGER_MASK(x)	((x) << 24)
-#define   DPIO_LANEDESKEW_STRAP_OVRD	(1 << 3)
-#define   DPIO_LEFT_TXFIFO_RST_MASTER	(1 << 1)
-#define   DPIO_RIGHT_TXFIFO_RST_MASTER	(1 << 0)
-#define VLV_PCS_DW11(ch) _PORT(ch, _VLV_PCS_DW11_CH0, _VLV_PCS_DW11_CH1)
-
-#define _VLV_PCS01_DW11_CH0		0x022c
-#define _VLV_PCS23_DW11_CH0		0x042c
-#define _VLV_PCS01_DW11_CH1		0x262c
-#define _VLV_PCS23_DW11_CH1		0x282c
-#define VLV_PCS01_DW11(ch) _PORT(ch, _VLV_PCS01_DW11_CH0, _VLV_PCS01_DW11_CH1)
-#define VLV_PCS23_DW11(ch) _PORT(ch, _VLV_PCS23_DW11_CH0, _VLV_PCS23_DW11_CH1)
-
-#define _VLV_PCS01_DW12_CH0		0x0230
-#define _VLV_PCS23_DW12_CH0		0x0430
-#define _VLV_PCS01_DW12_CH1		0x2630
-#define _VLV_PCS23_DW12_CH1		0x2830
-#define VLV_PCS01_DW12(ch) _PORT(ch, _VLV_PCS01_DW12_CH0, _VLV_PCS01_DW12_CH1)
-#define VLV_PCS23_DW12(ch) _PORT(ch, _VLV_PCS23_DW12_CH0, _VLV_PCS23_DW12_CH1)
-
-#define _VLV_PCS_DW12_CH0		0x8230
-#define _VLV_PCS_DW12_CH1		0x8430
-#define   DPIO_TX2_STAGGER_MULT(x)	((x) << 20)
-#define   DPIO_TX1_STAGGER_MULT(x)	((x) << 16)
-#define   DPIO_TX1_STAGGER_MASK(x)	((x) << 8)
-#define   DPIO_LANESTAGGER_STRAP_OVRD	(1 << 6)
-#define   DPIO_LANESTAGGER_STRAP(x)	((x) << 0)
-#define VLV_PCS_DW12(ch) _PORT(ch, _VLV_PCS_DW12_CH0, _VLV_PCS_DW12_CH1)
-
-#define _VLV_PCS_DW14_CH0		0x8238
-#define _VLV_PCS_DW14_CH1		0x8438
-#define	VLV_PCS_DW14(ch) _PORT(ch, _VLV_PCS_DW14_CH0, _VLV_PCS_DW14_CH1)
-
-#define _VLV_PCS_DW23_CH0		0x825c
-#define _VLV_PCS_DW23_CH1		0x845c
-#define VLV_PCS_DW23(ch) _PORT(ch, _VLV_PCS_DW23_CH0, _VLV_PCS_DW23_CH1)
-
-#define _VLV_TX_DW2_CH0			0x8288
-#define _VLV_TX_DW2_CH1			0x8488
-#define   DPIO_SWING_MARGIN000_SHIFT	16
-#define   DPIO_SWING_MARGIN000_MASK	(0xff << DPIO_SWING_MARGIN000_SHIFT)
-#define   DPIO_UNIQ_TRANS_SCALE_SHIFT	8
-#define VLV_TX_DW2(ch) _PORT(ch, _VLV_TX_DW2_CH0, _VLV_TX_DW2_CH1)
-
-#define _VLV_TX_DW3_CH0			0x828c
-#define _VLV_TX_DW3_CH1			0x848c
-/* The following bit for CHV phy */
-#define   DPIO_TX_UNIQ_TRANS_SCALE_EN	(1 << 27)
-#define   DPIO_SWING_MARGIN101_SHIFT	16
-#define   DPIO_SWING_MARGIN101_MASK	(0xff << DPIO_SWING_MARGIN101_SHIFT)
-#define VLV_TX_DW3(ch) _PORT(ch, _VLV_TX_DW3_CH0, _VLV_TX_DW3_CH1)
-
-#define _VLV_TX_DW4_CH0			0x8290
-#define _VLV_TX_DW4_CH1			0x8490
-#define   DPIO_SWING_DEEMPH9P5_SHIFT	24
-#define   DPIO_SWING_DEEMPH9P5_MASK	(0xff << DPIO_SWING_DEEMPH9P5_SHIFT)
-#define   DPIO_SWING_DEEMPH6P0_SHIFT	16
-#define   DPIO_SWING_DEEMPH6P0_MASK	(0xff << DPIO_SWING_DEEMPH6P0_SHIFT)
-#define VLV_TX_DW4(ch) _PORT(ch, _VLV_TX_DW4_CH0, _VLV_TX_DW4_CH1)
-
-#define _VLV_TX3_DW4_CH0		0x690
-#define _VLV_TX3_DW4_CH1		0x2a90
-#define VLV_TX3_DW4(ch) _PORT(ch, _VLV_TX3_DW4_CH0, _VLV_TX3_DW4_CH1)
-
-#define _VLV_TX_DW5_CH0			0x8294
-#define _VLV_TX_DW5_CH1			0x8494
-#define   DPIO_TX_OCALINIT_EN		(1 << 31)
-#define VLV_TX_DW5(ch) _PORT(ch, _VLV_TX_DW5_CH0, _VLV_TX_DW5_CH1)
-
-#define _VLV_TX_DW11_CH0		0x82ac
-#define _VLV_TX_DW11_CH1		0x84ac
-#define VLV_TX_DW11(ch) _PORT(ch, _VLV_TX_DW11_CH0, _VLV_TX_DW11_CH1)
-
-#define _VLV_TX_DW14_CH0		0x82b8
-#define _VLV_TX_DW14_CH1		0x84b8
-#define VLV_TX_DW14(ch) _PORT(ch, _VLV_TX_DW14_CH0, _VLV_TX_DW14_CH1)
-
-/* CHV dpPhy registers */
-#define _CHV_PLL_DW0_CH0		0x8000
-#define _CHV_PLL_DW0_CH1		0x8180
-#define CHV_PLL_DW0(ch) _PIPE(ch, _CHV_PLL_DW0_CH0, _CHV_PLL_DW0_CH1)
-
-#define _CHV_PLL_DW1_CH0		0x8004
-#define _CHV_PLL_DW1_CH1		0x8184
-#define   DPIO_CHV_N_DIV_SHIFT		8
-#define   DPIO_CHV_M1_DIV_BY_2		(0 << 0)
-#define CHV_PLL_DW1(ch) _PIPE(ch, _CHV_PLL_DW1_CH0, _CHV_PLL_DW1_CH1)
-
-#define _CHV_PLL_DW2_CH0		0x8008
-#define _CHV_PLL_DW2_CH1		0x8188
-#define CHV_PLL_DW2(ch) _PIPE(ch, _CHV_PLL_DW2_CH0, _CHV_PLL_DW2_CH1)
-
-#define _CHV_PLL_DW3_CH0		0x800c
-#define _CHV_PLL_DW3_CH1		0x818c
-#define  DPIO_CHV_FRAC_DIV_EN		(1 << 16)
-#define  DPIO_CHV_FIRST_MOD		(0 << 8)
-#define  DPIO_CHV_SECOND_MOD		(1 << 8)
-#define  DPIO_CHV_FEEDFWD_GAIN_SHIFT	0
-#define  DPIO_CHV_FEEDFWD_GAIN_MASK		(0xF << 0)
-#define CHV_PLL_DW3(ch) _PIPE(ch, _CHV_PLL_DW3_CH0, _CHV_PLL_DW3_CH1)
-
-#define _CHV_PLL_DW6_CH0		0x8018
-#define _CHV_PLL_DW6_CH1		0x8198
-#define   DPIO_CHV_GAIN_CTRL_SHIFT	16
-#define	  DPIO_CHV_INT_COEFF_SHIFT	8
-#define   DPIO_CHV_PROP_COEFF_SHIFT	0
-#define CHV_PLL_DW6(ch) _PIPE(ch, _CHV_PLL_DW6_CH0, _CHV_PLL_DW6_CH1)
-
-#define _CHV_PLL_DW8_CH0		0x8020
-#define _CHV_PLL_DW8_CH1		0x81A0
-#define   DPIO_CHV_TDC_TARGET_CNT_SHIFT 0
-#define   DPIO_CHV_TDC_TARGET_CNT_MASK  (0x3FF << 0)
-#define CHV_PLL_DW8(ch) _PIPE(ch, _CHV_PLL_DW8_CH0, _CHV_PLL_DW8_CH1)
-
-#define _CHV_PLL_DW9_CH0		0x8024
-#define _CHV_PLL_DW9_CH1		0x81A4
-#define  DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT		1 /* 3 bits */
-#define  DPIO_CHV_INT_LOCK_THRESHOLD_MASK		(7 << 1)
-#define  DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE	1 /* 1: coarse & 0 : fine  */
-#define CHV_PLL_DW9(ch) _PIPE(ch, _CHV_PLL_DW9_CH0, _CHV_PLL_DW9_CH1)
-
-#define _CHV_CMN_DW0_CH0               0x8100
-#define   DPIO_ALLDL_POWERDOWN_SHIFT_CH0	19
-#define   DPIO_ANYDL_POWERDOWN_SHIFT_CH0	18
-#define   DPIO_ALLDL_POWERDOWN			(1 << 1)
-#define   DPIO_ANYDL_POWERDOWN			(1 << 0)
-
-#define _CHV_CMN_DW5_CH0               0x8114
-#define   CHV_BUFRIGHTENA1_DISABLE	(0 << 20)
-#define   CHV_BUFRIGHTENA1_NORMAL	(1 << 20)
-#define   CHV_BUFRIGHTENA1_FORCE	(3 << 20)
-#define   CHV_BUFRIGHTENA1_MASK		(3 << 20)
-#define   CHV_BUFLEFTENA1_DISABLE	(0 << 22)
-#define   CHV_BUFLEFTENA1_NORMAL	(1 << 22)
-#define   CHV_BUFLEFTENA1_FORCE		(3 << 22)
-#define   CHV_BUFLEFTENA1_MASK		(3 << 22)
-
-#define _CHV_CMN_DW13_CH0		0x8134
-#define _CHV_CMN_DW0_CH1		0x8080
-#define   DPIO_CHV_S1_DIV_SHIFT		21
-#define   DPIO_CHV_P1_DIV_SHIFT		13 /* 3 bits */
-#define   DPIO_CHV_P2_DIV_SHIFT		8  /* 5 bits */
-#define   DPIO_CHV_K_DIV_SHIFT		4
-#define   DPIO_PLL_FREQLOCK		(1 << 1)
-#define   DPIO_PLL_LOCK			(1 << 0)
-#define CHV_CMN_DW13(ch) _PIPE(ch, _CHV_CMN_DW13_CH0, _CHV_CMN_DW0_CH1)
-
-#define _CHV_CMN_DW14_CH0		0x8138
-#define _CHV_CMN_DW1_CH1		0x8084
-#define   DPIO_AFC_RECAL		(1 << 14)
-#define   DPIO_DCLKP_EN			(1 << 13)
-#define   CHV_BUFLEFTENA2_DISABLE	(0 << 17) /* CL2 DW1 only */
-#define   CHV_BUFLEFTENA2_NORMAL	(1 << 17) /* CL2 DW1 only */
-#define   CHV_BUFLEFTENA2_FORCE		(3 << 17) /* CL2 DW1 only */
-#define   CHV_BUFLEFTENA2_MASK		(3 << 17) /* CL2 DW1 only */
-#define   CHV_BUFRIGHTENA2_DISABLE	(0 << 19) /* CL2 DW1 only */
-#define   CHV_BUFRIGHTENA2_NORMAL	(1 << 19) /* CL2 DW1 only */
-#define   CHV_BUFRIGHTENA2_FORCE	(3 << 19) /* CL2 DW1 only */
-#define   CHV_BUFRIGHTENA2_MASK		(3 << 19) /* CL2 DW1 only */
-#define CHV_CMN_DW14(ch) _PIPE(ch, _CHV_CMN_DW14_CH0, _CHV_CMN_DW1_CH1)
-
-#define _CHV_CMN_DW19_CH0		0x814c
-#define _CHV_CMN_DW6_CH1		0x8098
-#define   DPIO_ALLDL_POWERDOWN_SHIFT_CH1	30 /* CL2 DW6 only */
-#define   DPIO_ANYDL_POWERDOWN_SHIFT_CH1	29 /* CL2 DW6 only */
-#define   DPIO_DYNPWRDOWNEN_CH1		(1 << 28) /* CL2 DW6 only */
-#define   CHV_CMN_USEDCLKCHANNEL	(1 << 13)
-
-#define CHV_CMN_DW19(ch) _PIPE(ch, _CHV_CMN_DW19_CH0, _CHV_CMN_DW6_CH1)
-
-#define CHV_CMN_DW28			0x8170
-#define   DPIO_CL1POWERDOWNEN		(1 << 23)
-#define   DPIO_DYNPWRDOWNEN_CH0		(1 << 22)
-#define   DPIO_SUS_CLK_CONFIG_ON		(0 << 0)
-#define   DPIO_SUS_CLK_CONFIG_CLKREQ		(1 << 0)
-#define   DPIO_SUS_CLK_CONFIG_GATE		(2 << 0)
-#define   DPIO_SUS_CLK_CONFIG_GATE_CLKREQ	(3 << 0)
-
-#define CHV_CMN_DW30			0x8178
-#define   DPIO_CL2_LDOFUSE_PWRENB	(1 << 6)
-#define   DPIO_LRC_BYPASS		(1 << 3)
-
-#define _TXLANE(ch, lane, offset) ((ch ? 0x2400 : 0) + \
-					(lane) * 0x200 + (offset))
-
-#define CHV_TX_DW0(ch, lane) _TXLANE(ch, lane, 0x80)
-#define CHV_TX_DW1(ch, lane) _TXLANE(ch, lane, 0x84)
-#define CHV_TX_DW2(ch, lane) _TXLANE(ch, lane, 0x88)
-#define CHV_TX_DW3(ch, lane) _TXLANE(ch, lane, 0x8c)
-#define CHV_TX_DW4(ch, lane) _TXLANE(ch, lane, 0x90)
-#define CHV_TX_DW5(ch, lane) _TXLANE(ch, lane, 0x94)
-#define CHV_TX_DW6(ch, lane) _TXLANE(ch, lane, 0x98)
-#define CHV_TX_DW7(ch, lane) _TXLANE(ch, lane, 0x9c)
-#define CHV_TX_DW8(ch, lane) _TXLANE(ch, lane, 0xa0)
-#define CHV_TX_DW9(ch, lane) _TXLANE(ch, lane, 0xa4)
-#define CHV_TX_DW10(ch, lane) _TXLANE(ch, lane, 0xa8)
-#define CHV_TX_DW11(ch, lane) _TXLANE(ch, lane, 0xac)
-#define   DPIO_FRC_LATENCY_SHFIT	8
-#define CHV_TX_DW14(ch, lane) _TXLANE(ch, lane, 0xb8)
-#define   DPIO_UPAR_SHIFT		30
-
-/* BXT PHY registers */
-#define _BXT_PHY0_BASE			0x6C000
-#define _BXT_PHY1_BASE			0x162000
-#define _BXT_PHY2_BASE			0x163000
-#define BXT_PHY_BASE(phy)							\
-	 _PICK_EVEN_2RANGES(phy, 1,						\
-			    _BXT_PHY0_BASE, _BXT_PHY0_BASE,			\
-			    _BXT_PHY1_BASE, _BXT_PHY2_BASE)
-
-#define _BXT_PHY(phy, reg)						\
-	_MMIO(BXT_PHY_BASE(phy) - _BXT_PHY0_BASE + (reg))
-
-#define _BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1)		\
-	(BXT_PHY_BASE(phy) + _PIPE((ch), (reg_ch0) - _BXT_PHY0_BASE,	\
-					 (reg_ch1) - _BXT_PHY0_BASE))
-#define _MMIO_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1)		\
-	_MMIO(_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1))
-
 #define BXT_P_CR_GT_DISP_PWRON		_MMIO(0x138090)
 #define  MIPIO_RST_CTRL				(1 << 2)
 
@@ -577,250 +216,6 @@
 				  _PHY_CTL_FAMILY_DDI, _PHY_CTL_FAMILY_DDI,	\
 				  _PHY_CTL_FAMILY_EDP, _PHY_CTL_FAMILY_DDI_C))
 
-/* BXT PHY PLL registers */
-#define _PORT_PLL_A			0x46074
-#define _PORT_PLL_B			0x46078
-#define _PORT_PLL_C			0x4607c
-#define   PORT_PLL_ENABLE		REG_BIT(31)
-#define   PORT_PLL_LOCK			REG_BIT(30)
-#define   PORT_PLL_REF_SEL		REG_BIT(27)
-#define   PORT_PLL_POWER_ENABLE		REG_BIT(26)
-#define   PORT_PLL_POWER_STATE		REG_BIT(25)
-#define BXT_PORT_PLL_ENABLE(port)	_MMIO_PORT(port, _PORT_PLL_A, _PORT_PLL_B)
-
-#define _PORT_PLL_EBB_0_A		0x162034
-#define _PORT_PLL_EBB_0_B		0x6C034
-#define _PORT_PLL_EBB_0_C		0x6C340
-#define   PORT_PLL_P1_MASK		REG_GENMASK(15, 13)
-#define   PORT_PLL_P1(p1)		REG_FIELD_PREP(PORT_PLL_P1_MASK, (p1))
-#define   PORT_PLL_P2_MASK		REG_GENMASK(12, 8)
-#define   PORT_PLL_P2(p2)		REG_FIELD_PREP(PORT_PLL_P2_MASK, (p2))
-#define BXT_PORT_PLL_EBB_0(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_PLL_EBB_0_B, \
-							 _PORT_PLL_EBB_0_C)
-
-#define _PORT_PLL_EBB_4_A		0x162038
-#define _PORT_PLL_EBB_4_B		0x6C038
-#define _PORT_PLL_EBB_4_C		0x6C344
-#define   PORT_PLL_RECALIBRATE		REG_BIT(14)
-#define   PORT_PLL_10BIT_CLK_ENABLE	REG_BIT(13)
-#define BXT_PORT_PLL_EBB_4(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_PLL_EBB_4_B, \
-							 _PORT_PLL_EBB_4_C)
-
-#define _PORT_PLL_0_A			0x162100
-#define _PORT_PLL_0_B			0x6C100
-#define _PORT_PLL_0_C			0x6C380
-/* PORT_PLL_0_A */
-#define   PORT_PLL_M2_INT_MASK		REG_GENMASK(7, 0)
-#define   PORT_PLL_M2_INT(m2_int)	REG_FIELD_PREP(PORT_PLL_M2_INT_MASK, (m2_int))
-/* PORT_PLL_1_A */
-#define   PORT_PLL_N_MASK		REG_GENMASK(11, 8)
-#define   PORT_PLL_N(n)			REG_FIELD_PREP(PORT_PLL_N_MASK, (n))
-/* PORT_PLL_2_A */
-#define   PORT_PLL_M2_FRAC_MASK		REG_GENMASK(21, 0)
-#define   PORT_PLL_M2_FRAC(m2_frac)	REG_FIELD_PREP(PORT_PLL_M2_FRAC_MASK, (m2_frac))
-/* PORT_PLL_3_A */
-#define   PORT_PLL_M2_FRAC_ENABLE	REG_BIT(16)
-/* PORT_PLL_6_A */
-#define   PORT_PLL_GAIN_CTL_MASK	REG_GENMASK(18, 16)
-#define   PORT_PLL_GAIN_CTL(x)		REG_FIELD_PREP(PORT_PLL_GAIN_CTL_MASK, (x))
-#define   PORT_PLL_INT_COEFF_MASK	REG_GENMASK(12, 8)
-#define   PORT_PLL_INT_COEFF(x)		REG_FIELD_PREP(PORT_PLL_INT_COEFF_MASK, (x))
-#define   PORT_PLL_PROP_COEFF_MASK	REG_GENMASK(3, 0)
-#define   PORT_PLL_PROP_COEFF(x)	REG_FIELD_PREP(PORT_PLL_PROP_COEFF_MASK, (x))
-/* PORT_PLL_8_A */
-#define   PORT_PLL_TARGET_CNT_MASK	REG_GENMASK(9, 0)
-#define   PORT_PLL_TARGET_CNT(x)	REG_FIELD_PREP(PORT_PLL_TARGET_CNT_MASK, (x))
-/* PORT_PLL_9_A */
-#define  PORT_PLL_LOCK_THRESHOLD_MASK	REG_GENMASK(3, 1)
-#define  PORT_PLL_LOCK_THRESHOLD(x)	REG_FIELD_PREP(PORT_PLL_LOCK_THRESHOLD_MASK, (x))
-/* PORT_PLL_10_A */
-#define  PORT_PLL_DCO_AMP_OVR_EN_H	REG_BIT(27)
-#define  PORT_PLL_DCO_AMP_MASK		REG_GENMASK(13, 10)
-#define  PORT_PLL_DCO_AMP(x)		REG_FIELD_PREP(PORT_PLL_DCO_AMP_MASK, (x))
-#define _PORT_PLL_BASE(phy, ch)		_BXT_PHY_CH(phy, ch, \
-						    _PORT_PLL_0_B, \
-						    _PORT_PLL_0_C)
-#define BXT_PORT_PLL(phy, ch, idx)	_MMIO(_PORT_PLL_BASE(phy, ch) + \
-					      (idx) * 4)
-
-/* BXT PHY common lane registers */
-#define _PORT_CL1CM_DW0_A		0x162000
-#define _PORT_CL1CM_DW0_BC		0x6C000
-#define   PHY_POWER_GOOD		(1 << 16)
-#define   PHY_RESERVED			(1 << 7)
-#define BXT_PORT_CL1CM_DW0(phy)		_BXT_PHY((phy), _PORT_CL1CM_DW0_BC)
-
-#define _PORT_CL1CM_DW9_A		0x162024
-#define _PORT_CL1CM_DW9_BC		0x6C024
-#define   IREF0RC_OFFSET_SHIFT		8
-#define   IREF0RC_OFFSET_MASK		(0xFF << IREF0RC_OFFSET_SHIFT)
-#define BXT_PORT_CL1CM_DW9(phy)		_BXT_PHY((phy), _PORT_CL1CM_DW9_BC)
-
-#define _PORT_CL1CM_DW10_A		0x162028
-#define _PORT_CL1CM_DW10_BC		0x6C028
-#define   IREF1RC_OFFSET_SHIFT		8
-#define   IREF1RC_OFFSET_MASK		(0xFF << IREF1RC_OFFSET_SHIFT)
-#define BXT_PORT_CL1CM_DW10(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW10_BC)
-
-#define _PORT_CL1CM_DW28_A		0x162070
-#define _PORT_CL1CM_DW28_BC		0x6C070
-#define   OCL1_POWER_DOWN_EN		(1 << 23)
-#define   DW28_OLDO_DYN_PWR_DOWN_EN	(1 << 22)
-#define   SUS_CLK_CONFIG		0x3
-#define BXT_PORT_CL1CM_DW28(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW28_BC)
-
-#define _PORT_CL1CM_DW30_A		0x162078
-#define _PORT_CL1CM_DW30_BC		0x6C078
-#define   OCL2_LDOFUSE_PWR_DIS		(1 << 6)
-#define BXT_PORT_CL1CM_DW30(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW30_BC)
-
-/* The spec defines this only for BXT PHY0, but lets assume that this
- * would exist for PHY1 too if it had a second channel.
- */
-#define _PORT_CL2CM_DW6_A		0x162358
-#define _PORT_CL2CM_DW6_BC		0x6C358
-#define BXT_PORT_CL2CM_DW6(phy)		_BXT_PHY((phy), _PORT_CL2CM_DW6_BC)
-#define   DW6_OLDO_DYN_PWR_DOWN_EN	(1 << 28)
-
-/* BXT PHY Ref registers */
-#define _PORT_REF_DW3_A			0x16218C
-#define _PORT_REF_DW3_BC		0x6C18C
-#define   GRC_DONE			(1 << 22)
-#define BXT_PORT_REF_DW3(phy)		_BXT_PHY((phy), _PORT_REF_DW3_BC)
-
-#define _PORT_REF_DW6_A			0x162198
-#define _PORT_REF_DW6_BC		0x6C198
-#define   GRC_CODE_SHIFT		24
-#define   GRC_CODE_MASK			(0xFF << GRC_CODE_SHIFT)
-#define   GRC_CODE_FAST_SHIFT		16
-#define   GRC_CODE_FAST_MASK		(0xFF << GRC_CODE_FAST_SHIFT)
-#define   GRC_CODE_SLOW_SHIFT		8
-#define   GRC_CODE_SLOW_MASK		(0xFF << GRC_CODE_SLOW_SHIFT)
-#define   GRC_CODE_NOM_MASK		0xFF
-#define BXT_PORT_REF_DW6(phy)		_BXT_PHY((phy), _PORT_REF_DW6_BC)
-
-#define _PORT_REF_DW8_A			0x1621A0
-#define _PORT_REF_DW8_BC		0x6C1A0
-#define   GRC_DIS			(1 << 15)
-#define   GRC_RDY_OVRD			(1 << 1)
-#define BXT_PORT_REF_DW8(phy)		_BXT_PHY((phy), _PORT_REF_DW8_BC)
-
-/* BXT PHY PCS registers */
-#define _PORT_PCS_DW10_LN01_A		0x162428
-#define _PORT_PCS_DW10_LN01_B		0x6C428
-#define _PORT_PCS_DW10_LN01_C		0x6C828
-#define _PORT_PCS_DW10_GRP_A		0x162C28
-#define _PORT_PCS_DW10_GRP_B		0x6CC28
-#define _PORT_PCS_DW10_GRP_C		0x6CE28
-#define BXT_PORT_PCS_DW10_LN01(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_PCS_DW10_LN01_B, \
-							 _PORT_PCS_DW10_LN01_C)
-#define BXT_PORT_PCS_DW10_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_PCS_DW10_GRP_B, \
-							 _PORT_PCS_DW10_GRP_C)
-
-#define   TX2_SWING_CALC_INIT		(1 << 31)
-#define   TX1_SWING_CALC_INIT		(1 << 30)
-
-#define _PORT_PCS_DW12_LN01_A		0x162430
-#define _PORT_PCS_DW12_LN01_B		0x6C430
-#define _PORT_PCS_DW12_LN01_C		0x6C830
-#define _PORT_PCS_DW12_LN23_A		0x162630
-#define _PORT_PCS_DW12_LN23_B		0x6C630
-#define _PORT_PCS_DW12_LN23_C		0x6CA30
-#define _PORT_PCS_DW12_GRP_A		0x162c30
-#define _PORT_PCS_DW12_GRP_B		0x6CC30
-#define _PORT_PCS_DW12_GRP_C		0x6CE30
-#define   LANESTAGGER_STRAP_OVRD	(1 << 6)
-#define   LANE_STAGGER_MASK		0x1F
-#define BXT_PORT_PCS_DW12_LN01(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_PCS_DW12_LN01_B, \
-							 _PORT_PCS_DW12_LN01_C)
-#define BXT_PORT_PCS_DW12_LN23(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_PCS_DW12_LN23_B, \
-							 _PORT_PCS_DW12_LN23_C)
-#define BXT_PORT_PCS_DW12_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_PCS_DW12_GRP_B, \
-							 _PORT_PCS_DW12_GRP_C)
-
-/* BXT PHY TX registers */
-#define _BXT_LANE_OFFSET(lane)           (((lane) >> 1) * 0x200 +	\
-					  ((lane) & 1) * 0x80)
-
-#define _PORT_TX_DW2_LN0_A		0x162508
-#define _PORT_TX_DW2_LN0_B		0x6C508
-#define _PORT_TX_DW2_LN0_C		0x6C908
-#define _PORT_TX_DW2_GRP_A		0x162D08
-#define _PORT_TX_DW2_GRP_B		0x6CD08
-#define _PORT_TX_DW2_GRP_C		0x6CF08
-#define BXT_PORT_TX_DW2_LN0(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_TX_DW2_LN0_B, \
-							 _PORT_TX_DW2_LN0_C)
-#define BXT_PORT_TX_DW2_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_TX_DW2_GRP_B, \
-							 _PORT_TX_DW2_GRP_C)
-#define   MARGIN_000_SHIFT		16
-#define   MARGIN_000			(0xFF << MARGIN_000_SHIFT)
-#define   UNIQ_TRANS_SCALE_SHIFT	8
-#define   UNIQ_TRANS_SCALE		(0xFF << UNIQ_TRANS_SCALE_SHIFT)
-
-#define _PORT_TX_DW3_LN0_A		0x16250C
-#define _PORT_TX_DW3_LN0_B		0x6C50C
-#define _PORT_TX_DW3_LN0_C		0x6C90C
-#define _PORT_TX_DW3_GRP_A		0x162D0C
-#define _PORT_TX_DW3_GRP_B		0x6CD0C
-#define _PORT_TX_DW3_GRP_C		0x6CF0C
-#define BXT_PORT_TX_DW3_LN0(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_TX_DW3_LN0_B, \
-							 _PORT_TX_DW3_LN0_C)
-#define BXT_PORT_TX_DW3_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_TX_DW3_GRP_B, \
-							 _PORT_TX_DW3_GRP_C)
-#define   SCALE_DCOMP_METHOD		(1 << 26)
-#define   UNIQUE_TRANGE_EN_METHOD	(1 << 27)
-
-#define _PORT_TX_DW4_LN0_A		0x162510
-#define _PORT_TX_DW4_LN0_B		0x6C510
-#define _PORT_TX_DW4_LN0_C		0x6C910
-#define _PORT_TX_DW4_GRP_A		0x162D10
-#define _PORT_TX_DW4_GRP_B		0x6CD10
-#define _PORT_TX_DW4_GRP_C		0x6CF10
-#define BXT_PORT_TX_DW4_LN0(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_TX_DW4_LN0_B, \
-							 _PORT_TX_DW4_LN0_C)
-#define BXT_PORT_TX_DW4_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_TX_DW4_GRP_B, \
-							 _PORT_TX_DW4_GRP_C)
-#define   DEEMPH_SHIFT			24
-#define   DE_EMPHASIS			(0xFF << DEEMPH_SHIFT)
-
-#define _PORT_TX_DW5_LN0_A		0x162514
-#define _PORT_TX_DW5_LN0_B		0x6C514
-#define _PORT_TX_DW5_LN0_C		0x6C914
-#define _PORT_TX_DW5_GRP_A		0x162D14
-#define _PORT_TX_DW5_GRP_B		0x6CD14
-#define _PORT_TX_DW5_GRP_C		0x6CF14
-#define BXT_PORT_TX_DW5_LN0(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_TX_DW5_LN0_B, \
-							 _PORT_TX_DW5_LN0_C)
-#define BXT_PORT_TX_DW5_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
-							 _PORT_TX_DW5_GRP_B, \
-							 _PORT_TX_DW5_GRP_C)
-#define   DCC_DELAY_RANGE_1		(1 << 9)
-#define   DCC_DELAY_RANGE_2		(1 << 8)
-
-#define _PORT_TX_DW14_LN0_A		0x162538
-#define _PORT_TX_DW14_LN0_B		0x6C538
-#define _PORT_TX_DW14_LN0_C		0x6C938
-#define   LATENCY_OPTIM_SHIFT		30
-#define   LATENCY_OPTIM			(1 << LATENCY_OPTIM_SHIFT)
-#define BXT_PORT_TX_DW14_LN(phy, ch, lane)				\
-	_MMIO(_BXT_PHY_CH(phy, ch, _PORT_TX_DW14_LN0_B,			\
-				   _PORT_TX_DW14_LN0_C) +		\
-	      _BXT_LANE_OFFSET(lane))
-
 /* UAIMI scratch pad register 1 */
 #define UAIMI_SPR1			_MMIO(0x4F074)
 /* SKL VccIO mask */
@@ -1228,22 +623,6 @@
 #define I915_ASLE_INTERRUPT				(1 << 0)
 #define I915_BSD_USER_INTERRUPT				(1 << 25)
 
-#define I915_HDMI_LPE_AUDIO_BASE	(VLV_DISPLAY_BASE + 0x65000)
-#define I915_HDMI_LPE_AUDIO_SIZE	0x1000
-
-/* DisplayPort Audio w/ LPE */
-#define VLV_AUD_CHICKEN_BIT_REG		_MMIO(VLV_DISPLAY_BASE + 0x62F38)
-#define VLV_CHICKEN_BIT_DBG_ENABLE	(1 << 0)
-
-#define _VLV_AUD_PORT_EN_B_DBG		(VLV_DISPLAY_BASE + 0x62F20)
-#define _VLV_AUD_PORT_EN_C_DBG		(VLV_DISPLAY_BASE + 0x62F30)
-#define _VLV_AUD_PORT_EN_D_DBG		(VLV_DISPLAY_BASE + 0x62F34)
-#define VLV_AUD_PORT_EN_DBG(port)	_MMIO_PORT3((port) - PORT_B,	   \
-						    _VLV_AUD_PORT_EN_B_DBG, \
-						    _VLV_AUD_PORT_EN_C_DBG, \
-						    _VLV_AUD_PORT_EN_D_DBG)
-#define VLV_AMP_MUTE		        (1 << 1)
-
 #define GEN6_BSD_RNCID			_MMIO(0x12198)
 
 #define GEN7_FF_THREAD_MODE		_MMIO(0x20a0)
@@ -1264,109 +643,6 @@
 #define   GEN7_FF_DS_SCHED_LOAD_BALANCE	(0x1 << 4)  /* Default */
 #define   GEN7_FF_DS_SCHED_HW		(0x0 << 4)
 
-/*
- * Framebuffer compression (915+ only)
- */
-
-#define FBC_CFB_BASE		_MMIO(0x3200) /* 4k page aligned */
-#define FBC_LL_BASE		_MMIO(0x3204) /* 4k page aligned */
-#define FBC_CONTROL		_MMIO(0x3208)
-#define   FBC_CTL_EN			REG_BIT(31)
-#define   FBC_CTL_PERIODIC		REG_BIT(30)
-#define   FBC_CTL_INTERVAL_MASK		REG_GENMASK(29, 16)
-#define   FBC_CTL_INTERVAL(x)		REG_FIELD_PREP(FBC_CTL_INTERVAL_MASK, (x))
-#define   FBC_CTL_STOP_ON_MOD		REG_BIT(15)
-#define   FBC_CTL_UNCOMPRESSIBLE	REG_BIT(14) /* i915+ */
-#define   FBC_CTL_C3_IDLE		REG_BIT(13) /* i945gm only */
-#define   FBC_CTL_STRIDE_MASK		REG_GENMASK(12, 5)
-#define   FBC_CTL_STRIDE(x)		REG_FIELD_PREP(FBC_CTL_STRIDE_MASK, (x))
-#define   FBC_CTL_FENCENO_MASK		REG_GENMASK(3, 0)
-#define   FBC_CTL_FENCENO(x)		REG_FIELD_PREP(FBC_CTL_FENCENO_MASK, (x))
-#define FBC_COMMAND		_MMIO(0x320c)
-#define   FBC_CMD_COMPRESS		REG_BIT(0)
-#define FBC_STATUS		_MMIO(0x3210)
-#define   FBC_STAT_COMPRESSING		REG_BIT(31)
-#define   FBC_STAT_COMPRESSED		REG_BIT(30)
-#define   FBC_STAT_MODIFIED		REG_BIT(29)
-#define   FBC_STAT_CURRENT_LINE_MASK	REG_GENMASK(10, 0)
-#define FBC_CONTROL2		_MMIO(0x3214) /* i965gm only */
-#define   FBC_CTL_FENCE_DBL		REG_BIT(4)
-#define   FBC_CTL_IDLE_MASK		REG_GENMASK(3, 2)
-#define   FBC_CTL_IDLE_IMM		REG_FIELD_PREP(FBC_CTL_IDLE_MASK, 0)
-#define   FBC_CTL_IDLE_FULL		REG_FIELD_PREP(FBC_CTL_IDLE_MASK, 1)
-#define   FBC_CTL_IDLE_LINE		REG_FIELD_PREP(FBC_CTL_IDLE_MASK, 2)
-#define   FBC_CTL_IDLE_DEBUG		REG_FIELD_PREP(FBC_CTL_IDLE_MASK, 3)
-#define   FBC_CTL_CPU_FENCE_EN		REG_BIT(1)
-#define   FBC_CTL_PLANE_MASK		REG_GENMASK(1, 0)
-#define   FBC_CTL_PLANE(i9xx_plane)	REG_FIELD_PREP(FBC_CTL_PLANE_MASK, (i9xx_plane))
-#define FBC_FENCE_OFF		_MMIO(0x3218)  /* i965gm only, BSpec typo has 321Bh */
-#define FBC_MOD_NUM		_MMIO(0x3220)  /* i965gm only */
-#define   FBC_MOD_NUM_MASK		REG_GENMASK(31, 1)
-#define   FBC_MOD_NUM_VALID		REG_BIT(0)
-#define FBC_TAG(i)		_MMIO(0x3300 + (i) * 4) /* 49 reisters */
-#define   FBC_TAG_MASK			REG_GENMASK(1, 0) /* 16 tags per register */
-#define   FBC_TAG_MODIFIED		REG_FIELD_PREP(FBC_TAG_MASK, 0)
-#define   FBC_TAG_UNCOMPRESSED		REG_FIELD_PREP(FBC_TAG_MASK, 1)
-#define   FBC_TAG_UNCOMPRESSIBLE	REG_FIELD_PREP(FBC_TAG_MASK, 2)
-#define   FBC_TAG_COMPRESSED		REG_FIELD_PREP(FBC_TAG_MASK, 3)
-
-#define FBC_LL_SIZE		(1536)
-
-/* Framebuffer compression for GM45+ */
-#define DPFC_CB_BASE			_MMIO(0x3200)
-#define ILK_DPFC_CB_BASE(fbc_id)	_MMIO_PIPE((fbc_id), 0x43200, 0x43240)
-#define DPFC_CONTROL			_MMIO(0x3208)
-#define ILK_DPFC_CONTROL(fbc_id)	_MMIO_PIPE((fbc_id), 0x43208, 0x43248)
-#define   DPFC_CTL_EN				REG_BIT(31)
-#define   DPFC_CTL_PLANE_MASK_G4X		REG_BIT(30) /* g4x-snb */
-#define   DPFC_CTL_PLANE_G4X(i9xx_plane)	REG_FIELD_PREP(DPFC_CTL_PLANE_MASK_G4X, (i9xx_plane))
-#define   DPFC_CTL_FENCE_EN_G4X			REG_BIT(29) /* g4x-snb */
-#define   DPFC_CTL_PLANE_MASK_IVB		REG_GENMASK(30, 29) /* ivb only */
-#define   DPFC_CTL_PLANE_IVB(i9xx_plane)	REG_FIELD_PREP(DPFC_CTL_PLANE_MASK_IVB, (i9xx_plane))
-#define   DPFC_CTL_FENCE_EN_IVB			REG_BIT(28) /* ivb+ */
-#define   DPFC_CTL_PERSISTENT_MODE		REG_BIT(25) /* g4x-snb */
-#define   DPFC_CTL_PLANE_BINDING_MASK		REG_GENMASK(12, 11) /* lnl+ */
-#define   DPFC_CTL_PLANE_BINDING(plane_id)	REG_FIELD_PREP(DPFC_CTL_PLANE_BINDING_MASK, (plane_id))
-#define   DPFC_CTL_FALSE_COLOR			REG_BIT(10) /* ivb+ */
-#define   DPFC_CTL_SR_EN			REG_BIT(10) /* g4x only */
-#define   DPFC_CTL_SR_EXIT_DIS			REG_BIT(9) /* g4x only */
-#define   DPFC_CTL_LIMIT_MASK			REG_GENMASK(7, 6)
-#define   DPFC_CTL_LIMIT_1X			REG_FIELD_PREP(DPFC_CTL_LIMIT_MASK, 0)
-#define   DPFC_CTL_LIMIT_2X			REG_FIELD_PREP(DPFC_CTL_LIMIT_MASK, 1)
-#define   DPFC_CTL_LIMIT_4X			REG_FIELD_PREP(DPFC_CTL_LIMIT_MASK, 2)
-#define   DPFC_CTL_FENCENO_MASK			REG_GENMASK(3, 0)
-#define   DPFC_CTL_FENCENO(fence)		REG_FIELD_PREP(DPFC_CTL_FENCENO_MASK, (fence))
-#define DPFC_RECOMP_CTL			_MMIO(0x320c)
-#define ILK_DPFC_RECOMP_CTL(fbc_id)	_MMIO_PIPE((fbc_id), 0x4320c, 0x4324c)
-#define   DPFC_RECOMP_STALL_EN			REG_BIT(27)
-#define   DPFC_RECOMP_STALL_WM_MASK		REG_GENMASK(26, 16)
-#define   DPFC_RECOMP_TIMER_COUNT_MASK		REG_GENMASK(5, 0)
-#define DPFC_STATUS			_MMIO(0x3210)
-#define ILK_DPFC_STATUS(fbc_id)		_MMIO_PIPE((fbc_id), 0x43210, 0x43250)
-#define   DPFC_INVAL_SEG_MASK			REG_GENMASK(26, 16)
-#define   DPFC_COMP_SEG_MASK			REG_GENMASK(10, 0)
-#define DPFC_STATUS2			_MMIO(0x3214)
-#define ILK_DPFC_STATUS2(fbc_id)	_MMIO_PIPE((fbc_id), 0x43214, 0x43254)
-#define   DPFC_COMP_SEG_MASK_IVB		REG_GENMASK(11, 0)
-#define DPFC_FENCE_YOFF			_MMIO(0x3218)
-#define ILK_DPFC_FENCE_YOFF(fbc_id)	_MMIO_PIPE((fbc_id), 0x43218, 0x43258)
-#define DPFC_CHICKEN			_MMIO(0x3224)
-#define ILK_DPFC_CHICKEN(fbc_id)	_MMIO_PIPE((fbc_id), 0x43224, 0x43264)
-#define   DPFC_HT_MODIFY			REG_BIT(31) /* pre-ivb */
-#define   DPFC_NUKE_ON_ANY_MODIFICATION		REG_BIT(23) /* bdw+ */
-#define   DPFC_CHICKEN_COMP_DUMMY_PIXEL		REG_BIT(14) /* glk+ */
-#define   DPFC_CHICKEN_FORCE_SLB_INVALIDATION	REG_BIT(13) /* icl+ */
-#define   DPFC_DISABLE_DUMMY0			REG_BIT(8) /* ivb+ */
-
-#define GLK_FBC_STRIDE(fbc_id)	_MMIO_PIPE((fbc_id), 0x43228, 0x43268)
-#define   FBC_STRIDE_OVERRIDE	REG_BIT(15)
-#define   FBC_STRIDE_MASK	REG_GENMASK(14, 0)
-#define   FBC_STRIDE(x)		REG_FIELD_PREP(FBC_STRIDE_MASK, (x))
-
-#define ILK_FBC_RT_BASE		_MMIO(0x2128)
-#define   ILK_FBC_RT_VALID	REG_BIT(0)
-#define   SNB_FBC_FRONT_BUFFER	REG_BIT(1)
-
 #define ILK_DISPLAY_CHICKEN1	_MMIO(0x42000)
 #define   ILK_FBCQ_DIS			REG_BIT(22)
 #define   ILK_PABSTRETCH_DIS		REG_BIT(21)
@@ -1382,37 +658,18 @@
 #define   IVB_SPR_STRETCH_MAX_X2	REG_FIELD_PREP(IVB_SPR_STRETCH_MAX_MASK, 2)
 #define   IVB_SPR_STRETCH_MAX_X1	REG_FIELD_PREP(IVB_SPR_STRETCH_MAX_MASK, 3)
 
-
-/*
- * Framebuffer compression for Sandybridge
- *
- * The following two registers are of type GTTMMADR
- */
-#define SNB_DPFC_CTL_SA		_MMIO(0x100100)
-#define   SNB_DPFC_FENCE_EN		REG_BIT(29)
-#define   SNB_DPFC_FENCENO_MASK		REG_GENMASK(4, 0)
-#define   SNB_DPFC_FENCENO(fence)	REG_FIELD_PREP(SNB_DPFC_FENCENO_MASK, (fence))
-#define SNB_DPFC_CPU_FENCE_OFFSET	_MMIO(0x100104)
-
-/* Framebuffer compression for Ivybridge */
-#define IVB_FBC_RT_BASE			_MMIO(0x7020)
-#define IVB_FBC_RT_BASE_UPPER		_MMIO(0x7024)
-
 #define IPS_CTL		_MMIO(0x43408)
 #define   IPS_ENABLE		REG_BIT(31)
 #define   IPS_FALSE_COLOR	REG_BIT(4)
 
-#define MSG_FBC_REND_STATE(fbc_id)	_MMIO_PIPE((fbc_id), 0x50380, 0x50384)
-#define   FBC_REND_NUKE			REG_BIT(2)
-#define   FBC_REND_CACHE_CLEAN		REG_BIT(1)
-
 /*
  * Clock control & power management
  */
-#define _DPLL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x6014)
-#define _DPLL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x6018)
-#define _CHV_DPLL_C (DISPLAY_MMIO_BASE(dev_priv) + 0x6030)
-#define DPLL(pipe) _MMIO_PIPE3((pipe), _DPLL_A, _DPLL_B, _CHV_DPLL_C)
+#define _DPLL_A			0x6014
+#define _DPLL_B			0x6018
+#define _CHV_DPLL_C		0x6030
+#define DPLL(pipe)		_MMIO_BASE_PIPE3(DISPLAY_MMIO_BASE(dev_priv), \
+						 (pipe), _DPLL_A, _DPLL_B, _CHV_DPLL_C)
 
 #define VGA0	_MMIO(0x6000)
 #define VGA1	_MMIO(0x6004)
@@ -1508,10 +765,11 @@
 #define   SDVO_MULTIPLIER_SHIFT_HIRES		4
 #define   SDVO_MULTIPLIER_SHIFT_VGA		0
 
-#define _DPLL_A_MD (DISPLAY_MMIO_BASE(dev_priv) + 0x601c)
-#define _DPLL_B_MD (DISPLAY_MMIO_BASE(dev_priv) + 0x6020)
-#define _CHV_DPLL_C_MD (DISPLAY_MMIO_BASE(dev_priv) + 0x603c)
-#define DPLL_MD(pipe) _MMIO_PIPE3((pipe), _DPLL_A_MD, _DPLL_B_MD, _CHV_DPLL_C_MD)
+#define _DPLL_A_MD		0x601c
+#define _DPLL_B_MD		0x6020
+#define _CHV_DPLL_C_MD		0x603c
+#define DPLL_MD(pipe)		_MMIO_BASE_PIPE3(DISPLAY_MMIO_BASE(dev_priv), \
+						 (pipe), _DPLL_A_MD, _DPLL_B_MD, _CHV_DPLL_C_MD)
 
 /*
  * UDI pixel divider, controlling how many pixels are stuffed into a packet.
@@ -1716,42 +974,10 @@
 
 #define GMBUSFREQ_VLV		_MMIO(VLV_DISPLAY_BASE + 0x6510)
 
-/*
- * Palette regs
- */
-#define _PALETTE_A		0xa000
-#define _PALETTE_B		0xa800
-#define _CHV_PALETTE_C		0xc000
-/* 8bit mode / i965+ 10.6 interpolated mode ldw/udw */
-#define   PALETTE_RED_MASK		REG_GENMASK(23, 16)
-#define   PALETTE_GREEN_MASK		REG_GENMASK(15, 8)
-#define   PALETTE_BLUE_MASK		REG_GENMASK(7, 0)
-/* pre-i965 10bit interpolated mode ldw */
-#define   PALETTE_10BIT_RED_LDW_MASK	REG_GENMASK(23, 16)
-#define   PALETTE_10BIT_GREEN_LDW_MASK	REG_GENMASK(15, 8)
-#define   PALETTE_10BIT_BLUE_LDW_MASK	REG_GENMASK(7, 0)
-/* pre-i965 10bit interpolated mode udw */
-#define   PALETTE_10BIT_RED_EXP_MASK	REG_GENMASK(23, 22)
-#define   PALETTE_10BIT_RED_MANT_MASK	REG_GENMASK(21, 18)
-#define   PALETTE_10BIT_RED_UDW_MASK	REG_GENMASK(17, 16)
-#define   PALETTE_10BIT_GREEN_EXP_MASK	REG_GENMASK(15, 14)
-#define   PALETTE_10BIT_GREEN_MANT_MASK	REG_GENMASK(13, 10)
-#define   PALETTE_10BIT_GREEN_UDW_MASK	REG_GENMASK(9, 8)
-#define   PALETTE_10BIT_BLUE_EXP_MASK	REG_GENMASK(7, 6)
-#define   PALETTE_10BIT_BLUE_MANT_MASK	REG_GENMASK(5, 2)
-#define   PALETTE_10BIT_BLUE_UDW_MASK	REG_GENMASK(1, 0)
-#define PALETTE(pipe, i) _MMIO(DISPLAY_MMIO_BASE(dev_priv) +			\
-			       _PICK_EVEN_2RANGES(pipe, 2,			\
-						  _PALETTE_A, _PALETTE_B,	\
-						  _CHV_PALETTE_C, _CHV_PALETTE_C) + \
-						  (i) * 4)
-
 #define PEG_BAND_GAP_DATA	_MMIO(0x14d68)
 
 #define BXT_RP_STATE_CAP        _MMIO(0x138170)
 #define GEN9_RP_STATE_LIMITS	_MMIO(0x138148)
-#define XEHPSDV_RP_STATE_CAP	_MMIO(0x250014)
-#define PVC_RP_STATE_CAP	_MMIO(0x281014)
 
 #define MTL_RP_STATE_CAP	_MMIO(0x138000)
 #define MTL_MEDIAP_STATE_CAP	_MMIO(0x138020)
@@ -1911,18 +1137,18 @@
 #define _PIPE_CRC_RES_4_B_IVB		0x61070
 #define _PIPE_CRC_RES_5_B_IVB		0x61074
 
-#define PIPE_CRC_CTL(pipe)		_MMIO_TRANS2(pipe, _PIPE_CRC_CTL_A)
-#define PIPE_CRC_RES_1_IVB(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_1_A_IVB)
-#define PIPE_CRC_RES_2_IVB(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_2_A_IVB)
-#define PIPE_CRC_RES_3_IVB(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_3_A_IVB)
-#define PIPE_CRC_RES_4_IVB(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_4_A_IVB)
-#define PIPE_CRC_RES_5_IVB(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_5_A_IVB)
+#define PIPE_CRC_CTL(pipe)		_MMIO_TRANS2(dev_priv, pipe, _PIPE_CRC_CTL_A)
+#define PIPE_CRC_RES_1_IVB(pipe)	_MMIO_TRANS2(dev_priv, pipe, _PIPE_CRC_RES_1_A_IVB)
+#define PIPE_CRC_RES_2_IVB(pipe)	_MMIO_TRANS2(dev_priv, pipe, _PIPE_CRC_RES_2_A_IVB)
+#define PIPE_CRC_RES_3_IVB(pipe)	_MMIO_TRANS2(dev_priv, pipe, _PIPE_CRC_RES_3_A_IVB)
+#define PIPE_CRC_RES_4_IVB(pipe)	_MMIO_TRANS2(dev_priv, pipe, _PIPE_CRC_RES_4_A_IVB)
+#define PIPE_CRC_RES_5_IVB(pipe)	_MMIO_TRANS2(dev_priv, pipe, _PIPE_CRC_RES_5_A_IVB)
 
-#define PIPE_CRC_RES_RED(pipe)		_MMIO_TRANS2(pipe, _PIPE_CRC_RES_RED_A)
-#define PIPE_CRC_RES_GREEN(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_GREEN_A)
-#define PIPE_CRC_RES_BLUE(pipe)		_MMIO_TRANS2(pipe, _PIPE_CRC_RES_BLUE_A)
-#define PIPE_CRC_RES_RES1_I915(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_RES1_A_I915)
-#define PIPE_CRC_RES_RES2_G4X(pipe)	_MMIO_TRANS2(pipe, _PIPE_CRC_RES_RES2_A_G4X)
+#define PIPE_CRC_RES_RED(pipe)		_MMIO_TRANS2(dev_priv, pipe, _PIPE_CRC_RES_RED_A)
+#define PIPE_CRC_RES_GREEN(pipe)	_MMIO_TRANS2(dev_priv, pipe, _PIPE_CRC_RES_GREEN_A)
+#define PIPE_CRC_RES_BLUE(pipe)		_MMIO_TRANS2(dev_priv, pipe, _PIPE_CRC_RES_BLUE_A)
+#define PIPE_CRC_RES_RES1_I915(pipe)	_MMIO_TRANS2(dev_priv, pipe, _PIPE_CRC_RES_RES1_A_I915)
+#define PIPE_CRC_RES_RES2_G4X(pipe)	_MMIO_TRANS2(dev_priv, pipe, _PIPE_CRC_RES_RES2_A_G4X)
 
 /* Pipe/transcoder A timing regs */
 #define _TRANS_HTOTAL_A		0x60000
@@ -1991,23 +1217,23 @@
 #define _TRANS_VSYNC_DSI1	0x6b814
 #define _TRANS_VSYNCSHIFT_DSI1	0x6b828
 
-#define TRANS_HTOTAL(trans)	_MMIO_TRANS2((trans), _TRANS_HTOTAL_A)
-#define TRANS_HBLANK(trans)	_MMIO_TRANS2((trans), _TRANS_HBLANK_A)
-#define TRANS_HSYNC(trans)	_MMIO_TRANS2((trans), _TRANS_HSYNC_A)
-#define TRANS_VTOTAL(trans)	_MMIO_TRANS2((trans), _TRANS_VTOTAL_A)
-#define TRANS_VBLANK(trans)	_MMIO_TRANS2((trans), _TRANS_VBLANK_A)
-#define TRANS_VSYNC(trans)	_MMIO_TRANS2((trans), _TRANS_VSYNC_A)
-#define BCLRPAT(trans)		_MMIO_TRANS2((trans), _BCLRPAT_A)
-#define TRANS_VSYNCSHIFT(trans)	_MMIO_TRANS2((trans), _TRANS_VSYNCSHIFT_A)
-#define PIPESRC(pipe)		_MMIO_TRANS2((pipe), _PIPEASRC)
-#define TRANS_MULT(trans)	_MMIO_TRANS2((trans), _TRANS_MULT_A)
+#define TRANS_HTOTAL(trans)	_MMIO_TRANS2(dev_priv, (trans), _TRANS_HTOTAL_A)
+#define TRANS_HBLANK(trans)	_MMIO_TRANS2(dev_priv, (trans), _TRANS_HBLANK_A)
+#define TRANS_HSYNC(trans)	_MMIO_TRANS2(dev_priv, (trans), _TRANS_HSYNC_A)
+#define TRANS_VTOTAL(trans)	_MMIO_TRANS2(dev_priv, (trans), _TRANS_VTOTAL_A)
+#define TRANS_VBLANK(trans)	_MMIO_TRANS2(dev_priv, (trans), _TRANS_VBLANK_A)
+#define TRANS_VSYNC(trans)	_MMIO_TRANS2(dev_priv, (trans), _TRANS_VSYNC_A)
+#define BCLRPAT(trans)		_MMIO_TRANS2(dev_priv, (trans), _BCLRPAT_A)
+#define TRANS_VSYNCSHIFT(trans)	_MMIO_TRANS2(dev_priv, (trans), _TRANS_VSYNCSHIFT_A)
+#define PIPESRC(pipe)		_MMIO_TRANS2(dev_priv, (pipe), _PIPEASRC)
+#define TRANS_MULT(trans)	_MMIO_TRANS2(dev_priv, (trans), _TRANS_MULT_A)
 
 /* VRR registers */
 #define _TRANS_VRR_CTL_A		0x60420
 #define _TRANS_VRR_CTL_B		0x61420
 #define _TRANS_VRR_CTL_C		0x62420
 #define _TRANS_VRR_CTL_D		0x63420
-#define TRANS_VRR_CTL(trans)			_MMIO_TRANS2(trans, _TRANS_VRR_CTL_A)
+#define TRANS_VRR_CTL(trans)			_MMIO_TRANS2(dev_priv, trans, _TRANS_VRR_CTL_A)
 #define   VRR_CTL_VRR_ENABLE			REG_BIT(31)
 #define   VRR_CTL_IGN_MAX_SHIFT			REG_BIT(30)
 #define   VRR_CTL_FLIP_LINE_EN			REG_BIT(29)
@@ -2021,21 +1247,21 @@
 #define _TRANS_VRR_VMAX_B		0x61424
 #define _TRANS_VRR_VMAX_C		0x62424
 #define _TRANS_VRR_VMAX_D		0x63424
-#define TRANS_VRR_VMAX(trans)		_MMIO_TRANS2(trans, _TRANS_VRR_VMAX_A)
+#define TRANS_VRR_VMAX(trans)		_MMIO_TRANS2(dev_priv, trans, _TRANS_VRR_VMAX_A)
 #define   VRR_VMAX_MASK			REG_GENMASK(19, 0)
 
 #define _TRANS_VRR_VMIN_A		0x60434
 #define _TRANS_VRR_VMIN_B		0x61434
 #define _TRANS_VRR_VMIN_C		0x62434
 #define _TRANS_VRR_VMIN_D		0x63434
-#define TRANS_VRR_VMIN(trans)		_MMIO_TRANS2(trans, _TRANS_VRR_VMIN_A)
+#define TRANS_VRR_VMIN(trans)		_MMIO_TRANS2(dev_priv, trans, _TRANS_VRR_VMIN_A)
 #define   VRR_VMIN_MASK			REG_GENMASK(15, 0)
 
 #define _TRANS_VRR_VMAXSHIFT_A		0x60428
 #define _TRANS_VRR_VMAXSHIFT_B		0x61428
 #define _TRANS_VRR_VMAXSHIFT_C		0x62428
 #define _TRANS_VRR_VMAXSHIFT_D		0x63428
-#define TRANS_VRR_VMAXSHIFT(trans)	_MMIO_TRANS2(trans, \
+#define TRANS_VRR_VMAXSHIFT(trans)	_MMIO_TRANS2(dev_priv, trans, \
 					_TRANS_VRR_VMAXSHIFT_A)
 #define   VRR_VMAXSHIFT_DEC_MASK	REG_GENMASK(29, 16)
 #define   VRR_VMAXSHIFT_DEC		REG_BIT(16)
@@ -2045,7 +1271,7 @@
 #define _TRANS_VRR_STATUS_B		0x6142C
 #define _TRANS_VRR_STATUS_C		0x6242C
 #define _TRANS_VRR_STATUS_D		0x6342C
-#define TRANS_VRR_STATUS(trans)		_MMIO_TRANS2(trans, _TRANS_VRR_STATUS_A)
+#define TRANS_VRR_STATUS(trans)		_MMIO_TRANS2(dev_priv, trans, _TRANS_VRR_STATUS_A)
 #define   VRR_STATUS_VMAX_REACHED	REG_BIT(31)
 #define   VRR_STATUS_NOFLIP_TILL_BNDR	REG_BIT(30)
 #define   VRR_STATUS_FLIP_BEF_BNDR	REG_BIT(29)
@@ -2065,7 +1291,7 @@
 #define _TRANS_VRR_VTOTAL_PREV_B	0x61480
 #define _TRANS_VRR_VTOTAL_PREV_C	0x62480
 #define _TRANS_VRR_VTOTAL_PREV_D	0x63480
-#define TRANS_VRR_VTOTAL_PREV(trans)	_MMIO_TRANS2(trans, \
+#define TRANS_VRR_VTOTAL_PREV(trans)	_MMIO_TRANS2(dev_priv, trans, \
 					_TRANS_VRR_VTOTAL_PREV_A)
 #define   VRR_VTOTAL_FLIP_BEFR_BNDR	REG_BIT(31)
 #define   VRR_VTOTAL_FLIP_AFTER_BNDR	REG_BIT(30)
@@ -2076,7 +1302,7 @@
 #define _TRANS_VRR_FLIPLINE_B		0x61438
 #define _TRANS_VRR_FLIPLINE_C		0x62438
 #define _TRANS_VRR_FLIPLINE_D		0x63438
-#define TRANS_VRR_FLIPLINE(trans)	_MMIO_TRANS2(trans, \
+#define TRANS_VRR_FLIPLINE(trans)	_MMIO_TRANS2(dev_priv, trans, \
 					_TRANS_VRR_FLIPLINE_A)
 #define   VRR_FLIPLINE_MASK		REG_GENMASK(19, 0)
 
@@ -2084,17 +1310,24 @@
 #define _TRANS_VRR_STATUS2_B		0x6143C
 #define _TRANS_VRR_STATUS2_C		0x6243C
 #define _TRANS_VRR_STATUS2_D		0x6343C
-#define TRANS_VRR_STATUS2(trans)	_MMIO_TRANS2(trans, _TRANS_VRR_STATUS2_A)
+#define TRANS_VRR_STATUS2(trans)	_MMIO_TRANS2(dev_priv, trans, _TRANS_VRR_STATUS2_A)
 #define   VRR_STATUS2_VERT_LN_CNT_MASK	REG_GENMASK(19, 0)
 
 #define _TRANS_PUSH_A			0x60A70
 #define _TRANS_PUSH_B			0x61A70
 #define _TRANS_PUSH_C			0x62A70
 #define _TRANS_PUSH_D			0x63A70
-#define TRANS_PUSH(trans)		_MMIO_TRANS2(trans, _TRANS_PUSH_A)
+#define TRANS_PUSH(trans)		_MMIO_TRANS2(dev_priv, trans, _TRANS_PUSH_A)
 #define   TRANS_PUSH_EN			REG_BIT(31)
 #define   TRANS_PUSH_SEND		REG_BIT(30)
 
+#define _TRANS_VRR_VSYNC_A		0x60078
+#define TRANS_VRR_VSYNC(trans)		_MMIO_TRANS2(dev_priv, trans, _TRANS_VRR_VSYNC_A)
+#define VRR_VSYNC_END_MASK		REG_GENMASK(28, 16)
+#define VRR_VSYNC_END(vsync_end)	REG_FIELD_PREP(VRR_VSYNC_END_MASK, (vsync_end))
+#define VRR_VSYNC_START_MASK		REG_GENMASK(12, 0)
+#define VRR_VSYNC_START(vsync_start)	REG_FIELD_PREP(VRR_VSYNC_START_MASK, (vsync_start))
+
 /* VGA port control */
 #define ADPA			_MMIO(0x61100)
 #define PCH_ADPA                _MMIO(0xe1100)
@@ -2312,6 +1545,7 @@
  * (Haswell and newer) to see which VIDEO_DIP_DATA byte corresponds to each byte
  * of the infoframe structure specified by CEA-861. */
 #define   VIDEO_DIP_DATA_SIZE	32
+#define   VIDEO_DIP_ASYNC_DATA_SIZE	36
 #define   VIDEO_DIP_GMP_DATA_SIZE	36
 #define   VIDEO_DIP_VSC_DATA_SIZE	36
 #define   VIDEO_DIP_PPS_DATA_SIZE	132
@@ -2350,6 +1584,8 @@
 #define   VIDEO_DIP_ENABLE_VS_HSW	(1 << 8)
 #define   VIDEO_DIP_ENABLE_GMP_HSW	(1 << 4)
 #define   VIDEO_DIP_ENABLE_SPD_HSW	(1 << 0)
+/* ADL and later: */
+#define   VIDEO_DIP_ENABLE_AS_ADL	REG_BIT(23)
 
 /* Panel fitting */
 #define PFIT_CONTROL	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61230)
@@ -2588,6 +1824,9 @@
 #define   TRANSCONF_DITHER_TYPE_ST1		REG_FIELD_PREP(TRANSCONF_DITHER_TYPE_MASK, 1)
 #define   TRANSCONF_DITHER_TYPE_ST2		REG_FIELD_PREP(TRANSCONF_DITHER_TYPE_MASK, 2)
 #define   TRANSCONF_DITHER_TYPE_TEMP		REG_FIELD_PREP(TRANSCONF_DITHER_TYPE_MASK, 3)
+#define   TRANSCONF_PIXEL_COUNT_SCALING_MASK	REG_GENMASK(1, 0)
+#define   TRANSCONF_PIXEL_COUNT_SCALING_X4	1
+
 #define _PIPEASTAT		0x70024
 #define   PIPE_FIFO_UNDERRUN_STATUS		(1UL << 31)
 #define   SPRITE1_FLIP_DONE_INT_EN_VLV		(1UL << 30)
@@ -2639,18 +1878,18 @@
 #define PIPESTAT_INT_ENABLE_MASK		0x7fff0000
 #define PIPESTAT_INT_STATUS_MASK		0x0000ffff
 
-#define TRANSCONF(trans)	_MMIO_PIPE2((trans), _TRANSACONF)
-#define PIPEDSL(pipe)		_MMIO_PIPE2(pipe, _PIPEADSL)
-#define PIPEFRAME(pipe)		_MMIO_PIPE2(pipe, _PIPEAFRAMEHIGH)
-#define PIPEFRAMEPIXEL(pipe)	_MMIO_PIPE2(pipe, _PIPEAFRAMEPIXEL)
-#define PIPESTAT(pipe)		_MMIO_PIPE2(pipe, _PIPEASTAT)
+#define TRANSCONF(trans)	_MMIO_PIPE2(dev_priv, (trans), _TRANSACONF)
+#define PIPEDSL(pipe)		_MMIO_PIPE2(dev_priv, pipe, _PIPEADSL)
+#define PIPEFRAME(pipe)		_MMIO_PIPE2(dev_priv, pipe, _PIPEAFRAMEHIGH)
+#define PIPEFRAMEPIXEL(pipe)	_MMIO_PIPE2(dev_priv, pipe, _PIPEAFRAMEPIXEL)
+#define PIPESTAT(pipe)		_MMIO_PIPE2(dev_priv, pipe, _PIPEASTAT)
 
 #define  _PIPEAGCMAX           0x70010
 #define  _PIPEBGCMAX           0x71010
-#define PIPEGCMAX(pipe, i)     _MMIO_PIPE2(pipe, _PIPEAGCMAX + (i) * 4) /* u1.16 */
+#define PIPEGCMAX(pipe, i)     _MMIO_PIPE2(dev_priv, pipe, _PIPEAGCMAX + (i) * 4) /* u1.16 */
 
 #define _PIPE_ARB_CTL_A			0x70028 /* icl+ */
-#define PIPE_ARB_CTL(pipe)		_MMIO_PIPE2(pipe, _PIPE_ARB_CTL_A)
+#define PIPE_ARB_CTL(pipe)		_MMIO_PIPE2(dev_priv, pipe, _PIPE_ARB_CTL_A)
 #define   PIPE_ARB_USE_PROG_SLOTS	REG_BIT(13)
 
 #define _PIPE_MISC_A			0x70030
@@ -2694,7 +1933,7 @@
 #define PIPE_MISC2(pipe)		_MMIO_PIPE(pipe, _PIPE_MISC2_A, _PIPE_MISC2_B)
 
 #define _ICL_PIPE_A_STATUS			0x70058
-#define ICL_PIPESTATUS(pipe)			_MMIO_PIPE2(pipe, _ICL_PIPE_A_STATUS)
+#define ICL_PIPESTATUS(pipe)			_MMIO_PIPE2(dev_priv, pipe, _ICL_PIPE_A_STATUS)
 #define   PIPE_STATUS_UNDERRUN				REG_BIT(31)
 #define   PIPE_STATUS_SOFT_UNDERRUN_XELPD		REG_BIT(28)
 #define   PIPE_STATUS_HARD_UNDERRUN_XELPD		REG_BIT(27)
@@ -2969,8 +2208,8 @@
 #define _WM0_PIPEA_ILK		0x45100
 #define _WM0_PIPEB_ILK		0x45104
 #define _WM0_PIPEC_IVB		0x45200
-#define WM0_PIPE_ILK(pipe)	_MMIO_PIPE3((pipe), _WM0_PIPEA_ILK, \
-					    _WM0_PIPEB_ILK, _WM0_PIPEC_IVB)
+#define WM0_PIPE_ILK(pipe)	_MMIO_BASE_PIPE3(0, (pipe), _WM0_PIPEA_ILK, \
+						 _WM0_PIPEB_ILK, _WM0_PIPEC_IVB)
 #define  WM0_PIPE_PRIMARY_MASK	REG_GENMASK(31, 16)
 #define  WM0_PIPE_SPRITE_MASK	REG_GENMASK(15, 8)
 #define  WM0_PIPE_CURSOR_MASK	REG_GENMASK(7, 0)
@@ -3024,8 +2263,8 @@
 /* GM45+ just has to be different */
 #define _PIPEA_FRMCOUNT_G4X	0x70040
 #define _PIPEA_FLIPCOUNT_G4X	0x70044
-#define PIPE_FRMCOUNT_G4X(pipe) _MMIO_PIPE2(pipe, _PIPEA_FRMCOUNT_G4X)
-#define PIPE_FLIPCOUNT_G4X(pipe) _MMIO_PIPE2(pipe, _PIPEA_FLIPCOUNT_G4X)
+#define PIPE_FRMCOUNT_G4X(pipe) _MMIO_PIPE2(dev_priv, pipe, _PIPEA_FRMCOUNT_G4X)
+#define PIPE_FLIPCOUNT_G4X(pipe) _MMIO_PIPE2(dev_priv, pipe, _PIPEA_FLIPCOUNT_G4X)
 
 /* Cursor A & B regs */
 #define _CURACNTR		0x70080
@@ -3053,6 +2292,7 @@
 #define   MCURSOR_MODE_DISABLE		0x00
 #define   MCURSOR_MODE_128_32B_AX	0x02
 #define   MCURSOR_MODE_256_32B_AX	0x03
+#define   MCURSOR_MODE_64_2B		0x04
 #define   MCURSOR_MODE_64_32B_AX	0x07
 #define   MCURSOR_MODE_128_ARGB_AX	(0x20 | MCURSOR_MODE_128_32B_AX)
 #define   MCURSOR_MODE_256_ARGB_AX	(0x20 | MCURSOR_MODE_256_32B_AX)
@@ -3085,14 +2325,14 @@
 #define _CURBBASE_IVB		0x71084
 #define _CURBPOS_IVB		0x71088
 
-#define CURCNTR(pipe) _MMIO_CURSOR2(pipe, _CURACNTR)
-#define CURBASE(pipe) _MMIO_CURSOR2(pipe, _CURABASE)
-#define CURPOS(pipe) _MMIO_CURSOR2(pipe, _CURAPOS)
-#define CURPOS_ERLY_TPT(pipe) _MMIO_CURSOR2(pipe, _CURAPOS_ERLY_TPT)
-#define CURSIZE(pipe) _MMIO_CURSOR2(pipe, _CURASIZE)
-#define CUR_FBC_CTL(pipe) _MMIO_CURSOR2(pipe, _CUR_FBC_CTL_A)
-#define CUR_CHICKEN(pipe) _MMIO_CURSOR2(pipe, _CUR_CHICKEN_A)
-#define CURSURFLIVE(pipe) _MMIO_CURSOR2(pipe, _CURASURFLIVE)
+#define CURCNTR(pipe) _MMIO_CURSOR2(dev_priv, pipe, _CURACNTR)
+#define CURBASE(pipe) _MMIO_CURSOR2(dev_priv, pipe, _CURABASE)
+#define CURPOS(pipe) _MMIO_CURSOR2(dev_priv, pipe, _CURAPOS)
+#define CURPOS_ERLY_TPT(pipe) _MMIO_CURSOR2(dev_priv, pipe, _CURAPOS_ERLY_TPT)
+#define CURSIZE(pipe) _MMIO_CURSOR2(dev_priv, pipe, _CURASIZE)
+#define CUR_FBC_CTL(pipe) _MMIO_CURSOR2(dev_priv, pipe, _CUR_FBC_CTL_A)
+#define CUR_CHICKEN(pipe) _MMIO_CURSOR2(dev_priv, pipe, _CUR_CHICKEN_A)
+#define CURSURFLIVE(pipe) _MMIO_CURSOR2(dev_priv, pipe, _CURASURFLIVE)
 
 /* Display A control */
 #define _DSPAADDR_VLV				0x7017C /* vlv/chv */
@@ -3149,18 +2389,18 @@
 #define _DSPASURFLIVE				0x701AC
 #define _DSPAGAMC				0x701E0
 
-#define DSPADDR_VLV(plane)	_MMIO_PIPE2(plane, _DSPAADDR_VLV)
-#define DSPCNTR(plane)		_MMIO_PIPE2(plane, _DSPACNTR)
-#define DSPADDR(plane)		_MMIO_PIPE2(plane, _DSPAADDR)
-#define DSPSTRIDE(plane)	_MMIO_PIPE2(plane, _DSPASTRIDE)
-#define DSPPOS(plane)		_MMIO_PIPE2(plane, _DSPAPOS)
-#define DSPSIZE(plane)		_MMIO_PIPE2(plane, _DSPASIZE)
-#define DSPSURF(plane)		_MMIO_PIPE2(plane, _DSPASURF)
-#define DSPTILEOFF(plane)	_MMIO_PIPE2(plane, _DSPATILEOFF)
+#define DSPADDR_VLV(plane)	_MMIO_PIPE2(dev_priv, plane, _DSPAADDR_VLV)
+#define DSPCNTR(plane)		_MMIO_PIPE2(dev_priv, plane, _DSPACNTR)
+#define DSPADDR(plane)		_MMIO_PIPE2(dev_priv, plane, _DSPAADDR)
+#define DSPSTRIDE(plane)	_MMIO_PIPE2(dev_priv, plane, _DSPASTRIDE)
+#define DSPPOS(plane)		_MMIO_PIPE2(dev_priv, plane, _DSPAPOS)
+#define DSPSIZE(plane)		_MMIO_PIPE2(dev_priv, plane, _DSPASIZE)
+#define DSPSURF(plane)		_MMIO_PIPE2(dev_priv, plane, _DSPASURF)
+#define DSPTILEOFF(plane)	_MMIO_PIPE2(dev_priv, plane, _DSPATILEOFF)
 #define DSPLINOFF(plane)	DSPADDR(plane)
-#define DSPOFFSET(plane)	_MMIO_PIPE2(plane, _DSPAOFFSET)
-#define DSPSURFLIVE(plane)	_MMIO_PIPE2(plane, _DSPASURFLIVE)
-#define DSPGAMC(plane, i)	_MMIO_PIPE2(plane, _DSPAGAMC + (5 - (i)) * 4) /* plane C only, 6 x u0.8 */
+#define DSPOFFSET(plane)	_MMIO_PIPE2(dev_priv, plane, _DSPAOFFSET)
+#define DSPSURFLIVE(plane)	_MMIO_PIPE2(dev_priv, plane, _DSPASURFLIVE)
+#define DSPGAMC(plane, i)	_MMIO_PIPE2(dev_priv, plane, _DSPAGAMC + (5 - (i)) * 4) /* plane C only, 6 x u0.8 */
 
 /* CHV pipe B blender and primary plane */
 #define _CHV_BLEND_A		0x60a00
@@ -3187,11 +2427,11 @@
 #define   PRIM_CONST_ALPHA_MASK		REG_GENMASK(7, 0)
 #define   PRIM_CONST_ALPHA(alpha)	REG_FIELD_PREP(PRIM_CONST_ALPHA_MASK, (alpha))
 
-#define CHV_BLEND(pipe)		_MMIO_TRANS2(pipe, _CHV_BLEND_A)
-#define CHV_CANVAS(pipe)	_MMIO_TRANS2(pipe, _CHV_CANVAS_A)
-#define PRIMPOS(plane)		_MMIO_TRANS2(plane, _PRIMPOS_A)
-#define PRIMSIZE(plane)		_MMIO_TRANS2(plane, _PRIMSIZE_A)
-#define PRIMCNSTALPHA(plane)	_MMIO_TRANS2(plane, _PRIMCNSTALPHA_A)
+#define CHV_BLEND(pipe)		_MMIO_TRANS2(dev_priv, pipe, _CHV_BLEND_A)
+#define CHV_CANVAS(pipe)	_MMIO_TRANS2(dev_priv, pipe, _CHV_CANVAS_A)
+#define PRIMPOS(plane)		_MMIO_TRANS2(dev_priv, plane, _PRIMPOS_A)
+#define PRIMSIZE(plane)		_MMIO_TRANS2(dev_priv, plane, _PRIMSIZE_A)
+#define PRIMCNSTALPHA(plane)	_MMIO_TRANS2(dev_priv, plane, _PRIMCNSTALPHA_A)
 
 /* Display/Sprite base address macros */
 #define DISP_BASEADDR_MASK	(0xfffff000)
@@ -3241,346 +2481,6 @@
 #define _PIPEDSI0CONF		0x7b008
 #define _PIPEDSI1CONF		0x7b808
 
-/* Sprite A control */
-#define _DVSACNTR		0x72180
-#define   DVS_ENABLE			REG_BIT(31)
-#define   DVS_PIPE_GAMMA_ENABLE		REG_BIT(30)
-#define   DVS_YUV_RANGE_CORRECTION_DISABLE	REG_BIT(27)
-#define   DVS_FORMAT_MASK		REG_GENMASK(26, 25)
-#define   DVS_FORMAT_YUV422		REG_FIELD_PREP(DVS_FORMAT_MASK, 0)
-#define   DVS_FORMAT_RGBX101010		REG_FIELD_PREP(DVS_FORMAT_MASK, 1)
-#define   DVS_FORMAT_RGBX888		REG_FIELD_PREP(DVS_FORMAT_MASK, 2)
-#define   DVS_FORMAT_RGBX161616		REG_FIELD_PREP(DVS_FORMAT_MASK, 3)
-#define   DVS_PIPE_CSC_ENABLE		REG_BIT(24)
-#define   DVS_SOURCE_KEY		REG_BIT(22)
-#define   DVS_RGB_ORDER_XBGR		REG_BIT(20)
-#define   DVS_YUV_FORMAT_BT709		REG_BIT(18)
-#define   DVS_YUV_ORDER_MASK		REG_GENMASK(17, 16)
-#define   DVS_YUV_ORDER_YUYV		REG_FIELD_PREP(DVS_YUV_ORDER_MASK, 0)
-#define   DVS_YUV_ORDER_UYVY		REG_FIELD_PREP(DVS_YUV_ORDER_MASK, 1)
-#define   DVS_YUV_ORDER_YVYU		REG_FIELD_PREP(DVS_YUV_ORDER_MASK, 2)
-#define   DVS_YUV_ORDER_VYUY		REG_FIELD_PREP(DVS_YUV_ORDER_MASK, 3)
-#define   DVS_ROTATE_180		REG_BIT(15)
-#define   DVS_TRICKLE_FEED_DISABLE	REG_BIT(14)
-#define   DVS_TILED			REG_BIT(10)
-#define   DVS_DEST_KEY			REG_BIT(2)
-#define _DVSALINOFF		0x72184
-#define _DVSASTRIDE		0x72188
-#define _DVSAPOS		0x7218c
-#define   DVS_POS_Y_MASK		REG_GENMASK(31, 16)
-#define   DVS_POS_Y(y)			REG_FIELD_PREP(DVS_POS_Y_MASK, (y))
-#define   DVS_POS_X_MASK		REG_GENMASK(15, 0)
-#define   DVS_POS_X(x)			REG_FIELD_PREP(DVS_POS_X_MASK, (x))
-#define _DVSASIZE		0x72190
-#define   DVS_HEIGHT_MASK		REG_GENMASK(31, 16)
-#define   DVS_HEIGHT(h)			REG_FIELD_PREP(DVS_HEIGHT_MASK, (h))
-#define   DVS_WIDTH_MASK		REG_GENMASK(15, 0)
-#define   DVS_WIDTH(w)			REG_FIELD_PREP(DVS_WIDTH_MASK, (w))
-#define _DVSAKEYVAL		0x72194
-#define _DVSAKEYMSK		0x72198
-#define _DVSASURF		0x7219c
-#define   DVS_ADDR_MASK			REG_GENMASK(31, 12)
-#define _DVSAKEYMAXVAL		0x721a0
-#define _DVSATILEOFF		0x721a4
-#define   DVS_OFFSET_Y_MASK		REG_GENMASK(31, 16)
-#define   DVS_OFFSET_Y(y)		REG_FIELD_PREP(DVS_OFFSET_Y_MASK, (y))
-#define   DVS_OFFSET_X_MASK		REG_GENMASK(15, 0)
-#define   DVS_OFFSET_X(x)		REG_FIELD_PREP(DVS_OFFSET_X_MASK, (x))
-#define _DVSASURFLIVE		0x721ac
-#define _DVSAGAMC_G4X		0x721e0 /* g4x */
-#define _DVSASCALE		0x72204
-#define   DVS_SCALE_ENABLE		REG_BIT(31)
-#define   DVS_FILTER_MASK		REG_GENMASK(30, 29)
-#define   DVS_FILTER_MEDIUM		REG_FIELD_PREP(DVS_FILTER_MASK, 0)
-#define   DVS_FILTER_ENHANCING		REG_FIELD_PREP(DVS_FILTER_MASK, 1)
-#define   DVS_FILTER_SOFTENING		REG_FIELD_PREP(DVS_FILTER_MASK, 2)
-#define   DVS_VERTICAL_OFFSET_HALF	REG_BIT(28) /* must be enabled below */
-#define   DVS_VERTICAL_OFFSET_ENABLE	REG_BIT(27)
-#define   DVS_SRC_WIDTH_MASK		REG_GENMASK(26, 16)
-#define   DVS_SRC_WIDTH(w)		REG_FIELD_PREP(DVS_SRC_WIDTH_MASK, (w))
-#define   DVS_SRC_HEIGHT_MASK		REG_GENMASK(10, 0)
-#define   DVS_SRC_HEIGHT(h)		REG_FIELD_PREP(DVS_SRC_HEIGHT_MASK, (h))
-#define _DVSAGAMC_ILK		0x72300 /* ilk/snb */
-#define _DVSAGAMCMAX_ILK	0x72340 /* ilk/snb */
-
-#define _DVSBCNTR		0x73180
-#define _DVSBLINOFF		0x73184
-#define _DVSBSTRIDE		0x73188
-#define _DVSBPOS		0x7318c
-#define _DVSBSIZE		0x73190
-#define _DVSBKEYVAL		0x73194
-#define _DVSBKEYMSK		0x73198
-#define _DVSBSURF		0x7319c
-#define _DVSBKEYMAXVAL		0x731a0
-#define _DVSBTILEOFF		0x731a4
-#define _DVSBSURFLIVE		0x731ac
-#define _DVSBGAMC_G4X		0x731e0 /* g4x */
-#define _DVSBSCALE		0x73204
-#define _DVSBGAMC_ILK		0x73300 /* ilk/snb */
-#define _DVSBGAMCMAX_ILK	0x73340 /* ilk/snb */
-
-#define DVSCNTR(pipe) _MMIO_PIPE(pipe, _DVSACNTR, _DVSBCNTR)
-#define DVSLINOFF(pipe) _MMIO_PIPE(pipe, _DVSALINOFF, _DVSBLINOFF)
-#define DVSSTRIDE(pipe) _MMIO_PIPE(pipe, _DVSASTRIDE, _DVSBSTRIDE)
-#define DVSPOS(pipe) _MMIO_PIPE(pipe, _DVSAPOS, _DVSBPOS)
-#define DVSSURF(pipe) _MMIO_PIPE(pipe, _DVSASURF, _DVSBSURF)
-#define DVSKEYMAX(pipe) _MMIO_PIPE(pipe, _DVSAKEYMAXVAL, _DVSBKEYMAXVAL)
-#define DVSSIZE(pipe) _MMIO_PIPE(pipe, _DVSASIZE, _DVSBSIZE)
-#define DVSSCALE(pipe) _MMIO_PIPE(pipe, _DVSASCALE, _DVSBSCALE)
-#define DVSTILEOFF(pipe) _MMIO_PIPE(pipe, _DVSATILEOFF, _DVSBTILEOFF)
-#define DVSKEYVAL(pipe) _MMIO_PIPE(pipe, _DVSAKEYVAL, _DVSBKEYVAL)
-#define DVSKEYMSK(pipe) _MMIO_PIPE(pipe, _DVSAKEYMSK, _DVSBKEYMSK)
-#define DVSSURFLIVE(pipe) _MMIO_PIPE(pipe, _DVSASURFLIVE, _DVSBSURFLIVE)
-#define DVSGAMC_G4X(pipe, i) _MMIO(_PIPE(pipe, _DVSAGAMC_G4X, _DVSBGAMC_G4X) + (5 - (i)) * 4) /* 6 x u0.8 */
-#define DVSGAMC_ILK(pipe, i) _MMIO(_PIPE(pipe, _DVSAGAMC_ILK, _DVSBGAMC_ILK) + (i) * 4) /* 16 x u0.10 */
-#define DVSGAMCMAX_ILK(pipe, i) _MMIO(_PIPE(pipe, _DVSAGAMCMAX_ILK, _DVSBGAMCMAX_ILK) + (i) * 4) /* 3 x u1.10 */
-
-#define _SPRA_CTL		0x70280
-#define   SPRITE_ENABLE				REG_BIT(31)
-#define   SPRITE_PIPE_GAMMA_ENABLE		REG_BIT(30)
-#define   SPRITE_YUV_RANGE_CORRECTION_DISABLE	REG_BIT(28)
-#define   SPRITE_FORMAT_MASK			REG_GENMASK(27, 25)
-#define   SPRITE_FORMAT_YUV422			REG_FIELD_PREP(SPRITE_FORMAT_MASK, 0)
-#define   SPRITE_FORMAT_RGBX101010		REG_FIELD_PREP(SPRITE_FORMAT_MASK, 1)
-#define   SPRITE_FORMAT_RGBX888			REG_FIELD_PREP(SPRITE_FORMAT_MASK, 2)
-#define   SPRITE_FORMAT_RGBX161616		REG_FIELD_PREP(SPRITE_FORMAT_MASK, 3)
-#define   SPRITE_FORMAT_YUV444			REG_FIELD_PREP(SPRITE_FORMAT_MASK, 4)
-#define   SPRITE_FORMAT_XR_BGR101010		REG_FIELD_PREP(SPRITE_FORMAT_MASK, 5) /* Extended range */
-#define   SPRITE_PIPE_CSC_ENABLE		REG_BIT(24)
-#define   SPRITE_SOURCE_KEY			REG_BIT(22)
-#define   SPRITE_RGB_ORDER_RGBX			REG_BIT(20) /* only for 888 and 161616 */
-#define   SPRITE_YUV_TO_RGB_CSC_DISABLE		REG_BIT(19)
-#define   SPRITE_YUV_TO_RGB_CSC_FORMAT_BT709	REG_BIT(18) /* 0 is BT601 */
-#define   SPRITE_YUV_ORDER_MASK			REG_GENMASK(17, 16)
-#define   SPRITE_YUV_ORDER_YUYV			REG_FIELD_PREP(SPRITE_YUV_ORDER_MASK, 0)
-#define   SPRITE_YUV_ORDER_UYVY			REG_FIELD_PREP(SPRITE_YUV_ORDER_MASK, 1)
-#define   SPRITE_YUV_ORDER_YVYU			REG_FIELD_PREP(SPRITE_YUV_ORDER_MASK, 2)
-#define   SPRITE_YUV_ORDER_VYUY			REG_FIELD_PREP(SPRITE_YUV_ORDER_MASK, 3)
-#define   SPRITE_ROTATE_180			REG_BIT(15)
-#define   SPRITE_TRICKLE_FEED_DISABLE		REG_BIT(14)
-#define   SPRITE_PLANE_GAMMA_DISABLE		REG_BIT(13)
-#define   SPRITE_TILED				REG_BIT(10)
-#define   SPRITE_DEST_KEY			REG_BIT(2)
-#define _SPRA_LINOFF		0x70284
-#define _SPRA_STRIDE		0x70288
-#define _SPRA_POS		0x7028c
-#define   SPRITE_POS_Y_MASK	REG_GENMASK(31, 16)
-#define   SPRITE_POS_Y(y)	REG_FIELD_PREP(SPRITE_POS_Y_MASK, (y))
-#define   SPRITE_POS_X_MASK	REG_GENMASK(15, 0)
-#define   SPRITE_POS_X(x)	REG_FIELD_PREP(SPRITE_POS_X_MASK, (x))
-#define _SPRA_SIZE		0x70290
-#define   SPRITE_HEIGHT_MASK	REG_GENMASK(31, 16)
-#define   SPRITE_HEIGHT(h)	REG_FIELD_PREP(SPRITE_HEIGHT_MASK, (h))
-#define   SPRITE_WIDTH_MASK	REG_GENMASK(15, 0)
-#define   SPRITE_WIDTH(w)	REG_FIELD_PREP(SPRITE_WIDTH_MASK, (w))
-#define _SPRA_KEYVAL		0x70294
-#define _SPRA_KEYMSK		0x70298
-#define _SPRA_SURF		0x7029c
-#define   SPRITE_ADDR_MASK	REG_GENMASK(31, 12)
-#define _SPRA_KEYMAX		0x702a0
-#define _SPRA_TILEOFF		0x702a4
-#define   SPRITE_OFFSET_Y_MASK	REG_GENMASK(31, 16)
-#define   SPRITE_OFFSET_Y(y)	REG_FIELD_PREP(SPRITE_OFFSET_Y_MASK, (y))
-#define   SPRITE_OFFSET_X_MASK	REG_GENMASK(15, 0)
-#define   SPRITE_OFFSET_X(x)	REG_FIELD_PREP(SPRITE_OFFSET_X_MASK, (x))
-#define _SPRA_OFFSET		0x702a4
-#define _SPRA_SURFLIVE		0x702ac
-#define _SPRA_SCALE		0x70304
-#define   SPRITE_SCALE_ENABLE			REG_BIT(31)
-#define   SPRITE_FILTER_MASK			REG_GENMASK(30, 29)
-#define   SPRITE_FILTER_MEDIUM			REG_FIELD_PREP(SPRITE_FILTER_MASK, 0)
-#define   SPRITE_FILTER_ENHANCING		REG_FIELD_PREP(SPRITE_FILTER_MASK, 1)
-#define   SPRITE_FILTER_SOFTENING		REG_FIELD_PREP(SPRITE_FILTER_MASK, 2)
-#define   SPRITE_VERTICAL_OFFSET_HALF		REG_BIT(28) /* must be enabled below */
-#define   SPRITE_VERTICAL_OFFSET_ENABLE		REG_BIT(27)
-#define   SPRITE_SRC_WIDTH_MASK			REG_GENMASK(26, 16)
-#define   SPRITE_SRC_WIDTH(w)			REG_FIELD_PREP(SPRITE_SRC_WIDTH_MASK, (w))
-#define   SPRITE_SRC_HEIGHT_MASK		REG_GENMASK(10, 0)
-#define   SPRITE_SRC_HEIGHT(h)			REG_FIELD_PREP(SPRITE_SRC_HEIGHT_MASK, (h))
-#define _SPRA_GAMC		0x70400
-#define _SPRA_GAMC16		0x70440
-#define _SPRA_GAMC17		0x7044c
-
-#define _SPRB_CTL		0x71280
-#define _SPRB_LINOFF		0x71284
-#define _SPRB_STRIDE		0x71288
-#define _SPRB_POS		0x7128c
-#define _SPRB_SIZE		0x71290
-#define _SPRB_KEYVAL		0x71294
-#define _SPRB_KEYMSK		0x71298
-#define _SPRB_SURF		0x7129c
-#define _SPRB_KEYMAX		0x712a0
-#define _SPRB_TILEOFF		0x712a4
-#define _SPRB_OFFSET		0x712a4
-#define _SPRB_SURFLIVE		0x712ac
-#define _SPRB_SCALE		0x71304
-#define _SPRB_GAMC		0x71400
-#define _SPRB_GAMC16		0x71440
-#define _SPRB_GAMC17		0x7144c
-
-#define SPRCTL(pipe) _MMIO_PIPE(pipe, _SPRA_CTL, _SPRB_CTL)
-#define SPRLINOFF(pipe) _MMIO_PIPE(pipe, _SPRA_LINOFF, _SPRB_LINOFF)
-#define SPRSTRIDE(pipe) _MMIO_PIPE(pipe, _SPRA_STRIDE, _SPRB_STRIDE)
-#define SPRPOS(pipe) _MMIO_PIPE(pipe, _SPRA_POS, _SPRB_POS)
-#define SPRSIZE(pipe) _MMIO_PIPE(pipe, _SPRA_SIZE, _SPRB_SIZE)
-#define SPRKEYVAL(pipe) _MMIO_PIPE(pipe, _SPRA_KEYVAL, _SPRB_KEYVAL)
-#define SPRKEYMSK(pipe) _MMIO_PIPE(pipe, _SPRA_KEYMSK, _SPRB_KEYMSK)
-#define SPRSURF(pipe) _MMIO_PIPE(pipe, _SPRA_SURF, _SPRB_SURF)
-#define SPRKEYMAX(pipe) _MMIO_PIPE(pipe, _SPRA_KEYMAX, _SPRB_KEYMAX)
-#define SPRTILEOFF(pipe) _MMIO_PIPE(pipe, _SPRA_TILEOFF, _SPRB_TILEOFF)
-#define SPROFFSET(pipe) _MMIO_PIPE(pipe, _SPRA_OFFSET, _SPRB_OFFSET)
-#define SPRSCALE(pipe) _MMIO_PIPE(pipe, _SPRA_SCALE, _SPRB_SCALE)
-#define SPRGAMC(pipe, i) _MMIO(_PIPE(pipe, _SPRA_GAMC, _SPRB_GAMC) + (i) * 4) /* 16 x u0.10 */
-#define SPRGAMC16(pipe, i) _MMIO(_PIPE(pipe, _SPRA_GAMC16, _SPRB_GAMC16) + (i) * 4) /* 3 x u1.10 */
-#define SPRGAMC17(pipe, i) _MMIO(_PIPE(pipe, _SPRA_GAMC17, _SPRB_GAMC17) + (i) * 4) /* 3 x u2.10 */
-#define SPRSURFLIVE(pipe) _MMIO_PIPE(pipe, _SPRA_SURFLIVE, _SPRB_SURFLIVE)
-
-#define _SPACNTR		(VLV_DISPLAY_BASE + 0x72180)
-#define   SP_ENABLE			REG_BIT(31)
-#define   SP_PIPE_GAMMA_ENABLE		REG_BIT(30)
-#define   SP_FORMAT_MASK		REG_GENMASK(29, 26)
-#define   SP_FORMAT_YUV422		REG_FIELD_PREP(SP_FORMAT_MASK, 0)
-#define   SP_FORMAT_8BPP		REG_FIELD_PREP(SP_FORMAT_MASK, 2)
-#define   SP_FORMAT_BGR565		REG_FIELD_PREP(SP_FORMAT_MASK, 5)
-#define   SP_FORMAT_BGRX8888		REG_FIELD_PREP(SP_FORMAT_MASK, 6)
-#define   SP_FORMAT_BGRA8888		REG_FIELD_PREP(SP_FORMAT_MASK, 7)
-#define   SP_FORMAT_RGBX1010102		REG_FIELD_PREP(SP_FORMAT_MASK, 8)
-#define   SP_FORMAT_RGBA1010102		REG_FIELD_PREP(SP_FORMAT_MASK, 9)
-#define   SP_FORMAT_BGRX1010102		REG_FIELD_PREP(SP_FORMAT_MASK, 10) /* CHV pipe B */
-#define   SP_FORMAT_BGRA1010102		REG_FIELD_PREP(SP_FORMAT_MASK, 11) /* CHV pipe B */
-#define   SP_FORMAT_RGBX8888		REG_FIELD_PREP(SP_FORMAT_MASK, 14)
-#define   SP_FORMAT_RGBA8888		REG_FIELD_PREP(SP_FORMAT_MASK, 15)
-#define   SP_ALPHA_PREMULTIPLY		REG_BIT(23) /* CHV pipe B */
-#define   SP_SOURCE_KEY			REG_BIT(22)
-#define   SP_YUV_FORMAT_BT709		REG_BIT(18)
-#define   SP_YUV_ORDER_MASK		REG_GENMASK(17, 16)
-#define   SP_YUV_ORDER_YUYV		REG_FIELD_PREP(SP_YUV_ORDER_MASK, 0)
-#define   SP_YUV_ORDER_UYVY		REG_FIELD_PREP(SP_YUV_ORDER_MASK, 1)
-#define   SP_YUV_ORDER_YVYU		REG_FIELD_PREP(SP_YUV_ORDER_MASK, 2)
-#define   SP_YUV_ORDER_VYUY		REG_FIELD_PREP(SP_YUV_ORDER_MASK, 3)
-#define   SP_ROTATE_180			REG_BIT(15)
-#define   SP_TILED			REG_BIT(10)
-#define   SP_MIRROR			REG_BIT(8) /* CHV pipe B */
-#define _SPALINOFF		(VLV_DISPLAY_BASE + 0x72184)
-#define _SPASTRIDE		(VLV_DISPLAY_BASE + 0x72188)
-#define _SPAPOS			(VLV_DISPLAY_BASE + 0x7218c)
-#define   SP_POS_Y_MASK			REG_GENMASK(31, 16)
-#define   SP_POS_Y(y)			REG_FIELD_PREP(SP_POS_Y_MASK, (y))
-#define   SP_POS_X_MASK			REG_GENMASK(15, 0)
-#define   SP_POS_X(x)			REG_FIELD_PREP(SP_POS_X_MASK, (x))
-#define _SPASIZE		(VLV_DISPLAY_BASE + 0x72190)
-#define   SP_HEIGHT_MASK		REG_GENMASK(31, 16)
-#define   SP_HEIGHT(h)			REG_FIELD_PREP(SP_HEIGHT_MASK, (h))
-#define   SP_WIDTH_MASK			REG_GENMASK(15, 0)
-#define   SP_WIDTH(w)			REG_FIELD_PREP(SP_WIDTH_MASK, (w))
-#define _SPAKEYMINVAL		(VLV_DISPLAY_BASE + 0x72194)
-#define _SPAKEYMSK		(VLV_DISPLAY_BASE + 0x72198)
-#define _SPASURF		(VLV_DISPLAY_BASE + 0x7219c)
-#define   SP_ADDR_MASK			REG_GENMASK(31, 12)
-#define _SPAKEYMAXVAL		(VLV_DISPLAY_BASE + 0x721a0)
-#define _SPATILEOFF		(VLV_DISPLAY_BASE + 0x721a4)
-#define   SP_OFFSET_Y_MASK		REG_GENMASK(31, 16)
-#define   SP_OFFSET_Y(y)		REG_FIELD_PREP(SP_OFFSET_Y_MASK, (y))
-#define   SP_OFFSET_X_MASK		REG_GENMASK(15, 0)
-#define   SP_OFFSET_X(x)		REG_FIELD_PREP(SP_OFFSET_X_MASK, (x))
-#define _SPACONSTALPHA		(VLV_DISPLAY_BASE + 0x721a8)
-#define   SP_CONST_ALPHA_ENABLE		REG_BIT(31)
-#define   SP_CONST_ALPHA_MASK		REG_GENMASK(7, 0)
-#define   SP_CONST_ALPHA(alpha)		REG_FIELD_PREP(SP_CONST_ALPHA_MASK, (alpha))
-#define _SPASURFLIVE		(VLV_DISPLAY_BASE + 0x721ac)
-#define _SPACLRC0		(VLV_DISPLAY_BASE + 0x721d0)
-#define   SP_CONTRAST_MASK		REG_GENMASK(26, 18)
-#define   SP_CONTRAST(x)		REG_FIELD_PREP(SP_CONTRAST_MASK, (x)) /* u3.6 */
-#define   SP_BRIGHTNESS_MASK		REG_GENMASK(7, 0)
-#define   SP_BRIGHTNESS(x)		REG_FIELD_PREP(SP_BRIGHTNESS_MASK, (x)) /* s8 */
-#define _SPACLRC1		(VLV_DISPLAY_BASE + 0x721d4)
-#define   SP_SH_SIN_MASK		REG_GENMASK(26, 16)
-#define   SP_SH_SIN(x)			REG_FIELD_PREP(SP_SH_SIN_MASK, (x)) /* s4.7 */
-#define   SP_SH_COS_MASK		REG_GENMASK(9, 0)
-#define   SP_SH_COS(x)			REG_FIELD_PREP(SP_SH_COS_MASK, (x)) /* u3.7 */
-#define _SPAGAMC		(VLV_DISPLAY_BASE + 0x721e0)
-
-#define _SPBCNTR		(VLV_DISPLAY_BASE + 0x72280)
-#define _SPBLINOFF		(VLV_DISPLAY_BASE + 0x72284)
-#define _SPBSTRIDE		(VLV_DISPLAY_BASE + 0x72288)
-#define _SPBPOS			(VLV_DISPLAY_BASE + 0x7228c)
-#define _SPBSIZE		(VLV_DISPLAY_BASE + 0x72290)
-#define _SPBKEYMINVAL		(VLV_DISPLAY_BASE + 0x72294)
-#define _SPBKEYMSK		(VLV_DISPLAY_BASE + 0x72298)
-#define _SPBSURF		(VLV_DISPLAY_BASE + 0x7229c)
-#define _SPBKEYMAXVAL		(VLV_DISPLAY_BASE + 0x722a0)
-#define _SPBTILEOFF		(VLV_DISPLAY_BASE + 0x722a4)
-#define _SPBCONSTALPHA		(VLV_DISPLAY_BASE + 0x722a8)
-#define _SPBSURFLIVE		(VLV_DISPLAY_BASE + 0x722ac)
-#define _SPBCLRC0		(VLV_DISPLAY_BASE + 0x722d0)
-#define _SPBCLRC1		(VLV_DISPLAY_BASE + 0x722d4)
-#define _SPBGAMC		(VLV_DISPLAY_BASE + 0x722e0)
-
-#define _VLV_SPR(pipe, plane_id, reg_a, reg_b) \
-	_PIPE((pipe) * 2 + (plane_id) - PLANE_SPRITE0, (reg_a), (reg_b))
-#define _MMIO_VLV_SPR(pipe, plane_id, reg_a, reg_b) \
-	_MMIO(_VLV_SPR((pipe), (plane_id), (reg_a), (reg_b)))
-
-#define SPCNTR(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPACNTR, _SPBCNTR)
-#define SPLINOFF(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPALINOFF, _SPBLINOFF)
-#define SPSTRIDE(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPASTRIDE, _SPBSTRIDE)
-#define SPPOS(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPAPOS, _SPBPOS)
-#define SPSIZE(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPASIZE, _SPBSIZE)
-#define SPKEYMINVAL(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPAKEYMINVAL, _SPBKEYMINVAL)
-#define SPKEYMSK(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPAKEYMSK, _SPBKEYMSK)
-#define SPSURF(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPASURF, _SPBSURF)
-#define SPKEYMAXVAL(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPAKEYMAXVAL, _SPBKEYMAXVAL)
-#define SPTILEOFF(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPATILEOFF, _SPBTILEOFF)
-#define SPCONSTALPHA(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPACONSTALPHA, _SPBCONSTALPHA)
-#define SPSURFLIVE(pipe, plane_id)	_MMIO_VLV_SPR((pipe), (plane_id), _SPASURFLIVE, _SPBSURFLIVE)
-#define SPCLRC0(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPACLRC0, _SPBCLRC0)
-#define SPCLRC1(pipe, plane_id)		_MMIO_VLV_SPR((pipe), (plane_id), _SPACLRC1, _SPBCLRC1)
-#define SPGAMC(pipe, plane_id, i)	_MMIO(_VLV_SPR((pipe), (plane_id), _SPAGAMC, _SPBGAMC) + (5 - (i)) * 4) /* 6 x u0.10 */
-
-/*
- * CHV pipe B sprite CSC
- *
- * |cr|   |c0 c1 c2|   |cr + cr_ioff|   |cr_ooff|
- * |yg| = |c3 c4 c5| x |yg + yg_ioff| + |yg_ooff|
- * |cb|   |c6 c7 c8|   |cb + cr_ioff|   |cb_ooff|
- */
-#define _MMIO_CHV_SPCSC(plane_id, reg) \
-	_MMIO(VLV_DISPLAY_BASE + ((plane_id) - PLANE_SPRITE0) * 0x1000 + (reg))
-
-#define SPCSCYGOFF(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d900)
-#define SPCSCCBOFF(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d904)
-#define SPCSCCROFF(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d908)
-#define  SPCSC_OOFF_MASK	REG_GENMASK(26, 16)
-#define  SPCSC_OOFF(x)		REG_FIELD_PREP(SPCSC_OOFF_MASK, (x) & 0x7ff) /* s11 */
-#define  SPCSC_IOFF_MASK	REG_GENMASK(10, 0)
-#define  SPCSC_IOFF(x)		REG_FIELD_PREP(SPCSC_IOFF_MASK, (x) & 0x7ff) /* s11 */
-
-#define SPCSCC01(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d90c)
-#define SPCSCC23(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d910)
-#define SPCSCC45(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d914)
-#define SPCSCC67(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d918)
-#define SPCSCC8(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d91c)
-#define  SPCSC_C1_MASK		REG_GENMASK(30, 16)
-#define  SPCSC_C1(x)		REG_FIELD_PREP(SPCSC_C1_MASK, (x) & 0x7fff) /* s3.12 */
-#define  SPCSC_C0_MASK		REG_GENMASK(14, 0)
-#define  SPCSC_C0(x)		REG_FIELD_PREP(SPCSC_C0_MASK, (x) & 0x7fff) /* s3.12 */
-
-#define SPCSCYGICLAMP(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d920)
-#define SPCSCCBICLAMP(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d924)
-#define SPCSCCRICLAMP(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d928)
-#define  SPCSC_IMAX_MASK	REG_GENMASK(26, 16)
-#define  SPCSC_IMAX(x)		REG_FIELD_PREP(SPCSC_IMAX_MASK, (x) & 0x7ff) /* s11 */
-#define  SPCSC_IMIN_MASK	REG_GENMASK(10, 0)
-#define  SPCSC_IMIN(x)		REG_FIELD_PREP(SPCSC_IMIN_MASK, (x) & 0x7ff) /* s11 */
-
-#define SPCSCYGOCLAMP(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d92c)
-#define SPCSCCBOCLAMP(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d930)
-#define SPCSCCROCLAMP(plane_id)	_MMIO_CHV_SPCSC(plane_id, 0x6d934)
-#define  SPCSC_OMAX_MASK	REG_GENMASK(25, 16)
-#define  SPCSC_OMAX(x)		REG_FIELD_PREP(SPCSC_OMAX_MASK, (x)) /* u10 */
-#define  SPCSC_OMIN_MASK	REG_GENMASK(9, 0)
-#define  SPCSC_OMIN(x)		REG_FIELD_PREP(SPCSC_OMIN_MASK, (x)) /* u10 */
-
 /* Skylake plane registers */
 
 #define _PLANE_CTL_1_A				0x70180
@@ -3990,14 +2890,14 @@
 #define _PIPEB_LINK_M2		0x61048
 #define _PIPEB_LINK_N2		0x6104c
 
-#define PIPE_DATA_M1(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_M1)
-#define PIPE_DATA_N1(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_N1)
-#define PIPE_DATA_M2(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_M2)
-#define PIPE_DATA_N2(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_N2)
-#define PIPE_LINK_M1(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_M1)
-#define PIPE_LINK_N1(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_N1)
-#define PIPE_LINK_M2(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_M2)
-#define PIPE_LINK_N2(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_N2)
+#define PIPE_DATA_M1(tran) _MMIO_TRANS2(dev_priv, tran, _PIPEA_DATA_M1)
+#define PIPE_DATA_N1(tran) _MMIO_TRANS2(dev_priv, tran, _PIPEA_DATA_N1)
+#define PIPE_DATA_M2(tran) _MMIO_TRANS2(dev_priv, tran, _PIPEA_DATA_M2)
+#define PIPE_DATA_N2(tran) _MMIO_TRANS2(dev_priv, tran, _PIPEA_DATA_N2)
+#define PIPE_LINK_M1(tran) _MMIO_TRANS2(dev_priv, tran, _PIPEA_LINK_M1)
+#define PIPE_LINK_N1(tran) _MMIO_TRANS2(dev_priv, tran, _PIPEA_LINK_N1)
+#define PIPE_LINK_M2(tran) _MMIO_TRANS2(dev_priv, tran, _PIPEA_LINK_M2)
+#define PIPE_LINK_N2(tran) _MMIO_TRANS2(dev_priv, tran, _PIPEA_LINK_N2)
 
 /* CPU panel fitter */
 /* IVB+ has 3 fitters, 0 is 7x5 capable, the other two only 3x3 */
@@ -4555,6 +3455,11 @@
 #define   GLK_CL1_PWR_DOWN		REG_BIT(11)
 #define   GLK_CL0_PWR_DOWN		REG_BIT(10)
 
+#define CHICKEN_MISC_3		_MMIO(0x42088)
+#define   DP_MST_DPT_DPTP_ALIGN_WA(trans)	REG_BIT(9 + (trans) - TRANSCODER_A)
+#define   DP_MST_SHORT_HBLANK_WA(trans)		REG_BIT(5 + (trans) - TRANSCODER_A)
+#define   DP_MST_FEC_BS_JITTER_WA(trans)	REG_BIT(0 + (trans) - TRANSCODER_A)
+
 #define CHICKEN_MISC_4		_MMIO(0x4208c)
 #define   CHICKEN_FBC_STRIDE_OVERRIDE	REG_BIT(13)
 #define   CHICKEN_FBC_STRIDE_MASK	REG_GENMASK(12, 0)
@@ -4611,7 +3516,9 @@
 #define   DDIE_TRAINING_OVERRIDE_ENABLE	REG_BIT(17) /* CHICKEN_TRANS_A only */
 #define   DDIE_TRAINING_OVERRIDE_VALUE	REG_BIT(16) /* CHICKEN_TRANS_A only */
 #define   PSR2_ADD_VERTICAL_LINE_COUNT	REG_BIT(15)
+#define   DP_FEC_BS_JITTER_WA		REG_BIT(15)
 #define   PSR2_VSC_ENABLE_PROG_HEADER	REG_BIT(12)
+#define   DP_DSC_INSERT_SF_AT_EOL_WA	REG_BIT(4)
 
 #define DISP_ARB_CTL	_MMIO(0x45000)
 #define   DISP_FBC_MEMORY_WAKE		REG_BIT(31)
@@ -5010,27 +3917,29 @@
 #define TVIDEO_DIP_GCP(pipe) _MMIO_PIPE(pipe, _VIDEO_DIP_GCP_A, _VIDEO_DIP_GCP_B)
 
 /* Per-transcoder DIP controls (VLV) */
-#define _VLV_VIDEO_DIP_CTL_A		(VLV_DISPLAY_BASE + 0x60200)
-#define _VLV_VIDEO_DIP_DATA_A		(VLV_DISPLAY_BASE + 0x60208)
-#define _VLV_VIDEO_DIP_GDCP_PAYLOAD_A	(VLV_DISPLAY_BASE + 0x60210)
-
-#define _VLV_VIDEO_DIP_CTL_B		(VLV_DISPLAY_BASE + 0x61170)
-#define _VLV_VIDEO_DIP_DATA_B		(VLV_DISPLAY_BASE + 0x61174)
-#define _VLV_VIDEO_DIP_GDCP_PAYLOAD_B	(VLV_DISPLAY_BASE + 0x61178)
-
-#define _CHV_VIDEO_DIP_CTL_C		(VLV_DISPLAY_BASE + 0x611f0)
-#define _CHV_VIDEO_DIP_DATA_C		(VLV_DISPLAY_BASE + 0x611f4)
-#define _CHV_VIDEO_DIP_GDCP_PAYLOAD_C	(VLV_DISPLAY_BASE + 0x611f8)
-
-#define VLV_TVIDEO_DIP_CTL(pipe) \
-	_MMIO_PIPE3((pipe), _VLV_VIDEO_DIP_CTL_A, \
-	       _VLV_VIDEO_DIP_CTL_B, _CHV_VIDEO_DIP_CTL_C)
-#define VLV_TVIDEO_DIP_DATA(pipe) \
-	_MMIO_PIPE3((pipe), _VLV_VIDEO_DIP_DATA_A, \
-	       _VLV_VIDEO_DIP_DATA_B, _CHV_VIDEO_DIP_DATA_C)
-#define VLV_TVIDEO_DIP_GCP(pipe) \
-	_MMIO_PIPE3((pipe), _VLV_VIDEO_DIP_GDCP_PAYLOAD_A, \
-		_VLV_VIDEO_DIP_GDCP_PAYLOAD_B, _CHV_VIDEO_DIP_GDCP_PAYLOAD_C)
+#define _VLV_VIDEO_DIP_CTL_A		0x60200
+#define _VLV_VIDEO_DIP_CTL_B		0x61170
+#define _CHV_VIDEO_DIP_CTL_C		0x611f0
+#define VLV_TVIDEO_DIP_CTL(pipe)	_MMIO_BASE_PIPE3(VLV_DISPLAY_BASE, (pipe), \
+							 _VLV_VIDEO_DIP_CTL_A, \
+							 _VLV_VIDEO_DIP_CTL_B, \
+							 _CHV_VIDEO_DIP_CTL_C)
+
+#define _VLV_VIDEO_DIP_DATA_A		0x60208
+#define _VLV_VIDEO_DIP_DATA_B		0x61174
+#define _CHV_VIDEO_DIP_DATA_C		0x611f4
+#define VLV_TVIDEO_DIP_DATA(pipe)	_MMIO_BASE_PIPE3(VLV_DISPLAY_BASE, (pipe), \
+							 _VLV_VIDEO_DIP_DATA_A, \
+							 _VLV_VIDEO_DIP_DATA_B, \
+							 _CHV_VIDEO_DIP_DATA_C)
+
+#define _VLV_VIDEO_DIP_GDCP_PAYLOAD_A	0x60210
+#define _VLV_VIDEO_DIP_GDCP_PAYLOAD_B	0x61178
+#define _CHV_VIDEO_DIP_GDCP_PAYLOAD_C	0x611f8
+#define VLV_TVIDEO_DIP_GCP(pipe)	_MMIO_BASE_PIPE3(VLV_DISPLAY_BASE, (pipe), \
+							 _VLV_VIDEO_DIP_GDCP_PAYLOAD_A, \
+							 _VLV_VIDEO_DIP_GDCP_PAYLOAD_B, \
+							 _CHV_VIDEO_DIP_GDCP_PAYLOAD_C)
 
 /* Haswell DIP controls */
 
@@ -5040,6 +3949,7 @@
 #define _HSW_VIDEO_DIP_SPD_DATA_A	0x602A0
 #define _HSW_VIDEO_DIP_GMP_DATA_A	0x602E0
 #define _HSW_VIDEO_DIP_VSC_DATA_A	0x60320
+#define	_ADL_VIDEO_DIP_AS_DATA_A	0x60484
 #define _GLK_VIDEO_DIP_DRM_DATA_A	0x60440
 #define _HSW_VIDEO_DIP_AVI_ECC_A	0x60240
 #define _HSW_VIDEO_DIP_VS_ECC_A		0x60280
@@ -5054,6 +3964,7 @@
 #define _HSW_VIDEO_DIP_SPD_DATA_B	0x612A0
 #define _HSW_VIDEO_DIP_GMP_DATA_B	0x612E0
 #define _HSW_VIDEO_DIP_VSC_DATA_B	0x61320
+#define _ADL_VIDEO_DIP_AS_DATA_B	0x61484
 #define _GLK_VIDEO_DIP_DRM_DATA_B	0x61440
 #define _HSW_VIDEO_DIP_BVI_ECC_B	0x61240
 #define _HSW_VIDEO_DIP_VS_ECC_B		0x61280
@@ -5073,22 +3984,25 @@
 #define _ICL_VIDEO_DIP_PPS_ECC_A	0x603D4
 #define _ICL_VIDEO_DIP_PPS_ECC_B	0x613D4
 
-#define HSW_TVIDEO_DIP_CTL(trans)		_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_CTL_A)
-#define HSW_TVIDEO_DIP_GCP(trans)		_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_GCP_A)
-#define HSW_TVIDEO_DIP_AVI_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_AVI_DATA_A + (i) * 4)
-#define HSW_TVIDEO_DIP_VS_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_VS_DATA_A + (i) * 4)
-#define HSW_TVIDEO_DIP_SPD_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_SPD_DATA_A + (i) * 4)
-#define HSW_TVIDEO_DIP_GMP_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_GMP_DATA_A + (i) * 4)
-#define HSW_TVIDEO_DIP_VSC_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_VSC_DATA_A + (i) * 4)
-#define GLK_TVIDEO_DIP_DRM_DATA(trans, i)	_MMIO_TRANS2(trans, _GLK_VIDEO_DIP_DRM_DATA_A + (i) * 4)
-#define ICL_VIDEO_DIP_PPS_DATA(trans, i)	_MMIO_TRANS2(trans, _ICL_VIDEO_DIP_PPS_DATA_A + (i) * 4)
-#define ICL_VIDEO_DIP_PPS_ECC(trans, i)		_MMIO_TRANS2(trans, _ICL_VIDEO_DIP_PPS_ECC_A + (i) * 4)
+#define HSW_TVIDEO_DIP_CTL(trans)		_MMIO_TRANS2(dev_priv, trans, _HSW_VIDEO_DIP_CTL_A)
+#define HSW_TVIDEO_DIP_GCP(trans)		_MMIO_TRANS2(dev_priv, trans, _HSW_VIDEO_DIP_GCP_A)
+#define HSW_TVIDEO_DIP_AVI_DATA(trans, i)	_MMIO_TRANS2(dev_priv, trans, _HSW_VIDEO_DIP_AVI_DATA_A + (i) * 4)
+#define HSW_TVIDEO_DIP_VS_DATA(trans, i)	_MMIO_TRANS2(dev_priv, trans, _HSW_VIDEO_DIP_VS_DATA_A + (i) * 4)
+#define HSW_TVIDEO_DIP_SPD_DATA(trans, i)	_MMIO_TRANS2(dev_priv, trans, _HSW_VIDEO_DIP_SPD_DATA_A + (i) * 4)
+#define HSW_TVIDEO_DIP_GMP_DATA(trans, i)	_MMIO_TRANS2(dev_priv, trans, _HSW_VIDEO_DIP_GMP_DATA_A + (i) * 4)
+#define HSW_TVIDEO_DIP_VSC_DATA(trans, i)	_MMIO_TRANS2(dev_priv, trans, _HSW_VIDEO_DIP_VSC_DATA_A + (i) * 4)
+#define GLK_TVIDEO_DIP_DRM_DATA(trans, i)	_MMIO_TRANS2(dev_priv, trans, _GLK_VIDEO_DIP_DRM_DATA_A + (i) * 4)
+#define ICL_VIDEO_DIP_PPS_DATA(trans, i)	_MMIO_TRANS2(dev_priv, trans, _ICL_VIDEO_DIP_PPS_DATA_A + (i) * 4)
+#define ICL_VIDEO_DIP_PPS_ECC(trans, i)		_MMIO_TRANS2(dev_priv, trans, _ICL_VIDEO_DIP_PPS_ECC_A + (i) * 4)
+/*ADLP and later: */
+#define ADL_TVIDEO_DIP_AS_SDP_DATA(trans, i)	_MMIO_TRANS2(dev_priv, trans,\
+							     _ADL_VIDEO_DIP_AS_DATA_A + (i) * 4)
 
 #define _HSW_STEREO_3D_CTL_A		0x70020
 #define   S3D_ENABLE			(1 << 31)
 #define _HSW_STEREO_3D_CTL_B		0x71020
 
-#define HSW_STEREO_3D_CTL(trans)	_MMIO_PIPE2(trans, _HSW_STEREO_3D_CTL_A)
+#define HSW_STEREO_3D_CTL(trans)	_MMIO_PIPE2(dev_priv, trans, _HSW_STEREO_3D_CTL_A)
 
 #define _PCH_TRANS_HTOTAL_B          0xe1000
 #define _PCH_TRANS_HBLANK_B          0xe1004
@@ -5401,7 +4315,7 @@
 #define	    POWER_SETUP_I1_SHIFT		6	/* 10.6 fixed point format */
 #define	    POWER_SETUP_I1_DATA_MASK		REG_GENMASK(15, 0)
 #define GEN12_PCODE_READ_SAGV_BLOCK_TIME_US	0x23
-#define   XEHP_PCODE_FREQUENCY_CONFIG		0x6e	/* xehpsdv, pvc */
+#define   XEHP_PCODE_FREQUENCY_CONFIG		0x6e	/* pvc */
 /* XEHP_PCODE_FREQUENCY_CONFIG sub-commands (param1) */
 #define     PCODE_MBOX_FC_SC_READ_FUSED_P0	0x0
 #define     PCODE_MBOX_FC_SC_READ_FUSED_PN	0x1
@@ -5566,15 +4480,6 @@ enum skl_power_gate {
 	((pw_idx) - ICL_PW_CTL_IDX_PW_1 + SKL_PG1)
 #define  SKL_FUSE_PG_DIST_STATUS(pg)		(1 << (27 - (pg)))
 
-#define _ICL_AUX_REG_IDX(pw_idx)	((pw_idx) - ICL_PW_CTL_IDX_AUX_A)
-#define _ICL_AUX_ANAOVRD1_A		0x162398
-#define _ICL_AUX_ANAOVRD1_B		0x6C398
-#define ICL_AUX_ANAOVRD1(pw_idx)	_MMIO(_PICK(_ICL_AUX_REG_IDX(pw_idx), \
-						    _ICL_AUX_ANAOVRD1_A, \
-						    _ICL_AUX_ANAOVRD1_B))
-#define   ICL_AUX_ANAOVRD1_LDO_BYPASS	(1 << 7)
-#define   ICL_AUX_ANAOVRD1_ENABLE	(1 << 0)
-
 /* Per-pipe DDI Function Control */
 #define _TRANS_DDI_FUNC_CTL_A		0x60400
 #define _TRANS_DDI_FUNC_CTL_B		0x61400
@@ -5583,7 +4488,7 @@ enum skl_power_gate {
 #define _TRANS_DDI_FUNC_CTL_EDP		0x6F400
 #define _TRANS_DDI_FUNC_CTL_DSI0	0x6b400
 #define _TRANS_DDI_FUNC_CTL_DSI1	0x6bc00
-#define TRANS_DDI_FUNC_CTL(tran) _MMIO_TRANS2(tran, _TRANS_DDI_FUNC_CTL_A)
+#define TRANS_DDI_FUNC_CTL(tran) _MMIO_TRANS2(dev_priv, tran, _TRANS_DDI_FUNC_CTL_A)
 
 #define  TRANS_DDI_FUNC_ENABLE		(1 << 31)
 /* Those bits are ignored by pipe EDP since it can only connect to DDI A */
@@ -5638,7 +4543,7 @@ enum skl_power_gate {
 #define _TRANS_DDI_FUNC_CTL2_EDP	0x6f404
 #define _TRANS_DDI_FUNC_CTL2_DSI0	0x6b404
 #define _TRANS_DDI_FUNC_CTL2_DSI1	0x6bc04
-#define TRANS_DDI_FUNC_CTL2(tran)	_MMIO_TRANS2(tran, _TRANS_DDI_FUNC_CTL2_A)
+#define TRANS_DDI_FUNC_CTL2(tran)	_MMIO_TRANS2(dev_priv, tran, _TRANS_DDI_FUNC_CTL2_A)
 #define  PORT_SYNC_MODE_ENABLE			REG_BIT(4)
 #define  PORT_SYNC_MODE_MASTER_SELECT_MASK	REG_GENMASK(2, 0)
 #define  PORT_SYNC_MODE_MASTER_SELECT(x)	REG_FIELD_PREP(PORT_SYNC_MODE_MASTER_SELECT_MASK, (x))
@@ -5651,7 +4556,7 @@ enum skl_power_gate {
 #define _DP_TP_CTL_B			0x64140
 #define _TGL_DP_TP_CTL_A		0x60540
 #define DP_TP_CTL(port) _MMIO_PORT(port, _DP_TP_CTL_A, _DP_TP_CTL_B)
-#define TGL_DP_TP_CTL(tran) _MMIO_TRANS2((tran), _TGL_DP_TP_CTL_A)
+#define TGL_DP_TP_CTL(tran) _MMIO_TRANS2(dev_priv, (tran), _TGL_DP_TP_CTL_A)
 #define  DP_TP_CTL_ENABLE			(1 << 31)
 #define  DP_TP_CTL_FEC_ENABLE			(1 << 30)
 #define  DP_TP_CTL_MODE_SST			(0 << 27)
@@ -5677,7 +4582,7 @@ enum skl_power_gate {
 #define _DP_TP_STATUS_B			0x64144
 #define _TGL_DP_TP_STATUS_A		0x60544
 #define DP_TP_STATUS(port) _MMIO_PORT(port, _DP_TP_STATUS_A, _DP_TP_STATUS_B)
-#define TGL_DP_TP_STATUS(tran) _MMIO_TRANS2((tran), _TGL_DP_TP_STATUS_A)
+#define TGL_DP_TP_STATUS(tran) _MMIO_TRANS2(dev_priv, (tran), _TGL_DP_TP_STATUS_A)
 #define  DP_TP_STATUS_FEC_ENABLE_LIVE		(1 << 28)
 #define  DP_TP_STATUS_IDLE_DONE			(1 << 25)
 #define  DP_TP_STATUS_ACT_SENT			(1 << 24)
@@ -5858,14 +4763,14 @@ enum skl_power_gate {
 #define _TRANSB_MSA_MISC		0x61410
 #define _TRANSC_MSA_MISC		0x62410
 #define _TRANS_EDP_MSA_MISC		0x6f410
-#define TRANS_MSA_MISC(tran) _MMIO_TRANS2(tran, _TRANSA_MSA_MISC)
+#define TRANS_MSA_MISC(tran) _MMIO_TRANS2(dev_priv, tran, _TRANSA_MSA_MISC)
 /* See DP_MSA_MISC_* for the bit definitions */
 
 #define _TRANS_A_SET_CONTEXT_LATENCY		0x6007C
 #define _TRANS_B_SET_CONTEXT_LATENCY		0x6107C
 #define _TRANS_C_SET_CONTEXT_LATENCY		0x6207C
 #define _TRANS_D_SET_CONTEXT_LATENCY		0x6307C
-#define TRANS_SET_CONTEXT_LATENCY(tran)		_MMIO_TRANS2(tran, _TRANS_A_SET_CONTEXT_LATENCY)
+#define TRANS_SET_CONTEXT_LATENCY(tran)		_MMIO_TRANS2(dev_priv, tran, _TRANS_A_SET_CONTEXT_LATENCY)
 #define  TRANS_SET_CONTEXT_LATENCY_MASK		REG_GENMASK(15, 0)
 #define  TRANS_SET_CONTEXT_LATENCY_VALUE(x)	REG_FIELD_PREP(TRANS_SET_CONTEXT_LATENCY_MASK, (x))
 
@@ -5900,7 +4805,9 @@ enum skl_power_gate {
 #define  CDCLK_FREQ_540		REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 1)
 #define  CDCLK_FREQ_337_308		REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 2)
 #define  CDCLK_FREQ_675_617		REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 3)
-#define  MDCLK_SOURCE_SEL_CDCLK_PLL	REG_BIT(25)
+#define  MDCLK_SOURCE_SEL_MASK		REG_GENMASK(25, 25)
+#define  MDCLK_SOURCE_SEL_CD2XCLK	REG_FIELD_PREP(MDCLK_SOURCE_SEL_MASK, 0)
+#define  MDCLK_SOURCE_SEL_CDCLK_PLL	REG_FIELD_PREP(MDCLK_SOURCE_SEL_MASK, 1)
 #define  BXT_CDCLK_CD2X_DIV_SEL_MASK	REG_GENMASK(23, 22)
 #define  BXT_CDCLK_CD2X_DIV_SEL_1	REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 0)
 #define  BXT_CDCLK_CD2X_DIV_SEL_1_5	REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 1)
@@ -6317,7 +5224,7 @@ enum skl_power_gate {
 
 #define _VLV_PIPE_MSA_MISC_A			0x70048
 #define VLV_PIPE_MSA_MISC(pipe)		\
-			_MMIO_PIPE2(pipe, _VLV_PIPE_MSA_MISC_A)
+			_MMIO_PIPE2(dev_priv, pipe, _VLV_PIPE_MSA_MISC_A)
 #define   VLV_MSA_MISC1_HW_ENABLE			REG_BIT(31)
 #define   VLV_MSA_MISC1_SW_S3D_MASK			REG_GENMASK(2, 0) /* MSA MISC1 3:1 */
 
@@ -6390,7 +5297,7 @@ enum skl_power_gate {
 
 #define _MTL_CLKGATE_DIS_TRANS_A			0x604E8
 #define _MTL_CLKGATE_DIS_TRANS_B			0x614E8
-#define MTL_CLKGATE_DIS_TRANS(trans)			_MMIO_TRANS2(trans, _MTL_CLKGATE_DIS_TRANS_A)
+#define MTL_CLKGATE_DIS_TRANS(trans)			_MMIO_TRANS2(dev_priv, trans, _MTL_CLKGATE_DIS_TRANS_A)
 #define  MTL_CLKGATE_DIS_TRANS_DMASC_GATING_DIS		REG_BIT(7)
 
 #define MTL_MEM_SS_INFO_GLOBAL			_MMIO(0x45700)
diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
index 0d735d5c2b35..942345548bc3 100644
--- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
+++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
@@ -126,7 +126,7 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
 	return 0;
 
 err_free_blocks:
-	drm_buddy_free_list(mm, &bman_res->blocks);
+	drm_buddy_free_list(mm, &bman_res->blocks, 0);
 	mutex_unlock(&bman->lock);
 err_free_res:
 	ttm_resource_fini(man, &bman_res->base);
@@ -141,7 +141,7 @@ static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man,
 	struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
 
 	mutex_lock(&bman->lock);
-	drm_buddy_free_list(&bman->mm, &bman_res->blocks);
+	drm_buddy_free_list(&bman->mm, &bman_res->blocks, 0);
 	bman->visible_avail += bman_res->used_visible_size;
 	mutex_unlock(&bman->lock);
 
@@ -345,7 +345,7 @@ int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type)
 	ttm_set_driver_manager(bdev, type, NULL);
 
 	mutex_lock(&bman->lock);
-	drm_buddy_free_list(mm, &bman->reserved);
+	drm_buddy_free_list(mm, &bman->reserved, 0);
 	drm_buddy_fini(mm);
 	bman->visible_avail += bman->visible_reserved;
 	WARN_ON_ONCE(bman->visible_avail != bman->visible_size);
diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h
index b45ef0560611..06ec6ceb61d5 100644
--- a/drivers/gpu/drm/i915/i915_utils.h
+++ b/drivers/gpu/drm/i915/i915_utils.h
@@ -73,20 +73,6 @@ bool i915_error_injected(void);
 	__i915_printk(i915, i915_error_injected() ? KERN_DEBUG : KERN_ERR, \
 		      fmt, ##__VA_ARGS__)
 
-#if defined(GCC_VERSION) && GCC_VERSION >= 70000
-#define add_overflows_t(T, A, B) \
-	__builtin_add_overflow_p((A), (B), (T)0)
-#else
-#define add_overflows_t(T, A, B) ({ \
-	typeof(A) a = (A); \
-	typeof(B) b = (B); \
-	(T)(a + b) < a; \
-})
-#endif
-
-#define add_overflows(A, B) \
-	add_overflows_t(typeof((A) + (B)), (A), (B))
-
 #define range_overflows(start, size, max) ({ \
 	typeof(start) start__ = (start); \
 	typeof(size) size__ = (size); \
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index b70715b1411d..d2f064d2525c 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -1776,8 +1776,6 @@ static void release_references(struct i915_vma *vma, struct intel_gt *gt,
 	if (vm_ddestroy)
 		i915_vm_resv_put(vma->vm);
 
-	/* Wait for async active retire */
-	i915_active_wait(&vma->active);
 	i915_active_fini(&vma->active);
 	GEM_WARN_ON(vma->resource);
 	i915_vma_free(vma);
diff --git a/drivers/gpu/drm/i915/intel_clock_gating.c b/drivers/gpu/drm/i915/intel_clock_gating.c
index 9c21ce69bd98..1dc5281b2ade 100644
--- a/drivers/gpu/drm/i915/intel_clock_gating.c
+++ b/drivers/gpu/drm/i915/intel_clock_gating.c
@@ -28,6 +28,7 @@
 #include "display/intel_de.h"
 #include "display/intel_display.h"
 #include "display/intel_display_trace.h"
+#include "display/intel_fbc_regs.h"
 #include "display/skl_watermark.h"
 
 #include "gt/intel_engine_regs.h"
@@ -105,12 +106,6 @@ static void bxt_init_clock_gating(struct drm_i915_private *i915)
 	 * Display WA #0562: bxt
 	 */
 	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
-
-	/*
-	 * WaFbcHighMemBwCorruptionAvoidance:bxt
-	 * Display WA #0883: bxt
-	 */
-	intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 }
 
 static void glk_init_clock_gating(struct drm_i915_private *i915)
@@ -349,13 +344,6 @@ static void gen8_set_l3sqc_credits(struct drm_i915_private *i915,
 	intel_uncore_write(&i915->uncore, GEN7_MISCCPCTL, misccpctl);
 }
 
-static void xehpsdv_init_clock_gating(struct drm_i915_private *i915)
-{
-	/* Wa_22010146351:xehpsdv */
-	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
-		intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
-}
-
 static void dg2_init_clock_gating(struct drm_i915_private *i915)
 {
 	/* Wa_22010954014:dg2 */
@@ -363,17 +351,6 @@ static void dg2_init_clock_gating(struct drm_i915_private *i915)
 			 SGSI_SIDECLK_DIS);
 }
 
-static void pvc_init_clock_gating(struct drm_i915_private *i915)
-{
-	/* Wa_14012385139:pvc */
-	if (IS_PVC_BD_STEP(i915, STEP_A0, STEP_B0))
-		intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
-
-	/* Wa_22010954014:pvc */
-	if (IS_PVC_BD_STEP(i915, STEP_A0, STEP_B0))
-		intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
-}
-
 static void cnp_init_clock_gating(struct drm_i915_private *i915)
 {
 	if (!HAS_PCH_CNP(i915))
@@ -396,13 +373,6 @@ static void cfl_init_clock_gating(struct drm_i915_private *i915)
 	 * Display WA #0562: cfl
 	 */
 	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
-
-	/*
-	 * WaFbcNukeOnHostModify:cfl
-	 * Display WA #0873: cfl
-	 */
-	intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-			 0, DPFC_NUKE_ON_ANY_MODIFICATION);
 }
 
 static void kbl_init_clock_gating(struct drm_i915_private *i915)
@@ -427,13 +397,6 @@ static void kbl_init_clock_gating(struct drm_i915_private *i915)
 	 * Display WA #0562: kbl
 	 */
 	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
-
-	/*
-	 * WaFbcNukeOnHostModify:kbl
-	 * Display WA #0873: kbl
-	 */
-	intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-			 0, DPFC_NUKE_ON_ANY_MODIFICATION);
 }
 
 static void skl_init_clock_gating(struct drm_i915_private *i915)
@@ -452,19 +415,6 @@ static void skl_init_clock_gating(struct drm_i915_private *i915)
 	 * Display WA #0562: skl
 	 */
 	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
-
-	/*
-	 * WaFbcNukeOnHostModify:skl
-	 * Display WA #0873: skl
-	 */
-	intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-			 0, DPFC_NUKE_ON_ANY_MODIFICATION);
-
-	/*
-	 * WaFbcHighMemBwCorruptionAvoidance:skl
-	 * Display WA #0883: skl
-	 */
-	intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 }
 
 static void bdw_init_clock_gating(struct drm_i915_private *i915)
@@ -762,9 +712,7 @@ static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs =
 	.init_clock_gating = platform##_init_clock_gating,		\
 }
 
-CG_FUNCS(pvc);
 CG_FUNCS(dg2);
-CG_FUNCS(xehpsdv);
 CG_FUNCS(cfl);
 CG_FUNCS(skl);
 CG_FUNCS(kbl);
@@ -797,12 +745,8 @@ CG_FUNCS(nop);
  */
 void intel_clock_gating_hooks_init(struct drm_i915_private *i915)
 {
-	if (IS_PONTEVECCHIO(i915))
-		i915->clock_gating_funcs = &pvc_clock_gating_funcs;
-	else if (IS_DG2(i915))
+	if (IS_DG2(i915))
 		i915->clock_gating_funcs = &dg2_clock_gating_funcs;
-	else if (IS_XEHPSDV(i915))
-		i915->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
 	else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
 		i915->clock_gating_funcs = &cfl_clock_gating_funcs;
 	else if (IS_SKYLAKE(i915))
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 59bea1398c91..a0a43ea07f11 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -70,9 +70,7 @@ static const char * const platform_names[] = {
 	PLATFORM_NAME(DG1),
 	PLATFORM_NAME(ALDERLAKE_S),
 	PLATFORM_NAME(ALDERLAKE_P),
-	PLATFORM_NAME(XEHPSDV),
 	PLATFORM_NAME(DG2),
-	PLATFORM_NAME(PONTEVECCHIO),
 	PLATFORM_NAME(METEORLAKE),
 };
 #undef PLATFORM_NAME
diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h
index eba2f0b919c8..d1a2abc7e513 100644
--- a/drivers/gpu/drm/i915/intel_device_info.h
+++ b/drivers/gpu/drm/i915/intel_device_info.h
@@ -87,9 +87,7 @@ enum intel_platform {
 	INTEL_DG1,
 	INTEL_ALDERLAKE_S,
 	INTEL_ALDERLAKE_P,
-	INTEL_XEHPSDV,
 	INTEL_DG2,
-	INTEL_PONTEVECCHIO,
 	INTEL_METEORLAKE,
 	INTEL_MAX_PLATFORMS
 };
diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c
index 9b6d87c8b583..5a01d60e5186 100644
--- a/drivers/gpu/drm/i915/intel_gvt.c
+++ b/drivers/gpu/drm/i915/intel_gvt.c
@@ -28,6 +28,7 @@
 #include "gt/intel_context.h"
 #include "gt/intel_ring.h"
 #include "gt/shmem_utils.h"
+#include <linux/vmalloc.h>
 
 /**
  * DOC: Intel GVT-g host support
diff --git a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
index 87ecc5104fd9..e1a35f70b544 100644
--- a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
+++ b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
@@ -3,6 +3,7 @@
  * Copyright © 2020 Intel Corporation
  */
 
+#include "display/bxt_dpio_phy_regs.h"
 #include "display/intel_audio_regs.h"
 #include "display/intel_backlight_regs.h"
 #include "display/intel_color_regs.h"
@@ -10,9 +11,11 @@
 #include "display/intel_dmc_regs.h"
 #include "display/intel_dp_aux_regs.h"
 #include "display/intel_dpio_phy.h"
+#include "display/intel_fbc_regs.h"
 #include "display/intel_fdi_regs.h"
 #include "display/intel_lvds_regs.h"
 #include "display/intel_psr_regs.h"
+#include "display/intel_sprite_regs.h"
 #include "display/skl_watermark_regs.h"
 #include "display/vlv_dsi_pll_regs.h"
 #include "gt/intel_engine_regs.h"
@@ -1155,11 +1158,11 @@ static int iterate_bxt_mmio(struct intel_gvt_mmio_table_iter *iter)
 	MMIO_D(BXT_PORT_PCS_DW12_LN01(DPIO_PHY0, DPIO_CH0));
 	MMIO_D(BXT_PORT_PCS_DW12_LN23(DPIO_PHY0, DPIO_CH0));
 	MMIO_D(BXT_PORT_PCS_DW12_GRP(DPIO_PHY0, DPIO_CH0));
-	MMIO_D(BXT_PORT_TX_DW2_LN0(DPIO_PHY0, DPIO_CH0));
+	MMIO_D(BXT_PORT_TX_DW2_LN(DPIO_PHY0, DPIO_CH0, 0));
 	MMIO_D(BXT_PORT_TX_DW2_GRP(DPIO_PHY0, DPIO_CH0));
-	MMIO_D(BXT_PORT_TX_DW3_LN0(DPIO_PHY0, DPIO_CH0));
+	MMIO_D(BXT_PORT_TX_DW3_LN(DPIO_PHY0, DPIO_CH0, 0));
 	MMIO_D(BXT_PORT_TX_DW3_GRP(DPIO_PHY0, DPIO_CH0));
-	MMIO_D(BXT_PORT_TX_DW4_LN0(DPIO_PHY0, DPIO_CH0));
+	MMIO_D(BXT_PORT_TX_DW4_LN(DPIO_PHY0, DPIO_CH0, 0));
 	MMIO_D(BXT_PORT_TX_DW4_GRP(DPIO_PHY0, DPIO_CH0));
 	MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY0, DPIO_CH0, 0));
 	MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY0, DPIO_CH0, 1));
@@ -1180,11 +1183,11 @@ static int iterate_bxt_mmio(struct intel_gvt_mmio_table_iter *iter)
 	MMIO_D(BXT_PORT_PCS_DW12_LN01(DPIO_PHY0, DPIO_CH1));
 	MMIO_D(BXT_PORT_PCS_DW12_LN23(DPIO_PHY0, DPIO_CH1));
 	MMIO_D(BXT_PORT_PCS_DW12_GRP(DPIO_PHY0, DPIO_CH1));
-	MMIO_D(BXT_PORT_TX_DW2_LN0(DPIO_PHY0, DPIO_CH1));
+	MMIO_D(BXT_PORT_TX_DW2_LN(DPIO_PHY0, DPIO_CH1, 0));
 	MMIO_D(BXT_PORT_TX_DW2_GRP(DPIO_PHY0, DPIO_CH1));
-	MMIO_D(BXT_PORT_TX_DW3_LN0(DPIO_PHY0, DPIO_CH1));
+	MMIO_D(BXT_PORT_TX_DW3_LN(DPIO_PHY0, DPIO_CH1, 0));
 	MMIO_D(BXT_PORT_TX_DW3_GRP(DPIO_PHY0, DPIO_CH1));
-	MMIO_D(BXT_PORT_TX_DW4_LN0(DPIO_PHY0, DPIO_CH1));
+	MMIO_D(BXT_PORT_TX_DW4_LN(DPIO_PHY0, DPIO_CH1, 0));
 	MMIO_D(BXT_PORT_TX_DW4_GRP(DPIO_PHY0, DPIO_CH1));
 	MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY0, DPIO_CH1, 0));
 	MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY0, DPIO_CH1, 1));
@@ -1205,11 +1208,11 @@ static int iterate_bxt_mmio(struct intel_gvt_mmio_table_iter *iter)
 	MMIO_D(BXT_PORT_PCS_DW12_LN01(DPIO_PHY1, DPIO_CH0));
 	MMIO_D(BXT_PORT_PCS_DW12_LN23(DPIO_PHY1, DPIO_CH0));
 	MMIO_D(BXT_PORT_PCS_DW12_GRP(DPIO_PHY1, DPIO_CH0));
-	MMIO_D(BXT_PORT_TX_DW2_LN0(DPIO_PHY1, DPIO_CH0));
+	MMIO_D(BXT_PORT_TX_DW2_LN(DPIO_PHY1, DPIO_CH0, 0));
 	MMIO_D(BXT_PORT_TX_DW2_GRP(DPIO_PHY1, DPIO_CH0));
-	MMIO_D(BXT_PORT_TX_DW3_LN0(DPIO_PHY1, DPIO_CH0));
+	MMIO_D(BXT_PORT_TX_DW3_LN(DPIO_PHY1, DPIO_CH0, 0));
 	MMIO_D(BXT_PORT_TX_DW3_GRP(DPIO_PHY1, DPIO_CH0));
-	MMIO_D(BXT_PORT_TX_DW4_LN0(DPIO_PHY1, DPIO_CH0));
+	MMIO_D(BXT_PORT_TX_DW4_LN(DPIO_PHY1, DPIO_CH0, 0));
 	MMIO_D(BXT_PORT_TX_DW4_GRP(DPIO_PHY1, DPIO_CH0));
 	MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY1, DPIO_CH0, 0));
 	MMIO_D(BXT_PORT_TX_DW14_LN(DPIO_PHY1, DPIO_CH0, 1));
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index d4e844128826..2d0647aca964 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -272,15 +272,11 @@ intel_wakeref_t intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm)
  * intel_runtime_pm_get_noresume - grab a runtime pm reference
  * @rpm: the intel_runtime_pm structure
  *
- * This function grabs a device-level runtime pm reference (mostly used for GEM
- * code to ensure the GTT or GT is on).
+ * This function grabs a device-level runtime pm reference.
  *
- * It will _not_ power up the device but instead only check that it's powered
- * on.  Therefore it is only valid to call this functions from contexts where
- * the device is known to be powered up and where trying to power it up would
- * result in hilarity and deadlocks. That pretty much means only the system
- * suspend/resume code where this is used to grab runtime pm references for
- * delayed setup down in work items.
+ * It will _not_ resume the device but instead only get an extra wakeref.
+ * Therefore it is only valid to call this functions from contexts where
+ * the device is known to be active and with another wakeref previously hold.
  *
  * Any runtime pm reference obtained by this function must have a symmetric
  * call to intel_runtime_pm_put() to release the reference again.
@@ -289,7 +285,7 @@ intel_wakeref_t intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm)
  */
 intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm)
 {
-	assert_rpm_wakelock_held(rpm);
+	assert_rpm_raw_wakeref_held(rpm);
 	pm_runtime_get_noresume(rpm->kdev);
 
 	intel_runtime_pm_acquire(rpm, true);
diff --git a/drivers/gpu/drm/i915/intel_step.c b/drivers/gpu/drm/i915/intel_step.c
index b4162f1be765..a5adfb5d8fd2 100644
--- a/drivers/gpu/drm/i915/intel_step.c
+++ b/drivers/gpu/drm/i915/intel_step.c
@@ -102,13 +102,6 @@ static const struct intel_step_info adlp_revids[] = {
 	[0xC] = { COMMON_GT_MEDIA_STEP(C0), .display_step = STEP_D0 },
 };
 
-static const struct intel_step_info xehpsdv_revids[] = {
-	[0x0] = { COMMON_GT_MEDIA_STEP(A0) },
-	[0x1] = { COMMON_GT_MEDIA_STEP(A1) },
-	[0x4] = { COMMON_GT_MEDIA_STEP(B0) },
-	[0x8] = { COMMON_GT_MEDIA_STEP(C0) },
-};
-
 static const struct intel_step_info dg2_g10_revid_step_tbl[] = {
 	[0x0] = { COMMON_GT_MEDIA_STEP(A0), .display_step = STEP_A0 },
 	[0x1] = { COMMON_GT_MEDIA_STEP(A1), .display_step = STEP_A0 },
@@ -153,8 +146,6 @@ static u8 gmd_to_intel_step(struct drm_i915_private *i915,
 	return step;
 }
 
-static void pvc_step_init(struct drm_i915_private *i915, int pci_revid);
-
 void intel_step_init(struct drm_i915_private *i915)
 {
 	const struct intel_step_info *revids = NULL;
@@ -178,10 +169,7 @@ void intel_step_init(struct drm_i915_private *i915)
 		return;
 	}
 
-	if (IS_PONTEVECCHIO(i915)) {
-		pvc_step_init(i915, revid);
-		return;
-	} else if (IS_DG2_G10(i915)) {
+	if (IS_DG2_G10(i915)) {
 		revids = dg2_g10_revid_step_tbl;
 		size = ARRAY_SIZE(dg2_g10_revid_step_tbl);
 	} else if (IS_DG2_G11(i915)) {
@@ -190,9 +178,6 @@ void intel_step_init(struct drm_i915_private *i915)
 	} else if (IS_DG2_G12(i915)) {
 		revids = dg2_g12_revid_step_tbl;
 		size = ARRAY_SIZE(dg2_g12_revid_step_tbl);
-	} else if (IS_XEHPSDV(i915)) {
-		revids = xehpsdv_revids;
-		size = ARRAY_SIZE(xehpsdv_revids);
 	} else if (IS_ALDERLAKE_P_N(i915)) {
 		revids = adlp_n_revids;
 		size = ARRAY_SIZE(adlp_n_revids);
@@ -277,69 +262,6 @@ void intel_step_init(struct drm_i915_private *i915)
 	RUNTIME_INFO(i915)->step = step;
 }
 
-#define PVC_BD_REVID	GENMASK(5, 3)
-#define PVC_CT_REVID	GENMASK(2, 0)
-
-static const int pvc_bd_subids[] = {
-	[0x0] = STEP_A0,
-	[0x3] = STEP_B0,
-	[0x4] = STEP_B1,
-	[0x5] = STEP_B3,
-};
-
-static const int pvc_ct_subids[] = {
-	[0x3] = STEP_A0,
-	[0x5] = STEP_B0,
-	[0x6] = STEP_B1,
-	[0x7] = STEP_C0,
-};
-
-static int
-pvc_step_lookup(struct drm_i915_private *i915, const char *type,
-		const int *table, int size, int subid)
-{
-	if (subid < size && table[subid] != STEP_NONE)
-		return table[subid];
-
-	drm_warn(&i915->drm, "Unknown %s id 0x%02x\n", type, subid);
-
-	/*
-	 * As on other platforms, try to use the next higher ID if we land on a
-	 * gap in the table.
-	 */
-	while (subid < size && table[subid] == STEP_NONE)
-		subid++;
-
-	if (subid < size) {
-		drm_dbg(&i915->drm, "Using steppings for %s id 0x%02x\n",
-			type, subid);
-		return table[subid];
-	}
-
-	drm_dbg(&i915->drm, "Using future steppings\n");
-	return STEP_FUTURE;
-}
-
-/*
- * PVC needs special handling since we don't lookup the
- * revid in a table, but rather specific bitfields within
- * the revid for various components.
- */
-static void pvc_step_init(struct drm_i915_private *i915, int pci_revid)
-{
-	int ct_subid, bd_subid;
-
-	bd_subid = FIELD_GET(PVC_BD_REVID, pci_revid);
-	ct_subid = FIELD_GET(PVC_CT_REVID, pci_revid);
-
-	RUNTIME_INFO(i915)->step.basedie_step =
-		pvc_step_lookup(i915, "Base Die", pvc_bd_subids,
-				ARRAY_SIZE(pvc_bd_subids), bd_subid);
-	RUNTIME_INFO(i915)->step.graphics_step =
-		pvc_step_lookup(i915, "Compute Tile", pvc_ct_subids,
-				ARRAY_SIZE(pvc_ct_subids), ct_subid);
-}
-
 #define STEP_NAME_CASE(name)	\
 	case STEP_##name:	\
 		return #name;
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 76400e9c40f0..729409a4bada 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -1106,45 +1106,6 @@ static const struct i915_range dg2_shadowed_regs[] = {
 	{ .start = 0x1F8510, .end = 0x1F8550 },
 };
 
-static const struct i915_range pvc_shadowed_regs[] = {
-	{ .start =   0x2030, .end =   0x2030 },
-	{ .start =   0x2510, .end =   0x2550 },
-	{ .start =   0xA008, .end =   0xA00C },
-	{ .start =   0xA188, .end =   0xA188 },
-	{ .start =   0xA278, .end =   0xA278 },
-	{ .start =   0xA540, .end =   0xA56C },
-	{ .start =   0xC4C8, .end =   0xC4C8 },
-	{ .start =   0xC4E0, .end =   0xC4E0 },
-	{ .start =   0xC600, .end =   0xC600 },
-	{ .start =   0xC658, .end =   0xC658 },
-	{ .start =  0x22030, .end =  0x22030 },
-	{ .start =  0x22510, .end =  0x22550 },
-	{ .start = 0x1C0030, .end = 0x1C0030 },
-	{ .start = 0x1C0510, .end = 0x1C0550 },
-	{ .start = 0x1C4030, .end = 0x1C4030 },
-	{ .start = 0x1C4510, .end = 0x1C4550 },
-	{ .start = 0x1C8030, .end = 0x1C8030 },
-	{ .start = 0x1C8510, .end = 0x1C8550 },
-	{ .start = 0x1D0030, .end = 0x1D0030 },
-	{ .start = 0x1D0510, .end = 0x1D0550 },
-	{ .start = 0x1D4030, .end = 0x1D4030 },
-	{ .start = 0x1D4510, .end = 0x1D4550 },
-	{ .start = 0x1D8030, .end = 0x1D8030 },
-	{ .start = 0x1D8510, .end = 0x1D8550 },
-	{ .start = 0x1E0030, .end = 0x1E0030 },
-	{ .start = 0x1E0510, .end = 0x1E0550 },
-	{ .start = 0x1E4030, .end = 0x1E4030 },
-	{ .start = 0x1E4510, .end = 0x1E4550 },
-	{ .start = 0x1E8030, .end = 0x1E8030 },
-	{ .start = 0x1E8510, .end = 0x1E8550 },
-	{ .start = 0x1F0030, .end = 0x1F0030 },
-	{ .start = 0x1F0510, .end = 0x1F0550 },
-	{ .start = 0x1F4030, .end = 0x1F4030 },
-	{ .start = 0x1F4510, .end = 0x1F4550 },
-	{ .start = 0x1F8030, .end = 0x1F8030 },
-	{ .start = 0x1F8510, .end = 0x1F8550 },
-};
-
 static const struct i915_range mtl_shadowed_regs[] = {
 	{ .start =   0x2030, .end =   0x2030 },
 	{ .start =   0x2510, .end =   0x2550 },
@@ -1471,195 +1432,31 @@ static const struct intel_forcewake_range __gen12_fw_ranges[] = {
 		0x1d3f00 - 0x1d3fff: VD2 */
 };
 
-/*
- * Graphics IP version 12.55 brings a slight change to the 0xd800 range,
- * switching it from the GT domain to the render domain.
- */
-#define XEHP_FWRANGES(FW_RANGE_D800)					\
-	GEN_FW_RANGE(0x0, 0x1fff, 0), /*					\
-		  0x0 -  0xaff: reserved					\
-		0xb00 - 0x1fff: always on */					\
-	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),				\
-	GEN_FW_RANGE(0x2700, 0x4aff, FORCEWAKE_GT),				\
-	GEN_FW_RANGE(0x4b00, 0x51ff, 0), /*					\
-		0x4b00 - 0x4fff: reserved					\
-		0x5000 - 0x51ff: always on */					\
-	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),				\
-	GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),				\
-	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),				\
-	GEN_FW_RANGE(0x8160, 0x81ff, 0), /*					\
-		0x8160 - 0x817f: reserved					\
-		0x8180 - 0x81ff: always on */					\
-	GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT),				\
-	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),				\
-	GEN_FW_RANGE(0x8500, 0x8cff, FORCEWAKE_GT), /*				\
-		0x8500 - 0x87ff: gt						\
-		0x8800 - 0x8c7f: reserved					\
-		0x8c80 - 0x8cff: gt (DG2 only) */				\
-	GEN_FW_RANGE(0x8d00, 0x8fff, FORCEWAKE_RENDER), /*			\
-		0x8d00 - 0x8dff: render (DG2 only)				\
-		0x8e00 - 0x8fff: reserved */					\
-	GEN_FW_RANGE(0x9000, 0x94cf, FORCEWAKE_GT), /*				\
-		0x9000 - 0x947f: gt						\
-		0x9480 - 0x94cf: reserved */					\
-	GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),				\
-	GEN_FW_RANGE(0x9560, 0x967f, 0), /*					\
-		0x9560 - 0x95ff: always on					\
-		0x9600 - 0x967f: reserved */					\
-	GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /*			\
-		0x9680 - 0x96ff: render (DG2 only)				\
-		0x9700 - 0x97ff: reserved */					\
-	GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /*				\
-		0x9800 - 0xb4ff: gt						\
-		0xb500 - 0xbfff: reserved					\
-		0xc000 - 0xcfff: gt */						\
-	GEN_FW_RANGE(0xd000, 0xd7ff, 0),					\
-	GEN_FW_RANGE(0xd800, 0xd87f, FW_RANGE_D800),			\
-	GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT),				\
-	GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),				\
-	GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*				\
-		0xdd00 - 0xddff: gt						\
-		0xde00 - 0xde7f: reserved */					\
-	GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*			\
-		0xde80 - 0xdfff: render						\
-		0xe000 - 0xe0ff: reserved					\
-		0xe100 - 0xe8ff: render */					\
-	GEN_FW_RANGE(0xe900, 0xffff, FORCEWAKE_GT), /*				\
-		0xe900 - 0xe9ff: gt						\
-		0xea00 - 0xefff: reserved					\
-		0xf000 - 0xffff: gt */						\
-	GEN_FW_RANGE(0x10000, 0x12fff, 0), /*					\
-		0x10000 - 0x11fff: reserved					\
-		0x12000 - 0x127ff: always on					\
-		0x12800 - 0x12fff: reserved */					\
-	GEN_FW_RANGE(0x13000, 0x131ff, FORCEWAKE_MEDIA_VDBOX0), /* DG2 only */	\
-	GEN_FW_RANGE(0x13200, 0x13fff, FORCEWAKE_MEDIA_VDBOX2), /*		\
-		0x13200 - 0x133ff: VD2 (DG2 only)				\
-		0x13400 - 0x13fff: reserved */					\
-	GEN_FW_RANGE(0x14000, 0x141ff, FORCEWAKE_MEDIA_VDBOX0), /* XEHPSDV only */	\
-	GEN_FW_RANGE(0x14200, 0x143ff, FORCEWAKE_MEDIA_VDBOX2), /* XEHPSDV only */	\
-	GEN_FW_RANGE(0x14400, 0x145ff, FORCEWAKE_MEDIA_VDBOX4), /* XEHPSDV only */	\
-	GEN_FW_RANGE(0x14600, 0x147ff, FORCEWAKE_MEDIA_VDBOX6), /* XEHPSDV only */	\
-	GEN_FW_RANGE(0x14800, 0x14fff, FORCEWAKE_RENDER),			\
-	GEN_FW_RANGE(0x15000, 0x16dff, FORCEWAKE_GT), /*			\
-		0x15000 - 0x15fff: gt (DG2 only)				\
-		0x16000 - 0x16dff: reserved */					\
-	GEN_FW_RANGE(0x16e00, 0x1ffff, FORCEWAKE_RENDER),			\
-	GEN_FW_RANGE(0x20000, 0x21fff, FORCEWAKE_MEDIA_VDBOX0), /*		\
-		0x20000 - 0x20fff: VD0 (XEHPSDV only)				\
-		0x21000 - 0x21fff: reserved */					\
-	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),				\
-	GEN_FW_RANGE(0x24000, 0x2417f, 0), /*					\
-		0x24000 - 0x2407f: always on					\
-		0x24080 - 0x2417f: reserved */					\
-	GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /*			\
-		0x24180 - 0x241ff: gt						\
-		0x24200 - 0x249ff: reserved */					\
-	GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /*			\
-		0x24a00 - 0x24a7f: render					\
-		0x24a80 - 0x251ff: reserved */					\
-	GEN_FW_RANGE(0x25200, 0x25fff, FORCEWAKE_GT), /*			\
-		0x25200 - 0x252ff: gt						\
-		0x25300 - 0x25fff: reserved */					\
-	GEN_FW_RANGE(0x26000, 0x2ffff, FORCEWAKE_RENDER), /*			\
-		0x26000 - 0x27fff: render					\
-		0x28000 - 0x29fff: reserved					\
-		0x2a000 - 0x2ffff: undocumented */				\
-	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),				\
-	GEN_FW_RANGE(0x40000, 0x1bffff, 0),					\
-	GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*		\
-		0x1c0000 - 0x1c2bff: VD0					\
-		0x1c2c00 - 0x1c2cff: reserved					\
-		0x1c2d00 - 0x1c2dff: VD0					\
-		0x1c2e00 - 0x1c3eff: VD0 (DG2 only)				\
-		0x1c3f00 - 0x1c3fff: VD0 */					\
-	GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), /*		\
-		0x1c4000 - 0x1c6bff: VD1					\
-		0x1c6c00 - 0x1c6cff: reserved					\
-		0x1c6d00 - 0x1c6dff: VD1					\
-		0x1c6e00 - 0x1c7fff: reserved */				\
-	GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*		\
-		0x1c8000 - 0x1ca0ff: VE0					\
-		0x1ca100 - 0x1cbfff: reserved */				\
-	GEN_FW_RANGE(0x1cc000, 0x1ccfff, FORCEWAKE_MEDIA_VDBOX0),		\
-	GEN_FW_RANGE(0x1cd000, 0x1cdfff, FORCEWAKE_MEDIA_VDBOX2),		\
-	GEN_FW_RANGE(0x1ce000, 0x1cefff, FORCEWAKE_MEDIA_VDBOX4),		\
-	GEN_FW_RANGE(0x1cf000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX6),		\
-	GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /*		\
-		0x1d0000 - 0x1d2bff: VD2					\
-		0x1d2c00 - 0x1d2cff: reserved					\
-		0x1d2d00 - 0x1d2dff: VD2					\
-		0x1d2e00 - 0x1d3dff: VD2 (DG2 only)				\
-		0x1d3e00 - 0x1d3eff: reserved					\
-		0x1d3f00 - 0x1d3fff: VD2 */					\
-	GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), /*		\
-		0x1d4000 - 0x1d6bff: VD3					\
-		0x1d6c00 - 0x1d6cff: reserved					\
-		0x1d6d00 - 0x1d6dff: VD3					\
-		0x1d6e00 - 0x1d7fff: reserved */				\
-	GEN_FW_RANGE(0x1d8000, 0x1dffff, FORCEWAKE_MEDIA_VEBOX1), /*		\
-		0x1d8000 - 0x1da0ff: VE1					\
-		0x1da100 - 0x1dffff: reserved */				\
-	GEN_FW_RANGE(0x1e0000, 0x1e3fff, FORCEWAKE_MEDIA_VDBOX4), /*		\
-		0x1e0000 - 0x1e2bff: VD4					\
-		0x1e2c00 - 0x1e2cff: reserved					\
-		0x1e2d00 - 0x1e2dff: VD4					\
-		0x1e2e00 - 0x1e3eff: reserved					\
-		0x1e3f00 - 0x1e3fff: VD4 */					\
-	GEN_FW_RANGE(0x1e4000, 0x1e7fff, FORCEWAKE_MEDIA_VDBOX5), /*		\
-		0x1e4000 - 0x1e6bff: VD5					\
-		0x1e6c00 - 0x1e6cff: reserved					\
-		0x1e6d00 - 0x1e6dff: VD5					\
-		0x1e6e00 - 0x1e7fff: reserved */				\
-	GEN_FW_RANGE(0x1e8000, 0x1effff, FORCEWAKE_MEDIA_VEBOX2), /*		\
-		0x1e8000 - 0x1ea0ff: VE2					\
-		0x1ea100 - 0x1effff: reserved */				\
-	GEN_FW_RANGE(0x1f0000, 0x1f3fff, FORCEWAKE_MEDIA_VDBOX6), /*		\
-		0x1f0000 - 0x1f2bff: VD6					\
-		0x1f2c00 - 0x1f2cff: reserved					\
-		0x1f2d00 - 0x1f2dff: VD6					\
-		0x1f2e00 - 0x1f3eff: reserved					\
-		0x1f3f00 - 0x1f3fff: VD6 */					\
-	GEN_FW_RANGE(0x1f4000, 0x1f7fff, FORCEWAKE_MEDIA_VDBOX7), /*		\
-		0x1f4000 - 0x1f6bff: VD7					\
-		0x1f6c00 - 0x1f6cff: reserved					\
-		0x1f6d00 - 0x1f6dff: VD7					\
-		0x1f6e00 - 0x1f7fff: reserved */				\
-	GEN_FW_RANGE(0x1f8000, 0x1fa0ff, FORCEWAKE_MEDIA_VEBOX3),
-
-static const struct intel_forcewake_range __xehp_fw_ranges[] = {
-	XEHP_FWRANGES(FORCEWAKE_GT)
-};
-
 static const struct intel_forcewake_range __dg2_fw_ranges[] = {
-	XEHP_FWRANGES(FORCEWAKE_RENDER)
-};
-
-static const struct intel_forcewake_range __pvc_fw_ranges[] = {
-	GEN_FW_RANGE(0x0, 0xaff, 0),
-	GEN_FW_RANGE(0xb00, 0xbff, FORCEWAKE_GT),
-	GEN_FW_RANGE(0xc00, 0xfff, 0),
-	GEN_FW_RANGE(0x1000, 0x1fff, FORCEWAKE_GT),
+	GEN_FW_RANGE(0x0, 0x1fff, 0), /*
+		  0x0 -  0xaff: reserved
+		0xb00 - 0x1fff: always on */
 	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
-	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
-	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
-	GEN_FW_RANGE(0x4000, 0x813f, FORCEWAKE_GT), /*
-		0x4000 - 0x4aff: gt
+	GEN_FW_RANGE(0x2700, 0x4aff, FORCEWAKE_GT),
+	GEN_FW_RANGE(0x4b00, 0x51ff, 0), /*
 		0x4b00 - 0x4fff: reserved
-		0x5000 - 0x51ff: gt
-		0x5200 - 0x52ff: reserved
-		0x5300 - 0x53ff: gt
-		0x5400 - 0x7fff: reserved
-		0x8000 - 0x813f: gt */
-	GEN_FW_RANGE(0x8140, 0x817f, FORCEWAKE_RENDER),
-	GEN_FW_RANGE(0x8180, 0x81ff, 0),
-	GEN_FW_RANGE(0x8200, 0x94cf, FORCEWAKE_GT), /*
-		0x8200 - 0x82ff: gt
-		0x8300 - 0x84ff: reserved
-		0x8500 - 0x887f: gt
-		0x8880 - 0x8a7f: reserved
-		0x8a80 - 0x8aff: gt
-		0x8b00 - 0x8fff: reserved
+		0x5000 - 0x51ff: always on */
+	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
+	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x8160, 0x81ff, 0), /*
+		0x8160 - 0x817f: reserved
+		0x8180 - 0x81ff: always on */
+	GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT),
+	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x8500, 0x8cff, FORCEWAKE_GT), /*
+		0x8500 - 0x87ff: gt
+		0x8800 - 0x8c7f: reserved
+		0x8c80 - 0x8cff: gt (DG2 only) */
+	GEN_FW_RANGE(0x8d00, 0x8fff, FORCEWAKE_RENDER), /*
+		0x8d00 - 0x8dff: render (DG2 only)
+		0x8e00 - 0x8fff: reserved */
+	GEN_FW_RANGE(0x9000, 0x94cf, FORCEWAKE_GT), /*
 		0x9000 - 0x947f: gt
 		0x9480 - 0x94cf: reserved */
 	GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
@@ -1673,65 +1470,114 @@ static const struct intel_forcewake_range __pvc_fw_ranges[] = {
 		0x9800 - 0xb4ff: gt
 		0xb500 - 0xbfff: reserved
 		0xc000 - 0xcfff: gt */
-	GEN_FW_RANGE(0xd000, 0xd3ff, 0),
-	GEN_FW_RANGE(0xd400, 0xdbff, FORCEWAKE_GT),
+	GEN_FW_RANGE(0xd000, 0xd7ff, 0),
+	GEN_FW_RANGE(0xd800, 0xd87f, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT),
 	GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),
 	GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*
 		0xdd00 - 0xddff: gt
 		0xde00 - 0xde7f: reserved */
 	GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*
-		0xde80 - 0xdeff: render
-		0xdf00 - 0xe1ff: reserved
-		0xe200 - 0xe7ff: render
-		0xe800 - 0xe8ff: reserved */
-	GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_GT), /*
-		 0xe900 -  0xe9ff: gt
-		 0xea00 -  0xebff: reserved
-		 0xec00 -  0xffff: gt
-		0x10000 - 0x11fff: reserved */
-	GEN_FW_RANGE(0x12000, 0x12fff, 0), /*
+		0xde80 - 0xdfff: render
+		0xe000 - 0xe0ff: reserved
+		0xe100 - 0xe8ff: render */
+	GEN_FW_RANGE(0xe900, 0xffff, FORCEWAKE_GT), /*
+		0xe900 - 0xe9ff: gt
+		0xea00 - 0xefff: reserved
+		0xf000 - 0xffff: gt */
+	GEN_FW_RANGE(0x10000, 0x12fff, 0), /*
+		0x10000 - 0x11fff: reserved
 		0x12000 - 0x127ff: always on
 		0x12800 - 0x12fff: reserved */
-	GEN_FW_RANGE(0x13000, 0x19fff, FORCEWAKE_GT), /*
-		0x13000 - 0x135ff: gt
-		0x13600 - 0x147ff: reserved
-		0x14800 - 0x153ff: gt
-		0x15400 - 0x19fff: reserved */
-	GEN_FW_RANGE(0x1a000, 0x21fff, FORCEWAKE_RENDER), /*
-		0x1a000 - 0x1ffff: render
+	GEN_FW_RANGE(0x13000, 0x131ff, FORCEWAKE_MEDIA_VDBOX0),
+	GEN_FW_RANGE(0x13200, 0x147ff, FORCEWAKE_MEDIA_VDBOX2), /*
+		0x13200 - 0x133ff: VD2 (DG2 only)
+		0x13400 - 0x147ff: reserved */
+	GEN_FW_RANGE(0x14800, 0x14fff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x15000, 0x16dff, FORCEWAKE_GT), /*
+		0x15000 - 0x15fff: gt (DG2 only)
+		0x16000 - 0x16dff: reserved */
+	GEN_FW_RANGE(0x16e00, 0x21fff, FORCEWAKE_RENDER), /*
+		0x16e00 - 0x1ffff: render
 		0x20000 - 0x21fff: reserved */
 	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
 	GEN_FW_RANGE(0x24000, 0x2417f, 0), /*
-		24000 - 0x2407f: always on
-		24080 - 0x2417f: reserved */
-	GEN_FW_RANGE(0x24180, 0x25fff, FORCEWAKE_GT), /*
+		0x24000 - 0x2407f: always on
+		0x24080 - 0x2417f: reserved */
+	GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /*
 		0x24180 - 0x241ff: gt
-		0x24200 - 0x251ff: reserved
+		0x24200 - 0x249ff: reserved */
+	GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /*
+		0x24a00 - 0x24a7f: render
+		0x24a80 - 0x251ff: reserved */
+	GEN_FW_RANGE(0x25200, 0x25fff, FORCEWAKE_GT), /*
 		0x25200 - 0x252ff: gt
 		0x25300 - 0x25fff: reserved */
 	GEN_FW_RANGE(0x26000, 0x2ffff, FORCEWAKE_RENDER), /*
 		0x26000 - 0x27fff: render
-		0x28000 - 0x2ffff: reserved */
+		0x28000 - 0x29fff: reserved
+		0x2a000 - 0x2ffff: undocumented */
 	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
 	GEN_FW_RANGE(0x40000, 0x1bffff, 0),
 	GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*
 		0x1c0000 - 0x1c2bff: VD0
 		0x1c2c00 - 0x1c2cff: reserved
 		0x1c2d00 - 0x1c2dff: VD0
-		0x1c2e00 - 0x1c3eff: reserved
+		0x1c2e00 - 0x1c3eff: VD0
 		0x1c3f00 - 0x1c3fff: VD0 */
-	GEN_FW_RANGE(0x1c4000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX1), /*
-		0x1c4000 - 0x1c6aff: VD1
-		0x1c6b00 - 0x1c7eff: reserved
-		0x1c7f00 - 0x1c7fff: VD1
-		0x1c8000 - 0x1cffff: reserved */
-	GEN_FW_RANGE(0x1d0000, 0x23ffff, FORCEWAKE_MEDIA_VDBOX2), /*
-		0x1d0000 - 0x1d2aff: VD2
-		0x1d2b00 - 0x1d3eff: reserved
-		0x1d3f00 - 0x1d3fff: VD2
-		0x1d4000 - 0x23ffff: reserved */
-	GEN_FW_RANGE(0x240000, 0x3dffff, 0),
-	GEN_FW_RANGE(0x3e0000, 0x3effff, FORCEWAKE_GT),
+	GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), /*
+		0x1c4000 - 0x1c6bff: VD1
+		0x1c6c00 - 0x1c6cff: reserved
+		0x1c6d00 - 0x1c6dff: VD1
+		0x1c6e00 - 0x1c7fff: reserved */
+	GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
+		0x1c8000 - 0x1ca0ff: VE0
+		0x1ca100 - 0x1cbfff: reserved */
+	GEN_FW_RANGE(0x1cc000, 0x1ccfff, FORCEWAKE_MEDIA_VDBOX0),
+	GEN_FW_RANGE(0x1cd000, 0x1cdfff, FORCEWAKE_MEDIA_VDBOX2),
+	GEN_FW_RANGE(0x1ce000, 0x1cefff, FORCEWAKE_MEDIA_VDBOX4),
+	GEN_FW_RANGE(0x1cf000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX6),
+	GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /*
+		0x1d0000 - 0x1d2bff: VD2
+		0x1d2c00 - 0x1d2cff: reserved
+		0x1d2d00 - 0x1d2dff: VD2
+		0x1d2e00 - 0x1d3dff: VD2
+		0x1d3e00 - 0x1d3eff: reserved
+		0x1d3f00 - 0x1d3fff: VD2 */
+	GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), /*
+		0x1d4000 - 0x1d6bff: VD3
+		0x1d6c00 - 0x1d6cff: reserved
+		0x1d6d00 - 0x1d6dff: VD3
+		0x1d6e00 - 0x1d7fff: reserved */
+	GEN_FW_RANGE(0x1d8000, 0x1dffff, FORCEWAKE_MEDIA_VEBOX1), /*
+		0x1d8000 - 0x1da0ff: VE1
+		0x1da100 - 0x1dffff: reserved */
+	GEN_FW_RANGE(0x1e0000, 0x1e3fff, FORCEWAKE_MEDIA_VDBOX4), /*
+		0x1e0000 - 0x1e2bff: VD4
+		0x1e2c00 - 0x1e2cff: reserved
+		0x1e2d00 - 0x1e2dff: VD4
+		0x1e2e00 - 0x1e3eff: reserved
+		0x1e3f00 - 0x1e3fff: VD4 */
+	GEN_FW_RANGE(0x1e4000, 0x1e7fff, FORCEWAKE_MEDIA_VDBOX5), /*
+		0x1e4000 - 0x1e6bff: VD5
+		0x1e6c00 - 0x1e6cff: reserved
+		0x1e6d00 - 0x1e6dff: VD5
+		0x1e6e00 - 0x1e7fff: reserved */
+	GEN_FW_RANGE(0x1e8000, 0x1effff, FORCEWAKE_MEDIA_VEBOX2), /*
+		0x1e8000 - 0x1ea0ff: VE2
+		0x1ea100 - 0x1effff: reserved */
+	GEN_FW_RANGE(0x1f0000, 0x1f3fff, FORCEWAKE_MEDIA_VDBOX6), /*
+		0x1f0000 - 0x1f2bff: VD6
+		0x1f2c00 - 0x1f2cff: reserved
+		0x1f2d00 - 0x1f2dff: VD6
+		0x1f2e00 - 0x1f3eff: reserved
+		0x1f3f00 - 0x1f3fff: VD6 */
+	GEN_FW_RANGE(0x1f4000, 0x1f7fff, FORCEWAKE_MEDIA_VDBOX7), /*
+		0x1f4000 - 0x1f6bff: VD7
+		0x1f6c00 - 0x1f6cff: reserved
+		0x1f6d00 - 0x1f6dff: VD7
+		0x1f6e00 - 0x1f7fff: reserved */
+	GEN_FW_RANGE(0x1f8000, 0x1fa0ff, FORCEWAKE_MEDIA_VEBOX3),
 };
 
 static const struct intel_forcewake_range __mtl_fw_ranges[] = {
@@ -2576,18 +2422,10 @@ static int uncore_forcewake_init(struct intel_uncore *uncore)
 		ASSIGN_FW_DOMAINS_TABLE(uncore, __mtl_fw_ranges);
 		ASSIGN_SHADOW_TABLE(uncore, mtl_shadowed_regs);
 		ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
-	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 60)) {
-		ASSIGN_FW_DOMAINS_TABLE(uncore, __pvc_fw_ranges);
-		ASSIGN_SHADOW_TABLE(uncore, pvc_shadowed_regs);
-		ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
 	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
 		ASSIGN_FW_DOMAINS_TABLE(uncore, __dg2_fw_ranges);
 		ASSIGN_SHADOW_TABLE(uncore, dg2_shadowed_regs);
 		ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
-	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
-		ASSIGN_FW_DOMAINS_TABLE(uncore, __xehp_fw_ranges);
-		ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs);
-		ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
 	} else if (GRAPHICS_VER(i915) >= 12) {
 		ASSIGN_FW_DOMAINS_TABLE(uncore, __gen12_fw_ranges);
 		ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs);
@@ -2734,7 +2572,7 @@ void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore,
 		 * the forcewake domain if any of the other engines
 		 * in the same media slice are present.
 		 */
-		if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 50) && i % 2 == 0) {
+		if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 55) && i % 2 == 0) {
 			if ((i + 1 < I915_MAX_VCS) && HAS_ENGINE(gt, _VCS(i + 1)))
 				continue;
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_selftest.c b/drivers/gpu/drm/i915/selftests/i915_selftest.c
index ee79e0809a6d..fee76c1d2f45 100644
--- a/drivers/gpu/drm/i915/selftests/i915_selftest.c
+++ b/drivers/gpu/drm/i915/selftests/i915_selftest.c
@@ -154,6 +154,30 @@ __wait_gsc_proxy_completed(struct drm_i915_private *i915)
 		pr_warn(DRIVER_NAME "Timed out waiting for gsc_proxy_completion!\n");
 }
 
+static void
+__wait_gsc_huc_load_completed(struct drm_i915_private *i915)
+{
+	/* this only applies to DG2, so we only care about GT0 */
+	struct intel_huc *huc = &to_gt(i915)->uc.huc;
+	bool need_to_wait = (IS_ENABLED(CONFIG_INTEL_MEI_PXP) &&
+			     intel_huc_wait_required(huc));
+	/*
+	 * The GSC and PXP mei bringup depends on the kernel boot ordering, so
+	 * to account for the worst case scenario the HuC code waits for up to
+	 * 10s for the GSC driver to load and then another 5s for the PXP
+	 * component to bind before giving up, even though those steps normally
+	 * complete in less than a second from the i915 load. We match that
+	 * timeout here, but we expect to bail early due to the fence being
+	 * signalled even in a failure case, as it is extremely unlikely that
+	 * both components will use their full timeout.
+	 */
+	unsigned long timeout_ms = 15000;
+
+	if (need_to_wait &&
+	    wait_for(i915_sw_fence_done(&huc->delayed_load.fence), timeout_ms))
+		pr_warn(DRIVER_NAME "Timed out waiting for huc load via GSC!\n");
+}
+
 static int __run_selftests(const char *name,
 			   struct selftest *st,
 			   unsigned int count,
@@ -228,14 +252,16 @@ int i915_mock_selftests(void)
 
 int i915_live_selftests(struct pci_dev *pdev)
 {
+	struct drm_i915_private *i915 = pdev_to_i915(pdev);
 	int err;
 
 	if (!i915_selftest.live)
 		return 0;
 
-	__wait_gsc_proxy_completed(pdev_to_i915(pdev));
+	__wait_gsc_proxy_completed(i915);
+	__wait_gsc_huc_load_completed(i915);
 
-	err = run_selftests(live, pdev_to_i915(pdev));
+	err = run_selftests(live, i915);
 	if (err) {
 		i915_selftest.live = err;
 		return err;
@@ -251,14 +277,16 @@ int i915_live_selftests(struct pci_dev *pdev)
 
 int i915_perf_selftests(struct pci_dev *pdev)
 {
+	struct drm_i915_private *i915 = pdev_to_i915(pdev);
 	int err;
 
 	if (!i915_selftest.perf)
 		return 0;
 
-	__wait_gsc_proxy_completed(pdev_to_i915(pdev));
+	__wait_gsc_proxy_completed(i915);
+	__wait_gsc_huc_load_completed(i915);
 
-	err = run_selftests(perf, pdev_to_i915(pdev));
+	err = run_selftests(perf, i915);
 	if (err) {
 		i915_selftest.perf = err;
 		return err;
diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c
index 4f98aa8a861e..41eaa9b7f67d 100644
--- a/drivers/gpu/drm/i915/selftests/intel_uncore.c
+++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c
@@ -71,7 +71,6 @@ static int intel_shadow_table_check(void)
 		{ gen11_shadowed_regs, ARRAY_SIZE(gen11_shadowed_regs) },
 		{ gen12_shadowed_regs, ARRAY_SIZE(gen12_shadowed_regs) },
 		{ dg2_shadowed_regs, ARRAY_SIZE(dg2_shadowed_regs) },
-		{ pvc_shadowed_regs, ARRAY_SIZE(pvc_shadowed_regs) },
 		{ mtl_shadowed_regs, ARRAY_SIZE(mtl_shadowed_regs) },
 		{ xelpmp_shadowed_regs, ARRAY_SIZE(xelpmp_shadowed_regs) },
 	};
@@ -119,8 +118,6 @@ int intel_uncore_mock_selftests(void)
 		{ __gen9_fw_ranges, ARRAY_SIZE(__gen9_fw_ranges), true },
 		{ __gen11_fw_ranges, ARRAY_SIZE(__gen11_fw_ranges), true },
 		{ __gen12_fw_ranges, ARRAY_SIZE(__gen12_fw_ranges), true },
-		{ __xehp_fw_ranges, ARRAY_SIZE(__xehp_fw_ranges), true },
-		{ __pvc_fw_ranges, ARRAY_SIZE(__pvc_fw_ranges), true },
 		{ __mtl_fw_ranges, ARRAY_SIZE(__mtl_fw_ranges), true },
 		{ __xelpmp_fw_ranges, ARRAY_SIZE(__xelpmp_fw_ranges), true },
 	};
diff --git a/drivers/gpu/drm/i915/soc/intel_dram.c b/drivers/gpu/drm/i915/soc/intel_dram.c
index 15492b69f698..e3287f1de774 100644
--- a/drivers/gpu/drm/i915/soc/intel_dram.c
+++ b/drivers/gpu/drm/i915/soc/intel_dram.c
@@ -681,6 +681,8 @@ void intel_dram_detect(struct drm_i915_private *i915)
 	if (ret)
 		return;
 
+	drm_dbg_kms(&i915->drm, "Num qgv points %u\n", dram_info->num_qgv_points);
+
 	drm_dbg_kms(&i915->drm, "DRAM channels: %u\n", dram_info->num_channels);
 
 	drm_dbg_kms(&i915->drm, "Watermark level 0 adjustment needed: %s\n",
diff --git a/drivers/gpu/drm/i915/vlv_sideband.c b/drivers/gpu/drm/i915/vlv_sideband.c
index ffa195560d0d..68291412f4cb 100644
--- a/drivers/gpu/drm/i915/vlv_sideband.c
+++ b/drivers/gpu/drm/i915/vlv_sideband.c
@@ -9,7 +9,6 @@
 #include "vlv_sideband.h"
 
 #include "display/intel_dpio_phy.h"
-#include "display/intel_display_types.h"
 
 /*
  * IOSF sideband, see VLV2_SidebandMsg_HAS.docx and
diff --git a/drivers/gpu/drm/imagination/Makefile b/drivers/gpu/drm/imagination/Makefile
index ec6db8e9b403..9bc6a3884c22 100644
--- a/drivers/gpu/drm/imagination/Makefile
+++ b/drivers/gpu/drm/imagination/Makefile
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only OR MIT
 # Copyright (c) 2023 Imagination Technologies Ltd.
 
-subdir-ccflags-y := -I$(srctree)/$(src)
+subdir-ccflags-y := -I$(src)
 
 powervr-y := \
 	pvr_ccb.o \
diff --git a/drivers/gpu/drm/imagination/pvr_fw_mips.h b/drivers/gpu/drm/imagination/pvr_fw_mips.h
index 408dbe63a90c..a0c5c41c8aa2 100644
--- a/drivers/gpu/drm/imagination/pvr_fw_mips.h
+++ b/drivers/gpu/drm/imagination/pvr_fw_mips.h
@@ -7,13 +7,14 @@
 #include "pvr_rogue_mips.h"
 
 #include <asm/page.h>
+#include <linux/math.h>
 #include <linux/types.h>
 
 /* Forward declaration from pvr_gem.h. */
 struct pvr_gem_object;
 
-#define PVR_MIPS_PT_PAGE_COUNT ((ROGUE_MIPSFW_MAX_NUM_PAGETABLE_PAGES * ROGUE_MIPSFW_PAGE_SIZE_4K) \
-				>> PAGE_SHIFT)
+#define PVR_MIPS_PT_PAGE_COUNT DIV_ROUND_UP(ROGUE_MIPSFW_MAX_NUM_PAGETABLE_PAGES * ROGUE_MIPSFW_PAGE_SIZE_4K, PAGE_SIZE)
+
 /**
  * struct pvr_fw_mips_data - MIPS-specific data
  */
diff --git a/drivers/gpu/drm/imagination/pvr_fw_trace.c b/drivers/gpu/drm/imagination/pvr_fw_trace.c
index 31199e45b72e..73707daa4e52 100644
--- a/drivers/gpu/drm/imagination/pvr_fw_trace.c
+++ b/drivers/gpu/drm/imagination/pvr_fw_trace.c
@@ -12,6 +12,7 @@
 
 #include <linux/build_bug.h>
 #include <linux/dcache.h>
+#include <linux/debugfs.h>
 #include <linux/sysfs.h>
 #include <linux/types.h>
 
diff --git a/drivers/gpu/drm/imagination/pvr_vm_mips.c b/drivers/gpu/drm/imagination/pvr_vm_mips.c
index b7fef3c797e6..94af854547d6 100644
--- a/drivers/gpu/drm/imagination/pvr_vm_mips.c
+++ b/drivers/gpu/drm/imagination/pvr_vm_mips.c
@@ -14,6 +14,7 @@
 #include <linux/err.h>
 #include <linux/slab.h>
 #include <linux/types.h>
+#include <linux/vmalloc.h>
 
 /**
  * pvr_vm_mips_init() - Initialise MIPS FW pagetable
@@ -46,7 +47,7 @@ pvr_vm_mips_init(struct pvr_device *pvr_dev)
 	if (!mips_data)
 		return -ENOMEM;
 
-	for (page_nr = 0; page_nr < ARRAY_SIZE(mips_data->pt_pages); page_nr++) {
+	for (page_nr = 0; page_nr < PVR_MIPS_PT_PAGE_COUNT; page_nr++) {
 		mips_data->pt_pages[page_nr] = alloc_page(GFP_KERNEL | __GFP_ZERO);
 		if (!mips_data->pt_pages[page_nr]) {
 			err = -ENOMEM;
@@ -102,7 +103,7 @@ pvr_vm_mips_fini(struct pvr_device *pvr_dev)
 	int page_nr;
 
 	vunmap(mips_data->pt);
-	for (page_nr = ARRAY_SIZE(mips_data->pt_pages) - 1; page_nr >= 0; page_nr--) {
+	for (page_nr = PVR_MIPS_PT_PAGE_COUNT - 1; page_nr >= 0; page_nr--) {
 		dma_unmap_page(from_pvr_device(pvr_dev)->dev,
 			       mips_data->pt_dma_addr[page_nr], PAGE_SIZE, DMA_TO_DEVICE);
 
diff --git a/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c
index dade8b59feae..704c549750f9 100644
--- a/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c
+++ b/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c
@@ -773,6 +773,13 @@ static const struct drm_plane_helper_funcs ipu_plane_helper_funcs = {
 	.atomic_update = ipu_plane_atomic_update,
 };
 
+static const struct drm_plane_helper_funcs ipu_primary_plane_helper_funcs = {
+	.atomic_check = ipu_plane_atomic_check,
+	.atomic_disable = ipu_plane_atomic_disable,
+	.atomic_update = ipu_plane_atomic_update,
+	.get_scanout_buffer = drm_fb_dma_get_scanout_buffer,
+};
+
 bool ipu_plane_atomic_update_pending(struct drm_plane *plane)
 {
 	struct ipu_plane *ipu_plane = to_ipu_plane(plane);
@@ -916,7 +923,10 @@ struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu,
 	ipu_plane->dma = dma;
 	ipu_plane->dp_flow = dp;
 
-	drm_plane_helper_add(&ipu_plane->base, &ipu_plane_helper_funcs);
+	if (type == DRM_PLANE_TYPE_PRIMARY)
+		drm_plane_helper_add(&ipu_plane->base, &ipu_primary_plane_helper_funcs);
+	else
+		drm_plane_helper_add(&ipu_plane->base, &ipu_plane_helper_funcs);
 
 	if (dp == IPU_DP_FLOW_SYNC_BG || dp == IPU_DP_FLOW_SYNC_FG)
 		ret = drm_plane_create_zpos_property(&ipu_plane->base, zpos, 0,
diff --git a/drivers/gpu/drm/lima/lima_bcast.c b/drivers/gpu/drm/lima/lima_bcast.c
index fbc43f243c54..6d000504e1a4 100644
--- a/drivers/gpu/drm/lima/lima_bcast.c
+++ b/drivers/gpu/drm/lima/lima_bcast.c
@@ -43,6 +43,18 @@ void lima_bcast_suspend(struct lima_ip *ip)
 
 }
 
+int lima_bcast_mask_irq(struct lima_ip *ip)
+{
+	bcast_write(LIMA_BCAST_BROADCAST_MASK, 0);
+	bcast_write(LIMA_BCAST_INTERRUPT_MASK, 0);
+	return 0;
+}
+
+int lima_bcast_reset(struct lima_ip *ip)
+{
+	return lima_bcast_hw_init(ip);
+}
+
 int lima_bcast_init(struct lima_ip *ip)
 {
 	int i;
diff --git a/drivers/gpu/drm/lima/lima_bcast.h b/drivers/gpu/drm/lima/lima_bcast.h
index 465ee587bceb..cd08841e4787 100644
--- a/drivers/gpu/drm/lima/lima_bcast.h
+++ b/drivers/gpu/drm/lima/lima_bcast.h
@@ -13,4 +13,7 @@ void lima_bcast_fini(struct lima_ip *ip);
 
 void lima_bcast_enable(struct lima_device *dev, int num_pp);
 
+int lima_bcast_mask_irq(struct lima_ip *ip);
+int lima_bcast_reset(struct lima_ip *ip);
+
 #endif
diff --git a/drivers/gpu/drm/lima/lima_drv.c b/drivers/gpu/drm/lima/lima_drv.c
index 10fd9154cc46..739c865b556f 100644
--- a/drivers/gpu/drm/lima/lima_drv.c
+++ b/drivers/gpu/drm/lima/lima_drv.c
@@ -371,6 +371,7 @@ static int lima_pdev_probe(struct platform_device *pdev)
 {
 	struct lima_device *ldev;
 	struct drm_device *ddev;
+	const struct lima_compatible *comp;
 	int err;
 
 	err = lima_sched_slab_init();
@@ -384,7 +385,13 @@ static int lima_pdev_probe(struct platform_device *pdev)
 	}
 
 	ldev->dev = &pdev->dev;
-	ldev->id = (enum lima_gpu_id)of_device_get_match_data(&pdev->dev);
+	comp = of_device_get_match_data(&pdev->dev);
+	if (!comp) {
+		err = -ENODEV;
+		goto err_out0;
+	}
+
+	ldev->id = comp->id;
 
 	platform_set_drvdata(pdev, ldev);
 
@@ -459,9 +466,17 @@ static void lima_pdev_remove(struct platform_device *pdev)
 	lima_sched_slab_fini();
 }
 
+static const struct lima_compatible lima_mali400_data = {
+	.id = lima_gpu_mali400,
+};
+
+static const struct lima_compatible lima_mali450_data = {
+	.id = lima_gpu_mali450,
+};
+
 static const struct of_device_id dt_match[] = {
-	{ .compatible = "arm,mali-400", .data = (void *)lima_gpu_mali400 },
-	{ .compatible = "arm,mali-450", .data = (void *)lima_gpu_mali450 },
+	{ .compatible = "arm,mali-400", .data = &lima_mali400_data },
+	{ .compatible = "arm,mali-450", .data = &lima_mali450_data },
 	{}
 };
 MODULE_DEVICE_TABLE(of, dt_match);
diff --git a/drivers/gpu/drm/lima/lima_drv.h b/drivers/gpu/drm/lima/lima_drv.h
index c738d288547b..6706c19b166e 100644
--- a/drivers/gpu/drm/lima/lima_drv.h
+++ b/drivers/gpu/drm/lima/lima_drv.h
@@ -7,6 +7,7 @@
 #include <drm/drm_file.h>
 
 #include "lima_ctx.h"
+#include "lima_device.h"
 
 extern int lima_sched_timeout_ms;
 extern uint lima_heap_init_nr_pages;
@@ -39,6 +40,10 @@ struct lima_submit {
 	struct lima_sched_task *task;
 };
 
+struct lima_compatible {
+	enum lima_gpu_id id;
+};
+
 static inline struct lima_drm_priv *
 to_lima_drm_priv(struct drm_file *file)
 {
diff --git a/drivers/gpu/drm/lima/lima_gem.c b/drivers/gpu/drm/lima/lima_gem.c
index 7ea244d876ca..9bb997dbb4b9 100644
--- a/drivers/gpu/drm/lima/lima_gem.c
+++ b/drivers/gpu/drm/lima/lima_gem.c
@@ -185,7 +185,7 @@ static int lima_gem_pin(struct drm_gem_object *obj)
 	if (bo->heap_size)
 		return -EINVAL;
 
-	return drm_gem_shmem_pin(&bo->base);
+	return drm_gem_shmem_pin_locked(&bo->base);
 }
 
 static int lima_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map)
diff --git a/drivers/gpu/drm/lima/lima_gp.c b/drivers/gpu/drm/lima/lima_gp.c
index 6b354e2fb61d..3282997a0358 100644
--- a/drivers/gpu/drm/lima/lima_gp.c
+++ b/drivers/gpu/drm/lima/lima_gp.c
@@ -233,6 +233,13 @@ static void lima_gp_task_mmu_error(struct lima_sched_pipe *pipe)
 	lima_sched_pipe_task_done(pipe);
 }
 
+static void lima_gp_task_mask_irq(struct lima_sched_pipe *pipe)
+{
+	struct lima_ip *ip = pipe->processor[0];
+
+	gp_write(LIMA_GP_INT_MASK, 0);
+}
+
 static int lima_gp_task_recover(struct lima_sched_pipe *pipe)
 {
 	struct lima_ip *ip = pipe->processor[0];
@@ -338,7 +345,9 @@ int lima_gp_init(struct lima_ip *ip)
 
 void lima_gp_fini(struct lima_ip *ip)
 {
+	struct lima_device *dev = ip->dev;
 
+	devm_free_irq(dev->dev, ip->irq, ip);
 }
 
 int lima_gp_pipe_init(struct lima_device *dev)
@@ -365,6 +374,7 @@ int lima_gp_pipe_init(struct lima_device *dev)
 	pipe->task_error = lima_gp_task_error;
 	pipe->task_mmu_error = lima_gp_task_mmu_error;
 	pipe->task_recover = lima_gp_task_recover;
+	pipe->task_mask_irq = lima_gp_task_mask_irq;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/lima/lima_mmu.c b/drivers/gpu/drm/lima/lima_mmu.c
index e18317c5ca8c..6611e2836bf0 100644
--- a/drivers/gpu/drm/lima/lima_mmu.c
+++ b/drivers/gpu/drm/lima/lima_mmu.c
@@ -118,7 +118,12 @@ int lima_mmu_init(struct lima_ip *ip)
 
 void lima_mmu_fini(struct lima_ip *ip)
 {
+	struct lima_device *dev = ip->dev;
+
+	if (ip->id == lima_ip_ppmmu_bcast)
+		return;
 
+	devm_free_irq(dev->dev, ip->irq, ip);
 }
 
 void lima_mmu_flush_tlb(struct lima_ip *ip)
diff --git a/drivers/gpu/drm/lima/lima_pp.c b/drivers/gpu/drm/lima/lima_pp.c
index d0d2db0ef1ce..eaab4788dff4 100644
--- a/drivers/gpu/drm/lima/lima_pp.c
+++ b/drivers/gpu/drm/lima/lima_pp.c
@@ -286,7 +286,9 @@ int lima_pp_init(struct lima_ip *ip)
 
 void lima_pp_fini(struct lima_ip *ip)
 {
+	struct lima_device *dev = ip->dev;
 
+	devm_free_irq(dev->dev, ip->irq, ip);
 }
 
 int lima_pp_bcast_resume(struct lima_ip *ip)
@@ -319,7 +321,9 @@ int lima_pp_bcast_init(struct lima_ip *ip)
 
 void lima_pp_bcast_fini(struct lima_ip *ip)
 {
+	struct lima_device *dev = ip->dev;
 
+	devm_free_irq(dev->dev, ip->irq, ip);
 }
 
 static int lima_pp_task_validate(struct lima_sched_pipe *pipe,
@@ -429,6 +433,9 @@ static void lima_pp_task_error(struct lima_sched_pipe *pipe)
 
 		lima_pp_hard_reset(ip);
 	}
+
+	if (pipe->bcast_processor)
+		lima_bcast_reset(pipe->bcast_processor);
 }
 
 static void lima_pp_task_mmu_error(struct lima_sched_pipe *pipe)
@@ -437,6 +444,20 @@ static void lima_pp_task_mmu_error(struct lima_sched_pipe *pipe)
 		lima_sched_pipe_task_done(pipe);
 }
 
+static void lima_pp_task_mask_irq(struct lima_sched_pipe *pipe)
+{
+	int i;
+
+	for (i = 0; i < pipe->num_processor; i++) {
+		struct lima_ip *ip = pipe->processor[i];
+
+		pp_write(LIMA_PP_INT_MASK, 0);
+	}
+
+	if (pipe->bcast_processor)
+		lima_bcast_mask_irq(pipe->bcast_processor);
+}
+
 static struct kmem_cache *lima_pp_task_slab;
 static int lima_pp_task_slab_refcnt;
 
@@ -468,6 +489,7 @@ int lima_pp_pipe_init(struct lima_device *dev)
 	pipe->task_fini = lima_pp_task_fini;
 	pipe->task_error = lima_pp_task_error;
 	pipe->task_mmu_error = lima_pp_task_mmu_error;
+	pipe->task_mask_irq = lima_pp_task_mask_irq;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c
index 00b19adfc888..bbf3f8feab94 100644
--- a/drivers/gpu/drm/lima/lima_sched.c
+++ b/drivers/gpu/drm/lima/lima_sched.c
@@ -422,12 +422,21 @@ static enum drm_gpu_sched_stat lima_sched_timedout_job(struct drm_sched_job *job
 	 */
 	for (i = 0; i < pipe->num_processor; i++)
 		synchronize_irq(pipe->processor[i]->irq);
+	if (pipe->bcast_processor)
+		synchronize_irq(pipe->bcast_processor->irq);
 
 	if (dma_fence_is_signaled(task->fence)) {
 		DRM_WARN("%s unexpectedly high interrupt latency\n", lima_ip_name(ip));
 		return DRM_GPU_SCHED_STAT_NOMINAL;
 	}
 
+	/*
+	 * The task might still finish while this timeout handler runs.
+	 * To prevent a race condition on its completion, mask all irqs
+	 * on the running core until the next hard reset completes.
+	 */
+	pipe->task_mask_irq(pipe);
+
 	if (!pipe->error)
 		DRM_ERROR("%s job timeout\n", lima_ip_name(ip));
 
diff --git a/drivers/gpu/drm/lima/lima_sched.h b/drivers/gpu/drm/lima/lima_sched.h
index 6bd4f3b70109..85b23ba901d5 100644
--- a/drivers/gpu/drm/lima/lima_sched.h
+++ b/drivers/gpu/drm/lima/lima_sched.h
@@ -80,6 +80,7 @@ struct lima_sched_pipe {
 	void (*task_error)(struct lima_sched_pipe *pipe);
 	void (*task_mmu_error)(struct lima_sched_pipe *pipe);
 	int (*task_recover)(struct lima_sched_pipe *pipe);
+	void (*task_mask_irq)(struct lima_sched_pipe *pipe);
 
 	struct work_struct recover_work;
 };
diff --git a/drivers/gpu/drm/lima/lima_trace.h b/drivers/gpu/drm/lima/lima_trace.h
index 494b9790b1da..3a349d10304e 100644
--- a/drivers/gpu/drm/lima/lima_trace.h
+++ b/drivers/gpu/drm/lima/lima_trace.h
@@ -24,7 +24,7 @@ DECLARE_EVENT_CLASS(lima_task,
 		__entry->task_id = task->base.id;
 		__entry->context = task->base.s_fence->finished.context;
 		__entry->seqno = task->base.s_fence->finished.seqno;
-		__assign_str(pipe, task->base.sched->name);
+		__assign_str(pipe);
 		),
 
 	TP_printk("task=%llu, context=%u seqno=%u pipe=%s",
diff --git a/drivers/gpu/drm/loongson/lsdc_crtc.c b/drivers/gpu/drm/loongson/lsdc_crtc.c
index 827acab580fa..03958b79f251 100644
--- a/drivers/gpu/drm/loongson/lsdc_crtc.c
+++ b/drivers/gpu/drm/loongson/lsdc_crtc.c
@@ -3,6 +3,7 @@
  * Copyright (C) 2023 Loongson Technology Corporation Limited
  */
 
+#include <linux/debugfs.h>
 #include <linux/delay.h>
 
 #include <drm/drm_atomic.h>
diff --git a/drivers/gpu/drm/loongson/lsdc_gem.c b/drivers/gpu/drm/loongson/lsdc_gem.c
index 04293df2f0de..a720d8f53209 100644
--- a/drivers/gpu/drm/loongson/lsdc_gem.c
+++ b/drivers/gpu/drm/loongson/lsdc_gem.c
@@ -19,33 +19,24 @@ static int lsdc_gem_prime_pin(struct drm_gem_object *obj)
 	struct lsdc_bo *lbo = gem_to_lsdc_bo(obj);
 	int ret;
 
-	ret = lsdc_bo_reserve(lbo);
-	if (unlikely(ret))
-		return ret;
+	dma_resv_assert_held(obj->resv);
 
 	ret = lsdc_bo_pin(lbo, LSDC_GEM_DOMAIN_GTT, NULL);
 	if (likely(ret == 0))
 		lbo->sharing_count++;
 
-	lsdc_bo_unreserve(lbo);
-
 	return ret;
 }
 
 static void lsdc_gem_prime_unpin(struct drm_gem_object *obj)
 {
 	struct lsdc_bo *lbo = gem_to_lsdc_bo(obj);
-	int ret;
 
-	ret = lsdc_bo_reserve(lbo);
-	if (unlikely(ret))
-		return;
+	dma_resv_assert_held(obj->resv);
 
 	lsdc_bo_unpin(lbo);
 	if (lbo->sharing_count)
 		lbo->sharing_count--;
-
-	lsdc_bo_unreserve(lbo);
 }
 
 static struct sg_table *lsdc_gem_prime_get_sg_table(struct drm_gem_object *obj)
diff --git a/drivers/gpu/drm/mediatek/Kconfig b/drivers/gpu/drm/mediatek/Kconfig
index 76cab28e010c..96cbe020f493 100644
--- a/drivers/gpu/drm/mediatek/Kconfig
+++ b/drivers/gpu/drm/mediatek/Kconfig
@@ -26,7 +26,7 @@ config DRM_MEDIATEK_DP
 	select PHY_MTK_DP
 	select DRM_DISPLAY_HELPER
 	select DRM_DISPLAY_DP_HELPER
-	select DRM_DP_AUX_BUS
+	select DRM_DISPLAY_DP_AUX_BUS
 	help
 	  DRM/KMS Display Port driver for MediaTek SoCs.
 
diff --git a/drivers/gpu/drm/mediatek/Makefile b/drivers/gpu/drm/mediatek/Makefile
index 5e4436403b8d..32a2ed6c0cfe 100644
--- a/drivers/gpu/drm/mediatek/Makefile
+++ b/drivers/gpu/drm/mediatek/Makefile
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 
-mediatek-drm-y := mtk_disp_aal.o \
+mediatek-drm-y := mtk_crtc.o \
+		  mtk_ddp_comp.o \
+		  mtk_disp_aal.o \
 		  mtk_disp_ccorr.o \
 		  mtk_disp_color.o \
 		  mtk_disp_gamma.o \
@@ -8,16 +10,14 @@ mediatek-drm-y := mtk_disp_aal.o \
 		  mtk_disp_ovl.o \
 		  mtk_disp_ovl_adaptor.o \
 		  mtk_disp_rdma.o \
-		  mtk_drm_crtc.o \
-		  mtk_drm_ddp_comp.o \
 		  mtk_drm_drv.o \
-		  mtk_drm_gem.o \
-		  mtk_drm_plane.o \
 		  mtk_dsi.o \
 		  mtk_dpi.o \
 		  mtk_ethdr.o \
+		  mtk_gem.o \
 		  mtk_mdp_rdma.o \
-		  mtk_padding.o
+		  mtk_padding.o \
+		  mtk_plane.o
 
 obj-$(CONFIG_DRM_MEDIATEK) += mediatek-drm.o
 
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c b/drivers/gpu/drm/mediatek/mtk_crtc.c
index a04499c4f9ca..6f34f573e127 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
+++ b/drivers/gpu/drm/mediatek/mtk_crtc.c
@@ -19,14 +19,14 @@
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_drm_drv.h"
-#include "mtk_drm_crtc.h"
-#include "mtk_drm_ddp_comp.h"
-#include "mtk_drm_gem.h"
-#include "mtk_drm_plane.h"
+#include "mtk_gem.h"
+#include "mtk_plane.h"
 
 /*
- * struct mtk_drm_crtc - MediaTek specific crtc structure.
+ * struct mtk_crtc - MediaTek specific crtc structure.
  * @base: crtc object.
  * @enabled: records whether crtc_enable succeeded
  * @planes: array of 4 drm_plane structures, one for each overlay plane
@@ -38,7 +38,7 @@
  *
  * TODO: Needs update: this header is missing a bunch of member descriptions.
  */
-struct mtk_drm_crtc {
+struct mtk_crtc {
 	struct drm_crtc			base;
 	bool				enabled;
 
@@ -80,9 +80,9 @@ struct mtk_crtc_state {
 	unsigned int			pending_vrefresh;
 };
 
-static inline struct mtk_drm_crtc *to_mtk_crtc(struct drm_crtc *c)
+static inline struct mtk_crtc *to_mtk_crtc(struct drm_crtc *c)
 {
-	return container_of(c, struct mtk_drm_crtc, base);
+	return container_of(c, struct mtk_crtc, base);
 }
 
 static inline struct mtk_crtc_state *to_mtk_crtc_state(struct drm_crtc_state *s)
@@ -90,7 +90,7 @@ static inline struct mtk_crtc_state *to_mtk_crtc_state(struct drm_crtc_state *s)
 	return container_of(s, struct mtk_crtc_state, base);
 }
 
-static void mtk_drm_crtc_finish_page_flip(struct mtk_drm_crtc *mtk_crtc)
+static void mtk_crtc_finish_page_flip(struct mtk_crtc *mtk_crtc)
 {
 	struct drm_crtc *crtc = &mtk_crtc->base;
 	unsigned long flags;
@@ -104,11 +104,11 @@ static void mtk_drm_crtc_finish_page_flip(struct mtk_drm_crtc *mtk_crtc)
 	}
 }
 
-static void mtk_drm_finish_page_flip(struct mtk_drm_crtc *mtk_crtc)
+static void mtk_drm_finish_page_flip(struct mtk_crtc *mtk_crtc)
 {
 	drm_crtc_handle_vblank(&mtk_crtc->base);
 	if (!mtk_crtc->config_updating && mtk_crtc->pending_needs_vblank) {
-		mtk_drm_crtc_finish_page_flip(mtk_crtc);
+		mtk_crtc_finish_page_flip(mtk_crtc);
 		mtk_crtc->pending_needs_vblank = false;
 	}
 }
@@ -151,9 +151,9 @@ static void mtk_drm_cmdq_pkt_destroy(struct cmdq_pkt *pkt)
 }
 #endif
 
-static void mtk_drm_crtc_destroy(struct drm_crtc *crtc)
+static void mtk_crtc_destroy(struct drm_crtc *crtc)
 {
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	int i;
 
 	mtk_mutex_put(mtk_crtc->mutex);
@@ -176,7 +176,7 @@ static void mtk_drm_crtc_destroy(struct drm_crtc *crtc)
 	drm_crtc_cleanup(crtc);
 }
 
-static void mtk_drm_crtc_reset(struct drm_crtc *crtc)
+static void mtk_crtc_reset(struct drm_crtc *crtc)
 {
 	struct mtk_crtc_state *state;
 
@@ -191,7 +191,7 @@ static void mtk_drm_crtc_reset(struct drm_crtc *crtc)
 		__drm_atomic_helper_crtc_reset(crtc, &state->base);
 }
 
-static struct drm_crtc_state *mtk_drm_crtc_duplicate_state(struct drm_crtc *crtc)
+static struct drm_crtc_state *mtk_crtc_duplicate_state(struct drm_crtc *crtc)
 {
 	struct mtk_crtc_state *state;
 
@@ -208,18 +208,17 @@ static struct drm_crtc_state *mtk_drm_crtc_duplicate_state(struct drm_crtc *crtc
 	return &state->base;
 }
 
-static void mtk_drm_crtc_destroy_state(struct drm_crtc *crtc,
-				       struct drm_crtc_state *state)
+static void mtk_crtc_destroy_state(struct drm_crtc *crtc,
+				   struct drm_crtc_state *state)
 {
 	__drm_atomic_helper_crtc_destroy_state(state);
 	kfree(to_mtk_crtc_state(state));
 }
 
 static enum drm_mode_status
-mtk_drm_crtc_mode_valid(struct drm_crtc *crtc,
-			const struct drm_display_mode *mode)
+mtk_crtc_mode_valid(struct drm_crtc *crtc, const struct drm_display_mode *mode)
 {
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	enum drm_mode_status status = MODE_OK;
 	int i;
 
@@ -231,15 +230,15 @@ mtk_drm_crtc_mode_valid(struct drm_crtc *crtc,
 	return status;
 }
 
-static bool mtk_drm_crtc_mode_fixup(struct drm_crtc *crtc,
-				    const struct drm_display_mode *mode,
-				    struct drm_display_mode *adjusted_mode)
+static bool mtk_crtc_mode_fixup(struct drm_crtc *crtc,
+				const struct drm_display_mode *mode,
+				struct drm_display_mode *adjusted_mode)
 {
 	/* Nothing to do here, but this callback is mandatory. */
 	return true;
 }
 
-static void mtk_drm_crtc_mode_set_nofb(struct drm_crtc *crtc)
+static void mtk_crtc_mode_set_nofb(struct drm_crtc *crtc)
 {
 	struct mtk_crtc_state *state = to_mtk_crtc_state(crtc->state);
 
@@ -250,7 +249,7 @@ static void mtk_drm_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	state->pending_config = true;
 }
 
-static int mtk_crtc_ddp_clk_enable(struct mtk_drm_crtc *mtk_crtc)
+static int mtk_crtc_ddp_clk_enable(struct mtk_crtc *mtk_crtc)
 {
 	int ret;
 	int i;
@@ -270,7 +269,7 @@ err:
 	return ret;
 }
 
-static void mtk_crtc_ddp_clk_disable(struct mtk_drm_crtc *mtk_crtc)
+static void mtk_crtc_ddp_clk_disable(struct mtk_crtc *mtk_crtc)
 {
 	int i;
 
@@ -279,11 +278,11 @@ static void mtk_crtc_ddp_clk_disable(struct mtk_drm_crtc *mtk_crtc)
 }
 
 static
-struct mtk_ddp_comp *mtk_drm_ddp_comp_for_plane(struct drm_crtc *crtc,
-						struct drm_plane *plane,
-						unsigned int *local_layer)
+struct mtk_ddp_comp *mtk_ddp_comp_for_plane(struct drm_crtc *crtc,
+					    struct drm_plane *plane,
+					    unsigned int *local_layer)
 {
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	struct mtk_ddp_comp *comp;
 	int i, count = 0;
 	unsigned int local_index = plane - mtk_crtc->planes;
@@ -306,7 +305,7 @@ static void ddp_cmdq_cb(struct mbox_client *cl, void *mssg)
 {
 	struct cmdq_cb_data *data = mssg;
 	struct cmdq_client *cmdq_cl = container_of(cl, struct cmdq_client, client);
-	struct mtk_drm_crtc *mtk_crtc = container_of(cmdq_cl, struct mtk_drm_crtc, cmdq_client);
+	struct mtk_crtc *mtk_crtc = container_of(cmdq_cl, struct mtk_crtc, cmdq_client);
 	struct mtk_crtc_state *state;
 	unsigned int i;
 
@@ -346,7 +345,7 @@ static void ddp_cmdq_cb(struct mbox_client *cl, void *mssg)
 }
 #endif
 
-static int mtk_crtc_ddp_hw_init(struct mtk_drm_crtc *mtk_crtc)
+static int mtk_crtc_ddp_hw_init(struct mtk_crtc *mtk_crtc)
 {
 	struct drm_crtc *crtc = &mtk_crtc->base;
 	struct drm_connector *connector;
@@ -431,7 +430,7 @@ static int mtk_crtc_ddp_hw_init(struct mtk_drm_crtc *mtk_crtc)
 
 		/* should not enable layer before crtc enabled */
 		plane_state->pending.enable = false;
-		comp = mtk_drm_ddp_comp_for_plane(crtc, plane, &local_layer);
+		comp = mtk_ddp_comp_for_plane(crtc, plane, &local_layer);
 		if (comp)
 			mtk_ddp_comp_layer_config(comp, local_layer,
 						  plane_state, NULL);
@@ -446,7 +445,7 @@ err_pm_runtime_put:
 	return ret;
 }
 
-static void mtk_crtc_ddp_hw_fini(struct mtk_drm_crtc *mtk_crtc)
+static void mtk_crtc_ddp_hw_fini(struct mtk_crtc *mtk_crtc)
 {
 	struct drm_device *drm = mtk_crtc->base.dev;
 	struct drm_crtc *crtc = &mtk_crtc->base;
@@ -491,7 +490,7 @@ static void mtk_crtc_ddp_hw_fini(struct mtk_drm_crtc *mtk_crtc)
 static void mtk_crtc_ddp_config(struct drm_crtc *crtc,
 				struct cmdq_pkt *cmdq_handle)
 {
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	struct mtk_crtc_state *state = to_mtk_crtc_state(mtk_crtc->base.state);
 	struct mtk_ddp_comp *comp = mtk_crtc->ddp_comp[0];
 	unsigned int i;
@@ -522,8 +521,7 @@ static void mtk_crtc_ddp_config(struct drm_crtc *crtc,
 			if (!plane_state->pending.config)
 				continue;
 
-			comp = mtk_drm_ddp_comp_for_plane(crtc, plane,
-							  &local_layer);
+			comp = mtk_ddp_comp_for_plane(crtc, plane, &local_layer);
 
 			if (comp)
 				mtk_ddp_comp_layer_config(comp, local_layer,
@@ -547,8 +545,7 @@ static void mtk_crtc_ddp_config(struct drm_crtc *crtc,
 			if (!plane_state->pending.async_config)
 				continue;
 
-			comp = mtk_drm_ddp_comp_for_plane(crtc, plane,
-							  &local_layer);
+			comp = mtk_ddp_comp_for_plane(crtc, plane, &local_layer);
 
 			if (comp)
 				mtk_ddp_comp_layer_config(comp, local_layer,
@@ -563,8 +560,7 @@ static void mtk_crtc_ddp_config(struct drm_crtc *crtc,
 	}
 }
 
-static void mtk_drm_crtc_update_config(struct mtk_drm_crtc *mtk_crtc,
-				       bool needs_vblank)
+static void mtk_crtc_update_config(struct mtk_crtc *mtk_crtc, bool needs_vblank)
 {
 #if IS_REACHABLE(CONFIG_MTK_CMDQ)
 	struct cmdq_pkt *cmdq_handle = &mtk_crtc->cmdq_handle;
@@ -636,7 +632,7 @@ static void mtk_drm_crtc_update_config(struct mtk_drm_crtc *mtk_crtc,
 static void mtk_crtc_ddp_irq(void *data)
 {
 	struct drm_crtc *crtc = data;
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	struct mtk_drm_private *priv = crtc->dev->dev_private;
 
 #if IS_REACHABLE(CONFIG_MTK_CMDQ)
@@ -652,9 +648,9 @@ static void mtk_crtc_ddp_irq(void *data)
 	mtk_drm_finish_page_flip(mtk_crtc);
 }
 
-static int mtk_drm_crtc_enable_vblank(struct drm_crtc *crtc)
+static int mtk_crtc_enable_vblank(struct drm_crtc *crtc)
 {
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	struct mtk_ddp_comp *comp = mtk_crtc->ddp_comp[0];
 
 	mtk_ddp_comp_enable_vblank(comp);
@@ -662,22 +658,22 @@ static int mtk_drm_crtc_enable_vblank(struct drm_crtc *crtc)
 	return 0;
 }
 
-static void mtk_drm_crtc_disable_vblank(struct drm_crtc *crtc)
+static void mtk_crtc_disable_vblank(struct drm_crtc *crtc)
 {
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	struct mtk_ddp_comp *comp = mtk_crtc->ddp_comp[0];
 
 	mtk_ddp_comp_disable_vblank(comp);
 }
 
-static void mtk_drm_crtc_update_output(struct drm_crtc *crtc,
-				       struct drm_atomic_state *state)
+static void mtk_crtc_update_output(struct drm_crtc *crtc,
+				   struct drm_atomic_state *state)
 {
 	int crtc_index = drm_crtc_index(crtc);
 	int i;
 	struct device *dev;
 	struct drm_crtc_state *crtc_state = state->crtcs[crtc_index].new_state;
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	struct mtk_drm_private *priv;
 	unsigned int encoder_mask = crtc_state->encoder_mask;
 
@@ -707,33 +703,33 @@ static void mtk_drm_crtc_update_output(struct drm_crtc *crtc,
 	}
 }
 
-int mtk_drm_crtc_plane_check(struct drm_crtc *crtc, struct drm_plane *plane,
-			     struct mtk_plane_state *state)
+int mtk_crtc_plane_check(struct drm_crtc *crtc, struct drm_plane *plane,
+			 struct mtk_plane_state *state)
 {
 	unsigned int local_layer;
 	struct mtk_ddp_comp *comp;
 
-	comp = mtk_drm_ddp_comp_for_plane(crtc, plane, &local_layer);
+	comp = mtk_ddp_comp_for_plane(crtc, plane, &local_layer);
 	if (comp)
 		return mtk_ddp_comp_layer_check(comp, local_layer, state);
 	return 0;
 }
 
-void mtk_drm_crtc_async_update(struct drm_crtc *crtc, struct drm_plane *plane,
-			       struct drm_atomic_state *state)
+void mtk_crtc_async_update(struct drm_crtc *crtc, struct drm_plane *plane,
+			   struct drm_atomic_state *state)
 {
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 
 	if (!mtk_crtc->enabled)
 		return;
 
-	mtk_drm_crtc_update_config(mtk_crtc, false);
+	mtk_crtc_update_config(mtk_crtc, false);
 }
 
-static void mtk_drm_crtc_atomic_enable(struct drm_crtc *crtc,
-				       struct drm_atomic_state *state)
+static void mtk_crtc_atomic_enable(struct drm_crtc *crtc,
+				   struct drm_atomic_state *state)
 {
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	struct mtk_ddp_comp *comp = mtk_crtc->ddp_comp[0];
 	int ret;
 
@@ -745,7 +741,7 @@ static void mtk_drm_crtc_atomic_enable(struct drm_crtc *crtc,
 		return;
 	}
 
-	mtk_drm_crtc_update_output(crtc, state);
+	mtk_crtc_update_output(crtc, state);
 
 	ret = mtk_crtc_ddp_hw_init(mtk_crtc);
 	if (ret) {
@@ -757,10 +753,10 @@ static void mtk_drm_crtc_atomic_enable(struct drm_crtc *crtc,
 	mtk_crtc->enabled = true;
 }
 
-static void mtk_drm_crtc_atomic_disable(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
+static void mtk_crtc_atomic_disable(struct drm_crtc *crtc,
+				    struct drm_atomic_state *state)
 {
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	struct mtk_ddp_comp *comp = mtk_crtc->ddp_comp[0];
 	int i;
 
@@ -779,7 +775,7 @@ static void mtk_drm_crtc_atomic_disable(struct drm_crtc *crtc,
 	}
 	mtk_crtc->pending_planes = true;
 
-	mtk_drm_crtc_update_config(mtk_crtc, false);
+	mtk_crtc_update_config(mtk_crtc, false);
 #if IS_REACHABLE(CONFIG_MTK_CMDQ)
 	/* Wait for planes to be disabled by cmdq */
 	if (mtk_crtc->cmdq_client.chan)
@@ -797,13 +793,13 @@ static void mtk_drm_crtc_atomic_disable(struct drm_crtc *crtc,
 	mtk_crtc->enabled = false;
 }
 
-static void mtk_drm_crtc_atomic_begin(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+static void mtk_crtc_atomic_begin(struct drm_crtc *crtc,
+				  struct drm_atomic_state *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
 	struct mtk_crtc_state *mtk_crtc_state = to_mtk_crtc_state(crtc_state);
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	unsigned long flags;
 
 	if (mtk_crtc->event && mtk_crtc_state->base.event)
@@ -821,10 +817,10 @@ static void mtk_drm_crtc_atomic_begin(struct drm_crtc *crtc,
 	}
 }
 
-static void mtk_drm_crtc_atomic_flush(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+static void mtk_crtc_atomic_flush(struct drm_crtc *crtc,
+				  struct drm_atomic_state *state)
 {
-	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
+	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	int i;
 
 	if (crtc->state->color_mgmt_changed)
@@ -832,33 +828,32 @@ static void mtk_drm_crtc_atomic_flush(struct drm_crtc *crtc,
 			mtk_ddp_gamma_set(mtk_crtc->ddp_comp[i], crtc->state);
 			mtk_ddp_ctm_set(mtk_crtc->ddp_comp[i], crtc->state);
 		}
-	mtk_drm_crtc_update_config(mtk_crtc, !!mtk_crtc->event);
+	mtk_crtc_update_config(mtk_crtc, !!mtk_crtc->event);
 }
 
 static const struct drm_crtc_funcs mtk_crtc_funcs = {
 	.set_config		= drm_atomic_helper_set_config,
 	.page_flip		= drm_atomic_helper_page_flip,
-	.destroy		= mtk_drm_crtc_destroy,
-	.reset			= mtk_drm_crtc_reset,
-	.atomic_duplicate_state	= mtk_drm_crtc_duplicate_state,
-	.atomic_destroy_state	= mtk_drm_crtc_destroy_state,
-	.enable_vblank		= mtk_drm_crtc_enable_vblank,
-	.disable_vblank		= mtk_drm_crtc_disable_vblank,
+	.destroy		= mtk_crtc_destroy,
+	.reset			= mtk_crtc_reset,
+	.atomic_duplicate_state	= mtk_crtc_duplicate_state,
+	.atomic_destroy_state	= mtk_crtc_destroy_state,
+	.enable_vblank		= mtk_crtc_enable_vblank,
+	.disable_vblank		= mtk_crtc_disable_vblank,
 };
 
 static const struct drm_crtc_helper_funcs mtk_crtc_helper_funcs = {
-	.mode_fixup	= mtk_drm_crtc_mode_fixup,
-	.mode_set_nofb	= mtk_drm_crtc_mode_set_nofb,
-	.mode_valid	= mtk_drm_crtc_mode_valid,
-	.atomic_begin	= mtk_drm_crtc_atomic_begin,
-	.atomic_flush	= mtk_drm_crtc_atomic_flush,
-	.atomic_enable	= mtk_drm_crtc_atomic_enable,
-	.atomic_disable	= mtk_drm_crtc_atomic_disable,
+	.mode_fixup	= mtk_crtc_mode_fixup,
+	.mode_set_nofb	= mtk_crtc_mode_set_nofb,
+	.mode_valid	= mtk_crtc_mode_valid,
+	.atomic_begin	= mtk_crtc_atomic_begin,
+	.atomic_flush	= mtk_crtc_atomic_flush,
+	.atomic_enable	= mtk_crtc_atomic_enable,
+	.atomic_disable	= mtk_crtc_atomic_disable,
 };
 
-static int mtk_drm_crtc_init(struct drm_device *drm,
-			     struct mtk_drm_crtc *mtk_crtc,
-			     unsigned int pipe)
+static int mtk_crtc_init(struct drm_device *drm, struct mtk_crtc *mtk_crtc,
+			 unsigned int pipe)
 {
 	struct drm_plane *primary = NULL;
 	struct drm_plane *cursor = NULL;
@@ -885,8 +880,7 @@ err_cleanup_crtc:
 	return ret;
 }
 
-static int mtk_drm_crtc_num_comp_planes(struct mtk_drm_crtc *mtk_crtc,
-					int comp_idx)
+static int mtk_crtc_num_comp_planes(struct mtk_crtc *mtk_crtc, int comp_idx)
 {
 	struct mtk_ddp_comp *comp;
 
@@ -904,8 +898,8 @@ static int mtk_drm_crtc_num_comp_planes(struct mtk_drm_crtc *mtk_crtc,
 }
 
 static inline
-enum drm_plane_type mtk_drm_crtc_plane_type(unsigned int plane_idx,
-					    unsigned int num_planes)
+enum drm_plane_type mtk_crtc_plane_type(unsigned int plane_idx,
+					unsigned int num_planes)
 {
 	if (plane_idx == 0)
 		return DRM_PLANE_TYPE_PRIMARY;
@@ -916,11 +910,11 @@ enum drm_plane_type mtk_drm_crtc_plane_type(unsigned int plane_idx,
 
 }
 
-static int mtk_drm_crtc_init_comp_planes(struct drm_device *drm_dev,
-					 struct mtk_drm_crtc *mtk_crtc,
-					 int comp_idx, int pipe)
+static int mtk_crtc_init_comp_planes(struct drm_device *drm_dev,
+				     struct mtk_crtc *mtk_crtc,
+				     int comp_idx, int pipe)
 {
-	int num_planes = mtk_drm_crtc_num_comp_planes(mtk_crtc, comp_idx);
+	int num_planes = mtk_crtc_num_comp_planes(mtk_crtc, comp_idx);
 	struct mtk_ddp_comp *comp = mtk_crtc->ddp_comp[comp_idx];
 	int i, ret;
 
@@ -928,8 +922,7 @@ static int mtk_drm_crtc_init_comp_planes(struct drm_device *drm_dev,
 		ret = mtk_plane_init(drm_dev,
 				&mtk_crtc->planes[mtk_crtc->layer_nr],
 				BIT(pipe),
-				mtk_drm_crtc_plane_type(mtk_crtc->layer_nr,
-							num_planes),
+				mtk_crtc_plane_type(mtk_crtc->layer_nr, num_planes),
 				mtk_ddp_comp_supported_rotations(comp),
 				mtk_ddp_comp_get_formats(comp),
 				mtk_ddp_comp_get_num_formats(comp));
@@ -941,9 +934,9 @@ static int mtk_drm_crtc_init_comp_planes(struct drm_device *drm_dev,
 	return 0;
 }
 
-struct device *mtk_drm_crtc_dma_dev_get(struct drm_crtc *crtc)
+struct device *mtk_crtc_dma_dev_get(struct drm_crtc *crtc)
 {
-	struct mtk_drm_crtc *mtk_crtc = NULL;
+	struct mtk_crtc *mtk_crtc = NULL;
 
 	if (!crtc)
 		return NULL;
@@ -955,14 +948,14 @@ struct device *mtk_drm_crtc_dma_dev_get(struct drm_crtc *crtc)
 	return mtk_crtc->dma_dev;
 }
 
-int mtk_drm_crtc_create(struct drm_device *drm_dev,
-			const unsigned int *path, unsigned int path_len,
-			int priv_data_index, const struct mtk_drm_route *conn_routes,
-			unsigned int num_conn_routes)
+int mtk_crtc_create(struct drm_device *drm_dev, const unsigned int *path,
+		    unsigned int path_len, int priv_data_index,
+		    const struct mtk_drm_route *conn_routes,
+		    unsigned int num_conn_routes)
 {
 	struct mtk_drm_private *priv = drm_dev->dev_private;
 	struct device *dev = drm_dev->dev;
-	struct mtk_drm_crtc *mtk_crtc;
+	struct mtk_crtc *mtk_crtc;
 	unsigned int num_comp_planes = 0;
 	int ret;
 	int i;
@@ -1009,10 +1002,10 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 
 	mtk_crtc->mmsys_dev = priv->mmsys_dev;
 	mtk_crtc->ddp_comp_nr = path_len;
-	mtk_crtc->ddp_comp = devm_kmalloc_array(dev,
-						mtk_crtc->ddp_comp_nr + (conn_routes ? 1 : 0),
-						sizeof(*mtk_crtc->ddp_comp),
-						GFP_KERNEL);
+	mtk_crtc->ddp_comp = devm_kcalloc(dev,
+					  mtk_crtc->ddp_comp_nr + (conn_routes ? 1 : 0),
+					  sizeof(*mtk_crtc->ddp_comp),
+					  GFP_KERNEL);
 	if (!mtk_crtc->ddp_comp)
 		return -ENOMEM;
 
@@ -1047,7 +1040,7 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 	}
 
 	for (i = 0; i < mtk_crtc->ddp_comp_nr; i++)
-		num_comp_planes += mtk_drm_crtc_num_comp_planes(mtk_crtc, i);
+		num_comp_planes += mtk_crtc_num_comp_planes(mtk_crtc, i);
 
 	mtk_crtc->planes = devm_kcalloc(dev, num_comp_planes,
 					sizeof(struct drm_plane), GFP_KERNEL);
@@ -1055,8 +1048,7 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 		return -ENOMEM;
 
 	for (i = 0; i < mtk_crtc->ddp_comp_nr; i++) {
-		ret = mtk_drm_crtc_init_comp_planes(drm_dev, mtk_crtc, i,
-						    crtc_i);
+		ret = mtk_crtc_init_comp_planes(drm_dev, mtk_crtc, i, crtc_i);
 		if (ret)
 			return ret;
 	}
@@ -1068,7 +1060,7 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 	 */
 	mtk_crtc->dma_dev = mtk_ddp_comp_dma_dev_get(&priv->ddp_comp[path[0]]);
 
-	ret = mtk_drm_crtc_init(drm_dev, mtk_crtc, crtc_i);
+	ret = mtk_crtc_init(drm_dev, mtk_crtc, crtc_i);
 	if (ret < 0)
 		return ret;
 
@@ -1138,7 +1130,7 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 		mtk_crtc->num_conn_routes = num_conn_routes;
 		mtk_crtc->conn_routes = conn_routes;
 
-		/* increase ddp_comp_nr at the end of mtk_drm_crtc_create */
+		/* increase ddp_comp_nr at the end of mtk_crtc_create */
 		mtk_crtc->ddp_comp_nr++;
 	}
 
diff --git a/drivers/gpu/drm/mediatek/mtk_crtc.h b/drivers/gpu/drm/mediatek/mtk_crtc.h
new file mode 100644
index 000000000000..388e900b6f4d
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_crtc.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2015 MediaTek Inc.
+ */
+
+#ifndef MTK_CRTC_H
+#define MTK_CRTC_H
+
+#include <drm/drm_crtc.h>
+#include "mtk_ddp_comp.h"
+#include "mtk_drm_drv.h"
+#include "mtk_plane.h"
+
+#define MTK_MAX_BPC	10
+#define MTK_MIN_BPC	3
+
+void mtk_crtc_commit(struct drm_crtc *crtc);
+int mtk_crtc_create(struct drm_device *drm_dev, const unsigned int *path,
+		    unsigned int path_len, int priv_data_index,
+		    const struct mtk_drm_route *conn_routes,
+		    unsigned int num_conn_routes);
+int mtk_crtc_plane_check(struct drm_crtc *crtc, struct drm_plane *plane,
+			 struct mtk_plane_state *state);
+void mtk_crtc_async_update(struct drm_crtc *crtc, struct drm_plane *plane,
+			   struct drm_atomic_state *plane_state);
+struct device *mtk_crtc_dma_dev_get(struct drm_crtc *crtc);
+
+#endif /* MTK_CRTC_H */
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c b/drivers/gpu/drm/mediatek/mtk_ddp_comp.c
index a515e96cfefc..17b036411292 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
+++ b/drivers/gpu/drm/mediatek/mtk_ddp_comp.c
@@ -14,11 +14,11 @@
 #include <linux/soc/mediatek/mtk-cmdq.h>
 #include <drm/drm_print.h>
 
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_disp_drv.h"
 #include "mtk_drm_drv.h"
-#include "mtk_drm_plane.h"
-#include "mtk_drm_ddp_comp.h"
-#include "mtk_drm_crtc.h"
+#include "mtk_plane.h"
 
 
 #define DISP_REG_DITHER_EN			0x0000
@@ -497,10 +497,10 @@ static const struct mtk_ddp_comp_match mtk_ddp_matches[DDP_COMPONENT_DRM_ID_MAX]
 	[DDP_COMPONENT_WDMA1]		= { MTK_DISP_WDMA,		1, NULL },
 };
 
-static bool mtk_drm_find_comp_in_ddp(struct device *dev,
-				     const unsigned int *path,
-				     unsigned int path_len,
-				     struct mtk_ddp_comp *ddp_comp)
+static bool mtk_ddp_comp_find(struct device *dev,
+			      const unsigned int *path,
+			      unsigned int path_len,
+			      struct mtk_ddp_comp *ddp_comp)
 {
 	unsigned int i;
 
@@ -514,10 +514,10 @@ static bool mtk_drm_find_comp_in_ddp(struct device *dev,
 	return false;
 }
 
-static unsigned int mtk_drm_find_comp_in_ddp_conn_path(struct device *dev,
-						       const struct mtk_drm_route *routes,
-						       unsigned int num_routes,
-						       struct mtk_ddp_comp *ddp_comp)
+static unsigned int mtk_ddp_comp_find_in_route(struct device *dev,
+					       const struct mtk_drm_route *routes,
+					       unsigned int num_routes,
+					       struct mtk_ddp_comp *ddp_comp)
 {
 	int ret;
 	unsigned int i;
@@ -554,26 +554,31 @@ int mtk_ddp_comp_get_id(struct device_node *node,
 	return -EINVAL;
 }
 
-unsigned int mtk_drm_find_possible_crtc_by_comp(struct drm_device *drm,
-						struct device *dev)
+unsigned int mtk_find_possible_crtcs(struct drm_device *drm, struct device *dev)
 {
 	struct mtk_drm_private *private = drm->dev_private;
 	unsigned int ret = 0;
 
-	if (mtk_drm_find_comp_in_ddp(dev, private->data->main_path, private->data->main_len,
-				     private->ddp_comp))
+	if (mtk_ddp_comp_find(dev,
+			      private->data->main_path,
+			      private->data->main_len,
+			      private->ddp_comp))
 		ret = BIT(0);
-	else if (mtk_drm_find_comp_in_ddp(dev, private->data->ext_path,
-					  private->data->ext_len, private->ddp_comp))
+	else if (mtk_ddp_comp_find(dev,
+				   private->data->ext_path,
+				   private->data->ext_len,
+				   private->ddp_comp))
 		ret = BIT(1);
-	else if (mtk_drm_find_comp_in_ddp(dev, private->data->third_path,
-					  private->data->third_len, private->ddp_comp))
+	else if (mtk_ddp_comp_find(dev,
+				   private->data->third_path,
+				   private->data->third_len,
+				   private->ddp_comp))
 		ret = BIT(2);
 	else
-		ret = mtk_drm_find_comp_in_ddp_conn_path(dev,
-							 private->data->conn_routes,
-							 private->data->num_conn_routes,
-							 private->ddp_comp);
+		ret = mtk_ddp_comp_find_in_route(dev,
+						 private->data->conn_routes,
+						 private->data->num_conn_routes,
+						 private->ddp_comp);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h b/drivers/gpu/drm/mediatek/mtk_ddp_comp.h
index 93d79a1366e9..26236691ce4c 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h
+++ b/drivers/gpu/drm/mediatek/mtk_ddp_comp.h
@@ -3,8 +3,8 @@
  * Copyright (c) 2015 MediaTek Inc.
  */
 
-#ifndef MTK_DRM_DDP_COMP_H
-#define MTK_DRM_DDP_COMP_H
+#ifndef MTK_DDP_COMP_H
+#define MTK_DDP_COMP_H
 
 #include <linux/io.h>
 #include <linux/pm_runtime.h>
@@ -326,8 +326,7 @@ static inline void mtk_ddp_comp_encoder_index_set(struct mtk_ddp_comp *comp)
 
 int mtk_ddp_comp_get_id(struct device_node *node,
 			enum mtk_ddp_comp_type comp_type);
-unsigned int mtk_drm_find_possible_crtc_by_comp(struct drm_device *drm,
-						struct device *dev);
+unsigned int mtk_find_possible_crtcs(struct drm_device *drm, struct device *dev);
 int mtk_ddp_comp_init(struct device_node *comp_node, struct mtk_ddp_comp *comp,
 		      unsigned int comp_id);
 enum mtk_ddp_comp_type mtk_ddp_comp_get_type(unsigned int comp_id);
@@ -340,4 +339,4 @@ void mtk_ddp_write_relaxed(struct cmdq_pkt *cmdq_pkt, unsigned int value,
 void mtk_ddp_write_mask(struct cmdq_pkt *cmdq_pkt, unsigned int value,
 			struct cmdq_client_reg *cmdq_reg, void __iomem *regs,
 			unsigned int offset, unsigned int mask);
-#endif /* MTK_DRM_DDP_COMP_H */
+#endif /* MTK_DDP_COMP_H */
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_aal.c b/drivers/gpu/drm/mediatek/mtk_disp_aal.c
index 40fe403086c3..3ce8f32b06d5 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_aal.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_aal.c
@@ -11,9 +11,9 @@
 #include <linux/platform_device.h>
 #include <linux/soc/mediatek/mtk-cmdq.h>
 
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_disp_drv.h"
-#include "mtk_drm_crtc.h"
-#include "mtk_drm_ddp_comp.h"
 #include "mtk_drm_drv.h"
 
 #define DISP_AAL_EN				0x0000
@@ -223,7 +223,6 @@ struct platform_driver mtk_disp_aal_driver = {
 	.remove_new	= mtk_disp_aal_remove,
 	.driver		= {
 		.name	= "mediatek-disp-aal",
-		.owner	= THIS_MODULE,
 		.of_match_table = mtk_disp_aal_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ccorr.c b/drivers/gpu/drm/mediatek/mtk_disp_ccorr.c
index 465cddce0d32..df35e90dd25f 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_ccorr.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_ccorr.c
@@ -10,9 +10,9 @@
 #include <linux/platform_device.h>
 #include <linux/soc/mediatek/mtk-cmdq.h>
 
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_disp_drv.h"
-#include "mtk_drm_crtc.h"
-#include "mtk_drm_ddp_comp.h"
 #include "mtk_drm_drv.h"
 
 #define DISP_CCORR_EN				0x0000
@@ -214,7 +214,6 @@ struct platform_driver mtk_disp_ccorr_driver = {
 	.remove_new	= mtk_disp_ccorr_remove,
 	.driver		= {
 		.name	= "mediatek-disp-ccorr",
-		.owner	= THIS_MODULE,
 		.of_match_table = mtk_disp_ccorr_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_color.c b/drivers/gpu/drm/mediatek/mtk_disp_color.c
index 78ea99f1444f..7f0085be5671 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_color.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_color.c
@@ -10,9 +10,9 @@
 #include <linux/platform_device.h>
 #include <linux/soc/mediatek/mtk-cmdq.h>
 
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_disp_drv.h"
-#include "mtk_drm_crtc.h"
-#include "mtk_drm_ddp_comp.h"
 #include "mtk_drm_drv.h"
 
 #define DISP_COLOR_CFG_MAIN			0x0400
@@ -164,7 +164,6 @@ struct platform_driver mtk_disp_color_driver = {
 	.remove_new	= mtk_disp_color_remove,
 	.driver		= {
 		.name	= "mediatek-disp-color",
-		.owner	= THIS_MODULE,
 		.of_match_table = mtk_disp_color_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_drv.h b/drivers/gpu/drm/mediatek/mtk_disp_drv.h
index 90e64467ea8f..082ac18fe04a 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_drv.h
+++ b/drivers/gpu/drm/mediatek/mtk_disp_drv.h
@@ -9,8 +9,8 @@
 #include <linux/soc/mediatek/mtk-cmdq.h>
 #include <linux/soc/mediatek/mtk-mmsys.h>
 #include <linux/soc/mediatek/mtk-mutex.h>
-#include "mtk_drm_plane.h"
 #include "mtk_mdp_rdma.h"
+#include "mtk_plane.h"
 
 int mtk_aal_clk_enable(struct device *dev);
 void mtk_aal_clk_disable(struct device *dev);
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_gamma.c b/drivers/gpu/drm/mediatek/mtk_disp_gamma.c
index c1bc8b00d938..ca8d1f3aca03 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_gamma.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_gamma.c
@@ -11,9 +11,9 @@
 #include <linux/platform_device.h>
 #include <linux/soc/mediatek/mtk-cmdq.h>
 
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_disp_drv.h"
-#include "mtk_drm_crtc.h"
-#include "mtk_drm_ddp_comp.h"
 #include "mtk_drm_drv.h"
 
 #define DISP_GAMMA_EN				0x0000
@@ -334,7 +334,6 @@ struct platform_driver mtk_disp_gamma_driver = {
 	.remove_new	= mtk_disp_gamma_remove,
 	.driver		= {
 		.name	= "mediatek-disp-gamma",
-		.owner	= THIS_MODULE,
 		.of_match_table = mtk_disp_gamma_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_merge.c b/drivers/gpu/drm/mediatek/mtk_disp_merge.c
index 32a29924bd54..77c057e0e671 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_merge.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_merge.c
@@ -10,7 +10,7 @@
 #include <linux/reset.h>
 #include <linux/soc/mediatek/mtk-cmdq.h>
 
-#include "mtk_drm_ddp_comp.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_drm_drv.h"
 #include "mtk_disp_drv.h"
 
@@ -376,7 +376,6 @@ struct platform_driver mtk_disp_merge_driver = {
 	.remove_new = mtk_disp_merge_remove,
 	.driver = {
 		.name = "mediatek-disp-merge",
-		.owner = THIS_MODULE,
 		.of_match_table = mtk_disp_merge_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
index 2bffe4245466..b552a02d7eae 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
@@ -15,9 +15,9 @@
 #include <linux/pm_runtime.h>
 #include <linux/soc/mediatek/mtk-cmdq.h>
 
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_disp_drv.h"
-#include "mtk_drm_crtc.h"
-#include "mtk_drm_ddp_comp.h"
 #include "mtk_drm_drv.h"
 
 #define DISP_REG_OVL_INTEN			0x0004
@@ -659,7 +659,6 @@ struct platform_driver mtk_disp_ovl_driver = {
 	.remove_new	= mtk_disp_ovl_remove,
 	.driver		= {
 		.name	= "mediatek-disp-ovl",
-		.owner	= THIS_MODULE,
 		.of_match_table = mtk_disp_ovl_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c b/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c
index 034d31824d4d..02dd7dcdfedb 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c
@@ -17,9 +17,9 @@
 #include <linux/soc/mediatek/mtk-mmsys.h>
 #include <linux/soc/mediatek/mtk-mutex.h>
 
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_disp_drv.h"
-#include "mtk_drm_crtc.h"
-#include "mtk_drm_ddp_comp.h"
 #include "mtk_drm_drv.h"
 #include "mtk_ethdr.h"
 
@@ -629,6 +629,5 @@ struct platform_driver mtk_disp_ovl_adaptor_driver = {
 	.remove_new	= mtk_disp_ovl_adaptor_remove,
 	.driver		= {
 		.name	= "mediatek-disp-ovl-adaptor",
-		.owner	= THIS_MODULE,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
index faa907f2f443..7b1a6e631200 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
@@ -13,9 +13,9 @@
 #include <linux/pm_runtime.h>
 #include <linux/soc/mediatek/mtk-cmdq.h>
 
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_disp_drv.h"
-#include "mtk_drm_crtc.h"
-#include "mtk_drm_ddp_comp.h"
 #include "mtk_drm_drv.h"
 
 #define DISP_REG_RDMA_INT_ENABLE		0x0000
@@ -428,7 +428,6 @@ struct platform_driver mtk_disp_rdma_driver = {
 	.remove_new	= mtk_disp_rdma_remove,
 	.driver		= {
 		.name	= "mediatek-disp-rdma",
-		.owner	= THIS_MODULE,
 		.of_match_table = mtk_disp_rdma_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_dp.c b/drivers/gpu/drm/mediatek/mtk_dp.c
index 0ba72102636a..536366956447 100644
--- a/drivers/gpu/drm/mediatek/mtk_dp.c
+++ b/drivers/gpu/drm/mediatek/mtk_dp.c
@@ -2104,7 +2104,7 @@ static ssize_t mtk_dp_aux_transfer(struct drm_dp_aux *mtk_aux,
 
 	if (mtk_dp->bridge.type != DRM_MODE_CONNECTOR_eDP &&
 	    !mtk_dp->train_info.cable_plugged_in) {
-		ret = -EAGAIN;
+		ret = -EIO;
 		goto err;
 	}
 
diff --git a/drivers/gpu/drm/mediatek/mtk_dpi.c b/drivers/gpu/drm/mediatek/mtk_dpi.c
index beb7d9d08e97..bfe8653005db 100644
--- a/drivers/gpu/drm/mediatek/mtk_dpi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dpi.c
@@ -26,9 +26,9 @@
 #include <drm/drm_of.h>
 #include <drm/drm_simple_kms_helper.h>
 
+#include "mtk_ddp_comp.h"
 #include "mtk_disp_drv.h"
 #include "mtk_dpi_regs.h"
-#include "mtk_drm_ddp_comp.h"
 #include "mtk_drm_drv.h"
 
 enum mtk_dpi_out_bit_num {
@@ -805,7 +805,7 @@ static int mtk_dpi_bind(struct device *dev, struct device *master, void *data)
 		return ret;
 	}
 
-	dpi->encoder.possible_crtcs = mtk_drm_find_possible_crtc_by_comp(drm_dev, dpi->dev);
+	dpi->encoder.possible_crtcs = mtk_find_possible_crtcs(drm_dev, dpi->dev);
 
 	ret = drm_bridge_attach(&dpi->encoder, &dpi->bridge, NULL,
 				DRM_BRIDGE_ATTACH_NO_CONNECTOR);
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.h b/drivers/gpu/drm/mediatek/mtk_drm_crtc.h
deleted file mode 100644
index 1f988ff1bf9f..000000000000
--- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2015 MediaTek Inc.
- */
-
-#ifndef MTK_DRM_CRTC_H
-#define MTK_DRM_CRTC_H
-
-#include <drm/drm_crtc.h>
-#include "mtk_drm_ddp_comp.h"
-#include "mtk_drm_drv.h"
-#include "mtk_drm_plane.h"
-
-#define MTK_MAX_BPC	10
-#define MTK_MIN_BPC	3
-
-void mtk_drm_crtc_commit(struct drm_crtc *crtc);
-int mtk_drm_crtc_create(struct drm_device *drm_dev,
-			const unsigned int *path,
-			unsigned int path_len,
-			int priv_data_index,
-			const struct mtk_drm_route *conn_routes,
-			unsigned int num_conn_routes);
-int mtk_drm_crtc_plane_check(struct drm_crtc *crtc, struct drm_plane *plane,
-			     struct mtk_plane_state *state);
-void mtk_drm_crtc_async_update(struct drm_crtc *crtc, struct drm_plane *plane,
-			       struct drm_atomic_state *plane_state);
-struct device *mtk_drm_crtc_dma_dev_get(struct drm_crtc *crtc);
-
-#endif /* MTK_DRM_CRTC_H */
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
index 74832c213092..b5f605751b0a 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
@@ -24,10 +24,10 @@
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
-#include "mtk_drm_crtc.h"
-#include "mtk_drm_ddp_comp.h"
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_drm_drv.h"
-#include "mtk_drm_gem.h"
+#include "mtk_gem.h"
 
 #define DRIVER_NAME "mediatek"
 #define DRIVER_DESC "Mediatek SoC DRM"
@@ -494,24 +494,24 @@ static int mtk_drm_kms_init(struct drm_device *drm)
 			priv_n = private->all_drm_private[j];
 
 			if (i == CRTC_MAIN && priv_n->data->main_len) {
-				ret = mtk_drm_crtc_create(drm, priv_n->data->main_path,
-							  priv_n->data->main_len, j,
-							  priv_n->data->conn_routes,
-							  priv_n->data->num_conn_routes);
+				ret = mtk_crtc_create(drm, priv_n->data->main_path,
+						      priv_n->data->main_len, j,
+						      priv_n->data->conn_routes,
+						      priv_n->data->num_conn_routes);
 				if (ret)
 					goto err_component_unbind;
 
 				continue;
 			} else if (i == CRTC_EXT && priv_n->data->ext_len) {
-				ret = mtk_drm_crtc_create(drm, priv_n->data->ext_path,
-							  priv_n->data->ext_len, j, NULL, 0);
+				ret = mtk_crtc_create(drm, priv_n->data->ext_path,
+						      priv_n->data->ext_len, j, NULL, 0);
 				if (ret)
 					goto err_component_unbind;
 
 				continue;
 			} else if (i == CRTC_THIRD && priv_n->data->third_len) {
-				ret = mtk_drm_crtc_create(drm, priv_n->data->third_path,
-							  priv_n->data->third_len, j, NULL, 0);
+				ret = mtk_crtc_create(drm, priv_n->data->third_path,
+						      priv_n->data->third_len, j, NULL, 0);
 				if (ret)
 					goto err_component_unbind;
 
@@ -523,7 +523,7 @@ static int mtk_drm_kms_init(struct drm_device *drm)
 	/* Use OVL device for all DMA memory allocations */
 	crtc = drm_crtc_from_index(drm, 0);
 	if (crtc)
-		dma_dev = mtk_drm_crtc_dma_dev_get(crtc);
+		dma_dev = mtk_crtc_dma_dev_get(crtc);
 	if (!dma_dev) {
 		ret = -ENODEV;
 		dev_err(drm->dev, "Need at least one OVL device\n");
@@ -576,8 +576,8 @@ DEFINE_DRM_GEM_FOPS(mtk_drm_fops);
  * We need to override this because the device used to import the memory is
  * not dev->dev, as drm_gem_prime_import() expects.
  */
-static struct drm_gem_object *mtk_drm_gem_prime_import(struct drm_device *dev,
-						       struct dma_buf *dma_buf)
+static struct drm_gem_object *mtk_gem_prime_import(struct drm_device *dev,
+						   struct dma_buf *dma_buf)
 {
 	struct mtk_drm_private *private = dev->dev_private;
 
@@ -587,9 +587,9 @@ static struct drm_gem_object *mtk_drm_gem_prime_import(struct drm_device *dev,
 static const struct drm_driver mtk_drm_driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC,
 
-	.dumb_create = mtk_drm_gem_dumb_create,
+	.dumb_create = mtk_gem_dumb_create,
 
-	.gem_prime_import = mtk_drm_gem_prime_import,
+	.gem_prime_import = mtk_gem_prime_import,
 	.gem_prime_import_sg_table = mtk_gem_prime_import_sg_table,
 	.fops = &mtk_drm_fops,
 
@@ -709,6 +709,8 @@ static const struct of_device_id mtk_ddp_comp_dt_ids[] = {
 	  .data = (void *)MTK_DISP_GAMMA, },
 	{ .compatible = "mediatek,mt8183-disp-gamma",
 	  .data = (void *)MTK_DISP_GAMMA, },
+	{ .compatible = "mediatek,mt8195-disp-gamma",
+	  .data = (void *)MTK_DISP_GAMMA, },
 	{ .compatible = "mediatek,mt8195-disp-merge",
 	  .data = (void *)MTK_DISP_MERGE },
 	{ .compatible = "mediatek,mt2701-disp-mutex",
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.h b/drivers/gpu/drm/mediatek/mtk_drm_drv.h
index 33fadb08dc1c..78d698ede1bf 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.h
@@ -7,13 +7,13 @@
 #define MTK_DRM_DRV_H
 
 #include <linux/io.h>
-#include "mtk_drm_ddp_comp.h"
+#include "mtk_ddp_comp.h"
 
 #define MAX_CONNECTOR	2
 #define DDP_COMPONENT_DRM_OVL_ADAPTOR (DDP_COMPONENT_ID_MAX + 1)
 #define DDP_COMPONENT_DRM_ID_MAX (DDP_COMPONENT_DRM_OVL_ADAPTOR + 1)
 
-enum mtk_drm_crtc_path {
+enum mtk_crtc_path {
 	CRTC_MAIN,
 	CRTC_EXT,
 	CRTC_THIRD,
diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c
index 9501f4019199..c255559cc56e 100644
--- a/drivers/gpu/drm/mediatek/mtk_dsi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dsi.c
@@ -28,8 +28,8 @@
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
 
+#include "mtk_ddp_comp.h"
 #include "mtk_disp_drv.h"
-#include "mtk_drm_ddp_comp.h"
 #include "mtk_drm_drv.h"
 
 #define DSI_START		0x00
@@ -242,22 +242,23 @@ static void mtk_dsi_phy_timconfig(struct mtk_dsi *dsi)
 	u32 data_rate_mhz = DIV_ROUND_UP(dsi->data_rate, HZ_PER_MHZ);
 	struct mtk_phy_timing *timing = &dsi->phy_timing;
 
-	timing->lpx = (60 * data_rate_mhz / (8 * 1000)) + 1;
-	timing->da_hs_prepare = (80 * data_rate_mhz + 4 * 1000) / 8000;
-	timing->da_hs_zero = (170 * data_rate_mhz + 10 * 1000) / 8000 + 1 -
+	timing->lpx = (80 * data_rate_mhz / (8 * 1000)) + 1;
+	timing->da_hs_prepare = (59 * data_rate_mhz + 4 * 1000) / 8000 + 1;
+	timing->da_hs_zero = (163 * data_rate_mhz + 11 * 1000) / 8000 + 1 -
 			     timing->da_hs_prepare;
-	timing->da_hs_trail = timing->da_hs_prepare + 1;
+	timing->da_hs_trail = (78 * data_rate_mhz + 7 * 1000) / 8000 + 1;
 
-	timing->ta_go = 4 * timing->lpx - 2;
-	timing->ta_sure = timing->lpx + 2;
-	timing->ta_get = 4 * timing->lpx;
-	timing->da_hs_exit = 2 * timing->lpx + 1;
+	timing->ta_go = 4 * timing->lpx;
+	timing->ta_sure = 3 * timing->lpx / 2;
+	timing->ta_get = 5 * timing->lpx;
+	timing->da_hs_exit = (118 * data_rate_mhz / (8 * 1000)) + 1;
 
-	timing->clk_hs_prepare = 70 * data_rate_mhz / (8 * 1000);
-	timing->clk_hs_post = timing->clk_hs_prepare + 8;
-	timing->clk_hs_trail = timing->clk_hs_prepare;
-	timing->clk_hs_zero = timing->clk_hs_trail * 4;
-	timing->clk_hs_exit = 2 * timing->clk_hs_trail;
+	timing->clk_hs_prepare = (57 * data_rate_mhz / (8 * 1000)) + 1;
+	timing->clk_hs_post = (65 * data_rate_mhz + 53 * 1000) / 8000 + 1;
+	timing->clk_hs_trail = (78 * data_rate_mhz + 7 * 1000) / 8000 + 1;
+	timing->clk_hs_zero = (330 * data_rate_mhz / (8 * 1000)) + 1 -
+			      timing->clk_hs_prepare;
+	timing->clk_hs_exit = (118 * data_rate_mhz / (8 * 1000)) + 1;
 
 	timcon0 = FIELD_PREP(LPX, timing->lpx) |
 		  FIELD_PREP(HS_PREP, timing->da_hs_prepare) |
@@ -662,7 +663,7 @@ static void mtk_dsi_poweroff(struct mtk_dsi *dsi)
 
 	/*
 	 * mtk_dsi_stop() and mtk_dsi_start() is asymmetric, since
-	 * mtk_dsi_stop() should be called after mtk_drm_crtc_atomic_disable(),
+	 * mtk_dsi_stop() should be called after mtk_crtc_atomic_disable(),
 	 * which needs irq for vblank, and mtk_dsi_stop() will disable irq.
 	 * mtk_dsi_start() needs to be called in mtk_output_dsi_enable(),
 	 * after dsi is fully set.
@@ -836,7 +837,7 @@ static int mtk_dsi_encoder_init(struct drm_device *drm, struct mtk_dsi *dsi)
 		return ret;
 	}
 
-	dsi->encoder.possible_crtcs = mtk_drm_find_possible_crtc_by_comp(drm, dsi->host.dev);
+	dsi->encoder.possible_crtcs = mtk_find_possible_crtcs(drm, dsi->host.dev);
 
 	ret = drm_bridge_attach(&dsi->encoder, &dsi->bridge, NULL,
 				DRM_BRIDGE_ATTACH_NO_CONNECTOR);
diff --git a/drivers/gpu/drm/mediatek/mtk_ethdr.c b/drivers/gpu/drm/mediatek/mtk_ethdr.c
index 6a5d0c345aab..156c6ff547e8 100644
--- a/drivers/gpu/drm/mediatek/mtk_ethdr.c
+++ b/drivers/gpu/drm/mediatek/mtk_ethdr.c
@@ -14,8 +14,8 @@
 #include <linux/soc/mediatek/mtk-cmdq.h>
 #include <linux/soc/mediatek/mtk-mmsys.h>
 
-#include "mtk_drm_crtc.h"
-#include "mtk_drm_ddp_comp.h"
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_drm_drv.h"
 #include "mtk_ethdr.h"
 
@@ -363,7 +363,6 @@ struct platform_driver mtk_ethdr_driver = {
 	.remove_new	= mtk_ethdr_remove,
 	.driver		= {
 		.name	= "mediatek-disp-ethdr",
-		.owner	= THIS_MODULE,
 		.of_match_table = mtk_ethdr_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_gem.c b/drivers/gpu/drm/mediatek/mtk_gem.c
index 4f2e3feabc0f..a172456d1d7b 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_gem.c
+++ b/drivers/gpu/drm/mediatek/mtk_gem.c
@@ -4,6 +4,7 @@
  */
 
 #include <linux/dma-buf.h>
+#include <linux/vmalloc.h>
 
 #include <drm/drm.h>
 #include <drm/drm_device.h>
@@ -12,37 +13,40 @@
 #include <drm/drm_prime.h>
 
 #include "mtk_drm_drv.h"
-#include "mtk_drm_gem.h"
+#include "mtk_gem.h"
 
-static int mtk_drm_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
+static int mtk_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
 
 static const struct vm_operations_struct vm_ops = {
 	.open = drm_gem_vm_open,
 	.close = drm_gem_vm_close,
 };
 
-static const struct drm_gem_object_funcs mtk_drm_gem_object_funcs = {
-	.free = mtk_drm_gem_free_object,
+static const struct drm_gem_object_funcs mtk_gem_object_funcs = {
+	.free = mtk_gem_free_object,
 	.get_sg_table = mtk_gem_prime_get_sg_table,
-	.vmap = mtk_drm_gem_prime_vmap,
-	.vunmap = mtk_drm_gem_prime_vunmap,
-	.mmap = mtk_drm_gem_object_mmap,
+	.vmap = mtk_gem_prime_vmap,
+	.vunmap = mtk_gem_prime_vunmap,
+	.mmap = mtk_gem_object_mmap,
 	.vm_ops = &vm_ops,
 };
 
-static struct mtk_drm_gem_obj *mtk_drm_gem_init(struct drm_device *dev,
-						unsigned long size)
+static struct mtk_gem_obj *mtk_gem_init(struct drm_device *dev,
+					unsigned long size)
 {
-	struct mtk_drm_gem_obj *mtk_gem_obj;
+	struct mtk_gem_obj *mtk_gem_obj;
 	int ret;
 
 	size = round_up(size, PAGE_SIZE);
 
+	if (size == 0)
+		return ERR_PTR(-EINVAL);
+
 	mtk_gem_obj = kzalloc(sizeof(*mtk_gem_obj), GFP_KERNEL);
 	if (!mtk_gem_obj)
 		return ERR_PTR(-ENOMEM);
 
-	mtk_gem_obj->base.funcs = &mtk_drm_gem_object_funcs;
+	mtk_gem_obj->base.funcs = &mtk_gem_object_funcs;
 
 	ret = drm_gem_object_init(dev, &mtk_gem_obj->base, size);
 	if (ret < 0) {
@@ -54,15 +58,15 @@ static struct mtk_drm_gem_obj *mtk_drm_gem_init(struct drm_device *dev,
 	return mtk_gem_obj;
 }
 
-struct mtk_drm_gem_obj *mtk_drm_gem_create(struct drm_device *dev,
-					   size_t size, bool alloc_kmap)
+struct mtk_gem_obj *mtk_gem_create(struct drm_device *dev,
+				   size_t size, bool alloc_kmap)
 {
 	struct mtk_drm_private *priv = dev->dev_private;
-	struct mtk_drm_gem_obj *mtk_gem;
+	struct mtk_gem_obj *mtk_gem;
 	struct drm_gem_object *obj;
 	int ret;
 
-	mtk_gem = mtk_drm_gem_init(dev, size);
+	mtk_gem = mtk_gem_init(dev, size);
 	if (IS_ERR(mtk_gem))
 		return ERR_CAST(mtk_gem);
 
@@ -97,9 +101,9 @@ err_gem_free:
 	return ERR_PTR(ret);
 }
 
-void mtk_drm_gem_free_object(struct drm_gem_object *obj)
+void mtk_gem_free_object(struct drm_gem_object *obj)
 {
-	struct mtk_drm_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
+	struct mtk_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
 	struct mtk_drm_private *priv = obj->dev->dev_private;
 
 	if (mtk_gem->sg)
@@ -114,10 +118,10 @@ void mtk_drm_gem_free_object(struct drm_gem_object *obj)
 	kfree(mtk_gem);
 }
 
-int mtk_drm_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
-			    struct drm_mode_create_dumb *args)
+int mtk_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
+			struct drm_mode_create_dumb *args)
 {
-	struct mtk_drm_gem_obj *mtk_gem;
+	struct mtk_gem_obj *mtk_gem;
 	int ret;
 
 	args->pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
@@ -130,7 +134,7 @@ int mtk_drm_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
 	args->size = args->pitch;
 	args->size *= args->height;
 
-	mtk_gem = mtk_drm_gem_create(dev, args->size, false);
+	mtk_gem = mtk_gem_create(dev, args->size, false);
 	if (IS_ERR(mtk_gem))
 		return PTR_ERR(mtk_gem);
 
@@ -148,16 +152,16 @@ int mtk_drm_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
 	return 0;
 
 err_handle_create:
-	mtk_drm_gem_free_object(&mtk_gem->base);
+	mtk_gem_free_object(&mtk_gem->base);
 	return ret;
 }
 
-static int mtk_drm_gem_object_mmap(struct drm_gem_object *obj,
-				   struct vm_area_struct *vma)
+static int mtk_gem_object_mmap(struct drm_gem_object *obj,
+			       struct vm_area_struct *vma)
 
 {
 	int ret;
-	struct mtk_drm_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
+	struct mtk_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
 	struct mtk_drm_private *priv = obj->dev->dev_private;
 
 	/*
@@ -188,7 +192,7 @@ static int mtk_drm_gem_object_mmap(struct drm_gem_object *obj,
  */
 struct sg_table *mtk_gem_prime_get_sg_table(struct drm_gem_object *obj)
 {
-	struct mtk_drm_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
+	struct mtk_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
 	struct mtk_drm_private *priv = obj->dev->dev_private;
 	struct sg_table *sgt;
 	int ret;
@@ -212,7 +216,7 @@ struct sg_table *mtk_gem_prime_get_sg_table(struct drm_gem_object *obj)
 struct drm_gem_object *mtk_gem_prime_import_sg_table(struct drm_device *dev,
 			struct dma_buf_attachment *attach, struct sg_table *sg)
 {
-	struct mtk_drm_gem_obj *mtk_gem;
+	struct mtk_gem_obj *mtk_gem;
 
 	/* check if the entries in the sg_table are contiguous */
 	if (drm_prime_get_contiguous_size(sg) < attach->dmabuf->size) {
@@ -220,7 +224,7 @@ struct drm_gem_object *mtk_gem_prime_import_sg_table(struct drm_device *dev,
 		return ERR_PTR(-EINVAL);
 	}
 
-	mtk_gem = mtk_drm_gem_init(dev, attach->dmabuf->size);
+	mtk_gem = mtk_gem_init(dev, attach->dmabuf->size);
 	if (IS_ERR(mtk_gem))
 		return ERR_CAST(mtk_gem);
 
@@ -230,9 +234,9 @@ struct drm_gem_object *mtk_gem_prime_import_sg_table(struct drm_device *dev,
 	return &mtk_gem->base;
 }
 
-int mtk_drm_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map)
+int mtk_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map)
 {
-	struct mtk_drm_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
+	struct mtk_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
 	struct sg_table *sgt = NULL;
 	unsigned int npages;
 
@@ -270,10 +274,9 @@ out:
 	return 0;
 }
 
-void mtk_drm_gem_prime_vunmap(struct drm_gem_object *obj,
-			      struct iosys_map *map)
+void mtk_gem_prime_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
 {
-	struct mtk_drm_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
+	struct mtk_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
 	void *vaddr = map->vaddr;
 
 	if (!mtk_gem->pages)
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_gem.h b/drivers/gpu/drm/mediatek/mtk_gem.h
index 78f23b07a02e..66e5f154f698 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_gem.h
+++ b/drivers/gpu/drm/mediatek/mtk_gem.h
@@ -3,8 +3,8 @@
  * Copyright (c) 2015 MediaTek Inc.
  */
 
-#ifndef _MTK_DRM_GEM_H_
-#define _MTK_DRM_GEM_H_
+#ifndef _MTK_GEM_H_
+#define _MTK_GEM_H_
 
 #include <drm/drm_gem.h>
 
@@ -22,7 +22,7 @@
  * P.S. this object would be transferred to user as kms_bo.handle so
  *	user can access the buffer through kms_bo.handle.
  */
-struct mtk_drm_gem_obj {
+struct mtk_gem_obj {
 	struct drm_gem_object	base;
 	void			*cookie;
 	void			*kvaddr;
@@ -32,18 +32,17 @@ struct mtk_drm_gem_obj {
 	struct page		**pages;
 };
 
-#define to_mtk_gem_obj(x)	container_of(x, struct mtk_drm_gem_obj, base)
+#define to_mtk_gem_obj(x) container_of(x, struct mtk_gem_obj, base)
 
-void mtk_drm_gem_free_object(struct drm_gem_object *gem);
-struct mtk_drm_gem_obj *mtk_drm_gem_create(struct drm_device *dev, size_t size,
-					   bool alloc_kmap);
-int mtk_drm_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
-			    struct drm_mode_create_dumb *args);
+void mtk_gem_free_object(struct drm_gem_object *gem);
+struct mtk_gem_obj *mtk_gem_create(struct drm_device *dev, size_t size,
+				   bool alloc_kmap);
+int mtk_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
+			struct drm_mode_create_dumb *args);
 struct sg_table *mtk_gem_prime_get_sg_table(struct drm_gem_object *obj);
 struct drm_gem_object *mtk_gem_prime_import_sg_table(struct drm_device *dev,
 			struct dma_buf_attachment *attach, struct sg_table *sg);
-int mtk_drm_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map);
-void mtk_drm_gem_prime_vunmap(struct drm_gem_object *obj,
-			      struct iosys_map *map);
+int mtk_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map);
+void mtk_gem_prime_vunmap(struct drm_gem_object *obj, struct iosys_map *map);
 
 #endif
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c
index c6bdc565e4a9..6e1cca97a654 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c
@@ -1695,7 +1695,7 @@ static int mtk_hdmi_register_audio_driver(struct device *dev)
 	return 0;
 }
 
-static int mtk_drm_hdmi_probe(struct platform_device *pdev)
+static int mtk_hdmi_probe(struct platform_device *pdev)
 {
 	struct mtk_hdmi *hdmi;
 	struct device *dev = &pdev->dev;
@@ -1754,7 +1754,7 @@ err_bridge_remove:
 	return ret;
 }
 
-static void mtk_drm_hdmi_remove(struct platform_device *pdev)
+static void mtk_hdmi_remove(struct platform_device *pdev)
 {
 	struct mtk_hdmi *hdmi = platform_get_drvdata(pdev);
 
@@ -1798,7 +1798,7 @@ static const struct mtk_hdmi_conf mtk_hdmi_conf_mt8167 = {
 	.cea_modes_only = true,
 };
 
-static const struct of_device_id mtk_drm_hdmi_of_ids[] = {
+static const struct of_device_id mtk_hdmi_of_ids[] = {
 	{ .compatible = "mediatek,mt2701-hdmi",
 	  .data = &mtk_hdmi_conf_mt2701,
 	},
@@ -1809,14 +1809,14 @@ static const struct of_device_id mtk_drm_hdmi_of_ids[] = {
 	},
 	{}
 };
-MODULE_DEVICE_TABLE(of, mtk_drm_hdmi_of_ids);
+MODULE_DEVICE_TABLE(of, mtk_hdmi_of_ids);
 
 static struct platform_driver mtk_hdmi_driver = {
-	.probe = mtk_drm_hdmi_probe,
-	.remove_new = mtk_drm_hdmi_remove,
+	.probe = mtk_hdmi_probe,
+	.remove_new = mtk_hdmi_remove,
 	.driver = {
 		.name = "mediatek-drm-hdmi",
-		.of_match_table = mtk_drm_hdmi_of_ids,
+		.of_match_table = mtk_hdmi_of_ids,
 		.pm = &mtk_hdmi_pm_ops,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c b/drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c
index 54e46e440e0f..52d55861f954 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c
@@ -284,8 +284,7 @@ static int mtk_hdmi_ddc_probe(struct platform_device *pdev)
 		return PTR_ERR(ddc->clk);
 	}
 
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	ddc->regs = devm_ioremap_resource(&pdev->dev, mem);
+	ddc->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &mem);
 	if (IS_ERR(ddc->regs))
 		return PTR_ERR(ddc->regs);
 
diff --git a/drivers/gpu/drm/mediatek/mtk_mdp_rdma.c b/drivers/gpu/drm/mediatek/mtk_mdp_rdma.c
index ee9ce9b6d078..925cbb7471ec 100644
--- a/drivers/gpu/drm/mediatek/mtk_mdp_rdma.c
+++ b/drivers/gpu/drm/mediatek/mtk_mdp_rdma.c
@@ -346,7 +346,6 @@ struct platform_driver mtk_mdp_rdma_driver = {
 	.remove_new = mtk_mdp_rdma_remove,
 	.driver = {
 		.name = "mediatek-mdp-rdma",
-		.owner = THIS_MODULE,
 		.of_match_table = mtk_mdp_rdma_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_padding.c b/drivers/gpu/drm/mediatek/mtk_padding.c
index 0d6451c149b6..85bc6768b6bc 100644
--- a/drivers/gpu/drm/mediatek/mtk_padding.c
+++ b/drivers/gpu/drm/mediatek/mtk_padding.c
@@ -11,9 +11,9 @@
 #include <linux/pm_runtime.h>
 #include <linux/soc/mediatek/mtk-cmdq.h>
 
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_disp_drv.h"
-#include "mtk_drm_crtc.h"
-#include "mtk_drm_ddp_comp.h"
 
 #define PADDING_CONTROL_REG	0x00
 #define PADDING_BYPASS			BIT(0)
@@ -154,7 +154,6 @@ struct platform_driver mtk_padding_driver = {
 	.remove		= mtk_padding_remove,
 	.driver		= {
 		.name	= "mediatek-disp-padding",
-		.owner	= THIS_MODULE,
 		.of_match_table = mtk_padding_driver_dt_match,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_plane.c b/drivers/gpu/drm/mediatek/mtk_plane.c
index ddc9355b06d5..4625deb21d40 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_plane.c
+++ b/drivers/gpu/drm/mediatek/mtk_plane.c
@@ -13,11 +13,11 @@
 #include <drm/drm_gem_atomic_helper.h>
 #include <linux/align.h>
 
-#include "mtk_drm_crtc.h"
-#include "mtk_drm_ddp_comp.h"
+#include "mtk_crtc.h"
+#include "mtk_ddp_comp.h"
 #include "mtk_drm_drv.h"
-#include "mtk_drm_gem.h"
-#include "mtk_drm_plane.h"
+#include "mtk_gem.h"
+#include "mtk_plane.h"
 
 static const u64 modifiers[] = {
 	DRM_FORMAT_MOD_LINEAR,
@@ -93,8 +93,8 @@ static bool mtk_plane_format_mod_supported(struct drm_plane *plane,
 	return true;
 }
 
-static void mtk_drm_plane_destroy_state(struct drm_plane *plane,
-					struct drm_plane_state *state)
+static void mtk_plane_destroy_state(struct drm_plane *plane,
+				    struct drm_plane_state *state)
 {
 	__drm_atomic_helper_plane_destroy_state(state);
 	kfree(to_mtk_plane_state(state));
@@ -117,8 +117,8 @@ static int mtk_plane_atomic_async_check(struct drm_plane *plane,
 	if (!plane->state->fb)
 		return -EINVAL;
 
-	ret = mtk_drm_crtc_plane_check(new_plane_state->crtc, plane,
-				       to_mtk_plane_state(new_plane_state));
+	ret = mtk_crtc_plane_check(new_plane_state->crtc, plane,
+				   to_mtk_plane_state(new_plane_state));
 	if (ret)
 		return ret;
 
@@ -135,7 +135,7 @@ static void mtk_plane_update_new_state(struct drm_plane_state *new_state,
 {
 	struct drm_framebuffer *fb = new_state->fb;
 	struct drm_gem_object *gem;
-	struct mtk_drm_gem_obj *mtk_gem;
+	struct mtk_gem_obj *mtk_gem;
 	unsigned int pitch, format;
 	u64 modifier;
 	dma_addr_t addr;
@@ -232,7 +232,7 @@ static void mtk_plane_atomic_async_update(struct drm_plane *plane,
 	swap(plane->state->fb, new_state->fb);
 	wmb(); /* Make sure the above parameters are set before update */
 	new_plane_state->pending.async_dirty = true;
-	mtk_drm_crtc_async_update(new_state->crtc, plane, state);
+	mtk_crtc_async_update(new_state->crtc, plane, state);
 }
 
 static const struct drm_plane_funcs mtk_plane_funcs = {
@@ -241,7 +241,7 @@ static const struct drm_plane_funcs mtk_plane_funcs = {
 	.destroy = drm_plane_cleanup,
 	.reset = mtk_plane_reset,
 	.atomic_duplicate_state = mtk_plane_duplicate_state,
-	.atomic_destroy_state = mtk_drm_plane_destroy_state,
+	.atomic_destroy_state = mtk_plane_destroy_state,
 	.format_mod_supported = mtk_plane_format_mod_supported,
 };
 
@@ -260,8 +260,8 @@ static int mtk_plane_atomic_check(struct drm_plane *plane,
 	if (WARN_ON(!new_plane_state->crtc))
 		return 0;
 
-	ret = mtk_drm_crtc_plane_check(new_plane_state->crtc, plane,
-				       to_mtk_plane_state(new_plane_state));
+	ret = mtk_crtc_plane_check(new_plane_state->crtc, plane,
+				   to_mtk_plane_state(new_plane_state));
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_plane.h b/drivers/gpu/drm/mediatek/mtk_plane.h
index 99aff7da0831..231bb7aac947 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_plane.h
+++ b/drivers/gpu/drm/mediatek/mtk_plane.h
@@ -4,8 +4,8 @@
  * Author: CK Hu <ck.hu@mediatek.com>
  */
 
-#ifndef _MTK_DRM_PLANE_H_
-#define _MTK_DRM_PLANE_H_
+#ifndef _MTK_PLANE_H_
+#define _MTK_PLANE_H_
 
 #include <drm/drm_crtc.h>
 #include <linux/types.h>
diff --git a/drivers/gpu/drm/meson/meson_dw_hdmi.c b/drivers/gpu/drm/meson/meson_dw_hdmi.c
index 5a9538bc0e26..5565f7777529 100644
--- a/drivers/gpu/drm/meson/meson_dw_hdmi.c
+++ b/drivers/gpu/drm/meson/meson_dw_hdmi.c
@@ -106,6 +106,8 @@
 #define HHI_HDMI_CLK_CNTL	0x1cc /* 0x73 */
 #define HHI_HDMI_PHY_CNTL0	0x3a0 /* 0xe8 */
 #define HHI_HDMI_PHY_CNTL1	0x3a4 /* 0xe9 */
+#define  PHY_CNTL1_INIT		0x03900000
+#define  PHY_INVERT		BIT(17)
 #define HHI_HDMI_PHY_CNTL2	0x3a8 /* 0xea */
 #define HHI_HDMI_PHY_CNTL3	0x3ac /* 0xeb */
 #define HHI_HDMI_PHY_CNTL4	0x3b0 /* 0xec */
@@ -130,6 +132,8 @@ struct meson_dw_hdmi_data {
 				    unsigned int addr);
 	void		(*dwc_write)(struct meson_dw_hdmi *dw_hdmi,
 				     unsigned int addr, unsigned int data);
+	u32 cntl0_init;
+	u32 cntl1_init;
 };
 
 struct meson_dw_hdmi {
@@ -384,26 +388,6 @@ static int dw_hdmi_phy_init(struct dw_hdmi *hdmi, void *data,
 	    dw_hdmi_bus_fmt_is_420(hdmi))
 		mode_is_420 = true;
 
-	/* Enable clocks */
-	regmap_update_bits(priv->hhi, HHI_HDMI_CLK_CNTL, 0xffff, 0x100);
-
-	/* Bring HDMITX MEM output of power down */
-	regmap_update_bits(priv->hhi, HHI_MEM_PD_REG0, 0xff << 8, 0);
-
-	/* Bring out of reset */
-	dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_SW_RESET,  0);
-
-	/* Enable internal pixclk, tmds_clk, spdif_clk, i2s_clk, cecclk */
-	dw_hdmi_top_write_bits(dw_hdmi, HDMITX_TOP_CLK_CNTL,
-			       0x3, 0x3);
-
-	/* Enable cec_clk and hdcp22_tmdsclk_en */
-	dw_hdmi_top_write_bits(dw_hdmi, HDMITX_TOP_CLK_CNTL,
-			       0x3 << 4, 0x3 << 4);
-
-	/* Enable normal output to PHY */
-	dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_BIST_CNTL, BIT(12));
-
 	/* TMDS pattern setup */
 	if (mode->clock > 340000 && !mode_is_420) {
 		dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_01,
@@ -425,20 +409,6 @@ static int dw_hdmi_phy_init(struct dw_hdmi *hdmi, void *data,
 	/* Setup PHY parameters */
 	meson_hdmi_phy_setup_mode(dw_hdmi, mode, mode_is_420);
 
-	/* Setup PHY */
-	regmap_update_bits(priv->hhi, HHI_HDMI_PHY_CNTL1,
-			   0xffff << 16, 0x0390 << 16);
-
-	/* BIT_INVERT */
-	if (dw_hdmi_is_compatible(dw_hdmi, "amlogic,meson-gxl-dw-hdmi") ||
-	    dw_hdmi_is_compatible(dw_hdmi, "amlogic,meson-gxm-dw-hdmi") ||
-	    dw_hdmi_is_compatible(dw_hdmi, "amlogic,meson-g12a-dw-hdmi"))
-		regmap_update_bits(priv->hhi, HHI_HDMI_PHY_CNTL1,
-				   BIT(17), 0);
-	else
-		regmap_update_bits(priv->hhi, HHI_HDMI_PHY_CNTL1,
-				   BIT(17), BIT(17));
-
 	/* Disable clock, fifo, fifo_wr */
 	regmap_update_bits(priv->hhi, HHI_HDMI_PHY_CNTL1, 0xf, 0);
 
@@ -492,7 +462,9 @@ static void dw_hdmi_phy_disable(struct dw_hdmi *hdmi,
 
 	DRM_DEBUG_DRIVER("\n");
 
-	regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL0, 0);
+	/* Fallback to init mode */
+	regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL1, dw_hdmi->data->cntl1_init);
+	regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL0, dw_hdmi->data->cntl0_init);
 }
 
 static enum drm_connector_status dw_hdmi_read_hpd(struct dw_hdmi *hdmi,
@@ -610,11 +582,22 @@ static const struct regmap_config meson_dw_hdmi_regmap_config = {
 	.fast_io = true,
 };
 
-static const struct meson_dw_hdmi_data meson_dw_hdmi_gx_data = {
+static const struct meson_dw_hdmi_data meson_dw_hdmi_gxbb_data = {
 	.top_read = dw_hdmi_top_read,
 	.top_write = dw_hdmi_top_write,
 	.dwc_read = dw_hdmi_dwc_read,
 	.dwc_write = dw_hdmi_dwc_write,
+	.cntl0_init = 0x0,
+	.cntl1_init = PHY_CNTL1_INIT | PHY_INVERT,
+};
+
+static const struct meson_dw_hdmi_data meson_dw_hdmi_gxl_data = {
+	.top_read = dw_hdmi_top_read,
+	.top_write = dw_hdmi_top_write,
+	.dwc_read = dw_hdmi_dwc_read,
+	.dwc_write = dw_hdmi_dwc_write,
+	.cntl0_init = 0x0,
+	.cntl1_init = PHY_CNTL1_INIT,
 };
 
 static const struct meson_dw_hdmi_data meson_dw_hdmi_g12a_data = {
@@ -622,6 +605,8 @@ static const struct meson_dw_hdmi_data meson_dw_hdmi_g12a_data = {
 	.top_write = dw_hdmi_g12a_top_write,
 	.dwc_read = dw_hdmi_g12a_dwc_read,
 	.dwc_write = dw_hdmi_g12a_dwc_write,
+	.cntl0_init = 0x000b4242, /* Bandgap */
+	.cntl1_init = PHY_CNTL1_INIT,
 };
 
 static void meson_dw_hdmi_init(struct meson_dw_hdmi *meson_dw_hdmi)
@@ -656,6 +641,13 @@ static void meson_dw_hdmi_init(struct meson_dw_hdmi *meson_dw_hdmi)
 	meson_dw_hdmi->data->top_write(meson_dw_hdmi,
 				       HDMITX_TOP_CLK_CNTL, 0xff);
 
+	/* Enable normal output to PHY */
+	meson_dw_hdmi->data->top_write(meson_dw_hdmi, HDMITX_TOP_BIST_CNTL, BIT(12));
+
+	/* Setup PHY */
+	regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL1, meson_dw_hdmi->data->cntl1_init);
+	regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL0, meson_dw_hdmi->data->cntl0_init);
+
 	/* Enable HDMI-TX Interrupt */
 	meson_dw_hdmi->data->top_write(meson_dw_hdmi, HDMITX_TOP_INTR_STAT_CLR,
 				       HDMITX_TOP_INTR_CORE);
@@ -865,11 +857,11 @@ static const struct dev_pm_ops meson_dw_hdmi_pm_ops = {
 
 static const struct of_device_id meson_dw_hdmi_of_table[] = {
 	{ .compatible = "amlogic,meson-gxbb-dw-hdmi",
-	  .data = &meson_dw_hdmi_gx_data },
+	  .data = &meson_dw_hdmi_gxbb_data },
 	{ .compatible = "amlogic,meson-gxl-dw-hdmi",
-	  .data = &meson_dw_hdmi_gx_data },
+	  .data = &meson_dw_hdmi_gxl_data },
 	{ .compatible = "amlogic,meson-gxm-dw-hdmi",
-	  .data = &meson_dw_hdmi_gx_data },
+	  .data = &meson_dw_hdmi_gxl_data },
 	{ .compatible = "amlogic,meson-g12a-dw-hdmi",
 	  .data = &meson_dw_hdmi_g12a_data },
 	{ }
diff --git a/drivers/gpu/drm/meson/meson_dw_mipi_dsi.c b/drivers/gpu/drm/meson/meson_dw_mipi_dsi.c
index a6bc1bdb3d0d..a10cff3ca1fe 100644
--- a/drivers/gpu/drm/meson/meson_dw_mipi_dsi.c
+++ b/drivers/gpu/drm/meson/meson_dw_mipi_dsi.c
@@ -95,6 +95,7 @@ static int dw_mipi_dsi_phy_init(void *priv_data)
 		return ret;
 	}
 
+	clk_disable_unprepare(mipi_dsi->px_clk);
 	ret = clk_set_rate(mipi_dsi->px_clk, mipi_dsi->mode->clock * 1000);
 
 	if (ret) {
@@ -103,6 +104,12 @@ static int dw_mipi_dsi_phy_init(void *priv_data)
 		return ret;
 	}
 
+	ret = clk_prepare_enable(mipi_dsi->px_clk);
+	if (ret) {
+		dev_err(mipi_dsi->dev, "Failed to enable DSI Pixel clock (ret %d)\n", ret);
+		return ret;
+	}
+
 	switch (mipi_dsi->dsi_device->format) {
 	case MIPI_DSI_FMT_RGB888:
 		dpi_data_format = DPI_COLOR_24BIT;
diff --git a/drivers/gpu/drm/meson/meson_vclk.c b/drivers/gpu/drm/meson/meson_vclk.c
index 2a82119eb58e..2a942dc6a6dc 100644
--- a/drivers/gpu/drm/meson/meson_vclk.c
+++ b/drivers/gpu/drm/meson/meson_vclk.c
@@ -790,13 +790,13 @@ meson_vclk_vic_supported_freq(struct meson_drm *priv, unsigned int phy_freq,
 				 FREQ_1000_1001(params[i].pixel_freq));
 		DRM_DEBUG_DRIVER("i = %d phy_freq = %d alt = %d\n",
 				 i, params[i].phy_freq,
-				 FREQ_1000_1001(params[i].phy_freq/10)*10);
+				 FREQ_1000_1001(params[i].phy_freq/1000)*1000);
 		/* Match strict frequency */
 		if (phy_freq == params[i].phy_freq &&
 		    vclk_freq == params[i].vclk_freq)
 			return MODE_OK;
 		/* Match 1000/1001 variant */
-		if (phy_freq == (FREQ_1000_1001(params[i].phy_freq/10)*10) &&
+		if (phy_freq == (FREQ_1000_1001(params[i].phy_freq/1000)*1000) &&
 		    vclk_freq == FREQ_1000_1001(params[i].vclk_freq))
 			return MODE_OK;
 	}
@@ -1070,7 +1070,7 @@ void meson_vclk_setup(struct meson_drm *priv, unsigned int target,
 
 	for (freq = 0 ; params[freq].pixel_freq ; ++freq) {
 		if ((phy_freq == params[freq].phy_freq ||
-		     phy_freq == FREQ_1000_1001(params[freq].phy_freq/10)*10) &&
+		     phy_freq == FREQ_1000_1001(params[freq].phy_freq/1000)*1000) &&
 		    (vclk_freq == params[freq].vclk_freq ||
 		     vclk_freq == FREQ_1000_1001(params[freq].vclk_freq))) {
 			if (vclk_freq != params[freq].vclk_freq)
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.h b/drivers/gpu/drm/mgag200/mgag200_drv.h
index 765e49fd8911..58a0e62eaf18 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.h
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.h
@@ -366,6 +366,7 @@ struct drm_crtc_state;
 struct drm_display_mode;
 struct drm_plane;
 struct drm_atomic_state;
+struct drm_scanout_buffer;
 
 extern const uint32_t mgag200_primary_plane_formats[];
 extern const size_t   mgag200_primary_plane_formats_size;
@@ -379,12 +380,16 @@ void mgag200_primary_plane_helper_atomic_enable(struct drm_plane *plane,
 						struct drm_atomic_state *state);
 void mgag200_primary_plane_helper_atomic_disable(struct drm_plane *plane,
 						 struct drm_atomic_state *old_state);
+int mgag200_primary_plane_helper_get_scanout_buffer(struct drm_plane *plane,
+						    struct drm_scanout_buffer *sb);
+
 #define MGAG200_PRIMARY_PLANE_HELPER_FUNCS \
 	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS, \
 	.atomic_check = mgag200_primary_plane_helper_atomic_check, \
 	.atomic_update = mgag200_primary_plane_helper_atomic_update, \
 	.atomic_enable = mgag200_primary_plane_helper_atomic_enable, \
-	.atomic_disable = mgag200_primary_plane_helper_atomic_disable
+	.atomic_disable = mgag200_primary_plane_helper_atomic_disable, \
+	.get_scanout_buffer = mgag200_primary_plane_helper_get_scanout_buffer
 
 #define MGAG200_PRIMARY_PLANE_FUNCS \
 	.update_plane = drm_atomic_helper_update_plane, \
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index e17cb4c5f774..fc54851d3384 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -21,6 +21,7 @@
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_panic.h>
 #include <drm/drm_print.h>
 
 #include "mgag200_drv.h"
@@ -546,6 +547,23 @@ void mgag200_primary_plane_helper_atomic_disable(struct drm_plane *plane,
 	msleep(20);
 }
 
+int mgag200_primary_plane_helper_get_scanout_buffer(struct drm_plane *plane,
+						    struct drm_scanout_buffer *sb)
+{
+	struct mga_device *mdev = to_mga_device(plane->dev);
+	struct iosys_map map = IOSYS_MAP_INIT_VADDR_IOMEM(mdev->vram);
+
+	if (plane->state && plane->state->fb) {
+		sb->format = plane->state->fb->format;
+		sb->width = plane->state->fb->width;
+		sb->height = plane->state->fb->height;
+		sb->pitch[0] = plane->state->fb->pitches[0];
+		sb->map[0] = map;
+		return 0;
+	}
+	return -ENODEV;
+}
+
 /*
  * CRTC
  */
diff --git a/drivers/gpu/drm/msm/.gitignore b/drivers/gpu/drm/msm/.gitignore
new file mode 100644
index 000000000000..9ab870da897d
--- /dev/null
+++ b/drivers/gpu/drm/msm/.gitignore
@@ -0,0 +1 @@
+generated/
diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig
index f202f26adab2..1931ecf73e32 100644
--- a/drivers/gpu/drm/msm/Kconfig
+++ b/drivers/gpu/drm/msm/Kconfig
@@ -14,7 +14,7 @@ config DRM_MSM
 	select IOMMU_IO_PGTABLE
 	select QCOM_MDT_LOADER if ARCH_QCOM
 	select REGULATOR
-	select DRM_DP_AUX_BUS
+	select DRM_DISPLAY_DP_AUX_BUS
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HELPER
 	select DRM_EXEC
@@ -54,6 +54,14 @@ config DRM_MSM_GPU_SUDO
 	  Only use this if you are a driver developer.  This should *not*
 	  be enabled for production kernels.  If unsure, say N.
 
+config DRM_MSM_VALIDATE_XML
+	bool "Validate XML register files against schema"
+	depends on DRM_MSM && EXPERT
+	depends on $(success,$(PYTHON3) -c "import lxml")
+	help
+	  Validate XML files with register definitions against rules-fd schema.
+	  This option is mostly targeting DRM MSM developers. If unsure, say N.
+
 config DRM_MSM_MDSS
 	bool
 	depends on DRM_MSM
diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index b21ae2880c71..eb788921ff4f 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -1,13 +1,15 @@
 # SPDX-License-Identifier: GPL-2.0
-ccflags-y := -I $(srctree)/$(src)
-ccflags-y += -I $(srctree)/$(src)/disp/dpu1
-ccflags-$(CONFIG_DRM_MSM_DSI) += -I $(srctree)/$(src)/dsi
-ccflags-$(CONFIG_DRM_MSM_DP) += -I $(srctree)/$(src)/dp
+ccflags-y := -I $(src)
+ccflags-y += -I $(obj)/generated
+ccflags-y += -I $(src)/disp/dpu1
+ccflags-$(CONFIG_DRM_MSM_DSI) += -I $(src)/dsi
+ccflags-$(CONFIG_DRM_MSM_DP) += -I $(src)/dp
 
-msm-y := \
+adreno-y := \
 	adreno/adreno_device.o \
 	adreno/adreno_gpu.o \
 	adreno/a2xx_gpu.o \
+	adreno/a2xx_gpummu.o \
 	adreno/a3xx_gpu.o \
 	adreno/a4xx_gpu.o \
 	adreno/a5xx_gpu.o \
@@ -17,7 +19,11 @@ msm-y := \
 	adreno/a6xx_gmu.o \
 	adreno/a6xx_hfi.o \
 
-msm-$(CONFIG_DRM_MSM_HDMI) += \
+adreno-$(CONFIG_DEBUG_FS) += adreno/a5xx_debugfs.o \
+
+adreno-$(CONFIG_DRM_MSM_GPU_STATE)	+= adreno/a6xx_gpu_state.o
+
+msm-display-$(CONFIG_DRM_MSM_HDMI) += \
 	hdmi/hdmi.o \
 	hdmi/hdmi_audio.o \
 	hdmi/hdmi_bridge.o \
@@ -30,7 +36,7 @@ msm-$(CONFIG_DRM_MSM_HDMI) += \
 	hdmi/hdmi_phy_8x74.o \
 	hdmi/hdmi_pll_8960.o \
 
-msm-$(CONFIG_DRM_MSM_MDP4) += \
+msm-display-$(CONFIG_DRM_MSM_MDP4) += \
 	disp/mdp4/mdp4_crtc.o \
 	disp/mdp4/mdp4_dsi_encoder.o \
 	disp/mdp4/mdp4_dtv_encoder.o \
@@ -41,7 +47,7 @@ msm-$(CONFIG_DRM_MSM_MDP4) += \
 	disp/mdp4/mdp4_kms.o \
 	disp/mdp4/mdp4_plane.o \
 
-msm-$(CONFIG_DRM_MSM_MDP5) += \
+msm-display-$(CONFIG_DRM_MSM_MDP5) += \
 	disp/mdp5/mdp5_cfg.o \
 	disp/mdp5/mdp5_cmd_encoder.o \
 	disp/mdp5/mdp5_ctl.o \
@@ -54,7 +60,7 @@ msm-$(CONFIG_DRM_MSM_MDP5) += \
 	disp/mdp5/mdp5_plane.o \
 	disp/mdp5/mdp5_smp.o \
 
-msm-$(CONFIG_DRM_MSM_DPU) += \
+msm-display-$(CONFIG_DRM_MSM_DPU) += \
 	disp/dpu1/dpu_core_perf.o \
 	disp/dpu1/dpu_crtc.o \
 	disp/dpu1/dpu_encoder.o \
@@ -84,14 +90,16 @@ msm-$(CONFIG_DRM_MSM_DPU) += \
 	disp/dpu1/dpu_vbif.o \
 	disp/dpu1/dpu_writeback.o
 
-msm-$(CONFIG_DRM_MSM_MDSS) += \
+msm-display-$(CONFIG_DRM_MSM_MDSS) += \
 	msm_mdss.o \
 
-msm-y += \
+msm-display-y += \
 	disp/mdp_format.o \
 	disp/mdp_kms.o \
 	disp/msm_disp_snapshot.o \
 	disp/msm_disp_snapshot_util.o \
+
+msm-y += \
 	msm_atomic.o \
 	msm_atomic_tracepoints.o \
 	msm_debugfs.o \
@@ -113,14 +121,13 @@ msm-y += \
 	msm_ringbuffer.o \
 	msm_submitqueue.o \
 	msm_gpu_tracepoints.o \
-	msm_gpummu.o
 
-msm-$(CONFIG_DEBUG_FS) += adreno/a5xx_debugfs.o \
-	dp/dp_debug.o
+msm-$(CONFIG_DRM_FBDEV_EMULATION) += msm_fbdev.o
 
-msm-$(CONFIG_DRM_MSM_GPU_STATE)	+= adreno/a6xx_gpu_state.o
+msm-display-$(CONFIG_DEBUG_FS) += \
+	dp/dp_debug.o
 
-msm-$(CONFIG_DRM_MSM_DP)+= dp/dp_aux.o \
+msm-display-$(CONFIG_DRM_MSM_DP)+= dp/dp_aux.o \
 	dp/dp_catalog.o \
 	dp/dp_ctrl.o \
 	dp/dp_display.o \
@@ -130,21 +137,76 @@ msm-$(CONFIG_DRM_MSM_DP)+= dp/dp_aux.o \
 	dp/dp_audio.o \
 	dp/dp_utils.o
 
-msm-$(CONFIG_DRM_FBDEV_EMULATION) += msm_fbdev.o
-
-msm-$(CONFIG_DRM_MSM_HDMI_HDCP) += hdmi/hdmi_hdcp.o
+msm-display-$(CONFIG_DRM_MSM_HDMI_HDCP) += hdmi/hdmi_hdcp.o
 
-msm-$(CONFIG_DRM_MSM_DSI) += dsi/dsi.o \
+msm-display-$(CONFIG_DRM_MSM_DSI) += dsi/dsi.o \
 			dsi/dsi_cfg.o \
 			dsi/dsi_host.o \
 			dsi/dsi_manager.o \
 			dsi/phy/dsi_phy.o
 
-msm-$(CONFIG_DRM_MSM_DSI_28NM_PHY) += dsi/phy/dsi_phy_28nm.o
-msm-$(CONFIG_DRM_MSM_DSI_20NM_PHY) += dsi/phy/dsi_phy_20nm.o
-msm-$(CONFIG_DRM_MSM_DSI_28NM_8960_PHY) += dsi/phy/dsi_phy_28nm_8960.o
-msm-$(CONFIG_DRM_MSM_DSI_14NM_PHY) += dsi/phy/dsi_phy_14nm.o
-msm-$(CONFIG_DRM_MSM_DSI_10NM_PHY) += dsi/phy/dsi_phy_10nm.o
-msm-$(CONFIG_DRM_MSM_DSI_7NM_PHY) += dsi/phy/dsi_phy_7nm.o
+msm-display-$(CONFIG_DRM_MSM_DSI_28NM_PHY) += dsi/phy/dsi_phy_28nm.o
+msm-display-$(CONFIG_DRM_MSM_DSI_20NM_PHY) += dsi/phy/dsi_phy_20nm.o
+msm-display-$(CONFIG_DRM_MSM_DSI_28NM_8960_PHY) += dsi/phy/dsi_phy_28nm_8960.o
+msm-display-$(CONFIG_DRM_MSM_DSI_14NM_PHY) += dsi/phy/dsi_phy_14nm.o
+msm-display-$(CONFIG_DRM_MSM_DSI_10NM_PHY) += dsi/phy/dsi_phy_10nm.o
+msm-display-$(CONFIG_DRM_MSM_DSI_7NM_PHY) += dsi/phy/dsi_phy_7nm.o
+
+msm-y += $(adreno-y) $(msm-display-y)
 
 obj-$(CONFIG_DRM_MSM)	+= msm.o
+
+ifeq (y,$(CONFIG_DRM_MSM_VALIDATE_XML))
+	headergen-opts += --validate
+else
+	headergen-opts += --no-validate
+endif
+
+quiet_cmd_headergen = GENHDR  $@
+      cmd_headergen = mkdir -p $(obj)/generated && $(PYTHON3) $(src)/registers/gen_header.py \
+		      $(headergen-opts) --rnn $(src)/registers --xml $< c-defines > $@
+
+$(obj)/generated/%.xml.h: $(src)/registers/adreno/%.xml \
+		$(src)/registers/adreno/adreno_common.xml \
+		$(src)/registers/adreno/adreno_pm4.xml \
+		$(src)/registers/freedreno_copyright.xml \
+		$(src)/registers/gen_header.py \
+		$(src)/registers/rules-fd.xsd \
+		FORCE
+	$(call if_changed,headergen)
+
+$(obj)/generated/%.xml.h: $(src)/registers/display/%.xml \
+		$(src)/registers/freedreno_copyright.xml \
+		$(src)/registers/gen_header.py \
+		$(src)/registers/rules-fd.xsd \
+		FORCE
+	$(call if_changed,headergen)
+
+ADRENO_HEADERS = \
+	generated/a2xx.xml.h \
+	generated/a3xx.xml.h \
+	generated/a4xx.xml.h \
+	generated/a5xx.xml.h \
+	generated/a6xx.xml.h \
+	generated/a6xx_gmu.xml.h \
+	generated/adreno_common.xml.h \
+	generated/adreno_pm4.xml.h \
+
+DISPLAY_HEADERS = \
+	generated/dsi_phy_7nm.xml.h \
+	generated/dsi_phy_10nm.xml.h \
+	generated/dsi_phy_14nm.xml.h \
+	generated/dsi_phy_20nm.xml.h \
+	generated/dsi_phy_28nm_8960.xml.h \
+	generated/dsi_phy_28nm.xml.h \
+	generated/dsi.xml.h \
+	generated/hdmi.xml.h \
+	generated/mdp4.xml.h \
+	generated/mdp5.xml.h \
+	generated/mdp_common.xml.h \
+	generated/sfpb.xml.h
+
+$(addprefix $(obj)/,$(adreno-y)): $(addprefix $(obj)/,$(ADRENO_HEADERS))
+$(addprefix $(obj)/,$(msm-display-y)): $(addprefix $(obj)/,$(DISPLAY_HEADERS))
+
+targets += $(ADRENO_HEADERS) $(DISPLAY_HEADERS)
diff --git a/drivers/gpu/drm/msm/adreno/a2xx.xml.h b/drivers/gpu/drm/msm/adreno/a2xx.xml.h
deleted file mode 100644
index 23141cbcea97..000000000000
--- a/drivers/gpu/drm/msm/adreno/a2xx.xml.h
+++ /dev/null
@@ -1,3251 +0,0 @@
-#ifndef A2XX_XML
-#define A2XX_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng gen_header.py tool in this git repository:
-http://gitlab.freedesktop.org/mesa/mesa/
-git clone https://gitlab.freedesktop.org/mesa/mesa.git
-
-The rules-ng-ng source files this header was generated from are:
-
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a2xx.xml          (  91929 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml  (   1572 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml (  15434 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml    (  85691 bytes, from Fri Feb 16 09:49:01 2024)
-
-Copyright (C) 2013-2024 by the following authors:
-- Rob Clark <robdclark@gmail.com> Rob Clark
-- Ilia Mirkin <imirkin@alum.mit.edu> Ilia Mirkin
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-*/
-
-#ifdef __KERNEL__
-#include <linux/bug.h>
-#define assert(x) BUG_ON(!(x))
-#else
-#include <assert.h>
-#endif
-
-#ifdef __cplusplus
-#define __struct_cast(X)
-#else
-#define __struct_cast(X) (struct X)
-#endif
-
-enum a2xx_rb_dither_type {
-	DITHER_PIXEL = 0,
-	DITHER_SUBPIXEL = 1,
-};
-
-enum a2xx_colorformatx {
-	COLORX_4_4_4_4 = 0,
-	COLORX_1_5_5_5 = 1,
-	COLORX_5_6_5 = 2,
-	COLORX_8 = 3,
-	COLORX_8_8 = 4,
-	COLORX_8_8_8_8 = 5,
-	COLORX_S8_8_8_8 = 6,
-	COLORX_16_FLOAT = 7,
-	COLORX_16_16_FLOAT = 8,
-	COLORX_16_16_16_16_FLOAT = 9,
-	COLORX_32_FLOAT = 10,
-	COLORX_32_32_FLOAT = 11,
-	COLORX_32_32_32_32_FLOAT = 12,
-	COLORX_2_3_3 = 13,
-	COLORX_8_8_8 = 14,
-};
-
-enum a2xx_sq_surfaceformat {
-	FMT_1_REVERSE = 0,
-	FMT_1 = 1,
-	FMT_8 = 2,
-	FMT_1_5_5_5 = 3,
-	FMT_5_6_5 = 4,
-	FMT_6_5_5 = 5,
-	FMT_8_8_8_8 = 6,
-	FMT_2_10_10_10 = 7,
-	FMT_8_A = 8,
-	FMT_8_B = 9,
-	FMT_8_8 = 10,
-	FMT_Cr_Y1_Cb_Y0 = 11,
-	FMT_Y1_Cr_Y0_Cb = 12,
-	FMT_5_5_5_1 = 13,
-	FMT_8_8_8_8_A = 14,
-	FMT_4_4_4_4 = 15,
-	FMT_8_8_8 = 16,
-	FMT_DXT1 = 18,
-	FMT_DXT2_3 = 19,
-	FMT_DXT4_5 = 20,
-	FMT_10_10_10_2 = 21,
-	FMT_24_8 = 22,
-	FMT_16 = 24,
-	FMT_16_16 = 25,
-	FMT_16_16_16_16 = 26,
-	FMT_16_EXPAND = 27,
-	FMT_16_16_EXPAND = 28,
-	FMT_16_16_16_16_EXPAND = 29,
-	FMT_16_FLOAT = 30,
-	FMT_16_16_FLOAT = 31,
-	FMT_16_16_16_16_FLOAT = 32,
-	FMT_32 = 33,
-	FMT_32_32 = 34,
-	FMT_32_32_32_32 = 35,
-	FMT_32_FLOAT = 36,
-	FMT_32_32_FLOAT = 37,
-	FMT_32_32_32_32_FLOAT = 38,
-	FMT_ATI_TC_RGB = 39,
-	FMT_ATI_TC_RGBA = 40,
-	FMT_ATI_TC_555_565_RGB = 41,
-	FMT_ATI_TC_555_565_RGBA = 42,
-	FMT_ATI_TC_RGBA_INTERP = 43,
-	FMT_ATI_TC_555_565_RGBA_INTERP = 44,
-	FMT_ETC1_RGBA_INTERP = 46,
-	FMT_ETC1_RGB = 47,
-	FMT_ETC1_RGBA = 48,
-	FMT_DXN = 49,
-	FMT_2_3_3 = 51,
-	FMT_2_10_10_10_AS_16_16_16_16 = 54,
-	FMT_10_10_10_2_AS_16_16_16_16 = 55,
-	FMT_32_32_32_FLOAT = 57,
-	FMT_DXT3A = 58,
-	FMT_DXT5A = 59,
-	FMT_CTX1 = 60,
-};
-
-enum a2xx_sq_ps_vtx_mode {
-	POSITION_1_VECTOR = 0,
-	POSITION_2_VECTORS_UNUSED = 1,
-	POSITION_2_VECTORS_SPRITE = 2,
-	POSITION_2_VECTORS_EDGE = 3,
-	POSITION_2_VECTORS_KILL = 4,
-	POSITION_2_VECTORS_SPRITE_KILL = 5,
-	POSITION_2_VECTORS_EDGE_KILL = 6,
-	MULTIPASS = 7,
-};
-
-enum a2xx_sq_sample_cntl {
-	CENTROIDS_ONLY = 0,
-	CENTERS_ONLY = 1,
-	CENTROIDS_AND_CENTERS = 2,
-};
-
-enum a2xx_dx_clip_space {
-	DXCLIP_OPENGL = 0,
-	DXCLIP_DIRECTX = 1,
-};
-
-enum a2xx_pa_su_sc_polymode {
-	POLY_DISABLED = 0,
-	POLY_DUALMODE = 1,
-};
-
-enum a2xx_rb_edram_mode {
-	EDRAM_NOP = 0,
-	COLOR_DEPTH = 4,
-	DEPTH_ONLY = 5,
-	EDRAM_COPY = 6,
-};
-
-enum a2xx_pa_sc_pattern_bit_order {
-	LITTLE = 0,
-	BIG = 1,
-};
-
-enum a2xx_pa_sc_auto_reset_cntl {
-	NEVER = 0,
-	EACH_PRIMITIVE = 1,
-	EACH_PACKET = 2,
-};
-
-enum a2xx_pa_pixcenter {
-	PIXCENTER_D3D = 0,
-	PIXCENTER_OGL = 1,
-};
-
-enum a2xx_pa_roundmode {
-	TRUNCATE = 0,
-	ROUND = 1,
-	ROUNDTOEVEN = 2,
-	ROUNDTOODD = 3,
-};
-
-enum a2xx_pa_quantmode {
-	ONE_SIXTEENTH = 0,
-	ONE_EIGTH = 1,
-	ONE_QUARTER = 2,
-	ONE_HALF = 3,
-	ONE = 4,
-};
-
-enum a2xx_rb_copy_sample_select {
-	SAMPLE_0 = 0,
-	SAMPLE_1 = 1,
-	SAMPLE_2 = 2,
-	SAMPLE_3 = 3,
-	SAMPLE_01 = 4,
-	SAMPLE_23 = 5,
-	SAMPLE_0123 = 6,
-};
-
-enum a2xx_rb_blend_opcode {
-	BLEND2_DST_PLUS_SRC = 0,
-	BLEND2_SRC_MINUS_DST = 1,
-	BLEND2_MIN_DST_SRC = 2,
-	BLEND2_MAX_DST_SRC = 3,
-	BLEND2_DST_MINUS_SRC = 4,
-	BLEND2_DST_PLUS_SRC_BIAS = 5,
-};
-
-enum a2xx_su_perfcnt_select {
-	PERF_PAPC_PASX_REQ = 0,
-	PERF_PAPC_PASX_FIRST_VECTOR = 2,
-	PERF_PAPC_PASX_SECOND_VECTOR = 3,
-	PERF_PAPC_PASX_FIRST_DEAD = 4,
-	PERF_PAPC_PASX_SECOND_DEAD = 5,
-	PERF_PAPC_PASX_VTX_KILL_DISCARD = 6,
-	PERF_PAPC_PASX_VTX_NAN_DISCARD = 7,
-	PERF_PAPC_PA_INPUT_PRIM = 8,
-	PERF_PAPC_PA_INPUT_NULL_PRIM = 9,
-	PERF_PAPC_PA_INPUT_EVENT_FLAG = 10,
-	PERF_PAPC_PA_INPUT_FIRST_PRIM_SLOT = 11,
-	PERF_PAPC_PA_INPUT_END_OF_PACKET = 12,
-	PERF_PAPC_CLPR_CULL_PRIM = 13,
-	PERF_PAPC_CLPR_VV_CULL_PRIM = 15,
-	PERF_PAPC_CLPR_VTX_KILL_CULL_PRIM = 17,
-	PERF_PAPC_CLPR_VTX_NAN_CULL_PRIM = 18,
-	PERF_PAPC_CLPR_CULL_TO_NULL_PRIM = 19,
-	PERF_PAPC_CLPR_VV_CLIP_PRIM = 21,
-	PERF_PAPC_CLPR_POINT_CLIP_CANDIDATE = 23,
-	PERF_PAPC_CLPR_CLIP_PLANE_CNT_1 = 24,
-	PERF_PAPC_CLPR_CLIP_PLANE_CNT_2 = 25,
-	PERF_PAPC_CLPR_CLIP_PLANE_CNT_3 = 26,
-	PERF_PAPC_CLPR_CLIP_PLANE_CNT_4 = 27,
-	PERF_PAPC_CLPR_CLIP_PLANE_CNT_5 = 28,
-	PERF_PAPC_CLPR_CLIP_PLANE_CNT_6 = 29,
-	PERF_PAPC_CLPR_CLIP_PLANE_NEAR = 30,
-	PERF_PAPC_CLPR_CLIP_PLANE_FAR = 31,
-	PERF_PAPC_CLPR_CLIP_PLANE_LEFT = 32,
-	PERF_PAPC_CLPR_CLIP_PLANE_RIGHT = 33,
-	PERF_PAPC_CLPR_CLIP_PLANE_TOP = 34,
-	PERF_PAPC_CLPR_CLIP_PLANE_BOTTOM = 35,
-	PERF_PAPC_CLSM_NULL_PRIM = 36,
-	PERF_PAPC_CLSM_TOTALLY_VISIBLE_PRIM = 37,
-	PERF_PAPC_CLSM_CLIP_PRIM = 38,
-	PERF_PAPC_CLSM_CULL_TO_NULL_PRIM = 39,
-	PERF_PAPC_CLSM_OUT_PRIM_CNT_1 = 40,
-	PERF_PAPC_CLSM_OUT_PRIM_CNT_2 = 41,
-	PERF_PAPC_CLSM_OUT_PRIM_CNT_3 = 42,
-	PERF_PAPC_CLSM_OUT_PRIM_CNT_4 = 43,
-	PERF_PAPC_CLSM_OUT_PRIM_CNT_5 = 44,
-	PERF_PAPC_CLSM_OUT_PRIM_CNT_6_7 = 45,
-	PERF_PAPC_CLSM_NON_TRIVIAL_CULL = 46,
-	PERF_PAPC_SU_INPUT_PRIM = 47,
-	PERF_PAPC_SU_INPUT_CLIP_PRIM = 48,
-	PERF_PAPC_SU_INPUT_NULL_PRIM = 49,
-	PERF_PAPC_SU_ZERO_AREA_CULL_PRIM = 50,
-	PERF_PAPC_SU_BACK_FACE_CULL_PRIM = 51,
-	PERF_PAPC_SU_FRONT_FACE_CULL_PRIM = 52,
-	PERF_PAPC_SU_POLYMODE_FACE_CULL = 53,
-	PERF_PAPC_SU_POLYMODE_BACK_CULL = 54,
-	PERF_PAPC_SU_POLYMODE_FRONT_CULL = 55,
-	PERF_PAPC_SU_POLYMODE_INVALID_FILL = 56,
-	PERF_PAPC_SU_OUTPUT_PRIM = 57,
-	PERF_PAPC_SU_OUTPUT_CLIP_PRIM = 58,
-	PERF_PAPC_SU_OUTPUT_NULL_PRIM = 59,
-	PERF_PAPC_SU_OUTPUT_EVENT_FLAG = 60,
-	PERF_PAPC_SU_OUTPUT_FIRST_PRIM_SLOT = 61,
-	PERF_PAPC_SU_OUTPUT_END_OF_PACKET = 62,
-	PERF_PAPC_SU_OUTPUT_POLYMODE_FACE = 63,
-	PERF_PAPC_SU_OUTPUT_POLYMODE_BACK = 64,
-	PERF_PAPC_SU_OUTPUT_POLYMODE_FRONT = 65,
-	PERF_PAPC_SU_OUT_CLIP_POLYMODE_FACE = 66,
-	PERF_PAPC_SU_OUT_CLIP_POLYMODE_BACK = 67,
-	PERF_PAPC_SU_OUT_CLIP_POLYMODE_FRONT = 68,
-	PERF_PAPC_PASX_REQ_IDLE = 69,
-	PERF_PAPC_PASX_REQ_BUSY = 70,
-	PERF_PAPC_PASX_REQ_STALLED = 71,
-	PERF_PAPC_PASX_REC_IDLE = 72,
-	PERF_PAPC_PASX_REC_BUSY = 73,
-	PERF_PAPC_PASX_REC_STARVED_SX = 74,
-	PERF_PAPC_PASX_REC_STALLED = 75,
-	PERF_PAPC_PASX_REC_STALLED_POS_MEM = 76,
-	PERF_PAPC_PASX_REC_STALLED_CCGSM_IN = 77,
-	PERF_PAPC_CCGSM_IDLE = 78,
-	PERF_PAPC_CCGSM_BUSY = 79,
-	PERF_PAPC_CCGSM_STALLED = 80,
-	PERF_PAPC_CLPRIM_IDLE = 81,
-	PERF_PAPC_CLPRIM_BUSY = 82,
-	PERF_PAPC_CLPRIM_STALLED = 83,
-	PERF_PAPC_CLPRIM_STARVED_CCGSM = 84,
-	PERF_PAPC_CLIPSM_IDLE = 85,
-	PERF_PAPC_CLIPSM_BUSY = 86,
-	PERF_PAPC_CLIPSM_WAIT_CLIP_VERT_ENGH = 87,
-	PERF_PAPC_CLIPSM_WAIT_HIGH_PRI_SEQ = 88,
-	PERF_PAPC_CLIPSM_WAIT_CLIPGA = 89,
-	PERF_PAPC_CLIPSM_WAIT_AVAIL_VTE_CLIP = 90,
-	PERF_PAPC_CLIPSM_WAIT_CLIP_OUTSM = 91,
-	PERF_PAPC_CLIPGA_IDLE = 92,
-	PERF_PAPC_CLIPGA_BUSY = 93,
-	PERF_PAPC_CLIPGA_STARVED_VTE_CLIP = 94,
-	PERF_PAPC_CLIPGA_STALLED = 95,
-	PERF_PAPC_CLIP_IDLE = 96,
-	PERF_PAPC_CLIP_BUSY = 97,
-	PERF_PAPC_SU_IDLE = 98,
-	PERF_PAPC_SU_BUSY = 99,
-	PERF_PAPC_SU_STARVED_CLIP = 100,
-	PERF_PAPC_SU_STALLED_SC = 101,
-	PERF_PAPC_SU_FACENESS_CULL = 102,
-};
-
-enum a2xx_sc_perfcnt_select {
-	SC_SR_WINDOW_VALID = 0,
-	SC_CW_WINDOW_VALID = 1,
-	SC_QM_WINDOW_VALID = 2,
-	SC_FW_WINDOW_VALID = 3,
-	SC_EZ_WINDOW_VALID = 4,
-	SC_IT_WINDOW_VALID = 5,
-	SC_STARVED_BY_PA = 6,
-	SC_STALLED_BY_RB_TILE = 7,
-	SC_STALLED_BY_RB_SAMP = 8,
-	SC_STARVED_BY_RB_EZ = 9,
-	SC_STALLED_BY_SAMPLE_FF = 10,
-	SC_STALLED_BY_SQ = 11,
-	SC_STALLED_BY_SP = 12,
-	SC_TOTAL_NO_PRIMS = 13,
-	SC_NON_EMPTY_PRIMS = 14,
-	SC_NO_TILES_PASSING_QM = 15,
-	SC_NO_PIXELS_PRE_EZ = 16,
-	SC_NO_PIXELS_POST_EZ = 17,
-};
-
-enum a2xx_vgt_perfcount_select {
-	VGT_SQ_EVENT_WINDOW_ACTIVE = 0,
-	VGT_SQ_SEND = 1,
-	VGT_SQ_STALLED = 2,
-	VGT_SQ_STARVED_BUSY = 3,
-	VGT_SQ_STARVED_IDLE = 4,
-	VGT_SQ_STATIC = 5,
-	VGT_PA_EVENT_WINDOW_ACTIVE = 6,
-	VGT_PA_CLIP_V_SEND = 7,
-	VGT_PA_CLIP_V_STALLED = 8,
-	VGT_PA_CLIP_V_STARVED_BUSY = 9,
-	VGT_PA_CLIP_V_STARVED_IDLE = 10,
-	VGT_PA_CLIP_V_STATIC = 11,
-	VGT_PA_CLIP_P_SEND = 12,
-	VGT_PA_CLIP_P_STALLED = 13,
-	VGT_PA_CLIP_P_STARVED_BUSY = 14,
-	VGT_PA_CLIP_P_STARVED_IDLE = 15,
-	VGT_PA_CLIP_P_STATIC = 16,
-	VGT_PA_CLIP_S_SEND = 17,
-	VGT_PA_CLIP_S_STALLED = 18,
-	VGT_PA_CLIP_S_STARVED_BUSY = 19,
-	VGT_PA_CLIP_S_STARVED_IDLE = 20,
-	VGT_PA_CLIP_S_STATIC = 21,
-	RBIU_FIFOS_EVENT_WINDOW_ACTIVE = 22,
-	RBIU_IMMED_DATA_FIFO_STARVED = 23,
-	RBIU_IMMED_DATA_FIFO_STALLED = 24,
-	RBIU_DMA_REQUEST_FIFO_STARVED = 25,
-	RBIU_DMA_REQUEST_FIFO_STALLED = 26,
-	RBIU_DRAW_INITIATOR_FIFO_STARVED = 27,
-	RBIU_DRAW_INITIATOR_FIFO_STALLED = 28,
-	BIN_PRIM_NEAR_CULL = 29,
-	BIN_PRIM_ZERO_CULL = 30,
-	BIN_PRIM_FAR_CULL = 31,
-	BIN_PRIM_BIN_CULL = 32,
-	BIN_PRIM_FACE_CULL = 33,
-	SPARE34 = 34,
-	SPARE35 = 35,
-	SPARE36 = 36,
-	SPARE37 = 37,
-	SPARE38 = 38,
-	SPARE39 = 39,
-	TE_SU_IN_VALID = 40,
-	TE_SU_IN_READ = 41,
-	TE_SU_IN_PRIM = 42,
-	TE_SU_IN_EOP = 43,
-	TE_SU_IN_NULL_PRIM = 44,
-	TE_WK_IN_VALID = 45,
-	TE_WK_IN_READ = 46,
-	TE_OUT_PRIM_VALID = 47,
-	TE_OUT_PRIM_READ = 48,
-};
-
-enum a2xx_tcr_perfcount_select {
-	DGMMPD_IPMUX0_STALL = 0,
-	DGMMPD_IPMUX_ALL_STALL = 4,
-	OPMUX0_L2_WRITES = 5,
-};
-
-enum a2xx_tp_perfcount_select {
-	POINT_QUADS = 0,
-	BILIN_QUADS = 1,
-	ANISO_QUADS = 2,
-	MIP_QUADS = 3,
-	VOL_QUADS = 4,
-	MIP_VOL_QUADS = 5,
-	MIP_ANISO_QUADS = 6,
-	VOL_ANISO_QUADS = 7,
-	ANISO_2_1_QUADS = 8,
-	ANISO_4_1_QUADS = 9,
-	ANISO_6_1_QUADS = 10,
-	ANISO_8_1_QUADS = 11,
-	ANISO_10_1_QUADS = 12,
-	ANISO_12_1_QUADS = 13,
-	ANISO_14_1_QUADS = 14,
-	ANISO_16_1_QUADS = 15,
-	MIP_VOL_ANISO_QUADS = 16,
-	ALIGN_2_QUADS = 17,
-	ALIGN_4_QUADS = 18,
-	PIX_0_QUAD = 19,
-	PIX_1_QUAD = 20,
-	PIX_2_QUAD = 21,
-	PIX_3_QUAD = 22,
-	PIX_4_QUAD = 23,
-	TP_MIPMAP_LOD0 = 24,
-	TP_MIPMAP_LOD1 = 25,
-	TP_MIPMAP_LOD2 = 26,
-	TP_MIPMAP_LOD3 = 27,
-	TP_MIPMAP_LOD4 = 28,
-	TP_MIPMAP_LOD5 = 29,
-	TP_MIPMAP_LOD6 = 30,
-	TP_MIPMAP_LOD7 = 31,
-	TP_MIPMAP_LOD8 = 32,
-	TP_MIPMAP_LOD9 = 33,
-	TP_MIPMAP_LOD10 = 34,
-	TP_MIPMAP_LOD11 = 35,
-	TP_MIPMAP_LOD12 = 36,
-	TP_MIPMAP_LOD13 = 37,
-	TP_MIPMAP_LOD14 = 38,
-};
-
-enum a2xx_tcm_perfcount_select {
-	QUAD0_RD_LAT_FIFO_EMPTY = 0,
-	QUAD0_RD_LAT_FIFO_4TH_FULL = 3,
-	QUAD0_RD_LAT_FIFO_HALF_FULL = 4,
-	QUAD0_RD_LAT_FIFO_FULL = 5,
-	QUAD0_RD_LAT_FIFO_LT_4TH_FULL = 6,
-	READ_STARVED_QUAD0 = 28,
-	READ_STARVED = 32,
-	READ_STALLED_QUAD0 = 33,
-	READ_STALLED = 37,
-	VALID_READ_QUAD0 = 38,
-	TC_TP_STARVED_QUAD0 = 42,
-	TC_TP_STARVED = 46,
-};
-
-enum a2xx_tcf_perfcount_select {
-	VALID_CYCLES = 0,
-	SINGLE_PHASES = 1,
-	ANISO_PHASES = 2,
-	MIP_PHASES = 3,
-	VOL_PHASES = 4,
-	MIP_VOL_PHASES = 5,
-	MIP_ANISO_PHASES = 6,
-	VOL_ANISO_PHASES = 7,
-	ANISO_2_1_PHASES = 8,
-	ANISO_4_1_PHASES = 9,
-	ANISO_6_1_PHASES = 10,
-	ANISO_8_1_PHASES = 11,
-	ANISO_10_1_PHASES = 12,
-	ANISO_12_1_PHASES = 13,
-	ANISO_14_1_PHASES = 14,
-	ANISO_16_1_PHASES = 15,
-	MIP_VOL_ANISO_PHASES = 16,
-	ALIGN_2_PHASES = 17,
-	ALIGN_4_PHASES = 18,
-	TPC_BUSY = 19,
-	TPC_STALLED = 20,
-	TPC_STARVED = 21,
-	TPC_WORKING = 22,
-	TPC_WALKER_BUSY = 23,
-	TPC_WALKER_STALLED = 24,
-	TPC_WALKER_WORKING = 25,
-	TPC_ALIGNER_BUSY = 26,
-	TPC_ALIGNER_STALLED = 27,
-	TPC_ALIGNER_STALLED_BY_BLEND = 28,
-	TPC_ALIGNER_STALLED_BY_CACHE = 29,
-	TPC_ALIGNER_WORKING = 30,
-	TPC_BLEND_BUSY = 31,
-	TPC_BLEND_SYNC = 32,
-	TPC_BLEND_STARVED = 33,
-	TPC_BLEND_WORKING = 34,
-	OPCODE_0x00 = 35,
-	OPCODE_0x01 = 36,
-	OPCODE_0x04 = 37,
-	OPCODE_0x10 = 38,
-	OPCODE_0x11 = 39,
-	OPCODE_0x12 = 40,
-	OPCODE_0x13 = 41,
-	OPCODE_0x18 = 42,
-	OPCODE_0x19 = 43,
-	OPCODE_0x1A = 44,
-	OPCODE_OTHER = 45,
-	IN_FIFO_0_EMPTY = 56,
-	IN_FIFO_0_LT_HALF_FULL = 57,
-	IN_FIFO_0_HALF_FULL = 58,
-	IN_FIFO_0_FULL = 59,
-	IN_FIFO_TPC_EMPTY = 72,
-	IN_FIFO_TPC_LT_HALF_FULL = 73,
-	IN_FIFO_TPC_HALF_FULL = 74,
-	IN_FIFO_TPC_FULL = 75,
-	TPC_TC_XFC = 76,
-	TPC_TC_STATE = 77,
-	TC_STALL = 78,
-	QUAD0_TAPS = 79,
-	QUADS = 83,
-	TCA_SYNC_STALL = 84,
-	TAG_STALL = 85,
-	TCB_SYNC_STALL = 88,
-	TCA_VALID = 89,
-	PROBES_VALID = 90,
-	MISS_STALL = 91,
-	FETCH_FIFO_STALL = 92,
-	TCO_STALL = 93,
-	ANY_STALL = 94,
-	TAG_MISSES = 95,
-	TAG_HITS = 96,
-	SUB_TAG_MISSES = 97,
-	SET0_INVALIDATES = 98,
-	SET1_INVALIDATES = 99,
-	SET2_INVALIDATES = 100,
-	SET3_INVALIDATES = 101,
-	SET0_TAG_MISSES = 102,
-	SET1_TAG_MISSES = 103,
-	SET2_TAG_MISSES = 104,
-	SET3_TAG_MISSES = 105,
-	SET0_TAG_HITS = 106,
-	SET1_TAG_HITS = 107,
-	SET2_TAG_HITS = 108,
-	SET3_TAG_HITS = 109,
-	SET0_SUB_TAG_MISSES = 110,
-	SET1_SUB_TAG_MISSES = 111,
-	SET2_SUB_TAG_MISSES = 112,
-	SET3_SUB_TAG_MISSES = 113,
-	SET0_EVICT1 = 114,
-	SET0_EVICT2 = 115,
-	SET0_EVICT3 = 116,
-	SET0_EVICT4 = 117,
-	SET0_EVICT5 = 118,
-	SET0_EVICT6 = 119,
-	SET0_EVICT7 = 120,
-	SET0_EVICT8 = 121,
-	SET1_EVICT1 = 130,
-	SET1_EVICT2 = 131,
-	SET1_EVICT3 = 132,
-	SET1_EVICT4 = 133,
-	SET1_EVICT5 = 134,
-	SET1_EVICT6 = 135,
-	SET1_EVICT7 = 136,
-	SET1_EVICT8 = 137,
-	SET2_EVICT1 = 146,
-	SET2_EVICT2 = 147,
-	SET2_EVICT3 = 148,
-	SET2_EVICT4 = 149,
-	SET2_EVICT5 = 150,
-	SET2_EVICT6 = 151,
-	SET2_EVICT7 = 152,
-	SET2_EVICT8 = 153,
-	SET3_EVICT1 = 162,
-	SET3_EVICT2 = 163,
-	SET3_EVICT3 = 164,
-	SET3_EVICT4 = 165,
-	SET3_EVICT5 = 166,
-	SET3_EVICT6 = 167,
-	SET3_EVICT7 = 168,
-	SET3_EVICT8 = 169,
-	FF_EMPTY = 178,
-	FF_LT_HALF_FULL = 179,
-	FF_HALF_FULL = 180,
-	FF_FULL = 181,
-	FF_XFC = 182,
-	FF_STALLED = 183,
-	FG_MASKS = 184,
-	FG_LEFT_MASKS = 185,
-	FG_LEFT_MASK_STALLED = 186,
-	FG_LEFT_NOT_DONE_STALL = 187,
-	FG_LEFT_FG_STALL = 188,
-	FG_LEFT_SECTORS = 189,
-	FG0_REQUESTS = 195,
-	FG0_STALLED = 196,
-	MEM_REQ512 = 199,
-	MEM_REQ_SENT = 200,
-	MEM_LOCAL_READ_REQ = 202,
-	TC0_MH_STALLED = 203,
-};
-
-enum a2xx_sq_perfcnt_select {
-	SQ_PIXEL_VECTORS_SUB = 0,
-	SQ_VERTEX_VECTORS_SUB = 1,
-	SQ_ALU0_ACTIVE_VTX_SIMD0 = 2,
-	SQ_ALU1_ACTIVE_VTX_SIMD0 = 3,
-	SQ_ALU0_ACTIVE_PIX_SIMD0 = 4,
-	SQ_ALU1_ACTIVE_PIX_SIMD0 = 5,
-	SQ_ALU0_ACTIVE_VTX_SIMD1 = 6,
-	SQ_ALU1_ACTIVE_VTX_SIMD1 = 7,
-	SQ_ALU0_ACTIVE_PIX_SIMD1 = 8,
-	SQ_ALU1_ACTIVE_PIX_SIMD1 = 9,
-	SQ_EXPORT_CYCLES = 10,
-	SQ_ALU_CST_WRITTEN = 11,
-	SQ_TEX_CST_WRITTEN = 12,
-	SQ_ALU_CST_STALL = 13,
-	SQ_ALU_TEX_STALL = 14,
-	SQ_INST_WRITTEN = 15,
-	SQ_BOOLEAN_WRITTEN = 16,
-	SQ_LOOPS_WRITTEN = 17,
-	SQ_PIXEL_SWAP_IN = 18,
-	SQ_PIXEL_SWAP_OUT = 19,
-	SQ_VERTEX_SWAP_IN = 20,
-	SQ_VERTEX_SWAP_OUT = 21,
-	SQ_ALU_VTX_INST_ISSUED = 22,
-	SQ_TEX_VTX_INST_ISSUED = 23,
-	SQ_VC_VTX_INST_ISSUED = 24,
-	SQ_CF_VTX_INST_ISSUED = 25,
-	SQ_ALU_PIX_INST_ISSUED = 26,
-	SQ_TEX_PIX_INST_ISSUED = 27,
-	SQ_VC_PIX_INST_ISSUED = 28,
-	SQ_CF_PIX_INST_ISSUED = 29,
-	SQ_ALU0_FIFO_EMPTY_SIMD0 = 30,
-	SQ_ALU1_FIFO_EMPTY_SIMD0 = 31,
-	SQ_ALU0_FIFO_EMPTY_SIMD1 = 32,
-	SQ_ALU1_FIFO_EMPTY_SIMD1 = 33,
-	SQ_ALU_NOPS = 34,
-	SQ_PRED_SKIP = 35,
-	SQ_SYNC_ALU_STALL_SIMD0_VTX = 36,
-	SQ_SYNC_ALU_STALL_SIMD1_VTX = 37,
-	SQ_SYNC_TEX_STALL_VTX = 38,
-	SQ_SYNC_VC_STALL_VTX = 39,
-	SQ_CONSTANTS_USED_SIMD0 = 40,
-	SQ_CONSTANTS_SENT_SP_SIMD0 = 41,
-	SQ_GPR_STALL_VTX = 42,
-	SQ_GPR_STALL_PIX = 43,
-	SQ_VTX_RS_STALL = 44,
-	SQ_PIX_RS_STALL = 45,
-	SQ_SX_PC_FULL = 46,
-	SQ_SX_EXP_BUFF_FULL = 47,
-	SQ_SX_POS_BUFF_FULL = 48,
-	SQ_INTERP_QUADS = 49,
-	SQ_INTERP_ACTIVE = 50,
-	SQ_IN_PIXEL_STALL = 51,
-	SQ_IN_VTX_STALL = 52,
-	SQ_VTX_CNT = 53,
-	SQ_VTX_VECTOR2 = 54,
-	SQ_VTX_VECTOR3 = 55,
-	SQ_VTX_VECTOR4 = 56,
-	SQ_PIXEL_VECTOR1 = 57,
-	SQ_PIXEL_VECTOR23 = 58,
-	SQ_PIXEL_VECTOR4 = 59,
-	SQ_CONSTANTS_USED_SIMD1 = 60,
-	SQ_CONSTANTS_SENT_SP_SIMD1 = 61,
-	SQ_SX_MEM_EXP_FULL = 62,
-	SQ_ALU0_ACTIVE_VTX_SIMD2 = 63,
-	SQ_ALU1_ACTIVE_VTX_SIMD2 = 64,
-	SQ_ALU0_ACTIVE_PIX_SIMD2 = 65,
-	SQ_ALU1_ACTIVE_PIX_SIMD2 = 66,
-	SQ_ALU0_ACTIVE_VTX_SIMD3 = 67,
-	SQ_PERFCOUNT_VTX_QUAL_TP_DONE = 68,
-	SQ_ALU0_ACTIVE_PIX_SIMD3 = 69,
-	SQ_PERFCOUNT_PIX_QUAL_TP_DONE = 70,
-	SQ_ALU0_FIFO_EMPTY_SIMD2 = 71,
-	SQ_ALU1_FIFO_EMPTY_SIMD2 = 72,
-	SQ_ALU0_FIFO_EMPTY_SIMD3 = 73,
-	SQ_ALU1_FIFO_EMPTY_SIMD3 = 74,
-	SQ_SYNC_ALU_STALL_SIMD2_VTX = 75,
-	SQ_PERFCOUNT_VTX_POP_THREAD = 76,
-	SQ_SYNC_ALU_STALL_SIMD0_PIX = 77,
-	SQ_SYNC_ALU_STALL_SIMD1_PIX = 78,
-	SQ_SYNC_ALU_STALL_SIMD2_PIX = 79,
-	SQ_PERFCOUNT_PIX_POP_THREAD = 80,
-	SQ_SYNC_TEX_STALL_PIX = 81,
-	SQ_SYNC_VC_STALL_PIX = 82,
-	SQ_CONSTANTS_USED_SIMD2 = 83,
-	SQ_CONSTANTS_SENT_SP_SIMD2 = 84,
-	SQ_PERFCOUNT_VTX_DEALLOC_ACK = 85,
-	SQ_PERFCOUNT_PIX_DEALLOC_ACK = 86,
-	SQ_ALU0_FIFO_FULL_SIMD0 = 87,
-	SQ_ALU1_FIFO_FULL_SIMD0 = 88,
-	SQ_ALU0_FIFO_FULL_SIMD1 = 89,
-	SQ_ALU1_FIFO_FULL_SIMD1 = 90,
-	SQ_ALU0_FIFO_FULL_SIMD2 = 91,
-	SQ_ALU1_FIFO_FULL_SIMD2 = 92,
-	SQ_ALU0_FIFO_FULL_SIMD3 = 93,
-	SQ_ALU1_FIFO_FULL_SIMD3 = 94,
-	VC_PERF_STATIC = 95,
-	VC_PERF_STALLED = 96,
-	VC_PERF_STARVED = 97,
-	VC_PERF_SEND = 98,
-	VC_PERF_ACTUAL_STARVED = 99,
-	PIXEL_THREAD_0_ACTIVE = 100,
-	VERTEX_THREAD_0_ACTIVE = 101,
-	PIXEL_THREAD_0_NUMBER = 102,
-	VERTEX_THREAD_0_NUMBER = 103,
-	VERTEX_EVENT_NUMBER = 104,
-	PIXEL_EVENT_NUMBER = 105,
-	PTRBUFF_EF_PUSH = 106,
-	PTRBUFF_EF_POP_EVENT = 107,
-	PTRBUFF_EF_POP_NEW_VTX = 108,
-	PTRBUFF_EF_POP_DEALLOC = 109,
-	PTRBUFF_EF_POP_PVECTOR = 110,
-	PTRBUFF_EF_POP_PVECTOR_X = 111,
-	PTRBUFF_EF_POP_PVECTOR_VNZ = 112,
-	PTRBUFF_PB_DEALLOC = 113,
-	PTRBUFF_PI_STATE_PPB_POP = 114,
-	PTRBUFF_PI_RTR = 115,
-	PTRBUFF_PI_READ_EN = 116,
-	PTRBUFF_PI_BUFF_SWAP = 117,
-	PTRBUFF_SQ_FREE_BUFF = 118,
-	PTRBUFF_SQ_DEC = 119,
-	PTRBUFF_SC_VALID_CNTL_EVENT = 120,
-	PTRBUFF_SC_VALID_IJ_XFER = 121,
-	PTRBUFF_SC_NEW_VECTOR_1_Q = 122,
-	PTRBUFF_QUAL_NEW_VECTOR = 123,
-	PTRBUFF_QUAL_EVENT = 124,
-	PTRBUFF_END_BUFFER = 125,
-	PTRBUFF_FILL_QUAD = 126,
-	VERTS_WRITTEN_SPI = 127,
-	TP_FETCH_INSTR_EXEC = 128,
-	TP_FETCH_INSTR_REQ = 129,
-	TP_DATA_RETURN = 130,
-	SPI_WRITE_CYCLES_SP = 131,
-	SPI_WRITES_SP = 132,
-	SP_ALU_INSTR_EXEC = 133,
-	SP_CONST_ADDR_TO_SQ = 134,
-	SP_PRED_KILLS_TO_SQ = 135,
-	SP_EXPORT_CYCLES_TO_SX = 136,
-	SP_EXPORTS_TO_SX = 137,
-	SQ_CYCLES_ELAPSED = 138,
-	SQ_TCFS_OPT_ALLOC_EXEC = 139,
-	SQ_TCFS_NO_OPT_ALLOC = 140,
-	SQ_ALU0_NO_OPT_ALLOC = 141,
-	SQ_ALU1_NO_OPT_ALLOC = 142,
-	SQ_TCFS_ARB_XFC_CNT = 143,
-	SQ_ALU0_ARB_XFC_CNT = 144,
-	SQ_ALU1_ARB_XFC_CNT = 145,
-	SQ_TCFS_CFS_UPDATE_CNT = 146,
-	SQ_ALU0_CFS_UPDATE_CNT = 147,
-	SQ_ALU1_CFS_UPDATE_CNT = 148,
-	SQ_VTX_PUSH_THREAD_CNT = 149,
-	SQ_VTX_POP_THREAD_CNT = 150,
-	SQ_PIX_PUSH_THREAD_CNT = 151,
-	SQ_PIX_POP_THREAD_CNT = 152,
-	SQ_PIX_TOTAL = 153,
-	SQ_PIX_KILLED = 154,
-};
-
-enum a2xx_sx_perfcnt_select {
-	SX_EXPORT_VECTORS = 0,
-	SX_DUMMY_QUADS = 1,
-	SX_ALPHA_FAIL = 2,
-	SX_RB_QUAD_BUSY = 3,
-	SX_RB_COLOR_BUSY = 4,
-	SX_RB_QUAD_STALL = 5,
-	SX_RB_COLOR_STALL = 6,
-};
-
-enum a2xx_rbbm_perfcount1_sel {
-	RBBM1_COUNT = 0,
-	RBBM1_NRT_BUSY = 1,
-	RBBM1_RB_BUSY = 2,
-	RBBM1_SQ_CNTX0_BUSY = 3,
-	RBBM1_SQ_CNTX17_BUSY = 4,
-	RBBM1_VGT_BUSY = 5,
-	RBBM1_VGT_NODMA_BUSY = 6,
-	RBBM1_PA_BUSY = 7,
-	RBBM1_SC_CNTX_BUSY = 8,
-	RBBM1_TPC_BUSY = 9,
-	RBBM1_TC_BUSY = 10,
-	RBBM1_SX_BUSY = 11,
-	RBBM1_CP_COHER_BUSY = 12,
-	RBBM1_CP_NRT_BUSY = 13,
-	RBBM1_GFX_IDLE_STALL = 14,
-	RBBM1_INTERRUPT = 15,
-};
-
-enum a2xx_cp_perfcount_sel {
-	ALWAYS_COUNT = 0,
-	TRANS_FIFO_FULL = 1,
-	TRANS_FIFO_AF = 2,
-	RCIU_PFPTRANS_WAIT = 3,
-	RCIU_NRTTRANS_WAIT = 6,
-	CSF_NRT_READ_WAIT = 8,
-	CSF_I1_FIFO_FULL = 9,
-	CSF_I2_FIFO_FULL = 10,
-	CSF_ST_FIFO_FULL = 11,
-	CSF_RING_ROQ_FULL = 13,
-	CSF_I1_ROQ_FULL = 14,
-	CSF_I2_ROQ_FULL = 15,
-	CSF_ST_ROQ_FULL = 16,
-	MIU_TAG_MEM_FULL = 18,
-	MIU_WRITECLEAN = 19,
-	MIU_NRT_WRITE_STALLED = 22,
-	MIU_NRT_READ_STALLED = 23,
-	ME_WRITE_CONFIRM_FIFO_FULL = 24,
-	ME_VS_DEALLOC_FIFO_FULL = 25,
-	ME_PS_DEALLOC_FIFO_FULL = 26,
-	ME_REGS_VS_EVENT_FIFO_FULL = 27,
-	ME_REGS_PS_EVENT_FIFO_FULL = 28,
-	ME_REGS_CF_EVENT_FIFO_FULL = 29,
-	ME_MICRO_RB_STARVED = 30,
-	ME_MICRO_I1_STARVED = 31,
-	ME_MICRO_I2_STARVED = 32,
-	ME_MICRO_ST_STARVED = 33,
-	RCIU_RBBM_DWORD_SENT = 40,
-	ME_BUSY_CLOCKS = 41,
-	ME_WAIT_CONTEXT_AVAIL = 42,
-	PFP_TYPE0_PACKET = 43,
-	PFP_TYPE3_PACKET = 44,
-	CSF_RB_WPTR_NEQ_RPTR = 45,
-	CSF_I1_SIZE_NEQ_ZERO = 46,
-	CSF_I2_SIZE_NEQ_ZERO = 47,
-	CSF_RBI1I2_FETCHING = 48,
-};
-
-enum a2xx_rb_perfcnt_select {
-	RBPERF_CNTX_BUSY = 0,
-	RBPERF_CNTX_BUSY_MAX = 1,
-	RBPERF_SX_QUAD_STARVED = 2,
-	RBPERF_SX_QUAD_STARVED_MAX = 3,
-	RBPERF_GA_GC_CH0_SYS_REQ = 4,
-	RBPERF_GA_GC_CH0_SYS_REQ_MAX = 5,
-	RBPERF_GA_GC_CH1_SYS_REQ = 6,
-	RBPERF_GA_GC_CH1_SYS_REQ_MAX = 7,
-	RBPERF_MH_STARVED = 8,
-	RBPERF_MH_STARVED_MAX = 9,
-	RBPERF_AZ_BC_COLOR_BUSY = 10,
-	RBPERF_AZ_BC_COLOR_BUSY_MAX = 11,
-	RBPERF_AZ_BC_Z_BUSY = 12,
-	RBPERF_AZ_BC_Z_BUSY_MAX = 13,
-	RBPERF_RB_SC_TILE_RTR_N = 14,
-	RBPERF_RB_SC_TILE_RTR_N_MAX = 15,
-	RBPERF_RB_SC_SAMP_RTR_N = 16,
-	RBPERF_RB_SC_SAMP_RTR_N_MAX = 17,
-	RBPERF_RB_SX_QUAD_RTR_N = 18,
-	RBPERF_RB_SX_QUAD_RTR_N_MAX = 19,
-	RBPERF_RB_SX_COLOR_RTR_N = 20,
-	RBPERF_RB_SX_COLOR_RTR_N_MAX = 21,
-	RBPERF_RB_SC_SAMP_LZ_BUSY = 22,
-	RBPERF_RB_SC_SAMP_LZ_BUSY_MAX = 23,
-	RBPERF_ZXP_STALL = 24,
-	RBPERF_ZXP_STALL_MAX = 25,
-	RBPERF_EVENT_PENDING = 26,
-	RBPERF_EVENT_PENDING_MAX = 27,
-	RBPERF_RB_MH_VALID = 28,
-	RBPERF_RB_MH_VALID_MAX = 29,
-	RBPERF_SX_RB_QUAD_SEND = 30,
-	RBPERF_SX_RB_COLOR_SEND = 31,
-	RBPERF_SC_RB_TILE_SEND = 32,
-	RBPERF_SC_RB_SAMPLE_SEND = 33,
-	RBPERF_SX_RB_MEM_EXPORT = 34,
-	RBPERF_SX_RB_QUAD_EVENT = 35,
-	RBPERF_SC_RB_TILE_EVENT_FILTERED = 36,
-	RBPERF_SC_RB_TILE_EVENT_ALL = 37,
-	RBPERF_RB_SC_EZ_SEND = 38,
-	RBPERF_RB_SX_INDEX_SEND = 39,
-	RBPERF_GMEM_INTFO_RD = 40,
-	RBPERF_GMEM_INTF1_RD = 41,
-	RBPERF_GMEM_INTFO_WR = 42,
-	RBPERF_GMEM_INTF1_WR = 43,
-	RBPERF_RB_CP_CONTEXT_DONE = 44,
-	RBPERF_RB_CP_CACHE_FLUSH = 45,
-	RBPERF_ZPASS_DONE = 46,
-	RBPERF_ZCMD_VALID = 47,
-	RBPERF_CCMD_VALID = 48,
-	RBPERF_ACCUM_GRANT = 49,
-	RBPERF_ACCUM_C0_GRANT = 50,
-	RBPERF_ACCUM_C1_GRANT = 51,
-	RBPERF_ACCUM_FULL_BE_WR = 52,
-	RBPERF_ACCUM_REQUEST_NO_GRANT = 53,
-	RBPERF_ACCUM_TIMEOUT_PULSE = 54,
-	RBPERF_ACCUM_LIN_TIMEOUT_PULSE = 55,
-	RBPERF_ACCUM_CAM_HIT_FLUSHING = 56,
-};
-
-enum a2xx_mh_perfcnt_select {
-	CP_R0_REQUESTS = 0,
-	CP_R1_REQUESTS = 1,
-	CP_R2_REQUESTS = 2,
-	CP_R3_REQUESTS = 3,
-	CP_R4_REQUESTS = 4,
-	CP_TOTAL_READ_REQUESTS = 5,
-	CP_TOTAL_WRITE_REQUESTS = 6,
-	CP_TOTAL_REQUESTS = 7,
-	CP_DATA_BYTES_WRITTEN = 8,
-	CP_WRITE_CLEAN_RESPONSES = 9,
-	CP_R0_READ_BURSTS_RECEIVED = 10,
-	CP_R1_READ_BURSTS_RECEIVED = 11,
-	CP_R2_READ_BURSTS_RECEIVED = 12,
-	CP_R3_READ_BURSTS_RECEIVED = 13,
-	CP_R4_READ_BURSTS_RECEIVED = 14,
-	CP_TOTAL_READ_BURSTS_RECEIVED = 15,
-	CP_R0_DATA_BEATS_READ = 16,
-	CP_R1_DATA_BEATS_READ = 17,
-	CP_R2_DATA_BEATS_READ = 18,
-	CP_R3_DATA_BEATS_READ = 19,
-	CP_R4_DATA_BEATS_READ = 20,
-	CP_TOTAL_DATA_BEATS_READ = 21,
-	VGT_R0_REQUESTS = 22,
-	VGT_R1_REQUESTS = 23,
-	VGT_TOTAL_REQUESTS = 24,
-	VGT_R0_READ_BURSTS_RECEIVED = 25,
-	VGT_R1_READ_BURSTS_RECEIVED = 26,
-	VGT_TOTAL_READ_BURSTS_RECEIVED = 27,
-	VGT_R0_DATA_BEATS_READ = 28,
-	VGT_R1_DATA_BEATS_READ = 29,
-	VGT_TOTAL_DATA_BEATS_READ = 30,
-	TC_TOTAL_REQUESTS = 31,
-	TC_ROQ_REQUESTS = 32,
-	TC_INFO_SENT = 33,
-	TC_READ_BURSTS_RECEIVED = 34,
-	TC_DATA_BEATS_READ = 35,
-	TCD_BURSTS_READ = 36,
-	RB_REQUESTS = 37,
-	RB_DATA_BYTES_WRITTEN = 38,
-	RB_WRITE_CLEAN_RESPONSES = 39,
-	AXI_READ_REQUESTS_ID_0 = 40,
-	AXI_READ_REQUESTS_ID_1 = 41,
-	AXI_READ_REQUESTS_ID_2 = 42,
-	AXI_READ_REQUESTS_ID_3 = 43,
-	AXI_READ_REQUESTS_ID_4 = 44,
-	AXI_READ_REQUESTS_ID_5 = 45,
-	AXI_READ_REQUESTS_ID_6 = 46,
-	AXI_READ_REQUESTS_ID_7 = 47,
-	AXI_TOTAL_READ_REQUESTS = 48,
-	AXI_WRITE_REQUESTS_ID_0 = 49,
-	AXI_WRITE_REQUESTS_ID_1 = 50,
-	AXI_WRITE_REQUESTS_ID_2 = 51,
-	AXI_WRITE_REQUESTS_ID_3 = 52,
-	AXI_WRITE_REQUESTS_ID_4 = 53,
-	AXI_WRITE_REQUESTS_ID_5 = 54,
-	AXI_WRITE_REQUESTS_ID_6 = 55,
-	AXI_WRITE_REQUESTS_ID_7 = 56,
-	AXI_TOTAL_WRITE_REQUESTS = 57,
-	AXI_TOTAL_REQUESTS_ID_0 = 58,
-	AXI_TOTAL_REQUESTS_ID_1 = 59,
-	AXI_TOTAL_REQUESTS_ID_2 = 60,
-	AXI_TOTAL_REQUESTS_ID_3 = 61,
-	AXI_TOTAL_REQUESTS_ID_4 = 62,
-	AXI_TOTAL_REQUESTS_ID_5 = 63,
-	AXI_TOTAL_REQUESTS_ID_6 = 64,
-	AXI_TOTAL_REQUESTS_ID_7 = 65,
-	AXI_TOTAL_REQUESTS = 66,
-	AXI_READ_CHANNEL_BURSTS_ID_0 = 67,
-	AXI_READ_CHANNEL_BURSTS_ID_1 = 68,
-	AXI_READ_CHANNEL_BURSTS_ID_2 = 69,
-	AXI_READ_CHANNEL_BURSTS_ID_3 = 70,
-	AXI_READ_CHANNEL_BURSTS_ID_4 = 71,
-	AXI_READ_CHANNEL_BURSTS_ID_5 = 72,
-	AXI_READ_CHANNEL_BURSTS_ID_6 = 73,
-	AXI_READ_CHANNEL_BURSTS_ID_7 = 74,
-	AXI_READ_CHANNEL_TOTAL_BURSTS = 75,
-	AXI_READ_CHANNEL_DATA_BEATS_READ_ID_0 = 76,
-	AXI_READ_CHANNEL_DATA_BEATS_READ_ID_1 = 77,
-	AXI_READ_CHANNEL_DATA_BEATS_READ_ID_2 = 78,
-	AXI_READ_CHANNEL_DATA_BEATS_READ_ID_3 = 79,
-	AXI_READ_CHANNEL_DATA_BEATS_READ_ID_4 = 80,
-	AXI_READ_CHANNEL_DATA_BEATS_READ_ID_5 = 81,
-	AXI_READ_CHANNEL_DATA_BEATS_READ_ID_6 = 82,
-	AXI_READ_CHANNEL_DATA_BEATS_READ_ID_7 = 83,
-	AXI_READ_CHANNEL_TOTAL_DATA_BEATS_READ = 84,
-	AXI_WRITE_CHANNEL_BURSTS_ID_0 = 85,
-	AXI_WRITE_CHANNEL_BURSTS_ID_1 = 86,
-	AXI_WRITE_CHANNEL_BURSTS_ID_2 = 87,
-	AXI_WRITE_CHANNEL_BURSTS_ID_3 = 88,
-	AXI_WRITE_CHANNEL_BURSTS_ID_4 = 89,
-	AXI_WRITE_CHANNEL_BURSTS_ID_5 = 90,
-	AXI_WRITE_CHANNEL_BURSTS_ID_6 = 91,
-	AXI_WRITE_CHANNEL_BURSTS_ID_7 = 92,
-	AXI_WRITE_CHANNEL_TOTAL_BURSTS = 93,
-	AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_0 = 94,
-	AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_1 = 95,
-	AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_2 = 96,
-	AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_3 = 97,
-	AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_4 = 98,
-	AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_5 = 99,
-	AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_6 = 100,
-	AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_7 = 101,
-	AXI_WRITE_CHANNEL_TOTAL_DATA_BYTES_WRITTEN = 102,
-	AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_0 = 103,
-	AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_1 = 104,
-	AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_2 = 105,
-	AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_3 = 106,
-	AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_4 = 107,
-	AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_5 = 108,
-	AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_6 = 109,
-	AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_7 = 110,
-	AXI_WRITE_RESPONSE_CHANNEL_TOTAL_RESPONSES = 111,
-	TOTAL_MMU_MISSES = 112,
-	MMU_READ_MISSES = 113,
-	MMU_WRITE_MISSES = 114,
-	TOTAL_MMU_HITS = 115,
-	MMU_READ_HITS = 116,
-	MMU_WRITE_HITS = 117,
-	SPLIT_MODE_TC_HITS = 118,
-	SPLIT_MODE_TC_MISSES = 119,
-	SPLIT_MODE_NON_TC_HITS = 120,
-	SPLIT_MODE_NON_TC_MISSES = 121,
-	STALL_AWAITING_TLB_MISS_FETCH = 122,
-	MMU_TLB_MISS_READ_BURSTS_RECEIVED = 123,
-	MMU_TLB_MISS_DATA_BEATS_READ = 124,
-	CP_CYCLES_HELD_OFF = 125,
-	VGT_CYCLES_HELD_OFF = 126,
-	TC_CYCLES_HELD_OFF = 127,
-	TC_ROQ_CYCLES_HELD_OFF = 128,
-	TC_CYCLES_HELD_OFF_TCD_FULL = 129,
-	RB_CYCLES_HELD_OFF = 130,
-	TOTAL_CYCLES_ANY_CLNT_HELD_OFF = 131,
-	TLB_MISS_CYCLES_HELD_OFF = 132,
-	AXI_READ_REQUEST_HELD_OFF = 133,
-	AXI_WRITE_REQUEST_HELD_OFF = 134,
-	AXI_REQUEST_HELD_OFF = 135,
-	AXI_REQUEST_HELD_OFF_INFLIGHT_LIMIT = 136,
-	AXI_WRITE_DATA_HELD_OFF = 137,
-	CP_SAME_PAGE_BANK_REQUESTS = 138,
-	VGT_SAME_PAGE_BANK_REQUESTS = 139,
-	TC_SAME_PAGE_BANK_REQUESTS = 140,
-	TC_ARB_HOLD_SAME_PAGE_BANK_REQUESTS = 141,
-	RB_SAME_PAGE_BANK_REQUESTS = 142,
-	TOTAL_SAME_PAGE_BANK_REQUESTS = 143,
-	CP_SAME_PAGE_BANK_REQUESTS_KILLED_FAIRNESS_LIMIT = 144,
-	VGT_SAME_PAGE_BANK_REQUESTS_KILLED_FAIRNESS_LIMIT = 145,
-	TC_SAME_PAGE_BANK_REQUESTS_KILLED_FAIRNESS_LIMIT = 146,
-	RB_SAME_PAGE_BANK_REQUESTS_KILLED_FAIRNESS_LIMIT = 147,
-	TOTAL_SAME_PAGE_BANK_KILLED_FAIRNESS_LIMIT = 148,
-	TOTAL_MH_READ_REQUESTS = 149,
-	TOTAL_MH_WRITE_REQUESTS = 150,
-	TOTAL_MH_REQUESTS = 151,
-	MH_BUSY = 152,
-	CP_NTH_ACCESS_SAME_PAGE_BANK_SEQUENCE = 153,
-	VGT_NTH_ACCESS_SAME_PAGE_BANK_SEQUENCE = 154,
-	TC_NTH_ACCESS_SAME_PAGE_BANK_SEQUENCE = 155,
-	RB_NTH_ACCESS_SAME_PAGE_BANK_SEQUENCE = 156,
-	TC_ROQ_N_VALID_ENTRIES = 157,
-	ARQ_N_ENTRIES = 158,
-	WDB_N_ENTRIES = 159,
-	MH_READ_LATENCY_OUTST_REQ_SUM = 160,
-	MC_READ_LATENCY_OUTST_REQ_SUM = 161,
-	MC_TOTAL_READ_REQUESTS = 162,
-	ELAPSED_CYCLES_MH_GATED_CLK = 163,
-	ELAPSED_CLK_CYCLES = 164,
-	CP_W_16B_REQUESTS = 165,
-	CP_W_32B_REQUESTS = 166,
-	TC_16B_REQUESTS = 167,
-	TC_32B_REQUESTS = 168,
-	PA_REQUESTS = 169,
-	PA_DATA_BYTES_WRITTEN = 170,
-	PA_WRITE_CLEAN_RESPONSES = 171,
-	PA_CYCLES_HELD_OFF = 172,
-	AXI_READ_REQUEST_DATA_BEATS_ID_0 = 173,
-	AXI_READ_REQUEST_DATA_BEATS_ID_1 = 174,
-	AXI_READ_REQUEST_DATA_BEATS_ID_2 = 175,
-	AXI_READ_REQUEST_DATA_BEATS_ID_3 = 176,
-	AXI_READ_REQUEST_DATA_BEATS_ID_4 = 177,
-	AXI_READ_REQUEST_DATA_BEATS_ID_5 = 178,
-	AXI_READ_REQUEST_DATA_BEATS_ID_6 = 179,
-	AXI_READ_REQUEST_DATA_BEATS_ID_7 = 180,
-	AXI_TOTAL_READ_REQUEST_DATA_BEATS = 181,
-};
-
-enum perf_mode_cnt {
-	PERF_STATE_RESET = 0,
-	PERF_STATE_ENABLE = 1,
-	PERF_STATE_FREEZE = 2,
-};
-
-enum adreno_mmu_clnt_beh {
-	BEH_NEVR = 0,
-	BEH_TRAN_RNG = 1,
-	BEH_TRAN_FLT = 2,
-};
-
-enum sq_tex_clamp {
-	SQ_TEX_WRAP = 0,
-	SQ_TEX_MIRROR = 1,
-	SQ_TEX_CLAMP_LAST_TEXEL = 2,
-	SQ_TEX_MIRROR_ONCE_LAST_TEXEL = 3,
-	SQ_TEX_CLAMP_HALF_BORDER = 4,
-	SQ_TEX_MIRROR_ONCE_HALF_BORDER = 5,
-	SQ_TEX_CLAMP_BORDER = 6,
-	SQ_TEX_MIRROR_ONCE_BORDER = 7,
-};
-
-enum sq_tex_swiz {
-	SQ_TEX_X = 0,
-	SQ_TEX_Y = 1,
-	SQ_TEX_Z = 2,
-	SQ_TEX_W = 3,
-	SQ_TEX_ZERO = 4,
-	SQ_TEX_ONE = 5,
-};
-
-enum sq_tex_filter {
-	SQ_TEX_FILTER_POINT = 0,
-	SQ_TEX_FILTER_BILINEAR = 1,
-	SQ_TEX_FILTER_BASEMAP = 2,
-	SQ_TEX_FILTER_USE_FETCH_CONST = 3,
-};
-
-enum sq_tex_aniso_filter {
-	SQ_TEX_ANISO_FILTER_DISABLED = 0,
-	SQ_TEX_ANISO_FILTER_MAX_1_1 = 1,
-	SQ_TEX_ANISO_FILTER_MAX_2_1 = 2,
-	SQ_TEX_ANISO_FILTER_MAX_4_1 = 3,
-	SQ_TEX_ANISO_FILTER_MAX_8_1 = 4,
-	SQ_TEX_ANISO_FILTER_MAX_16_1 = 5,
-	SQ_TEX_ANISO_FILTER_USE_FETCH_CONST = 7,
-};
-
-enum sq_tex_dimension {
-	SQ_TEX_DIMENSION_1D = 0,
-	SQ_TEX_DIMENSION_2D = 1,
-	SQ_TEX_DIMENSION_3D = 2,
-	SQ_TEX_DIMENSION_CUBE = 3,
-};
-
-enum sq_tex_border_color {
-	SQ_TEX_BORDER_COLOR_BLACK = 0,
-	SQ_TEX_BORDER_COLOR_WHITE = 1,
-	SQ_TEX_BORDER_COLOR_ACBYCR_BLACK = 2,
-	SQ_TEX_BORDER_COLOR_ACBCRY_BLACK = 3,
-};
-
-enum sq_tex_sign {
-	SQ_TEX_SIGN_UNSIGNED = 0,
-	SQ_TEX_SIGN_SIGNED = 1,
-	SQ_TEX_SIGN_UNSIGNED_BIASED = 2,
-	SQ_TEX_SIGN_GAMMA = 3,
-};
-
-enum sq_tex_endian {
-	SQ_TEX_ENDIAN_NONE = 0,
-	SQ_TEX_ENDIAN_8IN16 = 1,
-	SQ_TEX_ENDIAN_8IN32 = 2,
-	SQ_TEX_ENDIAN_16IN32 = 3,
-};
-
-enum sq_tex_clamp_policy {
-	SQ_TEX_CLAMP_POLICY_D3D = 0,
-	SQ_TEX_CLAMP_POLICY_OGL = 1,
-};
-
-enum sq_tex_num_format {
-	SQ_TEX_NUM_FORMAT_FRAC = 0,
-	SQ_TEX_NUM_FORMAT_INT = 1,
-};
-
-enum sq_tex_type {
-	SQ_TEX_TYPE_0 = 0,
-	SQ_TEX_TYPE_1 = 1,
-	SQ_TEX_TYPE_2 = 2,
-	SQ_TEX_TYPE_3 = 3,
-};
-
-#define REG_A2XX_RBBM_PATCH_RELEASE				0x00000001
-
-#define REG_A2XX_RBBM_CNTL					0x0000003b
-
-#define REG_A2XX_RBBM_SOFT_RESET				0x0000003c
-
-#define REG_A2XX_CP_PFP_UCODE_ADDR				0x000000c0
-
-#define REG_A2XX_CP_PFP_UCODE_DATA				0x000000c1
-
-#define REG_A2XX_MH_MMU_CONFIG					0x00000040
-#define A2XX_MH_MMU_CONFIG_MMU_ENABLE				0x00000001
-#define A2XX_MH_MMU_CONFIG_SPLIT_MODE_ENABLE			0x00000002
-#define A2XX_MH_MMU_CONFIG_RB_W_CLNT_BEHAVIOR__MASK		0x00000030
-#define A2XX_MH_MMU_CONFIG_RB_W_CLNT_BEHAVIOR__SHIFT		4
-static inline uint32_t A2XX_MH_MMU_CONFIG_RB_W_CLNT_BEHAVIOR(enum adreno_mmu_clnt_beh val)
-{
-	return ((val) << A2XX_MH_MMU_CONFIG_RB_W_CLNT_BEHAVIOR__SHIFT) & A2XX_MH_MMU_CONFIG_RB_W_CLNT_BEHAVIOR__MASK;
-}
-#define A2XX_MH_MMU_CONFIG_CP_W_CLNT_BEHAVIOR__MASK		0x000000c0
-#define A2XX_MH_MMU_CONFIG_CP_W_CLNT_BEHAVIOR__SHIFT		6
-static inline uint32_t A2XX_MH_MMU_CONFIG_CP_W_CLNT_BEHAVIOR(enum adreno_mmu_clnt_beh val)
-{
-	return ((val) << A2XX_MH_MMU_CONFIG_CP_W_CLNT_BEHAVIOR__SHIFT) & A2XX_MH_MMU_CONFIG_CP_W_CLNT_BEHAVIOR__MASK;
-}
-#define A2XX_MH_MMU_CONFIG_CP_R0_CLNT_BEHAVIOR__MASK		0x00000300
-#define A2XX_MH_MMU_CONFIG_CP_R0_CLNT_BEHAVIOR__SHIFT		8
-static inline uint32_t A2XX_MH_MMU_CONFIG_CP_R0_CLNT_BEHAVIOR(enum adreno_mmu_clnt_beh val)
-{
-	return ((val) << A2XX_MH_MMU_CONFIG_CP_R0_CLNT_BEHAVIOR__SHIFT) & A2XX_MH_MMU_CONFIG_CP_R0_CLNT_BEHAVIOR__MASK;
-}
-#define A2XX_MH_MMU_CONFIG_CP_R1_CLNT_BEHAVIOR__MASK		0x00000c00
-#define A2XX_MH_MMU_CONFIG_CP_R1_CLNT_BEHAVIOR__SHIFT		10
-static inline uint32_t A2XX_MH_MMU_CONFIG_CP_R1_CLNT_BEHAVIOR(enum adreno_mmu_clnt_beh val)
-{
-	return ((val) << A2XX_MH_MMU_CONFIG_CP_R1_CLNT_BEHAVIOR__SHIFT) & A2XX_MH_MMU_CONFIG_CP_R1_CLNT_BEHAVIOR__MASK;
-}
-#define A2XX_MH_MMU_CONFIG_CP_R2_CLNT_BEHAVIOR__MASK		0x00003000
-#define A2XX_MH_MMU_CONFIG_CP_R2_CLNT_BEHAVIOR__SHIFT		12
-static inline uint32_t A2XX_MH_MMU_CONFIG_CP_R2_CLNT_BEHAVIOR(enum adreno_mmu_clnt_beh val)
-{
-	return ((val) << A2XX_MH_MMU_CONFIG_CP_R2_CLNT_BEHAVIOR__SHIFT) & A2XX_MH_MMU_CONFIG_CP_R2_CLNT_BEHAVIOR__MASK;
-}
-#define A2XX_MH_MMU_CONFIG_CP_R3_CLNT_BEHAVIOR__MASK		0x0000c000
-#define A2XX_MH_MMU_CONFIG_CP_R3_CLNT_BEHAVIOR__SHIFT		14
-static inline uint32_t A2XX_MH_MMU_CONFIG_CP_R3_CLNT_BEHAVIOR(enum adreno_mmu_clnt_beh val)
-{
-	return ((val) << A2XX_MH_MMU_CONFIG_CP_R3_CLNT_BEHAVIOR__SHIFT) & A2XX_MH_MMU_CONFIG_CP_R3_CLNT_BEHAVIOR__MASK;
-}
-#define A2XX_MH_MMU_CONFIG_CP_R4_CLNT_BEHAVIOR__MASK		0x00030000
-#define A2XX_MH_MMU_CONFIG_CP_R4_CLNT_BEHAVIOR__SHIFT		16
-static inline uint32_t A2XX_MH_MMU_CONFIG_CP_R4_CLNT_BEHAVIOR(enum adreno_mmu_clnt_beh val)
-{
-	return ((val) << A2XX_MH_MMU_CONFIG_CP_R4_CLNT_BEHAVIOR__SHIFT) & A2XX_MH_MMU_CONFIG_CP_R4_CLNT_BEHAVIOR__MASK;
-}
-#define A2XX_MH_MMU_CONFIG_VGT_R0_CLNT_BEHAVIOR__MASK		0x000c0000
-#define A2XX_MH_MMU_CONFIG_VGT_R0_CLNT_BEHAVIOR__SHIFT		18
-static inline uint32_t A2XX_MH_MMU_CONFIG_VGT_R0_CLNT_BEHAVIOR(enum adreno_mmu_clnt_beh val)
-{
-	return ((val) << A2XX_MH_MMU_CONFIG_VGT_R0_CLNT_BEHAVIOR__SHIFT) & A2XX_MH_MMU_CONFIG_VGT_R0_CLNT_BEHAVIOR__MASK;
-}
-#define A2XX_MH_MMU_CONFIG_VGT_R1_CLNT_BEHAVIOR__MASK		0x00300000
-#define A2XX_MH_MMU_CONFIG_VGT_R1_CLNT_BEHAVIOR__SHIFT		20
-static inline uint32_t A2XX_MH_MMU_CONFIG_VGT_R1_CLNT_BEHAVIOR(enum adreno_mmu_clnt_beh val)
-{
-	return ((val) << A2XX_MH_MMU_CONFIG_VGT_R1_CLNT_BEHAVIOR__SHIFT) & A2XX_MH_MMU_CONFIG_VGT_R1_CLNT_BEHAVIOR__MASK;
-}
-#define A2XX_MH_MMU_CONFIG_TC_R_CLNT_BEHAVIOR__MASK		0x00c00000
-#define A2XX_MH_MMU_CONFIG_TC_R_CLNT_BEHAVIOR__SHIFT		22
-static inline uint32_t A2XX_MH_MMU_CONFIG_TC_R_CLNT_BEHAVIOR(enum adreno_mmu_clnt_beh val)
-{
-	return ((val) << A2XX_MH_MMU_CONFIG_TC_R_CLNT_BEHAVIOR__SHIFT) & A2XX_MH_MMU_CONFIG_TC_R_CLNT_BEHAVIOR__MASK;
-}
-#define A2XX_MH_MMU_CONFIG_PA_W_CLNT_BEHAVIOR__MASK		0x03000000
-#define A2XX_MH_MMU_CONFIG_PA_W_CLNT_BEHAVIOR__SHIFT		24
-static inline uint32_t A2XX_MH_MMU_CONFIG_PA_W_CLNT_BEHAVIOR(enum adreno_mmu_clnt_beh val)
-{
-	return ((val) << A2XX_MH_MMU_CONFIG_PA_W_CLNT_BEHAVIOR__SHIFT) & A2XX_MH_MMU_CONFIG_PA_W_CLNT_BEHAVIOR__MASK;
-}
-
-#define REG_A2XX_MH_MMU_VA_RANGE				0x00000041
-#define A2XX_MH_MMU_VA_RANGE_NUM_64KB_REGIONS__MASK		0x00000fff
-#define A2XX_MH_MMU_VA_RANGE_NUM_64KB_REGIONS__SHIFT		0
-static inline uint32_t A2XX_MH_MMU_VA_RANGE_NUM_64KB_REGIONS(uint32_t val)
-{
-	return ((val) << A2XX_MH_MMU_VA_RANGE_NUM_64KB_REGIONS__SHIFT) & A2XX_MH_MMU_VA_RANGE_NUM_64KB_REGIONS__MASK;
-}
-#define A2XX_MH_MMU_VA_RANGE_VA_BASE__MASK			0xfffff000
-#define A2XX_MH_MMU_VA_RANGE_VA_BASE__SHIFT			12
-static inline uint32_t A2XX_MH_MMU_VA_RANGE_VA_BASE(uint32_t val)
-{
-	return ((val) << A2XX_MH_MMU_VA_RANGE_VA_BASE__SHIFT) & A2XX_MH_MMU_VA_RANGE_VA_BASE__MASK;
-}
-
-#define REG_A2XX_MH_MMU_PT_BASE					0x00000042
-
-#define REG_A2XX_MH_MMU_PAGE_FAULT				0x00000043
-
-#define REG_A2XX_MH_MMU_TRAN_ERROR				0x00000044
-
-#define REG_A2XX_MH_MMU_INVALIDATE				0x00000045
-#define A2XX_MH_MMU_INVALIDATE_INVALIDATE_ALL			0x00000001
-#define A2XX_MH_MMU_INVALIDATE_INVALIDATE_TC			0x00000002
-
-#define REG_A2XX_MH_MMU_MPU_BASE				0x00000046
-
-#define REG_A2XX_MH_MMU_MPU_END					0x00000047
-
-#define REG_A2XX_NQWAIT_UNTIL					0x00000394
-
-#define REG_A2XX_RBBM_PERFCOUNTER0_SELECT			0x00000395
-
-#define REG_A2XX_RBBM_PERFCOUNTER1_SELECT			0x00000396
-
-#define REG_A2XX_RBBM_PERFCOUNTER0_LO				0x00000397
-
-#define REG_A2XX_RBBM_PERFCOUNTER0_HI				0x00000398
-
-#define REG_A2XX_RBBM_PERFCOUNTER1_LO				0x00000399
-
-#define REG_A2XX_RBBM_PERFCOUNTER1_HI				0x0000039a
-
-#define REG_A2XX_RBBM_DEBUG					0x0000039b
-
-#define REG_A2XX_RBBM_PM_OVERRIDE1				0x0000039c
-#define A2XX_RBBM_PM_OVERRIDE1_RBBM_AHBCLK_PM_OVERRIDE		0x00000001
-#define A2XX_RBBM_PM_OVERRIDE1_SC_REG_SCLK_PM_OVERRIDE		0x00000002
-#define A2XX_RBBM_PM_OVERRIDE1_SC_SCLK_PM_OVERRIDE		0x00000004
-#define A2XX_RBBM_PM_OVERRIDE1_SP_TOP_SCLK_PM_OVERRIDE		0x00000008
-#define A2XX_RBBM_PM_OVERRIDE1_SP_V0_SCLK_PM_OVERRIDE		0x00000010
-#define A2XX_RBBM_PM_OVERRIDE1_SQ_REG_SCLK_PM_OVERRIDE		0x00000020
-#define A2XX_RBBM_PM_OVERRIDE1_SQ_REG_FIFOS_SCLK_PM_OVERRIDE	0x00000040
-#define A2XX_RBBM_PM_OVERRIDE1_SQ_CONST_MEM_SCLK_PM_OVERRIDE	0x00000080
-#define A2XX_RBBM_PM_OVERRIDE1_SQ_SQ_SCLK_PM_OVERRIDE		0x00000100
-#define A2XX_RBBM_PM_OVERRIDE1_SX_SCLK_PM_OVERRIDE		0x00000200
-#define A2XX_RBBM_PM_OVERRIDE1_SX_REG_SCLK_PM_OVERRIDE		0x00000400
-#define A2XX_RBBM_PM_OVERRIDE1_TCM_TCO_SCLK_PM_OVERRIDE		0x00000800
-#define A2XX_RBBM_PM_OVERRIDE1_TCM_TCM_SCLK_PM_OVERRIDE		0x00001000
-#define A2XX_RBBM_PM_OVERRIDE1_TCM_TCD_SCLK_PM_OVERRIDE		0x00002000
-#define A2XX_RBBM_PM_OVERRIDE1_TCM_REG_SCLK_PM_OVERRIDE		0x00004000
-#define A2XX_RBBM_PM_OVERRIDE1_TPC_TPC_SCLK_PM_OVERRIDE		0x00008000
-#define A2XX_RBBM_PM_OVERRIDE1_TPC_REG_SCLK_PM_OVERRIDE		0x00010000
-#define A2XX_RBBM_PM_OVERRIDE1_TCF_TCA_SCLK_PM_OVERRIDE		0x00020000
-#define A2XX_RBBM_PM_OVERRIDE1_TCF_TCB_SCLK_PM_OVERRIDE		0x00040000
-#define A2XX_RBBM_PM_OVERRIDE1_TCF_TCB_READ_SCLK_PM_OVERRIDE	0x00080000
-#define A2XX_RBBM_PM_OVERRIDE1_TP_TP_SCLK_PM_OVERRIDE		0x00100000
-#define A2XX_RBBM_PM_OVERRIDE1_TP_REG_SCLK_PM_OVERRIDE		0x00200000
-#define A2XX_RBBM_PM_OVERRIDE1_CP_G_SCLK_PM_OVERRIDE		0x00400000
-#define A2XX_RBBM_PM_OVERRIDE1_CP_REG_SCLK_PM_OVERRIDE		0x00800000
-#define A2XX_RBBM_PM_OVERRIDE1_CP_G_REG_SCLK_PM_OVERRIDE	0x01000000
-#define A2XX_RBBM_PM_OVERRIDE1_SPI_SCLK_PM_OVERRIDE		0x02000000
-#define A2XX_RBBM_PM_OVERRIDE1_RB_REG_SCLK_PM_OVERRIDE		0x04000000
-#define A2XX_RBBM_PM_OVERRIDE1_RB_SCLK_PM_OVERRIDE		0x08000000
-#define A2XX_RBBM_PM_OVERRIDE1_MH_MH_SCLK_PM_OVERRIDE		0x10000000
-#define A2XX_RBBM_PM_OVERRIDE1_MH_REG_SCLK_PM_OVERRIDE		0x20000000
-#define A2XX_RBBM_PM_OVERRIDE1_MH_MMU_SCLK_PM_OVERRIDE		0x40000000
-#define A2XX_RBBM_PM_OVERRIDE1_MH_TCROQ_SCLK_PM_OVERRIDE	0x80000000
-
-#define REG_A2XX_RBBM_PM_OVERRIDE2				0x0000039d
-#define A2XX_RBBM_PM_OVERRIDE2_PA_REG_SCLK_PM_OVERRIDE		0x00000001
-#define A2XX_RBBM_PM_OVERRIDE2_PA_PA_SCLK_PM_OVERRIDE		0x00000002
-#define A2XX_RBBM_PM_OVERRIDE2_PA_AG_SCLK_PM_OVERRIDE		0x00000004
-#define A2XX_RBBM_PM_OVERRIDE2_VGT_REG_SCLK_PM_OVERRIDE		0x00000008
-#define A2XX_RBBM_PM_OVERRIDE2_VGT_FIFOS_SCLK_PM_OVERRIDE	0x00000010
-#define A2XX_RBBM_PM_OVERRIDE2_VGT_VGT_SCLK_PM_OVERRIDE		0x00000020
-#define A2XX_RBBM_PM_OVERRIDE2_DEBUG_PERF_SCLK_PM_OVERRIDE	0x00000040
-#define A2XX_RBBM_PM_OVERRIDE2_PERM_SCLK_PM_OVERRIDE		0x00000080
-#define A2XX_RBBM_PM_OVERRIDE2_GC_GA_GMEM0_PM_OVERRIDE		0x00000100
-#define A2XX_RBBM_PM_OVERRIDE2_GC_GA_GMEM1_PM_OVERRIDE		0x00000200
-#define A2XX_RBBM_PM_OVERRIDE2_GC_GA_GMEM2_PM_OVERRIDE		0x00000400
-#define A2XX_RBBM_PM_OVERRIDE2_GC_GA_GMEM3_PM_OVERRIDE		0x00000800
-
-#define REG_A2XX_RBBM_DEBUG_OUT					0x000003a0
-
-#define REG_A2XX_RBBM_DEBUG_CNTL				0x000003a1
-
-#define REG_A2XX_RBBM_READ_ERROR				0x000003b3
-
-#define REG_A2XX_RBBM_INT_CNTL					0x000003b4
-#define A2XX_RBBM_INT_CNTL_RDERR_INT_MASK			0x00000001
-#define A2XX_RBBM_INT_CNTL_DISPLAY_UPDATE_INT_MASK		0x00000002
-#define A2XX_RBBM_INT_CNTL_GUI_IDLE_INT_MASK			0x00080000
-
-#define REG_A2XX_RBBM_INT_STATUS				0x000003b5
-
-#define REG_A2XX_RBBM_INT_ACK					0x000003b6
-
-#define REG_A2XX_MASTER_INT_SIGNAL				0x000003b7
-#define A2XX_MASTER_INT_SIGNAL_MH_INT_STAT			0x00000020
-#define A2XX_MASTER_INT_SIGNAL_SQ_INT_STAT			0x04000000
-#define A2XX_MASTER_INT_SIGNAL_CP_INT_STAT			0x40000000
-#define A2XX_MASTER_INT_SIGNAL_RBBM_INT_STAT			0x80000000
-
-#define REG_A2XX_RBBM_PERIPHID1					0x000003f9
-
-#define REG_A2XX_RBBM_PERIPHID2					0x000003fa
-
-#define REG_A2XX_CP_PERFMON_CNTL				0x00000444
-#define A2XX_CP_PERFMON_CNTL_PERF_MODE_CNT__MASK		0x00000007
-#define A2XX_CP_PERFMON_CNTL_PERF_MODE_CNT__SHIFT		0
-static inline uint32_t A2XX_CP_PERFMON_CNTL_PERF_MODE_CNT(enum perf_mode_cnt val)
-{
-	return ((val) << A2XX_CP_PERFMON_CNTL_PERF_MODE_CNT__SHIFT) & A2XX_CP_PERFMON_CNTL_PERF_MODE_CNT__MASK;
-}
-
-#define REG_A2XX_CP_PERFCOUNTER_SELECT				0x00000445
-
-#define REG_A2XX_CP_PERFCOUNTER_LO				0x00000446
-
-#define REG_A2XX_CP_PERFCOUNTER_HI				0x00000447
-
-#define REG_A2XX_RBBM_STATUS					0x000005d0
-#define A2XX_RBBM_STATUS_CMDFIFO_AVAIL__MASK			0x0000001f
-#define A2XX_RBBM_STATUS_CMDFIFO_AVAIL__SHIFT			0
-static inline uint32_t A2XX_RBBM_STATUS_CMDFIFO_AVAIL(uint32_t val)
-{
-	return ((val) << A2XX_RBBM_STATUS_CMDFIFO_AVAIL__SHIFT) & A2XX_RBBM_STATUS_CMDFIFO_AVAIL__MASK;
-}
-#define A2XX_RBBM_STATUS_TC_BUSY				0x00000020
-#define A2XX_RBBM_STATUS_HIRQ_PENDING				0x00000100
-#define A2XX_RBBM_STATUS_CPRQ_PENDING				0x00000200
-#define A2XX_RBBM_STATUS_CFRQ_PENDING				0x00000400
-#define A2XX_RBBM_STATUS_PFRQ_PENDING				0x00000800
-#define A2XX_RBBM_STATUS_VGT_BUSY_NO_DMA			0x00001000
-#define A2XX_RBBM_STATUS_RBBM_WU_BUSY				0x00004000
-#define A2XX_RBBM_STATUS_CP_NRT_BUSY				0x00010000
-#define A2XX_RBBM_STATUS_MH_BUSY				0x00040000
-#define A2XX_RBBM_STATUS_MH_COHERENCY_BUSY			0x00080000
-#define A2XX_RBBM_STATUS_SX_BUSY				0x00200000
-#define A2XX_RBBM_STATUS_TPC_BUSY				0x00400000
-#define A2XX_RBBM_STATUS_SC_CNTX_BUSY				0x01000000
-#define A2XX_RBBM_STATUS_PA_BUSY				0x02000000
-#define A2XX_RBBM_STATUS_VGT_BUSY				0x04000000
-#define A2XX_RBBM_STATUS_SQ_CNTX17_BUSY				0x08000000
-#define A2XX_RBBM_STATUS_SQ_CNTX0_BUSY				0x10000000
-#define A2XX_RBBM_STATUS_RB_CNTX_BUSY				0x40000000
-#define A2XX_RBBM_STATUS_GUI_ACTIVE				0x80000000
-
-#define REG_A2XX_MH_ARBITER_CONFIG				0x00000a40
-#define A2XX_MH_ARBITER_CONFIG_SAME_PAGE_LIMIT__MASK		0x0000003f
-#define A2XX_MH_ARBITER_CONFIG_SAME_PAGE_LIMIT__SHIFT		0
-static inline uint32_t A2XX_MH_ARBITER_CONFIG_SAME_PAGE_LIMIT(uint32_t val)
-{
-	return ((val) << A2XX_MH_ARBITER_CONFIG_SAME_PAGE_LIMIT__SHIFT) & A2XX_MH_ARBITER_CONFIG_SAME_PAGE_LIMIT__MASK;
-}
-#define A2XX_MH_ARBITER_CONFIG_SAME_PAGE_GRANULARITY		0x00000040
-#define A2XX_MH_ARBITER_CONFIG_L1_ARB_ENABLE			0x00000080
-#define A2XX_MH_ARBITER_CONFIG_L1_ARB_HOLD_ENABLE		0x00000100
-#define A2XX_MH_ARBITER_CONFIG_L2_ARB_CONTROL			0x00000200
-#define A2XX_MH_ARBITER_CONFIG_PAGE_SIZE__MASK			0x00001c00
-#define A2XX_MH_ARBITER_CONFIG_PAGE_SIZE__SHIFT			10
-static inline uint32_t A2XX_MH_ARBITER_CONFIG_PAGE_SIZE(uint32_t val)
-{
-	return ((val) << A2XX_MH_ARBITER_CONFIG_PAGE_SIZE__SHIFT) & A2XX_MH_ARBITER_CONFIG_PAGE_SIZE__MASK;
-}
-#define A2XX_MH_ARBITER_CONFIG_TC_REORDER_ENABLE		0x00002000
-#define A2XX_MH_ARBITER_CONFIG_TC_ARB_HOLD_ENABLE		0x00004000
-#define A2XX_MH_ARBITER_CONFIG_IN_FLIGHT_LIMIT_ENABLE		0x00008000
-#define A2XX_MH_ARBITER_CONFIG_IN_FLIGHT_LIMIT__MASK		0x003f0000
-#define A2XX_MH_ARBITER_CONFIG_IN_FLIGHT_LIMIT__SHIFT		16
-static inline uint32_t A2XX_MH_ARBITER_CONFIG_IN_FLIGHT_LIMIT(uint32_t val)
-{
-	return ((val) << A2XX_MH_ARBITER_CONFIG_IN_FLIGHT_LIMIT__SHIFT) & A2XX_MH_ARBITER_CONFIG_IN_FLIGHT_LIMIT__MASK;
-}
-#define A2XX_MH_ARBITER_CONFIG_CP_CLNT_ENABLE			0x00400000
-#define A2XX_MH_ARBITER_CONFIG_VGT_CLNT_ENABLE			0x00800000
-#define A2XX_MH_ARBITER_CONFIG_TC_CLNT_ENABLE			0x01000000
-#define A2XX_MH_ARBITER_CONFIG_RB_CLNT_ENABLE			0x02000000
-#define A2XX_MH_ARBITER_CONFIG_PA_CLNT_ENABLE			0x04000000
-
-#define REG_A2XX_MH_INTERRUPT_MASK				0x00000a42
-#define A2XX_MH_INTERRUPT_MASK_AXI_READ_ERROR			0x00000001
-#define A2XX_MH_INTERRUPT_MASK_AXI_WRITE_ERROR			0x00000002
-#define A2XX_MH_INTERRUPT_MASK_MMU_PAGE_FAULT			0x00000004
-
-#define REG_A2XX_MH_INTERRUPT_STATUS				0x00000a43
-
-#define REG_A2XX_MH_INTERRUPT_CLEAR				0x00000a44
-
-#define REG_A2XX_MH_CLNT_INTF_CTRL_CONFIG1			0x00000a54
-
-#define REG_A2XX_MH_CLNT_INTF_CTRL_CONFIG2			0x00000a55
-
-#define REG_A2XX_A220_VSC_BIN_SIZE				0x00000c01
-#define A2XX_A220_VSC_BIN_SIZE_WIDTH__MASK			0x0000001f
-#define A2XX_A220_VSC_BIN_SIZE_WIDTH__SHIFT			0
-static inline uint32_t A2XX_A220_VSC_BIN_SIZE_WIDTH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A2XX_A220_VSC_BIN_SIZE_WIDTH__SHIFT) & A2XX_A220_VSC_BIN_SIZE_WIDTH__MASK;
-}
-#define A2XX_A220_VSC_BIN_SIZE_HEIGHT__MASK			0x000003e0
-#define A2XX_A220_VSC_BIN_SIZE_HEIGHT__SHIFT			5
-static inline uint32_t A2XX_A220_VSC_BIN_SIZE_HEIGHT(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A2XX_A220_VSC_BIN_SIZE_HEIGHT__SHIFT) & A2XX_A220_VSC_BIN_SIZE_HEIGHT__MASK;
-}
-
-#define REG_A2XX_VSC_PIPE(i0) (0x00000c06 + 0x3*(i0))
-
-static inline uint32_t REG_A2XX_VSC_PIPE_CONFIG(uint32_t i0) { return 0x00000c06 + 0x3*i0; }
-
-static inline uint32_t REG_A2XX_VSC_PIPE_DATA_ADDRESS(uint32_t i0) { return 0x00000c07 + 0x3*i0; }
-
-static inline uint32_t REG_A2XX_VSC_PIPE_DATA_LENGTH(uint32_t i0) { return 0x00000c08 + 0x3*i0; }
-
-#define REG_A2XX_PC_DEBUG_CNTL					0x00000c38
-
-#define REG_A2XX_PC_DEBUG_DATA					0x00000c39
-
-#define REG_A2XX_PA_SC_VIZ_QUERY_STATUS				0x00000c44
-
-#define REG_A2XX_GRAS_DEBUG_CNTL				0x00000c80
-
-#define REG_A2XX_PA_SU_DEBUG_CNTL				0x00000c80
-
-#define REG_A2XX_GRAS_DEBUG_DATA				0x00000c81
-
-#define REG_A2XX_PA_SU_DEBUG_DATA				0x00000c81
-
-#define REG_A2XX_PA_SU_FACE_DATA				0x00000c86
-#define A2XX_PA_SU_FACE_DATA_BASE_ADDR__MASK			0xffffffe0
-#define A2XX_PA_SU_FACE_DATA_BASE_ADDR__SHIFT			5
-static inline uint32_t A2XX_PA_SU_FACE_DATA_BASE_ADDR(uint32_t val)
-{
-	return ((val) << A2XX_PA_SU_FACE_DATA_BASE_ADDR__SHIFT) & A2XX_PA_SU_FACE_DATA_BASE_ADDR__MASK;
-}
-
-#define REG_A2XX_SQ_GPR_MANAGEMENT				0x00000d00
-#define A2XX_SQ_GPR_MANAGEMENT_REG_DYNAMIC			0x00000001
-#define A2XX_SQ_GPR_MANAGEMENT_REG_SIZE_PIX__MASK		0x00000ff0
-#define A2XX_SQ_GPR_MANAGEMENT_REG_SIZE_PIX__SHIFT		4
-static inline uint32_t A2XX_SQ_GPR_MANAGEMENT_REG_SIZE_PIX(uint32_t val)
-{
-	return ((val) << A2XX_SQ_GPR_MANAGEMENT_REG_SIZE_PIX__SHIFT) & A2XX_SQ_GPR_MANAGEMENT_REG_SIZE_PIX__MASK;
-}
-#define A2XX_SQ_GPR_MANAGEMENT_REG_SIZE_VTX__MASK		0x000ff000
-#define A2XX_SQ_GPR_MANAGEMENT_REG_SIZE_VTX__SHIFT		12
-static inline uint32_t A2XX_SQ_GPR_MANAGEMENT_REG_SIZE_VTX(uint32_t val)
-{
-	return ((val) << A2XX_SQ_GPR_MANAGEMENT_REG_SIZE_VTX__SHIFT) & A2XX_SQ_GPR_MANAGEMENT_REG_SIZE_VTX__MASK;
-}
-
-#define REG_A2XX_SQ_FLOW_CONTROL				0x00000d01
-
-#define REG_A2XX_SQ_INST_STORE_MANAGMENT			0x00000d02
-#define A2XX_SQ_INST_STORE_MANAGMENT_INST_BASE_PIX__MASK	0x00000fff
-#define A2XX_SQ_INST_STORE_MANAGMENT_INST_BASE_PIX__SHIFT	0
-static inline uint32_t A2XX_SQ_INST_STORE_MANAGMENT_INST_BASE_PIX(uint32_t val)
-{
-	return ((val) << A2XX_SQ_INST_STORE_MANAGMENT_INST_BASE_PIX__SHIFT) & A2XX_SQ_INST_STORE_MANAGMENT_INST_BASE_PIX__MASK;
-}
-#define A2XX_SQ_INST_STORE_MANAGMENT_INST_BASE_VTX__MASK	0x0fff0000
-#define A2XX_SQ_INST_STORE_MANAGMENT_INST_BASE_VTX__SHIFT	16
-static inline uint32_t A2XX_SQ_INST_STORE_MANAGMENT_INST_BASE_VTX(uint32_t val)
-{
-	return ((val) << A2XX_SQ_INST_STORE_MANAGMENT_INST_BASE_VTX__SHIFT) & A2XX_SQ_INST_STORE_MANAGMENT_INST_BASE_VTX__MASK;
-}
-
-#define REG_A2XX_SQ_DEBUG_MISC					0x00000d05
-
-#define REG_A2XX_SQ_INT_CNTL					0x00000d34
-
-#define REG_A2XX_SQ_INT_STATUS					0x00000d35
-
-#define REG_A2XX_SQ_INT_ACK					0x00000d36
-
-#define REG_A2XX_SQ_DEBUG_INPUT_FSM				0x00000dae
-
-#define REG_A2XX_SQ_DEBUG_CONST_MGR_FSM				0x00000daf
-
-#define REG_A2XX_SQ_DEBUG_TP_FSM				0x00000db0
-
-#define REG_A2XX_SQ_DEBUG_FSM_ALU_0				0x00000db1
-
-#define REG_A2XX_SQ_DEBUG_FSM_ALU_1				0x00000db2
-
-#define REG_A2XX_SQ_DEBUG_EXP_ALLOC				0x00000db3
-
-#define REG_A2XX_SQ_DEBUG_PTR_BUFF				0x00000db4
-
-#define REG_A2XX_SQ_DEBUG_GPR_VTX				0x00000db5
-
-#define REG_A2XX_SQ_DEBUG_GPR_PIX				0x00000db6
-
-#define REG_A2XX_SQ_DEBUG_TB_STATUS_SEL				0x00000db7
-
-#define REG_A2XX_SQ_DEBUG_VTX_TB_0				0x00000db8
-
-#define REG_A2XX_SQ_DEBUG_VTX_TB_1				0x00000db9
-
-#define REG_A2XX_SQ_DEBUG_VTX_TB_STATUS_REG			0x00000dba
-
-#define REG_A2XX_SQ_DEBUG_VTX_TB_STATE_MEM			0x00000dbb
-
-#define REG_A2XX_SQ_DEBUG_PIX_TB_0				0x00000dbc
-
-#define REG_A2XX_SQ_DEBUG_PIX_TB_STATUS_REG_0			0x00000dbd
-
-#define REG_A2XX_SQ_DEBUG_PIX_TB_STATUS_REG_1			0x00000dbe
-
-#define REG_A2XX_SQ_DEBUG_PIX_TB_STATUS_REG_2			0x00000dbf
-
-#define REG_A2XX_SQ_DEBUG_PIX_TB_STATUS_REG_3			0x00000dc0
-
-#define REG_A2XX_SQ_DEBUG_PIX_TB_STATE_MEM			0x00000dc1
-
-#define REG_A2XX_TC_CNTL_STATUS					0x00000e00
-#define A2XX_TC_CNTL_STATUS_L2_INVALIDATE			0x00000001
-
-#define REG_A2XX_TP0_CHICKEN					0x00000e1e
-
-#define REG_A2XX_RB_BC_CONTROL					0x00000f01
-#define A2XX_RB_BC_CONTROL_ACCUM_LINEAR_MODE_ENABLE		0x00000001
-#define A2XX_RB_BC_CONTROL_ACCUM_TIMEOUT_SELECT__MASK		0x00000006
-#define A2XX_RB_BC_CONTROL_ACCUM_TIMEOUT_SELECT__SHIFT		1
-static inline uint32_t A2XX_RB_BC_CONTROL_ACCUM_TIMEOUT_SELECT(uint32_t val)
-{
-	return ((val) << A2XX_RB_BC_CONTROL_ACCUM_TIMEOUT_SELECT__SHIFT) & A2XX_RB_BC_CONTROL_ACCUM_TIMEOUT_SELECT__MASK;
-}
-#define A2XX_RB_BC_CONTROL_DISABLE_EDRAM_CAM			0x00000008
-#define A2XX_RB_BC_CONTROL_DISABLE_EZ_FAST_CONTEXT_SWITCH	0x00000010
-#define A2XX_RB_BC_CONTROL_DISABLE_EZ_NULL_ZCMD_DROP		0x00000020
-#define A2XX_RB_BC_CONTROL_DISABLE_LZ_NULL_ZCMD_DROP		0x00000040
-#define A2XX_RB_BC_CONTROL_ENABLE_AZ_THROTTLE			0x00000080
-#define A2XX_RB_BC_CONTROL_AZ_THROTTLE_COUNT__MASK		0x00001f00
-#define A2XX_RB_BC_CONTROL_AZ_THROTTLE_COUNT__SHIFT		8
-static inline uint32_t A2XX_RB_BC_CONTROL_AZ_THROTTLE_COUNT(uint32_t val)
-{
-	return ((val) << A2XX_RB_BC_CONTROL_AZ_THROTTLE_COUNT__SHIFT) & A2XX_RB_BC_CONTROL_AZ_THROTTLE_COUNT__MASK;
-}
-#define A2XX_RB_BC_CONTROL_ENABLE_CRC_UPDATE			0x00004000
-#define A2XX_RB_BC_CONTROL_CRC_MODE				0x00008000
-#define A2XX_RB_BC_CONTROL_DISABLE_SAMPLE_COUNTERS		0x00010000
-#define A2XX_RB_BC_CONTROL_DISABLE_ACCUM			0x00020000
-#define A2XX_RB_BC_CONTROL_ACCUM_ALLOC_MASK__MASK		0x003c0000
-#define A2XX_RB_BC_CONTROL_ACCUM_ALLOC_MASK__SHIFT		18
-static inline uint32_t A2XX_RB_BC_CONTROL_ACCUM_ALLOC_MASK(uint32_t val)
-{
-	return ((val) << A2XX_RB_BC_CONTROL_ACCUM_ALLOC_MASK__SHIFT) & A2XX_RB_BC_CONTROL_ACCUM_ALLOC_MASK__MASK;
-}
-#define A2XX_RB_BC_CONTROL_LINEAR_PERFORMANCE_ENABLE		0x00400000
-#define A2XX_RB_BC_CONTROL_ACCUM_DATA_FIFO_LIMIT__MASK		0x07800000
-#define A2XX_RB_BC_CONTROL_ACCUM_DATA_FIFO_LIMIT__SHIFT		23
-static inline uint32_t A2XX_RB_BC_CONTROL_ACCUM_DATA_FIFO_LIMIT(uint32_t val)
-{
-	return ((val) << A2XX_RB_BC_CONTROL_ACCUM_DATA_FIFO_LIMIT__SHIFT) & A2XX_RB_BC_CONTROL_ACCUM_DATA_FIFO_LIMIT__MASK;
-}
-#define A2XX_RB_BC_CONTROL_MEM_EXPORT_TIMEOUT_SELECT__MASK	0x18000000
-#define A2XX_RB_BC_CONTROL_MEM_EXPORT_TIMEOUT_SELECT__SHIFT	27
-static inline uint32_t A2XX_RB_BC_CONTROL_MEM_EXPORT_TIMEOUT_SELECT(uint32_t val)
-{
-	return ((val) << A2XX_RB_BC_CONTROL_MEM_EXPORT_TIMEOUT_SELECT__SHIFT) & A2XX_RB_BC_CONTROL_MEM_EXPORT_TIMEOUT_SELECT__MASK;
-}
-#define A2XX_RB_BC_CONTROL_MEM_EXPORT_LINEAR_MODE_ENABLE	0x20000000
-#define A2XX_RB_BC_CONTROL_CRC_SYSTEM				0x40000000
-#define A2XX_RB_BC_CONTROL_RESERVED6				0x80000000
-
-#define REG_A2XX_RB_EDRAM_INFO					0x00000f02
-
-#define REG_A2XX_RB_DEBUG_CNTL					0x00000f26
-
-#define REG_A2XX_RB_DEBUG_DATA					0x00000f27
-
-#define REG_A2XX_RB_SURFACE_INFO				0x00002000
-#define A2XX_RB_SURFACE_INFO_SURFACE_PITCH__MASK		0x00003fff
-#define A2XX_RB_SURFACE_INFO_SURFACE_PITCH__SHIFT		0
-static inline uint32_t A2XX_RB_SURFACE_INFO_SURFACE_PITCH(uint32_t val)
-{
-	return ((val) << A2XX_RB_SURFACE_INFO_SURFACE_PITCH__SHIFT) & A2XX_RB_SURFACE_INFO_SURFACE_PITCH__MASK;
-}
-#define A2XX_RB_SURFACE_INFO_MSAA_SAMPLES__MASK			0x0000c000
-#define A2XX_RB_SURFACE_INFO_MSAA_SAMPLES__SHIFT		14
-static inline uint32_t A2XX_RB_SURFACE_INFO_MSAA_SAMPLES(uint32_t val)
-{
-	return ((val) << A2XX_RB_SURFACE_INFO_MSAA_SAMPLES__SHIFT) & A2XX_RB_SURFACE_INFO_MSAA_SAMPLES__MASK;
-}
-
-#define REG_A2XX_RB_COLOR_INFO					0x00002001
-#define A2XX_RB_COLOR_INFO_FORMAT__MASK				0x0000000f
-#define A2XX_RB_COLOR_INFO_FORMAT__SHIFT			0
-static inline uint32_t A2XX_RB_COLOR_INFO_FORMAT(enum a2xx_colorformatx val)
-{
-	return ((val) << A2XX_RB_COLOR_INFO_FORMAT__SHIFT) & A2XX_RB_COLOR_INFO_FORMAT__MASK;
-}
-#define A2XX_RB_COLOR_INFO_ROUND_MODE__MASK			0x00000030
-#define A2XX_RB_COLOR_INFO_ROUND_MODE__SHIFT			4
-static inline uint32_t A2XX_RB_COLOR_INFO_ROUND_MODE(uint32_t val)
-{
-	return ((val) << A2XX_RB_COLOR_INFO_ROUND_MODE__SHIFT) & A2XX_RB_COLOR_INFO_ROUND_MODE__MASK;
-}
-#define A2XX_RB_COLOR_INFO_LINEAR				0x00000040
-#define A2XX_RB_COLOR_INFO_ENDIAN__MASK				0x00000180
-#define A2XX_RB_COLOR_INFO_ENDIAN__SHIFT			7
-static inline uint32_t A2XX_RB_COLOR_INFO_ENDIAN(uint32_t val)
-{
-	return ((val) << A2XX_RB_COLOR_INFO_ENDIAN__SHIFT) & A2XX_RB_COLOR_INFO_ENDIAN__MASK;
-}
-#define A2XX_RB_COLOR_INFO_SWAP__MASK				0x00000600
-#define A2XX_RB_COLOR_INFO_SWAP__SHIFT				9
-static inline uint32_t A2XX_RB_COLOR_INFO_SWAP(uint32_t val)
-{
-	return ((val) << A2XX_RB_COLOR_INFO_SWAP__SHIFT) & A2XX_RB_COLOR_INFO_SWAP__MASK;
-}
-#define A2XX_RB_COLOR_INFO_BASE__MASK				0xfffff000
-#define A2XX_RB_COLOR_INFO_BASE__SHIFT				12
-static inline uint32_t A2XX_RB_COLOR_INFO_BASE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A2XX_RB_COLOR_INFO_BASE__SHIFT) & A2XX_RB_COLOR_INFO_BASE__MASK;
-}
-
-#define REG_A2XX_RB_DEPTH_INFO					0x00002002
-#define A2XX_RB_DEPTH_INFO_DEPTH_FORMAT__MASK			0x00000001
-#define A2XX_RB_DEPTH_INFO_DEPTH_FORMAT__SHIFT			0
-static inline uint32_t A2XX_RB_DEPTH_INFO_DEPTH_FORMAT(enum adreno_rb_depth_format val)
-{
-	return ((val) << A2XX_RB_DEPTH_INFO_DEPTH_FORMAT__SHIFT) & A2XX_RB_DEPTH_INFO_DEPTH_FORMAT__MASK;
-}
-#define A2XX_RB_DEPTH_INFO_DEPTH_BASE__MASK			0xfffff000
-#define A2XX_RB_DEPTH_INFO_DEPTH_BASE__SHIFT			12
-static inline uint32_t A2XX_RB_DEPTH_INFO_DEPTH_BASE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A2XX_RB_DEPTH_INFO_DEPTH_BASE__SHIFT) & A2XX_RB_DEPTH_INFO_DEPTH_BASE__MASK;
-}
-
-#define REG_A2XX_A225_RB_COLOR_INFO3				0x00002005
-
-#define REG_A2XX_COHER_DEST_BASE_0				0x00002006
-
-#define REG_A2XX_PA_SC_SCREEN_SCISSOR_TL			0x0000200e
-#define A2XX_PA_SC_SCREEN_SCISSOR_TL_WINDOW_OFFSET_DISABLE	0x80000000
-#define A2XX_PA_SC_SCREEN_SCISSOR_TL_X__MASK			0x00007fff
-#define A2XX_PA_SC_SCREEN_SCISSOR_TL_X__SHIFT			0
-static inline uint32_t A2XX_PA_SC_SCREEN_SCISSOR_TL_X(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_SCREEN_SCISSOR_TL_X__SHIFT) & A2XX_PA_SC_SCREEN_SCISSOR_TL_X__MASK;
-}
-#define A2XX_PA_SC_SCREEN_SCISSOR_TL_Y__MASK			0x7fff0000
-#define A2XX_PA_SC_SCREEN_SCISSOR_TL_Y__SHIFT			16
-static inline uint32_t A2XX_PA_SC_SCREEN_SCISSOR_TL_Y(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_SCREEN_SCISSOR_TL_Y__SHIFT) & A2XX_PA_SC_SCREEN_SCISSOR_TL_Y__MASK;
-}
-
-#define REG_A2XX_PA_SC_SCREEN_SCISSOR_BR			0x0000200f
-#define A2XX_PA_SC_SCREEN_SCISSOR_BR_WINDOW_OFFSET_DISABLE	0x80000000
-#define A2XX_PA_SC_SCREEN_SCISSOR_BR_X__MASK			0x00007fff
-#define A2XX_PA_SC_SCREEN_SCISSOR_BR_X__SHIFT			0
-static inline uint32_t A2XX_PA_SC_SCREEN_SCISSOR_BR_X(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_SCREEN_SCISSOR_BR_X__SHIFT) & A2XX_PA_SC_SCREEN_SCISSOR_BR_X__MASK;
-}
-#define A2XX_PA_SC_SCREEN_SCISSOR_BR_Y__MASK			0x7fff0000
-#define A2XX_PA_SC_SCREEN_SCISSOR_BR_Y__SHIFT			16
-static inline uint32_t A2XX_PA_SC_SCREEN_SCISSOR_BR_Y(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_SCREEN_SCISSOR_BR_Y__SHIFT) & A2XX_PA_SC_SCREEN_SCISSOR_BR_Y__MASK;
-}
-
-#define REG_A2XX_PA_SC_WINDOW_OFFSET				0x00002080
-#define A2XX_PA_SC_WINDOW_OFFSET_X__MASK			0x00007fff
-#define A2XX_PA_SC_WINDOW_OFFSET_X__SHIFT			0
-static inline uint32_t A2XX_PA_SC_WINDOW_OFFSET_X(int32_t val)
-{
-	return ((val) << A2XX_PA_SC_WINDOW_OFFSET_X__SHIFT) & A2XX_PA_SC_WINDOW_OFFSET_X__MASK;
-}
-#define A2XX_PA_SC_WINDOW_OFFSET_Y__MASK			0x7fff0000
-#define A2XX_PA_SC_WINDOW_OFFSET_Y__SHIFT			16
-static inline uint32_t A2XX_PA_SC_WINDOW_OFFSET_Y(int32_t val)
-{
-	return ((val) << A2XX_PA_SC_WINDOW_OFFSET_Y__SHIFT) & A2XX_PA_SC_WINDOW_OFFSET_Y__MASK;
-}
-#define A2XX_PA_SC_WINDOW_OFFSET_DISABLE			0x80000000
-
-#define REG_A2XX_PA_SC_WINDOW_SCISSOR_TL			0x00002081
-#define A2XX_PA_SC_WINDOW_SCISSOR_TL_WINDOW_OFFSET_DISABLE	0x80000000
-#define A2XX_PA_SC_WINDOW_SCISSOR_TL_X__MASK			0x00007fff
-#define A2XX_PA_SC_WINDOW_SCISSOR_TL_X__SHIFT			0
-static inline uint32_t A2XX_PA_SC_WINDOW_SCISSOR_TL_X(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_WINDOW_SCISSOR_TL_X__SHIFT) & A2XX_PA_SC_WINDOW_SCISSOR_TL_X__MASK;
-}
-#define A2XX_PA_SC_WINDOW_SCISSOR_TL_Y__MASK			0x7fff0000
-#define A2XX_PA_SC_WINDOW_SCISSOR_TL_Y__SHIFT			16
-static inline uint32_t A2XX_PA_SC_WINDOW_SCISSOR_TL_Y(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_WINDOW_SCISSOR_TL_Y__SHIFT) & A2XX_PA_SC_WINDOW_SCISSOR_TL_Y__MASK;
-}
-
-#define REG_A2XX_PA_SC_WINDOW_SCISSOR_BR			0x00002082
-#define A2XX_PA_SC_WINDOW_SCISSOR_BR_WINDOW_OFFSET_DISABLE	0x80000000
-#define A2XX_PA_SC_WINDOW_SCISSOR_BR_X__MASK			0x00007fff
-#define A2XX_PA_SC_WINDOW_SCISSOR_BR_X__SHIFT			0
-static inline uint32_t A2XX_PA_SC_WINDOW_SCISSOR_BR_X(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_WINDOW_SCISSOR_BR_X__SHIFT) & A2XX_PA_SC_WINDOW_SCISSOR_BR_X__MASK;
-}
-#define A2XX_PA_SC_WINDOW_SCISSOR_BR_Y__MASK			0x7fff0000
-#define A2XX_PA_SC_WINDOW_SCISSOR_BR_Y__SHIFT			16
-static inline uint32_t A2XX_PA_SC_WINDOW_SCISSOR_BR_Y(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_WINDOW_SCISSOR_BR_Y__SHIFT) & A2XX_PA_SC_WINDOW_SCISSOR_BR_Y__MASK;
-}
-
-#define REG_A2XX_UNKNOWN_2010					0x00002010
-
-#define REG_A2XX_VGT_MAX_VTX_INDX				0x00002100
-
-#define REG_A2XX_VGT_MIN_VTX_INDX				0x00002101
-
-#define REG_A2XX_VGT_INDX_OFFSET				0x00002102
-
-#define REG_A2XX_A225_PC_MULTI_PRIM_IB_RESET_INDX		0x00002103
-
-#define REG_A2XX_RB_COLOR_MASK					0x00002104
-#define A2XX_RB_COLOR_MASK_WRITE_RED				0x00000001
-#define A2XX_RB_COLOR_MASK_WRITE_GREEN				0x00000002
-#define A2XX_RB_COLOR_MASK_WRITE_BLUE				0x00000004
-#define A2XX_RB_COLOR_MASK_WRITE_ALPHA				0x00000008
-
-#define REG_A2XX_RB_BLEND_RED					0x00002105
-
-#define REG_A2XX_RB_BLEND_GREEN					0x00002106
-
-#define REG_A2XX_RB_BLEND_BLUE					0x00002107
-
-#define REG_A2XX_RB_BLEND_ALPHA					0x00002108
-
-#define REG_A2XX_RB_FOG_COLOR					0x00002109
-#define A2XX_RB_FOG_COLOR_FOG_RED__MASK				0x000000ff
-#define A2XX_RB_FOG_COLOR_FOG_RED__SHIFT			0
-static inline uint32_t A2XX_RB_FOG_COLOR_FOG_RED(uint32_t val)
-{
-	return ((val) << A2XX_RB_FOG_COLOR_FOG_RED__SHIFT) & A2XX_RB_FOG_COLOR_FOG_RED__MASK;
-}
-#define A2XX_RB_FOG_COLOR_FOG_GREEN__MASK			0x0000ff00
-#define A2XX_RB_FOG_COLOR_FOG_GREEN__SHIFT			8
-static inline uint32_t A2XX_RB_FOG_COLOR_FOG_GREEN(uint32_t val)
-{
-	return ((val) << A2XX_RB_FOG_COLOR_FOG_GREEN__SHIFT) & A2XX_RB_FOG_COLOR_FOG_GREEN__MASK;
-}
-#define A2XX_RB_FOG_COLOR_FOG_BLUE__MASK			0x00ff0000
-#define A2XX_RB_FOG_COLOR_FOG_BLUE__SHIFT			16
-static inline uint32_t A2XX_RB_FOG_COLOR_FOG_BLUE(uint32_t val)
-{
-	return ((val) << A2XX_RB_FOG_COLOR_FOG_BLUE__SHIFT) & A2XX_RB_FOG_COLOR_FOG_BLUE__MASK;
-}
-
-#define REG_A2XX_RB_STENCILREFMASK_BF				0x0000210c
-#define A2XX_RB_STENCILREFMASK_BF_STENCILREF__MASK		0x000000ff
-#define A2XX_RB_STENCILREFMASK_BF_STENCILREF__SHIFT		0
-static inline uint32_t A2XX_RB_STENCILREFMASK_BF_STENCILREF(uint32_t val)
-{
-	return ((val) << A2XX_RB_STENCILREFMASK_BF_STENCILREF__SHIFT) & A2XX_RB_STENCILREFMASK_BF_STENCILREF__MASK;
-}
-#define A2XX_RB_STENCILREFMASK_BF_STENCILMASK__MASK		0x0000ff00
-#define A2XX_RB_STENCILREFMASK_BF_STENCILMASK__SHIFT		8
-static inline uint32_t A2XX_RB_STENCILREFMASK_BF_STENCILMASK(uint32_t val)
-{
-	return ((val) << A2XX_RB_STENCILREFMASK_BF_STENCILMASK__SHIFT) & A2XX_RB_STENCILREFMASK_BF_STENCILMASK__MASK;
-}
-#define A2XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__MASK	0x00ff0000
-#define A2XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__SHIFT	16
-static inline uint32_t A2XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK(uint32_t val)
-{
-	return ((val) << A2XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__SHIFT) & A2XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__MASK;
-}
-
-#define REG_A2XX_RB_STENCILREFMASK				0x0000210d
-#define A2XX_RB_STENCILREFMASK_STENCILREF__MASK			0x000000ff
-#define A2XX_RB_STENCILREFMASK_STENCILREF__SHIFT		0
-static inline uint32_t A2XX_RB_STENCILREFMASK_STENCILREF(uint32_t val)
-{
-	return ((val) << A2XX_RB_STENCILREFMASK_STENCILREF__SHIFT) & A2XX_RB_STENCILREFMASK_STENCILREF__MASK;
-}
-#define A2XX_RB_STENCILREFMASK_STENCILMASK__MASK		0x0000ff00
-#define A2XX_RB_STENCILREFMASK_STENCILMASK__SHIFT		8
-static inline uint32_t A2XX_RB_STENCILREFMASK_STENCILMASK(uint32_t val)
-{
-	return ((val) << A2XX_RB_STENCILREFMASK_STENCILMASK__SHIFT) & A2XX_RB_STENCILREFMASK_STENCILMASK__MASK;
-}
-#define A2XX_RB_STENCILREFMASK_STENCILWRITEMASK__MASK		0x00ff0000
-#define A2XX_RB_STENCILREFMASK_STENCILWRITEMASK__SHIFT		16
-static inline uint32_t A2XX_RB_STENCILREFMASK_STENCILWRITEMASK(uint32_t val)
-{
-	return ((val) << A2XX_RB_STENCILREFMASK_STENCILWRITEMASK__SHIFT) & A2XX_RB_STENCILREFMASK_STENCILWRITEMASK__MASK;
-}
-
-#define REG_A2XX_RB_ALPHA_REF					0x0000210e
-
-#define REG_A2XX_PA_CL_VPORT_XSCALE				0x0000210f
-#define A2XX_PA_CL_VPORT_XSCALE__MASK				0xffffffff
-#define A2XX_PA_CL_VPORT_XSCALE__SHIFT				0
-static inline uint32_t A2XX_PA_CL_VPORT_XSCALE(float val)
-{
-	return ((fui(val)) << A2XX_PA_CL_VPORT_XSCALE__SHIFT) & A2XX_PA_CL_VPORT_XSCALE__MASK;
-}
-
-#define REG_A2XX_PA_CL_VPORT_XOFFSET				0x00002110
-#define A2XX_PA_CL_VPORT_XOFFSET__MASK				0xffffffff
-#define A2XX_PA_CL_VPORT_XOFFSET__SHIFT				0
-static inline uint32_t A2XX_PA_CL_VPORT_XOFFSET(float val)
-{
-	return ((fui(val)) << A2XX_PA_CL_VPORT_XOFFSET__SHIFT) & A2XX_PA_CL_VPORT_XOFFSET__MASK;
-}
-
-#define REG_A2XX_PA_CL_VPORT_YSCALE				0x00002111
-#define A2XX_PA_CL_VPORT_YSCALE__MASK				0xffffffff
-#define A2XX_PA_CL_VPORT_YSCALE__SHIFT				0
-static inline uint32_t A2XX_PA_CL_VPORT_YSCALE(float val)
-{
-	return ((fui(val)) << A2XX_PA_CL_VPORT_YSCALE__SHIFT) & A2XX_PA_CL_VPORT_YSCALE__MASK;
-}
-
-#define REG_A2XX_PA_CL_VPORT_YOFFSET				0x00002112
-#define A2XX_PA_CL_VPORT_YOFFSET__MASK				0xffffffff
-#define A2XX_PA_CL_VPORT_YOFFSET__SHIFT				0
-static inline uint32_t A2XX_PA_CL_VPORT_YOFFSET(float val)
-{
-	return ((fui(val)) << A2XX_PA_CL_VPORT_YOFFSET__SHIFT) & A2XX_PA_CL_VPORT_YOFFSET__MASK;
-}
-
-#define REG_A2XX_PA_CL_VPORT_ZSCALE				0x00002113
-#define A2XX_PA_CL_VPORT_ZSCALE__MASK				0xffffffff
-#define A2XX_PA_CL_VPORT_ZSCALE__SHIFT				0
-static inline uint32_t A2XX_PA_CL_VPORT_ZSCALE(float val)
-{
-	return ((fui(val)) << A2XX_PA_CL_VPORT_ZSCALE__SHIFT) & A2XX_PA_CL_VPORT_ZSCALE__MASK;
-}
-
-#define REG_A2XX_PA_CL_VPORT_ZOFFSET				0x00002114
-#define A2XX_PA_CL_VPORT_ZOFFSET__MASK				0xffffffff
-#define A2XX_PA_CL_VPORT_ZOFFSET__SHIFT				0
-static inline uint32_t A2XX_PA_CL_VPORT_ZOFFSET(float val)
-{
-	return ((fui(val)) << A2XX_PA_CL_VPORT_ZOFFSET__SHIFT) & A2XX_PA_CL_VPORT_ZOFFSET__MASK;
-}
-
-#define REG_A2XX_SQ_PROGRAM_CNTL				0x00002180
-#define A2XX_SQ_PROGRAM_CNTL_VS_REGS__MASK			0x000000ff
-#define A2XX_SQ_PROGRAM_CNTL_VS_REGS__SHIFT			0
-static inline uint32_t A2XX_SQ_PROGRAM_CNTL_VS_REGS(uint32_t val)
-{
-	return ((val) << A2XX_SQ_PROGRAM_CNTL_VS_REGS__SHIFT) & A2XX_SQ_PROGRAM_CNTL_VS_REGS__MASK;
-}
-#define A2XX_SQ_PROGRAM_CNTL_PS_REGS__MASK			0x0000ff00
-#define A2XX_SQ_PROGRAM_CNTL_PS_REGS__SHIFT			8
-static inline uint32_t A2XX_SQ_PROGRAM_CNTL_PS_REGS(uint32_t val)
-{
-	return ((val) << A2XX_SQ_PROGRAM_CNTL_PS_REGS__SHIFT) & A2XX_SQ_PROGRAM_CNTL_PS_REGS__MASK;
-}
-#define A2XX_SQ_PROGRAM_CNTL_VS_RESOURCE			0x00010000
-#define A2XX_SQ_PROGRAM_CNTL_PS_RESOURCE			0x00020000
-#define A2XX_SQ_PROGRAM_CNTL_PARAM_GEN				0x00040000
-#define A2XX_SQ_PROGRAM_CNTL_GEN_INDEX_PIX			0x00080000
-#define A2XX_SQ_PROGRAM_CNTL_VS_EXPORT_COUNT__MASK		0x00f00000
-#define A2XX_SQ_PROGRAM_CNTL_VS_EXPORT_COUNT__SHIFT		20
-static inline uint32_t A2XX_SQ_PROGRAM_CNTL_VS_EXPORT_COUNT(uint32_t val)
-{
-	return ((val) << A2XX_SQ_PROGRAM_CNTL_VS_EXPORT_COUNT__SHIFT) & A2XX_SQ_PROGRAM_CNTL_VS_EXPORT_COUNT__MASK;
-}
-#define A2XX_SQ_PROGRAM_CNTL_VS_EXPORT_MODE__MASK		0x07000000
-#define A2XX_SQ_PROGRAM_CNTL_VS_EXPORT_MODE__SHIFT		24
-static inline uint32_t A2XX_SQ_PROGRAM_CNTL_VS_EXPORT_MODE(enum a2xx_sq_ps_vtx_mode val)
-{
-	return ((val) << A2XX_SQ_PROGRAM_CNTL_VS_EXPORT_MODE__SHIFT) & A2XX_SQ_PROGRAM_CNTL_VS_EXPORT_MODE__MASK;
-}
-#define A2XX_SQ_PROGRAM_CNTL_PS_EXPORT_MODE__MASK		0x78000000
-#define A2XX_SQ_PROGRAM_CNTL_PS_EXPORT_MODE__SHIFT		27
-static inline uint32_t A2XX_SQ_PROGRAM_CNTL_PS_EXPORT_MODE(uint32_t val)
-{
-	return ((val) << A2XX_SQ_PROGRAM_CNTL_PS_EXPORT_MODE__SHIFT) & A2XX_SQ_PROGRAM_CNTL_PS_EXPORT_MODE__MASK;
-}
-#define A2XX_SQ_PROGRAM_CNTL_GEN_INDEX_VTX			0x80000000
-
-#define REG_A2XX_SQ_CONTEXT_MISC				0x00002181
-#define A2XX_SQ_CONTEXT_MISC_INST_PRED_OPTIMIZE			0x00000001
-#define A2XX_SQ_CONTEXT_MISC_SC_OUTPUT_SCREEN_XY		0x00000002
-#define A2XX_SQ_CONTEXT_MISC_SC_SAMPLE_CNTL__MASK		0x0000000c
-#define A2XX_SQ_CONTEXT_MISC_SC_SAMPLE_CNTL__SHIFT		2
-static inline uint32_t A2XX_SQ_CONTEXT_MISC_SC_SAMPLE_CNTL(enum a2xx_sq_sample_cntl val)
-{
-	return ((val) << A2XX_SQ_CONTEXT_MISC_SC_SAMPLE_CNTL__SHIFT) & A2XX_SQ_CONTEXT_MISC_SC_SAMPLE_CNTL__MASK;
-}
-#define A2XX_SQ_CONTEXT_MISC_PARAM_GEN_POS__MASK		0x0000ff00
-#define A2XX_SQ_CONTEXT_MISC_PARAM_GEN_POS__SHIFT		8
-static inline uint32_t A2XX_SQ_CONTEXT_MISC_PARAM_GEN_POS(uint32_t val)
-{
-	return ((val) << A2XX_SQ_CONTEXT_MISC_PARAM_GEN_POS__SHIFT) & A2XX_SQ_CONTEXT_MISC_PARAM_GEN_POS__MASK;
-}
-#define A2XX_SQ_CONTEXT_MISC_PERFCOUNTER_REF			0x00010000
-#define A2XX_SQ_CONTEXT_MISC_YEILD_OPTIMIZE			0x00020000
-#define A2XX_SQ_CONTEXT_MISC_TX_CACHE_SEL			0x00040000
-
-#define REG_A2XX_SQ_INTERPOLATOR_CNTL				0x00002182
-#define A2XX_SQ_INTERPOLATOR_CNTL_PARAM_SHADE__MASK		0x0000ffff
-#define A2XX_SQ_INTERPOLATOR_CNTL_PARAM_SHADE__SHIFT		0
-static inline uint32_t A2XX_SQ_INTERPOLATOR_CNTL_PARAM_SHADE(uint32_t val)
-{
-	return ((val) << A2XX_SQ_INTERPOLATOR_CNTL_PARAM_SHADE__SHIFT) & A2XX_SQ_INTERPOLATOR_CNTL_PARAM_SHADE__MASK;
-}
-#define A2XX_SQ_INTERPOLATOR_CNTL_SAMPLING_PATTERN__MASK	0xffff0000
-#define A2XX_SQ_INTERPOLATOR_CNTL_SAMPLING_PATTERN__SHIFT	16
-static inline uint32_t A2XX_SQ_INTERPOLATOR_CNTL_SAMPLING_PATTERN(uint32_t val)
-{
-	return ((val) << A2XX_SQ_INTERPOLATOR_CNTL_SAMPLING_PATTERN__SHIFT) & A2XX_SQ_INTERPOLATOR_CNTL_SAMPLING_PATTERN__MASK;
-}
-
-#define REG_A2XX_SQ_WRAPPING_0					0x00002183
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_0__MASK			0x0000000f
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_0__SHIFT			0
-static inline uint32_t A2XX_SQ_WRAPPING_0_PARAM_WRAP_0(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_0_PARAM_WRAP_0__SHIFT) & A2XX_SQ_WRAPPING_0_PARAM_WRAP_0__MASK;
-}
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_1__MASK			0x000000f0
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_1__SHIFT			4
-static inline uint32_t A2XX_SQ_WRAPPING_0_PARAM_WRAP_1(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_0_PARAM_WRAP_1__SHIFT) & A2XX_SQ_WRAPPING_0_PARAM_WRAP_1__MASK;
-}
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_2__MASK			0x00000f00
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_2__SHIFT			8
-static inline uint32_t A2XX_SQ_WRAPPING_0_PARAM_WRAP_2(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_0_PARAM_WRAP_2__SHIFT) & A2XX_SQ_WRAPPING_0_PARAM_WRAP_2__MASK;
-}
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_3__MASK			0x0000f000
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_3__SHIFT			12
-static inline uint32_t A2XX_SQ_WRAPPING_0_PARAM_WRAP_3(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_0_PARAM_WRAP_3__SHIFT) & A2XX_SQ_WRAPPING_0_PARAM_WRAP_3__MASK;
-}
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_4__MASK			0x000f0000
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_4__SHIFT			16
-static inline uint32_t A2XX_SQ_WRAPPING_0_PARAM_WRAP_4(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_0_PARAM_WRAP_4__SHIFT) & A2XX_SQ_WRAPPING_0_PARAM_WRAP_4__MASK;
-}
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_5__MASK			0x00f00000
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_5__SHIFT			20
-static inline uint32_t A2XX_SQ_WRAPPING_0_PARAM_WRAP_5(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_0_PARAM_WRAP_5__SHIFT) & A2XX_SQ_WRAPPING_0_PARAM_WRAP_5__MASK;
-}
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_6__MASK			0x0f000000
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_6__SHIFT			24
-static inline uint32_t A2XX_SQ_WRAPPING_0_PARAM_WRAP_6(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_0_PARAM_WRAP_6__SHIFT) & A2XX_SQ_WRAPPING_0_PARAM_WRAP_6__MASK;
-}
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_7__MASK			0xf0000000
-#define A2XX_SQ_WRAPPING_0_PARAM_WRAP_7__SHIFT			28
-static inline uint32_t A2XX_SQ_WRAPPING_0_PARAM_WRAP_7(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_0_PARAM_WRAP_7__SHIFT) & A2XX_SQ_WRAPPING_0_PARAM_WRAP_7__MASK;
-}
-
-#define REG_A2XX_SQ_WRAPPING_1					0x00002184
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_8__MASK			0x0000000f
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_8__SHIFT			0
-static inline uint32_t A2XX_SQ_WRAPPING_1_PARAM_WRAP_8(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_1_PARAM_WRAP_8__SHIFT) & A2XX_SQ_WRAPPING_1_PARAM_WRAP_8__MASK;
-}
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_9__MASK			0x000000f0
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_9__SHIFT			4
-static inline uint32_t A2XX_SQ_WRAPPING_1_PARAM_WRAP_9(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_1_PARAM_WRAP_9__SHIFT) & A2XX_SQ_WRAPPING_1_PARAM_WRAP_9__MASK;
-}
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_10__MASK			0x00000f00
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_10__SHIFT			8
-static inline uint32_t A2XX_SQ_WRAPPING_1_PARAM_WRAP_10(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_1_PARAM_WRAP_10__SHIFT) & A2XX_SQ_WRAPPING_1_PARAM_WRAP_10__MASK;
-}
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_11__MASK			0x0000f000
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_11__SHIFT			12
-static inline uint32_t A2XX_SQ_WRAPPING_1_PARAM_WRAP_11(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_1_PARAM_WRAP_11__SHIFT) & A2XX_SQ_WRAPPING_1_PARAM_WRAP_11__MASK;
-}
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_12__MASK			0x000f0000
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_12__SHIFT			16
-static inline uint32_t A2XX_SQ_WRAPPING_1_PARAM_WRAP_12(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_1_PARAM_WRAP_12__SHIFT) & A2XX_SQ_WRAPPING_1_PARAM_WRAP_12__MASK;
-}
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_13__MASK			0x00f00000
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_13__SHIFT			20
-static inline uint32_t A2XX_SQ_WRAPPING_1_PARAM_WRAP_13(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_1_PARAM_WRAP_13__SHIFT) & A2XX_SQ_WRAPPING_1_PARAM_WRAP_13__MASK;
-}
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_14__MASK			0x0f000000
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_14__SHIFT			24
-static inline uint32_t A2XX_SQ_WRAPPING_1_PARAM_WRAP_14(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_1_PARAM_WRAP_14__SHIFT) & A2XX_SQ_WRAPPING_1_PARAM_WRAP_14__MASK;
-}
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_15__MASK			0xf0000000
-#define A2XX_SQ_WRAPPING_1_PARAM_WRAP_15__SHIFT			28
-static inline uint32_t A2XX_SQ_WRAPPING_1_PARAM_WRAP_15(uint32_t val)
-{
-	return ((val) << A2XX_SQ_WRAPPING_1_PARAM_WRAP_15__SHIFT) & A2XX_SQ_WRAPPING_1_PARAM_WRAP_15__MASK;
-}
-
-#define REG_A2XX_SQ_PS_PROGRAM					0x000021f6
-#define A2XX_SQ_PS_PROGRAM_BASE__MASK				0x00000fff
-#define A2XX_SQ_PS_PROGRAM_BASE__SHIFT				0
-static inline uint32_t A2XX_SQ_PS_PROGRAM_BASE(uint32_t val)
-{
-	return ((val) << A2XX_SQ_PS_PROGRAM_BASE__SHIFT) & A2XX_SQ_PS_PROGRAM_BASE__MASK;
-}
-#define A2XX_SQ_PS_PROGRAM_SIZE__MASK				0x00fff000
-#define A2XX_SQ_PS_PROGRAM_SIZE__SHIFT				12
-static inline uint32_t A2XX_SQ_PS_PROGRAM_SIZE(uint32_t val)
-{
-	return ((val) << A2XX_SQ_PS_PROGRAM_SIZE__SHIFT) & A2XX_SQ_PS_PROGRAM_SIZE__MASK;
-}
-
-#define REG_A2XX_SQ_VS_PROGRAM					0x000021f7
-#define A2XX_SQ_VS_PROGRAM_BASE__MASK				0x00000fff
-#define A2XX_SQ_VS_PROGRAM_BASE__SHIFT				0
-static inline uint32_t A2XX_SQ_VS_PROGRAM_BASE(uint32_t val)
-{
-	return ((val) << A2XX_SQ_VS_PROGRAM_BASE__SHIFT) & A2XX_SQ_VS_PROGRAM_BASE__MASK;
-}
-#define A2XX_SQ_VS_PROGRAM_SIZE__MASK				0x00fff000
-#define A2XX_SQ_VS_PROGRAM_SIZE__SHIFT				12
-static inline uint32_t A2XX_SQ_VS_PROGRAM_SIZE(uint32_t val)
-{
-	return ((val) << A2XX_SQ_VS_PROGRAM_SIZE__SHIFT) & A2XX_SQ_VS_PROGRAM_SIZE__MASK;
-}
-
-#define REG_A2XX_VGT_EVENT_INITIATOR				0x000021f9
-
-#define REG_A2XX_VGT_DRAW_INITIATOR				0x000021fc
-#define A2XX_VGT_DRAW_INITIATOR_PRIM_TYPE__MASK			0x0000003f
-#define A2XX_VGT_DRAW_INITIATOR_PRIM_TYPE__SHIFT		0
-static inline uint32_t A2XX_VGT_DRAW_INITIATOR_PRIM_TYPE(enum pc_di_primtype val)
-{
-	return ((val) << A2XX_VGT_DRAW_INITIATOR_PRIM_TYPE__SHIFT) & A2XX_VGT_DRAW_INITIATOR_PRIM_TYPE__MASK;
-}
-#define A2XX_VGT_DRAW_INITIATOR_SOURCE_SELECT__MASK		0x000000c0
-#define A2XX_VGT_DRAW_INITIATOR_SOURCE_SELECT__SHIFT		6
-static inline uint32_t A2XX_VGT_DRAW_INITIATOR_SOURCE_SELECT(enum pc_di_src_sel val)
-{
-	return ((val) << A2XX_VGT_DRAW_INITIATOR_SOURCE_SELECT__SHIFT) & A2XX_VGT_DRAW_INITIATOR_SOURCE_SELECT__MASK;
-}
-#define A2XX_VGT_DRAW_INITIATOR_VIS_CULL__MASK			0x00000600
-#define A2XX_VGT_DRAW_INITIATOR_VIS_CULL__SHIFT			9
-static inline uint32_t A2XX_VGT_DRAW_INITIATOR_VIS_CULL(enum pc_di_vis_cull_mode val)
-{
-	return ((val) << A2XX_VGT_DRAW_INITIATOR_VIS_CULL__SHIFT) & A2XX_VGT_DRAW_INITIATOR_VIS_CULL__MASK;
-}
-#define A2XX_VGT_DRAW_INITIATOR_INDEX_SIZE__MASK		0x00000800
-#define A2XX_VGT_DRAW_INITIATOR_INDEX_SIZE__SHIFT		11
-static inline uint32_t A2XX_VGT_DRAW_INITIATOR_INDEX_SIZE(enum pc_di_index_size val)
-{
-	return ((val) << A2XX_VGT_DRAW_INITIATOR_INDEX_SIZE__SHIFT) & A2XX_VGT_DRAW_INITIATOR_INDEX_SIZE__MASK;
-}
-#define A2XX_VGT_DRAW_INITIATOR_NOT_EOP				0x00001000
-#define A2XX_VGT_DRAW_INITIATOR_SMALL_INDEX			0x00002000
-#define A2XX_VGT_DRAW_INITIATOR_PRE_DRAW_INITIATOR_ENABLE	0x00004000
-#define A2XX_VGT_DRAW_INITIATOR_NUM_INSTANCES__MASK		0xff000000
-#define A2XX_VGT_DRAW_INITIATOR_NUM_INSTANCES__SHIFT		24
-static inline uint32_t A2XX_VGT_DRAW_INITIATOR_NUM_INSTANCES(uint32_t val)
-{
-	return ((val) << A2XX_VGT_DRAW_INITIATOR_NUM_INSTANCES__SHIFT) & A2XX_VGT_DRAW_INITIATOR_NUM_INSTANCES__MASK;
-}
-
-#define REG_A2XX_VGT_IMMED_DATA					0x000021fd
-
-#define REG_A2XX_RB_DEPTHCONTROL				0x00002200
-#define A2XX_RB_DEPTHCONTROL_STENCIL_ENABLE			0x00000001
-#define A2XX_RB_DEPTHCONTROL_Z_ENABLE				0x00000002
-#define A2XX_RB_DEPTHCONTROL_Z_WRITE_ENABLE			0x00000004
-#define A2XX_RB_DEPTHCONTROL_EARLY_Z_ENABLE			0x00000008
-#define A2XX_RB_DEPTHCONTROL_ZFUNC__MASK			0x00000070
-#define A2XX_RB_DEPTHCONTROL_ZFUNC__SHIFT			4
-static inline uint32_t A2XX_RB_DEPTHCONTROL_ZFUNC(enum adreno_compare_func val)
-{
-	return ((val) << A2XX_RB_DEPTHCONTROL_ZFUNC__SHIFT) & A2XX_RB_DEPTHCONTROL_ZFUNC__MASK;
-}
-#define A2XX_RB_DEPTHCONTROL_BACKFACE_ENABLE			0x00000080
-#define A2XX_RB_DEPTHCONTROL_STENCILFUNC__MASK			0x00000700
-#define A2XX_RB_DEPTHCONTROL_STENCILFUNC__SHIFT			8
-static inline uint32_t A2XX_RB_DEPTHCONTROL_STENCILFUNC(enum adreno_compare_func val)
-{
-	return ((val) << A2XX_RB_DEPTHCONTROL_STENCILFUNC__SHIFT) & A2XX_RB_DEPTHCONTROL_STENCILFUNC__MASK;
-}
-#define A2XX_RB_DEPTHCONTROL_STENCILFAIL__MASK			0x00003800
-#define A2XX_RB_DEPTHCONTROL_STENCILFAIL__SHIFT			11
-static inline uint32_t A2XX_RB_DEPTHCONTROL_STENCILFAIL(enum adreno_stencil_op val)
-{
-	return ((val) << A2XX_RB_DEPTHCONTROL_STENCILFAIL__SHIFT) & A2XX_RB_DEPTHCONTROL_STENCILFAIL__MASK;
-}
-#define A2XX_RB_DEPTHCONTROL_STENCILZPASS__MASK			0x0001c000
-#define A2XX_RB_DEPTHCONTROL_STENCILZPASS__SHIFT		14
-static inline uint32_t A2XX_RB_DEPTHCONTROL_STENCILZPASS(enum adreno_stencil_op val)
-{
-	return ((val) << A2XX_RB_DEPTHCONTROL_STENCILZPASS__SHIFT) & A2XX_RB_DEPTHCONTROL_STENCILZPASS__MASK;
-}
-#define A2XX_RB_DEPTHCONTROL_STENCILZFAIL__MASK			0x000e0000
-#define A2XX_RB_DEPTHCONTROL_STENCILZFAIL__SHIFT		17
-static inline uint32_t A2XX_RB_DEPTHCONTROL_STENCILZFAIL(enum adreno_stencil_op val)
-{
-	return ((val) << A2XX_RB_DEPTHCONTROL_STENCILZFAIL__SHIFT) & A2XX_RB_DEPTHCONTROL_STENCILZFAIL__MASK;
-}
-#define A2XX_RB_DEPTHCONTROL_STENCILFUNC_BF__MASK		0x00700000
-#define A2XX_RB_DEPTHCONTROL_STENCILFUNC_BF__SHIFT		20
-static inline uint32_t A2XX_RB_DEPTHCONTROL_STENCILFUNC_BF(enum adreno_compare_func val)
-{
-	return ((val) << A2XX_RB_DEPTHCONTROL_STENCILFUNC_BF__SHIFT) & A2XX_RB_DEPTHCONTROL_STENCILFUNC_BF__MASK;
-}
-#define A2XX_RB_DEPTHCONTROL_STENCILFAIL_BF__MASK		0x03800000
-#define A2XX_RB_DEPTHCONTROL_STENCILFAIL_BF__SHIFT		23
-static inline uint32_t A2XX_RB_DEPTHCONTROL_STENCILFAIL_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A2XX_RB_DEPTHCONTROL_STENCILFAIL_BF__SHIFT) & A2XX_RB_DEPTHCONTROL_STENCILFAIL_BF__MASK;
-}
-#define A2XX_RB_DEPTHCONTROL_STENCILZPASS_BF__MASK		0x1c000000
-#define A2XX_RB_DEPTHCONTROL_STENCILZPASS_BF__SHIFT		26
-static inline uint32_t A2XX_RB_DEPTHCONTROL_STENCILZPASS_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A2XX_RB_DEPTHCONTROL_STENCILZPASS_BF__SHIFT) & A2XX_RB_DEPTHCONTROL_STENCILZPASS_BF__MASK;
-}
-#define A2XX_RB_DEPTHCONTROL_STENCILZFAIL_BF__MASK		0xe0000000
-#define A2XX_RB_DEPTHCONTROL_STENCILZFAIL_BF__SHIFT		29
-static inline uint32_t A2XX_RB_DEPTHCONTROL_STENCILZFAIL_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A2XX_RB_DEPTHCONTROL_STENCILZFAIL_BF__SHIFT) & A2XX_RB_DEPTHCONTROL_STENCILZFAIL_BF__MASK;
-}
-
-#define REG_A2XX_RB_BLEND_CONTROL				0x00002201
-#define A2XX_RB_BLEND_CONTROL_COLOR_SRCBLEND__MASK		0x0000001f
-#define A2XX_RB_BLEND_CONTROL_COLOR_SRCBLEND__SHIFT		0
-static inline uint32_t A2XX_RB_BLEND_CONTROL_COLOR_SRCBLEND(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A2XX_RB_BLEND_CONTROL_COLOR_SRCBLEND__SHIFT) & A2XX_RB_BLEND_CONTROL_COLOR_SRCBLEND__MASK;
-}
-#define A2XX_RB_BLEND_CONTROL_COLOR_COMB_FCN__MASK		0x000000e0
-#define A2XX_RB_BLEND_CONTROL_COLOR_COMB_FCN__SHIFT		5
-static inline uint32_t A2XX_RB_BLEND_CONTROL_COLOR_COMB_FCN(enum a2xx_rb_blend_opcode val)
-{
-	return ((val) << A2XX_RB_BLEND_CONTROL_COLOR_COMB_FCN__SHIFT) & A2XX_RB_BLEND_CONTROL_COLOR_COMB_FCN__MASK;
-}
-#define A2XX_RB_BLEND_CONTROL_COLOR_DESTBLEND__MASK		0x00001f00
-#define A2XX_RB_BLEND_CONTROL_COLOR_DESTBLEND__SHIFT		8
-static inline uint32_t A2XX_RB_BLEND_CONTROL_COLOR_DESTBLEND(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A2XX_RB_BLEND_CONTROL_COLOR_DESTBLEND__SHIFT) & A2XX_RB_BLEND_CONTROL_COLOR_DESTBLEND__MASK;
-}
-#define A2XX_RB_BLEND_CONTROL_ALPHA_SRCBLEND__MASK		0x001f0000
-#define A2XX_RB_BLEND_CONTROL_ALPHA_SRCBLEND__SHIFT		16
-static inline uint32_t A2XX_RB_BLEND_CONTROL_ALPHA_SRCBLEND(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A2XX_RB_BLEND_CONTROL_ALPHA_SRCBLEND__SHIFT) & A2XX_RB_BLEND_CONTROL_ALPHA_SRCBLEND__MASK;
-}
-#define A2XX_RB_BLEND_CONTROL_ALPHA_COMB_FCN__MASK		0x00e00000
-#define A2XX_RB_BLEND_CONTROL_ALPHA_COMB_FCN__SHIFT		21
-static inline uint32_t A2XX_RB_BLEND_CONTROL_ALPHA_COMB_FCN(enum a2xx_rb_blend_opcode val)
-{
-	return ((val) << A2XX_RB_BLEND_CONTROL_ALPHA_COMB_FCN__SHIFT) & A2XX_RB_BLEND_CONTROL_ALPHA_COMB_FCN__MASK;
-}
-#define A2XX_RB_BLEND_CONTROL_ALPHA_DESTBLEND__MASK		0x1f000000
-#define A2XX_RB_BLEND_CONTROL_ALPHA_DESTBLEND__SHIFT		24
-static inline uint32_t A2XX_RB_BLEND_CONTROL_ALPHA_DESTBLEND(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A2XX_RB_BLEND_CONTROL_ALPHA_DESTBLEND__SHIFT) & A2XX_RB_BLEND_CONTROL_ALPHA_DESTBLEND__MASK;
-}
-#define A2XX_RB_BLEND_CONTROL_BLEND_FORCE_ENABLE		0x20000000
-#define A2XX_RB_BLEND_CONTROL_BLEND_FORCE			0x40000000
-
-#define REG_A2XX_RB_COLORCONTROL				0x00002202
-#define A2XX_RB_COLORCONTROL_ALPHA_FUNC__MASK			0x00000007
-#define A2XX_RB_COLORCONTROL_ALPHA_FUNC__SHIFT			0
-static inline uint32_t A2XX_RB_COLORCONTROL_ALPHA_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A2XX_RB_COLORCONTROL_ALPHA_FUNC__SHIFT) & A2XX_RB_COLORCONTROL_ALPHA_FUNC__MASK;
-}
-#define A2XX_RB_COLORCONTROL_ALPHA_TEST_ENABLE			0x00000008
-#define A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_ENABLE		0x00000010
-#define A2XX_RB_COLORCONTROL_BLEND_DISABLE			0x00000020
-#define A2XX_RB_COLORCONTROL_VOB_ENABLE				0x00000040
-#define A2XX_RB_COLORCONTROL_VS_EXPORTS_FOG			0x00000080
-#define A2XX_RB_COLORCONTROL_ROP_CODE__MASK			0x00000f00
-#define A2XX_RB_COLORCONTROL_ROP_CODE__SHIFT			8
-static inline uint32_t A2XX_RB_COLORCONTROL_ROP_CODE(uint32_t val)
-{
-	return ((val) << A2XX_RB_COLORCONTROL_ROP_CODE__SHIFT) & A2XX_RB_COLORCONTROL_ROP_CODE__MASK;
-}
-#define A2XX_RB_COLORCONTROL_DITHER_MODE__MASK			0x00003000
-#define A2XX_RB_COLORCONTROL_DITHER_MODE__SHIFT			12
-static inline uint32_t A2XX_RB_COLORCONTROL_DITHER_MODE(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A2XX_RB_COLORCONTROL_DITHER_MODE__SHIFT) & A2XX_RB_COLORCONTROL_DITHER_MODE__MASK;
-}
-#define A2XX_RB_COLORCONTROL_DITHER_TYPE__MASK			0x0000c000
-#define A2XX_RB_COLORCONTROL_DITHER_TYPE__SHIFT			14
-static inline uint32_t A2XX_RB_COLORCONTROL_DITHER_TYPE(enum a2xx_rb_dither_type val)
-{
-	return ((val) << A2XX_RB_COLORCONTROL_DITHER_TYPE__SHIFT) & A2XX_RB_COLORCONTROL_DITHER_TYPE__MASK;
-}
-#define A2XX_RB_COLORCONTROL_PIXEL_FOG				0x00010000
-#define A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET0__MASK	0x03000000
-#define A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET0__SHIFT	24
-static inline uint32_t A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET0(uint32_t val)
-{
-	return ((val) << A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET0__SHIFT) & A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET0__MASK;
-}
-#define A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET1__MASK	0x0c000000
-#define A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET1__SHIFT	26
-static inline uint32_t A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET1(uint32_t val)
-{
-	return ((val) << A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET1__SHIFT) & A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET1__MASK;
-}
-#define A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET2__MASK	0x30000000
-#define A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET2__SHIFT	28
-static inline uint32_t A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET2(uint32_t val)
-{
-	return ((val) << A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET2__SHIFT) & A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET2__MASK;
-}
-#define A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET3__MASK	0xc0000000
-#define A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET3__SHIFT	30
-static inline uint32_t A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET3(uint32_t val)
-{
-	return ((val) << A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET3__SHIFT) & A2XX_RB_COLORCONTROL_ALPHA_TO_MASK_OFFSET3__MASK;
-}
-
-#define REG_A2XX_VGT_CURRENT_BIN_ID_MAX				0x00002203
-#define A2XX_VGT_CURRENT_BIN_ID_MAX_COLUMN__MASK		0x00000007
-#define A2XX_VGT_CURRENT_BIN_ID_MAX_COLUMN__SHIFT		0
-static inline uint32_t A2XX_VGT_CURRENT_BIN_ID_MAX_COLUMN(uint32_t val)
-{
-	return ((val) << A2XX_VGT_CURRENT_BIN_ID_MAX_COLUMN__SHIFT) & A2XX_VGT_CURRENT_BIN_ID_MAX_COLUMN__MASK;
-}
-#define A2XX_VGT_CURRENT_BIN_ID_MAX_ROW__MASK			0x00000038
-#define A2XX_VGT_CURRENT_BIN_ID_MAX_ROW__SHIFT			3
-static inline uint32_t A2XX_VGT_CURRENT_BIN_ID_MAX_ROW(uint32_t val)
-{
-	return ((val) << A2XX_VGT_CURRENT_BIN_ID_MAX_ROW__SHIFT) & A2XX_VGT_CURRENT_BIN_ID_MAX_ROW__MASK;
-}
-#define A2XX_VGT_CURRENT_BIN_ID_MAX_GUARD_BAND_MASK__MASK	0x000001c0
-#define A2XX_VGT_CURRENT_BIN_ID_MAX_GUARD_BAND_MASK__SHIFT	6
-static inline uint32_t A2XX_VGT_CURRENT_BIN_ID_MAX_GUARD_BAND_MASK(uint32_t val)
-{
-	return ((val) << A2XX_VGT_CURRENT_BIN_ID_MAX_GUARD_BAND_MASK__SHIFT) & A2XX_VGT_CURRENT_BIN_ID_MAX_GUARD_BAND_MASK__MASK;
-}
-
-#define REG_A2XX_PA_CL_CLIP_CNTL				0x00002204
-#define A2XX_PA_CL_CLIP_CNTL_CLIP_DISABLE			0x00010000
-#define A2XX_PA_CL_CLIP_CNTL_BOUNDARY_EDGE_FLAG_ENA		0x00040000
-#define A2XX_PA_CL_CLIP_CNTL_DX_CLIP_SPACE_DEF__MASK		0x00080000
-#define A2XX_PA_CL_CLIP_CNTL_DX_CLIP_SPACE_DEF__SHIFT		19
-static inline uint32_t A2XX_PA_CL_CLIP_CNTL_DX_CLIP_SPACE_DEF(enum a2xx_dx_clip_space val)
-{
-	return ((val) << A2XX_PA_CL_CLIP_CNTL_DX_CLIP_SPACE_DEF__SHIFT) & A2XX_PA_CL_CLIP_CNTL_DX_CLIP_SPACE_DEF__MASK;
-}
-#define A2XX_PA_CL_CLIP_CNTL_DIS_CLIP_ERR_DETECT		0x00100000
-#define A2XX_PA_CL_CLIP_CNTL_VTX_KILL_OR			0x00200000
-#define A2XX_PA_CL_CLIP_CNTL_XY_NAN_RETAIN			0x00400000
-#define A2XX_PA_CL_CLIP_CNTL_Z_NAN_RETAIN			0x00800000
-#define A2XX_PA_CL_CLIP_CNTL_W_NAN_RETAIN			0x01000000
-
-#define REG_A2XX_PA_SU_SC_MODE_CNTL				0x00002205
-#define A2XX_PA_SU_SC_MODE_CNTL_CULL_FRONT			0x00000001
-#define A2XX_PA_SU_SC_MODE_CNTL_CULL_BACK			0x00000002
-#define A2XX_PA_SU_SC_MODE_CNTL_FACE				0x00000004
-#define A2XX_PA_SU_SC_MODE_CNTL_POLYMODE__MASK			0x00000018
-#define A2XX_PA_SU_SC_MODE_CNTL_POLYMODE__SHIFT			3
-static inline uint32_t A2XX_PA_SU_SC_MODE_CNTL_POLYMODE(enum a2xx_pa_su_sc_polymode val)
-{
-	return ((val) << A2XX_PA_SU_SC_MODE_CNTL_POLYMODE__SHIFT) & A2XX_PA_SU_SC_MODE_CNTL_POLYMODE__MASK;
-}
-#define A2XX_PA_SU_SC_MODE_CNTL_FRONT_PTYPE__MASK		0x000000e0
-#define A2XX_PA_SU_SC_MODE_CNTL_FRONT_PTYPE__SHIFT		5
-static inline uint32_t A2XX_PA_SU_SC_MODE_CNTL_FRONT_PTYPE(enum adreno_pa_su_sc_draw val)
-{
-	return ((val) << A2XX_PA_SU_SC_MODE_CNTL_FRONT_PTYPE__SHIFT) & A2XX_PA_SU_SC_MODE_CNTL_FRONT_PTYPE__MASK;
-}
-#define A2XX_PA_SU_SC_MODE_CNTL_BACK_PTYPE__MASK		0x00000700
-#define A2XX_PA_SU_SC_MODE_CNTL_BACK_PTYPE__SHIFT		8
-static inline uint32_t A2XX_PA_SU_SC_MODE_CNTL_BACK_PTYPE(enum adreno_pa_su_sc_draw val)
-{
-	return ((val) << A2XX_PA_SU_SC_MODE_CNTL_BACK_PTYPE__SHIFT) & A2XX_PA_SU_SC_MODE_CNTL_BACK_PTYPE__MASK;
-}
-#define A2XX_PA_SU_SC_MODE_CNTL_POLY_OFFSET_FRONT_ENABLE	0x00000800
-#define A2XX_PA_SU_SC_MODE_CNTL_POLY_OFFSET_BACK_ENABLE		0x00001000
-#define A2XX_PA_SU_SC_MODE_CNTL_POLY_OFFSET_PARA_ENABLE		0x00002000
-#define A2XX_PA_SU_SC_MODE_CNTL_MSAA_ENABLE			0x00008000
-#define A2XX_PA_SU_SC_MODE_CNTL_VTX_WINDOW_OFFSET_ENABLE	0x00010000
-#define A2XX_PA_SU_SC_MODE_CNTL_LINE_STIPPLE_ENABLE		0x00040000
-#define A2XX_PA_SU_SC_MODE_CNTL_PROVOKING_VTX_LAST		0x00080000
-#define A2XX_PA_SU_SC_MODE_CNTL_PERSP_CORR_DIS			0x00100000
-#define A2XX_PA_SU_SC_MODE_CNTL_MULTI_PRIM_IB_ENA		0x00200000
-#define A2XX_PA_SU_SC_MODE_CNTL_QUAD_ORDER_ENABLE		0x00800000
-#define A2XX_PA_SU_SC_MODE_CNTL_WAIT_RB_IDLE_ALL_TRI		0x02000000
-#define A2XX_PA_SU_SC_MODE_CNTL_WAIT_RB_IDLE_FIRST_TRI_NEW_STATE	0x04000000
-#define A2XX_PA_SU_SC_MODE_CNTL_CLAMPED_FACENESS		0x10000000
-#define A2XX_PA_SU_SC_MODE_CNTL_ZERO_AREA_FACENESS		0x20000000
-#define A2XX_PA_SU_SC_MODE_CNTL_FACE_KILL_ENABLE		0x40000000
-#define A2XX_PA_SU_SC_MODE_CNTL_FACE_WRITE_ENABLE		0x80000000
-
-#define REG_A2XX_PA_CL_VTE_CNTL					0x00002206
-#define A2XX_PA_CL_VTE_CNTL_VPORT_X_SCALE_ENA			0x00000001
-#define A2XX_PA_CL_VTE_CNTL_VPORT_X_OFFSET_ENA			0x00000002
-#define A2XX_PA_CL_VTE_CNTL_VPORT_Y_SCALE_ENA			0x00000004
-#define A2XX_PA_CL_VTE_CNTL_VPORT_Y_OFFSET_ENA			0x00000008
-#define A2XX_PA_CL_VTE_CNTL_VPORT_Z_SCALE_ENA			0x00000010
-#define A2XX_PA_CL_VTE_CNTL_VPORT_Z_OFFSET_ENA			0x00000020
-#define A2XX_PA_CL_VTE_CNTL_VTX_XY_FMT				0x00000100
-#define A2XX_PA_CL_VTE_CNTL_VTX_Z_FMT				0x00000200
-#define A2XX_PA_CL_VTE_CNTL_VTX_W0_FMT				0x00000400
-#define A2XX_PA_CL_VTE_CNTL_PERFCOUNTER_REF			0x00000800
-
-#define REG_A2XX_VGT_CURRENT_BIN_ID_MIN				0x00002207
-#define A2XX_VGT_CURRENT_BIN_ID_MIN_COLUMN__MASK		0x00000007
-#define A2XX_VGT_CURRENT_BIN_ID_MIN_COLUMN__SHIFT		0
-static inline uint32_t A2XX_VGT_CURRENT_BIN_ID_MIN_COLUMN(uint32_t val)
-{
-	return ((val) << A2XX_VGT_CURRENT_BIN_ID_MIN_COLUMN__SHIFT) & A2XX_VGT_CURRENT_BIN_ID_MIN_COLUMN__MASK;
-}
-#define A2XX_VGT_CURRENT_BIN_ID_MIN_ROW__MASK			0x00000038
-#define A2XX_VGT_CURRENT_BIN_ID_MIN_ROW__SHIFT			3
-static inline uint32_t A2XX_VGT_CURRENT_BIN_ID_MIN_ROW(uint32_t val)
-{
-	return ((val) << A2XX_VGT_CURRENT_BIN_ID_MIN_ROW__SHIFT) & A2XX_VGT_CURRENT_BIN_ID_MIN_ROW__MASK;
-}
-#define A2XX_VGT_CURRENT_BIN_ID_MIN_GUARD_BAND_MASK__MASK	0x000001c0
-#define A2XX_VGT_CURRENT_BIN_ID_MIN_GUARD_BAND_MASK__SHIFT	6
-static inline uint32_t A2XX_VGT_CURRENT_BIN_ID_MIN_GUARD_BAND_MASK(uint32_t val)
-{
-	return ((val) << A2XX_VGT_CURRENT_BIN_ID_MIN_GUARD_BAND_MASK__SHIFT) & A2XX_VGT_CURRENT_BIN_ID_MIN_GUARD_BAND_MASK__MASK;
-}
-
-#define REG_A2XX_RB_MODECONTROL					0x00002208
-#define A2XX_RB_MODECONTROL_EDRAM_MODE__MASK			0x00000007
-#define A2XX_RB_MODECONTROL_EDRAM_MODE__SHIFT			0
-static inline uint32_t A2XX_RB_MODECONTROL_EDRAM_MODE(enum a2xx_rb_edram_mode val)
-{
-	return ((val) << A2XX_RB_MODECONTROL_EDRAM_MODE__SHIFT) & A2XX_RB_MODECONTROL_EDRAM_MODE__MASK;
-}
-
-#define REG_A2XX_A220_RB_LRZ_VSC_CONTROL			0x00002209
-
-#define REG_A2XX_RB_SAMPLE_POS					0x0000220a
-
-#define REG_A2XX_CLEAR_COLOR					0x0000220b
-#define A2XX_CLEAR_COLOR_RED__MASK				0x000000ff
-#define A2XX_CLEAR_COLOR_RED__SHIFT				0
-static inline uint32_t A2XX_CLEAR_COLOR_RED(uint32_t val)
-{
-	return ((val) << A2XX_CLEAR_COLOR_RED__SHIFT) & A2XX_CLEAR_COLOR_RED__MASK;
-}
-#define A2XX_CLEAR_COLOR_GREEN__MASK				0x0000ff00
-#define A2XX_CLEAR_COLOR_GREEN__SHIFT				8
-static inline uint32_t A2XX_CLEAR_COLOR_GREEN(uint32_t val)
-{
-	return ((val) << A2XX_CLEAR_COLOR_GREEN__SHIFT) & A2XX_CLEAR_COLOR_GREEN__MASK;
-}
-#define A2XX_CLEAR_COLOR_BLUE__MASK				0x00ff0000
-#define A2XX_CLEAR_COLOR_BLUE__SHIFT				16
-static inline uint32_t A2XX_CLEAR_COLOR_BLUE(uint32_t val)
-{
-	return ((val) << A2XX_CLEAR_COLOR_BLUE__SHIFT) & A2XX_CLEAR_COLOR_BLUE__MASK;
-}
-#define A2XX_CLEAR_COLOR_ALPHA__MASK				0xff000000
-#define A2XX_CLEAR_COLOR_ALPHA__SHIFT				24
-static inline uint32_t A2XX_CLEAR_COLOR_ALPHA(uint32_t val)
-{
-	return ((val) << A2XX_CLEAR_COLOR_ALPHA__SHIFT) & A2XX_CLEAR_COLOR_ALPHA__MASK;
-}
-
-#define REG_A2XX_A220_GRAS_CONTROL				0x00002210
-
-#define REG_A2XX_PA_SU_POINT_SIZE				0x00002280
-#define A2XX_PA_SU_POINT_SIZE_HEIGHT__MASK			0x0000ffff
-#define A2XX_PA_SU_POINT_SIZE_HEIGHT__SHIFT			0
-static inline uint32_t A2XX_PA_SU_POINT_SIZE_HEIGHT(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A2XX_PA_SU_POINT_SIZE_HEIGHT__SHIFT) & A2XX_PA_SU_POINT_SIZE_HEIGHT__MASK;
-}
-#define A2XX_PA_SU_POINT_SIZE_WIDTH__MASK			0xffff0000
-#define A2XX_PA_SU_POINT_SIZE_WIDTH__SHIFT			16
-static inline uint32_t A2XX_PA_SU_POINT_SIZE_WIDTH(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A2XX_PA_SU_POINT_SIZE_WIDTH__SHIFT) & A2XX_PA_SU_POINT_SIZE_WIDTH__MASK;
-}
-
-#define REG_A2XX_PA_SU_POINT_MINMAX				0x00002281
-#define A2XX_PA_SU_POINT_MINMAX_MIN__MASK			0x0000ffff
-#define A2XX_PA_SU_POINT_MINMAX_MIN__SHIFT			0
-static inline uint32_t A2XX_PA_SU_POINT_MINMAX_MIN(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A2XX_PA_SU_POINT_MINMAX_MIN__SHIFT) & A2XX_PA_SU_POINT_MINMAX_MIN__MASK;
-}
-#define A2XX_PA_SU_POINT_MINMAX_MAX__MASK			0xffff0000
-#define A2XX_PA_SU_POINT_MINMAX_MAX__SHIFT			16
-static inline uint32_t A2XX_PA_SU_POINT_MINMAX_MAX(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A2XX_PA_SU_POINT_MINMAX_MAX__SHIFT) & A2XX_PA_SU_POINT_MINMAX_MAX__MASK;
-}
-
-#define REG_A2XX_PA_SU_LINE_CNTL				0x00002282
-#define A2XX_PA_SU_LINE_CNTL_WIDTH__MASK			0x0000ffff
-#define A2XX_PA_SU_LINE_CNTL_WIDTH__SHIFT			0
-static inline uint32_t A2XX_PA_SU_LINE_CNTL_WIDTH(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A2XX_PA_SU_LINE_CNTL_WIDTH__SHIFT) & A2XX_PA_SU_LINE_CNTL_WIDTH__MASK;
-}
-
-#define REG_A2XX_PA_SC_LINE_STIPPLE				0x00002283
-#define A2XX_PA_SC_LINE_STIPPLE_LINE_PATTERN__MASK		0x0000ffff
-#define A2XX_PA_SC_LINE_STIPPLE_LINE_PATTERN__SHIFT		0
-static inline uint32_t A2XX_PA_SC_LINE_STIPPLE_LINE_PATTERN(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_LINE_STIPPLE_LINE_PATTERN__SHIFT) & A2XX_PA_SC_LINE_STIPPLE_LINE_PATTERN__MASK;
-}
-#define A2XX_PA_SC_LINE_STIPPLE_REPEAT_COUNT__MASK		0x00ff0000
-#define A2XX_PA_SC_LINE_STIPPLE_REPEAT_COUNT__SHIFT		16
-static inline uint32_t A2XX_PA_SC_LINE_STIPPLE_REPEAT_COUNT(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_LINE_STIPPLE_REPEAT_COUNT__SHIFT) & A2XX_PA_SC_LINE_STIPPLE_REPEAT_COUNT__MASK;
-}
-#define A2XX_PA_SC_LINE_STIPPLE_PATTERN_BIT_ORDER__MASK		0x10000000
-#define A2XX_PA_SC_LINE_STIPPLE_PATTERN_BIT_ORDER__SHIFT	28
-static inline uint32_t A2XX_PA_SC_LINE_STIPPLE_PATTERN_BIT_ORDER(enum a2xx_pa_sc_pattern_bit_order val)
-{
-	return ((val) << A2XX_PA_SC_LINE_STIPPLE_PATTERN_BIT_ORDER__SHIFT) & A2XX_PA_SC_LINE_STIPPLE_PATTERN_BIT_ORDER__MASK;
-}
-#define A2XX_PA_SC_LINE_STIPPLE_AUTO_RESET_CNTL__MASK		0x60000000
-#define A2XX_PA_SC_LINE_STIPPLE_AUTO_RESET_CNTL__SHIFT		29
-static inline uint32_t A2XX_PA_SC_LINE_STIPPLE_AUTO_RESET_CNTL(enum a2xx_pa_sc_auto_reset_cntl val)
-{
-	return ((val) << A2XX_PA_SC_LINE_STIPPLE_AUTO_RESET_CNTL__SHIFT) & A2XX_PA_SC_LINE_STIPPLE_AUTO_RESET_CNTL__MASK;
-}
-
-#define REG_A2XX_PA_SC_VIZ_QUERY				0x00002293
-#define A2XX_PA_SC_VIZ_QUERY_VIZ_QUERY_ENA			0x00000001
-#define A2XX_PA_SC_VIZ_QUERY_VIZ_QUERY_ID__MASK			0x0000007e
-#define A2XX_PA_SC_VIZ_QUERY_VIZ_QUERY_ID__SHIFT		1
-static inline uint32_t A2XX_PA_SC_VIZ_QUERY_VIZ_QUERY_ID(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_VIZ_QUERY_VIZ_QUERY_ID__SHIFT) & A2XX_PA_SC_VIZ_QUERY_VIZ_QUERY_ID__MASK;
-}
-#define A2XX_PA_SC_VIZ_QUERY_KILL_PIX_POST_EARLY_Z		0x00000100
-
-#define REG_A2XX_VGT_ENHANCE					0x00002294
-
-#define REG_A2XX_PA_SC_LINE_CNTL				0x00002300
-#define A2XX_PA_SC_LINE_CNTL_BRES_CNTL__MASK			0x0000ffff
-#define A2XX_PA_SC_LINE_CNTL_BRES_CNTL__SHIFT			0
-static inline uint32_t A2XX_PA_SC_LINE_CNTL_BRES_CNTL(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_LINE_CNTL_BRES_CNTL__SHIFT) & A2XX_PA_SC_LINE_CNTL_BRES_CNTL__MASK;
-}
-#define A2XX_PA_SC_LINE_CNTL_USE_BRES_CNTL			0x00000100
-#define A2XX_PA_SC_LINE_CNTL_EXPAND_LINE_WIDTH			0x00000200
-#define A2XX_PA_SC_LINE_CNTL_LAST_PIXEL				0x00000400
-
-#define REG_A2XX_PA_SC_AA_CONFIG				0x00002301
-#define A2XX_PA_SC_AA_CONFIG_MSAA_NUM_SAMPLES__MASK		0x00000007
-#define A2XX_PA_SC_AA_CONFIG_MSAA_NUM_SAMPLES__SHIFT		0
-static inline uint32_t A2XX_PA_SC_AA_CONFIG_MSAA_NUM_SAMPLES(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_AA_CONFIG_MSAA_NUM_SAMPLES__SHIFT) & A2XX_PA_SC_AA_CONFIG_MSAA_NUM_SAMPLES__MASK;
-}
-#define A2XX_PA_SC_AA_CONFIG_MAX_SAMPLE_DIST__MASK		0x0001e000
-#define A2XX_PA_SC_AA_CONFIG_MAX_SAMPLE_DIST__SHIFT		13
-static inline uint32_t A2XX_PA_SC_AA_CONFIG_MAX_SAMPLE_DIST(uint32_t val)
-{
-	return ((val) << A2XX_PA_SC_AA_CONFIG_MAX_SAMPLE_DIST__SHIFT) & A2XX_PA_SC_AA_CONFIG_MAX_SAMPLE_DIST__MASK;
-}
-
-#define REG_A2XX_PA_SU_VTX_CNTL					0x00002302
-#define A2XX_PA_SU_VTX_CNTL_PIX_CENTER__MASK			0x00000001
-#define A2XX_PA_SU_VTX_CNTL_PIX_CENTER__SHIFT			0
-static inline uint32_t A2XX_PA_SU_VTX_CNTL_PIX_CENTER(enum a2xx_pa_pixcenter val)
-{
-	return ((val) << A2XX_PA_SU_VTX_CNTL_PIX_CENTER__SHIFT) & A2XX_PA_SU_VTX_CNTL_PIX_CENTER__MASK;
-}
-#define A2XX_PA_SU_VTX_CNTL_ROUND_MODE__MASK			0x00000006
-#define A2XX_PA_SU_VTX_CNTL_ROUND_MODE__SHIFT			1
-static inline uint32_t A2XX_PA_SU_VTX_CNTL_ROUND_MODE(enum a2xx_pa_roundmode val)
-{
-	return ((val) << A2XX_PA_SU_VTX_CNTL_ROUND_MODE__SHIFT) & A2XX_PA_SU_VTX_CNTL_ROUND_MODE__MASK;
-}
-#define A2XX_PA_SU_VTX_CNTL_QUANT_MODE__MASK			0x00000380
-#define A2XX_PA_SU_VTX_CNTL_QUANT_MODE__SHIFT			7
-static inline uint32_t A2XX_PA_SU_VTX_CNTL_QUANT_MODE(enum a2xx_pa_quantmode val)
-{
-	return ((val) << A2XX_PA_SU_VTX_CNTL_QUANT_MODE__SHIFT) & A2XX_PA_SU_VTX_CNTL_QUANT_MODE__MASK;
-}
-
-#define REG_A2XX_PA_CL_GB_VERT_CLIP_ADJ				0x00002303
-#define A2XX_PA_CL_GB_VERT_CLIP_ADJ__MASK			0xffffffff
-#define A2XX_PA_CL_GB_VERT_CLIP_ADJ__SHIFT			0
-static inline uint32_t A2XX_PA_CL_GB_VERT_CLIP_ADJ(float val)
-{
-	return ((fui(val)) << A2XX_PA_CL_GB_VERT_CLIP_ADJ__SHIFT) & A2XX_PA_CL_GB_VERT_CLIP_ADJ__MASK;
-}
-
-#define REG_A2XX_PA_CL_GB_VERT_DISC_ADJ				0x00002304
-#define A2XX_PA_CL_GB_VERT_DISC_ADJ__MASK			0xffffffff
-#define A2XX_PA_CL_GB_VERT_DISC_ADJ__SHIFT			0
-static inline uint32_t A2XX_PA_CL_GB_VERT_DISC_ADJ(float val)
-{
-	return ((fui(val)) << A2XX_PA_CL_GB_VERT_DISC_ADJ__SHIFT) & A2XX_PA_CL_GB_VERT_DISC_ADJ__MASK;
-}
-
-#define REG_A2XX_PA_CL_GB_HORZ_CLIP_ADJ				0x00002305
-#define A2XX_PA_CL_GB_HORZ_CLIP_ADJ__MASK			0xffffffff
-#define A2XX_PA_CL_GB_HORZ_CLIP_ADJ__SHIFT			0
-static inline uint32_t A2XX_PA_CL_GB_HORZ_CLIP_ADJ(float val)
-{
-	return ((fui(val)) << A2XX_PA_CL_GB_HORZ_CLIP_ADJ__SHIFT) & A2XX_PA_CL_GB_HORZ_CLIP_ADJ__MASK;
-}
-
-#define REG_A2XX_PA_CL_GB_HORZ_DISC_ADJ				0x00002306
-#define A2XX_PA_CL_GB_HORZ_DISC_ADJ__MASK			0xffffffff
-#define A2XX_PA_CL_GB_HORZ_DISC_ADJ__SHIFT			0
-static inline uint32_t A2XX_PA_CL_GB_HORZ_DISC_ADJ(float val)
-{
-	return ((fui(val)) << A2XX_PA_CL_GB_HORZ_DISC_ADJ__SHIFT) & A2XX_PA_CL_GB_HORZ_DISC_ADJ__MASK;
-}
-
-#define REG_A2XX_SQ_VS_CONST					0x00002307
-#define A2XX_SQ_VS_CONST_BASE__MASK				0x000001ff
-#define A2XX_SQ_VS_CONST_BASE__SHIFT				0
-static inline uint32_t A2XX_SQ_VS_CONST_BASE(uint32_t val)
-{
-	return ((val) << A2XX_SQ_VS_CONST_BASE__SHIFT) & A2XX_SQ_VS_CONST_BASE__MASK;
-}
-#define A2XX_SQ_VS_CONST_SIZE__MASK				0x001ff000
-#define A2XX_SQ_VS_CONST_SIZE__SHIFT				12
-static inline uint32_t A2XX_SQ_VS_CONST_SIZE(uint32_t val)
-{
-	return ((val) << A2XX_SQ_VS_CONST_SIZE__SHIFT) & A2XX_SQ_VS_CONST_SIZE__MASK;
-}
-
-#define REG_A2XX_SQ_PS_CONST					0x00002308
-#define A2XX_SQ_PS_CONST_BASE__MASK				0x000001ff
-#define A2XX_SQ_PS_CONST_BASE__SHIFT				0
-static inline uint32_t A2XX_SQ_PS_CONST_BASE(uint32_t val)
-{
-	return ((val) << A2XX_SQ_PS_CONST_BASE__SHIFT) & A2XX_SQ_PS_CONST_BASE__MASK;
-}
-#define A2XX_SQ_PS_CONST_SIZE__MASK				0x001ff000
-#define A2XX_SQ_PS_CONST_SIZE__SHIFT				12
-static inline uint32_t A2XX_SQ_PS_CONST_SIZE(uint32_t val)
-{
-	return ((val) << A2XX_SQ_PS_CONST_SIZE__SHIFT) & A2XX_SQ_PS_CONST_SIZE__MASK;
-}
-
-#define REG_A2XX_SQ_DEBUG_MISC_0				0x00002309
-
-#define REG_A2XX_SQ_DEBUG_MISC_1				0x0000230a
-
-#define REG_A2XX_PA_SC_AA_MASK					0x00002312
-
-#define REG_A2XX_VGT_VERTEX_REUSE_BLOCK_CNTL			0x00002316
-#define A2XX_VGT_VERTEX_REUSE_BLOCK_CNTL_VTX_REUSE_DEPTH__MASK	0x00000007
-#define A2XX_VGT_VERTEX_REUSE_BLOCK_CNTL_VTX_REUSE_DEPTH__SHIFT	0
-static inline uint32_t A2XX_VGT_VERTEX_REUSE_BLOCK_CNTL_VTX_REUSE_DEPTH(uint32_t val)
-{
-	return ((val) << A2XX_VGT_VERTEX_REUSE_BLOCK_CNTL_VTX_REUSE_DEPTH__SHIFT) & A2XX_VGT_VERTEX_REUSE_BLOCK_CNTL_VTX_REUSE_DEPTH__MASK;
-}
-
-#define REG_A2XX_VGT_OUT_DEALLOC_CNTL				0x00002317
-#define A2XX_VGT_OUT_DEALLOC_CNTL_DEALLOC_DIST__MASK		0x00000003
-#define A2XX_VGT_OUT_DEALLOC_CNTL_DEALLOC_DIST__SHIFT		0
-static inline uint32_t A2XX_VGT_OUT_DEALLOC_CNTL_DEALLOC_DIST(uint32_t val)
-{
-	return ((val) << A2XX_VGT_OUT_DEALLOC_CNTL_DEALLOC_DIST__SHIFT) & A2XX_VGT_OUT_DEALLOC_CNTL_DEALLOC_DIST__MASK;
-}
-
-#define REG_A2XX_RB_COPY_CONTROL				0x00002318
-#define A2XX_RB_COPY_CONTROL_COPY_SAMPLE_SELECT__MASK		0x00000007
-#define A2XX_RB_COPY_CONTROL_COPY_SAMPLE_SELECT__SHIFT		0
-static inline uint32_t A2XX_RB_COPY_CONTROL_COPY_SAMPLE_SELECT(enum a2xx_rb_copy_sample_select val)
-{
-	return ((val) << A2XX_RB_COPY_CONTROL_COPY_SAMPLE_SELECT__SHIFT) & A2XX_RB_COPY_CONTROL_COPY_SAMPLE_SELECT__MASK;
-}
-#define A2XX_RB_COPY_CONTROL_DEPTH_CLEAR_ENABLE			0x00000008
-#define A2XX_RB_COPY_CONTROL_CLEAR_MASK__MASK			0x000000f0
-#define A2XX_RB_COPY_CONTROL_CLEAR_MASK__SHIFT			4
-static inline uint32_t A2XX_RB_COPY_CONTROL_CLEAR_MASK(uint32_t val)
-{
-	return ((val) << A2XX_RB_COPY_CONTROL_CLEAR_MASK__SHIFT) & A2XX_RB_COPY_CONTROL_CLEAR_MASK__MASK;
-}
-
-#define REG_A2XX_RB_COPY_DEST_BASE				0x00002319
-
-#define REG_A2XX_RB_COPY_DEST_PITCH				0x0000231a
-#define A2XX_RB_COPY_DEST_PITCH__MASK				0xffffffff
-#define A2XX_RB_COPY_DEST_PITCH__SHIFT				0
-static inline uint32_t A2XX_RB_COPY_DEST_PITCH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A2XX_RB_COPY_DEST_PITCH__SHIFT) & A2XX_RB_COPY_DEST_PITCH__MASK;
-}
-
-#define REG_A2XX_RB_COPY_DEST_INFO				0x0000231b
-#define A2XX_RB_COPY_DEST_INFO_DEST_ENDIAN__MASK		0x00000007
-#define A2XX_RB_COPY_DEST_INFO_DEST_ENDIAN__SHIFT		0
-static inline uint32_t A2XX_RB_COPY_DEST_INFO_DEST_ENDIAN(enum adreno_rb_surface_endian val)
-{
-	return ((val) << A2XX_RB_COPY_DEST_INFO_DEST_ENDIAN__SHIFT) & A2XX_RB_COPY_DEST_INFO_DEST_ENDIAN__MASK;
-}
-#define A2XX_RB_COPY_DEST_INFO_LINEAR				0x00000008
-#define A2XX_RB_COPY_DEST_INFO_FORMAT__MASK			0x000000f0
-#define A2XX_RB_COPY_DEST_INFO_FORMAT__SHIFT			4
-static inline uint32_t A2XX_RB_COPY_DEST_INFO_FORMAT(enum a2xx_colorformatx val)
-{
-	return ((val) << A2XX_RB_COPY_DEST_INFO_FORMAT__SHIFT) & A2XX_RB_COPY_DEST_INFO_FORMAT__MASK;
-}
-#define A2XX_RB_COPY_DEST_INFO_SWAP__MASK			0x00000300
-#define A2XX_RB_COPY_DEST_INFO_SWAP__SHIFT			8
-static inline uint32_t A2XX_RB_COPY_DEST_INFO_SWAP(uint32_t val)
-{
-	return ((val) << A2XX_RB_COPY_DEST_INFO_SWAP__SHIFT) & A2XX_RB_COPY_DEST_INFO_SWAP__MASK;
-}
-#define A2XX_RB_COPY_DEST_INFO_DITHER_MODE__MASK		0x00000c00
-#define A2XX_RB_COPY_DEST_INFO_DITHER_MODE__SHIFT		10
-static inline uint32_t A2XX_RB_COPY_DEST_INFO_DITHER_MODE(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A2XX_RB_COPY_DEST_INFO_DITHER_MODE__SHIFT) & A2XX_RB_COPY_DEST_INFO_DITHER_MODE__MASK;
-}
-#define A2XX_RB_COPY_DEST_INFO_DITHER_TYPE__MASK		0x00003000
-#define A2XX_RB_COPY_DEST_INFO_DITHER_TYPE__SHIFT		12
-static inline uint32_t A2XX_RB_COPY_DEST_INFO_DITHER_TYPE(enum a2xx_rb_dither_type val)
-{
-	return ((val) << A2XX_RB_COPY_DEST_INFO_DITHER_TYPE__SHIFT) & A2XX_RB_COPY_DEST_INFO_DITHER_TYPE__MASK;
-}
-#define A2XX_RB_COPY_DEST_INFO_WRITE_RED			0x00004000
-#define A2XX_RB_COPY_DEST_INFO_WRITE_GREEN			0x00008000
-#define A2XX_RB_COPY_DEST_INFO_WRITE_BLUE			0x00010000
-#define A2XX_RB_COPY_DEST_INFO_WRITE_ALPHA			0x00020000
-
-#define REG_A2XX_RB_COPY_DEST_OFFSET				0x0000231c
-#define A2XX_RB_COPY_DEST_OFFSET_X__MASK			0x00001fff
-#define A2XX_RB_COPY_DEST_OFFSET_X__SHIFT			0
-static inline uint32_t A2XX_RB_COPY_DEST_OFFSET_X(uint32_t val)
-{
-	return ((val) << A2XX_RB_COPY_DEST_OFFSET_X__SHIFT) & A2XX_RB_COPY_DEST_OFFSET_X__MASK;
-}
-#define A2XX_RB_COPY_DEST_OFFSET_Y__MASK			0x03ffe000
-#define A2XX_RB_COPY_DEST_OFFSET_Y__SHIFT			13
-static inline uint32_t A2XX_RB_COPY_DEST_OFFSET_Y(uint32_t val)
-{
-	return ((val) << A2XX_RB_COPY_DEST_OFFSET_Y__SHIFT) & A2XX_RB_COPY_DEST_OFFSET_Y__MASK;
-}
-
-#define REG_A2XX_RB_DEPTH_CLEAR					0x0000231d
-
-#define REG_A2XX_RB_SAMPLE_COUNT_CTL				0x00002324
-
-#define REG_A2XX_RB_COLOR_DEST_MASK				0x00002326
-
-#define REG_A2XX_A225_GRAS_UCP0X				0x00002340
-
-#define REG_A2XX_A225_GRAS_UCP5W				0x00002357
-
-#define REG_A2XX_A225_GRAS_UCP_ENABLED				0x00002360
-
-#define REG_A2XX_PA_SU_POLY_OFFSET_FRONT_SCALE			0x00002380
-
-#define REG_A2XX_PA_SU_POLY_OFFSET_FRONT_OFFSET			0x00002381
-
-#define REG_A2XX_PA_SU_POLY_OFFSET_BACK_SCALE			0x00002382
-
-#define REG_A2XX_PA_SU_POLY_OFFSET_BACK_OFFSET			0x00002383
-
-#define REG_A2XX_SQ_CONSTANT_0					0x00004000
-
-#define REG_A2XX_SQ_FETCH_0					0x00004800
-
-#define REG_A2XX_SQ_CF_BOOLEANS					0x00004900
-
-#define REG_A2XX_SQ_CF_LOOP					0x00004908
-
-#define REG_A2XX_COHER_SIZE_PM4					0x00000a29
-
-#define REG_A2XX_COHER_BASE_PM4					0x00000a2a
-
-#define REG_A2XX_COHER_STATUS_PM4				0x00000a2b
-
-#define REG_A2XX_PA_SU_PERFCOUNTER0_SELECT			0x00000c88
-
-#define REG_A2XX_PA_SU_PERFCOUNTER1_SELECT			0x00000c89
-
-#define REG_A2XX_PA_SU_PERFCOUNTER2_SELECT			0x00000c8a
-
-#define REG_A2XX_PA_SU_PERFCOUNTER3_SELECT			0x00000c8b
-
-#define REG_A2XX_PA_SU_PERFCOUNTER0_LOW				0x00000c8c
-
-#define REG_A2XX_PA_SU_PERFCOUNTER0_HI				0x00000c8d
-
-#define REG_A2XX_PA_SU_PERFCOUNTER1_LOW				0x00000c8e
-
-#define REG_A2XX_PA_SU_PERFCOUNTER1_HI				0x00000c8f
-
-#define REG_A2XX_PA_SU_PERFCOUNTER2_LOW				0x00000c90
-
-#define REG_A2XX_PA_SU_PERFCOUNTER2_HI				0x00000c91
-
-#define REG_A2XX_PA_SU_PERFCOUNTER3_LOW				0x00000c92
-
-#define REG_A2XX_PA_SU_PERFCOUNTER3_HI				0x00000c93
-
-#define REG_A2XX_PA_SC_PERFCOUNTER0_SELECT			0x00000c98
-
-#define REG_A2XX_PA_SC_PERFCOUNTER0_LOW				0x00000c99
-
-#define REG_A2XX_PA_SC_PERFCOUNTER0_HI				0x00000c9a
-
-#define REG_A2XX_VGT_PERFCOUNTER0_SELECT			0x00000c48
-
-#define REG_A2XX_VGT_PERFCOUNTER1_SELECT			0x00000c49
-
-#define REG_A2XX_VGT_PERFCOUNTER2_SELECT			0x00000c4a
-
-#define REG_A2XX_VGT_PERFCOUNTER3_SELECT			0x00000c4b
-
-#define REG_A2XX_VGT_PERFCOUNTER0_LOW				0x00000c4c
-
-#define REG_A2XX_VGT_PERFCOUNTER1_LOW				0x00000c4e
-
-#define REG_A2XX_VGT_PERFCOUNTER2_LOW				0x00000c50
-
-#define REG_A2XX_VGT_PERFCOUNTER3_LOW				0x00000c52
-
-#define REG_A2XX_VGT_PERFCOUNTER0_HI				0x00000c4d
-
-#define REG_A2XX_VGT_PERFCOUNTER1_HI				0x00000c4f
-
-#define REG_A2XX_VGT_PERFCOUNTER2_HI				0x00000c51
-
-#define REG_A2XX_VGT_PERFCOUNTER3_HI				0x00000c53
-
-#define REG_A2XX_TCR_PERFCOUNTER0_SELECT			0x00000e05
-
-#define REG_A2XX_TCR_PERFCOUNTER1_SELECT			0x00000e08
-
-#define REG_A2XX_TCR_PERFCOUNTER0_HI				0x00000e06
-
-#define REG_A2XX_TCR_PERFCOUNTER1_HI				0x00000e09
-
-#define REG_A2XX_TCR_PERFCOUNTER0_LOW				0x00000e07
-
-#define REG_A2XX_TCR_PERFCOUNTER1_LOW				0x00000e0a
-
-#define REG_A2XX_TP0_PERFCOUNTER0_SELECT			0x00000e1f
-
-#define REG_A2XX_TP0_PERFCOUNTER0_HI				0x00000e20
-
-#define REG_A2XX_TP0_PERFCOUNTER0_LOW				0x00000e21
-
-#define REG_A2XX_TP0_PERFCOUNTER1_SELECT			0x00000e22
-
-#define REG_A2XX_TP0_PERFCOUNTER1_HI				0x00000e23
-
-#define REG_A2XX_TP0_PERFCOUNTER1_LOW				0x00000e24
-
-#define REG_A2XX_TCM_PERFCOUNTER0_SELECT			0x00000e54
-
-#define REG_A2XX_TCM_PERFCOUNTER1_SELECT			0x00000e57
-
-#define REG_A2XX_TCM_PERFCOUNTER0_HI				0x00000e55
-
-#define REG_A2XX_TCM_PERFCOUNTER1_HI				0x00000e58
-
-#define REG_A2XX_TCM_PERFCOUNTER0_LOW				0x00000e56
-
-#define REG_A2XX_TCM_PERFCOUNTER1_LOW				0x00000e59
-
-#define REG_A2XX_TCF_PERFCOUNTER0_SELECT			0x00000e5a
-
-#define REG_A2XX_TCF_PERFCOUNTER1_SELECT			0x00000e5d
-
-#define REG_A2XX_TCF_PERFCOUNTER2_SELECT			0x00000e60
-
-#define REG_A2XX_TCF_PERFCOUNTER3_SELECT			0x00000e63
-
-#define REG_A2XX_TCF_PERFCOUNTER4_SELECT			0x00000e66
-
-#define REG_A2XX_TCF_PERFCOUNTER5_SELECT			0x00000e69
-
-#define REG_A2XX_TCF_PERFCOUNTER6_SELECT			0x00000e6c
-
-#define REG_A2XX_TCF_PERFCOUNTER7_SELECT			0x00000e6f
-
-#define REG_A2XX_TCF_PERFCOUNTER8_SELECT			0x00000e72
-
-#define REG_A2XX_TCF_PERFCOUNTER9_SELECT			0x00000e75
-
-#define REG_A2XX_TCF_PERFCOUNTER10_SELECT			0x00000e78
-
-#define REG_A2XX_TCF_PERFCOUNTER11_SELECT			0x00000e7b
-
-#define REG_A2XX_TCF_PERFCOUNTER0_HI				0x00000e5b
-
-#define REG_A2XX_TCF_PERFCOUNTER1_HI				0x00000e5e
-
-#define REG_A2XX_TCF_PERFCOUNTER2_HI				0x00000e61
-
-#define REG_A2XX_TCF_PERFCOUNTER3_HI				0x00000e64
-
-#define REG_A2XX_TCF_PERFCOUNTER4_HI				0x00000e67
-
-#define REG_A2XX_TCF_PERFCOUNTER5_HI				0x00000e6a
-
-#define REG_A2XX_TCF_PERFCOUNTER6_HI				0x00000e6d
-
-#define REG_A2XX_TCF_PERFCOUNTER7_HI				0x00000e70
-
-#define REG_A2XX_TCF_PERFCOUNTER8_HI				0x00000e73
-
-#define REG_A2XX_TCF_PERFCOUNTER9_HI				0x00000e76
-
-#define REG_A2XX_TCF_PERFCOUNTER10_HI				0x00000e79
-
-#define REG_A2XX_TCF_PERFCOUNTER11_HI				0x00000e7c
-
-#define REG_A2XX_TCF_PERFCOUNTER0_LOW				0x00000e5c
-
-#define REG_A2XX_TCF_PERFCOUNTER1_LOW				0x00000e5f
-
-#define REG_A2XX_TCF_PERFCOUNTER2_LOW				0x00000e62
-
-#define REG_A2XX_TCF_PERFCOUNTER3_LOW				0x00000e65
-
-#define REG_A2XX_TCF_PERFCOUNTER4_LOW				0x00000e68
-
-#define REG_A2XX_TCF_PERFCOUNTER5_LOW				0x00000e6b
-
-#define REG_A2XX_TCF_PERFCOUNTER6_LOW				0x00000e6e
-
-#define REG_A2XX_TCF_PERFCOUNTER7_LOW				0x00000e71
-
-#define REG_A2XX_TCF_PERFCOUNTER8_LOW				0x00000e74
-
-#define REG_A2XX_TCF_PERFCOUNTER9_LOW				0x00000e77
-
-#define REG_A2XX_TCF_PERFCOUNTER10_LOW				0x00000e7a
-
-#define REG_A2XX_TCF_PERFCOUNTER11_LOW				0x00000e7d
-
-#define REG_A2XX_SQ_PERFCOUNTER0_SELECT				0x00000dc8
-
-#define REG_A2XX_SQ_PERFCOUNTER1_SELECT				0x00000dc9
-
-#define REG_A2XX_SQ_PERFCOUNTER2_SELECT				0x00000dca
-
-#define REG_A2XX_SQ_PERFCOUNTER3_SELECT				0x00000dcb
-
-#define REG_A2XX_SQ_PERFCOUNTER0_LOW				0x00000dcc
-
-#define REG_A2XX_SQ_PERFCOUNTER0_HI				0x00000dcd
-
-#define REG_A2XX_SQ_PERFCOUNTER1_LOW				0x00000dce
-
-#define REG_A2XX_SQ_PERFCOUNTER1_HI				0x00000dcf
-
-#define REG_A2XX_SQ_PERFCOUNTER2_LOW				0x00000dd0
-
-#define REG_A2XX_SQ_PERFCOUNTER2_HI				0x00000dd1
-
-#define REG_A2XX_SQ_PERFCOUNTER3_LOW				0x00000dd2
-
-#define REG_A2XX_SQ_PERFCOUNTER3_HI				0x00000dd3
-
-#define REG_A2XX_SX_PERFCOUNTER0_SELECT				0x00000dd4
-
-#define REG_A2XX_SX_PERFCOUNTER0_LOW				0x00000dd8
-
-#define REG_A2XX_SX_PERFCOUNTER0_HI				0x00000dd9
-
-#define REG_A2XX_MH_PERFCOUNTER0_SELECT				0x00000a46
-
-#define REG_A2XX_MH_PERFCOUNTER1_SELECT				0x00000a4a
-
-#define REG_A2XX_MH_PERFCOUNTER0_CONFIG				0x00000a47
-
-#define REG_A2XX_MH_PERFCOUNTER1_CONFIG				0x00000a4b
-
-#define REG_A2XX_MH_PERFCOUNTER0_LOW				0x00000a48
-
-#define REG_A2XX_MH_PERFCOUNTER1_LOW				0x00000a4c
-
-#define REG_A2XX_MH_PERFCOUNTER0_HI				0x00000a49
-
-#define REG_A2XX_MH_PERFCOUNTER1_HI				0x00000a4d
-
-#define REG_A2XX_RB_PERFCOUNTER0_SELECT				0x00000f04
-
-#define REG_A2XX_RB_PERFCOUNTER1_SELECT				0x00000f05
-
-#define REG_A2XX_RB_PERFCOUNTER2_SELECT				0x00000f06
-
-#define REG_A2XX_RB_PERFCOUNTER3_SELECT				0x00000f07
-
-#define REG_A2XX_RB_PERFCOUNTER0_LOW				0x00000f08
-
-#define REG_A2XX_RB_PERFCOUNTER0_HI				0x00000f09
-
-#define REG_A2XX_RB_PERFCOUNTER1_LOW				0x00000f0a
-
-#define REG_A2XX_RB_PERFCOUNTER1_HI				0x00000f0b
-
-#define REG_A2XX_RB_PERFCOUNTER2_LOW				0x00000f0c
-
-#define REG_A2XX_RB_PERFCOUNTER2_HI				0x00000f0d
-
-#define REG_A2XX_RB_PERFCOUNTER3_LOW				0x00000f0e
-
-#define REG_A2XX_RB_PERFCOUNTER3_HI				0x00000f0f
-
-#define REG_A2XX_SQ_TEX_0					0x00000000
-#define A2XX_SQ_TEX_0_TYPE__MASK				0x00000003
-#define A2XX_SQ_TEX_0_TYPE__SHIFT				0
-static inline uint32_t A2XX_SQ_TEX_0_TYPE(enum sq_tex_type val)
-{
-	return ((val) << A2XX_SQ_TEX_0_TYPE__SHIFT) & A2XX_SQ_TEX_0_TYPE__MASK;
-}
-#define A2XX_SQ_TEX_0_SIGN_X__MASK				0x0000000c
-#define A2XX_SQ_TEX_0_SIGN_X__SHIFT				2
-static inline uint32_t A2XX_SQ_TEX_0_SIGN_X(enum sq_tex_sign val)
-{
-	return ((val) << A2XX_SQ_TEX_0_SIGN_X__SHIFT) & A2XX_SQ_TEX_0_SIGN_X__MASK;
-}
-#define A2XX_SQ_TEX_0_SIGN_Y__MASK				0x00000030
-#define A2XX_SQ_TEX_0_SIGN_Y__SHIFT				4
-static inline uint32_t A2XX_SQ_TEX_0_SIGN_Y(enum sq_tex_sign val)
-{
-	return ((val) << A2XX_SQ_TEX_0_SIGN_Y__SHIFT) & A2XX_SQ_TEX_0_SIGN_Y__MASK;
-}
-#define A2XX_SQ_TEX_0_SIGN_Z__MASK				0x000000c0
-#define A2XX_SQ_TEX_0_SIGN_Z__SHIFT				6
-static inline uint32_t A2XX_SQ_TEX_0_SIGN_Z(enum sq_tex_sign val)
-{
-	return ((val) << A2XX_SQ_TEX_0_SIGN_Z__SHIFT) & A2XX_SQ_TEX_0_SIGN_Z__MASK;
-}
-#define A2XX_SQ_TEX_0_SIGN_W__MASK				0x00000300
-#define A2XX_SQ_TEX_0_SIGN_W__SHIFT				8
-static inline uint32_t A2XX_SQ_TEX_0_SIGN_W(enum sq_tex_sign val)
-{
-	return ((val) << A2XX_SQ_TEX_0_SIGN_W__SHIFT) & A2XX_SQ_TEX_0_SIGN_W__MASK;
-}
-#define A2XX_SQ_TEX_0_CLAMP_X__MASK				0x00001c00
-#define A2XX_SQ_TEX_0_CLAMP_X__SHIFT				10
-static inline uint32_t A2XX_SQ_TEX_0_CLAMP_X(enum sq_tex_clamp val)
-{
-	return ((val) << A2XX_SQ_TEX_0_CLAMP_X__SHIFT) & A2XX_SQ_TEX_0_CLAMP_X__MASK;
-}
-#define A2XX_SQ_TEX_0_CLAMP_Y__MASK				0x0000e000
-#define A2XX_SQ_TEX_0_CLAMP_Y__SHIFT				13
-static inline uint32_t A2XX_SQ_TEX_0_CLAMP_Y(enum sq_tex_clamp val)
-{
-	return ((val) << A2XX_SQ_TEX_0_CLAMP_Y__SHIFT) & A2XX_SQ_TEX_0_CLAMP_Y__MASK;
-}
-#define A2XX_SQ_TEX_0_CLAMP_Z__MASK				0x00070000
-#define A2XX_SQ_TEX_0_CLAMP_Z__SHIFT				16
-static inline uint32_t A2XX_SQ_TEX_0_CLAMP_Z(enum sq_tex_clamp val)
-{
-	return ((val) << A2XX_SQ_TEX_0_CLAMP_Z__SHIFT) & A2XX_SQ_TEX_0_CLAMP_Z__MASK;
-}
-#define A2XX_SQ_TEX_0_PITCH__MASK				0x7fc00000
-#define A2XX_SQ_TEX_0_PITCH__SHIFT				22
-static inline uint32_t A2XX_SQ_TEX_0_PITCH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A2XX_SQ_TEX_0_PITCH__SHIFT) & A2XX_SQ_TEX_0_PITCH__MASK;
-}
-#define A2XX_SQ_TEX_0_TILED					0x80000000
-
-#define REG_A2XX_SQ_TEX_1					0x00000001
-#define A2XX_SQ_TEX_1_FORMAT__MASK				0x0000003f
-#define A2XX_SQ_TEX_1_FORMAT__SHIFT				0
-static inline uint32_t A2XX_SQ_TEX_1_FORMAT(enum a2xx_sq_surfaceformat val)
-{
-	return ((val) << A2XX_SQ_TEX_1_FORMAT__SHIFT) & A2XX_SQ_TEX_1_FORMAT__MASK;
-}
-#define A2XX_SQ_TEX_1_ENDIANNESS__MASK				0x000000c0
-#define A2XX_SQ_TEX_1_ENDIANNESS__SHIFT				6
-static inline uint32_t A2XX_SQ_TEX_1_ENDIANNESS(enum sq_tex_endian val)
-{
-	return ((val) << A2XX_SQ_TEX_1_ENDIANNESS__SHIFT) & A2XX_SQ_TEX_1_ENDIANNESS__MASK;
-}
-#define A2XX_SQ_TEX_1_REQUEST_SIZE__MASK			0x00000300
-#define A2XX_SQ_TEX_1_REQUEST_SIZE__SHIFT			8
-static inline uint32_t A2XX_SQ_TEX_1_REQUEST_SIZE(uint32_t val)
-{
-	return ((val) << A2XX_SQ_TEX_1_REQUEST_SIZE__SHIFT) & A2XX_SQ_TEX_1_REQUEST_SIZE__MASK;
-}
-#define A2XX_SQ_TEX_1_STACKED					0x00000400
-#define A2XX_SQ_TEX_1_CLAMP_POLICY__MASK			0x00000800
-#define A2XX_SQ_TEX_1_CLAMP_POLICY__SHIFT			11
-static inline uint32_t A2XX_SQ_TEX_1_CLAMP_POLICY(enum sq_tex_clamp_policy val)
-{
-	return ((val) << A2XX_SQ_TEX_1_CLAMP_POLICY__SHIFT) & A2XX_SQ_TEX_1_CLAMP_POLICY__MASK;
-}
-#define A2XX_SQ_TEX_1_BASE_ADDRESS__MASK			0xfffff000
-#define A2XX_SQ_TEX_1_BASE_ADDRESS__SHIFT			12
-static inline uint32_t A2XX_SQ_TEX_1_BASE_ADDRESS(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A2XX_SQ_TEX_1_BASE_ADDRESS__SHIFT) & A2XX_SQ_TEX_1_BASE_ADDRESS__MASK;
-}
-
-#define REG_A2XX_SQ_TEX_2					0x00000002
-#define A2XX_SQ_TEX_2_WIDTH__MASK				0x00001fff
-#define A2XX_SQ_TEX_2_WIDTH__SHIFT				0
-static inline uint32_t A2XX_SQ_TEX_2_WIDTH(uint32_t val)
-{
-	return ((val) << A2XX_SQ_TEX_2_WIDTH__SHIFT) & A2XX_SQ_TEX_2_WIDTH__MASK;
-}
-#define A2XX_SQ_TEX_2_HEIGHT__MASK				0x03ffe000
-#define A2XX_SQ_TEX_2_HEIGHT__SHIFT				13
-static inline uint32_t A2XX_SQ_TEX_2_HEIGHT(uint32_t val)
-{
-	return ((val) << A2XX_SQ_TEX_2_HEIGHT__SHIFT) & A2XX_SQ_TEX_2_HEIGHT__MASK;
-}
-#define A2XX_SQ_TEX_2_DEPTH__MASK				0xfc000000
-#define A2XX_SQ_TEX_2_DEPTH__SHIFT				26
-static inline uint32_t A2XX_SQ_TEX_2_DEPTH(uint32_t val)
-{
-	return ((val) << A2XX_SQ_TEX_2_DEPTH__SHIFT) & A2XX_SQ_TEX_2_DEPTH__MASK;
-}
-
-#define REG_A2XX_SQ_TEX_3					0x00000003
-#define A2XX_SQ_TEX_3_NUM_FORMAT__MASK				0x00000001
-#define A2XX_SQ_TEX_3_NUM_FORMAT__SHIFT				0
-static inline uint32_t A2XX_SQ_TEX_3_NUM_FORMAT(enum sq_tex_num_format val)
-{
-	return ((val) << A2XX_SQ_TEX_3_NUM_FORMAT__SHIFT) & A2XX_SQ_TEX_3_NUM_FORMAT__MASK;
-}
-#define A2XX_SQ_TEX_3_SWIZ_X__MASK				0x0000000e
-#define A2XX_SQ_TEX_3_SWIZ_X__SHIFT				1
-static inline uint32_t A2XX_SQ_TEX_3_SWIZ_X(enum sq_tex_swiz val)
-{
-	return ((val) << A2XX_SQ_TEX_3_SWIZ_X__SHIFT) & A2XX_SQ_TEX_3_SWIZ_X__MASK;
-}
-#define A2XX_SQ_TEX_3_SWIZ_Y__MASK				0x00000070
-#define A2XX_SQ_TEX_3_SWIZ_Y__SHIFT				4
-static inline uint32_t A2XX_SQ_TEX_3_SWIZ_Y(enum sq_tex_swiz val)
-{
-	return ((val) << A2XX_SQ_TEX_3_SWIZ_Y__SHIFT) & A2XX_SQ_TEX_3_SWIZ_Y__MASK;
-}
-#define A2XX_SQ_TEX_3_SWIZ_Z__MASK				0x00000380
-#define A2XX_SQ_TEX_3_SWIZ_Z__SHIFT				7
-static inline uint32_t A2XX_SQ_TEX_3_SWIZ_Z(enum sq_tex_swiz val)
-{
-	return ((val) << A2XX_SQ_TEX_3_SWIZ_Z__SHIFT) & A2XX_SQ_TEX_3_SWIZ_Z__MASK;
-}
-#define A2XX_SQ_TEX_3_SWIZ_W__MASK				0x00001c00
-#define A2XX_SQ_TEX_3_SWIZ_W__SHIFT				10
-static inline uint32_t A2XX_SQ_TEX_3_SWIZ_W(enum sq_tex_swiz val)
-{
-	return ((val) << A2XX_SQ_TEX_3_SWIZ_W__SHIFT) & A2XX_SQ_TEX_3_SWIZ_W__MASK;
-}
-#define A2XX_SQ_TEX_3_EXP_ADJUST__MASK				0x0007e000
-#define A2XX_SQ_TEX_3_EXP_ADJUST__SHIFT				13
-static inline uint32_t A2XX_SQ_TEX_3_EXP_ADJUST(int32_t val)
-{
-	return ((val) << A2XX_SQ_TEX_3_EXP_ADJUST__SHIFT) & A2XX_SQ_TEX_3_EXP_ADJUST__MASK;
-}
-#define A2XX_SQ_TEX_3_XY_MAG_FILTER__MASK			0x00180000
-#define A2XX_SQ_TEX_3_XY_MAG_FILTER__SHIFT			19
-static inline uint32_t A2XX_SQ_TEX_3_XY_MAG_FILTER(enum sq_tex_filter val)
-{
-	return ((val) << A2XX_SQ_TEX_3_XY_MAG_FILTER__SHIFT) & A2XX_SQ_TEX_3_XY_MAG_FILTER__MASK;
-}
-#define A2XX_SQ_TEX_3_XY_MIN_FILTER__MASK			0x00600000
-#define A2XX_SQ_TEX_3_XY_MIN_FILTER__SHIFT			21
-static inline uint32_t A2XX_SQ_TEX_3_XY_MIN_FILTER(enum sq_tex_filter val)
-{
-	return ((val) << A2XX_SQ_TEX_3_XY_MIN_FILTER__SHIFT) & A2XX_SQ_TEX_3_XY_MIN_FILTER__MASK;
-}
-#define A2XX_SQ_TEX_3_MIP_FILTER__MASK				0x01800000
-#define A2XX_SQ_TEX_3_MIP_FILTER__SHIFT				23
-static inline uint32_t A2XX_SQ_TEX_3_MIP_FILTER(enum sq_tex_filter val)
-{
-	return ((val) << A2XX_SQ_TEX_3_MIP_FILTER__SHIFT) & A2XX_SQ_TEX_3_MIP_FILTER__MASK;
-}
-#define A2XX_SQ_TEX_3_ANISO_FILTER__MASK			0x0e000000
-#define A2XX_SQ_TEX_3_ANISO_FILTER__SHIFT			25
-static inline uint32_t A2XX_SQ_TEX_3_ANISO_FILTER(enum sq_tex_aniso_filter val)
-{
-	return ((val) << A2XX_SQ_TEX_3_ANISO_FILTER__SHIFT) & A2XX_SQ_TEX_3_ANISO_FILTER__MASK;
-}
-#define A2XX_SQ_TEX_3_BORDER_SIZE__MASK				0x80000000
-#define A2XX_SQ_TEX_3_BORDER_SIZE__SHIFT			31
-static inline uint32_t A2XX_SQ_TEX_3_BORDER_SIZE(uint32_t val)
-{
-	return ((val) << A2XX_SQ_TEX_3_BORDER_SIZE__SHIFT) & A2XX_SQ_TEX_3_BORDER_SIZE__MASK;
-}
-
-#define REG_A2XX_SQ_TEX_4					0x00000004
-#define A2XX_SQ_TEX_4_VOL_MAG_FILTER__MASK			0x00000001
-#define A2XX_SQ_TEX_4_VOL_MAG_FILTER__SHIFT			0
-static inline uint32_t A2XX_SQ_TEX_4_VOL_MAG_FILTER(enum sq_tex_filter val)
-{
-	return ((val) << A2XX_SQ_TEX_4_VOL_MAG_FILTER__SHIFT) & A2XX_SQ_TEX_4_VOL_MAG_FILTER__MASK;
-}
-#define A2XX_SQ_TEX_4_VOL_MIN_FILTER__MASK			0x00000002
-#define A2XX_SQ_TEX_4_VOL_MIN_FILTER__SHIFT			1
-static inline uint32_t A2XX_SQ_TEX_4_VOL_MIN_FILTER(enum sq_tex_filter val)
-{
-	return ((val) << A2XX_SQ_TEX_4_VOL_MIN_FILTER__SHIFT) & A2XX_SQ_TEX_4_VOL_MIN_FILTER__MASK;
-}
-#define A2XX_SQ_TEX_4_MIP_MIN_LEVEL__MASK			0x0000003c
-#define A2XX_SQ_TEX_4_MIP_MIN_LEVEL__SHIFT			2
-static inline uint32_t A2XX_SQ_TEX_4_MIP_MIN_LEVEL(uint32_t val)
-{
-	return ((val) << A2XX_SQ_TEX_4_MIP_MIN_LEVEL__SHIFT) & A2XX_SQ_TEX_4_MIP_MIN_LEVEL__MASK;
-}
-#define A2XX_SQ_TEX_4_MIP_MAX_LEVEL__MASK			0x000003c0
-#define A2XX_SQ_TEX_4_MIP_MAX_LEVEL__SHIFT			6
-static inline uint32_t A2XX_SQ_TEX_4_MIP_MAX_LEVEL(uint32_t val)
-{
-	return ((val) << A2XX_SQ_TEX_4_MIP_MAX_LEVEL__SHIFT) & A2XX_SQ_TEX_4_MIP_MAX_LEVEL__MASK;
-}
-#define A2XX_SQ_TEX_4_MAX_ANISO_WALK				0x00000400
-#define A2XX_SQ_TEX_4_MIN_ANISO_WALK				0x00000800
-#define A2XX_SQ_TEX_4_LOD_BIAS__MASK				0x003ff000
-#define A2XX_SQ_TEX_4_LOD_BIAS__SHIFT				12
-static inline uint32_t A2XX_SQ_TEX_4_LOD_BIAS(float val)
-{
-	return ((((int32_t)(val * 32.0))) << A2XX_SQ_TEX_4_LOD_BIAS__SHIFT) & A2XX_SQ_TEX_4_LOD_BIAS__MASK;
-}
-#define A2XX_SQ_TEX_4_GRAD_EXP_ADJUST_H__MASK			0x07c00000
-#define A2XX_SQ_TEX_4_GRAD_EXP_ADJUST_H__SHIFT			22
-static inline uint32_t A2XX_SQ_TEX_4_GRAD_EXP_ADJUST_H(uint32_t val)
-{
-	return ((val) << A2XX_SQ_TEX_4_GRAD_EXP_ADJUST_H__SHIFT) & A2XX_SQ_TEX_4_GRAD_EXP_ADJUST_H__MASK;
-}
-#define A2XX_SQ_TEX_4_GRAD_EXP_ADJUST_V__MASK			0xf8000000
-#define A2XX_SQ_TEX_4_GRAD_EXP_ADJUST_V__SHIFT			27
-static inline uint32_t A2XX_SQ_TEX_4_GRAD_EXP_ADJUST_V(uint32_t val)
-{
-	return ((val) << A2XX_SQ_TEX_4_GRAD_EXP_ADJUST_V__SHIFT) & A2XX_SQ_TEX_4_GRAD_EXP_ADJUST_V__MASK;
-}
-
-#define REG_A2XX_SQ_TEX_5					0x00000005
-#define A2XX_SQ_TEX_5_BORDER_COLOR__MASK			0x00000003
-#define A2XX_SQ_TEX_5_BORDER_COLOR__SHIFT			0
-static inline uint32_t A2XX_SQ_TEX_5_BORDER_COLOR(enum sq_tex_border_color val)
-{
-	return ((val) << A2XX_SQ_TEX_5_BORDER_COLOR__SHIFT) & A2XX_SQ_TEX_5_BORDER_COLOR__MASK;
-}
-#define A2XX_SQ_TEX_5_FORCE_BCW_MAX				0x00000004
-#define A2XX_SQ_TEX_5_TRI_CLAMP__MASK				0x00000018
-#define A2XX_SQ_TEX_5_TRI_CLAMP__SHIFT				3
-static inline uint32_t A2XX_SQ_TEX_5_TRI_CLAMP(uint32_t val)
-{
-	return ((val) << A2XX_SQ_TEX_5_TRI_CLAMP__SHIFT) & A2XX_SQ_TEX_5_TRI_CLAMP__MASK;
-}
-#define A2XX_SQ_TEX_5_ANISO_BIAS__MASK				0x000001e0
-#define A2XX_SQ_TEX_5_ANISO_BIAS__SHIFT				5
-static inline uint32_t A2XX_SQ_TEX_5_ANISO_BIAS(float val)
-{
-	return ((((int32_t)(val * 1.0))) << A2XX_SQ_TEX_5_ANISO_BIAS__SHIFT) & A2XX_SQ_TEX_5_ANISO_BIAS__MASK;
-}
-#define A2XX_SQ_TEX_5_DIMENSION__MASK				0x00000600
-#define A2XX_SQ_TEX_5_DIMENSION__SHIFT				9
-static inline uint32_t A2XX_SQ_TEX_5_DIMENSION(enum sq_tex_dimension val)
-{
-	return ((val) << A2XX_SQ_TEX_5_DIMENSION__SHIFT) & A2XX_SQ_TEX_5_DIMENSION__MASK;
-}
-#define A2XX_SQ_TEX_5_PACKED_MIPS				0x00000800
-#define A2XX_SQ_TEX_5_MIP_ADDRESS__MASK				0xfffff000
-#define A2XX_SQ_TEX_5_MIP_ADDRESS__SHIFT			12
-static inline uint32_t A2XX_SQ_TEX_5_MIP_ADDRESS(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A2XX_SQ_TEX_5_MIP_ADDRESS__SHIFT) & A2XX_SQ_TEX_5_MIP_ADDRESS__MASK;
-}
-
-#ifdef __cplusplus
-#endif
-
-#endif /* A2XX_XML */
diff --git a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c
index 0d8133f3174b..0dc255ddf5ce 100644
--- a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c
@@ -113,7 +113,7 @@ static int a2xx_hw_init(struct msm_gpu *gpu)
 	uint32_t *ptr, len;
 	int i, ret;
 
-	msm_gpummu_params(gpu->aspace->mmu, &pt_base, &tran_error);
+	a2xx_gpummu_params(gpu->aspace->mmu, &pt_base, &tran_error);
 
 	DBG("%s", gpu->name);
 
@@ -469,7 +469,7 @@ static struct msm_gpu_state *a2xx_gpu_state_get(struct msm_gpu *gpu)
 static struct msm_gem_address_space *
 a2xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev)
 {
-	struct msm_mmu *mmu = msm_gpummu_new(&pdev->dev, gpu);
+	struct msm_mmu *mmu = a2xx_gpummu_new(&pdev->dev, gpu);
 	struct msm_gem_address_space *aspace;
 
 	aspace = msm_gem_address_space_create(mmu, "gpu", SZ_16M,
diff --git a/drivers/gpu/drm/msm/adreno/a2xx_gpu.h b/drivers/gpu/drm/msm/adreno/a2xx_gpu.h
index 161a075f94af..53702f19990f 100644
--- a/drivers/gpu/drm/msm/adreno/a2xx_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/a2xx_gpu.h
@@ -19,4 +19,8 @@ struct a2xx_gpu {
 };
 #define to_a2xx_gpu(x) container_of(x, struct a2xx_gpu, base)
 
+struct msm_mmu *a2xx_gpummu_new(struct device *dev, struct msm_gpu *gpu);
+void a2xx_gpummu_params(struct msm_mmu *mmu, dma_addr_t *pt_base,
+		dma_addr_t *tran_error);
+
 #endif /* __A2XX_GPU_H__ */
diff --git a/drivers/gpu/drm/msm/msm_gpummu.c b/drivers/gpu/drm/msm/adreno/a2xx_gpummu.c
index f7d1945e0c9f..39641551eeb6 100644
--- a/drivers/gpu/drm/msm/msm_gpummu.c
+++ b/drivers/gpu/drm/msm/adreno/a2xx_gpummu.c
@@ -5,30 +5,33 @@
 
 #include "msm_drv.h"
 #include "msm_mmu.h"
-#include "adreno/adreno_gpu.h"
-#include "adreno/a2xx.xml.h"
 
-struct msm_gpummu {
+#include "adreno_gpu.h"
+#include "a2xx_gpu.h"
+
+#include "a2xx.xml.h"
+
+struct a2xx_gpummu {
 	struct msm_mmu base;
 	struct msm_gpu *gpu;
 	dma_addr_t pt_base;
 	uint32_t *table;
 };
-#define to_msm_gpummu(x) container_of(x, struct msm_gpummu, base)
+#define to_a2xx_gpummu(x) container_of(x, struct a2xx_gpummu, base)
 
 #define GPUMMU_VA_START SZ_16M
 #define GPUMMU_VA_RANGE (0xfff * SZ_64K)
 #define GPUMMU_PAGE_SIZE SZ_4K
 #define TABLE_SIZE (sizeof(uint32_t) * GPUMMU_VA_RANGE / GPUMMU_PAGE_SIZE)
 
-static void msm_gpummu_detach(struct msm_mmu *mmu)
+static void a2xx_gpummu_detach(struct msm_mmu *mmu)
 {
 }
 
-static int msm_gpummu_map(struct msm_mmu *mmu, uint64_t iova,
+static int a2xx_gpummu_map(struct msm_mmu *mmu, uint64_t iova,
 		struct sg_table *sgt, size_t len, int prot)
 {
-	struct msm_gpummu *gpummu = to_msm_gpummu(mmu);
+	struct a2xx_gpummu *gpummu = to_a2xx_gpummu(mmu);
 	unsigned idx = (iova - GPUMMU_VA_START) / GPUMMU_PAGE_SIZE;
 	struct sg_dma_page_iter dma_iter;
 	unsigned prot_bits = 0;
@@ -53,9 +56,9 @@ static int msm_gpummu_map(struct msm_mmu *mmu, uint64_t iova,
 	return 0;
 }
 
-static int msm_gpummu_unmap(struct msm_mmu *mmu, uint64_t iova, size_t len)
+static int a2xx_gpummu_unmap(struct msm_mmu *mmu, uint64_t iova, size_t len)
 {
-	struct msm_gpummu *gpummu = to_msm_gpummu(mmu);
+	struct a2xx_gpummu *gpummu = to_a2xx_gpummu(mmu);
 	unsigned idx = (iova - GPUMMU_VA_START) / GPUMMU_PAGE_SIZE;
 	unsigned i;
 
@@ -68,13 +71,13 @@ static int msm_gpummu_unmap(struct msm_mmu *mmu, uint64_t iova, size_t len)
 	return 0;
 }
 
-static void msm_gpummu_resume_translation(struct msm_mmu *mmu)
+static void a2xx_gpummu_resume_translation(struct msm_mmu *mmu)
 {
 }
 
-static void msm_gpummu_destroy(struct msm_mmu *mmu)
+static void a2xx_gpummu_destroy(struct msm_mmu *mmu)
 {
-	struct msm_gpummu *gpummu = to_msm_gpummu(mmu);
+	struct a2xx_gpummu *gpummu = to_a2xx_gpummu(mmu);
 
 	dma_free_attrs(mmu->dev, TABLE_SIZE, gpummu->table, gpummu->pt_base,
 		DMA_ATTR_FORCE_CONTIGUOUS);
@@ -83,16 +86,16 @@ static void msm_gpummu_destroy(struct msm_mmu *mmu)
 }
 
 static const struct msm_mmu_funcs funcs = {
-		.detach = msm_gpummu_detach,
-		.map = msm_gpummu_map,
-		.unmap = msm_gpummu_unmap,
-		.destroy = msm_gpummu_destroy,
-		.resume_translation = msm_gpummu_resume_translation,
+		.detach = a2xx_gpummu_detach,
+		.map = a2xx_gpummu_map,
+		.unmap = a2xx_gpummu_unmap,
+		.destroy = a2xx_gpummu_destroy,
+		.resume_translation = a2xx_gpummu_resume_translation,
 };
 
-struct msm_mmu *msm_gpummu_new(struct device *dev, struct msm_gpu *gpu)
+struct msm_mmu *a2xx_gpummu_new(struct device *dev, struct msm_gpu *gpu)
 {
-	struct msm_gpummu *gpummu;
+	struct a2xx_gpummu *gpummu;
 
 	gpummu = kzalloc(sizeof(*gpummu), GFP_KERNEL);
 	if (!gpummu)
@@ -111,10 +114,10 @@ struct msm_mmu *msm_gpummu_new(struct device *dev, struct msm_gpu *gpu)
 	return &gpummu->base;
 }
 
-void msm_gpummu_params(struct msm_mmu *mmu, dma_addr_t *pt_base,
+void a2xx_gpummu_params(struct msm_mmu *mmu, dma_addr_t *pt_base,
 		dma_addr_t *tran_error)
 {
-	dma_addr_t base = to_msm_gpummu(mmu)->pt_base;
+	dma_addr_t base = to_a2xx_gpummu(mmu)->pt_base;
 
 	*pt_base = base;
 	*tran_error = base + TABLE_SIZE; /* 32-byte aligned */
diff --git a/drivers/gpu/drm/msm/adreno/a3xx.xml.h b/drivers/gpu/drm/msm/adreno/a3xx.xml.h
deleted file mode 100644
index 5edd740ad3bb..000000000000
--- a/drivers/gpu/drm/msm/adreno/a3xx.xml.h
+++ /dev/null
@@ -1,3268 +0,0 @@
-#ifndef A3XX_XML
-#define A3XX_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng gen_header.py tool in this git repository:
-http://gitlab.freedesktop.org/mesa/mesa/
-git clone https://gitlab.freedesktop.org/mesa/mesa.git
-
-The rules-ng-ng source files this header was generated from are:
-
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a3xx.xml          (  84323 bytes, from Wed Aug 23 10:39:39 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml  (   1572 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml (  15434 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml    (  85691 bytes, from Fri Feb 16 09:49:01 2024)
-
-Copyright (C) 2013-2024 by the following authors:
-- Rob Clark <robdclark@gmail.com> Rob Clark
-- Ilia Mirkin <imirkin@alum.mit.edu> Ilia Mirkin
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-*/
-
-#ifdef __KERNEL__
-#include <linux/bug.h>
-#define assert(x) BUG_ON(!(x))
-#else
-#include <assert.h>
-#endif
-
-#ifdef __cplusplus
-#define __struct_cast(X)
-#else
-#define __struct_cast(X) (struct X)
-#endif
-
-enum a3xx_tile_mode {
-	LINEAR = 0,
-	TILE_4X4 = 1,
-	TILE_32X32 = 2,
-	TILE_4X2 = 3,
-};
-
-enum a3xx_state_block_id {
-	HLSQ_BLOCK_ID_TP_TEX = 2,
-	HLSQ_BLOCK_ID_TP_MIPMAP = 3,
-	HLSQ_BLOCK_ID_SP_VS = 4,
-	HLSQ_BLOCK_ID_SP_FS = 6,
-};
-
-enum a3xx_cache_opcode {
-	INVALIDATE = 1,
-};
-
-enum a3xx_vtx_fmt {
-	VFMT_32_FLOAT = 0,
-	VFMT_32_32_FLOAT = 1,
-	VFMT_32_32_32_FLOAT = 2,
-	VFMT_32_32_32_32_FLOAT = 3,
-	VFMT_16_FLOAT = 4,
-	VFMT_16_16_FLOAT = 5,
-	VFMT_16_16_16_FLOAT = 6,
-	VFMT_16_16_16_16_FLOAT = 7,
-	VFMT_32_FIXED = 8,
-	VFMT_32_32_FIXED = 9,
-	VFMT_32_32_32_FIXED = 10,
-	VFMT_32_32_32_32_FIXED = 11,
-	VFMT_16_SINT = 16,
-	VFMT_16_16_SINT = 17,
-	VFMT_16_16_16_SINT = 18,
-	VFMT_16_16_16_16_SINT = 19,
-	VFMT_16_UINT = 20,
-	VFMT_16_16_UINT = 21,
-	VFMT_16_16_16_UINT = 22,
-	VFMT_16_16_16_16_UINT = 23,
-	VFMT_16_SNORM = 24,
-	VFMT_16_16_SNORM = 25,
-	VFMT_16_16_16_SNORM = 26,
-	VFMT_16_16_16_16_SNORM = 27,
-	VFMT_16_UNORM = 28,
-	VFMT_16_16_UNORM = 29,
-	VFMT_16_16_16_UNORM = 30,
-	VFMT_16_16_16_16_UNORM = 31,
-	VFMT_32_UINT = 32,
-	VFMT_32_32_UINT = 33,
-	VFMT_32_32_32_UINT = 34,
-	VFMT_32_32_32_32_UINT = 35,
-	VFMT_32_SINT = 36,
-	VFMT_32_32_SINT = 37,
-	VFMT_32_32_32_SINT = 38,
-	VFMT_32_32_32_32_SINT = 39,
-	VFMT_8_UINT = 40,
-	VFMT_8_8_UINT = 41,
-	VFMT_8_8_8_UINT = 42,
-	VFMT_8_8_8_8_UINT = 43,
-	VFMT_8_UNORM = 44,
-	VFMT_8_8_UNORM = 45,
-	VFMT_8_8_8_UNORM = 46,
-	VFMT_8_8_8_8_UNORM = 47,
-	VFMT_8_SINT = 48,
-	VFMT_8_8_SINT = 49,
-	VFMT_8_8_8_SINT = 50,
-	VFMT_8_8_8_8_SINT = 51,
-	VFMT_8_SNORM = 52,
-	VFMT_8_8_SNORM = 53,
-	VFMT_8_8_8_SNORM = 54,
-	VFMT_8_8_8_8_SNORM = 55,
-	VFMT_10_10_10_2_UINT = 56,
-	VFMT_10_10_10_2_UNORM = 57,
-	VFMT_10_10_10_2_SINT = 58,
-	VFMT_10_10_10_2_SNORM = 59,
-	VFMT_2_10_10_10_UINT = 60,
-	VFMT_2_10_10_10_UNORM = 61,
-	VFMT_2_10_10_10_SINT = 62,
-	VFMT_2_10_10_10_SNORM = 63,
-	VFMT_NONE = 255,
-};
-
-enum a3xx_tex_fmt {
-	TFMT_5_6_5_UNORM = 4,
-	TFMT_5_5_5_1_UNORM = 5,
-	TFMT_4_4_4_4_UNORM = 7,
-	TFMT_Z16_UNORM = 9,
-	TFMT_X8Z24_UNORM = 10,
-	TFMT_Z32_FLOAT = 11,
-	TFMT_UV_64X32 = 16,
-	TFMT_VU_64X32 = 17,
-	TFMT_Y_64X32 = 18,
-	TFMT_NV12_64X32 = 19,
-	TFMT_UV_LINEAR = 20,
-	TFMT_VU_LINEAR = 21,
-	TFMT_Y_LINEAR = 22,
-	TFMT_NV12_LINEAR = 23,
-	TFMT_I420_Y = 24,
-	TFMT_I420_U = 26,
-	TFMT_I420_V = 27,
-	TFMT_ATC_RGB = 32,
-	TFMT_ATC_RGBA_EXPLICIT = 33,
-	TFMT_ETC1 = 34,
-	TFMT_ATC_RGBA_INTERPOLATED = 35,
-	TFMT_DXT1 = 36,
-	TFMT_DXT3 = 37,
-	TFMT_DXT5 = 38,
-	TFMT_2_10_10_10_UNORM = 40,
-	TFMT_10_10_10_2_UNORM = 41,
-	TFMT_9_9_9_E5_FLOAT = 42,
-	TFMT_11_11_10_FLOAT = 43,
-	TFMT_A8_UNORM = 44,
-	TFMT_L8_UNORM = 45,
-	TFMT_L8_A8_UNORM = 47,
-	TFMT_8_UNORM = 48,
-	TFMT_8_8_UNORM = 49,
-	TFMT_8_8_8_UNORM = 50,
-	TFMT_8_8_8_8_UNORM = 51,
-	TFMT_8_SNORM = 52,
-	TFMT_8_8_SNORM = 53,
-	TFMT_8_8_8_SNORM = 54,
-	TFMT_8_8_8_8_SNORM = 55,
-	TFMT_8_UINT = 56,
-	TFMT_8_8_UINT = 57,
-	TFMT_8_8_8_UINT = 58,
-	TFMT_8_8_8_8_UINT = 59,
-	TFMT_8_SINT = 60,
-	TFMT_8_8_SINT = 61,
-	TFMT_8_8_8_SINT = 62,
-	TFMT_8_8_8_8_SINT = 63,
-	TFMT_16_FLOAT = 64,
-	TFMT_16_16_FLOAT = 65,
-	TFMT_16_16_16_16_FLOAT = 67,
-	TFMT_16_UINT = 68,
-	TFMT_16_16_UINT = 69,
-	TFMT_16_16_16_16_UINT = 71,
-	TFMT_16_SINT = 72,
-	TFMT_16_16_SINT = 73,
-	TFMT_16_16_16_16_SINT = 75,
-	TFMT_16_UNORM = 76,
-	TFMT_16_16_UNORM = 77,
-	TFMT_16_16_16_16_UNORM = 79,
-	TFMT_16_SNORM = 80,
-	TFMT_16_16_SNORM = 81,
-	TFMT_16_16_16_16_SNORM = 83,
-	TFMT_32_FLOAT = 84,
-	TFMT_32_32_FLOAT = 85,
-	TFMT_32_32_32_32_FLOAT = 87,
-	TFMT_32_UINT = 88,
-	TFMT_32_32_UINT = 89,
-	TFMT_32_32_32_32_UINT = 91,
-	TFMT_32_SINT = 92,
-	TFMT_32_32_SINT = 93,
-	TFMT_32_32_32_32_SINT = 95,
-	TFMT_2_10_10_10_UINT = 96,
-	TFMT_10_10_10_2_UINT = 97,
-	TFMT_ETC2_RG11_SNORM = 112,
-	TFMT_ETC2_RG11_UNORM = 113,
-	TFMT_ETC2_R11_SNORM = 114,
-	TFMT_ETC2_R11_UNORM = 115,
-	TFMT_ETC2_RGBA8 = 116,
-	TFMT_ETC2_RGB8A1 = 117,
-	TFMT_ETC2_RGB8 = 118,
-	TFMT_NONE = 255,
-};
-
-enum a3xx_color_fmt {
-	RB_R5G6B5_UNORM = 0,
-	RB_R5G5B5A1_UNORM = 1,
-	RB_R4G4B4A4_UNORM = 3,
-	RB_R8G8B8_UNORM = 4,
-	RB_R8G8B8A8_UNORM = 8,
-	RB_R8G8B8A8_SNORM = 9,
-	RB_R8G8B8A8_UINT = 10,
-	RB_R8G8B8A8_SINT = 11,
-	RB_R8G8_UNORM = 12,
-	RB_R8G8_SNORM = 13,
-	RB_R8G8_UINT = 14,
-	RB_R8G8_SINT = 15,
-	RB_R10G10B10A2_UNORM = 16,
-	RB_A2R10G10B10_UNORM = 17,
-	RB_R10G10B10A2_UINT = 18,
-	RB_A2R10G10B10_UINT = 19,
-	RB_A8_UNORM = 20,
-	RB_R8_UNORM = 21,
-	RB_R16_FLOAT = 24,
-	RB_R16G16_FLOAT = 25,
-	RB_R16G16B16A16_FLOAT = 27,
-	RB_R11G11B10_FLOAT = 28,
-	RB_R16_SNORM = 32,
-	RB_R16G16_SNORM = 33,
-	RB_R16G16B16A16_SNORM = 35,
-	RB_R16_UNORM = 36,
-	RB_R16G16_UNORM = 37,
-	RB_R16G16B16A16_UNORM = 39,
-	RB_R16_SINT = 40,
-	RB_R16G16_SINT = 41,
-	RB_R16G16B16A16_SINT = 43,
-	RB_R16_UINT = 44,
-	RB_R16G16_UINT = 45,
-	RB_R16G16B16A16_UINT = 47,
-	RB_R32_FLOAT = 48,
-	RB_R32G32_FLOAT = 49,
-	RB_R32G32B32A32_FLOAT = 51,
-	RB_R32_SINT = 52,
-	RB_R32G32_SINT = 53,
-	RB_R32G32B32A32_SINT = 55,
-	RB_R32_UINT = 56,
-	RB_R32G32_UINT = 57,
-	RB_R32G32B32A32_UINT = 59,
-	RB_NONE = 255,
-};
-
-enum a3xx_cp_perfcounter_select {
-	CP_ALWAYS_COUNT = 0,
-	CP_AHB_PFPTRANS_WAIT = 3,
-	CP_AHB_NRTTRANS_WAIT = 6,
-	CP_CSF_NRT_READ_WAIT = 8,
-	CP_CSF_I1_FIFO_FULL = 9,
-	CP_CSF_I2_FIFO_FULL = 10,
-	CP_CSF_ST_FIFO_FULL = 11,
-	CP_RESERVED_12 = 12,
-	CP_CSF_RING_ROQ_FULL = 13,
-	CP_CSF_I1_ROQ_FULL = 14,
-	CP_CSF_I2_ROQ_FULL = 15,
-	CP_CSF_ST_ROQ_FULL = 16,
-	CP_RESERVED_17 = 17,
-	CP_MIU_TAG_MEM_FULL = 18,
-	CP_MIU_NRT_WRITE_STALLED = 22,
-	CP_MIU_NRT_READ_STALLED = 23,
-	CP_ME_REGS_RB_DONE_FIFO_FULL = 26,
-	CP_ME_REGS_VS_EVENT_FIFO_FULL = 27,
-	CP_ME_REGS_PS_EVENT_FIFO_FULL = 28,
-	CP_ME_REGS_CF_EVENT_FIFO_FULL = 29,
-	CP_ME_MICRO_RB_STARVED = 30,
-	CP_AHB_RBBM_DWORD_SENT = 40,
-	CP_ME_BUSY_CLOCKS = 41,
-	CP_ME_WAIT_CONTEXT_AVAIL = 42,
-	CP_PFP_TYPE0_PACKET = 43,
-	CP_PFP_TYPE3_PACKET = 44,
-	CP_CSF_RB_WPTR_NEQ_RPTR = 45,
-	CP_CSF_I1_SIZE_NEQ_ZERO = 46,
-	CP_CSF_I2_SIZE_NEQ_ZERO = 47,
-	CP_CSF_RBI1I2_FETCHING = 48,
-};
-
-enum a3xx_gras_tse_perfcounter_select {
-	GRAS_TSEPERF_INPUT_PRIM = 0,
-	GRAS_TSEPERF_INPUT_NULL_PRIM = 1,
-	GRAS_TSEPERF_TRIVAL_REJ_PRIM = 2,
-	GRAS_TSEPERF_CLIPPED_PRIM = 3,
-	GRAS_TSEPERF_NEW_PRIM = 4,
-	GRAS_TSEPERF_ZERO_AREA_PRIM = 5,
-	GRAS_TSEPERF_FACENESS_CULLED_PRIM = 6,
-	GRAS_TSEPERF_ZERO_PIXEL_PRIM = 7,
-	GRAS_TSEPERF_OUTPUT_NULL_PRIM = 8,
-	GRAS_TSEPERF_OUTPUT_VISIBLE_PRIM = 9,
-	GRAS_TSEPERF_PRE_CLIP_PRIM = 10,
-	GRAS_TSEPERF_POST_CLIP_PRIM = 11,
-	GRAS_TSEPERF_WORKING_CYCLES = 12,
-	GRAS_TSEPERF_PC_STARVE = 13,
-	GRAS_TSERASPERF_STALL = 14,
-};
-
-enum a3xx_gras_ras_perfcounter_select {
-	GRAS_RASPERF_16X16_TILES = 0,
-	GRAS_RASPERF_8X8_TILES = 1,
-	GRAS_RASPERF_4X4_TILES = 2,
-	GRAS_RASPERF_WORKING_CYCLES = 3,
-	GRAS_RASPERF_STALL_CYCLES_BY_RB = 4,
-	GRAS_RASPERF_STALL_CYCLES_BY_VSC = 5,
-	GRAS_RASPERF_STARVE_CYCLES_BY_TSE = 6,
-};
-
-enum a3xx_hlsq_perfcounter_select {
-	HLSQ_PERF_SP_VS_CONSTANT = 0,
-	HLSQ_PERF_SP_VS_INSTRUCTIONS = 1,
-	HLSQ_PERF_SP_FS_CONSTANT = 2,
-	HLSQ_PERF_SP_FS_INSTRUCTIONS = 3,
-	HLSQ_PERF_TP_STATE = 4,
-	HLSQ_PERF_QUADS = 5,
-	HLSQ_PERF_PIXELS = 6,
-	HLSQ_PERF_VERTICES = 7,
-	HLSQ_PERF_FS8_THREADS = 8,
-	HLSQ_PERF_FS16_THREADS = 9,
-	HLSQ_PERF_FS32_THREADS = 10,
-	HLSQ_PERF_VS8_THREADS = 11,
-	HLSQ_PERF_VS16_THREADS = 12,
-	HLSQ_PERF_SP_VS_DATA_BYTES = 13,
-	HLSQ_PERF_SP_FS_DATA_BYTES = 14,
-	HLSQ_PERF_ACTIVE_CYCLES = 15,
-	HLSQ_PERF_STALL_CYCLES_SP_STATE = 16,
-	HLSQ_PERF_STALL_CYCLES_SP_VS = 17,
-	HLSQ_PERF_STALL_CYCLES_SP_FS = 18,
-	HLSQ_PERF_STALL_CYCLES_UCHE = 19,
-	HLSQ_PERF_RBBM_LOAD_CYCLES = 20,
-	HLSQ_PERF_DI_TO_VS_START_SP0 = 21,
-	HLSQ_PERF_DI_TO_FS_START_SP0 = 22,
-	HLSQ_PERF_VS_START_TO_DONE_SP0 = 23,
-	HLSQ_PERF_FS_START_TO_DONE_SP0 = 24,
-	HLSQ_PERF_SP_STATE_COPY_CYCLES_VS = 25,
-	HLSQ_PERF_SP_STATE_COPY_CYCLES_FS = 26,
-	HLSQ_PERF_UCHE_LATENCY_CYCLES = 27,
-	HLSQ_PERF_UCHE_LATENCY_COUNT = 28,
-};
-
-enum a3xx_pc_perfcounter_select {
-	PC_PCPERF_VISIBILITY_STREAMS = 0,
-	PC_PCPERF_TOTAL_INSTANCES = 1,
-	PC_PCPERF_PRIMITIVES_PC_VPC = 2,
-	PC_PCPERF_PRIMITIVES_KILLED_BY_VS = 3,
-	PC_PCPERF_PRIMITIVES_VISIBLE_BY_VS = 4,
-	PC_PCPERF_DRAWCALLS_KILLED_BY_VS = 5,
-	PC_PCPERF_DRAWCALLS_VISIBLE_BY_VS = 6,
-	PC_PCPERF_VERTICES_TO_VFD = 7,
-	PC_PCPERF_REUSED_VERTICES = 8,
-	PC_PCPERF_CYCLES_STALLED_BY_VFD = 9,
-	PC_PCPERF_CYCLES_STALLED_BY_TSE = 10,
-	PC_PCPERF_CYCLES_STALLED_BY_VBIF = 11,
-	PC_PCPERF_CYCLES_IS_WORKING = 12,
-};
-
-enum a3xx_rb_perfcounter_select {
-	RB_RBPERF_ACTIVE_CYCLES_ANY = 0,
-	RB_RBPERF_ACTIVE_CYCLES_ALL = 1,
-	RB_RBPERF_STARVE_CYCLES_BY_SP = 2,
-	RB_RBPERF_STARVE_CYCLES_BY_RAS = 3,
-	RB_RBPERF_STARVE_CYCLES_BY_MARB = 4,
-	RB_RBPERF_STALL_CYCLES_BY_MARB = 5,
-	RB_RBPERF_STALL_CYCLES_BY_HLSQ = 6,
-	RB_RBPERF_RB_MARB_DATA = 7,
-	RB_RBPERF_SP_RB_QUAD = 8,
-	RB_RBPERF_RAS_EARLY_Z_QUADS = 9,
-	RB_RBPERF_GMEM_CH0_READ = 10,
-	RB_RBPERF_GMEM_CH1_READ = 11,
-	RB_RBPERF_GMEM_CH0_WRITE = 12,
-	RB_RBPERF_GMEM_CH1_WRITE = 13,
-	RB_RBPERF_CP_CONTEXT_DONE = 14,
-	RB_RBPERF_CP_CACHE_FLUSH = 15,
-	RB_RBPERF_CP_ZPASS_DONE = 16,
-};
-
-enum a3xx_rbbm_perfcounter_select {
-	RBBM_ALAWYS_ON = 0,
-	RBBM_VBIF_BUSY = 1,
-	RBBM_TSE_BUSY = 2,
-	RBBM_RAS_BUSY = 3,
-	RBBM_PC_DCALL_BUSY = 4,
-	RBBM_PC_VSD_BUSY = 5,
-	RBBM_VFD_BUSY = 6,
-	RBBM_VPC_BUSY = 7,
-	RBBM_UCHE_BUSY = 8,
-	RBBM_VSC_BUSY = 9,
-	RBBM_HLSQ_BUSY = 10,
-	RBBM_ANY_RB_BUSY = 11,
-	RBBM_ANY_TEX_BUSY = 12,
-	RBBM_ANY_USP_BUSY = 13,
-	RBBM_ANY_MARB_BUSY = 14,
-	RBBM_ANY_ARB_BUSY = 15,
-	RBBM_AHB_STATUS_BUSY = 16,
-	RBBM_AHB_STATUS_STALLED = 17,
-	RBBM_AHB_STATUS_TXFR = 18,
-	RBBM_AHB_STATUS_TXFR_SPLIT = 19,
-	RBBM_AHB_STATUS_TXFR_ERROR = 20,
-	RBBM_AHB_STATUS_LONG_STALL = 21,
-	RBBM_RBBM_STATUS_MASKED = 22,
-};
-
-enum a3xx_sp_perfcounter_select {
-	SP_LM_LOAD_INSTRUCTIONS = 0,
-	SP_LM_STORE_INSTRUCTIONS = 1,
-	SP_LM_ATOMICS = 2,
-	SP_UCHE_LOAD_INSTRUCTIONS = 3,
-	SP_UCHE_STORE_INSTRUCTIONS = 4,
-	SP_UCHE_ATOMICS = 5,
-	SP_VS_TEX_INSTRUCTIONS = 6,
-	SP_VS_CFLOW_INSTRUCTIONS = 7,
-	SP_VS_EFU_INSTRUCTIONS = 8,
-	SP_VS_FULL_ALU_INSTRUCTIONS = 9,
-	SP_VS_HALF_ALU_INSTRUCTIONS = 10,
-	SP_FS_TEX_INSTRUCTIONS = 11,
-	SP_FS_CFLOW_INSTRUCTIONS = 12,
-	SP_FS_EFU_INSTRUCTIONS = 13,
-	SP_FS_FULL_ALU_INSTRUCTIONS = 14,
-	SP_FS_HALF_ALU_INSTRUCTIONS = 15,
-	SP_FS_BARY_INSTRUCTIONS = 16,
-	SP_VS_INSTRUCTIONS = 17,
-	SP_FS_INSTRUCTIONS = 18,
-	SP_ADDR_LOCK_COUNT = 19,
-	SP_UCHE_READ_TRANS = 20,
-	SP_UCHE_WRITE_TRANS = 21,
-	SP_EXPORT_VPC_TRANS = 22,
-	SP_EXPORT_RB_TRANS = 23,
-	SP_PIXELS_KILLED = 24,
-	SP_ICL1_REQUESTS = 25,
-	SP_ICL1_MISSES = 26,
-	SP_ICL0_REQUESTS = 27,
-	SP_ICL0_MISSES = 28,
-	SP_ALU_ACTIVE_CYCLES = 29,
-	SP_EFU_ACTIVE_CYCLES = 30,
-	SP_STALL_CYCLES_BY_VPC = 31,
-	SP_STALL_CYCLES_BY_TP = 32,
-	SP_STALL_CYCLES_BY_UCHE = 33,
-	SP_STALL_CYCLES_BY_RB = 34,
-	SP_ACTIVE_CYCLES_ANY = 35,
-	SP_ACTIVE_CYCLES_ALL = 36,
-};
-
-enum a3xx_tp_perfcounter_select {
-	TPL1_TPPERF_L1_REQUESTS = 0,
-	TPL1_TPPERF_TP0_L1_REQUESTS = 1,
-	TPL1_TPPERF_TP0_L1_MISSES = 2,
-	TPL1_TPPERF_TP1_L1_REQUESTS = 3,
-	TPL1_TPPERF_TP1_L1_MISSES = 4,
-	TPL1_TPPERF_TP2_L1_REQUESTS = 5,
-	TPL1_TPPERF_TP2_L1_MISSES = 6,
-	TPL1_TPPERF_TP3_L1_REQUESTS = 7,
-	TPL1_TPPERF_TP3_L1_MISSES = 8,
-	TPL1_TPPERF_OUTPUT_TEXELS_POINT = 9,
-	TPL1_TPPERF_OUTPUT_TEXELS_BILINEAR = 10,
-	TPL1_TPPERF_OUTPUT_TEXELS_MIP = 11,
-	TPL1_TPPERF_OUTPUT_TEXELS_ANISO = 12,
-	TPL1_TPPERF_BILINEAR_OPS = 13,
-	TPL1_TPPERF_QUADSQUADS_OFFSET = 14,
-	TPL1_TPPERF_QUADQUADS_SHADOW = 15,
-	TPL1_TPPERF_QUADS_ARRAY = 16,
-	TPL1_TPPERF_QUADS_PROJECTION = 17,
-	TPL1_TPPERF_QUADS_GRADIENT = 18,
-	TPL1_TPPERF_QUADS_1D2D = 19,
-	TPL1_TPPERF_QUADS_3DCUBE = 20,
-	TPL1_TPPERF_ZERO_LOD = 21,
-	TPL1_TPPERF_OUTPUT_TEXELS = 22,
-	TPL1_TPPERF_ACTIVE_CYCLES_ANY = 23,
-	TPL1_TPPERF_ACTIVE_CYCLES_ALL = 24,
-	TPL1_TPPERF_STALL_CYCLES_BY_ARB = 25,
-	TPL1_TPPERF_LATENCY = 26,
-	TPL1_TPPERF_LATENCY_TRANS = 27,
-};
-
-enum a3xx_vfd_perfcounter_select {
-	VFD_PERF_UCHE_BYTE_FETCHED = 0,
-	VFD_PERF_UCHE_TRANS = 1,
-	VFD_PERF_VPC_BYPASS_COMPONENTS = 2,
-	VFD_PERF_FETCH_INSTRUCTIONS = 3,
-	VFD_PERF_DECODE_INSTRUCTIONS = 4,
-	VFD_PERF_ACTIVE_CYCLES = 5,
-	VFD_PERF_STALL_CYCLES_UCHE = 6,
-	VFD_PERF_STALL_CYCLES_HLSQ = 7,
-	VFD_PERF_STALL_CYCLES_VPC_BYPASS = 8,
-	VFD_PERF_STALL_CYCLES_VPC_ALLOC = 9,
-};
-
-enum a3xx_vpc_perfcounter_select {
-	VPC_PERF_SP_LM_PRIMITIVES = 0,
-	VPC_PERF_COMPONENTS_FROM_SP = 1,
-	VPC_PERF_SP_LM_COMPONENTS = 2,
-	VPC_PERF_ACTIVE_CYCLES = 3,
-	VPC_PERF_STALL_CYCLES_LM = 4,
-	VPC_PERF_STALL_CYCLES_RAS = 5,
-};
-
-enum a3xx_uche_perfcounter_select {
-	UCHE_UCHEPERF_VBIF_READ_BEATS_TP = 0,
-	UCHE_UCHEPERF_VBIF_READ_BEATS_VFD = 1,
-	UCHE_UCHEPERF_VBIF_READ_BEATS_HLSQ = 2,
-	UCHE_UCHEPERF_VBIF_READ_BEATS_MARB = 3,
-	UCHE_UCHEPERF_VBIF_READ_BEATS_SP = 4,
-	UCHE_UCHEPERF_READ_REQUESTS_TP = 8,
-	UCHE_UCHEPERF_READ_REQUESTS_VFD = 9,
-	UCHE_UCHEPERF_READ_REQUESTS_HLSQ = 10,
-	UCHE_UCHEPERF_READ_REQUESTS_MARB = 11,
-	UCHE_UCHEPERF_READ_REQUESTS_SP = 12,
-	UCHE_UCHEPERF_WRITE_REQUESTS_MARB = 13,
-	UCHE_UCHEPERF_WRITE_REQUESTS_SP = 14,
-	UCHE_UCHEPERF_TAG_CHECK_FAILS = 15,
-	UCHE_UCHEPERF_EVICTS = 16,
-	UCHE_UCHEPERF_FLUSHES = 17,
-	UCHE_UCHEPERF_VBIF_LATENCY_CYCLES = 18,
-	UCHE_UCHEPERF_VBIF_LATENCY_SAMPLES = 19,
-	UCHE_UCHEPERF_ACTIVE_CYCLES = 20,
-};
-
-enum a3xx_intp_mode {
-	SMOOTH = 0,
-	FLAT = 1,
-	ZERO = 2,
-	ONE = 3,
-};
-
-enum a3xx_repl_mode {
-	S = 1,
-	T = 2,
-	ONE_T = 3,
-};
-
-enum a3xx_tex_filter {
-	A3XX_TEX_NEAREST = 0,
-	A3XX_TEX_LINEAR = 1,
-	A3XX_TEX_ANISO = 2,
-};
-
-enum a3xx_tex_clamp {
-	A3XX_TEX_REPEAT = 0,
-	A3XX_TEX_CLAMP_TO_EDGE = 1,
-	A3XX_TEX_MIRROR_REPEAT = 2,
-	A3XX_TEX_CLAMP_TO_BORDER = 3,
-	A3XX_TEX_MIRROR_CLAMP = 4,
-};
-
-enum a3xx_tex_aniso {
-	A3XX_TEX_ANISO_1 = 0,
-	A3XX_TEX_ANISO_2 = 1,
-	A3XX_TEX_ANISO_4 = 2,
-	A3XX_TEX_ANISO_8 = 3,
-	A3XX_TEX_ANISO_16 = 4,
-};
-
-enum a3xx_tex_swiz {
-	A3XX_TEX_X = 0,
-	A3XX_TEX_Y = 1,
-	A3XX_TEX_Z = 2,
-	A3XX_TEX_W = 3,
-	A3XX_TEX_ZERO = 4,
-	A3XX_TEX_ONE = 5,
-};
-
-enum a3xx_tex_type {
-	A3XX_TEX_1D = 0,
-	A3XX_TEX_2D = 1,
-	A3XX_TEX_CUBE = 2,
-	A3XX_TEX_3D = 3,
-};
-
-enum a3xx_tex_msaa {
-	A3XX_TPL1_MSAA1X = 0,
-	A3XX_TPL1_MSAA2X = 1,
-	A3XX_TPL1_MSAA4X = 2,
-	A3XX_TPL1_MSAA8X = 3,
-};
-
-#define A3XX_INT0_RBBM_GPU_IDLE					0x00000001
-#define A3XX_INT0_RBBM_AHB_ERROR				0x00000002
-#define A3XX_INT0_RBBM_REG_TIMEOUT				0x00000004
-#define A3XX_INT0_RBBM_ME_MS_TIMEOUT				0x00000008
-#define A3XX_INT0_RBBM_PFP_MS_TIMEOUT				0x00000010
-#define A3XX_INT0_RBBM_ATB_BUS_OVERFLOW				0x00000020
-#define A3XX_INT0_VFD_ERROR					0x00000040
-#define A3XX_INT0_CP_SW_INT					0x00000080
-#define A3XX_INT0_CP_T0_PACKET_IN_IB				0x00000100
-#define A3XX_INT0_CP_OPCODE_ERROR				0x00000200
-#define A3XX_INT0_CP_RESERVED_BIT_ERROR				0x00000400
-#define A3XX_INT0_CP_HW_FAULT					0x00000800
-#define A3XX_INT0_CP_DMA					0x00001000
-#define A3XX_INT0_CP_IB2_INT					0x00002000
-#define A3XX_INT0_CP_IB1_INT					0x00004000
-#define A3XX_INT0_CP_RB_INT					0x00008000
-#define A3XX_INT0_CP_REG_PROTECT_FAULT				0x00010000
-#define A3XX_INT0_CP_RB_DONE_TS					0x00020000
-#define A3XX_INT0_CP_VS_DONE_TS					0x00040000
-#define A3XX_INT0_CP_PS_DONE_TS					0x00080000
-#define A3XX_INT0_CACHE_FLUSH_TS				0x00100000
-#define A3XX_INT0_CP_AHB_ERROR_HALT				0x00200000
-#define A3XX_INT0_MISC_HANG_DETECT				0x01000000
-#define A3XX_INT0_UCHE_OOB_ACCESS				0x02000000
-
-#define REG_A3XX_RBBM_HW_VERSION				0x00000000
-
-#define REG_A3XX_RBBM_HW_RELEASE				0x00000001
-
-#define REG_A3XX_RBBM_HW_CONFIGURATION				0x00000002
-
-#define REG_A3XX_RBBM_CLOCK_CTL					0x00000010
-
-#define REG_A3XX_RBBM_SP_HYST_CNT				0x00000012
-
-#define REG_A3XX_RBBM_SW_RESET_CMD				0x00000018
-
-#define REG_A3XX_RBBM_AHB_CTL0					0x00000020
-
-#define REG_A3XX_RBBM_AHB_CTL1					0x00000021
-
-#define REG_A3XX_RBBM_AHB_CMD					0x00000022
-
-#define REG_A3XX_RBBM_AHB_ERROR_STATUS				0x00000027
-
-#define REG_A3XX_RBBM_GPR0_CTL					0x0000002e
-
-#define REG_A3XX_RBBM_STATUS					0x00000030
-#define A3XX_RBBM_STATUS_HI_BUSY				0x00000001
-#define A3XX_RBBM_STATUS_CP_ME_BUSY				0x00000002
-#define A3XX_RBBM_STATUS_CP_PFP_BUSY				0x00000004
-#define A3XX_RBBM_STATUS_CP_NRT_BUSY				0x00004000
-#define A3XX_RBBM_STATUS_VBIF_BUSY				0x00008000
-#define A3XX_RBBM_STATUS_TSE_BUSY				0x00010000
-#define A3XX_RBBM_STATUS_RAS_BUSY				0x00020000
-#define A3XX_RBBM_STATUS_RB_BUSY				0x00040000
-#define A3XX_RBBM_STATUS_PC_DCALL_BUSY				0x00080000
-#define A3XX_RBBM_STATUS_PC_VSD_BUSY				0x00100000
-#define A3XX_RBBM_STATUS_VFD_BUSY				0x00200000
-#define A3XX_RBBM_STATUS_VPC_BUSY				0x00400000
-#define A3XX_RBBM_STATUS_UCHE_BUSY				0x00800000
-#define A3XX_RBBM_STATUS_SP_BUSY				0x01000000
-#define A3XX_RBBM_STATUS_TPL1_BUSY				0x02000000
-#define A3XX_RBBM_STATUS_MARB_BUSY				0x04000000
-#define A3XX_RBBM_STATUS_VSC_BUSY				0x08000000
-#define A3XX_RBBM_STATUS_ARB_BUSY				0x10000000
-#define A3XX_RBBM_STATUS_HLSQ_BUSY				0x20000000
-#define A3XX_RBBM_STATUS_GPU_BUSY_NOHC				0x40000000
-#define A3XX_RBBM_STATUS_GPU_BUSY				0x80000000
-
-#define REG_A3XX_RBBM_NQWAIT_UNTIL				0x00000040
-
-#define REG_A3XX_RBBM_WAIT_IDLE_CLOCKS_CTL			0x00000033
-
-#define REG_A3XX_RBBM_INTERFACE_HANG_INT_CTL			0x00000050
-
-#define REG_A3XX_RBBM_INTERFACE_HANG_MASK_CTL0			0x00000051
-
-#define REG_A3XX_RBBM_INTERFACE_HANG_MASK_CTL1			0x00000054
-
-#define REG_A3XX_RBBM_INTERFACE_HANG_MASK_CTL2			0x00000057
-
-#define REG_A3XX_RBBM_INTERFACE_HANG_MASK_CTL3			0x0000005a
-
-#define REG_A3XX_RBBM_INT_SET_CMD				0x00000060
-#define REG_A3XX_RBBM_INT_CLEAR_CMD				0x00000061
-#define REG_A3XX_RBBM_INT_0_MASK				0x00000063
-#define REG_A3XX_RBBM_INT_0_STATUS				0x00000064
-#define REG_A3XX_RBBM_PERFCTR_CTL				0x00000080
-#define A3XX_RBBM_PERFCTR_CTL_ENABLE				0x00000001
-
-#define REG_A3XX_RBBM_PERFCTR_LOAD_CMD0				0x00000081
-
-#define REG_A3XX_RBBM_PERFCTR_LOAD_CMD1				0x00000082
-
-#define REG_A3XX_RBBM_PERFCTR_LOAD_VALUE_LO			0x00000084
-
-#define REG_A3XX_RBBM_PERFCTR_LOAD_VALUE_HI			0x00000085
-
-#define REG_A3XX_RBBM_PERFCOUNTER0_SELECT			0x00000086
-
-#define REG_A3XX_RBBM_PERFCOUNTER1_SELECT			0x00000087
-
-#define REG_A3XX_RBBM_GPU_BUSY_MASKED				0x00000088
-
-#define REG_A3XX_RBBM_PERFCTR_CP_0_LO				0x00000090
-
-#define REG_A3XX_RBBM_PERFCTR_CP_0_HI				0x00000091
-
-#define REG_A3XX_RBBM_PERFCTR_RBBM_0_LO				0x00000092
-
-#define REG_A3XX_RBBM_PERFCTR_RBBM_0_HI				0x00000093
-
-#define REG_A3XX_RBBM_PERFCTR_RBBM_1_LO				0x00000094
-
-#define REG_A3XX_RBBM_PERFCTR_RBBM_1_HI				0x00000095
-
-#define REG_A3XX_RBBM_PERFCTR_PC_0_LO				0x00000096
-
-#define REG_A3XX_RBBM_PERFCTR_PC_0_HI				0x00000097
-
-#define REG_A3XX_RBBM_PERFCTR_PC_1_LO				0x00000098
-
-#define REG_A3XX_RBBM_PERFCTR_PC_1_HI				0x00000099
-
-#define REG_A3XX_RBBM_PERFCTR_PC_2_LO				0x0000009a
-
-#define REG_A3XX_RBBM_PERFCTR_PC_2_HI				0x0000009b
-
-#define REG_A3XX_RBBM_PERFCTR_PC_3_LO				0x0000009c
-
-#define REG_A3XX_RBBM_PERFCTR_PC_3_HI				0x0000009d
-
-#define REG_A3XX_RBBM_PERFCTR_VFD_0_LO				0x0000009e
-
-#define REG_A3XX_RBBM_PERFCTR_VFD_0_HI				0x0000009f
-
-#define REG_A3XX_RBBM_PERFCTR_VFD_1_LO				0x000000a0
-
-#define REG_A3XX_RBBM_PERFCTR_VFD_1_HI				0x000000a1
-
-#define REG_A3XX_RBBM_PERFCTR_HLSQ_0_LO				0x000000a2
-
-#define REG_A3XX_RBBM_PERFCTR_HLSQ_0_HI				0x000000a3
-
-#define REG_A3XX_RBBM_PERFCTR_HLSQ_1_LO				0x000000a4
-
-#define REG_A3XX_RBBM_PERFCTR_HLSQ_1_HI				0x000000a5
-
-#define REG_A3XX_RBBM_PERFCTR_HLSQ_2_LO				0x000000a6
-
-#define REG_A3XX_RBBM_PERFCTR_HLSQ_2_HI				0x000000a7
-
-#define REG_A3XX_RBBM_PERFCTR_HLSQ_3_LO				0x000000a8
-
-#define REG_A3XX_RBBM_PERFCTR_HLSQ_3_HI				0x000000a9
-
-#define REG_A3XX_RBBM_PERFCTR_HLSQ_4_LO				0x000000aa
-
-#define REG_A3XX_RBBM_PERFCTR_HLSQ_4_HI				0x000000ab
-
-#define REG_A3XX_RBBM_PERFCTR_HLSQ_5_LO				0x000000ac
-
-#define REG_A3XX_RBBM_PERFCTR_HLSQ_5_HI				0x000000ad
-
-#define REG_A3XX_RBBM_PERFCTR_VPC_0_LO				0x000000ae
-
-#define REG_A3XX_RBBM_PERFCTR_VPC_0_HI				0x000000af
-
-#define REG_A3XX_RBBM_PERFCTR_VPC_1_LO				0x000000b0
-
-#define REG_A3XX_RBBM_PERFCTR_VPC_1_HI				0x000000b1
-
-#define REG_A3XX_RBBM_PERFCTR_TSE_0_LO				0x000000b2
-
-#define REG_A3XX_RBBM_PERFCTR_TSE_0_HI				0x000000b3
-
-#define REG_A3XX_RBBM_PERFCTR_TSE_1_LO				0x000000b4
-
-#define REG_A3XX_RBBM_PERFCTR_TSE_1_HI				0x000000b5
-
-#define REG_A3XX_RBBM_PERFCTR_RAS_0_LO				0x000000b6
-
-#define REG_A3XX_RBBM_PERFCTR_RAS_0_HI				0x000000b7
-
-#define REG_A3XX_RBBM_PERFCTR_RAS_1_LO				0x000000b8
-
-#define REG_A3XX_RBBM_PERFCTR_RAS_1_HI				0x000000b9
-
-#define REG_A3XX_RBBM_PERFCTR_UCHE_0_LO				0x000000ba
-
-#define REG_A3XX_RBBM_PERFCTR_UCHE_0_HI				0x000000bb
-
-#define REG_A3XX_RBBM_PERFCTR_UCHE_1_LO				0x000000bc
-
-#define REG_A3XX_RBBM_PERFCTR_UCHE_1_HI				0x000000bd
-
-#define REG_A3XX_RBBM_PERFCTR_UCHE_2_LO				0x000000be
-
-#define REG_A3XX_RBBM_PERFCTR_UCHE_2_HI				0x000000bf
-
-#define REG_A3XX_RBBM_PERFCTR_UCHE_3_LO				0x000000c0
-
-#define REG_A3XX_RBBM_PERFCTR_UCHE_3_HI				0x000000c1
-
-#define REG_A3XX_RBBM_PERFCTR_UCHE_4_LO				0x000000c2
-
-#define REG_A3XX_RBBM_PERFCTR_UCHE_4_HI				0x000000c3
-
-#define REG_A3XX_RBBM_PERFCTR_UCHE_5_LO				0x000000c4
-
-#define REG_A3XX_RBBM_PERFCTR_UCHE_5_HI				0x000000c5
-
-#define REG_A3XX_RBBM_PERFCTR_TP_0_LO				0x000000c6
-
-#define REG_A3XX_RBBM_PERFCTR_TP_0_HI				0x000000c7
-
-#define REG_A3XX_RBBM_PERFCTR_TP_1_LO				0x000000c8
-
-#define REG_A3XX_RBBM_PERFCTR_TP_1_HI				0x000000c9
-
-#define REG_A3XX_RBBM_PERFCTR_TP_2_LO				0x000000ca
-
-#define REG_A3XX_RBBM_PERFCTR_TP_2_HI				0x000000cb
-
-#define REG_A3XX_RBBM_PERFCTR_TP_3_LO				0x000000cc
-
-#define REG_A3XX_RBBM_PERFCTR_TP_3_HI				0x000000cd
-
-#define REG_A3XX_RBBM_PERFCTR_TP_4_LO				0x000000ce
-
-#define REG_A3XX_RBBM_PERFCTR_TP_4_HI				0x000000cf
-
-#define REG_A3XX_RBBM_PERFCTR_TP_5_LO				0x000000d0
-
-#define REG_A3XX_RBBM_PERFCTR_TP_5_HI				0x000000d1
-
-#define REG_A3XX_RBBM_PERFCTR_SP_0_LO				0x000000d2
-
-#define REG_A3XX_RBBM_PERFCTR_SP_0_HI				0x000000d3
-
-#define REG_A3XX_RBBM_PERFCTR_SP_1_LO				0x000000d4
-
-#define REG_A3XX_RBBM_PERFCTR_SP_1_HI				0x000000d5
-
-#define REG_A3XX_RBBM_PERFCTR_SP_2_LO				0x000000d6
-
-#define REG_A3XX_RBBM_PERFCTR_SP_2_HI				0x000000d7
-
-#define REG_A3XX_RBBM_PERFCTR_SP_3_LO				0x000000d8
-
-#define REG_A3XX_RBBM_PERFCTR_SP_3_HI				0x000000d9
-
-#define REG_A3XX_RBBM_PERFCTR_SP_4_LO				0x000000da
-
-#define REG_A3XX_RBBM_PERFCTR_SP_4_HI				0x000000db
-
-#define REG_A3XX_RBBM_PERFCTR_SP_5_LO				0x000000dc
-
-#define REG_A3XX_RBBM_PERFCTR_SP_5_HI				0x000000dd
-
-#define REG_A3XX_RBBM_PERFCTR_SP_6_LO				0x000000de
-
-#define REG_A3XX_RBBM_PERFCTR_SP_6_HI				0x000000df
-
-#define REG_A3XX_RBBM_PERFCTR_SP_7_LO				0x000000e0
-
-#define REG_A3XX_RBBM_PERFCTR_SP_7_HI				0x000000e1
-
-#define REG_A3XX_RBBM_PERFCTR_RB_0_LO				0x000000e2
-
-#define REG_A3XX_RBBM_PERFCTR_RB_0_HI				0x000000e3
-
-#define REG_A3XX_RBBM_PERFCTR_RB_1_LO				0x000000e4
-
-#define REG_A3XX_RBBM_PERFCTR_RB_1_HI				0x000000e5
-
-#define REG_A3XX_RBBM_PERFCTR_PWR_0_LO				0x000000ea
-
-#define REG_A3XX_RBBM_PERFCTR_PWR_0_HI				0x000000eb
-
-#define REG_A3XX_RBBM_PERFCTR_PWR_1_LO				0x000000ec
-
-#define REG_A3XX_RBBM_PERFCTR_PWR_1_HI				0x000000ed
-
-#define REG_A3XX_RBBM_RBBM_CTL					0x00000100
-
-#define REG_A3XX_RBBM_DEBUG_BUS_CTL				0x00000111
-
-#define REG_A3XX_RBBM_DEBUG_BUS_DATA_STATUS			0x00000112
-
-#define REG_A3XX_CP_PFP_UCODE_ADDR				0x000001c9
-
-#define REG_A3XX_CP_PFP_UCODE_DATA				0x000001ca
-
-#define REG_A3XX_CP_ROQ_ADDR					0x000001cc
-
-#define REG_A3XX_CP_ROQ_DATA					0x000001cd
-
-#define REG_A3XX_CP_MERCIU_ADDR					0x000001d1
-
-#define REG_A3XX_CP_MERCIU_DATA					0x000001d2
-
-#define REG_A3XX_CP_MERCIU_DATA2				0x000001d3
-
-#define REG_A3XX_CP_MEQ_ADDR					0x000001da
-
-#define REG_A3XX_CP_MEQ_DATA					0x000001db
-
-#define REG_A3XX_CP_WFI_PEND_CTR				0x000001f5
-
-#define REG_A3XX_RBBM_PM_OVERRIDE2				0x0000039d
-
-#define REG_A3XX_CP_PERFCOUNTER_SELECT				0x00000445
-
-#define REG_A3XX_CP_HW_FAULT					0x0000045c
-
-#define REG_A3XX_CP_PROTECT_CTRL				0x0000045e
-
-#define REG_A3XX_CP_PROTECT_STATUS				0x0000045f
-
-#define REG_A3XX_CP_PROTECT(i0) (0x00000460 + 0x1*(i0))
-
-static inline uint32_t REG_A3XX_CP_PROTECT_REG(uint32_t i0) { return 0x00000460 + 0x1*i0; }
-
-#define REG_A3XX_CP_AHB_FAULT					0x0000054d
-
-#define REG_A3XX_SQ_GPR_MANAGEMENT				0x00000d00
-
-#define REG_A3XX_SQ_INST_STORE_MANAGMENT			0x00000d02
-
-#define REG_A3XX_TP0_CHICKEN					0x00000e1e
-
-#define REG_A3XX_SP_GLOBAL_MEM_SIZE				0x00000e22
-
-#define REG_A3XX_SP_GLOBAL_MEM_ADDR				0x00000e23
-
-#define REG_A3XX_GRAS_CL_CLIP_CNTL				0x00002040
-#define A3XX_GRAS_CL_CLIP_CNTL_IJ_PERSP_CENTER			0x00001000
-#define A3XX_GRAS_CL_CLIP_CNTL_IJ_NON_PERSP_CENTER		0x00002000
-#define A3XX_GRAS_CL_CLIP_CNTL_IJ_PERSP_CENTROID		0x00004000
-#define A3XX_GRAS_CL_CLIP_CNTL_IJ_NON_PERSP_CENTROID		0x00008000
-#define A3XX_GRAS_CL_CLIP_CNTL_CLIP_DISABLE			0x00010000
-#define A3XX_GRAS_CL_CLIP_CNTL_ZFAR_CLIP_DISABLE		0x00020000
-#define A3XX_GRAS_CL_CLIP_CNTL_VP_CLIP_CODE_IGNORE		0x00080000
-#define A3XX_GRAS_CL_CLIP_CNTL_VP_XFORM_DISABLE			0x00100000
-#define A3XX_GRAS_CL_CLIP_CNTL_PERSP_DIVISION_DISABLE		0x00200000
-#define A3XX_GRAS_CL_CLIP_CNTL_ZERO_GB_SCALE_Z			0x00400000
-#define A3XX_GRAS_CL_CLIP_CNTL_ZCOORD				0x00800000
-#define A3XX_GRAS_CL_CLIP_CNTL_WCOORD				0x01000000
-#define A3XX_GRAS_CL_CLIP_CNTL_ZCLIP_DISABLE			0x02000000
-#define A3XX_GRAS_CL_CLIP_CNTL_NUM_USER_CLIP_PLANES__MASK	0x1c000000
-#define A3XX_GRAS_CL_CLIP_CNTL_NUM_USER_CLIP_PLANES__SHIFT	26
-static inline uint32_t A3XX_GRAS_CL_CLIP_CNTL_NUM_USER_CLIP_PLANES(uint32_t val)
-{
-	return ((val) << A3XX_GRAS_CL_CLIP_CNTL_NUM_USER_CLIP_PLANES__SHIFT) & A3XX_GRAS_CL_CLIP_CNTL_NUM_USER_CLIP_PLANES__MASK;
-}
-
-#define REG_A3XX_GRAS_CL_GB_CLIP_ADJ				0x00002044
-#define A3XX_GRAS_CL_GB_CLIP_ADJ_HORZ__MASK			0x000003ff
-#define A3XX_GRAS_CL_GB_CLIP_ADJ_HORZ__SHIFT			0
-static inline uint32_t A3XX_GRAS_CL_GB_CLIP_ADJ_HORZ(uint32_t val)
-{
-	return ((val) << A3XX_GRAS_CL_GB_CLIP_ADJ_HORZ__SHIFT) & A3XX_GRAS_CL_GB_CLIP_ADJ_HORZ__MASK;
-}
-#define A3XX_GRAS_CL_GB_CLIP_ADJ_VERT__MASK			0x000ffc00
-#define A3XX_GRAS_CL_GB_CLIP_ADJ_VERT__SHIFT			10
-static inline uint32_t A3XX_GRAS_CL_GB_CLIP_ADJ_VERT(uint32_t val)
-{
-	return ((val) << A3XX_GRAS_CL_GB_CLIP_ADJ_VERT__SHIFT) & A3XX_GRAS_CL_GB_CLIP_ADJ_VERT__MASK;
-}
-
-#define REG_A3XX_GRAS_CL_VPORT_XOFFSET				0x00002048
-#define A3XX_GRAS_CL_VPORT_XOFFSET__MASK			0xffffffff
-#define A3XX_GRAS_CL_VPORT_XOFFSET__SHIFT			0
-static inline uint32_t A3XX_GRAS_CL_VPORT_XOFFSET(float val)
-{
-	return ((fui(val)) << A3XX_GRAS_CL_VPORT_XOFFSET__SHIFT) & A3XX_GRAS_CL_VPORT_XOFFSET__MASK;
-}
-
-#define REG_A3XX_GRAS_CL_VPORT_XSCALE				0x00002049
-#define A3XX_GRAS_CL_VPORT_XSCALE__MASK				0xffffffff
-#define A3XX_GRAS_CL_VPORT_XSCALE__SHIFT			0
-static inline uint32_t A3XX_GRAS_CL_VPORT_XSCALE(float val)
-{
-	return ((fui(val)) << A3XX_GRAS_CL_VPORT_XSCALE__SHIFT) & A3XX_GRAS_CL_VPORT_XSCALE__MASK;
-}
-
-#define REG_A3XX_GRAS_CL_VPORT_YOFFSET				0x0000204a
-#define A3XX_GRAS_CL_VPORT_YOFFSET__MASK			0xffffffff
-#define A3XX_GRAS_CL_VPORT_YOFFSET__SHIFT			0
-static inline uint32_t A3XX_GRAS_CL_VPORT_YOFFSET(float val)
-{
-	return ((fui(val)) << A3XX_GRAS_CL_VPORT_YOFFSET__SHIFT) & A3XX_GRAS_CL_VPORT_YOFFSET__MASK;
-}
-
-#define REG_A3XX_GRAS_CL_VPORT_YSCALE				0x0000204b
-#define A3XX_GRAS_CL_VPORT_YSCALE__MASK				0xffffffff
-#define A3XX_GRAS_CL_VPORT_YSCALE__SHIFT			0
-static inline uint32_t A3XX_GRAS_CL_VPORT_YSCALE(float val)
-{
-	return ((fui(val)) << A3XX_GRAS_CL_VPORT_YSCALE__SHIFT) & A3XX_GRAS_CL_VPORT_YSCALE__MASK;
-}
-
-#define REG_A3XX_GRAS_CL_VPORT_ZOFFSET				0x0000204c
-#define A3XX_GRAS_CL_VPORT_ZOFFSET__MASK			0xffffffff
-#define A3XX_GRAS_CL_VPORT_ZOFFSET__SHIFT			0
-static inline uint32_t A3XX_GRAS_CL_VPORT_ZOFFSET(float val)
-{
-	return ((fui(val)) << A3XX_GRAS_CL_VPORT_ZOFFSET__SHIFT) & A3XX_GRAS_CL_VPORT_ZOFFSET__MASK;
-}
-
-#define REG_A3XX_GRAS_CL_VPORT_ZSCALE				0x0000204d
-#define A3XX_GRAS_CL_VPORT_ZSCALE__MASK				0xffffffff
-#define A3XX_GRAS_CL_VPORT_ZSCALE__SHIFT			0
-static inline uint32_t A3XX_GRAS_CL_VPORT_ZSCALE(float val)
-{
-	return ((fui(val)) << A3XX_GRAS_CL_VPORT_ZSCALE__SHIFT) & A3XX_GRAS_CL_VPORT_ZSCALE__MASK;
-}
-
-#define REG_A3XX_GRAS_SU_POINT_MINMAX				0x00002068
-#define A3XX_GRAS_SU_POINT_MINMAX_MIN__MASK			0x0000ffff
-#define A3XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT			0
-static inline uint32_t A3XX_GRAS_SU_POINT_MINMAX_MIN(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A3XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT) & A3XX_GRAS_SU_POINT_MINMAX_MIN__MASK;
-}
-#define A3XX_GRAS_SU_POINT_MINMAX_MAX__MASK			0xffff0000
-#define A3XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT			16
-static inline uint32_t A3XX_GRAS_SU_POINT_MINMAX_MAX(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A3XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT) & A3XX_GRAS_SU_POINT_MINMAX_MAX__MASK;
-}
-
-#define REG_A3XX_GRAS_SU_POINT_SIZE				0x00002069
-#define A3XX_GRAS_SU_POINT_SIZE__MASK				0xffffffff
-#define A3XX_GRAS_SU_POINT_SIZE__SHIFT				0
-static inline uint32_t A3XX_GRAS_SU_POINT_SIZE(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A3XX_GRAS_SU_POINT_SIZE__SHIFT) & A3XX_GRAS_SU_POINT_SIZE__MASK;
-}
-
-#define REG_A3XX_GRAS_SU_POLY_OFFSET_SCALE			0x0000206c
-#define A3XX_GRAS_SU_POLY_OFFSET_SCALE_VAL__MASK		0x00ffffff
-#define A3XX_GRAS_SU_POLY_OFFSET_SCALE_VAL__SHIFT		0
-static inline uint32_t A3XX_GRAS_SU_POLY_OFFSET_SCALE_VAL(float val)
-{
-	return ((((int32_t)(val * 1048576.0))) << A3XX_GRAS_SU_POLY_OFFSET_SCALE_VAL__SHIFT) & A3XX_GRAS_SU_POLY_OFFSET_SCALE_VAL__MASK;
-}
-
-#define REG_A3XX_GRAS_SU_POLY_OFFSET_OFFSET			0x0000206d
-#define A3XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK			0xffffffff
-#define A3XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT			0
-static inline uint32_t A3XX_GRAS_SU_POLY_OFFSET_OFFSET(float val)
-{
-	return ((((int32_t)(val * 64.0))) << A3XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT) & A3XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK;
-}
-
-#define REG_A3XX_GRAS_SU_MODE_CONTROL				0x00002070
-#define A3XX_GRAS_SU_MODE_CONTROL_CULL_FRONT			0x00000001
-#define A3XX_GRAS_SU_MODE_CONTROL_CULL_BACK			0x00000002
-#define A3XX_GRAS_SU_MODE_CONTROL_FRONT_CW			0x00000004
-#define A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__MASK		0x000007f8
-#define A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__SHIFT		3
-static inline uint32_t A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH(float val)
-{
-	return ((((int32_t)(val * 4.0))) << A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__SHIFT) & A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__MASK;
-}
-#define A3XX_GRAS_SU_MODE_CONTROL_POLY_OFFSET			0x00000800
-
-#define REG_A3XX_GRAS_SC_CONTROL				0x00002072
-#define A3XX_GRAS_SC_CONTROL_RENDER_MODE__MASK			0x000000f0
-#define A3XX_GRAS_SC_CONTROL_RENDER_MODE__SHIFT			4
-static inline uint32_t A3XX_GRAS_SC_CONTROL_RENDER_MODE(enum a3xx_render_mode val)
-{
-	return ((val) << A3XX_GRAS_SC_CONTROL_RENDER_MODE__SHIFT) & A3XX_GRAS_SC_CONTROL_RENDER_MODE__MASK;
-}
-#define A3XX_GRAS_SC_CONTROL_MSAA_SAMPLES__MASK			0x00000f00
-#define A3XX_GRAS_SC_CONTROL_MSAA_SAMPLES__SHIFT		8
-static inline uint32_t A3XX_GRAS_SC_CONTROL_MSAA_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A3XX_GRAS_SC_CONTROL_MSAA_SAMPLES__SHIFT) & A3XX_GRAS_SC_CONTROL_MSAA_SAMPLES__MASK;
-}
-#define A3XX_GRAS_SC_CONTROL_RASTER_MODE__MASK			0x0000f000
-#define A3XX_GRAS_SC_CONTROL_RASTER_MODE__SHIFT			12
-static inline uint32_t A3XX_GRAS_SC_CONTROL_RASTER_MODE(uint32_t val)
-{
-	return ((val) << A3XX_GRAS_SC_CONTROL_RASTER_MODE__SHIFT) & A3XX_GRAS_SC_CONTROL_RASTER_MODE__MASK;
-}
-
-#define REG_A3XX_GRAS_SC_SCREEN_SCISSOR_TL			0x00002074
-#define A3XX_GRAS_SC_SCREEN_SCISSOR_TL_WINDOW_OFFSET_DISABLE	0x80000000
-#define A3XX_GRAS_SC_SCREEN_SCISSOR_TL_X__MASK			0x00007fff
-#define A3XX_GRAS_SC_SCREEN_SCISSOR_TL_X__SHIFT			0
-static inline uint32_t A3XX_GRAS_SC_SCREEN_SCISSOR_TL_X(uint32_t val)
-{
-	return ((val) << A3XX_GRAS_SC_SCREEN_SCISSOR_TL_X__SHIFT) & A3XX_GRAS_SC_SCREEN_SCISSOR_TL_X__MASK;
-}
-#define A3XX_GRAS_SC_SCREEN_SCISSOR_TL_Y__MASK			0x7fff0000
-#define A3XX_GRAS_SC_SCREEN_SCISSOR_TL_Y__SHIFT			16
-static inline uint32_t A3XX_GRAS_SC_SCREEN_SCISSOR_TL_Y(uint32_t val)
-{
-	return ((val) << A3XX_GRAS_SC_SCREEN_SCISSOR_TL_Y__SHIFT) & A3XX_GRAS_SC_SCREEN_SCISSOR_TL_Y__MASK;
-}
-
-#define REG_A3XX_GRAS_SC_SCREEN_SCISSOR_BR			0x00002075
-#define A3XX_GRAS_SC_SCREEN_SCISSOR_BR_WINDOW_OFFSET_DISABLE	0x80000000
-#define A3XX_GRAS_SC_SCREEN_SCISSOR_BR_X__MASK			0x00007fff
-#define A3XX_GRAS_SC_SCREEN_SCISSOR_BR_X__SHIFT			0
-static inline uint32_t A3XX_GRAS_SC_SCREEN_SCISSOR_BR_X(uint32_t val)
-{
-	return ((val) << A3XX_GRAS_SC_SCREEN_SCISSOR_BR_X__SHIFT) & A3XX_GRAS_SC_SCREEN_SCISSOR_BR_X__MASK;
-}
-#define A3XX_GRAS_SC_SCREEN_SCISSOR_BR_Y__MASK			0x7fff0000
-#define A3XX_GRAS_SC_SCREEN_SCISSOR_BR_Y__SHIFT			16
-static inline uint32_t A3XX_GRAS_SC_SCREEN_SCISSOR_BR_Y(uint32_t val)
-{
-	return ((val) << A3XX_GRAS_SC_SCREEN_SCISSOR_BR_Y__SHIFT) & A3XX_GRAS_SC_SCREEN_SCISSOR_BR_Y__MASK;
-}
-
-#define REG_A3XX_GRAS_SC_WINDOW_SCISSOR_TL			0x00002079
-#define A3XX_GRAS_SC_WINDOW_SCISSOR_TL_WINDOW_OFFSET_DISABLE	0x80000000
-#define A3XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK			0x00007fff
-#define A3XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT			0
-static inline uint32_t A3XX_GRAS_SC_WINDOW_SCISSOR_TL_X(uint32_t val)
-{
-	return ((val) << A3XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT) & A3XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK;
-}
-#define A3XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK			0x7fff0000
-#define A3XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT			16
-static inline uint32_t A3XX_GRAS_SC_WINDOW_SCISSOR_TL_Y(uint32_t val)
-{
-	return ((val) << A3XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT) & A3XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK;
-}
-
-#define REG_A3XX_GRAS_SC_WINDOW_SCISSOR_BR			0x0000207a
-#define A3XX_GRAS_SC_WINDOW_SCISSOR_BR_WINDOW_OFFSET_DISABLE	0x80000000
-#define A3XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK			0x00007fff
-#define A3XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT			0
-static inline uint32_t A3XX_GRAS_SC_WINDOW_SCISSOR_BR_X(uint32_t val)
-{
-	return ((val) << A3XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT) & A3XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK;
-}
-#define A3XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK			0x7fff0000
-#define A3XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT			16
-static inline uint32_t A3XX_GRAS_SC_WINDOW_SCISSOR_BR_Y(uint32_t val)
-{
-	return ((val) << A3XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT) & A3XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK;
-}
-
-#define REG_A3XX_RB_MODE_CONTROL				0x000020c0
-#define A3XX_RB_MODE_CONTROL_GMEM_BYPASS			0x00000080
-#define A3XX_RB_MODE_CONTROL_RENDER_MODE__MASK			0x00000700
-#define A3XX_RB_MODE_CONTROL_RENDER_MODE__SHIFT			8
-static inline uint32_t A3XX_RB_MODE_CONTROL_RENDER_MODE(enum a3xx_render_mode val)
-{
-	return ((val) << A3XX_RB_MODE_CONTROL_RENDER_MODE__SHIFT) & A3XX_RB_MODE_CONTROL_RENDER_MODE__MASK;
-}
-#define A3XX_RB_MODE_CONTROL_MRT__MASK				0x00003000
-#define A3XX_RB_MODE_CONTROL_MRT__SHIFT				12
-static inline uint32_t A3XX_RB_MODE_CONTROL_MRT(uint32_t val)
-{
-	return ((val) << A3XX_RB_MODE_CONTROL_MRT__SHIFT) & A3XX_RB_MODE_CONTROL_MRT__MASK;
-}
-#define A3XX_RB_MODE_CONTROL_MARB_CACHE_SPLIT_MODE		0x00008000
-#define A3XX_RB_MODE_CONTROL_PACKER_TIMER_ENABLE		0x00010000
-
-#define REG_A3XX_RB_RENDER_CONTROL				0x000020c1
-#define A3XX_RB_RENDER_CONTROL_DUAL_COLOR_IN_ENABLE		0x00000001
-#define A3XX_RB_RENDER_CONTROL_YUV_IN_ENABLE			0x00000002
-#define A3XX_RB_RENDER_CONTROL_COV_VALUE_INPUT_ENABLE		0x00000004
-#define A3XX_RB_RENDER_CONTROL_FACENESS				0x00000008
-#define A3XX_RB_RENDER_CONTROL_BIN_WIDTH__MASK			0x00000ff0
-#define A3XX_RB_RENDER_CONTROL_BIN_WIDTH__SHIFT			4
-static inline uint32_t A3XX_RB_RENDER_CONTROL_BIN_WIDTH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A3XX_RB_RENDER_CONTROL_BIN_WIDTH__SHIFT) & A3XX_RB_RENDER_CONTROL_BIN_WIDTH__MASK;
-}
-#define A3XX_RB_RENDER_CONTROL_DISABLE_COLOR_PIPE		0x00001000
-#define A3XX_RB_RENDER_CONTROL_ENABLE_GMEM			0x00002000
-#define A3XX_RB_RENDER_CONTROL_COORD_MASK__MASK			0x0003c000
-#define A3XX_RB_RENDER_CONTROL_COORD_MASK__SHIFT		14
-static inline uint32_t A3XX_RB_RENDER_CONTROL_COORD_MASK(uint32_t val)
-{
-	return ((val) << A3XX_RB_RENDER_CONTROL_COORD_MASK__SHIFT) & A3XX_RB_RENDER_CONTROL_COORD_MASK__MASK;
-}
-#define A3XX_RB_RENDER_CONTROL_I_CLAMP_ENABLE			0x00080000
-#define A3XX_RB_RENDER_CONTROL_COV_VALUE_OUTPUT_ENABLE		0x00100000
-#define A3XX_RB_RENDER_CONTROL_ALPHA_TEST			0x00400000
-#define A3XX_RB_RENDER_CONTROL_ALPHA_TEST_FUNC__MASK		0x07000000
-#define A3XX_RB_RENDER_CONTROL_ALPHA_TEST_FUNC__SHIFT		24
-static inline uint32_t A3XX_RB_RENDER_CONTROL_ALPHA_TEST_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A3XX_RB_RENDER_CONTROL_ALPHA_TEST_FUNC__SHIFT) & A3XX_RB_RENDER_CONTROL_ALPHA_TEST_FUNC__MASK;
-}
-#define A3XX_RB_RENDER_CONTROL_ALPHA_TO_COVERAGE		0x40000000
-#define A3XX_RB_RENDER_CONTROL_ALPHA_TO_ONE			0x80000000
-
-#define REG_A3XX_RB_MSAA_CONTROL				0x000020c2
-#define A3XX_RB_MSAA_CONTROL_DISABLE				0x00000400
-#define A3XX_RB_MSAA_CONTROL_SAMPLES__MASK			0x0000f000
-#define A3XX_RB_MSAA_CONTROL_SAMPLES__SHIFT			12
-static inline uint32_t A3XX_RB_MSAA_CONTROL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A3XX_RB_MSAA_CONTROL_SAMPLES__SHIFT) & A3XX_RB_MSAA_CONTROL_SAMPLES__MASK;
-}
-#define A3XX_RB_MSAA_CONTROL_SAMPLE_MASK__MASK			0xffff0000
-#define A3XX_RB_MSAA_CONTROL_SAMPLE_MASK__SHIFT			16
-static inline uint32_t A3XX_RB_MSAA_CONTROL_SAMPLE_MASK(uint32_t val)
-{
-	return ((val) << A3XX_RB_MSAA_CONTROL_SAMPLE_MASK__SHIFT) & A3XX_RB_MSAA_CONTROL_SAMPLE_MASK__MASK;
-}
-
-#define REG_A3XX_RB_ALPHA_REF					0x000020c3
-#define A3XX_RB_ALPHA_REF_UINT__MASK				0x0000ff00
-#define A3XX_RB_ALPHA_REF_UINT__SHIFT				8
-static inline uint32_t A3XX_RB_ALPHA_REF_UINT(uint32_t val)
-{
-	return ((val) << A3XX_RB_ALPHA_REF_UINT__SHIFT) & A3XX_RB_ALPHA_REF_UINT__MASK;
-}
-#define A3XX_RB_ALPHA_REF_FLOAT__MASK				0xffff0000
-#define A3XX_RB_ALPHA_REF_FLOAT__SHIFT				16
-static inline uint32_t A3XX_RB_ALPHA_REF_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A3XX_RB_ALPHA_REF_FLOAT__SHIFT) & A3XX_RB_ALPHA_REF_FLOAT__MASK;
-}
-
-#define REG_A3XX_RB_MRT(i0) (0x000020c4 + 0x4*(i0))
-
-static inline uint32_t REG_A3XX_RB_MRT_CONTROL(uint32_t i0) { return 0x000020c4 + 0x4*i0; }
-#define A3XX_RB_MRT_CONTROL_READ_DEST_ENABLE			0x00000008
-#define A3XX_RB_MRT_CONTROL_BLEND				0x00000010
-#define A3XX_RB_MRT_CONTROL_BLEND2				0x00000020
-#define A3XX_RB_MRT_CONTROL_ROP_CODE__MASK			0x00000f00
-#define A3XX_RB_MRT_CONTROL_ROP_CODE__SHIFT			8
-static inline uint32_t A3XX_RB_MRT_CONTROL_ROP_CODE(enum a3xx_rop_code val)
-{
-	return ((val) << A3XX_RB_MRT_CONTROL_ROP_CODE__SHIFT) & A3XX_RB_MRT_CONTROL_ROP_CODE__MASK;
-}
-#define A3XX_RB_MRT_CONTROL_DITHER_MODE__MASK			0x00003000
-#define A3XX_RB_MRT_CONTROL_DITHER_MODE__SHIFT			12
-static inline uint32_t A3XX_RB_MRT_CONTROL_DITHER_MODE(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A3XX_RB_MRT_CONTROL_DITHER_MODE__SHIFT) & A3XX_RB_MRT_CONTROL_DITHER_MODE__MASK;
-}
-#define A3XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK		0x0f000000
-#define A3XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT		24
-static inline uint32_t A3XX_RB_MRT_CONTROL_COMPONENT_ENABLE(uint32_t val)
-{
-	return ((val) << A3XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT) & A3XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK;
-}
-
-static inline uint32_t REG_A3XX_RB_MRT_BUF_INFO(uint32_t i0) { return 0x000020c5 + 0x4*i0; }
-#define A3XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK			0x0000003f
-#define A3XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT		0
-static inline uint32_t A3XX_RB_MRT_BUF_INFO_COLOR_FORMAT(enum a3xx_color_fmt val)
-{
-	return ((val) << A3XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT) & A3XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK;
-}
-#define A3XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK		0x000000c0
-#define A3XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT		6
-static inline uint32_t A3XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE(enum a3xx_tile_mode val)
-{
-	return ((val) << A3XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT) & A3XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK;
-}
-#define A3XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK			0x00000c00
-#define A3XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT			10
-static inline uint32_t A3XX_RB_MRT_BUF_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A3XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT) & A3XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK;
-}
-#define A3XX_RB_MRT_BUF_INFO_COLOR_SRGB				0x00004000
-#define A3XX_RB_MRT_BUF_INFO_COLOR_BUF_PITCH__MASK		0xfffe0000
-#define A3XX_RB_MRT_BUF_INFO_COLOR_BUF_PITCH__SHIFT		17
-static inline uint32_t A3XX_RB_MRT_BUF_INFO_COLOR_BUF_PITCH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A3XX_RB_MRT_BUF_INFO_COLOR_BUF_PITCH__SHIFT) & A3XX_RB_MRT_BUF_INFO_COLOR_BUF_PITCH__MASK;
-}
-
-static inline uint32_t REG_A3XX_RB_MRT_BUF_BASE(uint32_t i0) { return 0x000020c6 + 0x4*i0; }
-#define A3XX_RB_MRT_BUF_BASE_COLOR_BUF_BASE__MASK		0xfffffff0
-#define A3XX_RB_MRT_BUF_BASE_COLOR_BUF_BASE__SHIFT		4
-static inline uint32_t A3XX_RB_MRT_BUF_BASE_COLOR_BUF_BASE(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A3XX_RB_MRT_BUF_BASE_COLOR_BUF_BASE__SHIFT) & A3XX_RB_MRT_BUF_BASE_COLOR_BUF_BASE__MASK;
-}
-
-static inline uint32_t REG_A3XX_RB_MRT_BLEND_CONTROL(uint32_t i0) { return 0x000020c7 + 0x4*i0; }
-#define A3XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK		0x0000001f
-#define A3XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT		0
-static inline uint32_t A3XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A3XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT) & A3XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK;
-}
-#define A3XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK	0x000000e0
-#define A3XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT	5
-static inline uint32_t A3XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
-{
-	return ((val) << A3XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT) & A3XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK;
-}
-#define A3XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK		0x00001f00
-#define A3XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT	8
-static inline uint32_t A3XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A3XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT) & A3XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK;
-}
-#define A3XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK	0x001f0000
-#define A3XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT	16
-static inline uint32_t A3XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A3XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT) & A3XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK;
-}
-#define A3XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK	0x00e00000
-#define A3XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT	21
-static inline uint32_t A3XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
-{
-	return ((val) << A3XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT) & A3XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK;
-}
-#define A3XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK	0x1f000000
-#define A3XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT	24
-static inline uint32_t A3XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A3XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT) & A3XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK;
-}
-#define A3XX_RB_MRT_BLEND_CONTROL_CLAMP_ENABLE			0x20000000
-
-#define REG_A3XX_RB_BLEND_RED					0x000020e4
-#define A3XX_RB_BLEND_RED_UINT__MASK				0x000000ff
-#define A3XX_RB_BLEND_RED_UINT__SHIFT				0
-static inline uint32_t A3XX_RB_BLEND_RED_UINT(uint32_t val)
-{
-	return ((val) << A3XX_RB_BLEND_RED_UINT__SHIFT) & A3XX_RB_BLEND_RED_UINT__MASK;
-}
-#define A3XX_RB_BLEND_RED_FLOAT__MASK				0xffff0000
-#define A3XX_RB_BLEND_RED_FLOAT__SHIFT				16
-static inline uint32_t A3XX_RB_BLEND_RED_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A3XX_RB_BLEND_RED_FLOAT__SHIFT) & A3XX_RB_BLEND_RED_FLOAT__MASK;
-}
-
-#define REG_A3XX_RB_BLEND_GREEN					0x000020e5
-#define A3XX_RB_BLEND_GREEN_UINT__MASK				0x000000ff
-#define A3XX_RB_BLEND_GREEN_UINT__SHIFT				0
-static inline uint32_t A3XX_RB_BLEND_GREEN_UINT(uint32_t val)
-{
-	return ((val) << A3XX_RB_BLEND_GREEN_UINT__SHIFT) & A3XX_RB_BLEND_GREEN_UINT__MASK;
-}
-#define A3XX_RB_BLEND_GREEN_FLOAT__MASK				0xffff0000
-#define A3XX_RB_BLEND_GREEN_FLOAT__SHIFT			16
-static inline uint32_t A3XX_RB_BLEND_GREEN_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A3XX_RB_BLEND_GREEN_FLOAT__SHIFT) & A3XX_RB_BLEND_GREEN_FLOAT__MASK;
-}
-
-#define REG_A3XX_RB_BLEND_BLUE					0x000020e6
-#define A3XX_RB_BLEND_BLUE_UINT__MASK				0x000000ff
-#define A3XX_RB_BLEND_BLUE_UINT__SHIFT				0
-static inline uint32_t A3XX_RB_BLEND_BLUE_UINT(uint32_t val)
-{
-	return ((val) << A3XX_RB_BLEND_BLUE_UINT__SHIFT) & A3XX_RB_BLEND_BLUE_UINT__MASK;
-}
-#define A3XX_RB_BLEND_BLUE_FLOAT__MASK				0xffff0000
-#define A3XX_RB_BLEND_BLUE_FLOAT__SHIFT				16
-static inline uint32_t A3XX_RB_BLEND_BLUE_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A3XX_RB_BLEND_BLUE_FLOAT__SHIFT) & A3XX_RB_BLEND_BLUE_FLOAT__MASK;
-}
-
-#define REG_A3XX_RB_BLEND_ALPHA					0x000020e7
-#define A3XX_RB_BLEND_ALPHA_UINT__MASK				0x000000ff
-#define A3XX_RB_BLEND_ALPHA_UINT__SHIFT				0
-static inline uint32_t A3XX_RB_BLEND_ALPHA_UINT(uint32_t val)
-{
-	return ((val) << A3XX_RB_BLEND_ALPHA_UINT__SHIFT) & A3XX_RB_BLEND_ALPHA_UINT__MASK;
-}
-#define A3XX_RB_BLEND_ALPHA_FLOAT__MASK				0xffff0000
-#define A3XX_RB_BLEND_ALPHA_FLOAT__SHIFT			16
-static inline uint32_t A3XX_RB_BLEND_ALPHA_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A3XX_RB_BLEND_ALPHA_FLOAT__SHIFT) & A3XX_RB_BLEND_ALPHA_FLOAT__MASK;
-}
-
-#define REG_A3XX_RB_CLEAR_COLOR_DW0				0x000020e8
-
-#define REG_A3XX_RB_CLEAR_COLOR_DW1				0x000020e9
-
-#define REG_A3XX_RB_CLEAR_COLOR_DW2				0x000020ea
-
-#define REG_A3XX_RB_CLEAR_COLOR_DW3				0x000020eb
-
-#define REG_A3XX_RB_COPY_CONTROL				0x000020ec
-#define A3XX_RB_COPY_CONTROL_MSAA_RESOLVE__MASK			0x00000003
-#define A3XX_RB_COPY_CONTROL_MSAA_RESOLVE__SHIFT		0
-static inline uint32_t A3XX_RB_COPY_CONTROL_MSAA_RESOLVE(enum a3xx_msaa_samples val)
-{
-	return ((val) << A3XX_RB_COPY_CONTROL_MSAA_RESOLVE__SHIFT) & A3XX_RB_COPY_CONTROL_MSAA_RESOLVE__MASK;
-}
-#define A3XX_RB_COPY_CONTROL_DEPTHCLEAR				0x00000008
-#define A3XX_RB_COPY_CONTROL_MODE__MASK				0x00000070
-#define A3XX_RB_COPY_CONTROL_MODE__SHIFT			4
-static inline uint32_t A3XX_RB_COPY_CONTROL_MODE(enum adreno_rb_copy_control_mode val)
-{
-	return ((val) << A3XX_RB_COPY_CONTROL_MODE__SHIFT) & A3XX_RB_COPY_CONTROL_MODE__MASK;
-}
-#define A3XX_RB_COPY_CONTROL_MSAA_SRGB_DOWNSAMPLE		0x00000080
-#define A3XX_RB_COPY_CONTROL_FASTCLEAR__MASK			0x00000f00
-#define A3XX_RB_COPY_CONTROL_FASTCLEAR__SHIFT			8
-static inline uint32_t A3XX_RB_COPY_CONTROL_FASTCLEAR(uint32_t val)
-{
-	return ((val) << A3XX_RB_COPY_CONTROL_FASTCLEAR__SHIFT) & A3XX_RB_COPY_CONTROL_FASTCLEAR__MASK;
-}
-#define A3XX_RB_COPY_CONTROL_DEPTH32_RESOLVE			0x00001000
-#define A3XX_RB_COPY_CONTROL_GMEM_BASE__MASK			0xffffc000
-#define A3XX_RB_COPY_CONTROL_GMEM_BASE__SHIFT			14
-static inline uint32_t A3XX_RB_COPY_CONTROL_GMEM_BASE(uint32_t val)
-{
-	assert(!(val & 0x3fff));
-	return (((val >> 14)) << A3XX_RB_COPY_CONTROL_GMEM_BASE__SHIFT) & A3XX_RB_COPY_CONTROL_GMEM_BASE__MASK;
-}
-
-#define REG_A3XX_RB_COPY_DEST_BASE				0x000020ed
-#define A3XX_RB_COPY_DEST_BASE_BASE__MASK			0xfffffff0
-#define A3XX_RB_COPY_DEST_BASE_BASE__SHIFT			4
-static inline uint32_t A3XX_RB_COPY_DEST_BASE_BASE(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A3XX_RB_COPY_DEST_BASE_BASE__SHIFT) & A3XX_RB_COPY_DEST_BASE_BASE__MASK;
-}
-
-#define REG_A3XX_RB_COPY_DEST_PITCH				0x000020ee
-#define A3XX_RB_COPY_DEST_PITCH_PITCH__MASK			0xffffffff
-#define A3XX_RB_COPY_DEST_PITCH_PITCH__SHIFT			0
-static inline uint32_t A3XX_RB_COPY_DEST_PITCH_PITCH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A3XX_RB_COPY_DEST_PITCH_PITCH__SHIFT) & A3XX_RB_COPY_DEST_PITCH_PITCH__MASK;
-}
-
-#define REG_A3XX_RB_COPY_DEST_INFO				0x000020ef
-#define A3XX_RB_COPY_DEST_INFO_TILE__MASK			0x00000003
-#define A3XX_RB_COPY_DEST_INFO_TILE__SHIFT			0
-static inline uint32_t A3XX_RB_COPY_DEST_INFO_TILE(enum a3xx_tile_mode val)
-{
-	return ((val) << A3XX_RB_COPY_DEST_INFO_TILE__SHIFT) & A3XX_RB_COPY_DEST_INFO_TILE__MASK;
-}
-#define A3XX_RB_COPY_DEST_INFO_FORMAT__MASK			0x000000fc
-#define A3XX_RB_COPY_DEST_INFO_FORMAT__SHIFT			2
-static inline uint32_t A3XX_RB_COPY_DEST_INFO_FORMAT(enum a3xx_color_fmt val)
-{
-	return ((val) << A3XX_RB_COPY_DEST_INFO_FORMAT__SHIFT) & A3XX_RB_COPY_DEST_INFO_FORMAT__MASK;
-}
-#define A3XX_RB_COPY_DEST_INFO_SWAP__MASK			0x00000300
-#define A3XX_RB_COPY_DEST_INFO_SWAP__SHIFT			8
-static inline uint32_t A3XX_RB_COPY_DEST_INFO_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A3XX_RB_COPY_DEST_INFO_SWAP__SHIFT) & A3XX_RB_COPY_DEST_INFO_SWAP__MASK;
-}
-#define A3XX_RB_COPY_DEST_INFO_DITHER_MODE__MASK		0x00000c00
-#define A3XX_RB_COPY_DEST_INFO_DITHER_MODE__SHIFT		10
-static inline uint32_t A3XX_RB_COPY_DEST_INFO_DITHER_MODE(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A3XX_RB_COPY_DEST_INFO_DITHER_MODE__SHIFT) & A3XX_RB_COPY_DEST_INFO_DITHER_MODE__MASK;
-}
-#define A3XX_RB_COPY_DEST_INFO_COMPONENT_ENABLE__MASK		0x0003c000
-#define A3XX_RB_COPY_DEST_INFO_COMPONENT_ENABLE__SHIFT		14
-static inline uint32_t A3XX_RB_COPY_DEST_INFO_COMPONENT_ENABLE(uint32_t val)
-{
-	return ((val) << A3XX_RB_COPY_DEST_INFO_COMPONENT_ENABLE__SHIFT) & A3XX_RB_COPY_DEST_INFO_COMPONENT_ENABLE__MASK;
-}
-#define A3XX_RB_COPY_DEST_INFO_ENDIAN__MASK			0x001c0000
-#define A3XX_RB_COPY_DEST_INFO_ENDIAN__SHIFT			18
-static inline uint32_t A3XX_RB_COPY_DEST_INFO_ENDIAN(enum adreno_rb_surface_endian val)
-{
-	return ((val) << A3XX_RB_COPY_DEST_INFO_ENDIAN__SHIFT) & A3XX_RB_COPY_DEST_INFO_ENDIAN__MASK;
-}
-
-#define REG_A3XX_RB_DEPTH_CONTROL				0x00002100
-#define A3XX_RB_DEPTH_CONTROL_FRAG_WRITES_Z			0x00000001
-#define A3XX_RB_DEPTH_CONTROL_Z_TEST_ENABLE			0x00000002
-#define A3XX_RB_DEPTH_CONTROL_Z_WRITE_ENABLE			0x00000004
-#define A3XX_RB_DEPTH_CONTROL_EARLY_Z_DISABLE			0x00000008
-#define A3XX_RB_DEPTH_CONTROL_ZFUNC__MASK			0x00000070
-#define A3XX_RB_DEPTH_CONTROL_ZFUNC__SHIFT			4
-static inline uint32_t A3XX_RB_DEPTH_CONTROL_ZFUNC(enum adreno_compare_func val)
-{
-	return ((val) << A3XX_RB_DEPTH_CONTROL_ZFUNC__SHIFT) & A3XX_RB_DEPTH_CONTROL_ZFUNC__MASK;
-}
-#define A3XX_RB_DEPTH_CONTROL_Z_CLAMP_ENABLE			0x00000080
-#define A3XX_RB_DEPTH_CONTROL_Z_READ_ENABLE			0x80000000
-
-#define REG_A3XX_RB_DEPTH_CLEAR					0x00002101
-
-#define REG_A3XX_RB_DEPTH_INFO					0x00002102
-#define A3XX_RB_DEPTH_INFO_DEPTH_FORMAT__MASK			0x00000003
-#define A3XX_RB_DEPTH_INFO_DEPTH_FORMAT__SHIFT			0
-static inline uint32_t A3XX_RB_DEPTH_INFO_DEPTH_FORMAT(enum adreno_rb_depth_format val)
-{
-	return ((val) << A3XX_RB_DEPTH_INFO_DEPTH_FORMAT__SHIFT) & A3XX_RB_DEPTH_INFO_DEPTH_FORMAT__MASK;
-}
-#define A3XX_RB_DEPTH_INFO_DEPTH_BASE__MASK			0xfffff800
-#define A3XX_RB_DEPTH_INFO_DEPTH_BASE__SHIFT			11
-static inline uint32_t A3XX_RB_DEPTH_INFO_DEPTH_BASE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A3XX_RB_DEPTH_INFO_DEPTH_BASE__SHIFT) & A3XX_RB_DEPTH_INFO_DEPTH_BASE__MASK;
-}
-
-#define REG_A3XX_RB_DEPTH_PITCH					0x00002103
-#define A3XX_RB_DEPTH_PITCH__MASK				0xffffffff
-#define A3XX_RB_DEPTH_PITCH__SHIFT				0
-static inline uint32_t A3XX_RB_DEPTH_PITCH(uint32_t val)
-{
-	assert(!(val & 0x7));
-	return (((val >> 3)) << A3XX_RB_DEPTH_PITCH__SHIFT) & A3XX_RB_DEPTH_PITCH__MASK;
-}
-
-#define REG_A3XX_RB_STENCIL_CONTROL				0x00002104
-#define A3XX_RB_STENCIL_CONTROL_STENCIL_ENABLE			0x00000001
-#define A3XX_RB_STENCIL_CONTROL_STENCIL_ENABLE_BF		0x00000002
-#define A3XX_RB_STENCIL_CONTROL_STENCIL_READ			0x00000004
-#define A3XX_RB_STENCIL_CONTROL_FUNC__MASK			0x00000700
-#define A3XX_RB_STENCIL_CONTROL_FUNC__SHIFT			8
-static inline uint32_t A3XX_RB_STENCIL_CONTROL_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A3XX_RB_STENCIL_CONTROL_FUNC__SHIFT) & A3XX_RB_STENCIL_CONTROL_FUNC__MASK;
-}
-#define A3XX_RB_STENCIL_CONTROL_FAIL__MASK			0x00003800
-#define A3XX_RB_STENCIL_CONTROL_FAIL__SHIFT			11
-static inline uint32_t A3XX_RB_STENCIL_CONTROL_FAIL(enum adreno_stencil_op val)
-{
-	return ((val) << A3XX_RB_STENCIL_CONTROL_FAIL__SHIFT) & A3XX_RB_STENCIL_CONTROL_FAIL__MASK;
-}
-#define A3XX_RB_STENCIL_CONTROL_ZPASS__MASK			0x0001c000
-#define A3XX_RB_STENCIL_CONTROL_ZPASS__SHIFT			14
-static inline uint32_t A3XX_RB_STENCIL_CONTROL_ZPASS(enum adreno_stencil_op val)
-{
-	return ((val) << A3XX_RB_STENCIL_CONTROL_ZPASS__SHIFT) & A3XX_RB_STENCIL_CONTROL_ZPASS__MASK;
-}
-#define A3XX_RB_STENCIL_CONTROL_ZFAIL__MASK			0x000e0000
-#define A3XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT			17
-static inline uint32_t A3XX_RB_STENCIL_CONTROL_ZFAIL(enum adreno_stencil_op val)
-{
-	return ((val) << A3XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT) & A3XX_RB_STENCIL_CONTROL_ZFAIL__MASK;
-}
-#define A3XX_RB_STENCIL_CONTROL_FUNC_BF__MASK			0x00700000
-#define A3XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT			20
-static inline uint32_t A3XX_RB_STENCIL_CONTROL_FUNC_BF(enum adreno_compare_func val)
-{
-	return ((val) << A3XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT) & A3XX_RB_STENCIL_CONTROL_FUNC_BF__MASK;
-}
-#define A3XX_RB_STENCIL_CONTROL_FAIL_BF__MASK			0x03800000
-#define A3XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT			23
-static inline uint32_t A3XX_RB_STENCIL_CONTROL_FAIL_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A3XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT) & A3XX_RB_STENCIL_CONTROL_FAIL_BF__MASK;
-}
-#define A3XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK			0x1c000000
-#define A3XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT			26
-static inline uint32_t A3XX_RB_STENCIL_CONTROL_ZPASS_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A3XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT) & A3XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK;
-}
-#define A3XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK			0xe0000000
-#define A3XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT			29
-static inline uint32_t A3XX_RB_STENCIL_CONTROL_ZFAIL_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A3XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT) & A3XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK;
-}
-
-#define REG_A3XX_RB_STENCIL_CLEAR				0x00002105
-
-#define REG_A3XX_RB_STENCIL_INFO				0x00002106
-#define A3XX_RB_STENCIL_INFO_STENCIL_BASE__MASK			0xfffff800
-#define A3XX_RB_STENCIL_INFO_STENCIL_BASE__SHIFT		11
-static inline uint32_t A3XX_RB_STENCIL_INFO_STENCIL_BASE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A3XX_RB_STENCIL_INFO_STENCIL_BASE__SHIFT) & A3XX_RB_STENCIL_INFO_STENCIL_BASE__MASK;
-}
-
-#define REG_A3XX_RB_STENCIL_PITCH				0x00002107
-#define A3XX_RB_STENCIL_PITCH__MASK				0xffffffff
-#define A3XX_RB_STENCIL_PITCH__SHIFT				0
-static inline uint32_t A3XX_RB_STENCIL_PITCH(uint32_t val)
-{
-	assert(!(val & 0x7));
-	return (((val >> 3)) << A3XX_RB_STENCIL_PITCH__SHIFT) & A3XX_RB_STENCIL_PITCH__MASK;
-}
-
-#define REG_A3XX_RB_STENCILREFMASK				0x00002108
-#define A3XX_RB_STENCILREFMASK_STENCILREF__MASK			0x000000ff
-#define A3XX_RB_STENCILREFMASK_STENCILREF__SHIFT		0
-static inline uint32_t A3XX_RB_STENCILREFMASK_STENCILREF(uint32_t val)
-{
-	return ((val) << A3XX_RB_STENCILREFMASK_STENCILREF__SHIFT) & A3XX_RB_STENCILREFMASK_STENCILREF__MASK;
-}
-#define A3XX_RB_STENCILREFMASK_STENCILMASK__MASK		0x0000ff00
-#define A3XX_RB_STENCILREFMASK_STENCILMASK__SHIFT		8
-static inline uint32_t A3XX_RB_STENCILREFMASK_STENCILMASK(uint32_t val)
-{
-	return ((val) << A3XX_RB_STENCILREFMASK_STENCILMASK__SHIFT) & A3XX_RB_STENCILREFMASK_STENCILMASK__MASK;
-}
-#define A3XX_RB_STENCILREFMASK_STENCILWRITEMASK__MASK		0x00ff0000
-#define A3XX_RB_STENCILREFMASK_STENCILWRITEMASK__SHIFT		16
-static inline uint32_t A3XX_RB_STENCILREFMASK_STENCILWRITEMASK(uint32_t val)
-{
-	return ((val) << A3XX_RB_STENCILREFMASK_STENCILWRITEMASK__SHIFT) & A3XX_RB_STENCILREFMASK_STENCILWRITEMASK__MASK;
-}
-
-#define REG_A3XX_RB_STENCILREFMASK_BF				0x00002109
-#define A3XX_RB_STENCILREFMASK_BF_STENCILREF__MASK		0x000000ff
-#define A3XX_RB_STENCILREFMASK_BF_STENCILREF__SHIFT		0
-static inline uint32_t A3XX_RB_STENCILREFMASK_BF_STENCILREF(uint32_t val)
-{
-	return ((val) << A3XX_RB_STENCILREFMASK_BF_STENCILREF__SHIFT) & A3XX_RB_STENCILREFMASK_BF_STENCILREF__MASK;
-}
-#define A3XX_RB_STENCILREFMASK_BF_STENCILMASK__MASK		0x0000ff00
-#define A3XX_RB_STENCILREFMASK_BF_STENCILMASK__SHIFT		8
-static inline uint32_t A3XX_RB_STENCILREFMASK_BF_STENCILMASK(uint32_t val)
-{
-	return ((val) << A3XX_RB_STENCILREFMASK_BF_STENCILMASK__SHIFT) & A3XX_RB_STENCILREFMASK_BF_STENCILMASK__MASK;
-}
-#define A3XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__MASK	0x00ff0000
-#define A3XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__SHIFT	16
-static inline uint32_t A3XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK(uint32_t val)
-{
-	return ((val) << A3XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__SHIFT) & A3XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__MASK;
-}
-
-#define REG_A3XX_RB_LRZ_VSC_CONTROL				0x0000210c
-#define A3XX_RB_LRZ_VSC_CONTROL_BINNING_ENABLE			0x00000002
-
-#define REG_A3XX_RB_WINDOW_OFFSET				0x0000210e
-#define A3XX_RB_WINDOW_OFFSET_X__MASK				0x0000ffff
-#define A3XX_RB_WINDOW_OFFSET_X__SHIFT				0
-static inline uint32_t A3XX_RB_WINDOW_OFFSET_X(uint32_t val)
-{
-	return ((val) << A3XX_RB_WINDOW_OFFSET_X__SHIFT) & A3XX_RB_WINDOW_OFFSET_X__MASK;
-}
-#define A3XX_RB_WINDOW_OFFSET_Y__MASK				0xffff0000
-#define A3XX_RB_WINDOW_OFFSET_Y__SHIFT				16
-static inline uint32_t A3XX_RB_WINDOW_OFFSET_Y(uint32_t val)
-{
-	return ((val) << A3XX_RB_WINDOW_OFFSET_Y__SHIFT) & A3XX_RB_WINDOW_OFFSET_Y__MASK;
-}
-
-#define REG_A3XX_RB_SAMPLE_COUNT_CONTROL			0x00002110
-#define A3XX_RB_SAMPLE_COUNT_CONTROL_RESET			0x00000001
-#define A3XX_RB_SAMPLE_COUNT_CONTROL_COPY			0x00000002
-
-#define REG_A3XX_RB_SAMPLE_COUNT_ADDR				0x00002111
-
-#define REG_A3XX_RB_Z_CLAMP_MIN					0x00002114
-
-#define REG_A3XX_RB_Z_CLAMP_MAX					0x00002115
-
-#define REG_A3XX_VGT_BIN_BASE					0x000021e1
-
-#define REG_A3XX_VGT_BIN_SIZE					0x000021e2
-
-#define REG_A3XX_PC_VSTREAM_CONTROL				0x000021e4
-#define A3XX_PC_VSTREAM_CONTROL_SIZE__MASK			0x003f0000
-#define A3XX_PC_VSTREAM_CONTROL_SIZE__SHIFT			16
-static inline uint32_t A3XX_PC_VSTREAM_CONTROL_SIZE(uint32_t val)
-{
-	return ((val) << A3XX_PC_VSTREAM_CONTROL_SIZE__SHIFT) & A3XX_PC_VSTREAM_CONTROL_SIZE__MASK;
-}
-#define A3XX_PC_VSTREAM_CONTROL_N__MASK				0x07c00000
-#define A3XX_PC_VSTREAM_CONTROL_N__SHIFT			22
-static inline uint32_t A3XX_PC_VSTREAM_CONTROL_N(uint32_t val)
-{
-	return ((val) << A3XX_PC_VSTREAM_CONTROL_N__SHIFT) & A3XX_PC_VSTREAM_CONTROL_N__MASK;
-}
-
-#define REG_A3XX_PC_VERTEX_REUSE_BLOCK_CNTL			0x000021ea
-
-#define REG_A3XX_PC_PRIM_VTX_CNTL				0x000021ec
-#define A3XX_PC_PRIM_VTX_CNTL_STRIDE_IN_VPC__MASK		0x0000001f
-#define A3XX_PC_PRIM_VTX_CNTL_STRIDE_IN_VPC__SHIFT		0
-static inline uint32_t A3XX_PC_PRIM_VTX_CNTL_STRIDE_IN_VPC(uint32_t val)
-{
-	return ((val) << A3XX_PC_PRIM_VTX_CNTL_STRIDE_IN_VPC__SHIFT) & A3XX_PC_PRIM_VTX_CNTL_STRIDE_IN_VPC__MASK;
-}
-#define A3XX_PC_PRIM_VTX_CNTL_POLYMODE_FRONT_PTYPE__MASK	0x000000e0
-#define A3XX_PC_PRIM_VTX_CNTL_POLYMODE_FRONT_PTYPE__SHIFT	5
-static inline uint32_t A3XX_PC_PRIM_VTX_CNTL_POLYMODE_FRONT_PTYPE(enum adreno_pa_su_sc_draw val)
-{
-	return ((val) << A3XX_PC_PRIM_VTX_CNTL_POLYMODE_FRONT_PTYPE__SHIFT) & A3XX_PC_PRIM_VTX_CNTL_POLYMODE_FRONT_PTYPE__MASK;
-}
-#define A3XX_PC_PRIM_VTX_CNTL_POLYMODE_BACK_PTYPE__MASK		0x00000700
-#define A3XX_PC_PRIM_VTX_CNTL_POLYMODE_BACK_PTYPE__SHIFT	8
-static inline uint32_t A3XX_PC_PRIM_VTX_CNTL_POLYMODE_BACK_PTYPE(enum adreno_pa_su_sc_draw val)
-{
-	return ((val) << A3XX_PC_PRIM_VTX_CNTL_POLYMODE_BACK_PTYPE__SHIFT) & A3XX_PC_PRIM_VTX_CNTL_POLYMODE_BACK_PTYPE__MASK;
-}
-#define A3XX_PC_PRIM_VTX_CNTL_POLYMODE_ENABLE			0x00001000
-#define A3XX_PC_PRIM_VTX_CNTL_PRIMITIVE_RESTART			0x00100000
-#define A3XX_PC_PRIM_VTX_CNTL_PROVOKING_VTX_LAST		0x02000000
-#define A3XX_PC_PRIM_VTX_CNTL_PSIZE				0x04000000
-
-#define REG_A3XX_PC_RESTART_INDEX				0x000021ed
-
-#define REG_A3XX_HLSQ_CONTROL_0_REG				0x00002200
-#define A3XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__MASK		0x00000030
-#define A3XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__SHIFT		4
-static inline uint32_t A3XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__SHIFT) & A3XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__MASK;
-}
-#define A3XX_HLSQ_CONTROL_0_REG_FSSUPERTHREADENABLE		0x00000040
-#define A3XX_HLSQ_CONTROL_0_REG_COMPUTEMODE			0x00000100
-#define A3XX_HLSQ_CONTROL_0_REG_SPSHADERRESTART			0x00000200
-#define A3XX_HLSQ_CONTROL_0_REG_RESERVED2			0x00000400
-#define A3XX_HLSQ_CONTROL_0_REG_CYCLETIMEOUTLIMITVPC__MASK	0x00fff000
-#define A3XX_HLSQ_CONTROL_0_REG_CYCLETIMEOUTLIMITVPC__SHIFT	12
-static inline uint32_t A3XX_HLSQ_CONTROL_0_REG_CYCLETIMEOUTLIMITVPC(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_0_REG_CYCLETIMEOUTLIMITVPC__SHIFT) & A3XX_HLSQ_CONTROL_0_REG_CYCLETIMEOUTLIMITVPC__MASK;
-}
-#define A3XX_HLSQ_CONTROL_0_REG_FSONLYTEX			0x02000000
-#define A3XX_HLSQ_CONTROL_0_REG_CHUNKDISABLE			0x04000000
-#define A3XX_HLSQ_CONTROL_0_REG_CONSTMODE__MASK			0x08000000
-#define A3XX_HLSQ_CONTROL_0_REG_CONSTMODE__SHIFT		27
-static inline uint32_t A3XX_HLSQ_CONTROL_0_REG_CONSTMODE(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_0_REG_CONSTMODE__SHIFT) & A3XX_HLSQ_CONTROL_0_REG_CONSTMODE__MASK;
-}
-#define A3XX_HLSQ_CONTROL_0_REG_LAZYUPDATEDISABLE		0x10000000
-#define A3XX_HLSQ_CONTROL_0_REG_SPCONSTFULLUPDATE		0x20000000
-#define A3XX_HLSQ_CONTROL_0_REG_TPFULLUPDATE			0x40000000
-#define A3XX_HLSQ_CONTROL_0_REG_SINGLECONTEXT			0x80000000
-
-#define REG_A3XX_HLSQ_CONTROL_1_REG				0x00002201
-#define A3XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__MASK		0x000000c0
-#define A3XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__SHIFT		6
-static inline uint32_t A3XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__SHIFT) & A3XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__MASK;
-}
-#define A3XX_HLSQ_CONTROL_1_REG_VSSUPERTHREADENABLE		0x00000100
-#define A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDXYREGID__MASK		0x00ff0000
-#define A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDXYREGID__SHIFT		16
-static inline uint32_t A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDXYREGID(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDXYREGID__SHIFT) & A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDXYREGID__MASK;
-}
-#define A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDZWREGID__MASK		0xff000000
-#define A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDZWREGID__SHIFT		24
-static inline uint32_t A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDZWREGID(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDZWREGID__SHIFT) & A3XX_HLSQ_CONTROL_1_REG_FRAGCOORDZWREGID__MASK;
-}
-
-#define REG_A3XX_HLSQ_CONTROL_2_REG				0x00002202
-#define A3XX_HLSQ_CONTROL_2_REG_FACENESSREGID__MASK		0x000003fc
-#define A3XX_HLSQ_CONTROL_2_REG_FACENESSREGID__SHIFT		2
-static inline uint32_t A3XX_HLSQ_CONTROL_2_REG_FACENESSREGID(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_2_REG_FACENESSREGID__SHIFT) & A3XX_HLSQ_CONTROL_2_REG_FACENESSREGID__MASK;
-}
-#define A3XX_HLSQ_CONTROL_2_REG_COVVALUEREGID__MASK		0x03fc0000
-#define A3XX_HLSQ_CONTROL_2_REG_COVVALUEREGID__SHIFT		18
-static inline uint32_t A3XX_HLSQ_CONTROL_2_REG_COVVALUEREGID(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_2_REG_COVVALUEREGID__SHIFT) & A3XX_HLSQ_CONTROL_2_REG_COVVALUEREGID__MASK;
-}
-#define A3XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD__MASK	0xfc000000
-#define A3XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD__SHIFT	26
-static inline uint32_t A3XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD__SHIFT) & A3XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD__MASK;
-}
-
-#define REG_A3XX_HLSQ_CONTROL_3_REG				0x00002203
-#define A3XX_HLSQ_CONTROL_3_REG_IJPERSPCENTERREGID__MASK	0x000000ff
-#define A3XX_HLSQ_CONTROL_3_REG_IJPERSPCENTERREGID__SHIFT	0
-static inline uint32_t A3XX_HLSQ_CONTROL_3_REG_IJPERSPCENTERREGID(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_3_REG_IJPERSPCENTERREGID__SHIFT) & A3XX_HLSQ_CONTROL_3_REG_IJPERSPCENTERREGID__MASK;
-}
-#define A3XX_HLSQ_CONTROL_3_REG_IJNONPERSPCENTERREGID__MASK	0x0000ff00
-#define A3XX_HLSQ_CONTROL_3_REG_IJNONPERSPCENTERREGID__SHIFT	8
-static inline uint32_t A3XX_HLSQ_CONTROL_3_REG_IJNONPERSPCENTERREGID(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_3_REG_IJNONPERSPCENTERREGID__SHIFT) & A3XX_HLSQ_CONTROL_3_REG_IJNONPERSPCENTERREGID__MASK;
-}
-#define A3XX_HLSQ_CONTROL_3_REG_IJPERSPCENTROIDREGID__MASK	0x00ff0000
-#define A3XX_HLSQ_CONTROL_3_REG_IJPERSPCENTROIDREGID__SHIFT	16
-static inline uint32_t A3XX_HLSQ_CONTROL_3_REG_IJPERSPCENTROIDREGID(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_3_REG_IJPERSPCENTROIDREGID__SHIFT) & A3XX_HLSQ_CONTROL_3_REG_IJPERSPCENTROIDREGID__MASK;
-}
-#define A3XX_HLSQ_CONTROL_3_REG_IJNONPERSPCENTROIDREGID__MASK	0xff000000
-#define A3XX_HLSQ_CONTROL_3_REG_IJNONPERSPCENTROIDREGID__SHIFT	24
-static inline uint32_t A3XX_HLSQ_CONTROL_3_REG_IJNONPERSPCENTROIDREGID(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONTROL_3_REG_IJNONPERSPCENTROIDREGID__SHIFT) & A3XX_HLSQ_CONTROL_3_REG_IJNONPERSPCENTROIDREGID__MASK;
-}
-
-#define REG_A3XX_HLSQ_VS_CONTROL_REG				0x00002204
-#define A3XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__MASK		0x000003ff
-#define A3XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__SHIFT		0
-static inline uint32_t A3XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__SHIFT) & A3XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__MASK;
-}
-#define A3XX_HLSQ_VS_CONTROL_REG_CONSTSTARTOFFSET__MASK		0x001ff000
-#define A3XX_HLSQ_VS_CONTROL_REG_CONSTSTARTOFFSET__SHIFT	12
-static inline uint32_t A3XX_HLSQ_VS_CONTROL_REG_CONSTSTARTOFFSET(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_VS_CONTROL_REG_CONSTSTARTOFFSET__SHIFT) & A3XX_HLSQ_VS_CONTROL_REG_CONSTSTARTOFFSET__MASK;
-}
-#define A3XX_HLSQ_VS_CONTROL_REG_INSTRLENGTH__MASK		0xff000000
-#define A3XX_HLSQ_VS_CONTROL_REG_INSTRLENGTH__SHIFT		24
-static inline uint32_t A3XX_HLSQ_VS_CONTROL_REG_INSTRLENGTH(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_VS_CONTROL_REG_INSTRLENGTH__SHIFT) & A3XX_HLSQ_VS_CONTROL_REG_INSTRLENGTH__MASK;
-}
-
-#define REG_A3XX_HLSQ_FS_CONTROL_REG				0x00002205
-#define A3XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__MASK		0x000003ff
-#define A3XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__SHIFT		0
-static inline uint32_t A3XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__SHIFT) & A3XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__MASK;
-}
-#define A3XX_HLSQ_FS_CONTROL_REG_CONSTSTARTOFFSET__MASK		0x001ff000
-#define A3XX_HLSQ_FS_CONTROL_REG_CONSTSTARTOFFSET__SHIFT	12
-static inline uint32_t A3XX_HLSQ_FS_CONTROL_REG_CONSTSTARTOFFSET(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_FS_CONTROL_REG_CONSTSTARTOFFSET__SHIFT) & A3XX_HLSQ_FS_CONTROL_REG_CONSTSTARTOFFSET__MASK;
-}
-#define A3XX_HLSQ_FS_CONTROL_REG_INSTRLENGTH__MASK		0xff000000
-#define A3XX_HLSQ_FS_CONTROL_REG_INSTRLENGTH__SHIFT		24
-static inline uint32_t A3XX_HLSQ_FS_CONTROL_REG_INSTRLENGTH(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_FS_CONTROL_REG_INSTRLENGTH__SHIFT) & A3XX_HLSQ_FS_CONTROL_REG_INSTRLENGTH__MASK;
-}
-
-#define REG_A3XX_HLSQ_CONST_VSPRESV_RANGE_REG			0x00002206
-#define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_STARTENTRY__MASK	0x000001ff
-#define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_STARTENTRY__SHIFT	0
-static inline uint32_t A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_STARTENTRY(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_STARTENTRY__SHIFT) & A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_STARTENTRY__MASK;
-}
-#define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_ENDENTRY__MASK	0x01ff0000
-#define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_ENDENTRY__SHIFT	16
-static inline uint32_t A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_ENDENTRY(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_ENDENTRY__SHIFT) & A3XX_HLSQ_CONST_VSPRESV_RANGE_REG_ENDENTRY__MASK;
-}
-
-#define REG_A3XX_HLSQ_CONST_FSPRESV_RANGE_REG			0x00002207
-#define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_STARTENTRY__MASK	0x000001ff
-#define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_STARTENTRY__SHIFT	0
-static inline uint32_t A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_STARTENTRY(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_STARTENTRY__SHIFT) & A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_STARTENTRY__MASK;
-}
-#define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_ENDENTRY__MASK	0x01ff0000
-#define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_ENDENTRY__SHIFT	16
-static inline uint32_t A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_ENDENTRY(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_ENDENTRY__SHIFT) & A3XX_HLSQ_CONST_FSPRESV_RANGE_REG_ENDENTRY__MASK;
-}
-
-#define REG_A3XX_HLSQ_CL_NDRANGE_0_REG				0x0000220a
-#define A3XX_HLSQ_CL_NDRANGE_0_REG_WORKDIM__MASK		0x00000003
-#define A3XX_HLSQ_CL_NDRANGE_0_REG_WORKDIM__SHIFT		0
-static inline uint32_t A3XX_HLSQ_CL_NDRANGE_0_REG_WORKDIM(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CL_NDRANGE_0_REG_WORKDIM__SHIFT) & A3XX_HLSQ_CL_NDRANGE_0_REG_WORKDIM__MASK;
-}
-#define A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE0__MASK		0x00000ffc
-#define A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE0__SHIFT		2
-static inline uint32_t A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE0(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE0__SHIFT) & A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE0__MASK;
-}
-#define A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE1__MASK		0x003ff000
-#define A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE1__SHIFT		12
-static inline uint32_t A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE1(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE1__SHIFT) & A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE1__MASK;
-}
-#define A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE2__MASK		0xffc00000
-#define A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE2__SHIFT		22
-static inline uint32_t A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE2(uint32_t val)
-{
-	return ((val) << A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE2__SHIFT) & A3XX_HLSQ_CL_NDRANGE_0_REG_LOCALSIZE2__MASK;
-}
-
-#define REG_A3XX_HLSQ_CL_GLOBAL_WORK(i0) (0x0000220b + 0x2*(i0))
-
-static inline uint32_t REG_A3XX_HLSQ_CL_GLOBAL_WORK_SIZE(uint32_t i0) { return 0x0000220b + 0x2*i0; }
-
-static inline uint32_t REG_A3XX_HLSQ_CL_GLOBAL_WORK_OFFSET(uint32_t i0) { return 0x0000220c + 0x2*i0; }
-
-#define REG_A3XX_HLSQ_CL_CONTROL_0_REG				0x00002211
-
-#define REG_A3XX_HLSQ_CL_CONTROL_1_REG				0x00002212
-
-#define REG_A3XX_HLSQ_CL_KERNEL_CONST_REG			0x00002214
-
-#define REG_A3XX_HLSQ_CL_KERNEL_GROUP(i0) (0x00002215 + 0x1*(i0))
-
-static inline uint32_t REG_A3XX_HLSQ_CL_KERNEL_GROUP_RATIO(uint32_t i0) { return 0x00002215 + 0x1*i0; }
-
-#define REG_A3XX_HLSQ_CL_KERNEL_GROUP_Y_REG			0x00002216
-
-#define REG_A3XX_HLSQ_CL_KERNEL_GROUP_Z_REG			0x00002217
-
-#define REG_A3XX_HLSQ_CL_WG_OFFSET_REG				0x0000221a
-
-#define REG_A3XX_VFD_CONTROL_0					0x00002240
-#define A3XX_VFD_CONTROL_0_TOTALATTRTOVS__MASK			0x0003ffff
-#define A3XX_VFD_CONTROL_0_TOTALATTRTOVS__SHIFT			0
-static inline uint32_t A3XX_VFD_CONTROL_0_TOTALATTRTOVS(uint32_t val)
-{
-	return ((val) << A3XX_VFD_CONTROL_0_TOTALATTRTOVS__SHIFT) & A3XX_VFD_CONTROL_0_TOTALATTRTOVS__MASK;
-}
-#define A3XX_VFD_CONTROL_0_PACKETSIZE__MASK			0x003c0000
-#define A3XX_VFD_CONTROL_0_PACKETSIZE__SHIFT			18
-static inline uint32_t A3XX_VFD_CONTROL_0_PACKETSIZE(uint32_t val)
-{
-	return ((val) << A3XX_VFD_CONTROL_0_PACKETSIZE__SHIFT) & A3XX_VFD_CONTROL_0_PACKETSIZE__MASK;
-}
-#define A3XX_VFD_CONTROL_0_STRMDECINSTRCNT__MASK		0x07c00000
-#define A3XX_VFD_CONTROL_0_STRMDECINSTRCNT__SHIFT		22
-static inline uint32_t A3XX_VFD_CONTROL_0_STRMDECINSTRCNT(uint32_t val)
-{
-	return ((val) << A3XX_VFD_CONTROL_0_STRMDECINSTRCNT__SHIFT) & A3XX_VFD_CONTROL_0_STRMDECINSTRCNT__MASK;
-}
-#define A3XX_VFD_CONTROL_0_STRMFETCHINSTRCNT__MASK		0xf8000000
-#define A3XX_VFD_CONTROL_0_STRMFETCHINSTRCNT__SHIFT		27
-static inline uint32_t A3XX_VFD_CONTROL_0_STRMFETCHINSTRCNT(uint32_t val)
-{
-	return ((val) << A3XX_VFD_CONTROL_0_STRMFETCHINSTRCNT__SHIFT) & A3XX_VFD_CONTROL_0_STRMFETCHINSTRCNT__MASK;
-}
-
-#define REG_A3XX_VFD_CONTROL_1					0x00002241
-#define A3XX_VFD_CONTROL_1_MAXSTORAGE__MASK			0x0000000f
-#define A3XX_VFD_CONTROL_1_MAXSTORAGE__SHIFT			0
-static inline uint32_t A3XX_VFD_CONTROL_1_MAXSTORAGE(uint32_t val)
-{
-	return ((val) << A3XX_VFD_CONTROL_1_MAXSTORAGE__SHIFT) & A3XX_VFD_CONTROL_1_MAXSTORAGE__MASK;
-}
-#define A3XX_VFD_CONTROL_1_MAXTHRESHOLD__MASK			0x000000f0
-#define A3XX_VFD_CONTROL_1_MAXTHRESHOLD__SHIFT			4
-static inline uint32_t A3XX_VFD_CONTROL_1_MAXTHRESHOLD(uint32_t val)
-{
-	return ((val) << A3XX_VFD_CONTROL_1_MAXTHRESHOLD__SHIFT) & A3XX_VFD_CONTROL_1_MAXTHRESHOLD__MASK;
-}
-#define A3XX_VFD_CONTROL_1_MINTHRESHOLD__MASK			0x00000f00
-#define A3XX_VFD_CONTROL_1_MINTHRESHOLD__SHIFT			8
-static inline uint32_t A3XX_VFD_CONTROL_1_MINTHRESHOLD(uint32_t val)
-{
-	return ((val) << A3XX_VFD_CONTROL_1_MINTHRESHOLD__SHIFT) & A3XX_VFD_CONTROL_1_MINTHRESHOLD__MASK;
-}
-#define A3XX_VFD_CONTROL_1_REGID4VTX__MASK			0x00ff0000
-#define A3XX_VFD_CONTROL_1_REGID4VTX__SHIFT			16
-static inline uint32_t A3XX_VFD_CONTROL_1_REGID4VTX(uint32_t val)
-{
-	return ((val) << A3XX_VFD_CONTROL_1_REGID4VTX__SHIFT) & A3XX_VFD_CONTROL_1_REGID4VTX__MASK;
-}
-#define A3XX_VFD_CONTROL_1_REGID4INST__MASK			0xff000000
-#define A3XX_VFD_CONTROL_1_REGID4INST__SHIFT			24
-static inline uint32_t A3XX_VFD_CONTROL_1_REGID4INST(uint32_t val)
-{
-	return ((val) << A3XX_VFD_CONTROL_1_REGID4INST__SHIFT) & A3XX_VFD_CONTROL_1_REGID4INST__MASK;
-}
-
-#define REG_A3XX_VFD_INDEX_MIN					0x00002242
-
-#define REG_A3XX_VFD_INDEX_MAX					0x00002243
-
-#define REG_A3XX_VFD_INSTANCEID_OFFSET				0x00002244
-
-#define REG_A3XX_VFD_INDEX_OFFSET				0x00002245
-
-#define REG_A3XX_VFD_FETCH(i0) (0x00002246 + 0x2*(i0))
-
-static inline uint32_t REG_A3XX_VFD_FETCH_INSTR_0(uint32_t i0) { return 0x00002246 + 0x2*i0; }
-#define A3XX_VFD_FETCH_INSTR_0_FETCHSIZE__MASK			0x0000007f
-#define A3XX_VFD_FETCH_INSTR_0_FETCHSIZE__SHIFT			0
-static inline uint32_t A3XX_VFD_FETCH_INSTR_0_FETCHSIZE(uint32_t val)
-{
-	return ((val) << A3XX_VFD_FETCH_INSTR_0_FETCHSIZE__SHIFT) & A3XX_VFD_FETCH_INSTR_0_FETCHSIZE__MASK;
-}
-#define A3XX_VFD_FETCH_INSTR_0_BUFSTRIDE__MASK			0x0000ff80
-#define A3XX_VFD_FETCH_INSTR_0_BUFSTRIDE__SHIFT			7
-static inline uint32_t A3XX_VFD_FETCH_INSTR_0_BUFSTRIDE(uint32_t val)
-{
-	return ((val) << A3XX_VFD_FETCH_INSTR_0_BUFSTRIDE__SHIFT) & A3XX_VFD_FETCH_INSTR_0_BUFSTRIDE__MASK;
-}
-#define A3XX_VFD_FETCH_INSTR_0_INSTANCED			0x00010000
-#define A3XX_VFD_FETCH_INSTR_0_SWITCHNEXT			0x00020000
-#define A3XX_VFD_FETCH_INSTR_0_INDEXCODE__MASK			0x00fc0000
-#define A3XX_VFD_FETCH_INSTR_0_INDEXCODE__SHIFT			18
-static inline uint32_t A3XX_VFD_FETCH_INSTR_0_INDEXCODE(uint32_t val)
-{
-	return ((val) << A3XX_VFD_FETCH_INSTR_0_INDEXCODE__SHIFT) & A3XX_VFD_FETCH_INSTR_0_INDEXCODE__MASK;
-}
-#define A3XX_VFD_FETCH_INSTR_0_STEPRATE__MASK			0xff000000
-#define A3XX_VFD_FETCH_INSTR_0_STEPRATE__SHIFT			24
-static inline uint32_t A3XX_VFD_FETCH_INSTR_0_STEPRATE(uint32_t val)
-{
-	return ((val) << A3XX_VFD_FETCH_INSTR_0_STEPRATE__SHIFT) & A3XX_VFD_FETCH_INSTR_0_STEPRATE__MASK;
-}
-
-static inline uint32_t REG_A3XX_VFD_FETCH_INSTR_1(uint32_t i0) { return 0x00002247 + 0x2*i0; }
-
-#define REG_A3XX_VFD_DECODE(i0) (0x00002266 + 0x1*(i0))
-
-static inline uint32_t REG_A3XX_VFD_DECODE_INSTR(uint32_t i0) { return 0x00002266 + 0x1*i0; }
-#define A3XX_VFD_DECODE_INSTR_WRITEMASK__MASK			0x0000000f
-#define A3XX_VFD_DECODE_INSTR_WRITEMASK__SHIFT			0
-static inline uint32_t A3XX_VFD_DECODE_INSTR_WRITEMASK(uint32_t val)
-{
-	return ((val) << A3XX_VFD_DECODE_INSTR_WRITEMASK__SHIFT) & A3XX_VFD_DECODE_INSTR_WRITEMASK__MASK;
-}
-#define A3XX_VFD_DECODE_INSTR_CONSTFILL				0x00000010
-#define A3XX_VFD_DECODE_INSTR_FORMAT__MASK			0x00000fc0
-#define A3XX_VFD_DECODE_INSTR_FORMAT__SHIFT			6
-static inline uint32_t A3XX_VFD_DECODE_INSTR_FORMAT(enum a3xx_vtx_fmt val)
-{
-	return ((val) << A3XX_VFD_DECODE_INSTR_FORMAT__SHIFT) & A3XX_VFD_DECODE_INSTR_FORMAT__MASK;
-}
-#define A3XX_VFD_DECODE_INSTR_REGID__MASK			0x000ff000
-#define A3XX_VFD_DECODE_INSTR_REGID__SHIFT			12
-static inline uint32_t A3XX_VFD_DECODE_INSTR_REGID(uint32_t val)
-{
-	return ((val) << A3XX_VFD_DECODE_INSTR_REGID__SHIFT) & A3XX_VFD_DECODE_INSTR_REGID__MASK;
-}
-#define A3XX_VFD_DECODE_INSTR_INT				0x00100000
-#define A3XX_VFD_DECODE_INSTR_SWAP__MASK			0x00c00000
-#define A3XX_VFD_DECODE_INSTR_SWAP__SHIFT			22
-static inline uint32_t A3XX_VFD_DECODE_INSTR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A3XX_VFD_DECODE_INSTR_SWAP__SHIFT) & A3XX_VFD_DECODE_INSTR_SWAP__MASK;
-}
-#define A3XX_VFD_DECODE_INSTR_SHIFTCNT__MASK			0x1f000000
-#define A3XX_VFD_DECODE_INSTR_SHIFTCNT__SHIFT			24
-static inline uint32_t A3XX_VFD_DECODE_INSTR_SHIFTCNT(uint32_t val)
-{
-	return ((val) << A3XX_VFD_DECODE_INSTR_SHIFTCNT__SHIFT) & A3XX_VFD_DECODE_INSTR_SHIFTCNT__MASK;
-}
-#define A3XX_VFD_DECODE_INSTR_LASTCOMPVALID			0x20000000
-#define A3XX_VFD_DECODE_INSTR_SWITCHNEXT			0x40000000
-
-#define REG_A3XX_VFD_VS_THREADING_THRESHOLD			0x0000227e
-#define A3XX_VFD_VS_THREADING_THRESHOLD_REGID_THRESHOLD__MASK	0x0000000f
-#define A3XX_VFD_VS_THREADING_THRESHOLD_REGID_THRESHOLD__SHIFT	0
-static inline uint32_t A3XX_VFD_VS_THREADING_THRESHOLD_REGID_THRESHOLD(uint32_t val)
-{
-	return ((val) << A3XX_VFD_VS_THREADING_THRESHOLD_REGID_THRESHOLD__SHIFT) & A3XX_VFD_VS_THREADING_THRESHOLD_REGID_THRESHOLD__MASK;
-}
-#define A3XX_VFD_VS_THREADING_THRESHOLD_REGID_VTXCNT__MASK	0x0000ff00
-#define A3XX_VFD_VS_THREADING_THRESHOLD_REGID_VTXCNT__SHIFT	8
-static inline uint32_t A3XX_VFD_VS_THREADING_THRESHOLD_REGID_VTXCNT(uint32_t val)
-{
-	return ((val) << A3XX_VFD_VS_THREADING_THRESHOLD_REGID_VTXCNT__SHIFT) & A3XX_VFD_VS_THREADING_THRESHOLD_REGID_VTXCNT__MASK;
-}
-
-#define REG_A3XX_VPC_ATTR					0x00002280
-#define A3XX_VPC_ATTR_TOTALATTR__MASK				0x000001ff
-#define A3XX_VPC_ATTR_TOTALATTR__SHIFT				0
-static inline uint32_t A3XX_VPC_ATTR_TOTALATTR(uint32_t val)
-{
-	return ((val) << A3XX_VPC_ATTR_TOTALATTR__SHIFT) & A3XX_VPC_ATTR_TOTALATTR__MASK;
-}
-#define A3XX_VPC_ATTR_PSIZE					0x00000200
-#define A3XX_VPC_ATTR_THRDASSIGN__MASK				0x0ffff000
-#define A3XX_VPC_ATTR_THRDASSIGN__SHIFT				12
-static inline uint32_t A3XX_VPC_ATTR_THRDASSIGN(uint32_t val)
-{
-	return ((val) << A3XX_VPC_ATTR_THRDASSIGN__SHIFT) & A3XX_VPC_ATTR_THRDASSIGN__MASK;
-}
-#define A3XX_VPC_ATTR_LMSIZE__MASK				0xf0000000
-#define A3XX_VPC_ATTR_LMSIZE__SHIFT				28
-static inline uint32_t A3XX_VPC_ATTR_LMSIZE(uint32_t val)
-{
-	return ((val) << A3XX_VPC_ATTR_LMSIZE__SHIFT) & A3XX_VPC_ATTR_LMSIZE__MASK;
-}
-
-#define REG_A3XX_VPC_PACK					0x00002281
-#define A3XX_VPC_PACK_NUMFPNONPOSVAR__MASK			0x0000ff00
-#define A3XX_VPC_PACK_NUMFPNONPOSVAR__SHIFT			8
-static inline uint32_t A3XX_VPC_PACK_NUMFPNONPOSVAR(uint32_t val)
-{
-	return ((val) << A3XX_VPC_PACK_NUMFPNONPOSVAR__SHIFT) & A3XX_VPC_PACK_NUMFPNONPOSVAR__MASK;
-}
-#define A3XX_VPC_PACK_NUMNONPOSVSVAR__MASK			0x00ff0000
-#define A3XX_VPC_PACK_NUMNONPOSVSVAR__SHIFT			16
-static inline uint32_t A3XX_VPC_PACK_NUMNONPOSVSVAR(uint32_t val)
-{
-	return ((val) << A3XX_VPC_PACK_NUMNONPOSVSVAR__SHIFT) & A3XX_VPC_PACK_NUMNONPOSVSVAR__MASK;
-}
-
-#define REG_A3XX_VPC_VARYING_INTERP(i0) (0x00002282 + 0x1*(i0))
-
-static inline uint32_t REG_A3XX_VPC_VARYING_INTERP_MODE(uint32_t i0) { return 0x00002282 + 0x1*i0; }
-#define A3XX_VPC_VARYING_INTERP_MODE_C0__MASK			0x00000003
-#define A3XX_VPC_VARYING_INTERP_MODE_C0__SHIFT			0
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_C0(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_C0__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_C0__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_C1__MASK			0x0000000c
-#define A3XX_VPC_VARYING_INTERP_MODE_C1__SHIFT			2
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_C1(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_C1__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_C1__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_C2__MASK			0x00000030
-#define A3XX_VPC_VARYING_INTERP_MODE_C2__SHIFT			4
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_C2(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_C2__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_C2__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_C3__MASK			0x000000c0
-#define A3XX_VPC_VARYING_INTERP_MODE_C3__SHIFT			6
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_C3(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_C3__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_C3__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_C4__MASK			0x00000300
-#define A3XX_VPC_VARYING_INTERP_MODE_C4__SHIFT			8
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_C4(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_C4__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_C4__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_C5__MASK			0x00000c00
-#define A3XX_VPC_VARYING_INTERP_MODE_C5__SHIFT			10
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_C5(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_C5__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_C5__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_C6__MASK			0x00003000
-#define A3XX_VPC_VARYING_INTERP_MODE_C6__SHIFT			12
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_C6(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_C6__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_C6__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_C7__MASK			0x0000c000
-#define A3XX_VPC_VARYING_INTERP_MODE_C7__SHIFT			14
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_C7(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_C7__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_C7__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_C8__MASK			0x00030000
-#define A3XX_VPC_VARYING_INTERP_MODE_C8__SHIFT			16
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_C8(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_C8__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_C8__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_C9__MASK			0x000c0000
-#define A3XX_VPC_VARYING_INTERP_MODE_C9__SHIFT			18
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_C9(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_C9__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_C9__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_CA__MASK			0x00300000
-#define A3XX_VPC_VARYING_INTERP_MODE_CA__SHIFT			20
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_CA(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_CA__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_CA__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_CB__MASK			0x00c00000
-#define A3XX_VPC_VARYING_INTERP_MODE_CB__SHIFT			22
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_CB(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_CB__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_CB__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_CC__MASK			0x03000000
-#define A3XX_VPC_VARYING_INTERP_MODE_CC__SHIFT			24
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_CC(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_CC__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_CC__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_CD__MASK			0x0c000000
-#define A3XX_VPC_VARYING_INTERP_MODE_CD__SHIFT			26
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_CD(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_CD__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_CD__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_CE__MASK			0x30000000
-#define A3XX_VPC_VARYING_INTERP_MODE_CE__SHIFT			28
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_CE(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_CE__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_CE__MASK;
-}
-#define A3XX_VPC_VARYING_INTERP_MODE_CF__MASK			0xc0000000
-#define A3XX_VPC_VARYING_INTERP_MODE_CF__SHIFT			30
-static inline uint32_t A3XX_VPC_VARYING_INTERP_MODE_CF(enum a3xx_intp_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_INTERP_MODE_CF__SHIFT) & A3XX_VPC_VARYING_INTERP_MODE_CF__MASK;
-}
-
-#define REG_A3XX_VPC_VARYING_PS_REPL(i0) (0x00002286 + 0x1*(i0))
-
-static inline uint32_t REG_A3XX_VPC_VARYING_PS_REPL_MODE(uint32_t i0) { return 0x00002286 + 0x1*i0; }
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C0__MASK			0x00000003
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C0__SHIFT			0
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_C0(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_C0__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_C0__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C1__MASK			0x0000000c
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C1__SHIFT			2
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_C1(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_C1__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_C1__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C2__MASK			0x00000030
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C2__SHIFT			4
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_C2(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_C2__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_C2__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C3__MASK			0x000000c0
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C3__SHIFT			6
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_C3(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_C3__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_C3__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C4__MASK			0x00000300
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C4__SHIFT			8
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_C4(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_C4__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_C4__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C5__MASK			0x00000c00
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C5__SHIFT			10
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_C5(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_C5__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_C5__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C6__MASK			0x00003000
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C6__SHIFT			12
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_C6(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_C6__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_C6__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C7__MASK			0x0000c000
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C7__SHIFT			14
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_C7(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_C7__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_C7__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C8__MASK			0x00030000
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C8__SHIFT			16
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_C8(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_C8__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_C8__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C9__MASK			0x000c0000
-#define A3XX_VPC_VARYING_PS_REPL_MODE_C9__SHIFT			18
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_C9(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_C9__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_C9__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_CA__MASK			0x00300000
-#define A3XX_VPC_VARYING_PS_REPL_MODE_CA__SHIFT			20
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_CA(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_CA__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_CA__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_CB__MASK			0x00c00000
-#define A3XX_VPC_VARYING_PS_REPL_MODE_CB__SHIFT			22
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_CB(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_CB__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_CB__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_CC__MASK			0x03000000
-#define A3XX_VPC_VARYING_PS_REPL_MODE_CC__SHIFT			24
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_CC(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_CC__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_CC__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_CD__MASK			0x0c000000
-#define A3XX_VPC_VARYING_PS_REPL_MODE_CD__SHIFT			26
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_CD(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_CD__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_CD__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_CE__MASK			0x30000000
-#define A3XX_VPC_VARYING_PS_REPL_MODE_CE__SHIFT			28
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_CE(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_CE__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_CE__MASK;
-}
-#define A3XX_VPC_VARYING_PS_REPL_MODE_CF__MASK			0xc0000000
-#define A3XX_VPC_VARYING_PS_REPL_MODE_CF__SHIFT			30
-static inline uint32_t A3XX_VPC_VARYING_PS_REPL_MODE_CF(enum a3xx_repl_mode val)
-{
-	return ((val) << A3XX_VPC_VARYING_PS_REPL_MODE_CF__SHIFT) & A3XX_VPC_VARYING_PS_REPL_MODE_CF__MASK;
-}
-
-#define REG_A3XX_VPC_VARY_CYLWRAP_ENABLE_0			0x0000228a
-
-#define REG_A3XX_VPC_VARY_CYLWRAP_ENABLE_1			0x0000228b
-
-#define REG_A3XX_SP_SP_CTRL_REG					0x000022c0
-#define A3XX_SP_SP_CTRL_REG_RESOLVE				0x00010000
-#define A3XX_SP_SP_CTRL_REG_CONSTMODE__MASK			0x00040000
-#define A3XX_SP_SP_CTRL_REG_CONSTMODE__SHIFT			18
-static inline uint32_t A3XX_SP_SP_CTRL_REG_CONSTMODE(uint32_t val)
-{
-	return ((val) << A3XX_SP_SP_CTRL_REG_CONSTMODE__SHIFT) & A3XX_SP_SP_CTRL_REG_CONSTMODE__MASK;
-}
-#define A3XX_SP_SP_CTRL_REG_BINNING				0x00080000
-#define A3XX_SP_SP_CTRL_REG_SLEEPMODE__MASK			0x00300000
-#define A3XX_SP_SP_CTRL_REG_SLEEPMODE__SHIFT			20
-static inline uint32_t A3XX_SP_SP_CTRL_REG_SLEEPMODE(uint32_t val)
-{
-	return ((val) << A3XX_SP_SP_CTRL_REG_SLEEPMODE__SHIFT) & A3XX_SP_SP_CTRL_REG_SLEEPMODE__MASK;
-}
-#define A3XX_SP_SP_CTRL_REG_L0MODE__MASK			0x00c00000
-#define A3XX_SP_SP_CTRL_REG_L0MODE__SHIFT			22
-static inline uint32_t A3XX_SP_SP_CTRL_REG_L0MODE(uint32_t val)
-{
-	return ((val) << A3XX_SP_SP_CTRL_REG_L0MODE__SHIFT) & A3XX_SP_SP_CTRL_REG_L0MODE__MASK;
-}
-
-#define REG_A3XX_SP_VS_CTRL_REG0				0x000022c4
-#define A3XX_SP_VS_CTRL_REG0_THREADMODE__MASK			0x00000001
-#define A3XX_SP_VS_CTRL_REG0_THREADMODE__SHIFT			0
-static inline uint32_t A3XX_SP_VS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
-{
-	return ((val) << A3XX_SP_VS_CTRL_REG0_THREADMODE__SHIFT) & A3XX_SP_VS_CTRL_REG0_THREADMODE__MASK;
-}
-#define A3XX_SP_VS_CTRL_REG0_INSTRBUFFERMODE__MASK		0x00000002
-#define A3XX_SP_VS_CTRL_REG0_INSTRBUFFERMODE__SHIFT		1
-static inline uint32_t A3XX_SP_VS_CTRL_REG0_INSTRBUFFERMODE(enum a3xx_instrbuffermode val)
-{
-	return ((val) << A3XX_SP_VS_CTRL_REG0_INSTRBUFFERMODE__SHIFT) & A3XX_SP_VS_CTRL_REG0_INSTRBUFFERMODE__MASK;
-}
-#define A3XX_SP_VS_CTRL_REG0_CACHEINVALID			0x00000004
-#define A3XX_SP_VS_CTRL_REG0_ALUSCHMODE				0x00000008
-#define A3XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
-#define A3XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
-static inline uint32_t A3XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A3XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A3XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
-#define A3XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
-static inline uint32_t A3XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A3XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A3XX_SP_VS_CTRL_REG0_THREADSIZE__MASK			0x00100000
-#define A3XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT			20
-static inline uint32_t A3XX_SP_VS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A3XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT) & A3XX_SP_VS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A3XX_SP_VS_CTRL_REG0_SUPERTHREADMODE			0x00200000
-#define A3XX_SP_VS_CTRL_REG0_LENGTH__MASK			0xff000000
-#define A3XX_SP_VS_CTRL_REG0_LENGTH__SHIFT			24
-static inline uint32_t A3XX_SP_VS_CTRL_REG0_LENGTH(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_CTRL_REG0_LENGTH__SHIFT) & A3XX_SP_VS_CTRL_REG0_LENGTH__MASK;
-}
-
-#define REG_A3XX_SP_VS_CTRL_REG1				0x000022c5
-#define A3XX_SP_VS_CTRL_REG1_CONSTLENGTH__MASK			0x000003ff
-#define A3XX_SP_VS_CTRL_REG1_CONSTLENGTH__SHIFT			0
-static inline uint32_t A3XX_SP_VS_CTRL_REG1_CONSTLENGTH(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_CTRL_REG1_CONSTLENGTH__SHIFT) & A3XX_SP_VS_CTRL_REG1_CONSTLENGTH__MASK;
-}
-#define A3XX_SP_VS_CTRL_REG1_CONSTFOOTPRINT__MASK		0x000ffc00
-#define A3XX_SP_VS_CTRL_REG1_CONSTFOOTPRINT__SHIFT		10
-static inline uint32_t A3XX_SP_VS_CTRL_REG1_CONSTFOOTPRINT(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_CTRL_REG1_CONSTFOOTPRINT__SHIFT) & A3XX_SP_VS_CTRL_REG1_CONSTFOOTPRINT__MASK;
-}
-#define A3XX_SP_VS_CTRL_REG1_INITIALOUTSTANDING__MASK		0x7f000000
-#define A3XX_SP_VS_CTRL_REG1_INITIALOUTSTANDING__SHIFT		24
-static inline uint32_t A3XX_SP_VS_CTRL_REG1_INITIALOUTSTANDING(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_CTRL_REG1_INITIALOUTSTANDING__SHIFT) & A3XX_SP_VS_CTRL_REG1_INITIALOUTSTANDING__MASK;
-}
-
-#define REG_A3XX_SP_VS_PARAM_REG				0x000022c6
-#define A3XX_SP_VS_PARAM_REG_POSREGID__MASK			0x000000ff
-#define A3XX_SP_VS_PARAM_REG_POSREGID__SHIFT			0
-static inline uint32_t A3XX_SP_VS_PARAM_REG_POSREGID(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_PARAM_REG_POSREGID__SHIFT) & A3XX_SP_VS_PARAM_REG_POSREGID__MASK;
-}
-#define A3XX_SP_VS_PARAM_REG_PSIZEREGID__MASK			0x0000ff00
-#define A3XX_SP_VS_PARAM_REG_PSIZEREGID__SHIFT			8
-static inline uint32_t A3XX_SP_VS_PARAM_REG_PSIZEREGID(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_PARAM_REG_PSIZEREGID__SHIFT) & A3XX_SP_VS_PARAM_REG_PSIZEREGID__MASK;
-}
-#define A3XX_SP_VS_PARAM_REG_POS2DMODE				0x00010000
-#define A3XX_SP_VS_PARAM_REG_TOTALVSOUTVAR__MASK		0x01f00000
-#define A3XX_SP_VS_PARAM_REG_TOTALVSOUTVAR__SHIFT		20
-static inline uint32_t A3XX_SP_VS_PARAM_REG_TOTALVSOUTVAR(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_PARAM_REG_TOTALVSOUTVAR__SHIFT) & A3XX_SP_VS_PARAM_REG_TOTALVSOUTVAR__MASK;
-}
-
-#define REG_A3XX_SP_VS_OUT(i0) (0x000022c7 + 0x1*(i0))
-
-static inline uint32_t REG_A3XX_SP_VS_OUT_REG(uint32_t i0) { return 0x000022c7 + 0x1*i0; }
-#define A3XX_SP_VS_OUT_REG_A_REGID__MASK			0x000000ff
-#define A3XX_SP_VS_OUT_REG_A_REGID__SHIFT			0
-static inline uint32_t A3XX_SP_VS_OUT_REG_A_REGID(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_OUT_REG_A_REGID__SHIFT) & A3XX_SP_VS_OUT_REG_A_REGID__MASK;
-}
-#define A3XX_SP_VS_OUT_REG_A_HALF				0x00000100
-#define A3XX_SP_VS_OUT_REG_A_COMPMASK__MASK			0x00001e00
-#define A3XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT			9
-static inline uint32_t A3XX_SP_VS_OUT_REG_A_COMPMASK(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT) & A3XX_SP_VS_OUT_REG_A_COMPMASK__MASK;
-}
-#define A3XX_SP_VS_OUT_REG_B_REGID__MASK			0x00ff0000
-#define A3XX_SP_VS_OUT_REG_B_REGID__SHIFT			16
-static inline uint32_t A3XX_SP_VS_OUT_REG_B_REGID(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_OUT_REG_B_REGID__SHIFT) & A3XX_SP_VS_OUT_REG_B_REGID__MASK;
-}
-#define A3XX_SP_VS_OUT_REG_B_HALF				0x01000000
-#define A3XX_SP_VS_OUT_REG_B_COMPMASK__MASK			0x1e000000
-#define A3XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT			25
-static inline uint32_t A3XX_SP_VS_OUT_REG_B_COMPMASK(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT) & A3XX_SP_VS_OUT_REG_B_COMPMASK__MASK;
-}
-
-#define REG_A3XX_SP_VS_VPC_DST(i0) (0x000022d0 + 0x1*(i0))
-
-static inline uint32_t REG_A3XX_SP_VS_VPC_DST_REG(uint32_t i0) { return 0x000022d0 + 0x1*i0; }
-#define A3XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK			0x0000007f
-#define A3XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT			0
-static inline uint32_t A3XX_SP_VS_VPC_DST_REG_OUTLOC0(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT) & A3XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK;
-}
-#define A3XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK			0x00007f00
-#define A3XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT			8
-static inline uint32_t A3XX_SP_VS_VPC_DST_REG_OUTLOC1(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT) & A3XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK;
-}
-#define A3XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK			0x007f0000
-#define A3XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT			16
-static inline uint32_t A3XX_SP_VS_VPC_DST_REG_OUTLOC2(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT) & A3XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK;
-}
-#define A3XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK			0x7f000000
-#define A3XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT			24
-static inline uint32_t A3XX_SP_VS_VPC_DST_REG_OUTLOC3(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT) & A3XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK;
-}
-
-#define REG_A3XX_SP_VS_OBJ_OFFSET_REG				0x000022d4
-#define A3XX_SP_VS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__MASK	0x0000ffff
-#define A3XX_SP_VS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__SHIFT	0
-static inline uint32_t A3XX_SP_VS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__SHIFT) & A3XX_SP_VS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__MASK;
-}
-#define A3XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK	0x01ff0000
-#define A3XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT	16
-static inline uint32_t A3XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT) & A3XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A3XX_SP_VS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK		0xfe000000
-#define A3XX_SP_VS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT	25
-static inline uint32_t A3XX_SP_VS_OBJ_OFFSET_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT) & A3XX_SP_VS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A3XX_SP_VS_OBJ_START_REG				0x000022d5
-
-#define REG_A3XX_SP_VS_PVT_MEM_PARAM_REG			0x000022d6
-#define A3XX_SP_VS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__MASK	0x000000ff
-#define A3XX_SP_VS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__SHIFT	0
-static inline uint32_t A3XX_SP_VS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x7f));
-	return (((val >> 7)) << A3XX_SP_VS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__SHIFT) & A3XX_SP_VS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__MASK;
-}
-#define A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__MASK	0x00ffff00
-#define A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__SHIFT	8
-static inline uint32_t A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKOFFSET(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__SHIFT) & A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__MASK;
-}
-#define A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__SHIFT) & A3XX_SP_VS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A3XX_SP_VS_PVT_MEM_ADDR_REG				0x000022d7
-#define A3XX_SP_VS_PVT_MEM_ADDR_REG_BURSTLEN__MASK		0x0000001f
-#define A3XX_SP_VS_PVT_MEM_ADDR_REG_BURSTLEN__SHIFT		0
-static inline uint32_t A3XX_SP_VS_PVT_MEM_ADDR_REG_BURSTLEN(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_PVT_MEM_ADDR_REG_BURSTLEN__SHIFT) & A3XX_SP_VS_PVT_MEM_ADDR_REG_BURSTLEN__MASK;
-}
-#define A3XX_SP_VS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__MASK	0xffffffe0
-#define A3XX_SP_VS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__SHIFT	5
-static inline uint32_t A3XX_SP_VS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A3XX_SP_VS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__SHIFT) & A3XX_SP_VS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__MASK;
-}
-
-#define REG_A3XX_SP_VS_PVT_MEM_SIZE_REG				0x000022d8
-
-#define REG_A3XX_SP_VS_LENGTH_REG				0x000022df
-#define A3XX_SP_VS_LENGTH_REG_SHADERLENGTH__MASK		0xffffffff
-#define A3XX_SP_VS_LENGTH_REG_SHADERLENGTH__SHIFT		0
-static inline uint32_t A3XX_SP_VS_LENGTH_REG_SHADERLENGTH(uint32_t val)
-{
-	return ((val) << A3XX_SP_VS_LENGTH_REG_SHADERLENGTH__SHIFT) & A3XX_SP_VS_LENGTH_REG_SHADERLENGTH__MASK;
-}
-
-#define REG_A3XX_SP_FS_CTRL_REG0				0x000022e0
-#define A3XX_SP_FS_CTRL_REG0_THREADMODE__MASK			0x00000001
-#define A3XX_SP_FS_CTRL_REG0_THREADMODE__SHIFT			0
-static inline uint32_t A3XX_SP_FS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
-{
-	return ((val) << A3XX_SP_FS_CTRL_REG0_THREADMODE__SHIFT) & A3XX_SP_FS_CTRL_REG0_THREADMODE__MASK;
-}
-#define A3XX_SP_FS_CTRL_REG0_INSTRBUFFERMODE__MASK		0x00000002
-#define A3XX_SP_FS_CTRL_REG0_INSTRBUFFERMODE__SHIFT		1
-static inline uint32_t A3XX_SP_FS_CTRL_REG0_INSTRBUFFERMODE(enum a3xx_instrbuffermode val)
-{
-	return ((val) << A3XX_SP_FS_CTRL_REG0_INSTRBUFFERMODE__SHIFT) & A3XX_SP_FS_CTRL_REG0_INSTRBUFFERMODE__MASK;
-}
-#define A3XX_SP_FS_CTRL_REG0_CACHEINVALID			0x00000004
-#define A3XX_SP_FS_CTRL_REG0_ALUSCHMODE				0x00000008
-#define A3XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
-#define A3XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
-static inline uint32_t A3XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A3XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A3XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
-#define A3XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
-static inline uint32_t A3XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A3XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A3XX_SP_FS_CTRL_REG0_FSBYPASSENABLE			0x00020000
-#define A3XX_SP_FS_CTRL_REG0_INOUTREGOVERLAP			0x00040000
-#define A3XX_SP_FS_CTRL_REG0_OUTORDERED				0x00080000
-#define A3XX_SP_FS_CTRL_REG0_THREADSIZE__MASK			0x00100000
-#define A3XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT			20
-static inline uint32_t A3XX_SP_FS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A3XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT) & A3XX_SP_FS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A3XX_SP_FS_CTRL_REG0_SUPERTHREADMODE			0x00200000
-#define A3XX_SP_FS_CTRL_REG0_PIXLODENABLE			0x00400000
-#define A3XX_SP_FS_CTRL_REG0_COMPUTEMODE			0x00800000
-#define A3XX_SP_FS_CTRL_REG0_LENGTH__MASK			0xff000000
-#define A3XX_SP_FS_CTRL_REG0_LENGTH__SHIFT			24
-static inline uint32_t A3XX_SP_FS_CTRL_REG0_LENGTH(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_CTRL_REG0_LENGTH__SHIFT) & A3XX_SP_FS_CTRL_REG0_LENGTH__MASK;
-}
-
-#define REG_A3XX_SP_FS_CTRL_REG1				0x000022e1
-#define A3XX_SP_FS_CTRL_REG1_CONSTLENGTH__MASK			0x000003ff
-#define A3XX_SP_FS_CTRL_REG1_CONSTLENGTH__SHIFT			0
-static inline uint32_t A3XX_SP_FS_CTRL_REG1_CONSTLENGTH(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_CTRL_REG1_CONSTLENGTH__SHIFT) & A3XX_SP_FS_CTRL_REG1_CONSTLENGTH__MASK;
-}
-#define A3XX_SP_FS_CTRL_REG1_CONSTFOOTPRINT__MASK		0x000ffc00
-#define A3XX_SP_FS_CTRL_REG1_CONSTFOOTPRINT__SHIFT		10
-static inline uint32_t A3XX_SP_FS_CTRL_REG1_CONSTFOOTPRINT(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_CTRL_REG1_CONSTFOOTPRINT__SHIFT) & A3XX_SP_FS_CTRL_REG1_CONSTFOOTPRINT__MASK;
-}
-#define A3XX_SP_FS_CTRL_REG1_INITIALOUTSTANDING__MASK		0x00f00000
-#define A3XX_SP_FS_CTRL_REG1_INITIALOUTSTANDING__SHIFT		20
-static inline uint32_t A3XX_SP_FS_CTRL_REG1_INITIALOUTSTANDING(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_CTRL_REG1_INITIALOUTSTANDING__SHIFT) & A3XX_SP_FS_CTRL_REG1_INITIALOUTSTANDING__MASK;
-}
-#define A3XX_SP_FS_CTRL_REG1_HALFPRECVAROFFSET__MASK		0x7f000000
-#define A3XX_SP_FS_CTRL_REG1_HALFPRECVAROFFSET__SHIFT		24
-static inline uint32_t A3XX_SP_FS_CTRL_REG1_HALFPRECVAROFFSET(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_CTRL_REG1_HALFPRECVAROFFSET__SHIFT) & A3XX_SP_FS_CTRL_REG1_HALFPRECVAROFFSET__MASK;
-}
-
-#define REG_A3XX_SP_FS_OBJ_OFFSET_REG				0x000022e2
-#define A3XX_SP_FS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__MASK	0x0000ffff
-#define A3XX_SP_FS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__SHIFT	0
-static inline uint32_t A3XX_SP_FS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__SHIFT) & A3XX_SP_FS_OBJ_OFFSET_REG_FIRSTEXECINSTROFFSET__MASK;
-}
-#define A3XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK	0x01ff0000
-#define A3XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT	16
-static inline uint32_t A3XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT) & A3XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A3XX_SP_FS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK		0xfe000000
-#define A3XX_SP_FS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT	25
-static inline uint32_t A3XX_SP_FS_OBJ_OFFSET_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT) & A3XX_SP_FS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A3XX_SP_FS_OBJ_START_REG				0x000022e3
-
-#define REG_A3XX_SP_FS_PVT_MEM_PARAM_REG			0x000022e4
-#define A3XX_SP_FS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__MASK	0x000000ff
-#define A3XX_SP_FS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__SHIFT	0
-static inline uint32_t A3XX_SP_FS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__SHIFT) & A3XX_SP_FS_PVT_MEM_PARAM_REG_MEMSIZEPERITEM__MASK;
-}
-#define A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__MASK	0x00ffff00
-#define A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__SHIFT	8
-static inline uint32_t A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKOFFSET(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__SHIFT) & A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKOFFSET__MASK;
-}
-#define A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__SHIFT) & A3XX_SP_FS_PVT_MEM_PARAM_REG_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A3XX_SP_FS_PVT_MEM_ADDR_REG				0x000022e5
-#define A3XX_SP_FS_PVT_MEM_ADDR_REG_BURSTLEN__MASK		0x0000001f
-#define A3XX_SP_FS_PVT_MEM_ADDR_REG_BURSTLEN__SHIFT		0
-static inline uint32_t A3XX_SP_FS_PVT_MEM_ADDR_REG_BURSTLEN(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_PVT_MEM_ADDR_REG_BURSTLEN__SHIFT) & A3XX_SP_FS_PVT_MEM_ADDR_REG_BURSTLEN__MASK;
-}
-#define A3XX_SP_FS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__MASK	0xffffffe0
-#define A3XX_SP_FS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__SHIFT	5
-static inline uint32_t A3XX_SP_FS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A3XX_SP_FS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__SHIFT) & A3XX_SP_FS_PVT_MEM_ADDR_REG_SHADERSTARTADDRESS__MASK;
-}
-
-#define REG_A3XX_SP_FS_PVT_MEM_SIZE_REG				0x000022e6
-
-#define REG_A3XX_SP_FS_FLAT_SHAD_MODE_REG_0			0x000022e8
-
-#define REG_A3XX_SP_FS_FLAT_SHAD_MODE_REG_1			0x000022e9
-
-#define REG_A3XX_SP_FS_OUTPUT_REG				0x000022ec
-#define A3XX_SP_FS_OUTPUT_REG_MRT__MASK				0x00000003
-#define A3XX_SP_FS_OUTPUT_REG_MRT__SHIFT			0
-static inline uint32_t A3XX_SP_FS_OUTPUT_REG_MRT(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_OUTPUT_REG_MRT__SHIFT) & A3XX_SP_FS_OUTPUT_REG_MRT__MASK;
-}
-#define A3XX_SP_FS_OUTPUT_REG_DEPTH_ENABLE			0x00000080
-#define A3XX_SP_FS_OUTPUT_REG_DEPTH_REGID__MASK			0x0000ff00
-#define A3XX_SP_FS_OUTPUT_REG_DEPTH_REGID__SHIFT		8
-static inline uint32_t A3XX_SP_FS_OUTPUT_REG_DEPTH_REGID(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_OUTPUT_REG_DEPTH_REGID__SHIFT) & A3XX_SP_FS_OUTPUT_REG_DEPTH_REGID__MASK;
-}
-
-#define REG_A3XX_SP_FS_MRT(i0) (0x000022f0 + 0x1*(i0))
-
-static inline uint32_t REG_A3XX_SP_FS_MRT_REG(uint32_t i0) { return 0x000022f0 + 0x1*i0; }
-#define A3XX_SP_FS_MRT_REG_REGID__MASK				0x000000ff
-#define A3XX_SP_FS_MRT_REG_REGID__SHIFT				0
-static inline uint32_t A3XX_SP_FS_MRT_REG_REGID(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_MRT_REG_REGID__SHIFT) & A3XX_SP_FS_MRT_REG_REGID__MASK;
-}
-#define A3XX_SP_FS_MRT_REG_HALF_PRECISION			0x00000100
-#define A3XX_SP_FS_MRT_REG_SINT					0x00000400
-#define A3XX_SP_FS_MRT_REG_UINT					0x00000800
-
-#define REG_A3XX_SP_FS_IMAGE_OUTPUT(i0) (0x000022f4 + 0x1*(i0))
-
-static inline uint32_t REG_A3XX_SP_FS_IMAGE_OUTPUT_REG(uint32_t i0) { return 0x000022f4 + 0x1*i0; }
-#define A3XX_SP_FS_IMAGE_OUTPUT_REG_MRTFORMAT__MASK		0x0000003f
-#define A3XX_SP_FS_IMAGE_OUTPUT_REG_MRTFORMAT__SHIFT		0
-static inline uint32_t A3XX_SP_FS_IMAGE_OUTPUT_REG_MRTFORMAT(enum a3xx_color_fmt val)
-{
-	return ((val) << A3XX_SP_FS_IMAGE_OUTPUT_REG_MRTFORMAT__SHIFT) & A3XX_SP_FS_IMAGE_OUTPUT_REG_MRTFORMAT__MASK;
-}
-
-#define REG_A3XX_SP_FS_LENGTH_REG				0x000022ff
-#define A3XX_SP_FS_LENGTH_REG_SHADERLENGTH__MASK		0xffffffff
-#define A3XX_SP_FS_LENGTH_REG_SHADERLENGTH__SHIFT		0
-static inline uint32_t A3XX_SP_FS_LENGTH_REG_SHADERLENGTH(uint32_t val)
-{
-	return ((val) << A3XX_SP_FS_LENGTH_REG_SHADERLENGTH__SHIFT) & A3XX_SP_FS_LENGTH_REG_SHADERLENGTH__MASK;
-}
-
-#define REG_A3XX_PA_SC_AA_CONFIG				0x00002301
-
-#define REG_A3XX_TPL1_TP_VS_TEX_OFFSET				0x00002340
-#define A3XX_TPL1_TP_VS_TEX_OFFSET_SAMPLEROFFSET__MASK		0x000000ff
-#define A3XX_TPL1_TP_VS_TEX_OFFSET_SAMPLEROFFSET__SHIFT		0
-static inline uint32_t A3XX_TPL1_TP_VS_TEX_OFFSET_SAMPLEROFFSET(uint32_t val)
-{
-	return ((val) << A3XX_TPL1_TP_VS_TEX_OFFSET_SAMPLEROFFSET__SHIFT) & A3XX_TPL1_TP_VS_TEX_OFFSET_SAMPLEROFFSET__MASK;
-}
-#define A3XX_TPL1_TP_VS_TEX_OFFSET_MEMOBJOFFSET__MASK		0x0000ff00
-#define A3XX_TPL1_TP_VS_TEX_OFFSET_MEMOBJOFFSET__SHIFT		8
-static inline uint32_t A3XX_TPL1_TP_VS_TEX_OFFSET_MEMOBJOFFSET(uint32_t val)
-{
-	return ((val) << A3XX_TPL1_TP_VS_TEX_OFFSET_MEMOBJOFFSET__SHIFT) & A3XX_TPL1_TP_VS_TEX_OFFSET_MEMOBJOFFSET__MASK;
-}
-#define A3XX_TPL1_TP_VS_TEX_OFFSET_BASETABLEPTR__MASK		0xffff0000
-#define A3XX_TPL1_TP_VS_TEX_OFFSET_BASETABLEPTR__SHIFT		16
-static inline uint32_t A3XX_TPL1_TP_VS_TEX_OFFSET_BASETABLEPTR(uint32_t val)
-{
-	return ((val) << A3XX_TPL1_TP_VS_TEX_OFFSET_BASETABLEPTR__SHIFT) & A3XX_TPL1_TP_VS_TEX_OFFSET_BASETABLEPTR__MASK;
-}
-
-#define REG_A3XX_TPL1_TP_VS_BORDER_COLOR_BASE_ADDR		0x00002341
-
-#define REG_A3XX_TPL1_TP_FS_TEX_OFFSET				0x00002342
-#define A3XX_TPL1_TP_FS_TEX_OFFSET_SAMPLEROFFSET__MASK		0x000000ff
-#define A3XX_TPL1_TP_FS_TEX_OFFSET_SAMPLEROFFSET__SHIFT		0
-static inline uint32_t A3XX_TPL1_TP_FS_TEX_OFFSET_SAMPLEROFFSET(uint32_t val)
-{
-	return ((val) << A3XX_TPL1_TP_FS_TEX_OFFSET_SAMPLEROFFSET__SHIFT) & A3XX_TPL1_TP_FS_TEX_OFFSET_SAMPLEROFFSET__MASK;
-}
-#define A3XX_TPL1_TP_FS_TEX_OFFSET_MEMOBJOFFSET__MASK		0x0000ff00
-#define A3XX_TPL1_TP_FS_TEX_OFFSET_MEMOBJOFFSET__SHIFT		8
-static inline uint32_t A3XX_TPL1_TP_FS_TEX_OFFSET_MEMOBJOFFSET(uint32_t val)
-{
-	return ((val) << A3XX_TPL1_TP_FS_TEX_OFFSET_MEMOBJOFFSET__SHIFT) & A3XX_TPL1_TP_FS_TEX_OFFSET_MEMOBJOFFSET__MASK;
-}
-#define A3XX_TPL1_TP_FS_TEX_OFFSET_BASETABLEPTR__MASK		0xffff0000
-#define A3XX_TPL1_TP_FS_TEX_OFFSET_BASETABLEPTR__SHIFT		16
-static inline uint32_t A3XX_TPL1_TP_FS_TEX_OFFSET_BASETABLEPTR(uint32_t val)
-{
-	return ((val) << A3XX_TPL1_TP_FS_TEX_OFFSET_BASETABLEPTR__SHIFT) & A3XX_TPL1_TP_FS_TEX_OFFSET_BASETABLEPTR__MASK;
-}
-
-#define REG_A3XX_TPL1_TP_FS_BORDER_COLOR_BASE_ADDR		0x00002343
-
-#define REG_A3XX_VBIF_CLKON					0x00003001
-
-#define REG_A3XX_VBIF_FIXED_SORT_EN				0x0000300c
-
-#define REG_A3XX_VBIF_FIXED_SORT_SEL0				0x0000300d
-
-#define REG_A3XX_VBIF_FIXED_SORT_SEL1				0x0000300e
-
-#define REG_A3XX_VBIF_ABIT_SORT					0x0000301c
-
-#define REG_A3XX_VBIF_ABIT_SORT_CONF				0x0000301d
-
-#define REG_A3XX_VBIF_GATE_OFF_WRREQ_EN				0x0000302a
-
-#define REG_A3XX_VBIF_IN_RD_LIM_CONF0				0x0000302c
-
-#define REG_A3XX_VBIF_IN_RD_LIM_CONF1				0x0000302d
-
-#define REG_A3XX_VBIF_IN_WR_LIM_CONF0				0x00003030
-
-#define REG_A3XX_VBIF_IN_WR_LIM_CONF1				0x00003031
-
-#define REG_A3XX_VBIF_OUT_RD_LIM_CONF0				0x00003034
-
-#define REG_A3XX_VBIF_OUT_WR_LIM_CONF0				0x00003035
-
-#define REG_A3XX_VBIF_DDR_OUT_MAX_BURST				0x00003036
-
-#define REG_A3XX_VBIF_ARB_CTL					0x0000303c
-
-#define REG_A3XX_VBIF_ROUND_ROBIN_QOS_ARB			0x00003049
-
-#define REG_A3XX_VBIF_OUT_AXI_AMEMTYPE_CONF0			0x00003058
-
-#define REG_A3XX_VBIF_OUT_AXI_AOOO_EN				0x0000305e
-
-#define REG_A3XX_VBIF_OUT_AXI_AOOO				0x0000305f
-
-#define REG_A3XX_VBIF_PERF_CNT_EN				0x00003070
-#define A3XX_VBIF_PERF_CNT_EN_CNT0				0x00000001
-#define A3XX_VBIF_PERF_CNT_EN_CNT1				0x00000002
-#define A3XX_VBIF_PERF_CNT_EN_PWRCNT0				0x00000004
-#define A3XX_VBIF_PERF_CNT_EN_PWRCNT1				0x00000008
-#define A3XX_VBIF_PERF_CNT_EN_PWRCNT2				0x00000010
-
-#define REG_A3XX_VBIF_PERF_CNT_CLR				0x00003071
-#define A3XX_VBIF_PERF_CNT_CLR_CNT0				0x00000001
-#define A3XX_VBIF_PERF_CNT_CLR_CNT1				0x00000002
-#define A3XX_VBIF_PERF_CNT_CLR_PWRCNT0				0x00000004
-#define A3XX_VBIF_PERF_CNT_CLR_PWRCNT1				0x00000008
-#define A3XX_VBIF_PERF_CNT_CLR_PWRCNT2				0x00000010
-
-#define REG_A3XX_VBIF_PERF_CNT_SEL				0x00003072
-
-#define REG_A3XX_VBIF_PERF_CNT0_LO				0x00003073
-
-#define REG_A3XX_VBIF_PERF_CNT0_HI				0x00003074
-
-#define REG_A3XX_VBIF_PERF_CNT1_LO				0x00003075
-
-#define REG_A3XX_VBIF_PERF_CNT1_HI				0x00003076
-
-#define REG_A3XX_VBIF_PERF_PWR_CNT0_LO				0x00003077
-
-#define REG_A3XX_VBIF_PERF_PWR_CNT0_HI				0x00003078
-
-#define REG_A3XX_VBIF_PERF_PWR_CNT1_LO				0x00003079
-
-#define REG_A3XX_VBIF_PERF_PWR_CNT1_HI				0x0000307a
-
-#define REG_A3XX_VBIF_PERF_PWR_CNT2_LO				0x0000307b
-
-#define REG_A3XX_VBIF_PERF_PWR_CNT2_HI				0x0000307c
-
-#define REG_A3XX_VSC_BIN_SIZE					0x00000c01
-#define A3XX_VSC_BIN_SIZE_WIDTH__MASK				0x0000001f
-#define A3XX_VSC_BIN_SIZE_WIDTH__SHIFT				0
-static inline uint32_t A3XX_VSC_BIN_SIZE_WIDTH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A3XX_VSC_BIN_SIZE_WIDTH__SHIFT) & A3XX_VSC_BIN_SIZE_WIDTH__MASK;
-}
-#define A3XX_VSC_BIN_SIZE_HEIGHT__MASK				0x000003e0
-#define A3XX_VSC_BIN_SIZE_HEIGHT__SHIFT				5
-static inline uint32_t A3XX_VSC_BIN_SIZE_HEIGHT(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A3XX_VSC_BIN_SIZE_HEIGHT__SHIFT) & A3XX_VSC_BIN_SIZE_HEIGHT__MASK;
-}
-
-#define REG_A3XX_VSC_SIZE_ADDRESS				0x00000c02
-
-#define REG_A3XX_VSC_PIPE(i0) (0x00000c06 + 0x3*(i0))
-
-static inline uint32_t REG_A3XX_VSC_PIPE_CONFIG(uint32_t i0) { return 0x00000c06 + 0x3*i0; }
-#define A3XX_VSC_PIPE_CONFIG_X__MASK				0x000003ff
-#define A3XX_VSC_PIPE_CONFIG_X__SHIFT				0
-static inline uint32_t A3XX_VSC_PIPE_CONFIG_X(uint32_t val)
-{
-	return ((val) << A3XX_VSC_PIPE_CONFIG_X__SHIFT) & A3XX_VSC_PIPE_CONFIG_X__MASK;
-}
-#define A3XX_VSC_PIPE_CONFIG_Y__MASK				0x000ffc00
-#define A3XX_VSC_PIPE_CONFIG_Y__SHIFT				10
-static inline uint32_t A3XX_VSC_PIPE_CONFIG_Y(uint32_t val)
-{
-	return ((val) << A3XX_VSC_PIPE_CONFIG_Y__SHIFT) & A3XX_VSC_PIPE_CONFIG_Y__MASK;
-}
-#define A3XX_VSC_PIPE_CONFIG_W__MASK				0x00f00000
-#define A3XX_VSC_PIPE_CONFIG_W__SHIFT				20
-static inline uint32_t A3XX_VSC_PIPE_CONFIG_W(uint32_t val)
-{
-	return ((val) << A3XX_VSC_PIPE_CONFIG_W__SHIFT) & A3XX_VSC_PIPE_CONFIG_W__MASK;
-}
-#define A3XX_VSC_PIPE_CONFIG_H__MASK				0x0f000000
-#define A3XX_VSC_PIPE_CONFIG_H__SHIFT				24
-static inline uint32_t A3XX_VSC_PIPE_CONFIG_H(uint32_t val)
-{
-	return ((val) << A3XX_VSC_PIPE_CONFIG_H__SHIFT) & A3XX_VSC_PIPE_CONFIG_H__MASK;
-}
-
-static inline uint32_t REG_A3XX_VSC_PIPE_DATA_ADDRESS(uint32_t i0) { return 0x00000c07 + 0x3*i0; }
-
-static inline uint32_t REG_A3XX_VSC_PIPE_DATA_LENGTH(uint32_t i0) { return 0x00000c08 + 0x3*i0; }
-
-#define REG_A3XX_VSC_BIN_CONTROL				0x00000c3c
-#define A3XX_VSC_BIN_CONTROL_BINNING_ENABLE			0x00000001
-
-#define REG_A3XX_UNKNOWN_0C3D					0x00000c3d
-
-#define REG_A3XX_PC_PERFCOUNTER0_SELECT				0x00000c48
-
-#define REG_A3XX_PC_PERFCOUNTER1_SELECT				0x00000c49
-
-#define REG_A3XX_PC_PERFCOUNTER2_SELECT				0x00000c4a
-
-#define REG_A3XX_PC_PERFCOUNTER3_SELECT				0x00000c4b
-
-#define REG_A3XX_GRAS_TSE_DEBUG_ECO				0x00000c81
-
-#define REG_A3XX_GRAS_PERFCOUNTER0_SELECT			0x00000c88
-
-#define REG_A3XX_GRAS_PERFCOUNTER1_SELECT			0x00000c89
-
-#define REG_A3XX_GRAS_PERFCOUNTER2_SELECT			0x00000c8a
-
-#define REG_A3XX_GRAS_PERFCOUNTER3_SELECT			0x00000c8b
-
-#define REG_A3XX_GRAS_CL_USER_PLANE(i0) (0x00000ca0 + 0x4*(i0))
-
-static inline uint32_t REG_A3XX_GRAS_CL_USER_PLANE_X(uint32_t i0) { return 0x00000ca0 + 0x4*i0; }
-
-static inline uint32_t REG_A3XX_GRAS_CL_USER_PLANE_Y(uint32_t i0) { return 0x00000ca1 + 0x4*i0; }
-
-static inline uint32_t REG_A3XX_GRAS_CL_USER_PLANE_Z(uint32_t i0) { return 0x00000ca2 + 0x4*i0; }
-
-static inline uint32_t REG_A3XX_GRAS_CL_USER_PLANE_W(uint32_t i0) { return 0x00000ca3 + 0x4*i0; }
-
-#define REG_A3XX_RB_GMEM_BASE_ADDR				0x00000cc0
-
-#define REG_A3XX_RB_DEBUG_ECO_CONTROLS_ADDR			0x00000cc1
-
-#define REG_A3XX_RB_PERFCOUNTER0_SELECT				0x00000cc6
-
-#define REG_A3XX_RB_PERFCOUNTER1_SELECT				0x00000cc7
-
-#define REG_A3XX_RB_FRAME_BUFFER_DIMENSION			0x00000ce0
-#define A3XX_RB_FRAME_BUFFER_DIMENSION_WIDTH__MASK		0x00003fff
-#define A3XX_RB_FRAME_BUFFER_DIMENSION_WIDTH__SHIFT		0
-static inline uint32_t A3XX_RB_FRAME_BUFFER_DIMENSION_WIDTH(uint32_t val)
-{
-	return ((val) << A3XX_RB_FRAME_BUFFER_DIMENSION_WIDTH__SHIFT) & A3XX_RB_FRAME_BUFFER_DIMENSION_WIDTH__MASK;
-}
-#define A3XX_RB_FRAME_BUFFER_DIMENSION_HEIGHT__MASK		0x0fffc000
-#define A3XX_RB_FRAME_BUFFER_DIMENSION_HEIGHT__SHIFT		14
-static inline uint32_t A3XX_RB_FRAME_BUFFER_DIMENSION_HEIGHT(uint32_t val)
-{
-	return ((val) << A3XX_RB_FRAME_BUFFER_DIMENSION_HEIGHT__SHIFT) & A3XX_RB_FRAME_BUFFER_DIMENSION_HEIGHT__MASK;
-}
-
-#define REG_A3XX_HLSQ_PERFCOUNTER0_SELECT			0x00000e00
-
-#define REG_A3XX_HLSQ_PERFCOUNTER1_SELECT			0x00000e01
-
-#define REG_A3XX_HLSQ_PERFCOUNTER2_SELECT			0x00000e02
-
-#define REG_A3XX_HLSQ_PERFCOUNTER3_SELECT			0x00000e03
-
-#define REG_A3XX_HLSQ_PERFCOUNTER4_SELECT			0x00000e04
-
-#define REG_A3XX_HLSQ_PERFCOUNTER5_SELECT			0x00000e05
-
-#define REG_A3XX_UNKNOWN_0E43					0x00000e43
-
-#define REG_A3XX_VFD_PERFCOUNTER0_SELECT			0x00000e44
-
-#define REG_A3XX_VFD_PERFCOUNTER1_SELECT			0x00000e45
-
-#define REG_A3XX_VPC_VPC_DEBUG_RAM_SEL				0x00000e61
-
-#define REG_A3XX_VPC_VPC_DEBUG_RAM_READ				0x00000e62
-
-#define REG_A3XX_VPC_PERFCOUNTER0_SELECT			0x00000e64
-
-#define REG_A3XX_VPC_PERFCOUNTER1_SELECT			0x00000e65
-
-#define REG_A3XX_UCHE_CACHE_MODE_CONTROL_REG			0x00000e82
-
-#define REG_A3XX_UCHE_PERFCOUNTER0_SELECT			0x00000e84
-
-#define REG_A3XX_UCHE_PERFCOUNTER1_SELECT			0x00000e85
-
-#define REG_A3XX_UCHE_PERFCOUNTER2_SELECT			0x00000e86
-
-#define REG_A3XX_UCHE_PERFCOUNTER3_SELECT			0x00000e87
-
-#define REG_A3XX_UCHE_PERFCOUNTER4_SELECT			0x00000e88
-
-#define REG_A3XX_UCHE_PERFCOUNTER5_SELECT			0x00000e89
-
-#define REG_A3XX_UCHE_CACHE_INVALIDATE0_REG			0x00000ea0
-#define A3XX_UCHE_CACHE_INVALIDATE0_REG_ADDR__MASK		0x0fffffff
-#define A3XX_UCHE_CACHE_INVALIDATE0_REG_ADDR__SHIFT		0
-static inline uint32_t A3XX_UCHE_CACHE_INVALIDATE0_REG_ADDR(uint32_t val)
-{
-	return ((val) << A3XX_UCHE_CACHE_INVALIDATE0_REG_ADDR__SHIFT) & A3XX_UCHE_CACHE_INVALIDATE0_REG_ADDR__MASK;
-}
-
-#define REG_A3XX_UCHE_CACHE_INVALIDATE1_REG			0x00000ea1
-#define A3XX_UCHE_CACHE_INVALIDATE1_REG_ADDR__MASK		0x0fffffff
-#define A3XX_UCHE_CACHE_INVALIDATE1_REG_ADDR__SHIFT		0
-static inline uint32_t A3XX_UCHE_CACHE_INVALIDATE1_REG_ADDR(uint32_t val)
-{
-	return ((val) << A3XX_UCHE_CACHE_INVALIDATE1_REG_ADDR__SHIFT) & A3XX_UCHE_CACHE_INVALIDATE1_REG_ADDR__MASK;
-}
-#define A3XX_UCHE_CACHE_INVALIDATE1_REG_OPCODE__MASK		0x30000000
-#define A3XX_UCHE_CACHE_INVALIDATE1_REG_OPCODE__SHIFT		28
-static inline uint32_t A3XX_UCHE_CACHE_INVALIDATE1_REG_OPCODE(enum a3xx_cache_opcode val)
-{
-	return ((val) << A3XX_UCHE_CACHE_INVALIDATE1_REG_OPCODE__SHIFT) & A3XX_UCHE_CACHE_INVALIDATE1_REG_OPCODE__MASK;
-}
-#define A3XX_UCHE_CACHE_INVALIDATE1_REG_ENTIRE_CACHE		0x80000000
-
-#define REG_A3XX_UNKNOWN_0EA6					0x00000ea6
-
-#define REG_A3XX_SP_PERFCOUNTER0_SELECT				0x00000ec4
-
-#define REG_A3XX_SP_PERFCOUNTER1_SELECT				0x00000ec5
-
-#define REG_A3XX_SP_PERFCOUNTER2_SELECT				0x00000ec6
-
-#define REG_A3XX_SP_PERFCOUNTER3_SELECT				0x00000ec7
-
-#define REG_A3XX_SP_PERFCOUNTER4_SELECT				0x00000ec8
-
-#define REG_A3XX_SP_PERFCOUNTER5_SELECT				0x00000ec9
-
-#define REG_A3XX_SP_PERFCOUNTER6_SELECT				0x00000eca
-
-#define REG_A3XX_SP_PERFCOUNTER7_SELECT				0x00000ecb
-
-#define REG_A3XX_UNKNOWN_0EE0					0x00000ee0
-
-#define REG_A3XX_UNKNOWN_0F03					0x00000f03
-
-#define REG_A3XX_TP_PERFCOUNTER0_SELECT				0x00000f04
-
-#define REG_A3XX_TP_PERFCOUNTER1_SELECT				0x00000f05
-
-#define REG_A3XX_TP_PERFCOUNTER2_SELECT				0x00000f06
-
-#define REG_A3XX_TP_PERFCOUNTER3_SELECT				0x00000f07
-
-#define REG_A3XX_TP_PERFCOUNTER4_SELECT				0x00000f08
-
-#define REG_A3XX_TP_PERFCOUNTER5_SELECT				0x00000f09
-
-#define REG_A3XX_VGT_CL_INITIATOR				0x000021f0
-
-#define REG_A3XX_VGT_EVENT_INITIATOR				0x000021f9
-
-#define REG_A3XX_VGT_DRAW_INITIATOR				0x000021fc
-#define A3XX_VGT_DRAW_INITIATOR_PRIM_TYPE__MASK			0x0000003f
-#define A3XX_VGT_DRAW_INITIATOR_PRIM_TYPE__SHIFT		0
-static inline uint32_t A3XX_VGT_DRAW_INITIATOR_PRIM_TYPE(enum pc_di_primtype val)
-{
-	return ((val) << A3XX_VGT_DRAW_INITIATOR_PRIM_TYPE__SHIFT) & A3XX_VGT_DRAW_INITIATOR_PRIM_TYPE__MASK;
-}
-#define A3XX_VGT_DRAW_INITIATOR_SOURCE_SELECT__MASK		0x000000c0
-#define A3XX_VGT_DRAW_INITIATOR_SOURCE_SELECT__SHIFT		6
-static inline uint32_t A3XX_VGT_DRAW_INITIATOR_SOURCE_SELECT(enum pc_di_src_sel val)
-{
-	return ((val) << A3XX_VGT_DRAW_INITIATOR_SOURCE_SELECT__SHIFT) & A3XX_VGT_DRAW_INITIATOR_SOURCE_SELECT__MASK;
-}
-#define A3XX_VGT_DRAW_INITIATOR_VIS_CULL__MASK			0x00000600
-#define A3XX_VGT_DRAW_INITIATOR_VIS_CULL__SHIFT			9
-static inline uint32_t A3XX_VGT_DRAW_INITIATOR_VIS_CULL(enum pc_di_vis_cull_mode val)
-{
-	return ((val) << A3XX_VGT_DRAW_INITIATOR_VIS_CULL__SHIFT) & A3XX_VGT_DRAW_INITIATOR_VIS_CULL__MASK;
-}
-#define A3XX_VGT_DRAW_INITIATOR_INDEX_SIZE__MASK		0x00000800
-#define A3XX_VGT_DRAW_INITIATOR_INDEX_SIZE__SHIFT		11
-static inline uint32_t A3XX_VGT_DRAW_INITIATOR_INDEX_SIZE(enum pc_di_index_size val)
-{
-	return ((val) << A3XX_VGT_DRAW_INITIATOR_INDEX_SIZE__SHIFT) & A3XX_VGT_DRAW_INITIATOR_INDEX_SIZE__MASK;
-}
-#define A3XX_VGT_DRAW_INITIATOR_NOT_EOP				0x00001000
-#define A3XX_VGT_DRAW_INITIATOR_SMALL_INDEX			0x00002000
-#define A3XX_VGT_DRAW_INITIATOR_PRE_DRAW_INITIATOR_ENABLE	0x00004000
-#define A3XX_VGT_DRAW_INITIATOR_NUM_INSTANCES__MASK		0xff000000
-#define A3XX_VGT_DRAW_INITIATOR_NUM_INSTANCES__SHIFT		24
-static inline uint32_t A3XX_VGT_DRAW_INITIATOR_NUM_INSTANCES(uint32_t val)
-{
-	return ((val) << A3XX_VGT_DRAW_INITIATOR_NUM_INSTANCES__SHIFT) & A3XX_VGT_DRAW_INITIATOR_NUM_INSTANCES__MASK;
-}
-
-#define REG_A3XX_VGT_IMMED_DATA					0x000021fd
-
-#define REG_A3XX_TEX_SAMP_0					0x00000000
-#define A3XX_TEX_SAMP_0_CLAMPENABLE				0x00000001
-#define A3XX_TEX_SAMP_0_MIPFILTER_LINEAR			0x00000002
-#define A3XX_TEX_SAMP_0_XY_MAG__MASK				0x0000000c
-#define A3XX_TEX_SAMP_0_XY_MAG__SHIFT				2
-static inline uint32_t A3XX_TEX_SAMP_0_XY_MAG(enum a3xx_tex_filter val)
-{
-	return ((val) << A3XX_TEX_SAMP_0_XY_MAG__SHIFT) & A3XX_TEX_SAMP_0_XY_MAG__MASK;
-}
-#define A3XX_TEX_SAMP_0_XY_MIN__MASK				0x00000030
-#define A3XX_TEX_SAMP_0_XY_MIN__SHIFT				4
-static inline uint32_t A3XX_TEX_SAMP_0_XY_MIN(enum a3xx_tex_filter val)
-{
-	return ((val) << A3XX_TEX_SAMP_0_XY_MIN__SHIFT) & A3XX_TEX_SAMP_0_XY_MIN__MASK;
-}
-#define A3XX_TEX_SAMP_0_WRAP_S__MASK				0x000001c0
-#define A3XX_TEX_SAMP_0_WRAP_S__SHIFT				6
-static inline uint32_t A3XX_TEX_SAMP_0_WRAP_S(enum a3xx_tex_clamp val)
-{
-	return ((val) << A3XX_TEX_SAMP_0_WRAP_S__SHIFT) & A3XX_TEX_SAMP_0_WRAP_S__MASK;
-}
-#define A3XX_TEX_SAMP_0_WRAP_T__MASK				0x00000e00
-#define A3XX_TEX_SAMP_0_WRAP_T__SHIFT				9
-static inline uint32_t A3XX_TEX_SAMP_0_WRAP_T(enum a3xx_tex_clamp val)
-{
-	return ((val) << A3XX_TEX_SAMP_0_WRAP_T__SHIFT) & A3XX_TEX_SAMP_0_WRAP_T__MASK;
-}
-#define A3XX_TEX_SAMP_0_WRAP_R__MASK				0x00007000
-#define A3XX_TEX_SAMP_0_WRAP_R__SHIFT				12
-static inline uint32_t A3XX_TEX_SAMP_0_WRAP_R(enum a3xx_tex_clamp val)
-{
-	return ((val) << A3XX_TEX_SAMP_0_WRAP_R__SHIFT) & A3XX_TEX_SAMP_0_WRAP_R__MASK;
-}
-#define A3XX_TEX_SAMP_0_ANISO__MASK				0x00038000
-#define A3XX_TEX_SAMP_0_ANISO__SHIFT				15
-static inline uint32_t A3XX_TEX_SAMP_0_ANISO(enum a3xx_tex_aniso val)
-{
-	return ((val) << A3XX_TEX_SAMP_0_ANISO__SHIFT) & A3XX_TEX_SAMP_0_ANISO__MASK;
-}
-#define A3XX_TEX_SAMP_0_COMPARE_FUNC__MASK			0x00700000
-#define A3XX_TEX_SAMP_0_COMPARE_FUNC__SHIFT			20
-static inline uint32_t A3XX_TEX_SAMP_0_COMPARE_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A3XX_TEX_SAMP_0_COMPARE_FUNC__SHIFT) & A3XX_TEX_SAMP_0_COMPARE_FUNC__MASK;
-}
-#define A3XX_TEX_SAMP_0_CUBEMAPSEAMLESSFILTOFF			0x01000000
-#define A3XX_TEX_SAMP_0_UNNORM_COORDS				0x80000000
-
-#define REG_A3XX_TEX_SAMP_1					0x00000001
-#define A3XX_TEX_SAMP_1_LOD_BIAS__MASK				0x000007ff
-#define A3XX_TEX_SAMP_1_LOD_BIAS__SHIFT				0
-static inline uint32_t A3XX_TEX_SAMP_1_LOD_BIAS(float val)
-{
-	return ((((int32_t)(val * 64.0))) << A3XX_TEX_SAMP_1_LOD_BIAS__SHIFT) & A3XX_TEX_SAMP_1_LOD_BIAS__MASK;
-}
-#define A3XX_TEX_SAMP_1_MAX_LOD__MASK				0x003ff000
-#define A3XX_TEX_SAMP_1_MAX_LOD__SHIFT				12
-static inline uint32_t A3XX_TEX_SAMP_1_MAX_LOD(float val)
-{
-	return ((((uint32_t)(val * 64.0))) << A3XX_TEX_SAMP_1_MAX_LOD__SHIFT) & A3XX_TEX_SAMP_1_MAX_LOD__MASK;
-}
-#define A3XX_TEX_SAMP_1_MIN_LOD__MASK				0xffc00000
-#define A3XX_TEX_SAMP_1_MIN_LOD__SHIFT				22
-static inline uint32_t A3XX_TEX_SAMP_1_MIN_LOD(float val)
-{
-	return ((((uint32_t)(val * 64.0))) << A3XX_TEX_SAMP_1_MIN_LOD__SHIFT) & A3XX_TEX_SAMP_1_MIN_LOD__MASK;
-}
-
-#define REG_A3XX_TEX_CONST_0					0x00000000
-#define A3XX_TEX_CONST_0_TILE_MODE__MASK			0x00000003
-#define A3XX_TEX_CONST_0_TILE_MODE__SHIFT			0
-static inline uint32_t A3XX_TEX_CONST_0_TILE_MODE(enum a3xx_tile_mode val)
-{
-	return ((val) << A3XX_TEX_CONST_0_TILE_MODE__SHIFT) & A3XX_TEX_CONST_0_TILE_MODE__MASK;
-}
-#define A3XX_TEX_CONST_0_SRGB					0x00000004
-#define A3XX_TEX_CONST_0_SWIZ_X__MASK				0x00000070
-#define A3XX_TEX_CONST_0_SWIZ_X__SHIFT				4
-static inline uint32_t A3XX_TEX_CONST_0_SWIZ_X(enum a3xx_tex_swiz val)
-{
-	return ((val) << A3XX_TEX_CONST_0_SWIZ_X__SHIFT) & A3XX_TEX_CONST_0_SWIZ_X__MASK;
-}
-#define A3XX_TEX_CONST_0_SWIZ_Y__MASK				0x00000380
-#define A3XX_TEX_CONST_0_SWIZ_Y__SHIFT				7
-static inline uint32_t A3XX_TEX_CONST_0_SWIZ_Y(enum a3xx_tex_swiz val)
-{
-	return ((val) << A3XX_TEX_CONST_0_SWIZ_Y__SHIFT) & A3XX_TEX_CONST_0_SWIZ_Y__MASK;
-}
-#define A3XX_TEX_CONST_0_SWIZ_Z__MASK				0x00001c00
-#define A3XX_TEX_CONST_0_SWIZ_Z__SHIFT				10
-static inline uint32_t A3XX_TEX_CONST_0_SWIZ_Z(enum a3xx_tex_swiz val)
-{
-	return ((val) << A3XX_TEX_CONST_0_SWIZ_Z__SHIFT) & A3XX_TEX_CONST_0_SWIZ_Z__MASK;
-}
-#define A3XX_TEX_CONST_0_SWIZ_W__MASK				0x0000e000
-#define A3XX_TEX_CONST_0_SWIZ_W__SHIFT				13
-static inline uint32_t A3XX_TEX_CONST_0_SWIZ_W(enum a3xx_tex_swiz val)
-{
-	return ((val) << A3XX_TEX_CONST_0_SWIZ_W__SHIFT) & A3XX_TEX_CONST_0_SWIZ_W__MASK;
-}
-#define A3XX_TEX_CONST_0_MIPLVLS__MASK				0x000f0000
-#define A3XX_TEX_CONST_0_MIPLVLS__SHIFT				16
-static inline uint32_t A3XX_TEX_CONST_0_MIPLVLS(uint32_t val)
-{
-	return ((val) << A3XX_TEX_CONST_0_MIPLVLS__SHIFT) & A3XX_TEX_CONST_0_MIPLVLS__MASK;
-}
-#define A3XX_TEX_CONST_0_MSAATEX__MASK				0x00300000
-#define A3XX_TEX_CONST_0_MSAATEX__SHIFT				20
-static inline uint32_t A3XX_TEX_CONST_0_MSAATEX(enum a3xx_tex_msaa val)
-{
-	return ((val) << A3XX_TEX_CONST_0_MSAATEX__SHIFT) & A3XX_TEX_CONST_0_MSAATEX__MASK;
-}
-#define A3XX_TEX_CONST_0_FMT__MASK				0x1fc00000
-#define A3XX_TEX_CONST_0_FMT__SHIFT				22
-static inline uint32_t A3XX_TEX_CONST_0_FMT(enum a3xx_tex_fmt val)
-{
-	return ((val) << A3XX_TEX_CONST_0_FMT__SHIFT) & A3XX_TEX_CONST_0_FMT__MASK;
-}
-#define A3XX_TEX_CONST_0_NOCONVERT				0x20000000
-#define A3XX_TEX_CONST_0_TYPE__MASK				0xc0000000
-#define A3XX_TEX_CONST_0_TYPE__SHIFT				30
-static inline uint32_t A3XX_TEX_CONST_0_TYPE(enum a3xx_tex_type val)
-{
-	return ((val) << A3XX_TEX_CONST_0_TYPE__SHIFT) & A3XX_TEX_CONST_0_TYPE__MASK;
-}
-
-#define REG_A3XX_TEX_CONST_1					0x00000001
-#define A3XX_TEX_CONST_1_HEIGHT__MASK				0x00003fff
-#define A3XX_TEX_CONST_1_HEIGHT__SHIFT				0
-static inline uint32_t A3XX_TEX_CONST_1_HEIGHT(uint32_t val)
-{
-	return ((val) << A3XX_TEX_CONST_1_HEIGHT__SHIFT) & A3XX_TEX_CONST_1_HEIGHT__MASK;
-}
-#define A3XX_TEX_CONST_1_WIDTH__MASK				0x0fffc000
-#define A3XX_TEX_CONST_1_WIDTH__SHIFT				14
-static inline uint32_t A3XX_TEX_CONST_1_WIDTH(uint32_t val)
-{
-	return ((val) << A3XX_TEX_CONST_1_WIDTH__SHIFT) & A3XX_TEX_CONST_1_WIDTH__MASK;
-}
-#define A3XX_TEX_CONST_1_PITCHALIGN__MASK			0xf0000000
-#define A3XX_TEX_CONST_1_PITCHALIGN__SHIFT			28
-static inline uint32_t A3XX_TEX_CONST_1_PITCHALIGN(uint32_t val)
-{
-	return ((val) << A3XX_TEX_CONST_1_PITCHALIGN__SHIFT) & A3XX_TEX_CONST_1_PITCHALIGN__MASK;
-}
-
-#define REG_A3XX_TEX_CONST_2					0x00000002
-#define A3XX_TEX_CONST_2_INDX__MASK				0x000001ff
-#define A3XX_TEX_CONST_2_INDX__SHIFT				0
-static inline uint32_t A3XX_TEX_CONST_2_INDX(uint32_t val)
-{
-	return ((val) << A3XX_TEX_CONST_2_INDX__SHIFT) & A3XX_TEX_CONST_2_INDX__MASK;
-}
-#define A3XX_TEX_CONST_2_PITCH__MASK				0x3ffff000
-#define A3XX_TEX_CONST_2_PITCH__SHIFT				12
-static inline uint32_t A3XX_TEX_CONST_2_PITCH(uint32_t val)
-{
-	return ((val) << A3XX_TEX_CONST_2_PITCH__SHIFT) & A3XX_TEX_CONST_2_PITCH__MASK;
-}
-#define A3XX_TEX_CONST_2_SWAP__MASK				0xc0000000
-#define A3XX_TEX_CONST_2_SWAP__SHIFT				30
-static inline uint32_t A3XX_TEX_CONST_2_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A3XX_TEX_CONST_2_SWAP__SHIFT) & A3XX_TEX_CONST_2_SWAP__MASK;
-}
-
-#define REG_A3XX_TEX_CONST_3					0x00000003
-#define A3XX_TEX_CONST_3_LAYERSZ1__MASK				0x0001ffff
-#define A3XX_TEX_CONST_3_LAYERSZ1__SHIFT			0
-static inline uint32_t A3XX_TEX_CONST_3_LAYERSZ1(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A3XX_TEX_CONST_3_LAYERSZ1__SHIFT) & A3XX_TEX_CONST_3_LAYERSZ1__MASK;
-}
-#define A3XX_TEX_CONST_3_DEPTH__MASK				0x0ffe0000
-#define A3XX_TEX_CONST_3_DEPTH__SHIFT				17
-static inline uint32_t A3XX_TEX_CONST_3_DEPTH(uint32_t val)
-{
-	return ((val) << A3XX_TEX_CONST_3_DEPTH__SHIFT) & A3XX_TEX_CONST_3_DEPTH__MASK;
-}
-#define A3XX_TEX_CONST_3_LAYERSZ2__MASK				0xf0000000
-#define A3XX_TEX_CONST_3_LAYERSZ2__SHIFT			28
-static inline uint32_t A3XX_TEX_CONST_3_LAYERSZ2(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A3XX_TEX_CONST_3_LAYERSZ2__SHIFT) & A3XX_TEX_CONST_3_LAYERSZ2__MASK;
-}
-
-#ifdef __cplusplus
-#endif
-
-#endif /* A3XX_XML */
diff --git a/drivers/gpu/drm/msm/adreno/a4xx.xml.h b/drivers/gpu/drm/msm/adreno/a4xx.xml.h
deleted file mode 100644
index 103a416a787f..000000000000
--- a/drivers/gpu/drm/msm/adreno/a4xx.xml.h
+++ /dev/null
@@ -1,4379 +0,0 @@
-#ifndef A4XX_XML
-#define A4XX_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng gen_header.py tool in this git repository:
-http://gitlab.freedesktop.org/mesa/mesa/
-git clone https://gitlab.freedesktop.org/mesa/mesa.git
-
-The rules-ng-ng source files this header was generated from are:
-
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a4xx.xml          ( 113474 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml  (   1572 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml (  15434 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml    (  85691 bytes, from Fri Feb 16 09:49:01 2024)
-
-Copyright (C) 2013-2024 by the following authors:
-- Rob Clark <robdclark@gmail.com> Rob Clark
-- Ilia Mirkin <imirkin@alum.mit.edu> Ilia Mirkin
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-*/
-
-#ifdef __KERNEL__
-#include <linux/bug.h>
-#define assert(x) BUG_ON(!(x))
-#else
-#include <assert.h>
-#endif
-
-#ifdef __cplusplus
-#define __struct_cast(X)
-#else
-#define __struct_cast(X) (struct X)
-#endif
-
-enum a4xx_color_fmt {
-	RB4_A8_UNORM = 1,
-	RB4_R8_UNORM = 2,
-	RB4_R8_SNORM = 3,
-	RB4_R8_UINT = 4,
-	RB4_R8_SINT = 5,
-	RB4_R4G4B4A4_UNORM = 8,
-	RB4_R5G5B5A1_UNORM = 10,
-	RB4_R5G6B5_UNORM = 14,
-	RB4_R8G8_UNORM = 15,
-	RB4_R8G8_SNORM = 16,
-	RB4_R8G8_UINT = 17,
-	RB4_R8G8_SINT = 18,
-	RB4_R16_UNORM = 19,
-	RB4_R16_SNORM = 20,
-	RB4_R16_FLOAT = 21,
-	RB4_R16_UINT = 22,
-	RB4_R16_SINT = 23,
-	RB4_R8G8B8_UNORM = 25,
-	RB4_R8G8B8A8_UNORM = 26,
-	RB4_R8G8B8A8_SNORM = 28,
-	RB4_R8G8B8A8_UINT = 29,
-	RB4_R8G8B8A8_SINT = 30,
-	RB4_R10G10B10A2_UNORM = 31,
-	RB4_R10G10B10A2_UINT = 34,
-	RB4_R11G11B10_FLOAT = 39,
-	RB4_R16G16_UNORM = 40,
-	RB4_R16G16_SNORM = 41,
-	RB4_R16G16_FLOAT = 42,
-	RB4_R16G16_UINT = 43,
-	RB4_R16G16_SINT = 44,
-	RB4_R32_FLOAT = 45,
-	RB4_R32_UINT = 46,
-	RB4_R32_SINT = 47,
-	RB4_R16G16B16A16_UNORM = 52,
-	RB4_R16G16B16A16_SNORM = 53,
-	RB4_R16G16B16A16_FLOAT = 54,
-	RB4_R16G16B16A16_UINT = 55,
-	RB4_R16G16B16A16_SINT = 56,
-	RB4_R32G32_FLOAT = 57,
-	RB4_R32G32_UINT = 58,
-	RB4_R32G32_SINT = 59,
-	RB4_R32G32B32A32_FLOAT = 60,
-	RB4_R32G32B32A32_UINT = 61,
-	RB4_R32G32B32A32_SINT = 62,
-	RB4_NONE = 255,
-};
-
-enum a4xx_tile_mode {
-	TILE4_LINEAR = 0,
-	TILE4_2 = 2,
-	TILE4_3 = 3,
-};
-
-enum a4xx_vtx_fmt {
-	VFMT4_32_FLOAT = 1,
-	VFMT4_32_32_FLOAT = 2,
-	VFMT4_32_32_32_FLOAT = 3,
-	VFMT4_32_32_32_32_FLOAT = 4,
-	VFMT4_16_FLOAT = 5,
-	VFMT4_16_16_FLOAT = 6,
-	VFMT4_16_16_16_FLOAT = 7,
-	VFMT4_16_16_16_16_FLOAT = 8,
-	VFMT4_32_FIXED = 9,
-	VFMT4_32_32_FIXED = 10,
-	VFMT4_32_32_32_FIXED = 11,
-	VFMT4_32_32_32_32_FIXED = 12,
-	VFMT4_11_11_10_FLOAT = 13,
-	VFMT4_16_SINT = 16,
-	VFMT4_16_16_SINT = 17,
-	VFMT4_16_16_16_SINT = 18,
-	VFMT4_16_16_16_16_SINT = 19,
-	VFMT4_16_UINT = 20,
-	VFMT4_16_16_UINT = 21,
-	VFMT4_16_16_16_UINT = 22,
-	VFMT4_16_16_16_16_UINT = 23,
-	VFMT4_16_SNORM = 24,
-	VFMT4_16_16_SNORM = 25,
-	VFMT4_16_16_16_SNORM = 26,
-	VFMT4_16_16_16_16_SNORM = 27,
-	VFMT4_16_UNORM = 28,
-	VFMT4_16_16_UNORM = 29,
-	VFMT4_16_16_16_UNORM = 30,
-	VFMT4_16_16_16_16_UNORM = 31,
-	VFMT4_32_UINT = 32,
-	VFMT4_32_32_UINT = 33,
-	VFMT4_32_32_32_UINT = 34,
-	VFMT4_32_32_32_32_UINT = 35,
-	VFMT4_32_SINT = 36,
-	VFMT4_32_32_SINT = 37,
-	VFMT4_32_32_32_SINT = 38,
-	VFMT4_32_32_32_32_SINT = 39,
-	VFMT4_8_UINT = 40,
-	VFMT4_8_8_UINT = 41,
-	VFMT4_8_8_8_UINT = 42,
-	VFMT4_8_8_8_8_UINT = 43,
-	VFMT4_8_UNORM = 44,
-	VFMT4_8_8_UNORM = 45,
-	VFMT4_8_8_8_UNORM = 46,
-	VFMT4_8_8_8_8_UNORM = 47,
-	VFMT4_8_SINT = 48,
-	VFMT4_8_8_SINT = 49,
-	VFMT4_8_8_8_SINT = 50,
-	VFMT4_8_8_8_8_SINT = 51,
-	VFMT4_8_SNORM = 52,
-	VFMT4_8_8_SNORM = 53,
-	VFMT4_8_8_8_SNORM = 54,
-	VFMT4_8_8_8_8_SNORM = 55,
-	VFMT4_10_10_10_2_UINT = 56,
-	VFMT4_10_10_10_2_UNORM = 57,
-	VFMT4_10_10_10_2_SINT = 58,
-	VFMT4_10_10_10_2_SNORM = 59,
-	VFMT4_2_10_10_10_UINT = 60,
-	VFMT4_2_10_10_10_UNORM = 61,
-	VFMT4_2_10_10_10_SINT = 62,
-	VFMT4_2_10_10_10_SNORM = 63,
-	VFMT4_NONE = 255,
-};
-
-enum a4xx_tex_fmt {
-	TFMT4_A8_UNORM = 3,
-	TFMT4_8_UNORM = 4,
-	TFMT4_8_SNORM = 5,
-	TFMT4_8_UINT = 6,
-	TFMT4_8_SINT = 7,
-	TFMT4_4_4_4_4_UNORM = 8,
-	TFMT4_5_5_5_1_UNORM = 9,
-	TFMT4_5_6_5_UNORM = 11,
-	TFMT4_L8_A8_UNORM = 13,
-	TFMT4_8_8_UNORM = 14,
-	TFMT4_8_8_SNORM = 15,
-	TFMT4_8_8_UINT = 16,
-	TFMT4_8_8_SINT = 17,
-	TFMT4_16_UNORM = 18,
-	TFMT4_16_SNORM = 19,
-	TFMT4_16_FLOAT = 20,
-	TFMT4_16_UINT = 21,
-	TFMT4_16_SINT = 22,
-	TFMT4_8_8_8_8_UNORM = 28,
-	TFMT4_8_8_8_8_SNORM = 29,
-	TFMT4_8_8_8_8_UINT = 30,
-	TFMT4_8_8_8_8_SINT = 31,
-	TFMT4_9_9_9_E5_FLOAT = 32,
-	TFMT4_10_10_10_2_UNORM = 33,
-	TFMT4_10_10_10_2_UINT = 34,
-	TFMT4_11_11_10_FLOAT = 37,
-	TFMT4_16_16_UNORM = 38,
-	TFMT4_16_16_SNORM = 39,
-	TFMT4_16_16_FLOAT = 40,
-	TFMT4_16_16_UINT = 41,
-	TFMT4_16_16_SINT = 42,
-	TFMT4_32_FLOAT = 43,
-	TFMT4_32_UINT = 44,
-	TFMT4_32_SINT = 45,
-	TFMT4_16_16_16_16_UNORM = 51,
-	TFMT4_16_16_16_16_SNORM = 52,
-	TFMT4_16_16_16_16_FLOAT = 53,
-	TFMT4_16_16_16_16_UINT = 54,
-	TFMT4_16_16_16_16_SINT = 55,
-	TFMT4_32_32_FLOAT = 56,
-	TFMT4_32_32_UINT = 57,
-	TFMT4_32_32_SINT = 58,
-	TFMT4_32_32_32_FLOAT = 59,
-	TFMT4_32_32_32_UINT = 60,
-	TFMT4_32_32_32_SINT = 61,
-	TFMT4_32_32_32_32_FLOAT = 63,
-	TFMT4_32_32_32_32_UINT = 64,
-	TFMT4_32_32_32_32_SINT = 65,
-	TFMT4_X8Z24_UNORM = 71,
-	TFMT4_DXT1 = 86,
-	TFMT4_DXT3 = 87,
-	TFMT4_DXT5 = 88,
-	TFMT4_RGTC1_UNORM = 90,
-	TFMT4_RGTC1_SNORM = 91,
-	TFMT4_RGTC2_UNORM = 94,
-	TFMT4_RGTC2_SNORM = 95,
-	TFMT4_BPTC_UFLOAT = 97,
-	TFMT4_BPTC_FLOAT = 98,
-	TFMT4_BPTC = 99,
-	TFMT4_ATC_RGB = 100,
-	TFMT4_ATC_RGBA_EXPLICIT = 101,
-	TFMT4_ATC_RGBA_INTERPOLATED = 102,
-	TFMT4_ETC2_RG11_UNORM = 103,
-	TFMT4_ETC2_RG11_SNORM = 104,
-	TFMT4_ETC2_R11_UNORM = 105,
-	TFMT4_ETC2_R11_SNORM = 106,
-	TFMT4_ETC1 = 107,
-	TFMT4_ETC2_RGB8 = 108,
-	TFMT4_ETC2_RGBA8 = 109,
-	TFMT4_ETC2_RGB8A1 = 110,
-	TFMT4_ASTC_4x4 = 111,
-	TFMT4_ASTC_5x4 = 112,
-	TFMT4_ASTC_5x5 = 113,
-	TFMT4_ASTC_6x5 = 114,
-	TFMT4_ASTC_6x6 = 115,
-	TFMT4_ASTC_8x5 = 116,
-	TFMT4_ASTC_8x6 = 117,
-	TFMT4_ASTC_8x8 = 118,
-	TFMT4_ASTC_10x5 = 119,
-	TFMT4_ASTC_10x6 = 120,
-	TFMT4_ASTC_10x8 = 121,
-	TFMT4_ASTC_10x10 = 122,
-	TFMT4_ASTC_12x10 = 123,
-	TFMT4_ASTC_12x12 = 124,
-	TFMT4_NONE = 255,
-};
-
-enum a4xx_depth_format {
-	DEPTH4_NONE = 0,
-	DEPTH4_16 = 1,
-	DEPTH4_24_8 = 2,
-	DEPTH4_32 = 3,
-};
-
-enum a4xx_ccu_perfcounter_select {
-	CCU_BUSY_CYCLES = 0,
-	CCU_RB_DEPTH_RETURN_STALL = 2,
-	CCU_RB_COLOR_RETURN_STALL = 3,
-	CCU_DEPTH_BLOCKS = 6,
-	CCU_COLOR_BLOCKS = 7,
-	CCU_DEPTH_BLOCK_HIT = 8,
-	CCU_COLOR_BLOCK_HIT = 9,
-	CCU_DEPTH_FLAG1_COUNT = 10,
-	CCU_DEPTH_FLAG2_COUNT = 11,
-	CCU_DEPTH_FLAG3_COUNT = 12,
-	CCU_DEPTH_FLAG4_COUNT = 13,
-	CCU_COLOR_FLAG1_COUNT = 14,
-	CCU_COLOR_FLAG2_COUNT = 15,
-	CCU_COLOR_FLAG3_COUNT = 16,
-	CCU_COLOR_FLAG4_COUNT = 17,
-	CCU_PARTIAL_BLOCK_READ = 18,
-};
-
-enum a4xx_cp_perfcounter_select {
-	CP_ALWAYS_COUNT = 0,
-	CP_BUSY = 1,
-	CP_PFP_IDLE = 2,
-	CP_PFP_BUSY_WORKING = 3,
-	CP_PFP_STALL_CYCLES_ANY = 4,
-	CP_PFP_STARVE_CYCLES_ANY = 5,
-	CP_PFP_STARVED_PER_LOAD_ADDR = 6,
-	CP_PFP_STALLED_PER_STORE_ADDR = 7,
-	CP_PFP_PC_PROFILE = 8,
-	CP_PFP_MATCH_PM4_PKT_PROFILE = 9,
-	CP_PFP_COND_INDIRECT_DISCARDED = 10,
-	CP_LONG_RESUMPTIONS = 11,
-	CP_RESUME_CYCLES = 12,
-	CP_RESUME_TO_BOUNDARY_CYCLES = 13,
-	CP_LONG_PREEMPTIONS = 14,
-	CP_PREEMPT_CYCLES = 15,
-	CP_PREEMPT_TO_BOUNDARY_CYCLES = 16,
-	CP_ME_FIFO_EMPTY_PFP_IDLE = 17,
-	CP_ME_FIFO_EMPTY_PFP_BUSY = 18,
-	CP_ME_FIFO_NOT_EMPTY_NOT_FULL = 19,
-	CP_ME_FIFO_FULL_ME_BUSY = 20,
-	CP_ME_FIFO_FULL_ME_NON_WORKING = 21,
-	CP_ME_WAITING_FOR_PACKETS = 22,
-	CP_ME_BUSY_WORKING = 23,
-	CP_ME_STARVE_CYCLES_ANY = 24,
-	CP_ME_STARVE_CYCLES_PER_PROFILE = 25,
-	CP_ME_STALL_CYCLES_PER_PROFILE = 26,
-	CP_ME_PC_PROFILE = 27,
-	CP_RCIU_FIFO_EMPTY = 28,
-	CP_RCIU_FIFO_NOT_EMPTY_NOT_FULL = 29,
-	CP_RCIU_FIFO_FULL = 30,
-	CP_RCIU_FIFO_FULL_NO_CONTEXT = 31,
-	CP_RCIU_FIFO_FULL_AHB_MASTER = 32,
-	CP_RCIU_FIFO_FULL_OTHER = 33,
-	CP_AHB_IDLE = 34,
-	CP_AHB_STALL_ON_GRANT_NO_SPLIT = 35,
-	CP_AHB_STALL_ON_GRANT_SPLIT = 36,
-	CP_AHB_STALL_ON_GRANT_SPLIT_PROFILE = 37,
-	CP_AHB_BUSY_WORKING = 38,
-	CP_AHB_BUSY_STALL_ON_HRDY = 39,
-	CP_AHB_BUSY_STALL_ON_HRDY_PROFILE = 40,
-};
-
-enum a4xx_gras_ras_perfcounter_select {
-	RAS_SUPER_TILES = 0,
-	RAS_8X8_TILES = 1,
-	RAS_4X4_TILES = 2,
-	RAS_BUSY_CYCLES = 3,
-	RAS_STALL_CYCLES_BY_RB = 4,
-	RAS_STALL_CYCLES_BY_VSC = 5,
-	RAS_STARVE_CYCLES_BY_TSE = 6,
-	RAS_SUPERTILE_CYCLES = 7,
-	RAS_TILE_CYCLES = 8,
-	RAS_FULLY_COVERED_SUPER_TILES = 9,
-	RAS_FULLY_COVERED_8X8_TILES = 10,
-	RAS_4X4_PRIM = 11,
-	RAS_8X4_4X8_PRIM = 12,
-	RAS_8X8_PRIM = 13,
-};
-
-enum a4xx_gras_tse_perfcounter_select {
-	TSE_INPUT_PRIM = 0,
-	TSE_INPUT_NULL_PRIM = 1,
-	TSE_TRIVAL_REJ_PRIM = 2,
-	TSE_CLIPPED_PRIM = 3,
-	TSE_NEW_PRIM = 4,
-	TSE_ZERO_AREA_PRIM = 5,
-	TSE_FACENESS_CULLED_PRIM = 6,
-	TSE_ZERO_PIXEL_PRIM = 7,
-	TSE_OUTPUT_NULL_PRIM = 8,
-	TSE_OUTPUT_VISIBLE_PRIM = 9,
-	TSE_PRE_CLIP_PRIM = 10,
-	TSE_POST_CLIP_PRIM = 11,
-	TSE_BUSY_CYCLES = 12,
-	TSE_PC_STARVE = 13,
-	TSE_RAS_STALL = 14,
-	TSE_STALL_BARYPLANE_FIFO_FULL = 15,
-	TSE_STALL_ZPLANE_FIFO_FULL = 16,
-};
-
-enum a4xx_hlsq_perfcounter_select {
-	HLSQ_SP_VS_STAGE_CONSTANT = 0,
-	HLSQ_SP_VS_STAGE_INSTRUCTIONS = 1,
-	HLSQ_SP_FS_STAGE_CONSTANT = 2,
-	HLSQ_SP_FS_STAGE_INSTRUCTIONS = 3,
-	HLSQ_TP_STATE = 4,
-	HLSQ_QUADS = 5,
-	HLSQ_PIXELS = 6,
-	HLSQ_VERTICES = 7,
-	HLSQ_SP_VS_STAGE_DATA_BYTES = 13,
-	HLSQ_SP_FS_STAGE_DATA_BYTES = 14,
-	HLSQ_BUSY_CYCLES = 15,
-	HLSQ_STALL_CYCLES_SP_STATE = 16,
-	HLSQ_STALL_CYCLES_SP_VS_STAGE = 17,
-	HLSQ_STALL_CYCLES_SP_FS_STAGE = 18,
-	HLSQ_STALL_CYCLES_UCHE = 19,
-	HLSQ_RBBM_LOAD_CYCLES = 20,
-	HLSQ_DI_TO_VS_START_SP = 21,
-	HLSQ_DI_TO_FS_START_SP = 22,
-	HLSQ_VS_STAGE_START_TO_DONE_SP = 23,
-	HLSQ_FS_STAGE_START_TO_DONE_SP = 24,
-	HLSQ_SP_STATE_COPY_CYCLES_VS_STAGE = 25,
-	HLSQ_SP_STATE_COPY_CYCLES_FS_STAGE = 26,
-	HLSQ_UCHE_LATENCY_CYCLES = 27,
-	HLSQ_UCHE_LATENCY_COUNT = 28,
-	HLSQ_STARVE_CYCLES_VFD = 29,
-};
-
-enum a4xx_pc_perfcounter_select {
-	PC_VIS_STREAMS_LOADED = 0,
-	PC_VPC_PRIMITIVES = 2,
-	PC_DEAD_PRIM = 3,
-	PC_LIVE_PRIM = 4,
-	PC_DEAD_DRAWCALLS = 5,
-	PC_LIVE_DRAWCALLS = 6,
-	PC_VERTEX_MISSES = 7,
-	PC_STALL_CYCLES_VFD = 9,
-	PC_STALL_CYCLES_TSE = 10,
-	PC_STALL_CYCLES_UCHE = 11,
-	PC_WORKING_CYCLES = 12,
-	PC_IA_VERTICES = 13,
-	PC_GS_PRIMITIVES = 14,
-	PC_HS_INVOCATIONS = 15,
-	PC_DS_INVOCATIONS = 16,
-	PC_DS_PRIMITIVES = 17,
-	PC_STARVE_CYCLES_FOR_INDEX = 20,
-	PC_STARVE_CYCLES_FOR_TESS_FACTOR = 21,
-	PC_STARVE_CYCLES_FOR_VIZ_STREAM = 22,
-	PC_STALL_CYCLES_TESS = 23,
-	PC_STARVE_CYCLES_FOR_POSITION = 24,
-	PC_MODE0_DRAWCALL = 25,
-	PC_MODE1_DRAWCALL = 26,
-	PC_MODE2_DRAWCALL = 27,
-	PC_MODE3_DRAWCALL = 28,
-	PC_MODE4_DRAWCALL = 29,
-	PC_PREDICATED_DEAD_DRAWCALL = 30,
-	PC_STALL_CYCLES_BY_TSE_ONLY = 31,
-	PC_STALL_CYCLES_BY_VPC_ONLY = 32,
-	PC_VPC_POS_DATA_TRANSACTION = 33,
-	PC_BUSY_CYCLES = 34,
-	PC_STARVE_CYCLES_DI = 35,
-	PC_STALL_CYCLES_VPC = 36,
-	TESS_WORKING_CYCLES = 37,
-	TESS_NUM_CYCLES_SETUP_WORKING = 38,
-	TESS_NUM_CYCLES_PTGEN_WORKING = 39,
-	TESS_NUM_CYCLES_CONNGEN_WORKING = 40,
-	TESS_BUSY_CYCLES = 41,
-	TESS_STARVE_CYCLES_PC = 42,
-	TESS_STALL_CYCLES_PC = 43,
-};
-
-enum a4xx_pwr_perfcounter_select {
-	PWR_CORE_CLOCK_CYCLES = 0,
-	PWR_BUSY_CLOCK_CYCLES = 1,
-};
-
-enum a4xx_rb_perfcounter_select {
-	RB_BUSY_CYCLES = 0,
-	RB_BUSY_CYCLES_BINNING = 1,
-	RB_BUSY_CYCLES_RENDERING = 2,
-	RB_BUSY_CYCLES_RESOLVE = 3,
-	RB_STARVE_CYCLES_BY_SP = 4,
-	RB_STARVE_CYCLES_BY_RAS = 5,
-	RB_STARVE_CYCLES_BY_MARB = 6,
-	RB_STALL_CYCLES_BY_MARB = 7,
-	RB_STALL_CYCLES_BY_HLSQ = 8,
-	RB_RB_RB_MARB_DATA = 9,
-	RB_SP_RB_QUAD = 10,
-	RB_RAS_RB_Z_QUADS = 11,
-	RB_GMEM_CH0_READ = 12,
-	RB_GMEM_CH1_READ = 13,
-	RB_GMEM_CH0_WRITE = 14,
-	RB_GMEM_CH1_WRITE = 15,
-	RB_CP_CONTEXT_DONE = 16,
-	RB_CP_CACHE_FLUSH = 17,
-	RB_CP_ZPASS_DONE = 18,
-	RB_STALL_FIFO0_FULL = 19,
-	RB_STALL_FIFO1_FULL = 20,
-	RB_STALL_FIFO2_FULL = 21,
-	RB_STALL_FIFO3_FULL = 22,
-	RB_RB_HLSQ_TRANSACTIONS = 23,
-	RB_Z_READ = 24,
-	RB_Z_WRITE = 25,
-	RB_C_READ = 26,
-	RB_C_WRITE = 27,
-	RB_C_READ_LATENCY = 28,
-	RB_Z_READ_LATENCY = 29,
-	RB_STALL_BY_UCHE = 30,
-	RB_MARB_UCHE_TRANSACTIONS = 31,
-	RB_CACHE_STALL_MISS = 32,
-	RB_CACHE_STALL_FIFO_FULL = 33,
-	RB_8BIT_BLENDER_UNITS_ACTIVE = 34,
-	RB_16BIT_BLENDER_UNITS_ACTIVE = 35,
-	RB_SAMPLER_UNITS_ACTIVE = 36,
-	RB_TOTAL_PASS = 38,
-	RB_Z_PASS = 39,
-	RB_Z_FAIL = 40,
-	RB_S_FAIL = 41,
-	RB_POWER0 = 42,
-	RB_POWER1 = 43,
-	RB_POWER2 = 44,
-	RB_POWER3 = 45,
-	RB_POWER4 = 46,
-	RB_POWER5 = 47,
-	RB_POWER6 = 48,
-	RB_POWER7 = 49,
-};
-
-enum a4xx_rbbm_perfcounter_select {
-	RBBM_ALWAYS_ON = 0,
-	RBBM_VBIF_BUSY = 1,
-	RBBM_TSE_BUSY = 2,
-	RBBM_RAS_BUSY = 3,
-	RBBM_PC_DCALL_BUSY = 4,
-	RBBM_PC_VSD_BUSY = 5,
-	RBBM_VFD_BUSY = 6,
-	RBBM_VPC_BUSY = 7,
-	RBBM_UCHE_BUSY = 8,
-	RBBM_VSC_BUSY = 9,
-	RBBM_HLSQ_BUSY = 10,
-	RBBM_ANY_RB_BUSY = 11,
-	RBBM_ANY_TPL1_BUSY = 12,
-	RBBM_ANY_SP_BUSY = 13,
-	RBBM_ANY_MARB_BUSY = 14,
-	RBBM_ANY_ARB_BUSY = 15,
-	RBBM_AHB_STATUS_BUSY = 16,
-	RBBM_AHB_STATUS_STALLED = 17,
-	RBBM_AHB_STATUS_TXFR = 18,
-	RBBM_AHB_STATUS_TXFR_SPLIT = 19,
-	RBBM_AHB_STATUS_TXFR_ERROR = 20,
-	RBBM_AHB_STATUS_LONG_STALL = 21,
-	RBBM_STATUS_MASKED = 22,
-	RBBM_CP_BUSY_GFX_CORE_IDLE = 23,
-	RBBM_TESS_BUSY = 24,
-	RBBM_COM_BUSY = 25,
-	RBBM_DCOM_BUSY = 32,
-	RBBM_ANY_CCU_BUSY = 33,
-	RBBM_DPM_BUSY = 34,
-};
-
-enum a4xx_sp_perfcounter_select {
-	SP_LM_LOAD_INSTRUCTIONS = 0,
-	SP_LM_STORE_INSTRUCTIONS = 1,
-	SP_LM_ATOMICS = 2,
-	SP_GM_LOAD_INSTRUCTIONS = 3,
-	SP_GM_STORE_INSTRUCTIONS = 4,
-	SP_GM_ATOMICS = 5,
-	SP_VS_STAGE_TEX_INSTRUCTIONS = 6,
-	SP_VS_STAGE_CFLOW_INSTRUCTIONS = 7,
-	SP_VS_STAGE_EFU_INSTRUCTIONS = 8,
-	SP_VS_STAGE_FULL_ALU_INSTRUCTIONS = 9,
-	SP_VS_STAGE_HALF_ALU_INSTRUCTIONS = 10,
-	SP_FS_STAGE_TEX_INSTRUCTIONS = 11,
-	SP_FS_STAGE_CFLOW_INSTRUCTIONS = 12,
-	SP_FS_STAGE_EFU_INSTRUCTIONS = 13,
-	SP_FS_STAGE_FULL_ALU_INSTRUCTIONS = 14,
-	SP_FS_STAGE_HALF_ALU_INSTRUCTIONS = 15,
-	SP_VS_INSTRUCTIONS = 17,
-	SP_FS_INSTRUCTIONS = 18,
-	SP_ADDR_LOCK_COUNT = 19,
-	SP_UCHE_READ_TRANS = 20,
-	SP_UCHE_WRITE_TRANS = 21,
-	SP_EXPORT_VPC_TRANS = 22,
-	SP_EXPORT_RB_TRANS = 23,
-	SP_PIXELS_KILLED = 24,
-	SP_ICL1_REQUESTS = 25,
-	SP_ICL1_MISSES = 26,
-	SP_ICL0_REQUESTS = 27,
-	SP_ICL0_MISSES = 28,
-	SP_ALU_WORKING_CYCLES = 29,
-	SP_EFU_WORKING_CYCLES = 30,
-	SP_STALL_CYCLES_BY_VPC = 31,
-	SP_STALL_CYCLES_BY_TP = 32,
-	SP_STALL_CYCLES_BY_UCHE = 33,
-	SP_STALL_CYCLES_BY_RB = 34,
-	SP_BUSY_CYCLES = 35,
-	SP_HS_INSTRUCTIONS = 36,
-	SP_DS_INSTRUCTIONS = 37,
-	SP_GS_INSTRUCTIONS = 38,
-	SP_CS_INSTRUCTIONS = 39,
-	SP_SCHEDULER_NON_WORKING = 40,
-	SP_WAVE_CONTEXTS = 41,
-	SP_WAVE_CONTEXT_CYCLES = 42,
-	SP_POWER0 = 43,
-	SP_POWER1 = 44,
-	SP_POWER2 = 45,
-	SP_POWER3 = 46,
-	SP_POWER4 = 47,
-	SP_POWER5 = 48,
-	SP_POWER6 = 49,
-	SP_POWER7 = 50,
-	SP_POWER8 = 51,
-	SP_POWER9 = 52,
-	SP_POWER10 = 53,
-	SP_POWER11 = 54,
-	SP_POWER12 = 55,
-	SP_POWER13 = 56,
-	SP_POWER14 = 57,
-	SP_POWER15 = 58,
-};
-
-enum a4xx_tp_perfcounter_select {
-	TP_L1_REQUESTS = 0,
-	TP_L1_MISSES = 1,
-	TP_QUADS_OFFSET = 8,
-	TP_QUAD_SHADOW = 9,
-	TP_QUADS_ARRAY = 10,
-	TP_QUADS_GRADIENT = 11,
-	TP_QUADS_1D2D = 12,
-	TP_QUADS_3DCUBE = 13,
-	TP_BUSY_CYCLES = 16,
-	TP_STALL_CYCLES_BY_ARB = 17,
-	TP_STATE_CACHE_REQUESTS = 20,
-	TP_STATE_CACHE_MISSES = 21,
-	TP_POWER0 = 22,
-	TP_POWER1 = 23,
-	TP_POWER2 = 24,
-	TP_POWER3 = 25,
-	TP_POWER4 = 26,
-	TP_POWER5 = 27,
-	TP_POWER6 = 28,
-	TP_POWER7 = 29,
-};
-
-enum a4xx_uche_perfcounter_select {
-	UCHE_VBIF_READ_BEATS_TP = 0,
-	UCHE_VBIF_READ_BEATS_VFD = 1,
-	UCHE_VBIF_READ_BEATS_HLSQ = 2,
-	UCHE_VBIF_READ_BEATS_MARB = 3,
-	UCHE_VBIF_READ_BEATS_SP = 4,
-	UCHE_READ_REQUESTS_TP = 5,
-	UCHE_READ_REQUESTS_VFD = 6,
-	UCHE_READ_REQUESTS_HLSQ = 7,
-	UCHE_READ_REQUESTS_MARB = 8,
-	UCHE_READ_REQUESTS_SP = 9,
-	UCHE_WRITE_REQUESTS_MARB = 10,
-	UCHE_WRITE_REQUESTS_SP = 11,
-	UCHE_TAG_CHECK_FAILS = 12,
-	UCHE_EVICTS = 13,
-	UCHE_FLUSHES = 14,
-	UCHE_VBIF_LATENCY_CYCLES = 15,
-	UCHE_VBIF_LATENCY_SAMPLES = 16,
-	UCHE_BUSY_CYCLES = 17,
-	UCHE_VBIF_READ_BEATS_PC = 18,
-	UCHE_READ_REQUESTS_PC = 19,
-	UCHE_WRITE_REQUESTS_VPC = 20,
-	UCHE_STALL_BY_VBIF = 21,
-	UCHE_WRITE_REQUESTS_VSC = 22,
-	UCHE_POWER0 = 23,
-	UCHE_POWER1 = 24,
-	UCHE_POWER2 = 25,
-	UCHE_POWER3 = 26,
-	UCHE_POWER4 = 27,
-	UCHE_POWER5 = 28,
-	UCHE_POWER6 = 29,
-	UCHE_POWER7 = 30,
-};
-
-enum a4xx_vbif_perfcounter_select {
-	AXI_READ_REQUESTS_ID_0 = 0,
-	AXI_READ_REQUESTS_ID_1 = 1,
-	AXI_READ_REQUESTS_ID_2 = 2,
-	AXI_READ_REQUESTS_ID_3 = 3,
-	AXI_READ_REQUESTS_ID_4 = 4,
-	AXI_READ_REQUESTS_ID_5 = 5,
-	AXI_READ_REQUESTS_ID_6 = 6,
-	AXI_READ_REQUESTS_ID_7 = 7,
-	AXI_READ_REQUESTS_ID_8 = 8,
-	AXI_READ_REQUESTS_ID_9 = 9,
-	AXI_READ_REQUESTS_ID_10 = 10,
-	AXI_READ_REQUESTS_ID_11 = 11,
-	AXI_READ_REQUESTS_ID_12 = 12,
-	AXI_READ_REQUESTS_ID_13 = 13,
-	AXI_READ_REQUESTS_ID_14 = 14,
-	AXI_READ_REQUESTS_ID_15 = 15,
-	AXI0_READ_REQUESTS_TOTAL = 16,
-	AXI1_READ_REQUESTS_TOTAL = 17,
-	AXI2_READ_REQUESTS_TOTAL = 18,
-	AXI3_READ_REQUESTS_TOTAL = 19,
-	AXI_READ_REQUESTS_TOTAL = 20,
-	AXI_WRITE_REQUESTS_ID_0 = 21,
-	AXI_WRITE_REQUESTS_ID_1 = 22,
-	AXI_WRITE_REQUESTS_ID_2 = 23,
-	AXI_WRITE_REQUESTS_ID_3 = 24,
-	AXI_WRITE_REQUESTS_ID_4 = 25,
-	AXI_WRITE_REQUESTS_ID_5 = 26,
-	AXI_WRITE_REQUESTS_ID_6 = 27,
-	AXI_WRITE_REQUESTS_ID_7 = 28,
-	AXI_WRITE_REQUESTS_ID_8 = 29,
-	AXI_WRITE_REQUESTS_ID_9 = 30,
-	AXI_WRITE_REQUESTS_ID_10 = 31,
-	AXI_WRITE_REQUESTS_ID_11 = 32,
-	AXI_WRITE_REQUESTS_ID_12 = 33,
-	AXI_WRITE_REQUESTS_ID_13 = 34,
-	AXI_WRITE_REQUESTS_ID_14 = 35,
-	AXI_WRITE_REQUESTS_ID_15 = 36,
-	AXI0_WRITE_REQUESTS_TOTAL = 37,
-	AXI1_WRITE_REQUESTS_TOTAL = 38,
-	AXI2_WRITE_REQUESTS_TOTAL = 39,
-	AXI3_WRITE_REQUESTS_TOTAL = 40,
-	AXI_WRITE_REQUESTS_TOTAL = 41,
-	AXI_TOTAL_REQUESTS = 42,
-	AXI_READ_DATA_BEATS_ID_0 = 43,
-	AXI_READ_DATA_BEATS_ID_1 = 44,
-	AXI_READ_DATA_BEATS_ID_2 = 45,
-	AXI_READ_DATA_BEATS_ID_3 = 46,
-	AXI_READ_DATA_BEATS_ID_4 = 47,
-	AXI_READ_DATA_BEATS_ID_5 = 48,
-	AXI_READ_DATA_BEATS_ID_6 = 49,
-	AXI_READ_DATA_BEATS_ID_7 = 50,
-	AXI_READ_DATA_BEATS_ID_8 = 51,
-	AXI_READ_DATA_BEATS_ID_9 = 52,
-	AXI_READ_DATA_BEATS_ID_10 = 53,
-	AXI_READ_DATA_BEATS_ID_11 = 54,
-	AXI_READ_DATA_BEATS_ID_12 = 55,
-	AXI_READ_DATA_BEATS_ID_13 = 56,
-	AXI_READ_DATA_BEATS_ID_14 = 57,
-	AXI_READ_DATA_BEATS_ID_15 = 58,
-	AXI0_READ_DATA_BEATS_TOTAL = 59,
-	AXI1_READ_DATA_BEATS_TOTAL = 60,
-	AXI2_READ_DATA_BEATS_TOTAL = 61,
-	AXI3_READ_DATA_BEATS_TOTAL = 62,
-	AXI_READ_DATA_BEATS_TOTAL = 63,
-	AXI_WRITE_DATA_BEATS_ID_0 = 64,
-	AXI_WRITE_DATA_BEATS_ID_1 = 65,
-	AXI_WRITE_DATA_BEATS_ID_2 = 66,
-	AXI_WRITE_DATA_BEATS_ID_3 = 67,
-	AXI_WRITE_DATA_BEATS_ID_4 = 68,
-	AXI_WRITE_DATA_BEATS_ID_5 = 69,
-	AXI_WRITE_DATA_BEATS_ID_6 = 70,
-	AXI_WRITE_DATA_BEATS_ID_7 = 71,
-	AXI_WRITE_DATA_BEATS_ID_8 = 72,
-	AXI_WRITE_DATA_BEATS_ID_9 = 73,
-	AXI_WRITE_DATA_BEATS_ID_10 = 74,
-	AXI_WRITE_DATA_BEATS_ID_11 = 75,
-	AXI_WRITE_DATA_BEATS_ID_12 = 76,
-	AXI_WRITE_DATA_BEATS_ID_13 = 77,
-	AXI_WRITE_DATA_BEATS_ID_14 = 78,
-	AXI_WRITE_DATA_BEATS_ID_15 = 79,
-	AXI0_WRITE_DATA_BEATS_TOTAL = 80,
-	AXI1_WRITE_DATA_BEATS_TOTAL = 81,
-	AXI2_WRITE_DATA_BEATS_TOTAL = 82,
-	AXI3_WRITE_DATA_BEATS_TOTAL = 83,
-	AXI_WRITE_DATA_BEATS_TOTAL = 84,
-	AXI_DATA_BEATS_TOTAL = 85,
-	CYCLES_HELD_OFF_ID_0 = 86,
-	CYCLES_HELD_OFF_ID_1 = 87,
-	CYCLES_HELD_OFF_ID_2 = 88,
-	CYCLES_HELD_OFF_ID_3 = 89,
-	CYCLES_HELD_OFF_ID_4 = 90,
-	CYCLES_HELD_OFF_ID_5 = 91,
-	CYCLES_HELD_OFF_ID_6 = 92,
-	CYCLES_HELD_OFF_ID_7 = 93,
-	CYCLES_HELD_OFF_ID_8 = 94,
-	CYCLES_HELD_OFF_ID_9 = 95,
-	CYCLES_HELD_OFF_ID_10 = 96,
-	CYCLES_HELD_OFF_ID_11 = 97,
-	CYCLES_HELD_OFF_ID_12 = 98,
-	CYCLES_HELD_OFF_ID_13 = 99,
-	CYCLES_HELD_OFF_ID_14 = 100,
-	CYCLES_HELD_OFF_ID_15 = 101,
-	AXI_READ_REQUEST_HELD_OFF = 102,
-	AXI_WRITE_REQUEST_HELD_OFF = 103,
-	AXI_REQUEST_HELD_OFF = 104,
-	AXI_WRITE_DATA_HELD_OFF = 105,
-	OCMEM_AXI_READ_REQUEST_HELD_OFF = 106,
-	OCMEM_AXI_WRITE_REQUEST_HELD_OFF = 107,
-	OCMEM_AXI_REQUEST_HELD_OFF = 108,
-	OCMEM_AXI_WRITE_DATA_HELD_OFF = 109,
-	ELAPSED_CYCLES_DDR = 110,
-	ELAPSED_CYCLES_OCMEM = 111,
-};
-
-enum a4xx_vfd_perfcounter_select {
-	VFD_UCHE_BYTE_FETCHED = 0,
-	VFD_UCHE_TRANS = 1,
-	VFD_FETCH_INSTRUCTIONS = 3,
-	VFD_BUSY_CYCLES = 5,
-	VFD_STALL_CYCLES_UCHE = 6,
-	VFD_STALL_CYCLES_HLSQ = 7,
-	VFD_STALL_CYCLES_VPC_BYPASS = 8,
-	VFD_STALL_CYCLES_VPC_ALLOC = 9,
-	VFD_MODE_0_FIBERS = 13,
-	VFD_MODE_1_FIBERS = 14,
-	VFD_MODE_2_FIBERS = 15,
-	VFD_MODE_3_FIBERS = 16,
-	VFD_MODE_4_FIBERS = 17,
-	VFD_BFIFO_STALL = 18,
-	VFD_NUM_VERTICES_TOTAL = 19,
-	VFD_PACKER_FULL = 20,
-	VFD_UCHE_REQUEST_FIFO_FULL = 21,
-	VFD_STARVE_CYCLES_PC = 22,
-	VFD_STARVE_CYCLES_UCHE = 23,
-};
-
-enum a4xx_vpc_perfcounter_select {
-	VPC_SP_LM_COMPONENTS = 2,
-	VPC_SP0_LM_BYTES = 3,
-	VPC_SP1_LM_BYTES = 4,
-	VPC_SP2_LM_BYTES = 5,
-	VPC_SP3_LM_BYTES = 6,
-	VPC_WORKING_CYCLES = 7,
-	VPC_STALL_CYCLES_LM = 8,
-	VPC_STARVE_CYCLES_RAS = 9,
-	VPC_STREAMOUT_CYCLES = 10,
-	VPC_UCHE_TRANSACTIONS = 12,
-	VPC_STALL_CYCLES_UCHE = 13,
-	VPC_BUSY_CYCLES = 14,
-	VPC_STARVE_CYCLES_SP = 15,
-};
-
-enum a4xx_vsc_perfcounter_select {
-	VSC_BUSY_CYCLES = 0,
-	VSC_WORKING_CYCLES = 1,
-	VSC_STALL_CYCLES_UCHE = 2,
-	VSC_STARVE_CYCLES_RAS = 3,
-	VSC_EOT_NUM = 4,
-};
-
-enum a4xx_tex_filter {
-	A4XX_TEX_NEAREST = 0,
-	A4XX_TEX_LINEAR = 1,
-	A4XX_TEX_ANISO = 2,
-};
-
-enum a4xx_tex_clamp {
-	A4XX_TEX_REPEAT = 0,
-	A4XX_TEX_CLAMP_TO_EDGE = 1,
-	A4XX_TEX_MIRROR_REPEAT = 2,
-	A4XX_TEX_CLAMP_TO_BORDER = 3,
-	A4XX_TEX_MIRROR_CLAMP = 4,
-};
-
-enum a4xx_tex_aniso {
-	A4XX_TEX_ANISO_1 = 0,
-	A4XX_TEX_ANISO_2 = 1,
-	A4XX_TEX_ANISO_4 = 2,
-	A4XX_TEX_ANISO_8 = 3,
-	A4XX_TEX_ANISO_16 = 4,
-};
-
-enum a4xx_tex_swiz {
-	A4XX_TEX_X = 0,
-	A4XX_TEX_Y = 1,
-	A4XX_TEX_Z = 2,
-	A4XX_TEX_W = 3,
-	A4XX_TEX_ZERO = 4,
-	A4XX_TEX_ONE = 5,
-};
-
-enum a4xx_tex_type {
-	A4XX_TEX_1D = 0,
-	A4XX_TEX_2D = 1,
-	A4XX_TEX_CUBE = 2,
-	A4XX_TEX_3D = 3,
-	A4XX_TEX_BUFFER = 4,
-};
-
-#define A4XX_CGC_HLSQ_EARLY_CYC__MASK				0x00700000
-#define A4XX_CGC_HLSQ_EARLY_CYC__SHIFT				20
-static inline uint32_t A4XX_CGC_HLSQ_EARLY_CYC(uint32_t val)
-{
-	return ((val) << A4XX_CGC_HLSQ_EARLY_CYC__SHIFT) & A4XX_CGC_HLSQ_EARLY_CYC__MASK;
-}
-
-#define A4XX_INT0_RBBM_GPU_IDLE					0x00000001
-#define A4XX_INT0_RBBM_AHB_ERROR				0x00000002
-#define A4XX_INT0_RBBM_REG_TIMEOUT				0x00000004
-#define A4XX_INT0_RBBM_ME_MS_TIMEOUT				0x00000008
-#define A4XX_INT0_RBBM_PFP_MS_TIMEOUT				0x00000010
-#define A4XX_INT0_RBBM_ATB_BUS_OVERFLOW				0x00000020
-#define A4XX_INT0_VFD_ERROR					0x00000040
-#define A4XX_INT0_CP_SW_INT					0x00000080
-#define A4XX_INT0_CP_T0_PACKET_IN_IB				0x00000100
-#define A4XX_INT0_CP_OPCODE_ERROR				0x00000200
-#define A4XX_INT0_CP_RESERVED_BIT_ERROR				0x00000400
-#define A4XX_INT0_CP_HW_FAULT					0x00000800
-#define A4XX_INT0_CP_DMA					0x00001000
-#define A4XX_INT0_CP_IB2_INT					0x00002000
-#define A4XX_INT0_CP_IB1_INT					0x00004000
-#define A4XX_INT0_CP_RB_INT					0x00008000
-#define A4XX_INT0_CP_REG_PROTECT_FAULT				0x00010000
-#define A4XX_INT0_CP_RB_DONE_TS					0x00020000
-#define A4XX_INT0_CP_VS_DONE_TS					0x00040000
-#define A4XX_INT0_CP_PS_DONE_TS					0x00080000
-#define A4XX_INT0_CACHE_FLUSH_TS				0x00100000
-#define A4XX_INT0_CP_AHB_ERROR_HALT				0x00200000
-#define A4XX_INT0_MISC_HANG_DETECT				0x01000000
-#define A4XX_INT0_UCHE_OOB_ACCESS				0x02000000
-
-#define REG_A4XX_RB_GMEM_BASE_ADDR				0x00000cc0
-
-#define REG_A4XX_RB_PERFCTR_RB_SEL_0				0x00000cc7
-
-#define REG_A4XX_RB_PERFCTR_RB_SEL_1				0x00000cc8
-
-#define REG_A4XX_RB_PERFCTR_RB_SEL_2				0x00000cc9
-
-#define REG_A4XX_RB_PERFCTR_RB_SEL_3				0x00000cca
-
-#define REG_A4XX_RB_PERFCTR_RB_SEL_4				0x00000ccb
-
-#define REG_A4XX_RB_PERFCTR_RB_SEL_5				0x00000ccc
-
-#define REG_A4XX_RB_PERFCTR_RB_SEL_6				0x00000ccd
-
-#define REG_A4XX_RB_PERFCTR_RB_SEL_7				0x00000cce
-
-#define REG_A4XX_RB_PERFCTR_CCU_SEL_0				0x00000ccf
-
-#define REG_A4XX_RB_PERFCTR_CCU_SEL_1				0x00000cd0
-
-#define REG_A4XX_RB_PERFCTR_CCU_SEL_2				0x00000cd1
-
-#define REG_A4XX_RB_PERFCTR_CCU_SEL_3				0x00000cd2
-
-#define REG_A4XX_RB_FRAME_BUFFER_DIMENSION			0x00000ce0
-#define A4XX_RB_FRAME_BUFFER_DIMENSION_WIDTH__MASK		0x00003fff
-#define A4XX_RB_FRAME_BUFFER_DIMENSION_WIDTH__SHIFT		0
-static inline uint32_t A4XX_RB_FRAME_BUFFER_DIMENSION_WIDTH(uint32_t val)
-{
-	return ((val) << A4XX_RB_FRAME_BUFFER_DIMENSION_WIDTH__SHIFT) & A4XX_RB_FRAME_BUFFER_DIMENSION_WIDTH__MASK;
-}
-#define A4XX_RB_FRAME_BUFFER_DIMENSION_HEIGHT__MASK		0x3fff0000
-#define A4XX_RB_FRAME_BUFFER_DIMENSION_HEIGHT__SHIFT		16
-static inline uint32_t A4XX_RB_FRAME_BUFFER_DIMENSION_HEIGHT(uint32_t val)
-{
-	return ((val) << A4XX_RB_FRAME_BUFFER_DIMENSION_HEIGHT__SHIFT) & A4XX_RB_FRAME_BUFFER_DIMENSION_HEIGHT__MASK;
-}
-
-#define REG_A4XX_RB_CLEAR_COLOR_DW0				0x000020cc
-
-#define REG_A4XX_RB_CLEAR_COLOR_DW1				0x000020cd
-
-#define REG_A4XX_RB_CLEAR_COLOR_DW2				0x000020ce
-
-#define REG_A4XX_RB_CLEAR_COLOR_DW3				0x000020cf
-
-#define REG_A4XX_RB_MODE_CONTROL				0x000020a0
-#define A4XX_RB_MODE_CONTROL_WIDTH__MASK			0x0000003f
-#define A4XX_RB_MODE_CONTROL_WIDTH__SHIFT			0
-static inline uint32_t A4XX_RB_MODE_CONTROL_WIDTH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A4XX_RB_MODE_CONTROL_WIDTH__SHIFT) & A4XX_RB_MODE_CONTROL_WIDTH__MASK;
-}
-#define A4XX_RB_MODE_CONTROL_HEIGHT__MASK			0x00003f00
-#define A4XX_RB_MODE_CONTROL_HEIGHT__SHIFT			8
-static inline uint32_t A4XX_RB_MODE_CONTROL_HEIGHT(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A4XX_RB_MODE_CONTROL_HEIGHT__SHIFT) & A4XX_RB_MODE_CONTROL_HEIGHT__MASK;
-}
-#define A4XX_RB_MODE_CONTROL_ENABLE_GMEM			0x00010000
-
-#define REG_A4XX_RB_RENDER_CONTROL				0x000020a1
-#define A4XX_RB_RENDER_CONTROL_BINNING_PASS			0x00000001
-#define A4XX_RB_RENDER_CONTROL_DISABLE_COLOR_PIPE		0x00000020
-
-#define REG_A4XX_RB_MSAA_CONTROL				0x000020a2
-#define A4XX_RB_MSAA_CONTROL_DISABLE				0x00001000
-#define A4XX_RB_MSAA_CONTROL_SAMPLES__MASK			0x0000e000
-#define A4XX_RB_MSAA_CONTROL_SAMPLES__SHIFT			13
-static inline uint32_t A4XX_RB_MSAA_CONTROL_SAMPLES(uint32_t val)
-{
-	return ((val) << A4XX_RB_MSAA_CONTROL_SAMPLES__SHIFT) & A4XX_RB_MSAA_CONTROL_SAMPLES__MASK;
-}
-
-#define REG_A4XX_RB_RENDER_CONTROL2				0x000020a3
-#define A4XX_RB_RENDER_CONTROL2_COORD_MASK__MASK		0x0000000f
-#define A4XX_RB_RENDER_CONTROL2_COORD_MASK__SHIFT		0
-static inline uint32_t A4XX_RB_RENDER_CONTROL2_COORD_MASK(uint32_t val)
-{
-	return ((val) << A4XX_RB_RENDER_CONTROL2_COORD_MASK__SHIFT) & A4XX_RB_RENDER_CONTROL2_COORD_MASK__MASK;
-}
-#define A4XX_RB_RENDER_CONTROL2_SAMPLEMASK			0x00000010
-#define A4XX_RB_RENDER_CONTROL2_FACENESS			0x00000020
-#define A4XX_RB_RENDER_CONTROL2_SAMPLEID			0x00000040
-#define A4XX_RB_RENDER_CONTROL2_MSAA_SAMPLES__MASK		0x00000380
-#define A4XX_RB_RENDER_CONTROL2_MSAA_SAMPLES__SHIFT		7
-static inline uint32_t A4XX_RB_RENDER_CONTROL2_MSAA_SAMPLES(uint32_t val)
-{
-	return ((val) << A4XX_RB_RENDER_CONTROL2_MSAA_SAMPLES__SHIFT) & A4XX_RB_RENDER_CONTROL2_MSAA_SAMPLES__MASK;
-}
-#define A4XX_RB_RENDER_CONTROL2_SAMPLEID_HR			0x00000800
-#define A4XX_RB_RENDER_CONTROL2_IJ_PERSP_PIXEL			0x00001000
-#define A4XX_RB_RENDER_CONTROL2_IJ_PERSP_CENTROID		0x00002000
-#define A4XX_RB_RENDER_CONTROL2_IJ_PERSP_SAMPLE			0x00004000
-#define A4XX_RB_RENDER_CONTROL2_SIZE				0x00008000
-
-#define REG_A4XX_RB_MRT(i0) (0x000020a4 + 0x5*(i0))
-
-static inline uint32_t REG_A4XX_RB_MRT_CONTROL(uint32_t i0) { return 0x000020a4 + 0x5*i0; }
-#define A4XX_RB_MRT_CONTROL_READ_DEST_ENABLE			0x00000008
-#define A4XX_RB_MRT_CONTROL_BLEND				0x00000010
-#define A4XX_RB_MRT_CONTROL_BLEND2				0x00000020
-#define A4XX_RB_MRT_CONTROL_ROP_ENABLE				0x00000040
-#define A4XX_RB_MRT_CONTROL_ROP_CODE__MASK			0x00000f00
-#define A4XX_RB_MRT_CONTROL_ROP_CODE__SHIFT			8
-static inline uint32_t A4XX_RB_MRT_CONTROL_ROP_CODE(enum a3xx_rop_code val)
-{
-	return ((val) << A4XX_RB_MRT_CONTROL_ROP_CODE__SHIFT) & A4XX_RB_MRT_CONTROL_ROP_CODE__MASK;
-}
-#define A4XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK		0x0f000000
-#define A4XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT		24
-static inline uint32_t A4XX_RB_MRT_CONTROL_COMPONENT_ENABLE(uint32_t val)
-{
-	return ((val) << A4XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT) & A4XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK;
-}
-
-static inline uint32_t REG_A4XX_RB_MRT_BUF_INFO(uint32_t i0) { return 0x000020a5 + 0x5*i0; }
-#define A4XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK			0x0000003f
-#define A4XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT		0
-static inline uint32_t A4XX_RB_MRT_BUF_INFO_COLOR_FORMAT(enum a4xx_color_fmt val)
-{
-	return ((val) << A4XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT) & A4XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK;
-}
-#define A4XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK		0x000000c0
-#define A4XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT		6
-static inline uint32_t A4XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE(enum a4xx_tile_mode val)
-{
-	return ((val) << A4XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT) & A4XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK;
-}
-#define A4XX_RB_MRT_BUF_INFO_DITHER_MODE__MASK			0x00000600
-#define A4XX_RB_MRT_BUF_INFO_DITHER_MODE__SHIFT			9
-static inline uint32_t A4XX_RB_MRT_BUF_INFO_DITHER_MODE(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A4XX_RB_MRT_BUF_INFO_DITHER_MODE__SHIFT) & A4XX_RB_MRT_BUF_INFO_DITHER_MODE__MASK;
-}
-#define A4XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK			0x00001800
-#define A4XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT			11
-static inline uint32_t A4XX_RB_MRT_BUF_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A4XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT) & A4XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK;
-}
-#define A4XX_RB_MRT_BUF_INFO_COLOR_SRGB				0x00002000
-#define A4XX_RB_MRT_BUF_INFO_COLOR_BUF_PITCH__MASK		0xffffc000
-#define A4XX_RB_MRT_BUF_INFO_COLOR_BUF_PITCH__SHIFT		14
-static inline uint32_t A4XX_RB_MRT_BUF_INFO_COLOR_BUF_PITCH(uint32_t val)
-{
-	assert(!(val & 0xf));
-	return (((val >> 4)) << A4XX_RB_MRT_BUF_INFO_COLOR_BUF_PITCH__SHIFT) & A4XX_RB_MRT_BUF_INFO_COLOR_BUF_PITCH__MASK;
-}
-
-static inline uint32_t REG_A4XX_RB_MRT_BASE(uint32_t i0) { return 0x000020a6 + 0x5*i0; }
-
-static inline uint32_t REG_A4XX_RB_MRT_CONTROL3(uint32_t i0) { return 0x000020a7 + 0x5*i0; }
-#define A4XX_RB_MRT_CONTROL3_STRIDE__MASK			0x03fffff8
-#define A4XX_RB_MRT_CONTROL3_STRIDE__SHIFT			3
-static inline uint32_t A4XX_RB_MRT_CONTROL3_STRIDE(uint32_t val)
-{
-	return ((val) << A4XX_RB_MRT_CONTROL3_STRIDE__SHIFT) & A4XX_RB_MRT_CONTROL3_STRIDE__MASK;
-}
-
-static inline uint32_t REG_A4XX_RB_MRT_BLEND_CONTROL(uint32_t i0) { return 0x000020a8 + 0x5*i0; }
-#define A4XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK		0x0000001f
-#define A4XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT		0
-static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A4XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT) & A4XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK;
-}
-#define A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK	0x000000e0
-#define A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT	5
-static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
-{
-	return ((val) << A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT) & A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK;
-}
-#define A4XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK		0x00001f00
-#define A4XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT	8
-static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A4XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT) & A4XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK;
-}
-#define A4XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK	0x001f0000
-#define A4XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT	16
-static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A4XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT) & A4XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK;
-}
-#define A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK	0x00e00000
-#define A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT	21
-static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
-{
-	return ((val) << A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT) & A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK;
-}
-#define A4XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK	0x1f000000
-#define A4XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT	24
-static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A4XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT) & A4XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK;
-}
-
-#define REG_A4XX_RB_BLEND_RED					0x000020f0
-#define A4XX_RB_BLEND_RED_UINT__MASK				0x000000ff
-#define A4XX_RB_BLEND_RED_UINT__SHIFT				0
-static inline uint32_t A4XX_RB_BLEND_RED_UINT(uint32_t val)
-{
-	return ((val) << A4XX_RB_BLEND_RED_UINT__SHIFT) & A4XX_RB_BLEND_RED_UINT__MASK;
-}
-#define A4XX_RB_BLEND_RED_SINT__MASK				0x0000ff00
-#define A4XX_RB_BLEND_RED_SINT__SHIFT				8
-static inline uint32_t A4XX_RB_BLEND_RED_SINT(uint32_t val)
-{
-	return ((val) << A4XX_RB_BLEND_RED_SINT__SHIFT) & A4XX_RB_BLEND_RED_SINT__MASK;
-}
-#define A4XX_RB_BLEND_RED_FLOAT__MASK				0xffff0000
-#define A4XX_RB_BLEND_RED_FLOAT__SHIFT				16
-static inline uint32_t A4XX_RB_BLEND_RED_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A4XX_RB_BLEND_RED_FLOAT__SHIFT) & A4XX_RB_BLEND_RED_FLOAT__MASK;
-}
-
-#define REG_A4XX_RB_BLEND_RED_F32				0x000020f1
-#define A4XX_RB_BLEND_RED_F32__MASK				0xffffffff
-#define A4XX_RB_BLEND_RED_F32__SHIFT				0
-static inline uint32_t A4XX_RB_BLEND_RED_F32(float val)
-{
-	return ((fui(val)) << A4XX_RB_BLEND_RED_F32__SHIFT) & A4XX_RB_BLEND_RED_F32__MASK;
-}
-
-#define REG_A4XX_RB_BLEND_GREEN					0x000020f2
-#define A4XX_RB_BLEND_GREEN_UINT__MASK				0x000000ff
-#define A4XX_RB_BLEND_GREEN_UINT__SHIFT				0
-static inline uint32_t A4XX_RB_BLEND_GREEN_UINT(uint32_t val)
-{
-	return ((val) << A4XX_RB_BLEND_GREEN_UINT__SHIFT) & A4XX_RB_BLEND_GREEN_UINT__MASK;
-}
-#define A4XX_RB_BLEND_GREEN_SINT__MASK				0x0000ff00
-#define A4XX_RB_BLEND_GREEN_SINT__SHIFT				8
-static inline uint32_t A4XX_RB_BLEND_GREEN_SINT(uint32_t val)
-{
-	return ((val) << A4XX_RB_BLEND_GREEN_SINT__SHIFT) & A4XX_RB_BLEND_GREEN_SINT__MASK;
-}
-#define A4XX_RB_BLEND_GREEN_FLOAT__MASK				0xffff0000
-#define A4XX_RB_BLEND_GREEN_FLOAT__SHIFT			16
-static inline uint32_t A4XX_RB_BLEND_GREEN_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A4XX_RB_BLEND_GREEN_FLOAT__SHIFT) & A4XX_RB_BLEND_GREEN_FLOAT__MASK;
-}
-
-#define REG_A4XX_RB_BLEND_GREEN_F32				0x000020f3
-#define A4XX_RB_BLEND_GREEN_F32__MASK				0xffffffff
-#define A4XX_RB_BLEND_GREEN_F32__SHIFT				0
-static inline uint32_t A4XX_RB_BLEND_GREEN_F32(float val)
-{
-	return ((fui(val)) << A4XX_RB_BLEND_GREEN_F32__SHIFT) & A4XX_RB_BLEND_GREEN_F32__MASK;
-}
-
-#define REG_A4XX_RB_BLEND_BLUE					0x000020f4
-#define A4XX_RB_BLEND_BLUE_UINT__MASK				0x000000ff
-#define A4XX_RB_BLEND_BLUE_UINT__SHIFT				0
-static inline uint32_t A4XX_RB_BLEND_BLUE_UINT(uint32_t val)
-{
-	return ((val) << A4XX_RB_BLEND_BLUE_UINT__SHIFT) & A4XX_RB_BLEND_BLUE_UINT__MASK;
-}
-#define A4XX_RB_BLEND_BLUE_SINT__MASK				0x0000ff00
-#define A4XX_RB_BLEND_BLUE_SINT__SHIFT				8
-static inline uint32_t A4XX_RB_BLEND_BLUE_SINT(uint32_t val)
-{
-	return ((val) << A4XX_RB_BLEND_BLUE_SINT__SHIFT) & A4XX_RB_BLEND_BLUE_SINT__MASK;
-}
-#define A4XX_RB_BLEND_BLUE_FLOAT__MASK				0xffff0000
-#define A4XX_RB_BLEND_BLUE_FLOAT__SHIFT				16
-static inline uint32_t A4XX_RB_BLEND_BLUE_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A4XX_RB_BLEND_BLUE_FLOAT__SHIFT) & A4XX_RB_BLEND_BLUE_FLOAT__MASK;
-}
-
-#define REG_A4XX_RB_BLEND_BLUE_F32				0x000020f5
-#define A4XX_RB_BLEND_BLUE_F32__MASK				0xffffffff
-#define A4XX_RB_BLEND_BLUE_F32__SHIFT				0
-static inline uint32_t A4XX_RB_BLEND_BLUE_F32(float val)
-{
-	return ((fui(val)) << A4XX_RB_BLEND_BLUE_F32__SHIFT) & A4XX_RB_BLEND_BLUE_F32__MASK;
-}
-
-#define REG_A4XX_RB_BLEND_ALPHA					0x000020f6
-#define A4XX_RB_BLEND_ALPHA_UINT__MASK				0x000000ff
-#define A4XX_RB_BLEND_ALPHA_UINT__SHIFT				0
-static inline uint32_t A4XX_RB_BLEND_ALPHA_UINT(uint32_t val)
-{
-	return ((val) << A4XX_RB_BLEND_ALPHA_UINT__SHIFT) & A4XX_RB_BLEND_ALPHA_UINT__MASK;
-}
-#define A4XX_RB_BLEND_ALPHA_SINT__MASK				0x0000ff00
-#define A4XX_RB_BLEND_ALPHA_SINT__SHIFT				8
-static inline uint32_t A4XX_RB_BLEND_ALPHA_SINT(uint32_t val)
-{
-	return ((val) << A4XX_RB_BLEND_ALPHA_SINT__SHIFT) & A4XX_RB_BLEND_ALPHA_SINT__MASK;
-}
-#define A4XX_RB_BLEND_ALPHA_FLOAT__MASK				0xffff0000
-#define A4XX_RB_BLEND_ALPHA_FLOAT__SHIFT			16
-static inline uint32_t A4XX_RB_BLEND_ALPHA_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A4XX_RB_BLEND_ALPHA_FLOAT__SHIFT) & A4XX_RB_BLEND_ALPHA_FLOAT__MASK;
-}
-
-#define REG_A4XX_RB_BLEND_ALPHA_F32				0x000020f7
-#define A4XX_RB_BLEND_ALPHA_F32__MASK				0xffffffff
-#define A4XX_RB_BLEND_ALPHA_F32__SHIFT				0
-static inline uint32_t A4XX_RB_BLEND_ALPHA_F32(float val)
-{
-	return ((fui(val)) << A4XX_RB_BLEND_ALPHA_F32__SHIFT) & A4XX_RB_BLEND_ALPHA_F32__MASK;
-}
-
-#define REG_A4XX_RB_ALPHA_CONTROL				0x000020f8
-#define A4XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK			0x000000ff
-#define A4XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT			0
-static inline uint32_t A4XX_RB_ALPHA_CONTROL_ALPHA_REF(uint32_t val)
-{
-	return ((val) << A4XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT) & A4XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK;
-}
-#define A4XX_RB_ALPHA_CONTROL_ALPHA_TEST			0x00000100
-#define A4XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__MASK		0x00000e00
-#define A4XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__SHIFT		9
-static inline uint32_t A4XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A4XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__SHIFT) & A4XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__MASK;
-}
-
-#define REG_A4XX_RB_FS_OUTPUT					0x000020f9
-#define A4XX_RB_FS_OUTPUT_ENABLE_BLEND__MASK			0x000000ff
-#define A4XX_RB_FS_OUTPUT_ENABLE_BLEND__SHIFT			0
-static inline uint32_t A4XX_RB_FS_OUTPUT_ENABLE_BLEND(uint32_t val)
-{
-	return ((val) << A4XX_RB_FS_OUTPUT_ENABLE_BLEND__SHIFT) & A4XX_RB_FS_OUTPUT_ENABLE_BLEND__MASK;
-}
-#define A4XX_RB_FS_OUTPUT_INDEPENDENT_BLEND			0x00000100
-#define A4XX_RB_FS_OUTPUT_SAMPLE_MASK__MASK			0xffff0000
-#define A4XX_RB_FS_OUTPUT_SAMPLE_MASK__SHIFT			16
-static inline uint32_t A4XX_RB_FS_OUTPUT_SAMPLE_MASK(uint32_t val)
-{
-	return ((val) << A4XX_RB_FS_OUTPUT_SAMPLE_MASK__SHIFT) & A4XX_RB_FS_OUTPUT_SAMPLE_MASK__MASK;
-}
-
-#define REG_A4XX_RB_SAMPLE_COUNT_CONTROL			0x000020fa
-#define A4XX_RB_SAMPLE_COUNT_CONTROL_COPY			0x00000002
-#define A4XX_RB_SAMPLE_COUNT_CONTROL_ADDR__MASK			0xfffffffc
-#define A4XX_RB_SAMPLE_COUNT_CONTROL_ADDR__SHIFT		2
-static inline uint32_t A4XX_RB_SAMPLE_COUNT_CONTROL_ADDR(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A4XX_RB_SAMPLE_COUNT_CONTROL_ADDR__SHIFT) & A4XX_RB_SAMPLE_COUNT_CONTROL_ADDR__MASK;
-}
-
-#define REG_A4XX_RB_RENDER_COMPONENTS				0x000020fb
-#define A4XX_RB_RENDER_COMPONENTS_RT0__MASK			0x0000000f
-#define A4XX_RB_RENDER_COMPONENTS_RT0__SHIFT			0
-static inline uint32_t A4XX_RB_RENDER_COMPONENTS_RT0(uint32_t val)
-{
-	return ((val) << A4XX_RB_RENDER_COMPONENTS_RT0__SHIFT) & A4XX_RB_RENDER_COMPONENTS_RT0__MASK;
-}
-#define A4XX_RB_RENDER_COMPONENTS_RT1__MASK			0x000000f0
-#define A4XX_RB_RENDER_COMPONENTS_RT1__SHIFT			4
-static inline uint32_t A4XX_RB_RENDER_COMPONENTS_RT1(uint32_t val)
-{
-	return ((val) << A4XX_RB_RENDER_COMPONENTS_RT1__SHIFT) & A4XX_RB_RENDER_COMPONENTS_RT1__MASK;
-}
-#define A4XX_RB_RENDER_COMPONENTS_RT2__MASK			0x00000f00
-#define A4XX_RB_RENDER_COMPONENTS_RT2__SHIFT			8
-static inline uint32_t A4XX_RB_RENDER_COMPONENTS_RT2(uint32_t val)
-{
-	return ((val) << A4XX_RB_RENDER_COMPONENTS_RT2__SHIFT) & A4XX_RB_RENDER_COMPONENTS_RT2__MASK;
-}
-#define A4XX_RB_RENDER_COMPONENTS_RT3__MASK			0x0000f000
-#define A4XX_RB_RENDER_COMPONENTS_RT3__SHIFT			12
-static inline uint32_t A4XX_RB_RENDER_COMPONENTS_RT3(uint32_t val)
-{
-	return ((val) << A4XX_RB_RENDER_COMPONENTS_RT3__SHIFT) & A4XX_RB_RENDER_COMPONENTS_RT3__MASK;
-}
-#define A4XX_RB_RENDER_COMPONENTS_RT4__MASK			0x000f0000
-#define A4XX_RB_RENDER_COMPONENTS_RT4__SHIFT			16
-static inline uint32_t A4XX_RB_RENDER_COMPONENTS_RT4(uint32_t val)
-{
-	return ((val) << A4XX_RB_RENDER_COMPONENTS_RT4__SHIFT) & A4XX_RB_RENDER_COMPONENTS_RT4__MASK;
-}
-#define A4XX_RB_RENDER_COMPONENTS_RT5__MASK			0x00f00000
-#define A4XX_RB_RENDER_COMPONENTS_RT5__SHIFT			20
-static inline uint32_t A4XX_RB_RENDER_COMPONENTS_RT5(uint32_t val)
-{
-	return ((val) << A4XX_RB_RENDER_COMPONENTS_RT5__SHIFT) & A4XX_RB_RENDER_COMPONENTS_RT5__MASK;
-}
-#define A4XX_RB_RENDER_COMPONENTS_RT6__MASK			0x0f000000
-#define A4XX_RB_RENDER_COMPONENTS_RT6__SHIFT			24
-static inline uint32_t A4XX_RB_RENDER_COMPONENTS_RT6(uint32_t val)
-{
-	return ((val) << A4XX_RB_RENDER_COMPONENTS_RT6__SHIFT) & A4XX_RB_RENDER_COMPONENTS_RT6__MASK;
-}
-#define A4XX_RB_RENDER_COMPONENTS_RT7__MASK			0xf0000000
-#define A4XX_RB_RENDER_COMPONENTS_RT7__SHIFT			28
-static inline uint32_t A4XX_RB_RENDER_COMPONENTS_RT7(uint32_t val)
-{
-	return ((val) << A4XX_RB_RENDER_COMPONENTS_RT7__SHIFT) & A4XX_RB_RENDER_COMPONENTS_RT7__MASK;
-}
-
-#define REG_A4XX_RB_COPY_CONTROL				0x000020fc
-#define A4XX_RB_COPY_CONTROL_MSAA_RESOLVE__MASK			0x00000003
-#define A4XX_RB_COPY_CONTROL_MSAA_RESOLVE__SHIFT		0
-static inline uint32_t A4XX_RB_COPY_CONTROL_MSAA_RESOLVE(enum a3xx_msaa_samples val)
-{
-	return ((val) << A4XX_RB_COPY_CONTROL_MSAA_RESOLVE__SHIFT) & A4XX_RB_COPY_CONTROL_MSAA_RESOLVE__MASK;
-}
-#define A4XX_RB_COPY_CONTROL_MODE__MASK				0x00000070
-#define A4XX_RB_COPY_CONTROL_MODE__SHIFT			4
-static inline uint32_t A4XX_RB_COPY_CONTROL_MODE(enum adreno_rb_copy_control_mode val)
-{
-	return ((val) << A4XX_RB_COPY_CONTROL_MODE__SHIFT) & A4XX_RB_COPY_CONTROL_MODE__MASK;
-}
-#define A4XX_RB_COPY_CONTROL_FASTCLEAR__MASK			0x00000f00
-#define A4XX_RB_COPY_CONTROL_FASTCLEAR__SHIFT			8
-static inline uint32_t A4XX_RB_COPY_CONTROL_FASTCLEAR(uint32_t val)
-{
-	return ((val) << A4XX_RB_COPY_CONTROL_FASTCLEAR__SHIFT) & A4XX_RB_COPY_CONTROL_FASTCLEAR__MASK;
-}
-#define A4XX_RB_COPY_CONTROL_GMEM_BASE__MASK			0xffffc000
-#define A4XX_RB_COPY_CONTROL_GMEM_BASE__SHIFT			14
-static inline uint32_t A4XX_RB_COPY_CONTROL_GMEM_BASE(uint32_t val)
-{
-	assert(!(val & 0x3fff));
-	return (((val >> 14)) << A4XX_RB_COPY_CONTROL_GMEM_BASE__SHIFT) & A4XX_RB_COPY_CONTROL_GMEM_BASE__MASK;
-}
-
-#define REG_A4XX_RB_COPY_DEST_BASE				0x000020fd
-#define A4XX_RB_COPY_DEST_BASE_BASE__MASK			0xffffffe0
-#define A4XX_RB_COPY_DEST_BASE_BASE__SHIFT			5
-static inline uint32_t A4XX_RB_COPY_DEST_BASE_BASE(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A4XX_RB_COPY_DEST_BASE_BASE__SHIFT) & A4XX_RB_COPY_DEST_BASE_BASE__MASK;
-}
-
-#define REG_A4XX_RB_COPY_DEST_PITCH				0x000020fe
-#define A4XX_RB_COPY_DEST_PITCH_PITCH__MASK			0xffffffff
-#define A4XX_RB_COPY_DEST_PITCH_PITCH__SHIFT			0
-static inline uint32_t A4XX_RB_COPY_DEST_PITCH_PITCH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A4XX_RB_COPY_DEST_PITCH_PITCH__SHIFT) & A4XX_RB_COPY_DEST_PITCH_PITCH__MASK;
-}
-
-#define REG_A4XX_RB_COPY_DEST_INFO				0x000020ff
-#define A4XX_RB_COPY_DEST_INFO_FORMAT__MASK			0x000000fc
-#define A4XX_RB_COPY_DEST_INFO_FORMAT__SHIFT			2
-static inline uint32_t A4XX_RB_COPY_DEST_INFO_FORMAT(enum a4xx_color_fmt val)
-{
-	return ((val) << A4XX_RB_COPY_DEST_INFO_FORMAT__SHIFT) & A4XX_RB_COPY_DEST_INFO_FORMAT__MASK;
-}
-#define A4XX_RB_COPY_DEST_INFO_SWAP__MASK			0x00000300
-#define A4XX_RB_COPY_DEST_INFO_SWAP__SHIFT			8
-static inline uint32_t A4XX_RB_COPY_DEST_INFO_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A4XX_RB_COPY_DEST_INFO_SWAP__SHIFT) & A4XX_RB_COPY_DEST_INFO_SWAP__MASK;
-}
-#define A4XX_RB_COPY_DEST_INFO_DITHER_MODE__MASK		0x00000c00
-#define A4XX_RB_COPY_DEST_INFO_DITHER_MODE__SHIFT		10
-static inline uint32_t A4XX_RB_COPY_DEST_INFO_DITHER_MODE(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A4XX_RB_COPY_DEST_INFO_DITHER_MODE__SHIFT) & A4XX_RB_COPY_DEST_INFO_DITHER_MODE__MASK;
-}
-#define A4XX_RB_COPY_DEST_INFO_COMPONENT_ENABLE__MASK		0x0003c000
-#define A4XX_RB_COPY_DEST_INFO_COMPONENT_ENABLE__SHIFT		14
-static inline uint32_t A4XX_RB_COPY_DEST_INFO_COMPONENT_ENABLE(uint32_t val)
-{
-	return ((val) << A4XX_RB_COPY_DEST_INFO_COMPONENT_ENABLE__SHIFT) & A4XX_RB_COPY_DEST_INFO_COMPONENT_ENABLE__MASK;
-}
-#define A4XX_RB_COPY_DEST_INFO_ENDIAN__MASK			0x001c0000
-#define A4XX_RB_COPY_DEST_INFO_ENDIAN__SHIFT			18
-static inline uint32_t A4XX_RB_COPY_DEST_INFO_ENDIAN(enum adreno_rb_surface_endian val)
-{
-	return ((val) << A4XX_RB_COPY_DEST_INFO_ENDIAN__SHIFT) & A4XX_RB_COPY_DEST_INFO_ENDIAN__MASK;
-}
-#define A4XX_RB_COPY_DEST_INFO_TILE__MASK			0x03000000
-#define A4XX_RB_COPY_DEST_INFO_TILE__SHIFT			24
-static inline uint32_t A4XX_RB_COPY_DEST_INFO_TILE(enum a4xx_tile_mode val)
-{
-	return ((val) << A4XX_RB_COPY_DEST_INFO_TILE__SHIFT) & A4XX_RB_COPY_DEST_INFO_TILE__MASK;
-}
-
-#define REG_A4XX_RB_FS_OUTPUT_REG				0x00002100
-#define A4XX_RB_FS_OUTPUT_REG_MRT__MASK				0x0000000f
-#define A4XX_RB_FS_OUTPUT_REG_MRT__SHIFT			0
-static inline uint32_t A4XX_RB_FS_OUTPUT_REG_MRT(uint32_t val)
-{
-	return ((val) << A4XX_RB_FS_OUTPUT_REG_MRT__SHIFT) & A4XX_RB_FS_OUTPUT_REG_MRT__MASK;
-}
-#define A4XX_RB_FS_OUTPUT_REG_FRAG_WRITES_Z			0x00000020
-
-#define REG_A4XX_RB_DEPTH_CONTROL				0x00002101
-#define A4XX_RB_DEPTH_CONTROL_FRAG_WRITES_Z			0x00000001
-#define A4XX_RB_DEPTH_CONTROL_Z_TEST_ENABLE			0x00000002
-#define A4XX_RB_DEPTH_CONTROL_Z_WRITE_ENABLE			0x00000004
-#define A4XX_RB_DEPTH_CONTROL_ZFUNC__MASK			0x00000070
-#define A4XX_RB_DEPTH_CONTROL_ZFUNC__SHIFT			4
-static inline uint32_t A4XX_RB_DEPTH_CONTROL_ZFUNC(enum adreno_compare_func val)
-{
-	return ((val) << A4XX_RB_DEPTH_CONTROL_ZFUNC__SHIFT) & A4XX_RB_DEPTH_CONTROL_ZFUNC__MASK;
-}
-#define A4XX_RB_DEPTH_CONTROL_Z_CLAMP_ENABLE			0x00000080
-#define A4XX_RB_DEPTH_CONTROL_EARLY_Z_DISABLE			0x00010000
-#define A4XX_RB_DEPTH_CONTROL_FORCE_FRAGZ_TO_FS			0x00020000
-#define A4XX_RB_DEPTH_CONTROL_Z_READ_ENABLE			0x80000000
-
-#define REG_A4XX_RB_DEPTH_CLEAR					0x00002102
-
-#define REG_A4XX_RB_DEPTH_INFO					0x00002103
-#define A4XX_RB_DEPTH_INFO_DEPTH_FORMAT__MASK			0x00000003
-#define A4XX_RB_DEPTH_INFO_DEPTH_FORMAT__SHIFT			0
-static inline uint32_t A4XX_RB_DEPTH_INFO_DEPTH_FORMAT(enum a4xx_depth_format val)
-{
-	return ((val) << A4XX_RB_DEPTH_INFO_DEPTH_FORMAT__SHIFT) & A4XX_RB_DEPTH_INFO_DEPTH_FORMAT__MASK;
-}
-#define A4XX_RB_DEPTH_INFO_DEPTH_BASE__MASK			0xfffff000
-#define A4XX_RB_DEPTH_INFO_DEPTH_BASE__SHIFT			12
-static inline uint32_t A4XX_RB_DEPTH_INFO_DEPTH_BASE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A4XX_RB_DEPTH_INFO_DEPTH_BASE__SHIFT) & A4XX_RB_DEPTH_INFO_DEPTH_BASE__MASK;
-}
-
-#define REG_A4XX_RB_DEPTH_PITCH					0x00002104
-#define A4XX_RB_DEPTH_PITCH__MASK				0xffffffff
-#define A4XX_RB_DEPTH_PITCH__SHIFT				0
-static inline uint32_t A4XX_RB_DEPTH_PITCH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A4XX_RB_DEPTH_PITCH__SHIFT) & A4XX_RB_DEPTH_PITCH__MASK;
-}
-
-#define REG_A4XX_RB_DEPTH_PITCH2				0x00002105
-#define A4XX_RB_DEPTH_PITCH2__MASK				0xffffffff
-#define A4XX_RB_DEPTH_PITCH2__SHIFT				0
-static inline uint32_t A4XX_RB_DEPTH_PITCH2(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A4XX_RB_DEPTH_PITCH2__SHIFT) & A4XX_RB_DEPTH_PITCH2__MASK;
-}
-
-#define REG_A4XX_RB_STENCIL_CONTROL				0x00002106
-#define A4XX_RB_STENCIL_CONTROL_STENCIL_ENABLE			0x00000001
-#define A4XX_RB_STENCIL_CONTROL_STENCIL_ENABLE_BF		0x00000002
-#define A4XX_RB_STENCIL_CONTROL_STENCIL_READ			0x00000004
-#define A4XX_RB_STENCIL_CONTROL_FUNC__MASK			0x00000700
-#define A4XX_RB_STENCIL_CONTROL_FUNC__SHIFT			8
-static inline uint32_t A4XX_RB_STENCIL_CONTROL_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A4XX_RB_STENCIL_CONTROL_FUNC__SHIFT) & A4XX_RB_STENCIL_CONTROL_FUNC__MASK;
-}
-#define A4XX_RB_STENCIL_CONTROL_FAIL__MASK			0x00003800
-#define A4XX_RB_STENCIL_CONTROL_FAIL__SHIFT			11
-static inline uint32_t A4XX_RB_STENCIL_CONTROL_FAIL(enum adreno_stencil_op val)
-{
-	return ((val) << A4XX_RB_STENCIL_CONTROL_FAIL__SHIFT) & A4XX_RB_STENCIL_CONTROL_FAIL__MASK;
-}
-#define A4XX_RB_STENCIL_CONTROL_ZPASS__MASK			0x0001c000
-#define A4XX_RB_STENCIL_CONTROL_ZPASS__SHIFT			14
-static inline uint32_t A4XX_RB_STENCIL_CONTROL_ZPASS(enum adreno_stencil_op val)
-{
-	return ((val) << A4XX_RB_STENCIL_CONTROL_ZPASS__SHIFT) & A4XX_RB_STENCIL_CONTROL_ZPASS__MASK;
-}
-#define A4XX_RB_STENCIL_CONTROL_ZFAIL__MASK			0x000e0000
-#define A4XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT			17
-static inline uint32_t A4XX_RB_STENCIL_CONTROL_ZFAIL(enum adreno_stencil_op val)
-{
-	return ((val) << A4XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT) & A4XX_RB_STENCIL_CONTROL_ZFAIL__MASK;
-}
-#define A4XX_RB_STENCIL_CONTROL_FUNC_BF__MASK			0x00700000
-#define A4XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT			20
-static inline uint32_t A4XX_RB_STENCIL_CONTROL_FUNC_BF(enum adreno_compare_func val)
-{
-	return ((val) << A4XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT) & A4XX_RB_STENCIL_CONTROL_FUNC_BF__MASK;
-}
-#define A4XX_RB_STENCIL_CONTROL_FAIL_BF__MASK			0x03800000
-#define A4XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT			23
-static inline uint32_t A4XX_RB_STENCIL_CONTROL_FAIL_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A4XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT) & A4XX_RB_STENCIL_CONTROL_FAIL_BF__MASK;
-}
-#define A4XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK			0x1c000000
-#define A4XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT			26
-static inline uint32_t A4XX_RB_STENCIL_CONTROL_ZPASS_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A4XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT) & A4XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK;
-}
-#define A4XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK			0xe0000000
-#define A4XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT			29
-static inline uint32_t A4XX_RB_STENCIL_CONTROL_ZFAIL_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A4XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT) & A4XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK;
-}
-
-#define REG_A4XX_RB_STENCIL_CONTROL2				0x00002107
-#define A4XX_RB_STENCIL_CONTROL2_STENCIL_BUFFER			0x00000001
-
-#define REG_A4XX_RB_STENCIL_INFO				0x00002108
-#define A4XX_RB_STENCIL_INFO_SEPARATE_STENCIL			0x00000001
-#define A4XX_RB_STENCIL_INFO_STENCIL_BASE__MASK			0xfffff000
-#define A4XX_RB_STENCIL_INFO_STENCIL_BASE__SHIFT		12
-static inline uint32_t A4XX_RB_STENCIL_INFO_STENCIL_BASE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A4XX_RB_STENCIL_INFO_STENCIL_BASE__SHIFT) & A4XX_RB_STENCIL_INFO_STENCIL_BASE__MASK;
-}
-
-#define REG_A4XX_RB_STENCIL_PITCH				0x00002109
-#define A4XX_RB_STENCIL_PITCH__MASK				0xffffffff
-#define A4XX_RB_STENCIL_PITCH__SHIFT				0
-static inline uint32_t A4XX_RB_STENCIL_PITCH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A4XX_RB_STENCIL_PITCH__SHIFT) & A4XX_RB_STENCIL_PITCH__MASK;
-}
-
-#define REG_A4XX_RB_STENCILREFMASK				0x0000210b
-#define A4XX_RB_STENCILREFMASK_STENCILREF__MASK			0x000000ff
-#define A4XX_RB_STENCILREFMASK_STENCILREF__SHIFT		0
-static inline uint32_t A4XX_RB_STENCILREFMASK_STENCILREF(uint32_t val)
-{
-	return ((val) << A4XX_RB_STENCILREFMASK_STENCILREF__SHIFT) & A4XX_RB_STENCILREFMASK_STENCILREF__MASK;
-}
-#define A4XX_RB_STENCILREFMASK_STENCILMASK__MASK		0x0000ff00
-#define A4XX_RB_STENCILREFMASK_STENCILMASK__SHIFT		8
-static inline uint32_t A4XX_RB_STENCILREFMASK_STENCILMASK(uint32_t val)
-{
-	return ((val) << A4XX_RB_STENCILREFMASK_STENCILMASK__SHIFT) & A4XX_RB_STENCILREFMASK_STENCILMASK__MASK;
-}
-#define A4XX_RB_STENCILREFMASK_STENCILWRITEMASK__MASK		0x00ff0000
-#define A4XX_RB_STENCILREFMASK_STENCILWRITEMASK__SHIFT		16
-static inline uint32_t A4XX_RB_STENCILREFMASK_STENCILWRITEMASK(uint32_t val)
-{
-	return ((val) << A4XX_RB_STENCILREFMASK_STENCILWRITEMASK__SHIFT) & A4XX_RB_STENCILREFMASK_STENCILWRITEMASK__MASK;
-}
-
-#define REG_A4XX_RB_STENCILREFMASK_BF				0x0000210c
-#define A4XX_RB_STENCILREFMASK_BF_STENCILREF__MASK		0x000000ff
-#define A4XX_RB_STENCILREFMASK_BF_STENCILREF__SHIFT		0
-static inline uint32_t A4XX_RB_STENCILREFMASK_BF_STENCILREF(uint32_t val)
-{
-	return ((val) << A4XX_RB_STENCILREFMASK_BF_STENCILREF__SHIFT) & A4XX_RB_STENCILREFMASK_BF_STENCILREF__MASK;
-}
-#define A4XX_RB_STENCILREFMASK_BF_STENCILMASK__MASK		0x0000ff00
-#define A4XX_RB_STENCILREFMASK_BF_STENCILMASK__SHIFT		8
-static inline uint32_t A4XX_RB_STENCILREFMASK_BF_STENCILMASK(uint32_t val)
-{
-	return ((val) << A4XX_RB_STENCILREFMASK_BF_STENCILMASK__SHIFT) & A4XX_RB_STENCILREFMASK_BF_STENCILMASK__MASK;
-}
-#define A4XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__MASK	0x00ff0000
-#define A4XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__SHIFT	16
-static inline uint32_t A4XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK(uint32_t val)
-{
-	return ((val) << A4XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__SHIFT) & A4XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__MASK;
-}
-
-#define REG_A4XX_RB_BIN_OFFSET					0x0000210d
-#define A4XX_RB_BIN_OFFSET_WINDOW_OFFSET_DISABLE		0x80000000
-#define A4XX_RB_BIN_OFFSET_X__MASK				0x00007fff
-#define A4XX_RB_BIN_OFFSET_X__SHIFT				0
-static inline uint32_t A4XX_RB_BIN_OFFSET_X(uint32_t val)
-{
-	return ((val) << A4XX_RB_BIN_OFFSET_X__SHIFT) & A4XX_RB_BIN_OFFSET_X__MASK;
-}
-#define A4XX_RB_BIN_OFFSET_Y__MASK				0x7fff0000
-#define A4XX_RB_BIN_OFFSET_Y__SHIFT				16
-static inline uint32_t A4XX_RB_BIN_OFFSET_Y(uint32_t val)
-{
-	return ((val) << A4XX_RB_BIN_OFFSET_Y__SHIFT) & A4XX_RB_BIN_OFFSET_Y__MASK;
-}
-
-#define REG_A4XX_RB_VPORT_Z_CLAMP(i0) (0x00002120 + 0x2*(i0))
-
-static inline uint32_t REG_A4XX_RB_VPORT_Z_CLAMP_MIN(uint32_t i0) { return 0x00002120 + 0x2*i0; }
-
-static inline uint32_t REG_A4XX_RB_VPORT_Z_CLAMP_MAX(uint32_t i0) { return 0x00002121 + 0x2*i0; }
-
-#define REG_A4XX_RBBM_HW_VERSION				0x00000000
-
-#define REG_A4XX_RBBM_HW_CONFIGURATION				0x00000002
-
-#define REG_A4XX_RBBM_CLOCK_CTL_TP(i0) (0x00000004 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_CTL_TP_REG(uint32_t i0) { return 0x00000004 + 0x1*i0; }
-
-#define REG_A4XX_RBBM_CLOCK_CTL2_TP(i0) (0x00000008 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_CTL2_TP_REG(uint32_t i0) { return 0x00000008 + 0x1*i0; }
-
-#define REG_A4XX_RBBM_CLOCK_HYST_TP(i0) (0x0000000c + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_HYST_TP_REG(uint32_t i0) { return 0x0000000c + 0x1*i0; }
-
-#define REG_A4XX_RBBM_CLOCK_DELAY_TP(i0) (0x00000010 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_DELAY_TP_REG(uint32_t i0) { return 0x00000010 + 0x1*i0; }
-
-#define REG_A4XX_RBBM_CLOCK_CTL_UCHE 				0x00000014
-
-#define REG_A4XX_RBBM_CLOCK_CTL2_UCHE				0x00000015
-
-#define REG_A4XX_RBBM_CLOCK_CTL3_UCHE				0x00000016
-
-#define REG_A4XX_RBBM_CLOCK_CTL4_UCHE				0x00000017
-
-#define REG_A4XX_RBBM_CLOCK_HYST_UCHE				0x00000018
-
-#define REG_A4XX_RBBM_CLOCK_DELAY_UCHE				0x00000019
-
-#define REG_A4XX_RBBM_CLOCK_MODE_GPC				0x0000001a
-
-#define REG_A4XX_RBBM_CLOCK_DELAY_GPC				0x0000001b
-
-#define REG_A4XX_RBBM_CLOCK_HYST_GPC				0x0000001c
-
-#define REG_A4XX_RBBM_CLOCK_CTL_TSE_RAS_RBBM			0x0000001d
-
-#define REG_A4XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM			0x0000001e
-
-#define REG_A4XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM			0x0000001f
-
-#define REG_A4XX_RBBM_CLOCK_CTL					0x00000020
-
-#define REG_A4XX_RBBM_SP_HYST_CNT				0x00000021
-
-#define REG_A4XX_RBBM_SW_RESET_CMD				0x00000022
-
-#define REG_A4XX_RBBM_AHB_CTL0					0x00000023
-
-#define REG_A4XX_RBBM_AHB_CTL1					0x00000024
-
-#define REG_A4XX_RBBM_AHB_CMD					0x00000025
-
-#define REG_A4XX_RBBM_RB_SUB_BLOCK_SEL_CTL			0x00000026
-
-#define REG_A4XX_RBBM_RAM_ACC_63_32				0x00000028
-
-#define REG_A4XX_RBBM_WAIT_IDLE_CLOCKS_CTL			0x0000002b
-
-#define REG_A4XX_RBBM_INTERFACE_HANG_INT_CTL			0x0000002f
-
-#define REG_A4XX_RBBM_INTERFACE_HANG_MASK_CTL4			0x00000034
-
-#define REG_A4XX_RBBM_INT_CLEAR_CMD				0x00000036
-
-#define REG_A4XX_RBBM_INT_0_MASK				0x00000037
-
-#define REG_A4XX_RBBM_RBBM_CTL					0x0000003e
-
-#define REG_A4XX_RBBM_AHB_DEBUG_CTL				0x0000003f
-
-#define REG_A4XX_RBBM_VBIF_DEBUG_CTL				0x00000041
-
-#define REG_A4XX_RBBM_CLOCK_CTL2				0x00000042
-
-#define REG_A4XX_RBBM_BLOCK_SW_RESET_CMD			0x00000045
-
-#define REG_A4XX_RBBM_RESET_CYCLES				0x00000047
-
-#define REG_A4XX_RBBM_EXT_TRACE_BUS_CTL				0x00000049
-
-#define REG_A4XX_RBBM_CFG_DEBBUS_SEL_A				0x0000004a
-
-#define REG_A4XX_RBBM_CFG_DEBBUS_SEL_B				0x0000004b
-
-#define REG_A4XX_RBBM_CFG_DEBBUS_SEL_C				0x0000004c
-
-#define REG_A4XX_RBBM_CFG_DEBBUS_SEL_D				0x0000004d
-
-#define REG_A4XX_RBBM_POWER_CNTL_IP				0x00000098
-#define A4XX_RBBM_POWER_CNTL_IP_SW_COLLAPSE			0x00000001
-#define A4XX_RBBM_POWER_CNTL_IP_SP_TP_PWR_ON			0x00100000
-
-#define REG_A4XX_RBBM_PERFCTR_CP_0_LO				0x0000009c
-
-#define REG_A4XX_RBBM_PERFCTR_CP_0_HI				0x0000009d
-
-#define REG_A4XX_RBBM_PERFCTR_CP_1_LO				0x0000009e
-
-#define REG_A4XX_RBBM_PERFCTR_CP_1_HI				0x0000009f
-
-#define REG_A4XX_RBBM_PERFCTR_CP_2_LO				0x000000a0
-
-#define REG_A4XX_RBBM_PERFCTR_CP_2_HI				0x000000a1
-
-#define REG_A4XX_RBBM_PERFCTR_CP_3_LO				0x000000a2
-
-#define REG_A4XX_RBBM_PERFCTR_CP_3_HI				0x000000a3
-
-#define REG_A4XX_RBBM_PERFCTR_CP_4_LO				0x000000a4
-
-#define REG_A4XX_RBBM_PERFCTR_CP_4_HI				0x000000a5
-
-#define REG_A4XX_RBBM_PERFCTR_CP_5_LO				0x000000a6
-
-#define REG_A4XX_RBBM_PERFCTR_CP_5_HI				0x000000a7
-
-#define REG_A4XX_RBBM_PERFCTR_CP_6_LO				0x000000a8
-
-#define REG_A4XX_RBBM_PERFCTR_CP_6_HI				0x000000a9
-
-#define REG_A4XX_RBBM_PERFCTR_CP_7_LO				0x000000aa
-
-#define REG_A4XX_RBBM_PERFCTR_CP_7_HI				0x000000ab
-
-#define REG_A4XX_RBBM_PERFCTR_RBBM_0_LO				0x000000ac
-
-#define REG_A4XX_RBBM_PERFCTR_RBBM_0_HI				0x000000ad
-
-#define REG_A4XX_RBBM_PERFCTR_RBBM_1_LO				0x000000ae
-
-#define REG_A4XX_RBBM_PERFCTR_RBBM_1_HI				0x000000af
-
-#define REG_A4XX_RBBM_PERFCTR_RBBM_2_LO				0x000000b0
-
-#define REG_A4XX_RBBM_PERFCTR_RBBM_2_HI				0x000000b1
-
-#define REG_A4XX_RBBM_PERFCTR_RBBM_3_LO				0x000000b2
-
-#define REG_A4XX_RBBM_PERFCTR_RBBM_3_HI				0x000000b3
-
-#define REG_A4XX_RBBM_PERFCTR_PC_0_LO				0x000000b4
-
-#define REG_A4XX_RBBM_PERFCTR_PC_0_HI				0x000000b5
-
-#define REG_A4XX_RBBM_PERFCTR_PC_1_LO				0x000000b6
-
-#define REG_A4XX_RBBM_PERFCTR_PC_1_HI				0x000000b7
-
-#define REG_A4XX_RBBM_PERFCTR_PC_2_LO				0x000000b8
-
-#define REG_A4XX_RBBM_PERFCTR_PC_2_HI				0x000000b9
-
-#define REG_A4XX_RBBM_PERFCTR_PC_3_LO				0x000000ba
-
-#define REG_A4XX_RBBM_PERFCTR_PC_3_HI				0x000000bb
-
-#define REG_A4XX_RBBM_PERFCTR_PC_4_LO				0x000000bc
-
-#define REG_A4XX_RBBM_PERFCTR_PC_4_HI				0x000000bd
-
-#define REG_A4XX_RBBM_PERFCTR_PC_5_LO				0x000000be
-
-#define REG_A4XX_RBBM_PERFCTR_PC_5_HI				0x000000bf
-
-#define REG_A4XX_RBBM_PERFCTR_PC_6_LO				0x000000c0
-
-#define REG_A4XX_RBBM_PERFCTR_PC_6_HI				0x000000c1
-
-#define REG_A4XX_RBBM_PERFCTR_PC_7_LO				0x000000c2
-
-#define REG_A4XX_RBBM_PERFCTR_PC_7_HI				0x000000c3
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_0_LO				0x000000c4
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_0_HI				0x000000c5
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_1_LO				0x000000c6
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_1_HI				0x000000c7
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_2_LO				0x000000c8
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_2_HI				0x000000c9
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_3_LO				0x000000ca
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_3_HI				0x000000cb
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_4_LO				0x000000cc
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_4_HI				0x000000cd
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_5_LO				0x000000ce
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_5_HI				0x000000cf
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_6_LO				0x000000d0
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_6_HI				0x000000d1
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_7_LO				0x000000d2
-
-#define REG_A4XX_RBBM_PERFCTR_VFD_7_HI				0x000000d3
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_0_LO				0x000000d4
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_0_HI				0x000000d5
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_1_LO				0x000000d6
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_1_HI				0x000000d7
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_2_LO				0x000000d8
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_2_HI				0x000000d9
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_3_LO				0x000000da
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_3_HI				0x000000db
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_4_LO				0x000000dc
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_4_HI				0x000000dd
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_5_LO				0x000000de
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_5_HI				0x000000df
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_6_LO				0x000000e0
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_6_HI				0x000000e1
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_7_LO				0x000000e2
-
-#define REG_A4XX_RBBM_PERFCTR_HLSQ_7_HI				0x000000e3
-
-#define REG_A4XX_RBBM_PERFCTR_VPC_0_LO				0x000000e4
-
-#define REG_A4XX_RBBM_PERFCTR_VPC_0_HI				0x000000e5
-
-#define REG_A4XX_RBBM_PERFCTR_VPC_1_LO				0x000000e6
-
-#define REG_A4XX_RBBM_PERFCTR_VPC_1_HI				0x000000e7
-
-#define REG_A4XX_RBBM_PERFCTR_VPC_2_LO				0x000000e8
-
-#define REG_A4XX_RBBM_PERFCTR_VPC_2_HI				0x000000e9
-
-#define REG_A4XX_RBBM_PERFCTR_VPC_3_LO				0x000000ea
-
-#define REG_A4XX_RBBM_PERFCTR_VPC_3_HI				0x000000eb
-
-#define REG_A4XX_RBBM_PERFCTR_CCU_0_LO				0x000000ec
-
-#define REG_A4XX_RBBM_PERFCTR_CCU_0_HI				0x000000ed
-
-#define REG_A4XX_RBBM_PERFCTR_CCU_1_LO				0x000000ee
-
-#define REG_A4XX_RBBM_PERFCTR_CCU_1_HI				0x000000ef
-
-#define REG_A4XX_RBBM_PERFCTR_CCU_2_LO				0x000000f0
-
-#define REG_A4XX_RBBM_PERFCTR_CCU_2_HI				0x000000f1
-
-#define REG_A4XX_RBBM_PERFCTR_CCU_3_LO				0x000000f2
-
-#define REG_A4XX_RBBM_PERFCTR_CCU_3_HI				0x000000f3
-
-#define REG_A4XX_RBBM_PERFCTR_TSE_0_LO				0x000000f4
-
-#define REG_A4XX_RBBM_PERFCTR_TSE_0_HI				0x000000f5
-
-#define REG_A4XX_RBBM_PERFCTR_TSE_1_LO				0x000000f6
-
-#define REG_A4XX_RBBM_PERFCTR_TSE_1_HI				0x000000f7
-
-#define REG_A4XX_RBBM_PERFCTR_TSE_2_LO				0x000000f8
-
-#define REG_A4XX_RBBM_PERFCTR_TSE_2_HI				0x000000f9
-
-#define REG_A4XX_RBBM_PERFCTR_TSE_3_LO				0x000000fa
-
-#define REG_A4XX_RBBM_PERFCTR_TSE_3_HI				0x000000fb
-
-#define REG_A4XX_RBBM_PERFCTR_RAS_0_LO				0x000000fc
-
-#define REG_A4XX_RBBM_PERFCTR_RAS_0_HI				0x000000fd
-
-#define REG_A4XX_RBBM_PERFCTR_RAS_1_LO				0x000000fe
-
-#define REG_A4XX_RBBM_PERFCTR_RAS_1_HI				0x000000ff
-
-#define REG_A4XX_RBBM_PERFCTR_RAS_2_LO				0x00000100
-
-#define REG_A4XX_RBBM_PERFCTR_RAS_2_HI				0x00000101
-
-#define REG_A4XX_RBBM_PERFCTR_RAS_3_LO				0x00000102
-
-#define REG_A4XX_RBBM_PERFCTR_RAS_3_HI				0x00000103
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_0_LO				0x00000104
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_0_HI				0x00000105
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_1_LO				0x00000106
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_1_HI				0x00000107
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_2_LO				0x00000108
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_2_HI				0x00000109
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_3_LO				0x0000010a
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_3_HI				0x0000010b
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_4_LO				0x0000010c
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_4_HI				0x0000010d
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_5_LO				0x0000010e
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_5_HI				0x0000010f
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_6_LO				0x00000110
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_6_HI				0x00000111
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_7_LO				0x00000112
-
-#define REG_A4XX_RBBM_PERFCTR_UCHE_7_HI				0x00000113
-
-#define REG_A4XX_RBBM_PERFCTR_TP_0_LO				0x00000114
-
-#define REG_A4XX_RBBM_PERFCTR_TP_0_HI				0x00000115
-
-#define REG_A4XX_RBBM_PERFCTR_TP_1_LO				0x00000116
-
-#define REG_A4XX_RBBM_PERFCTR_TP_1_HI				0x00000117
-
-#define REG_A4XX_RBBM_PERFCTR_TP_2_LO				0x00000118
-
-#define REG_A4XX_RBBM_PERFCTR_TP_2_HI				0x00000119
-
-#define REG_A4XX_RBBM_PERFCTR_TP_3_LO				0x0000011a
-
-#define REG_A4XX_RBBM_PERFCTR_TP_3_HI				0x0000011b
-
-#define REG_A4XX_RBBM_PERFCTR_TP_4_LO				0x0000011c
-
-#define REG_A4XX_RBBM_PERFCTR_TP_4_HI				0x0000011d
-
-#define REG_A4XX_RBBM_PERFCTR_TP_5_LO				0x0000011e
-
-#define REG_A4XX_RBBM_PERFCTR_TP_5_HI				0x0000011f
-
-#define REG_A4XX_RBBM_PERFCTR_TP_6_LO				0x00000120
-
-#define REG_A4XX_RBBM_PERFCTR_TP_6_HI				0x00000121
-
-#define REG_A4XX_RBBM_PERFCTR_TP_7_LO				0x00000122
-
-#define REG_A4XX_RBBM_PERFCTR_TP_7_HI				0x00000123
-
-#define REG_A4XX_RBBM_PERFCTR_SP_0_LO				0x00000124
-
-#define REG_A4XX_RBBM_PERFCTR_SP_0_HI				0x00000125
-
-#define REG_A4XX_RBBM_PERFCTR_SP_1_LO				0x00000126
-
-#define REG_A4XX_RBBM_PERFCTR_SP_1_HI				0x00000127
-
-#define REG_A4XX_RBBM_PERFCTR_SP_2_LO				0x00000128
-
-#define REG_A4XX_RBBM_PERFCTR_SP_2_HI				0x00000129
-
-#define REG_A4XX_RBBM_PERFCTR_SP_3_LO				0x0000012a
-
-#define REG_A4XX_RBBM_PERFCTR_SP_3_HI				0x0000012b
-
-#define REG_A4XX_RBBM_PERFCTR_SP_4_LO				0x0000012c
-
-#define REG_A4XX_RBBM_PERFCTR_SP_4_HI				0x0000012d
-
-#define REG_A4XX_RBBM_PERFCTR_SP_5_LO				0x0000012e
-
-#define REG_A4XX_RBBM_PERFCTR_SP_5_HI				0x0000012f
-
-#define REG_A4XX_RBBM_PERFCTR_SP_6_LO				0x00000130
-
-#define REG_A4XX_RBBM_PERFCTR_SP_6_HI				0x00000131
-
-#define REG_A4XX_RBBM_PERFCTR_SP_7_LO				0x00000132
-
-#define REG_A4XX_RBBM_PERFCTR_SP_7_HI				0x00000133
-
-#define REG_A4XX_RBBM_PERFCTR_SP_8_LO				0x00000134
-
-#define REG_A4XX_RBBM_PERFCTR_SP_8_HI				0x00000135
-
-#define REG_A4XX_RBBM_PERFCTR_SP_9_LO				0x00000136
-
-#define REG_A4XX_RBBM_PERFCTR_SP_9_HI				0x00000137
-
-#define REG_A4XX_RBBM_PERFCTR_SP_10_LO				0x00000138
-
-#define REG_A4XX_RBBM_PERFCTR_SP_10_HI				0x00000139
-
-#define REG_A4XX_RBBM_PERFCTR_SP_11_LO				0x0000013a
-
-#define REG_A4XX_RBBM_PERFCTR_SP_11_HI				0x0000013b
-
-#define REG_A4XX_RBBM_PERFCTR_RB_0_LO				0x0000013c
-
-#define REG_A4XX_RBBM_PERFCTR_RB_0_HI				0x0000013d
-
-#define REG_A4XX_RBBM_PERFCTR_RB_1_LO				0x0000013e
-
-#define REG_A4XX_RBBM_PERFCTR_RB_1_HI				0x0000013f
-
-#define REG_A4XX_RBBM_PERFCTR_RB_2_LO				0x00000140
-
-#define REG_A4XX_RBBM_PERFCTR_RB_2_HI				0x00000141
-
-#define REG_A4XX_RBBM_PERFCTR_RB_3_LO				0x00000142
-
-#define REG_A4XX_RBBM_PERFCTR_RB_3_HI				0x00000143
-
-#define REG_A4XX_RBBM_PERFCTR_RB_4_LO				0x00000144
-
-#define REG_A4XX_RBBM_PERFCTR_RB_4_HI				0x00000145
-
-#define REG_A4XX_RBBM_PERFCTR_RB_5_LO				0x00000146
-
-#define REG_A4XX_RBBM_PERFCTR_RB_5_HI				0x00000147
-
-#define REG_A4XX_RBBM_PERFCTR_RB_6_LO				0x00000148
-
-#define REG_A4XX_RBBM_PERFCTR_RB_6_HI				0x00000149
-
-#define REG_A4XX_RBBM_PERFCTR_RB_7_LO				0x0000014a
-
-#define REG_A4XX_RBBM_PERFCTR_RB_7_HI				0x0000014b
-
-#define REG_A4XX_RBBM_PERFCTR_VSC_0_LO				0x0000014c
-
-#define REG_A4XX_RBBM_PERFCTR_VSC_0_HI				0x0000014d
-
-#define REG_A4XX_RBBM_PERFCTR_VSC_1_LO				0x0000014e
-
-#define REG_A4XX_RBBM_PERFCTR_VSC_1_HI				0x0000014f
-
-#define REG_A4XX_RBBM_PERFCTR_PWR_0_LO				0x00000166
-
-#define REG_A4XX_RBBM_PERFCTR_PWR_0_HI				0x00000167
-
-#define REG_A4XX_RBBM_PERFCTR_PWR_1_LO				0x00000168
-
-#define REG_A4XX_RBBM_PERFCTR_PWR_1_HI				0x00000169
-
-#define REG_A4XX_RBBM_ALWAYSON_COUNTER_LO			0x0000016e
-
-#define REG_A4XX_RBBM_ALWAYSON_COUNTER_HI			0x0000016f
-
-#define REG_A4XX_RBBM_CLOCK_CTL_SP(i0) (0x00000068 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_CTL_SP_REG(uint32_t i0) { return 0x00000068 + 0x1*i0; }
-
-#define REG_A4XX_RBBM_CLOCK_CTL2_SP(i0) (0x0000006c + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_CTL2_SP_REG(uint32_t i0) { return 0x0000006c + 0x1*i0; }
-
-#define REG_A4XX_RBBM_CLOCK_HYST_SP(i0) (0x00000070 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_HYST_SP_REG(uint32_t i0) { return 0x00000070 + 0x1*i0; }
-
-#define REG_A4XX_RBBM_CLOCK_DELAY_SP(i0) (0x00000074 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_DELAY_SP_REG(uint32_t i0) { return 0x00000074 + 0x1*i0; }
-
-#define REG_A4XX_RBBM_CLOCK_CTL_RB(i0) (0x00000078 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_CTL_RB_REG(uint32_t i0) { return 0x00000078 + 0x1*i0; }
-
-#define REG_A4XX_RBBM_CLOCK_CTL2_RB(i0) (0x0000007c + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_CTL2_RB_REG(uint32_t i0) { return 0x0000007c + 0x1*i0; }
-
-#define REG_A4XX_RBBM_CLOCK_CTL_MARB_CCU(i0) (0x00000082 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_CTL_MARB_CCU_REG(uint32_t i0) { return 0x00000082 + 0x1*i0; }
-
-#define REG_A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU(i0) (0x00000086 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU_REG(uint32_t i0) { return 0x00000086 + 0x1*i0; }
-
-#define REG_A4XX_RBBM_CLOCK_HYST_COM_DCOM			0x00000080
-
-#define REG_A4XX_RBBM_CLOCK_CTL_COM_DCOM			0x00000081
-
-#define REG_A4XX_RBBM_CLOCK_CTL_HLSQ				0x0000008a
-
-#define REG_A4XX_RBBM_CLOCK_HYST_HLSQ				0x0000008b
-
-#define REG_A4XX_RBBM_CLOCK_DELAY_HLSQ				0x0000008c
-
-#define REG_A4XX_RBBM_CLOCK_DELAY_COM_DCOM			0x0000008d
-
-#define REG_A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1(i0) (0x0000008e + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1_REG(uint32_t i0) { return 0x0000008e + 0x1*i0; }
-
-#define REG_A4XX_RBBM_SP_REGFILE_SLEEP_CNTL_0			0x00000099
-
-#define REG_A4XX_RBBM_SP_REGFILE_SLEEP_CNTL_1			0x0000009a
-
-#define REG_A4XX_RBBM_PERFCTR_CTL				0x00000170
-
-#define REG_A4XX_RBBM_PERFCTR_LOAD_CMD0				0x00000171
-
-#define REG_A4XX_RBBM_PERFCTR_LOAD_CMD1				0x00000172
-
-#define REG_A4XX_RBBM_PERFCTR_LOAD_CMD2				0x00000173
-
-#define REG_A4XX_RBBM_PERFCTR_LOAD_VALUE_LO			0x00000174
-
-#define REG_A4XX_RBBM_PERFCTR_LOAD_VALUE_HI			0x00000175
-
-#define REG_A4XX_RBBM_PERFCTR_RBBM_SEL_0			0x00000176
-
-#define REG_A4XX_RBBM_PERFCTR_RBBM_SEL_1			0x00000177
-
-#define REG_A4XX_RBBM_PERFCTR_RBBM_SEL_2			0x00000178
-
-#define REG_A4XX_RBBM_PERFCTR_RBBM_SEL_3			0x00000179
-
-#define REG_A4XX_RBBM_GPU_BUSY_MASKED				0x0000017a
-
-#define REG_A4XX_RBBM_INT_0_STATUS				0x0000017d
-
-#define REG_A4XX_RBBM_CLOCK_STATUS				0x00000182
-
-#define REG_A4XX_RBBM_AHB_STATUS				0x00000189
-
-#define REG_A4XX_RBBM_AHB_ME_SPLIT_STATUS			0x0000018c
-
-#define REG_A4XX_RBBM_AHB_PFP_SPLIT_STATUS			0x0000018d
-
-#define REG_A4XX_RBBM_AHB_ERROR_STATUS				0x0000018f
-
-#define REG_A4XX_RBBM_STATUS					0x00000191
-#define A4XX_RBBM_STATUS_HI_BUSY				0x00000001
-#define A4XX_RBBM_STATUS_CP_ME_BUSY				0x00000002
-#define A4XX_RBBM_STATUS_CP_PFP_BUSY				0x00000004
-#define A4XX_RBBM_STATUS_CP_NRT_BUSY				0x00004000
-#define A4XX_RBBM_STATUS_VBIF_BUSY				0x00008000
-#define A4XX_RBBM_STATUS_TSE_BUSY				0x00010000
-#define A4XX_RBBM_STATUS_RAS_BUSY				0x00020000
-#define A4XX_RBBM_STATUS_RB_BUSY				0x00040000
-#define A4XX_RBBM_STATUS_PC_DCALL_BUSY				0x00080000
-#define A4XX_RBBM_STATUS_PC_VSD_BUSY				0x00100000
-#define A4XX_RBBM_STATUS_VFD_BUSY				0x00200000
-#define A4XX_RBBM_STATUS_VPC_BUSY				0x00400000
-#define A4XX_RBBM_STATUS_UCHE_BUSY				0x00800000
-#define A4XX_RBBM_STATUS_SP_BUSY				0x01000000
-#define A4XX_RBBM_STATUS_TPL1_BUSY				0x02000000
-#define A4XX_RBBM_STATUS_MARB_BUSY				0x04000000
-#define A4XX_RBBM_STATUS_VSC_BUSY				0x08000000
-#define A4XX_RBBM_STATUS_ARB_BUSY				0x10000000
-#define A4XX_RBBM_STATUS_HLSQ_BUSY				0x20000000
-#define A4XX_RBBM_STATUS_GPU_BUSY_NOHC				0x40000000
-#define A4XX_RBBM_STATUS_GPU_BUSY				0x80000000
-
-#define REG_A4XX_RBBM_INTERFACE_RRDY_STATUS5			0x0000019f
-
-#define REG_A4XX_RBBM_POWER_STATUS				0x000001b0
-#define A4XX_RBBM_POWER_STATUS_SP_TP_PWR_ON			0x00100000
-
-#define REG_A4XX_RBBM_WAIT_IDLE_CLOCKS_CTL2			0x000001b8
-
-#define REG_A4XX_CP_SCRATCH_UMASK				0x00000228
-
-#define REG_A4XX_CP_SCRATCH_ADDR				0x00000229
-
-#define REG_A4XX_CP_RB_BASE					0x00000200
-
-#define REG_A4XX_CP_RB_CNTL					0x00000201
-
-#define REG_A4XX_CP_RB_WPTR					0x00000205
-
-#define REG_A4XX_CP_RB_RPTR_ADDR				0x00000203
-
-#define REG_A4XX_CP_RB_RPTR					0x00000204
-
-#define REG_A4XX_CP_IB1_BASE					0x00000206
-
-#define REG_A4XX_CP_IB1_BUFSZ					0x00000207
-
-#define REG_A4XX_CP_IB2_BASE					0x00000208
-
-#define REG_A4XX_CP_IB2_BUFSZ					0x00000209
-
-#define REG_A4XX_CP_ME_NRT_ADDR					0x0000020c
-
-#define REG_A4XX_CP_ME_NRT_DATA					0x0000020d
-
-#define REG_A4XX_CP_ME_RB_DONE_DATA				0x00000217
-
-#define REG_A4XX_CP_QUEUE_THRESH2				0x00000219
-
-#define REG_A4XX_CP_MERCIU_SIZE					0x0000021b
-
-#define REG_A4XX_CP_ROQ_ADDR					0x0000021c
-
-#define REG_A4XX_CP_ROQ_DATA					0x0000021d
-
-#define REG_A4XX_CP_MEQ_ADDR					0x0000021e
-
-#define REG_A4XX_CP_MEQ_DATA					0x0000021f
-
-#define REG_A4XX_CP_MERCIU_ADDR					0x00000220
-
-#define REG_A4XX_CP_MERCIU_DATA					0x00000221
-
-#define REG_A4XX_CP_MERCIU_DATA2				0x00000222
-
-#define REG_A4XX_CP_PFP_UCODE_ADDR				0x00000223
-
-#define REG_A4XX_CP_PFP_UCODE_DATA				0x00000224
-
-#define REG_A4XX_CP_ME_RAM_WADDR				0x00000225
-
-#define REG_A4XX_CP_ME_RAM_RADDR				0x00000226
-
-#define REG_A4XX_CP_ME_RAM_DATA					0x00000227
-
-#define REG_A4XX_CP_PREEMPT					0x0000022a
-
-#define REG_A4XX_CP_CNTL					0x0000022c
-
-#define REG_A4XX_CP_ME_CNTL					0x0000022d
-
-#define REG_A4XX_CP_DEBUG					0x0000022e
-
-#define REG_A4XX_CP_DEBUG_ECO_CONTROL				0x00000231
-
-#define REG_A4XX_CP_DRAW_STATE_ADDR				0x00000232
-
-#define REG_A4XX_CP_PROTECT(i0) (0x00000240 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_CP_PROTECT_REG(uint32_t i0) { return 0x00000240 + 0x1*i0; }
-#define A4XX_CP_PROTECT_REG_BASE_ADDR__MASK			0x0001ffff
-#define A4XX_CP_PROTECT_REG_BASE_ADDR__SHIFT			0
-static inline uint32_t A4XX_CP_PROTECT_REG_BASE_ADDR(uint32_t val)
-{
-	return ((val) << A4XX_CP_PROTECT_REG_BASE_ADDR__SHIFT) & A4XX_CP_PROTECT_REG_BASE_ADDR__MASK;
-}
-#define A4XX_CP_PROTECT_REG_MASK_LEN__MASK			0x1f000000
-#define A4XX_CP_PROTECT_REG_MASK_LEN__SHIFT			24
-static inline uint32_t A4XX_CP_PROTECT_REG_MASK_LEN(uint32_t val)
-{
-	return ((val) << A4XX_CP_PROTECT_REG_MASK_LEN__SHIFT) & A4XX_CP_PROTECT_REG_MASK_LEN__MASK;
-}
-#define A4XX_CP_PROTECT_REG_TRAP_WRITE				0x20000000
-#define A4XX_CP_PROTECT_REG_TRAP_READ				0x40000000
-
-#define REG_A4XX_CP_PROTECT_CTRL				0x00000250
-
-#define REG_A4XX_CP_ST_BASE					0x000004c0
-
-#define REG_A4XX_CP_STQ_AVAIL					0x000004ce
-
-#define REG_A4XX_CP_MERCIU_STAT					0x000004d0
-
-#define REG_A4XX_CP_WFI_PEND_CTR				0x000004d2
-
-#define REG_A4XX_CP_HW_FAULT					0x000004d8
-
-#define REG_A4XX_CP_PROTECT_STATUS				0x000004da
-
-#define REG_A4XX_CP_EVENTS_IN_FLIGHT				0x000004dd
-
-#define REG_A4XX_CP_PERFCTR_CP_SEL_0				0x00000500
-
-#define REG_A4XX_CP_PERFCTR_CP_SEL_1				0x00000501
-
-#define REG_A4XX_CP_PERFCTR_CP_SEL_2				0x00000502
-
-#define REG_A4XX_CP_PERFCTR_CP_SEL_3				0x00000503
-
-#define REG_A4XX_CP_PERFCTR_CP_SEL_4				0x00000504
-
-#define REG_A4XX_CP_PERFCTR_CP_SEL_5				0x00000505
-
-#define REG_A4XX_CP_PERFCTR_CP_SEL_6				0x00000506
-
-#define REG_A4XX_CP_PERFCTR_CP_SEL_7				0x00000507
-
-#define REG_A4XX_CP_PERFCOMBINER_SELECT				0x0000050b
-
-#define REG_A4XX_CP_SCRATCH(i0) (0x00000578 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_CP_SCRATCH_REG(uint32_t i0) { return 0x00000578 + 0x1*i0; }
-
-#define REG_A4XX_SP_VS_STATUS					0x00000ec0
-
-#define REG_A4XX_SP_MODE_CONTROL				0x00000ec3
-
-#define REG_A4XX_SP_PERFCTR_SP_SEL_0				0x00000ec4
-
-#define REG_A4XX_SP_PERFCTR_SP_SEL_1				0x00000ec5
-
-#define REG_A4XX_SP_PERFCTR_SP_SEL_2				0x00000ec6
-
-#define REG_A4XX_SP_PERFCTR_SP_SEL_3				0x00000ec7
-
-#define REG_A4XX_SP_PERFCTR_SP_SEL_4				0x00000ec8
-
-#define REG_A4XX_SP_PERFCTR_SP_SEL_5				0x00000ec9
-
-#define REG_A4XX_SP_PERFCTR_SP_SEL_6				0x00000eca
-
-#define REG_A4XX_SP_PERFCTR_SP_SEL_7				0x00000ecb
-
-#define REG_A4XX_SP_PERFCTR_SP_SEL_8				0x00000ecc
-
-#define REG_A4XX_SP_PERFCTR_SP_SEL_9				0x00000ecd
-
-#define REG_A4XX_SP_PERFCTR_SP_SEL_10				0x00000ece
-
-#define REG_A4XX_SP_PERFCTR_SP_SEL_11				0x00000ecf
-
-#define REG_A4XX_SP_SP_CTRL_REG					0x000022c0
-#define A4XX_SP_SP_CTRL_REG_BINNING_PASS			0x00080000
-
-#define REG_A4XX_SP_INSTR_CACHE_CTRL				0x000022c1
-#define A4XX_SP_INSTR_CACHE_CTRL_VS_BUFFER			0x00000080
-#define A4XX_SP_INSTR_CACHE_CTRL_FS_BUFFER			0x00000100
-#define A4XX_SP_INSTR_CACHE_CTRL_INSTR_BUFFER			0x00000400
-
-#define REG_A4XX_SP_VS_CTRL_REG0				0x000022c4
-#define A4XX_SP_VS_CTRL_REG0_THREADMODE__MASK			0x00000001
-#define A4XX_SP_VS_CTRL_REG0_THREADMODE__SHIFT			0
-static inline uint32_t A4XX_SP_VS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
-{
-	return ((val) << A4XX_SP_VS_CTRL_REG0_THREADMODE__SHIFT) & A4XX_SP_VS_CTRL_REG0_THREADMODE__MASK;
-}
-#define A4XX_SP_VS_CTRL_REG0_VARYING				0x00000002
-#define A4XX_SP_VS_CTRL_REG0_CACHEINVALID			0x00000004
-#define A4XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
-#define A4XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
-static inline uint32_t A4XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A4XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
-#define A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
-static inline uint32_t A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A4XX_SP_VS_CTRL_REG0_INOUTREGOVERLAP__MASK		0x000c0000
-#define A4XX_SP_VS_CTRL_REG0_INOUTREGOVERLAP__SHIFT		18
-static inline uint32_t A4XX_SP_VS_CTRL_REG0_INOUTREGOVERLAP(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_CTRL_REG0_INOUTREGOVERLAP__SHIFT) & A4XX_SP_VS_CTRL_REG0_INOUTREGOVERLAP__MASK;
-}
-#define A4XX_SP_VS_CTRL_REG0_THREADSIZE__MASK			0x00100000
-#define A4XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT			20
-static inline uint32_t A4XX_SP_VS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A4XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT) & A4XX_SP_VS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A4XX_SP_VS_CTRL_REG0_SUPERTHREADMODE			0x00200000
-#define A4XX_SP_VS_CTRL_REG0_PIXLODENABLE			0x00400000
-
-#define REG_A4XX_SP_VS_CTRL_REG1				0x000022c5
-#define A4XX_SP_VS_CTRL_REG1_CONSTLENGTH__MASK			0x000000ff
-#define A4XX_SP_VS_CTRL_REG1_CONSTLENGTH__SHIFT			0
-static inline uint32_t A4XX_SP_VS_CTRL_REG1_CONSTLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_CTRL_REG1_CONSTLENGTH__SHIFT) & A4XX_SP_VS_CTRL_REG1_CONSTLENGTH__MASK;
-}
-#define A4XX_SP_VS_CTRL_REG1_INITIALOUTSTANDING__MASK		0x7f000000
-#define A4XX_SP_VS_CTRL_REG1_INITIALOUTSTANDING__SHIFT		24
-static inline uint32_t A4XX_SP_VS_CTRL_REG1_INITIALOUTSTANDING(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_CTRL_REG1_INITIALOUTSTANDING__SHIFT) & A4XX_SP_VS_CTRL_REG1_INITIALOUTSTANDING__MASK;
-}
-
-#define REG_A4XX_SP_VS_PARAM_REG				0x000022c6
-#define A4XX_SP_VS_PARAM_REG_POSREGID__MASK			0x000000ff
-#define A4XX_SP_VS_PARAM_REG_POSREGID__SHIFT			0
-static inline uint32_t A4XX_SP_VS_PARAM_REG_POSREGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_PARAM_REG_POSREGID__SHIFT) & A4XX_SP_VS_PARAM_REG_POSREGID__MASK;
-}
-#define A4XX_SP_VS_PARAM_REG_PSIZEREGID__MASK			0x0000ff00
-#define A4XX_SP_VS_PARAM_REG_PSIZEREGID__SHIFT			8
-static inline uint32_t A4XX_SP_VS_PARAM_REG_PSIZEREGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_PARAM_REG_PSIZEREGID__SHIFT) & A4XX_SP_VS_PARAM_REG_PSIZEREGID__MASK;
-}
-#define A4XX_SP_VS_PARAM_REG_TOTALVSOUTVAR__MASK		0xfff00000
-#define A4XX_SP_VS_PARAM_REG_TOTALVSOUTVAR__SHIFT		20
-static inline uint32_t A4XX_SP_VS_PARAM_REG_TOTALVSOUTVAR(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_PARAM_REG_TOTALVSOUTVAR__SHIFT) & A4XX_SP_VS_PARAM_REG_TOTALVSOUTVAR__MASK;
-}
-
-#define REG_A4XX_SP_VS_OUT(i0) (0x000022c7 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_SP_VS_OUT_REG(uint32_t i0) { return 0x000022c7 + 0x1*i0; }
-#define A4XX_SP_VS_OUT_REG_A_REGID__MASK			0x000001ff
-#define A4XX_SP_VS_OUT_REG_A_REGID__SHIFT			0
-static inline uint32_t A4XX_SP_VS_OUT_REG_A_REGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_OUT_REG_A_REGID__SHIFT) & A4XX_SP_VS_OUT_REG_A_REGID__MASK;
-}
-#define A4XX_SP_VS_OUT_REG_A_COMPMASK__MASK			0x00001e00
-#define A4XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT			9
-static inline uint32_t A4XX_SP_VS_OUT_REG_A_COMPMASK(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT) & A4XX_SP_VS_OUT_REG_A_COMPMASK__MASK;
-}
-#define A4XX_SP_VS_OUT_REG_B_REGID__MASK			0x01ff0000
-#define A4XX_SP_VS_OUT_REG_B_REGID__SHIFT			16
-static inline uint32_t A4XX_SP_VS_OUT_REG_B_REGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_OUT_REG_B_REGID__SHIFT) & A4XX_SP_VS_OUT_REG_B_REGID__MASK;
-}
-#define A4XX_SP_VS_OUT_REG_B_COMPMASK__MASK			0x1e000000
-#define A4XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT			25
-static inline uint32_t A4XX_SP_VS_OUT_REG_B_COMPMASK(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT) & A4XX_SP_VS_OUT_REG_B_COMPMASK__MASK;
-}
-
-#define REG_A4XX_SP_VS_VPC_DST(i0) (0x000022d8 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_SP_VS_VPC_DST_REG(uint32_t i0) { return 0x000022d8 + 0x1*i0; }
-#define A4XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK			0x000000ff
-#define A4XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT			0
-static inline uint32_t A4XX_SP_VS_VPC_DST_REG_OUTLOC0(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT) & A4XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK;
-}
-#define A4XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK			0x0000ff00
-#define A4XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT			8
-static inline uint32_t A4XX_SP_VS_VPC_DST_REG_OUTLOC1(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT) & A4XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK;
-}
-#define A4XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK			0x00ff0000
-#define A4XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT			16
-static inline uint32_t A4XX_SP_VS_VPC_DST_REG_OUTLOC2(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT) & A4XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK;
-}
-#define A4XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK			0xff000000
-#define A4XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT			24
-static inline uint32_t A4XX_SP_VS_VPC_DST_REG_OUTLOC3(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT) & A4XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK;
-}
-
-#define REG_A4XX_SP_VS_OBJ_OFFSET_REG				0x000022e0
-#define A4XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK	0x01ff0000
-#define A4XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT	16
-static inline uint32_t A4XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_SP_VS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A4XX_SP_VS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK		0xfe000000
-#define A4XX_SP_VS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT	25
-static inline uint32_t A4XX_SP_VS_OBJ_OFFSET_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_SP_VS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT) & A4XX_SP_VS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A4XX_SP_VS_OBJ_START				0x000022e1
-
-#define REG_A4XX_SP_VS_PVT_MEM_PARAM				0x000022e2
-
-#define REG_A4XX_SP_VS_PVT_MEM_ADDR				0x000022e3
-
-#define REG_A4XX_SP_VS_LENGTH_REG				0x000022e5
-
-#define REG_A4XX_SP_FS_CTRL_REG0				0x000022e8
-#define A4XX_SP_FS_CTRL_REG0_THREADMODE__MASK			0x00000001
-#define A4XX_SP_FS_CTRL_REG0_THREADMODE__SHIFT			0
-static inline uint32_t A4XX_SP_FS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
-{
-	return ((val) << A4XX_SP_FS_CTRL_REG0_THREADMODE__SHIFT) & A4XX_SP_FS_CTRL_REG0_THREADMODE__MASK;
-}
-#define A4XX_SP_FS_CTRL_REG0_VARYING				0x00000002
-#define A4XX_SP_FS_CTRL_REG0_CACHEINVALID			0x00000004
-#define A4XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
-#define A4XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
-static inline uint32_t A4XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A4XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A4XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
-#define A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
-static inline uint32_t A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A4XX_SP_FS_CTRL_REG0_INOUTREGOVERLAP__MASK		0x000c0000
-#define A4XX_SP_FS_CTRL_REG0_INOUTREGOVERLAP__SHIFT		18
-static inline uint32_t A4XX_SP_FS_CTRL_REG0_INOUTREGOVERLAP(uint32_t val)
-{
-	return ((val) << A4XX_SP_FS_CTRL_REG0_INOUTREGOVERLAP__SHIFT) & A4XX_SP_FS_CTRL_REG0_INOUTREGOVERLAP__MASK;
-}
-#define A4XX_SP_FS_CTRL_REG0_THREADSIZE__MASK			0x00100000
-#define A4XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT			20
-static inline uint32_t A4XX_SP_FS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A4XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT) & A4XX_SP_FS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A4XX_SP_FS_CTRL_REG0_SUPERTHREADMODE			0x00200000
-#define A4XX_SP_FS_CTRL_REG0_PIXLODENABLE			0x00400000
-
-#define REG_A4XX_SP_FS_CTRL_REG1				0x000022e9
-#define A4XX_SP_FS_CTRL_REG1_CONSTLENGTH__MASK			0x000000ff
-#define A4XX_SP_FS_CTRL_REG1_CONSTLENGTH__SHIFT			0
-static inline uint32_t A4XX_SP_FS_CTRL_REG1_CONSTLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_SP_FS_CTRL_REG1_CONSTLENGTH__SHIFT) & A4XX_SP_FS_CTRL_REG1_CONSTLENGTH__MASK;
-}
-#define A4XX_SP_FS_CTRL_REG1_FACENESS				0x00080000
-#define A4XX_SP_FS_CTRL_REG1_VARYING				0x00100000
-#define A4XX_SP_FS_CTRL_REG1_FRAGCOORD				0x00200000
-
-#define REG_A4XX_SP_FS_OBJ_OFFSET_REG				0x000022ea
-#define A4XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK	0x01ff0000
-#define A4XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT	16
-static inline uint32_t A4XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_SP_FS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A4XX_SP_FS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK		0xfe000000
-#define A4XX_SP_FS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT	25
-static inline uint32_t A4XX_SP_FS_OBJ_OFFSET_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_SP_FS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT) & A4XX_SP_FS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A4XX_SP_FS_OBJ_START				0x000022eb
-
-#define REG_A4XX_SP_FS_PVT_MEM_PARAM				0x000022ec
-
-#define REG_A4XX_SP_FS_PVT_MEM_ADDR				0x000022ed
-
-#define REG_A4XX_SP_FS_LENGTH_REG				0x000022ef
-
-#define REG_A4XX_SP_FS_OUTPUT_REG				0x000022f0
-#define A4XX_SP_FS_OUTPUT_REG_MRT__MASK				0x0000000f
-#define A4XX_SP_FS_OUTPUT_REG_MRT__SHIFT			0
-static inline uint32_t A4XX_SP_FS_OUTPUT_REG_MRT(uint32_t val)
-{
-	return ((val) << A4XX_SP_FS_OUTPUT_REG_MRT__SHIFT) & A4XX_SP_FS_OUTPUT_REG_MRT__MASK;
-}
-#define A4XX_SP_FS_OUTPUT_REG_DEPTH_ENABLE			0x00000080
-#define A4XX_SP_FS_OUTPUT_REG_DEPTH_REGID__MASK			0x0000ff00
-#define A4XX_SP_FS_OUTPUT_REG_DEPTH_REGID__SHIFT		8
-static inline uint32_t A4XX_SP_FS_OUTPUT_REG_DEPTH_REGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_FS_OUTPUT_REG_DEPTH_REGID__SHIFT) & A4XX_SP_FS_OUTPUT_REG_DEPTH_REGID__MASK;
-}
-#define A4XX_SP_FS_OUTPUT_REG_SAMPLEMASK_REGID__MASK		0xff000000
-#define A4XX_SP_FS_OUTPUT_REG_SAMPLEMASK_REGID__SHIFT		24
-static inline uint32_t A4XX_SP_FS_OUTPUT_REG_SAMPLEMASK_REGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_FS_OUTPUT_REG_SAMPLEMASK_REGID__SHIFT) & A4XX_SP_FS_OUTPUT_REG_SAMPLEMASK_REGID__MASK;
-}
-
-#define REG_A4XX_SP_FS_MRT(i0) (0x000022f1 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_SP_FS_MRT_REG(uint32_t i0) { return 0x000022f1 + 0x1*i0; }
-#define A4XX_SP_FS_MRT_REG_REGID__MASK				0x000000ff
-#define A4XX_SP_FS_MRT_REG_REGID__SHIFT				0
-static inline uint32_t A4XX_SP_FS_MRT_REG_REGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_FS_MRT_REG_REGID__SHIFT) & A4XX_SP_FS_MRT_REG_REGID__MASK;
-}
-#define A4XX_SP_FS_MRT_REG_HALF_PRECISION			0x00000100
-#define A4XX_SP_FS_MRT_REG_COLOR_SINT				0x00000400
-#define A4XX_SP_FS_MRT_REG_COLOR_UINT				0x00000800
-#define A4XX_SP_FS_MRT_REG_MRTFORMAT__MASK			0x0003f000
-#define A4XX_SP_FS_MRT_REG_MRTFORMAT__SHIFT			12
-static inline uint32_t A4XX_SP_FS_MRT_REG_MRTFORMAT(enum a4xx_color_fmt val)
-{
-	return ((val) << A4XX_SP_FS_MRT_REG_MRTFORMAT__SHIFT) & A4XX_SP_FS_MRT_REG_MRTFORMAT__MASK;
-}
-#define A4XX_SP_FS_MRT_REG_COLOR_SRGB				0x00040000
-
-#define REG_A4XX_SP_CS_CTRL_REG0				0x00002300
-#define A4XX_SP_CS_CTRL_REG0_THREADMODE__MASK			0x00000001
-#define A4XX_SP_CS_CTRL_REG0_THREADMODE__SHIFT			0
-static inline uint32_t A4XX_SP_CS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
-{
-	return ((val) << A4XX_SP_CS_CTRL_REG0_THREADMODE__SHIFT) & A4XX_SP_CS_CTRL_REG0_THREADMODE__MASK;
-}
-#define A4XX_SP_CS_CTRL_REG0_VARYING				0x00000002
-#define A4XX_SP_CS_CTRL_REG0_CACHEINVALID			0x00000004
-#define A4XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
-#define A4XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
-static inline uint32_t A4XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A4XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A4XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A4XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
-#define A4XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
-static inline uint32_t A4XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A4XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A4XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A4XX_SP_CS_CTRL_REG0_INOUTREGOVERLAP__MASK		0x000c0000
-#define A4XX_SP_CS_CTRL_REG0_INOUTREGOVERLAP__SHIFT		18
-static inline uint32_t A4XX_SP_CS_CTRL_REG0_INOUTREGOVERLAP(uint32_t val)
-{
-	return ((val) << A4XX_SP_CS_CTRL_REG0_INOUTREGOVERLAP__SHIFT) & A4XX_SP_CS_CTRL_REG0_INOUTREGOVERLAP__MASK;
-}
-#define A4XX_SP_CS_CTRL_REG0_THREADSIZE__MASK			0x00100000
-#define A4XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT			20
-static inline uint32_t A4XX_SP_CS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A4XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT) & A4XX_SP_CS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A4XX_SP_CS_CTRL_REG0_SUPERTHREADMODE			0x00200000
-#define A4XX_SP_CS_CTRL_REG0_PIXLODENABLE			0x00400000
-
-#define REG_A4XX_SP_CS_OBJ_OFFSET_REG				0x00002301
-
-#define REG_A4XX_SP_CS_OBJ_START				0x00002302
-
-#define REG_A4XX_SP_CS_PVT_MEM_PARAM				0x00002303
-
-#define REG_A4XX_SP_CS_PVT_MEM_ADDR				0x00002304
-
-#define REG_A4XX_SP_CS_PVT_MEM_SIZE				0x00002305
-
-#define REG_A4XX_SP_CS_LENGTH_REG				0x00002306
-
-#define REG_A4XX_SP_HS_OBJ_OFFSET_REG				0x0000230d
-#define A4XX_SP_HS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK	0x01ff0000
-#define A4XX_SP_HS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT	16
-static inline uint32_t A4XX_SP_HS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_SP_HS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_SP_HS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A4XX_SP_HS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK		0xfe000000
-#define A4XX_SP_HS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT	25
-static inline uint32_t A4XX_SP_HS_OBJ_OFFSET_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_SP_HS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT) & A4XX_SP_HS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A4XX_SP_HS_OBJ_START				0x0000230e
-
-#define REG_A4XX_SP_HS_PVT_MEM_PARAM				0x0000230f
-
-#define REG_A4XX_SP_HS_PVT_MEM_ADDR				0x00002310
-
-#define REG_A4XX_SP_HS_LENGTH_REG				0x00002312
-
-#define REG_A4XX_SP_DS_PARAM_REG				0x0000231a
-#define A4XX_SP_DS_PARAM_REG_POSREGID__MASK			0x000000ff
-#define A4XX_SP_DS_PARAM_REG_POSREGID__SHIFT			0
-static inline uint32_t A4XX_SP_DS_PARAM_REG_POSREGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_DS_PARAM_REG_POSREGID__SHIFT) & A4XX_SP_DS_PARAM_REG_POSREGID__MASK;
-}
-#define A4XX_SP_DS_PARAM_REG_TOTALGSOUTVAR__MASK		0xfff00000
-#define A4XX_SP_DS_PARAM_REG_TOTALGSOUTVAR__SHIFT		20
-static inline uint32_t A4XX_SP_DS_PARAM_REG_TOTALGSOUTVAR(uint32_t val)
-{
-	return ((val) << A4XX_SP_DS_PARAM_REG_TOTALGSOUTVAR__SHIFT) & A4XX_SP_DS_PARAM_REG_TOTALGSOUTVAR__MASK;
-}
-
-#define REG_A4XX_SP_DS_OUT(i0) (0x0000231b + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_SP_DS_OUT_REG(uint32_t i0) { return 0x0000231b + 0x1*i0; }
-#define A4XX_SP_DS_OUT_REG_A_REGID__MASK			0x000001ff
-#define A4XX_SP_DS_OUT_REG_A_REGID__SHIFT			0
-static inline uint32_t A4XX_SP_DS_OUT_REG_A_REGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_DS_OUT_REG_A_REGID__SHIFT) & A4XX_SP_DS_OUT_REG_A_REGID__MASK;
-}
-#define A4XX_SP_DS_OUT_REG_A_COMPMASK__MASK			0x00001e00
-#define A4XX_SP_DS_OUT_REG_A_COMPMASK__SHIFT			9
-static inline uint32_t A4XX_SP_DS_OUT_REG_A_COMPMASK(uint32_t val)
-{
-	return ((val) << A4XX_SP_DS_OUT_REG_A_COMPMASK__SHIFT) & A4XX_SP_DS_OUT_REG_A_COMPMASK__MASK;
-}
-#define A4XX_SP_DS_OUT_REG_B_REGID__MASK			0x01ff0000
-#define A4XX_SP_DS_OUT_REG_B_REGID__SHIFT			16
-static inline uint32_t A4XX_SP_DS_OUT_REG_B_REGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_DS_OUT_REG_B_REGID__SHIFT) & A4XX_SP_DS_OUT_REG_B_REGID__MASK;
-}
-#define A4XX_SP_DS_OUT_REG_B_COMPMASK__MASK			0x1e000000
-#define A4XX_SP_DS_OUT_REG_B_COMPMASK__SHIFT			25
-static inline uint32_t A4XX_SP_DS_OUT_REG_B_COMPMASK(uint32_t val)
-{
-	return ((val) << A4XX_SP_DS_OUT_REG_B_COMPMASK__SHIFT) & A4XX_SP_DS_OUT_REG_B_COMPMASK__MASK;
-}
-
-#define REG_A4XX_SP_DS_VPC_DST(i0) (0x0000232c + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_SP_DS_VPC_DST_REG(uint32_t i0) { return 0x0000232c + 0x1*i0; }
-#define A4XX_SP_DS_VPC_DST_REG_OUTLOC0__MASK			0x000000ff
-#define A4XX_SP_DS_VPC_DST_REG_OUTLOC0__SHIFT			0
-static inline uint32_t A4XX_SP_DS_VPC_DST_REG_OUTLOC0(uint32_t val)
-{
-	return ((val) << A4XX_SP_DS_VPC_DST_REG_OUTLOC0__SHIFT) & A4XX_SP_DS_VPC_DST_REG_OUTLOC0__MASK;
-}
-#define A4XX_SP_DS_VPC_DST_REG_OUTLOC1__MASK			0x0000ff00
-#define A4XX_SP_DS_VPC_DST_REG_OUTLOC1__SHIFT			8
-static inline uint32_t A4XX_SP_DS_VPC_DST_REG_OUTLOC1(uint32_t val)
-{
-	return ((val) << A4XX_SP_DS_VPC_DST_REG_OUTLOC1__SHIFT) & A4XX_SP_DS_VPC_DST_REG_OUTLOC1__MASK;
-}
-#define A4XX_SP_DS_VPC_DST_REG_OUTLOC2__MASK			0x00ff0000
-#define A4XX_SP_DS_VPC_DST_REG_OUTLOC2__SHIFT			16
-static inline uint32_t A4XX_SP_DS_VPC_DST_REG_OUTLOC2(uint32_t val)
-{
-	return ((val) << A4XX_SP_DS_VPC_DST_REG_OUTLOC2__SHIFT) & A4XX_SP_DS_VPC_DST_REG_OUTLOC2__MASK;
-}
-#define A4XX_SP_DS_VPC_DST_REG_OUTLOC3__MASK			0xff000000
-#define A4XX_SP_DS_VPC_DST_REG_OUTLOC3__SHIFT			24
-static inline uint32_t A4XX_SP_DS_VPC_DST_REG_OUTLOC3(uint32_t val)
-{
-	return ((val) << A4XX_SP_DS_VPC_DST_REG_OUTLOC3__SHIFT) & A4XX_SP_DS_VPC_DST_REG_OUTLOC3__MASK;
-}
-
-#define REG_A4XX_SP_DS_OBJ_OFFSET_REG				0x00002334
-#define A4XX_SP_DS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK	0x01ff0000
-#define A4XX_SP_DS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT	16
-static inline uint32_t A4XX_SP_DS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_SP_DS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_SP_DS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A4XX_SP_DS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK		0xfe000000
-#define A4XX_SP_DS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT	25
-static inline uint32_t A4XX_SP_DS_OBJ_OFFSET_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_SP_DS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT) & A4XX_SP_DS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A4XX_SP_DS_OBJ_START				0x00002335
-
-#define REG_A4XX_SP_DS_PVT_MEM_PARAM				0x00002336
-
-#define REG_A4XX_SP_DS_PVT_MEM_ADDR				0x00002337
-
-#define REG_A4XX_SP_DS_LENGTH_REG				0x00002339
-
-#define REG_A4XX_SP_GS_PARAM_REG				0x00002341
-#define A4XX_SP_GS_PARAM_REG_POSREGID__MASK			0x000000ff
-#define A4XX_SP_GS_PARAM_REG_POSREGID__SHIFT			0
-static inline uint32_t A4XX_SP_GS_PARAM_REG_POSREGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_PARAM_REG_POSREGID__SHIFT) & A4XX_SP_GS_PARAM_REG_POSREGID__MASK;
-}
-#define A4XX_SP_GS_PARAM_REG_PRIMREGID__MASK			0x0000ff00
-#define A4XX_SP_GS_PARAM_REG_PRIMREGID__SHIFT			8
-static inline uint32_t A4XX_SP_GS_PARAM_REG_PRIMREGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_PARAM_REG_PRIMREGID__SHIFT) & A4XX_SP_GS_PARAM_REG_PRIMREGID__MASK;
-}
-#define A4XX_SP_GS_PARAM_REG_TOTALGSOUTVAR__MASK		0xfff00000
-#define A4XX_SP_GS_PARAM_REG_TOTALGSOUTVAR__SHIFT		20
-static inline uint32_t A4XX_SP_GS_PARAM_REG_TOTALGSOUTVAR(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_PARAM_REG_TOTALGSOUTVAR__SHIFT) & A4XX_SP_GS_PARAM_REG_TOTALGSOUTVAR__MASK;
-}
-
-#define REG_A4XX_SP_GS_OUT(i0) (0x00002342 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_SP_GS_OUT_REG(uint32_t i0) { return 0x00002342 + 0x1*i0; }
-#define A4XX_SP_GS_OUT_REG_A_REGID__MASK			0x000001ff
-#define A4XX_SP_GS_OUT_REG_A_REGID__SHIFT			0
-static inline uint32_t A4XX_SP_GS_OUT_REG_A_REGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_OUT_REG_A_REGID__SHIFT) & A4XX_SP_GS_OUT_REG_A_REGID__MASK;
-}
-#define A4XX_SP_GS_OUT_REG_A_COMPMASK__MASK			0x00001e00
-#define A4XX_SP_GS_OUT_REG_A_COMPMASK__SHIFT			9
-static inline uint32_t A4XX_SP_GS_OUT_REG_A_COMPMASK(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_OUT_REG_A_COMPMASK__SHIFT) & A4XX_SP_GS_OUT_REG_A_COMPMASK__MASK;
-}
-#define A4XX_SP_GS_OUT_REG_B_REGID__MASK			0x01ff0000
-#define A4XX_SP_GS_OUT_REG_B_REGID__SHIFT			16
-static inline uint32_t A4XX_SP_GS_OUT_REG_B_REGID(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_OUT_REG_B_REGID__SHIFT) & A4XX_SP_GS_OUT_REG_B_REGID__MASK;
-}
-#define A4XX_SP_GS_OUT_REG_B_COMPMASK__MASK			0x1e000000
-#define A4XX_SP_GS_OUT_REG_B_COMPMASK__SHIFT			25
-static inline uint32_t A4XX_SP_GS_OUT_REG_B_COMPMASK(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_OUT_REG_B_COMPMASK__SHIFT) & A4XX_SP_GS_OUT_REG_B_COMPMASK__MASK;
-}
-
-#define REG_A4XX_SP_GS_VPC_DST(i0) (0x00002353 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_SP_GS_VPC_DST_REG(uint32_t i0) { return 0x00002353 + 0x1*i0; }
-#define A4XX_SP_GS_VPC_DST_REG_OUTLOC0__MASK			0x000000ff
-#define A4XX_SP_GS_VPC_DST_REG_OUTLOC0__SHIFT			0
-static inline uint32_t A4XX_SP_GS_VPC_DST_REG_OUTLOC0(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_VPC_DST_REG_OUTLOC0__SHIFT) & A4XX_SP_GS_VPC_DST_REG_OUTLOC0__MASK;
-}
-#define A4XX_SP_GS_VPC_DST_REG_OUTLOC1__MASK			0x0000ff00
-#define A4XX_SP_GS_VPC_DST_REG_OUTLOC1__SHIFT			8
-static inline uint32_t A4XX_SP_GS_VPC_DST_REG_OUTLOC1(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_VPC_DST_REG_OUTLOC1__SHIFT) & A4XX_SP_GS_VPC_DST_REG_OUTLOC1__MASK;
-}
-#define A4XX_SP_GS_VPC_DST_REG_OUTLOC2__MASK			0x00ff0000
-#define A4XX_SP_GS_VPC_DST_REG_OUTLOC2__SHIFT			16
-static inline uint32_t A4XX_SP_GS_VPC_DST_REG_OUTLOC2(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_VPC_DST_REG_OUTLOC2__SHIFT) & A4XX_SP_GS_VPC_DST_REG_OUTLOC2__MASK;
-}
-#define A4XX_SP_GS_VPC_DST_REG_OUTLOC3__MASK			0xff000000
-#define A4XX_SP_GS_VPC_DST_REG_OUTLOC3__SHIFT			24
-static inline uint32_t A4XX_SP_GS_VPC_DST_REG_OUTLOC3(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_VPC_DST_REG_OUTLOC3__SHIFT) & A4XX_SP_GS_VPC_DST_REG_OUTLOC3__MASK;
-}
-
-#define REG_A4XX_SP_GS_OBJ_OFFSET_REG				0x0000235b
-#define A4XX_SP_GS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK	0x01ff0000
-#define A4XX_SP_GS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT	16
-static inline uint32_t A4XX_SP_GS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_SP_GS_OBJ_OFFSET_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A4XX_SP_GS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK		0xfe000000
-#define A4XX_SP_GS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT	25
-static inline uint32_t A4XX_SP_GS_OBJ_OFFSET_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_SP_GS_OBJ_OFFSET_REG_SHADEROBJOFFSET__SHIFT) & A4XX_SP_GS_OBJ_OFFSET_REG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A4XX_SP_GS_OBJ_START				0x0000235c
-
-#define REG_A4XX_SP_GS_PVT_MEM_PARAM				0x0000235d
-
-#define REG_A4XX_SP_GS_PVT_MEM_ADDR				0x0000235e
-
-#define REG_A4XX_SP_GS_LENGTH_REG				0x00002360
-
-#define REG_A4XX_VPC_DEBUG_RAM_SEL				0x00000e60
-
-#define REG_A4XX_VPC_DEBUG_RAM_READ				0x00000e61
-
-#define REG_A4XX_VPC_DEBUG_ECO_CONTROL				0x00000e64
-
-#define REG_A4XX_VPC_PERFCTR_VPC_SEL_0				0x00000e65
-
-#define REG_A4XX_VPC_PERFCTR_VPC_SEL_1				0x00000e66
-
-#define REG_A4XX_VPC_PERFCTR_VPC_SEL_2				0x00000e67
-
-#define REG_A4XX_VPC_PERFCTR_VPC_SEL_3				0x00000e68
-
-#define REG_A4XX_VPC_ATTR					0x00002140
-#define A4XX_VPC_ATTR_TOTALATTR__MASK				0x000001ff
-#define A4XX_VPC_ATTR_TOTALATTR__SHIFT				0
-static inline uint32_t A4XX_VPC_ATTR_TOTALATTR(uint32_t val)
-{
-	return ((val) << A4XX_VPC_ATTR_TOTALATTR__SHIFT) & A4XX_VPC_ATTR_TOTALATTR__MASK;
-}
-#define A4XX_VPC_ATTR_PSIZE					0x00000200
-#define A4XX_VPC_ATTR_THRDASSIGN__MASK				0x00003000
-#define A4XX_VPC_ATTR_THRDASSIGN__SHIFT				12
-static inline uint32_t A4XX_VPC_ATTR_THRDASSIGN(uint32_t val)
-{
-	return ((val) << A4XX_VPC_ATTR_THRDASSIGN__SHIFT) & A4XX_VPC_ATTR_THRDASSIGN__MASK;
-}
-#define A4XX_VPC_ATTR_ENABLE					0x02000000
-
-#define REG_A4XX_VPC_PACK					0x00002141
-#define A4XX_VPC_PACK_NUMBYPASSVAR__MASK			0x000000ff
-#define A4XX_VPC_PACK_NUMBYPASSVAR__SHIFT			0
-static inline uint32_t A4XX_VPC_PACK_NUMBYPASSVAR(uint32_t val)
-{
-	return ((val) << A4XX_VPC_PACK_NUMBYPASSVAR__SHIFT) & A4XX_VPC_PACK_NUMBYPASSVAR__MASK;
-}
-#define A4XX_VPC_PACK_NUMFPNONPOSVAR__MASK			0x0000ff00
-#define A4XX_VPC_PACK_NUMFPNONPOSVAR__SHIFT			8
-static inline uint32_t A4XX_VPC_PACK_NUMFPNONPOSVAR(uint32_t val)
-{
-	return ((val) << A4XX_VPC_PACK_NUMFPNONPOSVAR__SHIFT) & A4XX_VPC_PACK_NUMFPNONPOSVAR__MASK;
-}
-#define A4XX_VPC_PACK_NUMNONPOSVSVAR__MASK			0x00ff0000
-#define A4XX_VPC_PACK_NUMNONPOSVSVAR__SHIFT			16
-static inline uint32_t A4XX_VPC_PACK_NUMNONPOSVSVAR(uint32_t val)
-{
-	return ((val) << A4XX_VPC_PACK_NUMNONPOSVSVAR__SHIFT) & A4XX_VPC_PACK_NUMNONPOSVSVAR__MASK;
-}
-
-#define REG_A4XX_VPC_VARYING_INTERP(i0) (0x00002142 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_VPC_VARYING_INTERP_MODE(uint32_t i0) { return 0x00002142 + 0x1*i0; }
-
-#define REG_A4XX_VPC_VARYING_PS_REPL(i0) (0x0000214a + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_VPC_VARYING_PS_REPL_MODE(uint32_t i0) { return 0x0000214a + 0x1*i0; }
-
-#define REG_A4XX_VPC_SO_FLUSH_WADDR_3				0x0000216e
-
-#define REG_A4XX_VSC_BIN_SIZE					0x00000c00
-#define A4XX_VSC_BIN_SIZE_WIDTH__MASK				0x0000001f
-#define A4XX_VSC_BIN_SIZE_WIDTH__SHIFT				0
-static inline uint32_t A4XX_VSC_BIN_SIZE_WIDTH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A4XX_VSC_BIN_SIZE_WIDTH__SHIFT) & A4XX_VSC_BIN_SIZE_WIDTH__MASK;
-}
-#define A4XX_VSC_BIN_SIZE_HEIGHT__MASK				0x000003e0
-#define A4XX_VSC_BIN_SIZE_HEIGHT__SHIFT				5
-static inline uint32_t A4XX_VSC_BIN_SIZE_HEIGHT(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A4XX_VSC_BIN_SIZE_HEIGHT__SHIFT) & A4XX_VSC_BIN_SIZE_HEIGHT__MASK;
-}
-
-#define REG_A4XX_VSC_SIZE_ADDRESS				0x00000c01
-
-#define REG_A4XX_VSC_SIZE_ADDRESS2				0x00000c02
-
-#define REG_A4XX_VSC_DEBUG_ECO_CONTROL				0x00000c03
-
-#define REG_A4XX_VSC_PIPE_CONFIG(i0) (0x00000c08 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_VSC_PIPE_CONFIG_REG(uint32_t i0) { return 0x00000c08 + 0x1*i0; }
-#define A4XX_VSC_PIPE_CONFIG_REG_X__MASK			0x000003ff
-#define A4XX_VSC_PIPE_CONFIG_REG_X__SHIFT			0
-static inline uint32_t A4XX_VSC_PIPE_CONFIG_REG_X(uint32_t val)
-{
-	return ((val) << A4XX_VSC_PIPE_CONFIG_REG_X__SHIFT) & A4XX_VSC_PIPE_CONFIG_REG_X__MASK;
-}
-#define A4XX_VSC_PIPE_CONFIG_REG_Y__MASK			0x000ffc00
-#define A4XX_VSC_PIPE_CONFIG_REG_Y__SHIFT			10
-static inline uint32_t A4XX_VSC_PIPE_CONFIG_REG_Y(uint32_t val)
-{
-	return ((val) << A4XX_VSC_PIPE_CONFIG_REG_Y__SHIFT) & A4XX_VSC_PIPE_CONFIG_REG_Y__MASK;
-}
-#define A4XX_VSC_PIPE_CONFIG_REG_W__MASK			0x00f00000
-#define A4XX_VSC_PIPE_CONFIG_REG_W__SHIFT			20
-static inline uint32_t A4XX_VSC_PIPE_CONFIG_REG_W(uint32_t val)
-{
-	return ((val) << A4XX_VSC_PIPE_CONFIG_REG_W__SHIFT) & A4XX_VSC_PIPE_CONFIG_REG_W__MASK;
-}
-#define A4XX_VSC_PIPE_CONFIG_REG_H__MASK			0x0f000000
-#define A4XX_VSC_PIPE_CONFIG_REG_H__SHIFT			24
-static inline uint32_t A4XX_VSC_PIPE_CONFIG_REG_H(uint32_t val)
-{
-	return ((val) << A4XX_VSC_PIPE_CONFIG_REG_H__SHIFT) & A4XX_VSC_PIPE_CONFIG_REG_H__MASK;
-}
-
-#define REG_A4XX_VSC_PIPE_DATA_ADDRESS(i0) (0x00000c10 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_VSC_PIPE_DATA_ADDRESS_REG(uint32_t i0) { return 0x00000c10 + 0x1*i0; }
-
-#define REG_A4XX_VSC_PIPE_DATA_LENGTH(i0) (0x00000c18 + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_VSC_PIPE_DATA_LENGTH_REG(uint32_t i0) { return 0x00000c18 + 0x1*i0; }
-
-#define REG_A4XX_VSC_PIPE_PARTIAL_POSN_1			0x00000c41
-
-#define REG_A4XX_VSC_PERFCTR_VSC_SEL_0				0x00000c50
-
-#define REG_A4XX_VSC_PERFCTR_VSC_SEL_1				0x00000c51
-
-#define REG_A4XX_VFD_DEBUG_CONTROL				0x00000e40
-
-#define REG_A4XX_VFD_PERFCTR_VFD_SEL_0				0x00000e43
-
-#define REG_A4XX_VFD_PERFCTR_VFD_SEL_1				0x00000e44
-
-#define REG_A4XX_VFD_PERFCTR_VFD_SEL_2				0x00000e45
-
-#define REG_A4XX_VFD_PERFCTR_VFD_SEL_3				0x00000e46
-
-#define REG_A4XX_VFD_PERFCTR_VFD_SEL_4				0x00000e47
-
-#define REG_A4XX_VFD_PERFCTR_VFD_SEL_5				0x00000e48
-
-#define REG_A4XX_VFD_PERFCTR_VFD_SEL_6				0x00000e49
-
-#define REG_A4XX_VFD_PERFCTR_VFD_SEL_7				0x00000e4a
-
-#define REG_A4XX_VGT_CL_INITIATOR				0x000021d0
-
-#define REG_A4XX_VGT_EVENT_INITIATOR				0x000021d9
-
-#define REG_A4XX_VFD_CONTROL_0					0x00002200
-#define A4XX_VFD_CONTROL_0_TOTALATTRTOVS__MASK			0x000000ff
-#define A4XX_VFD_CONTROL_0_TOTALATTRTOVS__SHIFT			0
-static inline uint32_t A4XX_VFD_CONTROL_0_TOTALATTRTOVS(uint32_t val)
-{
-	return ((val) << A4XX_VFD_CONTROL_0_TOTALATTRTOVS__SHIFT) & A4XX_VFD_CONTROL_0_TOTALATTRTOVS__MASK;
-}
-#define A4XX_VFD_CONTROL_0_BYPASSATTROVS__MASK			0x0001fe00
-#define A4XX_VFD_CONTROL_0_BYPASSATTROVS__SHIFT			9
-static inline uint32_t A4XX_VFD_CONTROL_0_BYPASSATTROVS(uint32_t val)
-{
-	return ((val) << A4XX_VFD_CONTROL_0_BYPASSATTROVS__SHIFT) & A4XX_VFD_CONTROL_0_BYPASSATTROVS__MASK;
-}
-#define A4XX_VFD_CONTROL_0_STRMDECINSTRCNT__MASK		0x03f00000
-#define A4XX_VFD_CONTROL_0_STRMDECINSTRCNT__SHIFT		20
-static inline uint32_t A4XX_VFD_CONTROL_0_STRMDECINSTRCNT(uint32_t val)
-{
-	return ((val) << A4XX_VFD_CONTROL_0_STRMDECINSTRCNT__SHIFT) & A4XX_VFD_CONTROL_0_STRMDECINSTRCNT__MASK;
-}
-#define A4XX_VFD_CONTROL_0_STRMFETCHINSTRCNT__MASK		0xfc000000
-#define A4XX_VFD_CONTROL_0_STRMFETCHINSTRCNT__SHIFT		26
-static inline uint32_t A4XX_VFD_CONTROL_0_STRMFETCHINSTRCNT(uint32_t val)
-{
-	return ((val) << A4XX_VFD_CONTROL_0_STRMFETCHINSTRCNT__SHIFT) & A4XX_VFD_CONTROL_0_STRMFETCHINSTRCNT__MASK;
-}
-
-#define REG_A4XX_VFD_CONTROL_1					0x00002201
-#define A4XX_VFD_CONTROL_1_MAXSTORAGE__MASK			0x0000ffff
-#define A4XX_VFD_CONTROL_1_MAXSTORAGE__SHIFT			0
-static inline uint32_t A4XX_VFD_CONTROL_1_MAXSTORAGE(uint32_t val)
-{
-	return ((val) << A4XX_VFD_CONTROL_1_MAXSTORAGE__SHIFT) & A4XX_VFD_CONTROL_1_MAXSTORAGE__MASK;
-}
-#define A4XX_VFD_CONTROL_1_REGID4VTX__MASK			0x00ff0000
-#define A4XX_VFD_CONTROL_1_REGID4VTX__SHIFT			16
-static inline uint32_t A4XX_VFD_CONTROL_1_REGID4VTX(uint32_t val)
-{
-	return ((val) << A4XX_VFD_CONTROL_1_REGID4VTX__SHIFT) & A4XX_VFD_CONTROL_1_REGID4VTX__MASK;
-}
-#define A4XX_VFD_CONTROL_1_REGID4INST__MASK			0xff000000
-#define A4XX_VFD_CONTROL_1_REGID4INST__SHIFT			24
-static inline uint32_t A4XX_VFD_CONTROL_1_REGID4INST(uint32_t val)
-{
-	return ((val) << A4XX_VFD_CONTROL_1_REGID4INST__SHIFT) & A4XX_VFD_CONTROL_1_REGID4INST__MASK;
-}
-
-#define REG_A4XX_VFD_CONTROL_2					0x00002202
-
-#define REG_A4XX_VFD_CONTROL_3					0x00002203
-#define A4XX_VFD_CONTROL_3_REGID_VTXCNT__MASK			0x0000ff00
-#define A4XX_VFD_CONTROL_3_REGID_VTXCNT__SHIFT			8
-static inline uint32_t A4XX_VFD_CONTROL_3_REGID_VTXCNT(uint32_t val)
-{
-	return ((val) << A4XX_VFD_CONTROL_3_REGID_VTXCNT__SHIFT) & A4XX_VFD_CONTROL_3_REGID_VTXCNT__MASK;
-}
-#define A4XX_VFD_CONTROL_3_REGID_TESSX__MASK			0x00ff0000
-#define A4XX_VFD_CONTROL_3_REGID_TESSX__SHIFT			16
-static inline uint32_t A4XX_VFD_CONTROL_3_REGID_TESSX(uint32_t val)
-{
-	return ((val) << A4XX_VFD_CONTROL_3_REGID_TESSX__SHIFT) & A4XX_VFD_CONTROL_3_REGID_TESSX__MASK;
-}
-#define A4XX_VFD_CONTROL_3_REGID_TESSY__MASK			0xff000000
-#define A4XX_VFD_CONTROL_3_REGID_TESSY__SHIFT			24
-static inline uint32_t A4XX_VFD_CONTROL_3_REGID_TESSY(uint32_t val)
-{
-	return ((val) << A4XX_VFD_CONTROL_3_REGID_TESSY__SHIFT) & A4XX_VFD_CONTROL_3_REGID_TESSY__MASK;
-}
-
-#define REG_A4XX_VFD_CONTROL_4					0x00002204
-
-#define REG_A4XX_VFD_INDEX_OFFSET				0x00002208
-
-#define REG_A4XX_VFD_FETCH(i0) (0x0000220a + 0x4*(i0))
-
-static inline uint32_t REG_A4XX_VFD_FETCH_INSTR_0(uint32_t i0) { return 0x0000220a + 0x4*i0; }
-#define A4XX_VFD_FETCH_INSTR_0_FETCHSIZE__MASK			0x0000007f
-#define A4XX_VFD_FETCH_INSTR_0_FETCHSIZE__SHIFT			0
-static inline uint32_t A4XX_VFD_FETCH_INSTR_0_FETCHSIZE(uint32_t val)
-{
-	return ((val) << A4XX_VFD_FETCH_INSTR_0_FETCHSIZE__SHIFT) & A4XX_VFD_FETCH_INSTR_0_FETCHSIZE__MASK;
-}
-#define A4XX_VFD_FETCH_INSTR_0_BUFSTRIDE__MASK			0x0001ff80
-#define A4XX_VFD_FETCH_INSTR_0_BUFSTRIDE__SHIFT			7
-static inline uint32_t A4XX_VFD_FETCH_INSTR_0_BUFSTRIDE(uint32_t val)
-{
-	return ((val) << A4XX_VFD_FETCH_INSTR_0_BUFSTRIDE__SHIFT) & A4XX_VFD_FETCH_INSTR_0_BUFSTRIDE__MASK;
-}
-#define A4XX_VFD_FETCH_INSTR_0_SWITCHNEXT			0x00080000
-#define A4XX_VFD_FETCH_INSTR_0_INSTANCED			0x00100000
-
-static inline uint32_t REG_A4XX_VFD_FETCH_INSTR_1(uint32_t i0) { return 0x0000220b + 0x4*i0; }
-
-static inline uint32_t REG_A4XX_VFD_FETCH_INSTR_2(uint32_t i0) { return 0x0000220c + 0x4*i0; }
-#define A4XX_VFD_FETCH_INSTR_2_SIZE__MASK			0xffffffff
-#define A4XX_VFD_FETCH_INSTR_2_SIZE__SHIFT			0
-static inline uint32_t A4XX_VFD_FETCH_INSTR_2_SIZE(uint32_t val)
-{
-	return ((val) << A4XX_VFD_FETCH_INSTR_2_SIZE__SHIFT) & A4XX_VFD_FETCH_INSTR_2_SIZE__MASK;
-}
-
-static inline uint32_t REG_A4XX_VFD_FETCH_INSTR_3(uint32_t i0) { return 0x0000220d + 0x4*i0; }
-#define A4XX_VFD_FETCH_INSTR_3_STEPRATE__MASK			0x000001ff
-#define A4XX_VFD_FETCH_INSTR_3_STEPRATE__SHIFT			0
-static inline uint32_t A4XX_VFD_FETCH_INSTR_3_STEPRATE(uint32_t val)
-{
-	return ((val) << A4XX_VFD_FETCH_INSTR_3_STEPRATE__SHIFT) & A4XX_VFD_FETCH_INSTR_3_STEPRATE__MASK;
-}
-
-#define REG_A4XX_VFD_DECODE(i0) (0x0000228a + 0x1*(i0))
-
-static inline uint32_t REG_A4XX_VFD_DECODE_INSTR(uint32_t i0) { return 0x0000228a + 0x1*i0; }
-#define A4XX_VFD_DECODE_INSTR_WRITEMASK__MASK			0x0000000f
-#define A4XX_VFD_DECODE_INSTR_WRITEMASK__SHIFT			0
-static inline uint32_t A4XX_VFD_DECODE_INSTR_WRITEMASK(uint32_t val)
-{
-	return ((val) << A4XX_VFD_DECODE_INSTR_WRITEMASK__SHIFT) & A4XX_VFD_DECODE_INSTR_WRITEMASK__MASK;
-}
-#define A4XX_VFD_DECODE_INSTR_CONSTFILL				0x00000010
-#define A4XX_VFD_DECODE_INSTR_FORMAT__MASK			0x00000fc0
-#define A4XX_VFD_DECODE_INSTR_FORMAT__SHIFT			6
-static inline uint32_t A4XX_VFD_DECODE_INSTR_FORMAT(enum a4xx_vtx_fmt val)
-{
-	return ((val) << A4XX_VFD_DECODE_INSTR_FORMAT__SHIFT) & A4XX_VFD_DECODE_INSTR_FORMAT__MASK;
-}
-#define A4XX_VFD_DECODE_INSTR_REGID__MASK			0x000ff000
-#define A4XX_VFD_DECODE_INSTR_REGID__SHIFT			12
-static inline uint32_t A4XX_VFD_DECODE_INSTR_REGID(uint32_t val)
-{
-	return ((val) << A4XX_VFD_DECODE_INSTR_REGID__SHIFT) & A4XX_VFD_DECODE_INSTR_REGID__MASK;
-}
-#define A4XX_VFD_DECODE_INSTR_INT				0x00100000
-#define A4XX_VFD_DECODE_INSTR_SWAP__MASK			0x00c00000
-#define A4XX_VFD_DECODE_INSTR_SWAP__SHIFT			22
-static inline uint32_t A4XX_VFD_DECODE_INSTR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A4XX_VFD_DECODE_INSTR_SWAP__SHIFT) & A4XX_VFD_DECODE_INSTR_SWAP__MASK;
-}
-#define A4XX_VFD_DECODE_INSTR_SHIFTCNT__MASK			0x1f000000
-#define A4XX_VFD_DECODE_INSTR_SHIFTCNT__SHIFT			24
-static inline uint32_t A4XX_VFD_DECODE_INSTR_SHIFTCNT(uint32_t val)
-{
-	return ((val) << A4XX_VFD_DECODE_INSTR_SHIFTCNT__SHIFT) & A4XX_VFD_DECODE_INSTR_SHIFTCNT__MASK;
-}
-#define A4XX_VFD_DECODE_INSTR_LASTCOMPVALID			0x20000000
-#define A4XX_VFD_DECODE_INSTR_SWITCHNEXT			0x40000000
-
-#define REG_A4XX_TPL1_DEBUG_ECO_CONTROL				0x00000f00
-
-#define REG_A4XX_TPL1_TP_MODE_CONTROL				0x00000f03
-
-#define REG_A4XX_TPL1_PERFCTR_TP_SEL_0				0x00000f04
-
-#define REG_A4XX_TPL1_PERFCTR_TP_SEL_1				0x00000f05
-
-#define REG_A4XX_TPL1_PERFCTR_TP_SEL_2				0x00000f06
-
-#define REG_A4XX_TPL1_PERFCTR_TP_SEL_3				0x00000f07
-
-#define REG_A4XX_TPL1_PERFCTR_TP_SEL_4				0x00000f08
-
-#define REG_A4XX_TPL1_PERFCTR_TP_SEL_5				0x00000f09
-
-#define REG_A4XX_TPL1_PERFCTR_TP_SEL_6				0x00000f0a
-
-#define REG_A4XX_TPL1_PERFCTR_TP_SEL_7				0x00000f0b
-
-#define REG_A4XX_TPL1_TP_TEX_OFFSET				0x00002380
-
-#define REG_A4XX_TPL1_TP_TEX_COUNT				0x00002381
-#define A4XX_TPL1_TP_TEX_COUNT_VS__MASK				0x000000ff
-#define A4XX_TPL1_TP_TEX_COUNT_VS__SHIFT			0
-static inline uint32_t A4XX_TPL1_TP_TEX_COUNT_VS(uint32_t val)
-{
-	return ((val) << A4XX_TPL1_TP_TEX_COUNT_VS__SHIFT) & A4XX_TPL1_TP_TEX_COUNT_VS__MASK;
-}
-#define A4XX_TPL1_TP_TEX_COUNT_HS__MASK				0x0000ff00
-#define A4XX_TPL1_TP_TEX_COUNT_HS__SHIFT			8
-static inline uint32_t A4XX_TPL1_TP_TEX_COUNT_HS(uint32_t val)
-{
-	return ((val) << A4XX_TPL1_TP_TEX_COUNT_HS__SHIFT) & A4XX_TPL1_TP_TEX_COUNT_HS__MASK;
-}
-#define A4XX_TPL1_TP_TEX_COUNT_DS__MASK				0x00ff0000
-#define A4XX_TPL1_TP_TEX_COUNT_DS__SHIFT			16
-static inline uint32_t A4XX_TPL1_TP_TEX_COUNT_DS(uint32_t val)
-{
-	return ((val) << A4XX_TPL1_TP_TEX_COUNT_DS__SHIFT) & A4XX_TPL1_TP_TEX_COUNT_DS__MASK;
-}
-#define A4XX_TPL1_TP_TEX_COUNT_GS__MASK				0xff000000
-#define A4XX_TPL1_TP_TEX_COUNT_GS__SHIFT			24
-static inline uint32_t A4XX_TPL1_TP_TEX_COUNT_GS(uint32_t val)
-{
-	return ((val) << A4XX_TPL1_TP_TEX_COUNT_GS__SHIFT) & A4XX_TPL1_TP_TEX_COUNT_GS__MASK;
-}
-
-#define REG_A4XX_TPL1_TP_VS_BORDER_COLOR_BASE_ADDR		0x00002384
-
-#define REG_A4XX_TPL1_TP_HS_BORDER_COLOR_BASE_ADDR		0x00002387
-
-#define REG_A4XX_TPL1_TP_DS_BORDER_COLOR_BASE_ADDR		0x0000238a
-
-#define REG_A4XX_TPL1_TP_GS_BORDER_COLOR_BASE_ADDR		0x0000238d
-
-#define REG_A4XX_TPL1_TP_FS_TEX_COUNT				0x000023a0
-#define A4XX_TPL1_TP_FS_TEX_COUNT_FS__MASK			0x000000ff
-#define A4XX_TPL1_TP_FS_TEX_COUNT_FS__SHIFT			0
-static inline uint32_t A4XX_TPL1_TP_FS_TEX_COUNT_FS(uint32_t val)
-{
-	return ((val) << A4XX_TPL1_TP_FS_TEX_COUNT_FS__SHIFT) & A4XX_TPL1_TP_FS_TEX_COUNT_FS__MASK;
-}
-#define A4XX_TPL1_TP_FS_TEX_COUNT_CS__MASK			0x0000ff00
-#define A4XX_TPL1_TP_FS_TEX_COUNT_CS__SHIFT			8
-static inline uint32_t A4XX_TPL1_TP_FS_TEX_COUNT_CS(uint32_t val)
-{
-	return ((val) << A4XX_TPL1_TP_FS_TEX_COUNT_CS__SHIFT) & A4XX_TPL1_TP_FS_TEX_COUNT_CS__MASK;
-}
-
-#define REG_A4XX_TPL1_TP_FS_BORDER_COLOR_BASE_ADDR		0x000023a1
-
-#define REG_A4XX_TPL1_TP_CS_BORDER_COLOR_BASE_ADDR		0x000023a4
-
-#define REG_A4XX_TPL1_TP_CS_SAMPLER_BASE_ADDR			0x000023a5
-
-#define REG_A4XX_TPL1_TP_CS_TEXMEMOBJ_BASE_ADDR			0x000023a6
-
-#define REG_A4XX_GRAS_TSE_STATUS				0x00000c80
-
-#define REG_A4XX_GRAS_DEBUG_ECO_CONTROL				0x00000c81
-
-#define REG_A4XX_GRAS_PERFCTR_TSE_SEL_0				0x00000c88
-
-#define REG_A4XX_GRAS_PERFCTR_TSE_SEL_1				0x00000c89
-
-#define REG_A4XX_GRAS_PERFCTR_TSE_SEL_2				0x00000c8a
-
-#define REG_A4XX_GRAS_PERFCTR_TSE_SEL_3				0x00000c8b
-
-#define REG_A4XX_GRAS_PERFCTR_RAS_SEL_0				0x00000c8c
-
-#define REG_A4XX_GRAS_PERFCTR_RAS_SEL_1				0x00000c8d
-
-#define REG_A4XX_GRAS_PERFCTR_RAS_SEL_2				0x00000c8e
-
-#define REG_A4XX_GRAS_PERFCTR_RAS_SEL_3				0x00000c8f
-
-#define REG_A4XX_GRAS_CL_CLIP_CNTL				0x00002000
-#define A4XX_GRAS_CL_CLIP_CNTL_CLIP_DISABLE			0x00008000
-#define A4XX_GRAS_CL_CLIP_CNTL_ZNEAR_CLIP_DISABLE		0x00010000
-#define A4XX_GRAS_CL_CLIP_CNTL_ZFAR_CLIP_DISABLE		0x00020000
-#define A4XX_GRAS_CL_CLIP_CNTL_ZERO_GB_SCALE_Z			0x00400000
-
-#define REG_A4XX_GRAS_CNTL					0x00002003
-#define A4XX_GRAS_CNTL_IJ_PERSP					0x00000001
-#define A4XX_GRAS_CNTL_IJ_LINEAR				0x00000002
-
-#define REG_A4XX_GRAS_CL_GB_CLIP_ADJ				0x00002004
-#define A4XX_GRAS_CL_GB_CLIP_ADJ_HORZ__MASK			0x000003ff
-#define A4XX_GRAS_CL_GB_CLIP_ADJ_HORZ__SHIFT			0
-static inline uint32_t A4XX_GRAS_CL_GB_CLIP_ADJ_HORZ(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_CL_GB_CLIP_ADJ_HORZ__SHIFT) & A4XX_GRAS_CL_GB_CLIP_ADJ_HORZ__MASK;
-}
-#define A4XX_GRAS_CL_GB_CLIP_ADJ_VERT__MASK			0x000ffc00
-#define A4XX_GRAS_CL_GB_CLIP_ADJ_VERT__SHIFT			10
-static inline uint32_t A4XX_GRAS_CL_GB_CLIP_ADJ_VERT(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_CL_GB_CLIP_ADJ_VERT__SHIFT) & A4XX_GRAS_CL_GB_CLIP_ADJ_VERT__MASK;
-}
-
-#define REG_A4XX_GRAS_CL_VPORT_XOFFSET_0			0x00002008
-#define A4XX_GRAS_CL_VPORT_XOFFSET_0__MASK			0xffffffff
-#define A4XX_GRAS_CL_VPORT_XOFFSET_0__SHIFT			0
-static inline uint32_t A4XX_GRAS_CL_VPORT_XOFFSET_0(float val)
-{
-	return ((fui(val)) << A4XX_GRAS_CL_VPORT_XOFFSET_0__SHIFT) & A4XX_GRAS_CL_VPORT_XOFFSET_0__MASK;
-}
-
-#define REG_A4XX_GRAS_CL_VPORT_XSCALE_0				0x00002009
-#define A4XX_GRAS_CL_VPORT_XSCALE_0__MASK			0xffffffff
-#define A4XX_GRAS_CL_VPORT_XSCALE_0__SHIFT			0
-static inline uint32_t A4XX_GRAS_CL_VPORT_XSCALE_0(float val)
-{
-	return ((fui(val)) << A4XX_GRAS_CL_VPORT_XSCALE_0__SHIFT) & A4XX_GRAS_CL_VPORT_XSCALE_0__MASK;
-}
-
-#define REG_A4XX_GRAS_CL_VPORT_YOFFSET_0			0x0000200a
-#define A4XX_GRAS_CL_VPORT_YOFFSET_0__MASK			0xffffffff
-#define A4XX_GRAS_CL_VPORT_YOFFSET_0__SHIFT			0
-static inline uint32_t A4XX_GRAS_CL_VPORT_YOFFSET_0(float val)
-{
-	return ((fui(val)) << A4XX_GRAS_CL_VPORT_YOFFSET_0__SHIFT) & A4XX_GRAS_CL_VPORT_YOFFSET_0__MASK;
-}
-
-#define REG_A4XX_GRAS_CL_VPORT_YSCALE_0				0x0000200b
-#define A4XX_GRAS_CL_VPORT_YSCALE_0__MASK			0xffffffff
-#define A4XX_GRAS_CL_VPORT_YSCALE_0__SHIFT			0
-static inline uint32_t A4XX_GRAS_CL_VPORT_YSCALE_0(float val)
-{
-	return ((fui(val)) << A4XX_GRAS_CL_VPORT_YSCALE_0__SHIFT) & A4XX_GRAS_CL_VPORT_YSCALE_0__MASK;
-}
-
-#define REG_A4XX_GRAS_CL_VPORT_ZOFFSET_0			0x0000200c
-#define A4XX_GRAS_CL_VPORT_ZOFFSET_0__MASK			0xffffffff
-#define A4XX_GRAS_CL_VPORT_ZOFFSET_0__SHIFT			0
-static inline uint32_t A4XX_GRAS_CL_VPORT_ZOFFSET_0(float val)
-{
-	return ((fui(val)) << A4XX_GRAS_CL_VPORT_ZOFFSET_0__SHIFT) & A4XX_GRAS_CL_VPORT_ZOFFSET_0__MASK;
-}
-
-#define REG_A4XX_GRAS_CL_VPORT_ZSCALE_0				0x0000200d
-#define A4XX_GRAS_CL_VPORT_ZSCALE_0__MASK			0xffffffff
-#define A4XX_GRAS_CL_VPORT_ZSCALE_0__SHIFT			0
-static inline uint32_t A4XX_GRAS_CL_VPORT_ZSCALE_0(float val)
-{
-	return ((fui(val)) << A4XX_GRAS_CL_VPORT_ZSCALE_0__SHIFT) & A4XX_GRAS_CL_VPORT_ZSCALE_0__MASK;
-}
-
-#define REG_A4XX_GRAS_SU_POINT_MINMAX				0x00002070
-#define A4XX_GRAS_SU_POINT_MINMAX_MIN__MASK			0x0000ffff
-#define A4XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT			0
-static inline uint32_t A4XX_GRAS_SU_POINT_MINMAX_MIN(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A4XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT) & A4XX_GRAS_SU_POINT_MINMAX_MIN__MASK;
-}
-#define A4XX_GRAS_SU_POINT_MINMAX_MAX__MASK			0xffff0000
-#define A4XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT			16
-static inline uint32_t A4XX_GRAS_SU_POINT_MINMAX_MAX(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A4XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT) & A4XX_GRAS_SU_POINT_MINMAX_MAX__MASK;
-}
-
-#define REG_A4XX_GRAS_SU_POINT_SIZE				0x00002071
-#define A4XX_GRAS_SU_POINT_SIZE__MASK				0xffffffff
-#define A4XX_GRAS_SU_POINT_SIZE__SHIFT				0
-static inline uint32_t A4XX_GRAS_SU_POINT_SIZE(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A4XX_GRAS_SU_POINT_SIZE__SHIFT) & A4XX_GRAS_SU_POINT_SIZE__MASK;
-}
-
-#define REG_A4XX_GRAS_ALPHA_CONTROL				0x00002073
-#define A4XX_GRAS_ALPHA_CONTROL_ALPHA_TEST_ENABLE		0x00000004
-#define A4XX_GRAS_ALPHA_CONTROL_FORCE_FRAGZ_TO_FS		0x00000008
-
-#define REG_A4XX_GRAS_SU_POLY_OFFSET_SCALE			0x00002074
-#define A4XX_GRAS_SU_POLY_OFFSET_SCALE__MASK			0xffffffff
-#define A4XX_GRAS_SU_POLY_OFFSET_SCALE__SHIFT			0
-static inline uint32_t A4XX_GRAS_SU_POLY_OFFSET_SCALE(float val)
-{
-	return ((fui(val)) << A4XX_GRAS_SU_POLY_OFFSET_SCALE__SHIFT) & A4XX_GRAS_SU_POLY_OFFSET_SCALE__MASK;
-}
-
-#define REG_A4XX_GRAS_SU_POLY_OFFSET_OFFSET			0x00002075
-#define A4XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK			0xffffffff
-#define A4XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT			0
-static inline uint32_t A4XX_GRAS_SU_POLY_OFFSET_OFFSET(float val)
-{
-	return ((fui(val)) << A4XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT) & A4XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK;
-}
-
-#define REG_A4XX_GRAS_SU_POLY_OFFSET_CLAMP			0x00002076
-#define A4XX_GRAS_SU_POLY_OFFSET_CLAMP__MASK			0xffffffff
-#define A4XX_GRAS_SU_POLY_OFFSET_CLAMP__SHIFT			0
-static inline uint32_t A4XX_GRAS_SU_POLY_OFFSET_CLAMP(float val)
-{
-	return ((fui(val)) << A4XX_GRAS_SU_POLY_OFFSET_CLAMP__SHIFT) & A4XX_GRAS_SU_POLY_OFFSET_CLAMP__MASK;
-}
-
-#define REG_A4XX_GRAS_DEPTH_CONTROL				0x00002077
-#define A4XX_GRAS_DEPTH_CONTROL_FORMAT__MASK			0x00000003
-#define A4XX_GRAS_DEPTH_CONTROL_FORMAT__SHIFT			0
-static inline uint32_t A4XX_GRAS_DEPTH_CONTROL_FORMAT(enum a4xx_depth_format val)
-{
-	return ((val) << A4XX_GRAS_DEPTH_CONTROL_FORMAT__SHIFT) & A4XX_GRAS_DEPTH_CONTROL_FORMAT__MASK;
-}
-
-#define REG_A4XX_GRAS_SU_MODE_CONTROL				0x00002078
-#define A4XX_GRAS_SU_MODE_CONTROL_CULL_FRONT			0x00000001
-#define A4XX_GRAS_SU_MODE_CONTROL_CULL_BACK			0x00000002
-#define A4XX_GRAS_SU_MODE_CONTROL_FRONT_CW			0x00000004
-#define A4XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__MASK		0x000007f8
-#define A4XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__SHIFT		3
-static inline uint32_t A4XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH(float val)
-{
-	return ((((int32_t)(val * 4.0))) << A4XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__SHIFT) & A4XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__MASK;
-}
-#define A4XX_GRAS_SU_MODE_CONTROL_POLY_OFFSET			0x00000800
-#define A4XX_GRAS_SU_MODE_CONTROL_MSAA_ENABLE			0x00002000
-#define A4XX_GRAS_SU_MODE_CONTROL_RENDERING_PASS		0x00100000
-
-#define REG_A4XX_GRAS_SC_CONTROL				0x0000207b
-#define A4XX_GRAS_SC_CONTROL_RENDER_MODE__MASK			0x0000000c
-#define A4XX_GRAS_SC_CONTROL_RENDER_MODE__SHIFT			2
-static inline uint32_t A4XX_GRAS_SC_CONTROL_RENDER_MODE(enum a3xx_render_mode val)
-{
-	return ((val) << A4XX_GRAS_SC_CONTROL_RENDER_MODE__SHIFT) & A4XX_GRAS_SC_CONTROL_RENDER_MODE__MASK;
-}
-#define A4XX_GRAS_SC_CONTROL_MSAA_SAMPLES__MASK			0x00000380
-#define A4XX_GRAS_SC_CONTROL_MSAA_SAMPLES__SHIFT		7
-static inline uint32_t A4XX_GRAS_SC_CONTROL_MSAA_SAMPLES(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_CONTROL_MSAA_SAMPLES__SHIFT) & A4XX_GRAS_SC_CONTROL_MSAA_SAMPLES__MASK;
-}
-#define A4XX_GRAS_SC_CONTROL_MSAA_DISABLE			0x00000800
-#define A4XX_GRAS_SC_CONTROL_RASTER_MODE__MASK			0x0000f000
-#define A4XX_GRAS_SC_CONTROL_RASTER_MODE__SHIFT			12
-static inline uint32_t A4XX_GRAS_SC_CONTROL_RASTER_MODE(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_CONTROL_RASTER_MODE__SHIFT) & A4XX_GRAS_SC_CONTROL_RASTER_MODE__MASK;
-}
-
-#define REG_A4XX_GRAS_SC_SCREEN_SCISSOR_TL			0x0000207c
-#define A4XX_GRAS_SC_SCREEN_SCISSOR_TL_WINDOW_OFFSET_DISABLE	0x80000000
-#define A4XX_GRAS_SC_SCREEN_SCISSOR_TL_X__MASK			0x00007fff
-#define A4XX_GRAS_SC_SCREEN_SCISSOR_TL_X__SHIFT			0
-static inline uint32_t A4XX_GRAS_SC_SCREEN_SCISSOR_TL_X(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_SCREEN_SCISSOR_TL_X__SHIFT) & A4XX_GRAS_SC_SCREEN_SCISSOR_TL_X__MASK;
-}
-#define A4XX_GRAS_SC_SCREEN_SCISSOR_TL_Y__MASK			0x7fff0000
-#define A4XX_GRAS_SC_SCREEN_SCISSOR_TL_Y__SHIFT			16
-static inline uint32_t A4XX_GRAS_SC_SCREEN_SCISSOR_TL_Y(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_SCREEN_SCISSOR_TL_Y__SHIFT) & A4XX_GRAS_SC_SCREEN_SCISSOR_TL_Y__MASK;
-}
-
-#define REG_A4XX_GRAS_SC_SCREEN_SCISSOR_BR			0x0000207d
-#define A4XX_GRAS_SC_SCREEN_SCISSOR_BR_WINDOW_OFFSET_DISABLE	0x80000000
-#define A4XX_GRAS_SC_SCREEN_SCISSOR_BR_X__MASK			0x00007fff
-#define A4XX_GRAS_SC_SCREEN_SCISSOR_BR_X__SHIFT			0
-static inline uint32_t A4XX_GRAS_SC_SCREEN_SCISSOR_BR_X(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_SCREEN_SCISSOR_BR_X__SHIFT) & A4XX_GRAS_SC_SCREEN_SCISSOR_BR_X__MASK;
-}
-#define A4XX_GRAS_SC_SCREEN_SCISSOR_BR_Y__MASK			0x7fff0000
-#define A4XX_GRAS_SC_SCREEN_SCISSOR_BR_Y__SHIFT			16
-static inline uint32_t A4XX_GRAS_SC_SCREEN_SCISSOR_BR_Y(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_SCREEN_SCISSOR_BR_Y__SHIFT) & A4XX_GRAS_SC_SCREEN_SCISSOR_BR_Y__MASK;
-}
-
-#define REG_A4XX_GRAS_SC_WINDOW_SCISSOR_BR			0x0000209c
-#define A4XX_GRAS_SC_WINDOW_SCISSOR_BR_WINDOW_OFFSET_DISABLE	0x80000000
-#define A4XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK			0x00007fff
-#define A4XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT			0
-static inline uint32_t A4XX_GRAS_SC_WINDOW_SCISSOR_BR_X(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT) & A4XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK;
-}
-#define A4XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK			0x7fff0000
-#define A4XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT			16
-static inline uint32_t A4XX_GRAS_SC_WINDOW_SCISSOR_BR_Y(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT) & A4XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK;
-}
-
-#define REG_A4XX_GRAS_SC_WINDOW_SCISSOR_TL			0x0000209d
-#define A4XX_GRAS_SC_WINDOW_SCISSOR_TL_WINDOW_OFFSET_DISABLE	0x80000000
-#define A4XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK			0x00007fff
-#define A4XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT			0
-static inline uint32_t A4XX_GRAS_SC_WINDOW_SCISSOR_TL_X(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT) & A4XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK;
-}
-#define A4XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK			0x7fff0000
-#define A4XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT			16
-static inline uint32_t A4XX_GRAS_SC_WINDOW_SCISSOR_TL_Y(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT) & A4XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK;
-}
-
-#define REG_A4XX_GRAS_SC_EXTENT_WINDOW_BR			0x0000209e
-#define A4XX_GRAS_SC_EXTENT_WINDOW_BR_WINDOW_OFFSET_DISABLE	0x80000000
-#define A4XX_GRAS_SC_EXTENT_WINDOW_BR_X__MASK			0x00007fff
-#define A4XX_GRAS_SC_EXTENT_WINDOW_BR_X__SHIFT			0
-static inline uint32_t A4XX_GRAS_SC_EXTENT_WINDOW_BR_X(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_EXTENT_WINDOW_BR_X__SHIFT) & A4XX_GRAS_SC_EXTENT_WINDOW_BR_X__MASK;
-}
-#define A4XX_GRAS_SC_EXTENT_WINDOW_BR_Y__MASK			0x7fff0000
-#define A4XX_GRAS_SC_EXTENT_WINDOW_BR_Y__SHIFT			16
-static inline uint32_t A4XX_GRAS_SC_EXTENT_WINDOW_BR_Y(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_EXTENT_WINDOW_BR_Y__SHIFT) & A4XX_GRAS_SC_EXTENT_WINDOW_BR_Y__MASK;
-}
-
-#define REG_A4XX_GRAS_SC_EXTENT_WINDOW_TL			0x0000209f
-#define A4XX_GRAS_SC_EXTENT_WINDOW_TL_WINDOW_OFFSET_DISABLE	0x80000000
-#define A4XX_GRAS_SC_EXTENT_WINDOW_TL_X__MASK			0x00007fff
-#define A4XX_GRAS_SC_EXTENT_WINDOW_TL_X__SHIFT			0
-static inline uint32_t A4XX_GRAS_SC_EXTENT_WINDOW_TL_X(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_EXTENT_WINDOW_TL_X__SHIFT) & A4XX_GRAS_SC_EXTENT_WINDOW_TL_X__MASK;
-}
-#define A4XX_GRAS_SC_EXTENT_WINDOW_TL_Y__MASK			0x7fff0000
-#define A4XX_GRAS_SC_EXTENT_WINDOW_TL_Y__SHIFT			16
-static inline uint32_t A4XX_GRAS_SC_EXTENT_WINDOW_TL_Y(uint32_t val)
-{
-	return ((val) << A4XX_GRAS_SC_EXTENT_WINDOW_TL_Y__SHIFT) & A4XX_GRAS_SC_EXTENT_WINDOW_TL_Y__MASK;
-}
-
-#define REG_A4XX_UCHE_CACHE_MODE_CONTROL			0x00000e80
-
-#define REG_A4XX_UCHE_TRAP_BASE_LO				0x00000e83
-
-#define REG_A4XX_UCHE_TRAP_BASE_HI				0x00000e84
-
-#define REG_A4XX_UCHE_CACHE_STATUS				0x00000e88
-
-#define REG_A4XX_UCHE_INVALIDATE0				0x00000e8a
-
-#define REG_A4XX_UCHE_INVALIDATE1				0x00000e8b
-
-#define REG_A4XX_UCHE_CACHE_WAYS_VFD				0x00000e8c
-
-#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_0			0x00000e8e
-
-#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_1			0x00000e8f
-
-#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_2			0x00000e90
-
-#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_3			0x00000e91
-
-#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_4			0x00000e92
-
-#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_5			0x00000e93
-
-#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_6			0x00000e94
-
-#define REG_A4XX_UCHE_PERFCTR_UCHE_SEL_7			0x00000e95
-
-#define REG_A4XX_HLSQ_TIMEOUT_THRESHOLD				0x00000e00
-
-#define REG_A4XX_HLSQ_DEBUG_ECO_CONTROL				0x00000e04
-
-#define REG_A4XX_HLSQ_MODE_CONTROL				0x00000e05
-
-#define REG_A4XX_HLSQ_PERF_PIPE_MASK				0x00000e0e
-
-#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_0			0x00000e06
-
-#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_1			0x00000e07
-
-#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_2			0x00000e08
-
-#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_3			0x00000e09
-
-#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_4			0x00000e0a
-
-#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_5			0x00000e0b
-
-#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_6			0x00000e0c
-
-#define REG_A4XX_HLSQ_PERFCTR_HLSQ_SEL_7			0x00000e0d
-
-#define REG_A4XX_HLSQ_CONTROL_0_REG				0x000023c0
-#define A4XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__MASK		0x00000010
-#define A4XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__SHIFT		4
-static inline uint32_t A4XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__SHIFT) & A4XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__MASK;
-}
-#define A4XX_HLSQ_CONTROL_0_REG_FSSUPERTHREADENABLE		0x00000040
-#define A4XX_HLSQ_CONTROL_0_REG_SPSHADERRESTART			0x00000200
-#define A4XX_HLSQ_CONTROL_0_REG_RESERVED2			0x00000400
-#define A4XX_HLSQ_CONTROL_0_REG_CHUNKDISABLE			0x04000000
-#define A4XX_HLSQ_CONTROL_0_REG_CONSTMODE__MASK			0x08000000
-#define A4XX_HLSQ_CONTROL_0_REG_CONSTMODE__SHIFT		27
-static inline uint32_t A4XX_HLSQ_CONTROL_0_REG_CONSTMODE(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_0_REG_CONSTMODE__SHIFT) & A4XX_HLSQ_CONTROL_0_REG_CONSTMODE__MASK;
-}
-#define A4XX_HLSQ_CONTROL_0_REG_LAZYUPDATEDISABLE		0x10000000
-#define A4XX_HLSQ_CONTROL_0_REG_SPCONSTFULLUPDATE		0x20000000
-#define A4XX_HLSQ_CONTROL_0_REG_TPFULLUPDATE			0x40000000
-#define A4XX_HLSQ_CONTROL_0_REG_SINGLECONTEXT			0x80000000
-
-#define REG_A4XX_HLSQ_CONTROL_1_REG				0x000023c1
-#define A4XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__MASK		0x00000040
-#define A4XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__SHIFT		6
-static inline uint32_t A4XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__SHIFT) & A4XX_HLSQ_CONTROL_1_REG_VSTHREADSIZE__MASK;
-}
-#define A4XX_HLSQ_CONTROL_1_REG_VSSUPERTHREADENABLE		0x00000100
-#define A4XX_HLSQ_CONTROL_1_REG_RESERVED1			0x00000200
-#define A4XX_HLSQ_CONTROL_1_REG_COORDREGID__MASK		0x00ff0000
-#define A4XX_HLSQ_CONTROL_1_REG_COORDREGID__SHIFT		16
-static inline uint32_t A4XX_HLSQ_CONTROL_1_REG_COORDREGID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_1_REG_COORDREGID__SHIFT) & A4XX_HLSQ_CONTROL_1_REG_COORDREGID__MASK;
-}
-#define A4XX_HLSQ_CONTROL_1_REG_ZWCOORDREGID__MASK		0xff000000
-#define A4XX_HLSQ_CONTROL_1_REG_ZWCOORDREGID__SHIFT		24
-static inline uint32_t A4XX_HLSQ_CONTROL_1_REG_ZWCOORDREGID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_1_REG_ZWCOORDREGID__SHIFT) & A4XX_HLSQ_CONTROL_1_REG_ZWCOORDREGID__MASK;
-}
-
-#define REG_A4XX_HLSQ_CONTROL_2_REG				0x000023c2
-#define A4XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD__MASK	0xfc000000
-#define A4XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD__SHIFT	26
-static inline uint32_t A4XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD__SHIFT) & A4XX_HLSQ_CONTROL_2_REG_PRIMALLOCTHRESHOLD__MASK;
-}
-#define A4XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK			0x000003fc
-#define A4XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT		2
-static inline uint32_t A4XX_HLSQ_CONTROL_2_REG_FACEREGID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT) & A4XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK;
-}
-#define A4XX_HLSQ_CONTROL_2_REG_SAMPLEID_REGID__MASK		0x0003fc00
-#define A4XX_HLSQ_CONTROL_2_REG_SAMPLEID_REGID__SHIFT		10
-static inline uint32_t A4XX_HLSQ_CONTROL_2_REG_SAMPLEID_REGID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_2_REG_SAMPLEID_REGID__SHIFT) & A4XX_HLSQ_CONTROL_2_REG_SAMPLEID_REGID__MASK;
-}
-#define A4XX_HLSQ_CONTROL_2_REG_SAMPLEMASK_REGID__MASK		0x03fc0000
-#define A4XX_HLSQ_CONTROL_2_REG_SAMPLEMASK_REGID__SHIFT		18
-static inline uint32_t A4XX_HLSQ_CONTROL_2_REG_SAMPLEMASK_REGID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_2_REG_SAMPLEMASK_REGID__SHIFT) & A4XX_HLSQ_CONTROL_2_REG_SAMPLEMASK_REGID__MASK;
-}
-
-#define REG_A4XX_HLSQ_CONTROL_3_REG				0x000023c3
-#define A4XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__MASK		0x000000ff
-#define A4XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__SHIFT		0
-static inline uint32_t A4XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__SHIFT) & A4XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__MASK;
-}
-#define A4XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__MASK		0x0000ff00
-#define A4XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__SHIFT		8
-static inline uint32_t A4XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__SHIFT) & A4XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__MASK;
-}
-#define A4XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__MASK		0x00ff0000
-#define A4XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__SHIFT	16
-static inline uint32_t A4XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__SHIFT) & A4XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__MASK;
-}
-#define A4XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__MASK	0xff000000
-#define A4XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__SHIFT	24
-static inline uint32_t A4XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__SHIFT) & A4XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__MASK;
-}
-
-#define REG_A4XX_HLSQ_CONTROL_4_REG				0x000023c4
-#define A4XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__MASK		0x000000ff
-#define A4XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__SHIFT		0
-static inline uint32_t A4XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__SHIFT) & A4XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__MASK;
-}
-#define A4XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__MASK		0x0000ff00
-#define A4XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__SHIFT		8
-static inline uint32_t A4XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__SHIFT) & A4XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__MASK;
-}
-
-#define REG_A4XX_HLSQ_VS_CONTROL_REG				0x000023c5
-#define A4XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__MASK		0x000000ff
-#define A4XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__SHIFT		0
-static inline uint32_t A4XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__MASK;
-}
-#define A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x00007f00
-#define A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	8
-static inline uint32_t A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A4XX_HLSQ_VS_CONTROL_REG_SSBO_ENABLE			0x00008000
-#define A4XX_HLSQ_VS_CONTROL_REG_ENABLED			0x00010000
-#define A4XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00fe0000
-#define A4XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		17
-static inline uint32_t A4XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A4XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET__MASK;
-}
-#define A4XX_HLSQ_VS_CONTROL_REG_INSTRLENGTH__MASK		0xff000000
-#define A4XX_HLSQ_VS_CONTROL_REG_INSTRLENGTH__SHIFT		24
-static inline uint32_t A4XX_HLSQ_VS_CONTROL_REG_INSTRLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_VS_CONTROL_REG_INSTRLENGTH__SHIFT) & A4XX_HLSQ_VS_CONTROL_REG_INSTRLENGTH__MASK;
-}
-
-#define REG_A4XX_HLSQ_FS_CONTROL_REG				0x000023c6
-#define A4XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__MASK		0x000000ff
-#define A4XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__SHIFT		0
-static inline uint32_t A4XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__MASK;
-}
-#define A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x00007f00
-#define A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	8
-static inline uint32_t A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A4XX_HLSQ_FS_CONTROL_REG_SSBO_ENABLE			0x00008000
-#define A4XX_HLSQ_FS_CONTROL_REG_ENABLED			0x00010000
-#define A4XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00fe0000
-#define A4XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		17
-static inline uint32_t A4XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A4XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET__MASK;
-}
-#define A4XX_HLSQ_FS_CONTROL_REG_INSTRLENGTH__MASK		0xff000000
-#define A4XX_HLSQ_FS_CONTROL_REG_INSTRLENGTH__SHIFT		24
-static inline uint32_t A4XX_HLSQ_FS_CONTROL_REG_INSTRLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_FS_CONTROL_REG_INSTRLENGTH__SHIFT) & A4XX_HLSQ_FS_CONTROL_REG_INSTRLENGTH__MASK;
-}
-
-#define REG_A4XX_HLSQ_HS_CONTROL_REG				0x000023c7
-#define A4XX_HLSQ_HS_CONTROL_REG_CONSTLENGTH__MASK		0x000000ff
-#define A4XX_HLSQ_HS_CONTROL_REG_CONSTLENGTH__SHIFT		0
-static inline uint32_t A4XX_HLSQ_HS_CONTROL_REG_CONSTLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_HS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_HS_CONTROL_REG_CONSTLENGTH__MASK;
-}
-#define A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x00007f00
-#define A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	8
-static inline uint32_t A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A4XX_HLSQ_HS_CONTROL_REG_SSBO_ENABLE			0x00008000
-#define A4XX_HLSQ_HS_CONTROL_REG_ENABLED			0x00010000
-#define A4XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00fe0000
-#define A4XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		17
-static inline uint32_t A4XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A4XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET__MASK;
-}
-#define A4XX_HLSQ_HS_CONTROL_REG_INSTRLENGTH__MASK		0xff000000
-#define A4XX_HLSQ_HS_CONTROL_REG_INSTRLENGTH__SHIFT		24
-static inline uint32_t A4XX_HLSQ_HS_CONTROL_REG_INSTRLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_HS_CONTROL_REG_INSTRLENGTH__SHIFT) & A4XX_HLSQ_HS_CONTROL_REG_INSTRLENGTH__MASK;
-}
-
-#define REG_A4XX_HLSQ_DS_CONTROL_REG				0x000023c8
-#define A4XX_HLSQ_DS_CONTROL_REG_CONSTLENGTH__MASK		0x000000ff
-#define A4XX_HLSQ_DS_CONTROL_REG_CONSTLENGTH__SHIFT		0
-static inline uint32_t A4XX_HLSQ_DS_CONTROL_REG_CONSTLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_DS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_DS_CONTROL_REG_CONSTLENGTH__MASK;
-}
-#define A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x00007f00
-#define A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	8
-static inline uint32_t A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A4XX_HLSQ_DS_CONTROL_REG_SSBO_ENABLE			0x00008000
-#define A4XX_HLSQ_DS_CONTROL_REG_ENABLED			0x00010000
-#define A4XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00fe0000
-#define A4XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		17
-static inline uint32_t A4XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A4XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET__MASK;
-}
-#define A4XX_HLSQ_DS_CONTROL_REG_INSTRLENGTH__MASK		0xff000000
-#define A4XX_HLSQ_DS_CONTROL_REG_INSTRLENGTH__SHIFT		24
-static inline uint32_t A4XX_HLSQ_DS_CONTROL_REG_INSTRLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_DS_CONTROL_REG_INSTRLENGTH__SHIFT) & A4XX_HLSQ_DS_CONTROL_REG_INSTRLENGTH__MASK;
-}
-
-#define REG_A4XX_HLSQ_GS_CONTROL_REG				0x000023c9
-#define A4XX_HLSQ_GS_CONTROL_REG_CONSTLENGTH__MASK		0x000000ff
-#define A4XX_HLSQ_GS_CONTROL_REG_CONSTLENGTH__SHIFT		0
-static inline uint32_t A4XX_HLSQ_GS_CONTROL_REG_CONSTLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_GS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_GS_CONTROL_REG_CONSTLENGTH__MASK;
-}
-#define A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x00007f00
-#define A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	8
-static inline uint32_t A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A4XX_HLSQ_GS_CONTROL_REG_SSBO_ENABLE			0x00008000
-#define A4XX_HLSQ_GS_CONTROL_REG_ENABLED			0x00010000
-#define A4XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00fe0000
-#define A4XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		17
-static inline uint32_t A4XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A4XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET__MASK;
-}
-#define A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH__MASK		0xff000000
-#define A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH__SHIFT		24
-static inline uint32_t A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH__SHIFT) & A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH__MASK;
-}
-
-#define REG_A4XX_HLSQ_CS_CONTROL_REG				0x000023ca
-#define A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH__MASK		0x000000ff
-#define A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH__SHIFT		0
-static inline uint32_t A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH__MASK;
-}
-#define A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x00007f00
-#define A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	8
-static inline uint32_t A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
-}
-#define A4XX_HLSQ_CS_CONTROL_REG_SSBO_ENABLE			0x00008000
-#define A4XX_HLSQ_CS_CONTROL_REG_ENABLED			0x00010000
-#define A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00fe0000
-#define A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		17
-static inline uint32_t A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET__MASK;
-}
-#define A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH__MASK		0xff000000
-#define A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH__SHIFT		24
-static inline uint32_t A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH__SHIFT) & A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH__MASK;
-}
-
-#define REG_A4XX_HLSQ_CL_NDRANGE_0				0x000023cd
-#define A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM__MASK			0x00000003
-#define A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM__SHIFT			0
-static inline uint32_t A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM__SHIFT) & A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM__MASK;
-}
-#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX__MASK			0x00000ffc
-#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX__SHIFT		2
-static inline uint32_t A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX__SHIFT) & A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX__MASK;
-}
-#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY__MASK			0x003ff000
-#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY__SHIFT		12
-static inline uint32_t A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY__SHIFT) & A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY__MASK;
-}
-#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ__MASK			0xffc00000
-#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ__SHIFT		22
-static inline uint32_t A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ__SHIFT) & A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ__MASK;
-}
-
-#define REG_A4XX_HLSQ_CL_NDRANGE_1				0x000023ce
-#define A4XX_HLSQ_CL_NDRANGE_1_SIZE_X__MASK			0xffffffff
-#define A4XX_HLSQ_CL_NDRANGE_1_SIZE_X__SHIFT			0
-static inline uint32_t A4XX_HLSQ_CL_NDRANGE_1_SIZE_X(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_NDRANGE_1_SIZE_X__SHIFT) & A4XX_HLSQ_CL_NDRANGE_1_SIZE_X__MASK;
-}
-
-#define REG_A4XX_HLSQ_CL_NDRANGE_2				0x000023cf
-
-#define REG_A4XX_HLSQ_CL_NDRANGE_3				0x000023d0
-#define A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y__MASK			0xffffffff
-#define A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y__SHIFT			0
-static inline uint32_t A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y__SHIFT) & A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y__MASK;
-}
-
-#define REG_A4XX_HLSQ_CL_NDRANGE_4				0x000023d1
-
-#define REG_A4XX_HLSQ_CL_NDRANGE_5				0x000023d2
-#define A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z__MASK			0xffffffff
-#define A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z__SHIFT			0
-static inline uint32_t A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z__SHIFT) & A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z__MASK;
-}
-
-#define REG_A4XX_HLSQ_CL_NDRANGE_6				0x000023d3
-
-#define REG_A4XX_HLSQ_CL_CONTROL_0				0x000023d4
-#define A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID__MASK		0x00000fff
-#define A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID__SHIFT		0
-static inline uint32_t A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID__SHIFT) & A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID__MASK;
-}
-#define A4XX_HLSQ_CL_CONTROL_0_KERNELDIMCONSTID__MASK		0x00fff000
-#define A4XX_HLSQ_CL_CONTROL_0_KERNELDIMCONSTID__SHIFT		12
-static inline uint32_t A4XX_HLSQ_CL_CONTROL_0_KERNELDIMCONSTID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_CONTROL_0_KERNELDIMCONSTID__SHIFT) & A4XX_HLSQ_CL_CONTROL_0_KERNELDIMCONSTID__MASK;
-}
-#define A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID__MASK		0xff000000
-#define A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID__SHIFT		24
-static inline uint32_t A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID__SHIFT) & A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID__MASK;
-}
-
-#define REG_A4XX_HLSQ_CL_CONTROL_1				0x000023d5
-#define A4XX_HLSQ_CL_CONTROL_1_UNK0CONSTID__MASK		0x00000fff
-#define A4XX_HLSQ_CL_CONTROL_1_UNK0CONSTID__SHIFT		0
-static inline uint32_t A4XX_HLSQ_CL_CONTROL_1_UNK0CONSTID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_CONTROL_1_UNK0CONSTID__SHIFT) & A4XX_HLSQ_CL_CONTROL_1_UNK0CONSTID__MASK;
-}
-#define A4XX_HLSQ_CL_CONTROL_1_WORKGROUPSIZECONSTID__MASK	0x00fff000
-#define A4XX_HLSQ_CL_CONTROL_1_WORKGROUPSIZECONSTID__SHIFT	12
-static inline uint32_t A4XX_HLSQ_CL_CONTROL_1_WORKGROUPSIZECONSTID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_CONTROL_1_WORKGROUPSIZECONSTID__SHIFT) & A4XX_HLSQ_CL_CONTROL_1_WORKGROUPSIZECONSTID__MASK;
-}
-
-#define REG_A4XX_HLSQ_CL_KERNEL_CONST				0x000023d6
-#define A4XX_HLSQ_CL_KERNEL_CONST_UNK0CONSTID__MASK		0x00000fff
-#define A4XX_HLSQ_CL_KERNEL_CONST_UNK0CONSTID__SHIFT		0
-static inline uint32_t A4XX_HLSQ_CL_KERNEL_CONST_UNK0CONSTID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_KERNEL_CONST_UNK0CONSTID__SHIFT) & A4XX_HLSQ_CL_KERNEL_CONST_UNK0CONSTID__MASK;
-}
-#define A4XX_HLSQ_CL_KERNEL_CONST_NUMWGCONSTID__MASK		0x00fff000
-#define A4XX_HLSQ_CL_KERNEL_CONST_NUMWGCONSTID__SHIFT		12
-static inline uint32_t A4XX_HLSQ_CL_KERNEL_CONST_NUMWGCONSTID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_KERNEL_CONST_NUMWGCONSTID__SHIFT) & A4XX_HLSQ_CL_KERNEL_CONST_NUMWGCONSTID__MASK;
-}
-
-#define REG_A4XX_HLSQ_CL_KERNEL_GROUP_X				0x000023d7
-
-#define REG_A4XX_HLSQ_CL_KERNEL_GROUP_Y				0x000023d8
-
-#define REG_A4XX_HLSQ_CL_KERNEL_GROUP_Z				0x000023d9
-
-#define REG_A4XX_HLSQ_CL_WG_OFFSET				0x000023da
-#define A4XX_HLSQ_CL_WG_OFFSET_UNK0CONSTID__MASK		0x00000fff
-#define A4XX_HLSQ_CL_WG_OFFSET_UNK0CONSTID__SHIFT		0
-static inline uint32_t A4XX_HLSQ_CL_WG_OFFSET_UNK0CONSTID(uint32_t val)
-{
-	return ((val) << A4XX_HLSQ_CL_WG_OFFSET_UNK0CONSTID__SHIFT) & A4XX_HLSQ_CL_WG_OFFSET_UNK0CONSTID__MASK;
-}
-
-#define REG_A4XX_HLSQ_UPDATE_CONTROL				0x000023db
-
-#define REG_A4XX_PC_BINNING_COMMAND				0x00000d00
-#define A4XX_PC_BINNING_COMMAND_BINNING_ENABLE			0x00000001
-
-#define REG_A4XX_PC_TESSFACTOR_ADDR				0x00000d08
-
-#define REG_A4XX_PC_DRAWCALL_SETUP_OVERRIDE			0x00000d0c
-
-#define REG_A4XX_PC_PERFCTR_PC_SEL_0				0x00000d10
-
-#define REG_A4XX_PC_PERFCTR_PC_SEL_1				0x00000d11
-
-#define REG_A4XX_PC_PERFCTR_PC_SEL_2				0x00000d12
-
-#define REG_A4XX_PC_PERFCTR_PC_SEL_3				0x00000d13
-
-#define REG_A4XX_PC_PERFCTR_PC_SEL_4				0x00000d14
-
-#define REG_A4XX_PC_PERFCTR_PC_SEL_5				0x00000d15
-
-#define REG_A4XX_PC_PERFCTR_PC_SEL_6				0x00000d16
-
-#define REG_A4XX_PC_PERFCTR_PC_SEL_7				0x00000d17
-
-#define REG_A4XX_PC_BIN_BASE					0x000021c0
-
-#define REG_A4XX_PC_VSTREAM_CONTROL				0x000021c2
-#define A4XX_PC_VSTREAM_CONTROL_SIZE__MASK			0x003f0000
-#define A4XX_PC_VSTREAM_CONTROL_SIZE__SHIFT			16
-static inline uint32_t A4XX_PC_VSTREAM_CONTROL_SIZE(uint32_t val)
-{
-	return ((val) << A4XX_PC_VSTREAM_CONTROL_SIZE__SHIFT) & A4XX_PC_VSTREAM_CONTROL_SIZE__MASK;
-}
-#define A4XX_PC_VSTREAM_CONTROL_N__MASK				0x07c00000
-#define A4XX_PC_VSTREAM_CONTROL_N__SHIFT			22
-static inline uint32_t A4XX_PC_VSTREAM_CONTROL_N(uint32_t val)
-{
-	return ((val) << A4XX_PC_VSTREAM_CONTROL_N__SHIFT) & A4XX_PC_VSTREAM_CONTROL_N__MASK;
-}
-
-#define REG_A4XX_PC_PRIM_VTX_CNTL				0x000021c4
-#define A4XX_PC_PRIM_VTX_CNTL_VAROUT__MASK			0x0000000f
-#define A4XX_PC_PRIM_VTX_CNTL_VAROUT__SHIFT			0
-static inline uint32_t A4XX_PC_PRIM_VTX_CNTL_VAROUT(uint32_t val)
-{
-	return ((val) << A4XX_PC_PRIM_VTX_CNTL_VAROUT__SHIFT) & A4XX_PC_PRIM_VTX_CNTL_VAROUT__MASK;
-}
-#define A4XX_PC_PRIM_VTX_CNTL_PRIMITIVE_RESTART			0x00100000
-#define A4XX_PC_PRIM_VTX_CNTL_PROVOKING_VTX_LAST		0x02000000
-#define A4XX_PC_PRIM_VTX_CNTL_PSIZE				0x04000000
-
-#define REG_A4XX_PC_PRIM_VTX_CNTL2				0x000021c5
-#define A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_FRONT_PTYPE__MASK	0x00000007
-#define A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_FRONT_PTYPE__SHIFT	0
-static inline uint32_t A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_FRONT_PTYPE(enum adreno_pa_su_sc_draw val)
-{
-	return ((val) << A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_FRONT_PTYPE__SHIFT) & A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_FRONT_PTYPE__MASK;
-}
-#define A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_BACK_PTYPE__MASK	0x00000038
-#define A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_BACK_PTYPE__SHIFT	3
-static inline uint32_t A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_BACK_PTYPE(enum adreno_pa_su_sc_draw val)
-{
-	return ((val) << A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_BACK_PTYPE__SHIFT) & A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_BACK_PTYPE__MASK;
-}
-#define A4XX_PC_PRIM_VTX_CNTL2_POLYMODE_ENABLE			0x00000040
-
-#define REG_A4XX_PC_RESTART_INDEX				0x000021c6
-
-#define REG_A4XX_PC_GS_PARAM					0x000021e5
-#define A4XX_PC_GS_PARAM_MAX_VERTICES__MASK			0x000003ff
-#define A4XX_PC_GS_PARAM_MAX_VERTICES__SHIFT			0
-static inline uint32_t A4XX_PC_GS_PARAM_MAX_VERTICES(uint32_t val)
-{
-	return ((val) << A4XX_PC_GS_PARAM_MAX_VERTICES__SHIFT) & A4XX_PC_GS_PARAM_MAX_VERTICES__MASK;
-}
-#define A4XX_PC_GS_PARAM_INVOCATIONS__MASK			0x0000f800
-#define A4XX_PC_GS_PARAM_INVOCATIONS__SHIFT			11
-static inline uint32_t A4XX_PC_GS_PARAM_INVOCATIONS(uint32_t val)
-{
-	return ((val) << A4XX_PC_GS_PARAM_INVOCATIONS__SHIFT) & A4XX_PC_GS_PARAM_INVOCATIONS__MASK;
-}
-#define A4XX_PC_GS_PARAM_PRIMTYPE__MASK				0x01800000
-#define A4XX_PC_GS_PARAM_PRIMTYPE__SHIFT			23
-static inline uint32_t A4XX_PC_GS_PARAM_PRIMTYPE(enum adreno_pa_su_sc_draw val)
-{
-	return ((val) << A4XX_PC_GS_PARAM_PRIMTYPE__SHIFT) & A4XX_PC_GS_PARAM_PRIMTYPE__MASK;
-}
-#define A4XX_PC_GS_PARAM_LAYER					0x80000000
-
-#define REG_A4XX_PC_HS_PARAM					0x000021e7
-#define A4XX_PC_HS_PARAM_VERTICES_OUT__MASK			0x0000003f
-#define A4XX_PC_HS_PARAM_VERTICES_OUT__SHIFT			0
-static inline uint32_t A4XX_PC_HS_PARAM_VERTICES_OUT(uint32_t val)
-{
-	return ((val) << A4XX_PC_HS_PARAM_VERTICES_OUT__SHIFT) & A4XX_PC_HS_PARAM_VERTICES_OUT__MASK;
-}
-#define A4XX_PC_HS_PARAM_SPACING__MASK				0x00600000
-#define A4XX_PC_HS_PARAM_SPACING__SHIFT				21
-static inline uint32_t A4XX_PC_HS_PARAM_SPACING(enum a4xx_tess_spacing val)
-{
-	return ((val) << A4XX_PC_HS_PARAM_SPACING__SHIFT) & A4XX_PC_HS_PARAM_SPACING__MASK;
-}
-#define A4XX_PC_HS_PARAM_CW					0x00800000
-#define A4XX_PC_HS_PARAM_CONNECTED				0x01000000
-
-#define REG_A4XX_VBIF_VERSION					0x00003000
-
-#define REG_A4XX_VBIF_CLKON					0x00003001
-#define A4XX_VBIF_CLKON_FORCE_ON_TESTBUS			0x00000001
-
-#define REG_A4XX_VBIF_ABIT_SORT					0x0000301c
-
-#define REG_A4XX_VBIF_ABIT_SORT_CONF				0x0000301d
-
-#define REG_A4XX_VBIF_GATE_OFF_WRREQ_EN				0x0000302a
-
-#define REG_A4XX_VBIF_IN_RD_LIM_CONF0				0x0000302c
-
-#define REG_A4XX_VBIF_IN_RD_LIM_CONF1				0x0000302d
-
-#define REG_A4XX_VBIF_IN_WR_LIM_CONF0				0x00003030
-
-#define REG_A4XX_VBIF_IN_WR_LIM_CONF1				0x00003031
-
-#define REG_A4XX_VBIF_ROUND_ROBIN_QOS_ARB			0x00003049
-
-#define REG_A4XX_VBIF_PERF_CNT_EN0				0x000030c0
-
-#define REG_A4XX_VBIF_PERF_CNT_EN1				0x000030c1
-
-#define REG_A4XX_VBIF_PERF_CNT_EN2				0x000030c2
-
-#define REG_A4XX_VBIF_PERF_CNT_EN3				0x000030c3
-
-#define REG_A4XX_VBIF_PERF_CNT_SEL0				0x000030d0
-
-#define REG_A4XX_VBIF_PERF_CNT_SEL1				0x000030d1
-
-#define REG_A4XX_VBIF_PERF_CNT_SEL2				0x000030d2
-
-#define REG_A4XX_VBIF_PERF_CNT_SEL3				0x000030d3
-
-#define REG_A4XX_VBIF_PERF_CNT_LOW0				0x000030d8
-
-#define REG_A4XX_VBIF_PERF_CNT_LOW1				0x000030d9
-
-#define REG_A4XX_VBIF_PERF_CNT_LOW2				0x000030da
-
-#define REG_A4XX_VBIF_PERF_CNT_LOW3				0x000030db
-
-#define REG_A4XX_VBIF_PERF_CNT_HIGH0				0x000030e0
-
-#define REG_A4XX_VBIF_PERF_CNT_HIGH1				0x000030e1
-
-#define REG_A4XX_VBIF_PERF_CNT_HIGH2				0x000030e2
-
-#define REG_A4XX_VBIF_PERF_CNT_HIGH3				0x000030e3
-
-#define REG_A4XX_VBIF_PERF_PWR_CNT_EN0				0x00003100
-
-#define REG_A4XX_VBIF_PERF_PWR_CNT_EN1				0x00003101
-
-#define REG_A4XX_VBIF_PERF_PWR_CNT_EN2				0x00003102
-
-#define REG_A4XX_UNKNOWN_0CC5					0x00000cc5
-
-#define REG_A4XX_UNKNOWN_0CC6					0x00000cc6
-
-#define REG_A4XX_UNKNOWN_0D01					0x00000d01
-
-#define REG_A4XX_UNKNOWN_0E42					0x00000e42
-
-#define REG_A4XX_UNKNOWN_0EC2					0x00000ec2
-
-#define REG_A4XX_UNKNOWN_2001					0x00002001
-
-#define REG_A4XX_UNKNOWN_209B					0x0000209b
-
-#define REG_A4XX_UNKNOWN_20EF					0x000020ef
-
-#define REG_A4XX_UNKNOWN_2152					0x00002152
-
-#define REG_A4XX_UNKNOWN_2153					0x00002153
-
-#define REG_A4XX_UNKNOWN_2154					0x00002154
-
-#define REG_A4XX_UNKNOWN_2155					0x00002155
-
-#define REG_A4XX_UNKNOWN_2156					0x00002156
-
-#define REG_A4XX_UNKNOWN_2157					0x00002157
-
-#define REG_A4XX_UNKNOWN_21C3					0x000021c3
-
-#define REG_A4XX_UNKNOWN_21E6					0x000021e6
-
-#define REG_A4XX_UNKNOWN_2209					0x00002209
-
-#define REG_A4XX_UNKNOWN_22D7					0x000022d7
-
-#define REG_A4XX_UNKNOWN_2352					0x00002352
-
-#define REG_A4XX_TEX_SAMP_0					0x00000000
-#define A4XX_TEX_SAMP_0_MIPFILTER_LINEAR_NEAR			0x00000001
-#define A4XX_TEX_SAMP_0_XY_MAG__MASK				0x00000006
-#define A4XX_TEX_SAMP_0_XY_MAG__SHIFT				1
-static inline uint32_t A4XX_TEX_SAMP_0_XY_MAG(enum a4xx_tex_filter val)
-{
-	return ((val) << A4XX_TEX_SAMP_0_XY_MAG__SHIFT) & A4XX_TEX_SAMP_0_XY_MAG__MASK;
-}
-#define A4XX_TEX_SAMP_0_XY_MIN__MASK				0x00000018
-#define A4XX_TEX_SAMP_0_XY_MIN__SHIFT				3
-static inline uint32_t A4XX_TEX_SAMP_0_XY_MIN(enum a4xx_tex_filter val)
-{
-	return ((val) << A4XX_TEX_SAMP_0_XY_MIN__SHIFT) & A4XX_TEX_SAMP_0_XY_MIN__MASK;
-}
-#define A4XX_TEX_SAMP_0_WRAP_S__MASK				0x000000e0
-#define A4XX_TEX_SAMP_0_WRAP_S__SHIFT				5
-static inline uint32_t A4XX_TEX_SAMP_0_WRAP_S(enum a4xx_tex_clamp val)
-{
-	return ((val) << A4XX_TEX_SAMP_0_WRAP_S__SHIFT) & A4XX_TEX_SAMP_0_WRAP_S__MASK;
-}
-#define A4XX_TEX_SAMP_0_WRAP_T__MASK				0x00000700
-#define A4XX_TEX_SAMP_0_WRAP_T__SHIFT				8
-static inline uint32_t A4XX_TEX_SAMP_0_WRAP_T(enum a4xx_tex_clamp val)
-{
-	return ((val) << A4XX_TEX_SAMP_0_WRAP_T__SHIFT) & A4XX_TEX_SAMP_0_WRAP_T__MASK;
-}
-#define A4XX_TEX_SAMP_0_WRAP_R__MASK				0x00003800
-#define A4XX_TEX_SAMP_0_WRAP_R__SHIFT				11
-static inline uint32_t A4XX_TEX_SAMP_0_WRAP_R(enum a4xx_tex_clamp val)
-{
-	return ((val) << A4XX_TEX_SAMP_0_WRAP_R__SHIFT) & A4XX_TEX_SAMP_0_WRAP_R__MASK;
-}
-#define A4XX_TEX_SAMP_0_ANISO__MASK				0x0001c000
-#define A4XX_TEX_SAMP_0_ANISO__SHIFT				14
-static inline uint32_t A4XX_TEX_SAMP_0_ANISO(enum a4xx_tex_aniso val)
-{
-	return ((val) << A4XX_TEX_SAMP_0_ANISO__SHIFT) & A4XX_TEX_SAMP_0_ANISO__MASK;
-}
-#define A4XX_TEX_SAMP_0_LOD_BIAS__MASK				0xfff80000
-#define A4XX_TEX_SAMP_0_LOD_BIAS__SHIFT				19
-static inline uint32_t A4XX_TEX_SAMP_0_LOD_BIAS(float val)
-{
-	return ((((int32_t)(val * 256.0))) << A4XX_TEX_SAMP_0_LOD_BIAS__SHIFT) & A4XX_TEX_SAMP_0_LOD_BIAS__MASK;
-}
-
-#define REG_A4XX_TEX_SAMP_1					0x00000001
-#define A4XX_TEX_SAMP_1_COMPARE_FUNC__MASK			0x0000000e
-#define A4XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT			1
-static inline uint32_t A4XX_TEX_SAMP_1_COMPARE_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A4XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT) & A4XX_TEX_SAMP_1_COMPARE_FUNC__MASK;
-}
-#define A4XX_TEX_SAMP_1_CUBEMAPSEAMLESSFILTOFF			0x00000010
-#define A4XX_TEX_SAMP_1_UNNORM_COORDS				0x00000020
-#define A4XX_TEX_SAMP_1_MIPFILTER_LINEAR_FAR			0x00000040
-#define A4XX_TEX_SAMP_1_MAX_LOD__MASK				0x000fff00
-#define A4XX_TEX_SAMP_1_MAX_LOD__SHIFT				8
-static inline uint32_t A4XX_TEX_SAMP_1_MAX_LOD(float val)
-{
-	return ((((uint32_t)(val * 256.0))) << A4XX_TEX_SAMP_1_MAX_LOD__SHIFT) & A4XX_TEX_SAMP_1_MAX_LOD__MASK;
-}
-#define A4XX_TEX_SAMP_1_MIN_LOD__MASK				0xfff00000
-#define A4XX_TEX_SAMP_1_MIN_LOD__SHIFT				20
-static inline uint32_t A4XX_TEX_SAMP_1_MIN_LOD(float val)
-{
-	return ((((uint32_t)(val * 256.0))) << A4XX_TEX_SAMP_1_MIN_LOD__SHIFT) & A4XX_TEX_SAMP_1_MIN_LOD__MASK;
-}
-
-#define REG_A4XX_TEX_CONST_0					0x00000000
-#define A4XX_TEX_CONST_0_TILED					0x00000001
-#define A4XX_TEX_CONST_0_SRGB					0x00000004
-#define A4XX_TEX_CONST_0_SWIZ_X__MASK				0x00000070
-#define A4XX_TEX_CONST_0_SWIZ_X__SHIFT				4
-static inline uint32_t A4XX_TEX_CONST_0_SWIZ_X(enum a4xx_tex_swiz val)
-{
-	return ((val) << A4XX_TEX_CONST_0_SWIZ_X__SHIFT) & A4XX_TEX_CONST_0_SWIZ_X__MASK;
-}
-#define A4XX_TEX_CONST_0_SWIZ_Y__MASK				0x00000380
-#define A4XX_TEX_CONST_0_SWIZ_Y__SHIFT				7
-static inline uint32_t A4XX_TEX_CONST_0_SWIZ_Y(enum a4xx_tex_swiz val)
-{
-	return ((val) << A4XX_TEX_CONST_0_SWIZ_Y__SHIFT) & A4XX_TEX_CONST_0_SWIZ_Y__MASK;
-}
-#define A4XX_TEX_CONST_0_SWIZ_Z__MASK				0x00001c00
-#define A4XX_TEX_CONST_0_SWIZ_Z__SHIFT				10
-static inline uint32_t A4XX_TEX_CONST_0_SWIZ_Z(enum a4xx_tex_swiz val)
-{
-	return ((val) << A4XX_TEX_CONST_0_SWIZ_Z__SHIFT) & A4XX_TEX_CONST_0_SWIZ_Z__MASK;
-}
-#define A4XX_TEX_CONST_0_SWIZ_W__MASK				0x0000e000
-#define A4XX_TEX_CONST_0_SWIZ_W__SHIFT				13
-static inline uint32_t A4XX_TEX_CONST_0_SWIZ_W(enum a4xx_tex_swiz val)
-{
-	return ((val) << A4XX_TEX_CONST_0_SWIZ_W__SHIFT) & A4XX_TEX_CONST_0_SWIZ_W__MASK;
-}
-#define A4XX_TEX_CONST_0_MIPLVLS__MASK				0x000f0000
-#define A4XX_TEX_CONST_0_MIPLVLS__SHIFT				16
-static inline uint32_t A4XX_TEX_CONST_0_MIPLVLS(uint32_t val)
-{
-	return ((val) << A4XX_TEX_CONST_0_MIPLVLS__SHIFT) & A4XX_TEX_CONST_0_MIPLVLS__MASK;
-}
-#define A4XX_TEX_CONST_0_FMT__MASK				0x1fc00000
-#define A4XX_TEX_CONST_0_FMT__SHIFT				22
-static inline uint32_t A4XX_TEX_CONST_0_FMT(enum a4xx_tex_fmt val)
-{
-	return ((val) << A4XX_TEX_CONST_0_FMT__SHIFT) & A4XX_TEX_CONST_0_FMT__MASK;
-}
-#define A4XX_TEX_CONST_0_TYPE__MASK				0xe0000000
-#define A4XX_TEX_CONST_0_TYPE__SHIFT				29
-static inline uint32_t A4XX_TEX_CONST_0_TYPE(enum a4xx_tex_type val)
-{
-	return ((val) << A4XX_TEX_CONST_0_TYPE__SHIFT) & A4XX_TEX_CONST_0_TYPE__MASK;
-}
-
-#define REG_A4XX_TEX_CONST_1					0x00000001
-#define A4XX_TEX_CONST_1_HEIGHT__MASK				0x00007fff
-#define A4XX_TEX_CONST_1_HEIGHT__SHIFT				0
-static inline uint32_t A4XX_TEX_CONST_1_HEIGHT(uint32_t val)
-{
-	return ((val) << A4XX_TEX_CONST_1_HEIGHT__SHIFT) & A4XX_TEX_CONST_1_HEIGHT__MASK;
-}
-#define A4XX_TEX_CONST_1_WIDTH__MASK				0x3fff8000
-#define A4XX_TEX_CONST_1_WIDTH__SHIFT				15
-static inline uint32_t A4XX_TEX_CONST_1_WIDTH(uint32_t val)
-{
-	return ((val) << A4XX_TEX_CONST_1_WIDTH__SHIFT) & A4XX_TEX_CONST_1_WIDTH__MASK;
-}
-
-#define REG_A4XX_TEX_CONST_2					0x00000002
-#define A4XX_TEX_CONST_2_PITCHALIGN__MASK			0x0000000f
-#define A4XX_TEX_CONST_2_PITCHALIGN__SHIFT			0
-static inline uint32_t A4XX_TEX_CONST_2_PITCHALIGN(uint32_t val)
-{
-	return ((val) << A4XX_TEX_CONST_2_PITCHALIGN__SHIFT) & A4XX_TEX_CONST_2_PITCHALIGN__MASK;
-}
-#define A4XX_TEX_CONST_2_BUFFER					0x00000040
-#define A4XX_TEX_CONST_2_PITCH__MASK				0x3ffffe00
-#define A4XX_TEX_CONST_2_PITCH__SHIFT				9
-static inline uint32_t A4XX_TEX_CONST_2_PITCH(uint32_t val)
-{
-	return ((val) << A4XX_TEX_CONST_2_PITCH__SHIFT) & A4XX_TEX_CONST_2_PITCH__MASK;
-}
-#define A4XX_TEX_CONST_2_SWAP__MASK				0xc0000000
-#define A4XX_TEX_CONST_2_SWAP__SHIFT				30
-static inline uint32_t A4XX_TEX_CONST_2_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A4XX_TEX_CONST_2_SWAP__SHIFT) & A4XX_TEX_CONST_2_SWAP__MASK;
-}
-
-#define REG_A4XX_TEX_CONST_3					0x00000003
-#define A4XX_TEX_CONST_3_LAYERSZ__MASK				0x00003fff
-#define A4XX_TEX_CONST_3_LAYERSZ__SHIFT				0
-static inline uint32_t A4XX_TEX_CONST_3_LAYERSZ(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A4XX_TEX_CONST_3_LAYERSZ__SHIFT) & A4XX_TEX_CONST_3_LAYERSZ__MASK;
-}
-#define A4XX_TEX_CONST_3_DEPTH__MASK				0x7ffc0000
-#define A4XX_TEX_CONST_3_DEPTH__SHIFT				18
-static inline uint32_t A4XX_TEX_CONST_3_DEPTH(uint32_t val)
-{
-	return ((val) << A4XX_TEX_CONST_3_DEPTH__SHIFT) & A4XX_TEX_CONST_3_DEPTH__MASK;
-}
-
-#define REG_A4XX_TEX_CONST_4					0x00000004
-#define A4XX_TEX_CONST_4_LAYERSZ__MASK				0x0000000f
-#define A4XX_TEX_CONST_4_LAYERSZ__SHIFT				0
-static inline uint32_t A4XX_TEX_CONST_4_LAYERSZ(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A4XX_TEX_CONST_4_LAYERSZ__SHIFT) & A4XX_TEX_CONST_4_LAYERSZ__MASK;
-}
-#define A4XX_TEX_CONST_4_BASE__MASK				0xffffffe0
-#define A4XX_TEX_CONST_4_BASE__SHIFT				5
-static inline uint32_t A4XX_TEX_CONST_4_BASE(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A4XX_TEX_CONST_4_BASE__SHIFT) & A4XX_TEX_CONST_4_BASE__MASK;
-}
-
-#define REG_A4XX_TEX_CONST_5					0x00000005
-
-#define REG_A4XX_TEX_CONST_6					0x00000006
-
-#define REG_A4XX_TEX_CONST_7					0x00000007
-
-#define REG_A4XX_SSBO_0_0					0x00000000
-#define A4XX_SSBO_0_0_BASE__MASK				0xffffffe0
-#define A4XX_SSBO_0_0_BASE__SHIFT				5
-static inline uint32_t A4XX_SSBO_0_0_BASE(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A4XX_SSBO_0_0_BASE__SHIFT) & A4XX_SSBO_0_0_BASE__MASK;
-}
-
-#define REG_A4XX_SSBO_0_1					0x00000001
-#define A4XX_SSBO_0_1_PITCH__MASK				0x003fffff
-#define A4XX_SSBO_0_1_PITCH__SHIFT				0
-static inline uint32_t A4XX_SSBO_0_1_PITCH(uint32_t val)
-{
-	return ((val) << A4XX_SSBO_0_1_PITCH__SHIFT) & A4XX_SSBO_0_1_PITCH__MASK;
-}
-
-#define REG_A4XX_SSBO_0_2					0x00000002
-#define A4XX_SSBO_0_2_ARRAY_PITCH__MASK				0x03fff000
-#define A4XX_SSBO_0_2_ARRAY_PITCH__SHIFT			12
-static inline uint32_t A4XX_SSBO_0_2_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A4XX_SSBO_0_2_ARRAY_PITCH__SHIFT) & A4XX_SSBO_0_2_ARRAY_PITCH__MASK;
-}
-
-#define REG_A4XX_SSBO_0_3					0x00000003
-#define A4XX_SSBO_0_3_CPP__MASK					0x0000003f
-#define A4XX_SSBO_0_3_CPP__SHIFT				0
-static inline uint32_t A4XX_SSBO_0_3_CPP(uint32_t val)
-{
-	return ((val) << A4XX_SSBO_0_3_CPP__SHIFT) & A4XX_SSBO_0_3_CPP__MASK;
-}
-
-#define REG_A4XX_SSBO_1_0					0x00000000
-#define A4XX_SSBO_1_0_CPP__MASK					0x0000001f
-#define A4XX_SSBO_1_0_CPP__SHIFT				0
-static inline uint32_t A4XX_SSBO_1_0_CPP(uint32_t val)
-{
-	return ((val) << A4XX_SSBO_1_0_CPP__SHIFT) & A4XX_SSBO_1_0_CPP__MASK;
-}
-#define A4XX_SSBO_1_0_FMT__MASK					0x0000ff00
-#define A4XX_SSBO_1_0_FMT__SHIFT				8
-static inline uint32_t A4XX_SSBO_1_0_FMT(enum a4xx_color_fmt val)
-{
-	return ((val) << A4XX_SSBO_1_0_FMT__SHIFT) & A4XX_SSBO_1_0_FMT__MASK;
-}
-#define A4XX_SSBO_1_0_WIDTH__MASK				0xffff0000
-#define A4XX_SSBO_1_0_WIDTH__SHIFT				16
-static inline uint32_t A4XX_SSBO_1_0_WIDTH(uint32_t val)
-{
-	return ((val) << A4XX_SSBO_1_0_WIDTH__SHIFT) & A4XX_SSBO_1_0_WIDTH__MASK;
-}
-
-#define REG_A4XX_SSBO_1_1					0x00000001
-#define A4XX_SSBO_1_1_HEIGHT__MASK				0x0000ffff
-#define A4XX_SSBO_1_1_HEIGHT__SHIFT				0
-static inline uint32_t A4XX_SSBO_1_1_HEIGHT(uint32_t val)
-{
-	return ((val) << A4XX_SSBO_1_1_HEIGHT__SHIFT) & A4XX_SSBO_1_1_HEIGHT__MASK;
-}
-#define A4XX_SSBO_1_1_DEPTH__MASK				0xffff0000
-#define A4XX_SSBO_1_1_DEPTH__SHIFT				16
-static inline uint32_t A4XX_SSBO_1_1_DEPTH(uint32_t val)
-{
-	return ((val) << A4XX_SSBO_1_1_DEPTH__SHIFT) & A4XX_SSBO_1_1_DEPTH__MASK;
-}
-
-#ifdef __cplusplus
-#endif
-
-#endif /* A4XX_XML */
diff --git a/drivers/gpu/drm/msm/adreno/a5xx.xml.h b/drivers/gpu/drm/msm/adreno/a5xx.xml.h
deleted file mode 100644
index d66306c14986..000000000000
--- a/drivers/gpu/drm/msm/adreno/a5xx.xml.h
+++ /dev/null
@@ -1,5572 +0,0 @@
-#ifndef A5XX_XML
-#define A5XX_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng gen_header.py tool in this git repository:
-http://gitlab.freedesktop.org/mesa/mesa/
-git clone https://gitlab.freedesktop.org/mesa/mesa.git
-
-The rules-ng-ng source files this header was generated from are:
-
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a5xx.xml          ( 151693 bytes, from Wed Aug 23 10:39:39 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml  (   1572 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml (  15434 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml    (  85691 bytes, from Fri Feb 16 09:49:01 2024)
-
-Copyright (C) 2013-2024 by the following authors:
-- Rob Clark <robdclark@gmail.com> Rob Clark
-- Ilia Mirkin <imirkin@alum.mit.edu> Ilia Mirkin
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-*/
-
-#ifdef __KERNEL__
-#include <linux/bug.h>
-#define assert(x) BUG_ON(!(x))
-#else
-#include <assert.h>
-#endif
-
-#ifdef __cplusplus
-#define __struct_cast(X)
-#else
-#define __struct_cast(X) (struct X)
-#endif
-
-enum a5xx_color_fmt {
-	RB5_A8_UNORM = 2,
-	RB5_R8_UNORM = 3,
-	RB5_R8_SNORM = 4,
-	RB5_R8_UINT = 5,
-	RB5_R8_SINT = 6,
-	RB5_R4G4B4A4_UNORM = 8,
-	RB5_R5G5B5A1_UNORM = 10,
-	RB5_R5G6B5_UNORM = 14,
-	RB5_R8G8_UNORM = 15,
-	RB5_R8G8_SNORM = 16,
-	RB5_R8G8_UINT = 17,
-	RB5_R8G8_SINT = 18,
-	RB5_R16_UNORM = 21,
-	RB5_R16_SNORM = 22,
-	RB5_R16_FLOAT = 23,
-	RB5_R16_UINT = 24,
-	RB5_R16_SINT = 25,
-	RB5_R8G8B8A8_UNORM = 48,
-	RB5_R8G8B8_UNORM = 49,
-	RB5_R8G8B8A8_SNORM = 50,
-	RB5_R8G8B8A8_UINT = 51,
-	RB5_R8G8B8A8_SINT = 52,
-	RB5_R10G10B10A2_UNORM = 55,
-	RB5_R10G10B10A2_UINT = 58,
-	RB5_R11G11B10_FLOAT = 66,
-	RB5_R16G16_UNORM = 67,
-	RB5_R16G16_SNORM = 68,
-	RB5_R16G16_FLOAT = 69,
-	RB5_R16G16_UINT = 70,
-	RB5_R16G16_SINT = 71,
-	RB5_R32_FLOAT = 74,
-	RB5_R32_UINT = 75,
-	RB5_R32_SINT = 76,
-	RB5_R16G16B16A16_UNORM = 96,
-	RB5_R16G16B16A16_SNORM = 97,
-	RB5_R16G16B16A16_FLOAT = 98,
-	RB5_R16G16B16A16_UINT = 99,
-	RB5_R16G16B16A16_SINT = 100,
-	RB5_R32G32_FLOAT = 103,
-	RB5_R32G32_UINT = 104,
-	RB5_R32G32_SINT = 105,
-	RB5_R32G32B32A32_FLOAT = 130,
-	RB5_R32G32B32A32_UINT = 131,
-	RB5_R32G32B32A32_SINT = 132,
-	RB5_NONE = 255,
-};
-
-enum a5xx_tile_mode {
-	TILE5_LINEAR = 0,
-	TILE5_2 = 2,
-	TILE5_3 = 3,
-};
-
-enum a5xx_vtx_fmt {
-	VFMT5_8_UNORM = 3,
-	VFMT5_8_SNORM = 4,
-	VFMT5_8_UINT = 5,
-	VFMT5_8_SINT = 6,
-	VFMT5_8_8_UNORM = 15,
-	VFMT5_8_8_SNORM = 16,
-	VFMT5_8_8_UINT = 17,
-	VFMT5_8_8_SINT = 18,
-	VFMT5_16_UNORM = 21,
-	VFMT5_16_SNORM = 22,
-	VFMT5_16_FLOAT = 23,
-	VFMT5_16_UINT = 24,
-	VFMT5_16_SINT = 25,
-	VFMT5_8_8_8_UNORM = 33,
-	VFMT5_8_8_8_SNORM = 34,
-	VFMT5_8_8_8_UINT = 35,
-	VFMT5_8_8_8_SINT = 36,
-	VFMT5_8_8_8_8_UNORM = 48,
-	VFMT5_8_8_8_8_SNORM = 50,
-	VFMT5_8_8_8_8_UINT = 51,
-	VFMT5_8_8_8_8_SINT = 52,
-	VFMT5_10_10_10_2_UNORM = 54,
-	VFMT5_10_10_10_2_SNORM = 57,
-	VFMT5_10_10_10_2_UINT = 58,
-	VFMT5_10_10_10_2_SINT = 59,
-	VFMT5_11_11_10_FLOAT = 66,
-	VFMT5_16_16_UNORM = 67,
-	VFMT5_16_16_SNORM = 68,
-	VFMT5_16_16_FLOAT = 69,
-	VFMT5_16_16_UINT = 70,
-	VFMT5_16_16_SINT = 71,
-	VFMT5_32_UNORM = 72,
-	VFMT5_32_SNORM = 73,
-	VFMT5_32_FLOAT = 74,
-	VFMT5_32_UINT = 75,
-	VFMT5_32_SINT = 76,
-	VFMT5_32_FIXED = 77,
-	VFMT5_16_16_16_UNORM = 88,
-	VFMT5_16_16_16_SNORM = 89,
-	VFMT5_16_16_16_FLOAT = 90,
-	VFMT5_16_16_16_UINT = 91,
-	VFMT5_16_16_16_SINT = 92,
-	VFMT5_16_16_16_16_UNORM = 96,
-	VFMT5_16_16_16_16_SNORM = 97,
-	VFMT5_16_16_16_16_FLOAT = 98,
-	VFMT5_16_16_16_16_UINT = 99,
-	VFMT5_16_16_16_16_SINT = 100,
-	VFMT5_32_32_UNORM = 101,
-	VFMT5_32_32_SNORM = 102,
-	VFMT5_32_32_FLOAT = 103,
-	VFMT5_32_32_UINT = 104,
-	VFMT5_32_32_SINT = 105,
-	VFMT5_32_32_FIXED = 106,
-	VFMT5_32_32_32_UNORM = 112,
-	VFMT5_32_32_32_SNORM = 113,
-	VFMT5_32_32_32_UINT = 114,
-	VFMT5_32_32_32_SINT = 115,
-	VFMT5_32_32_32_FLOAT = 116,
-	VFMT5_32_32_32_FIXED = 117,
-	VFMT5_32_32_32_32_UNORM = 128,
-	VFMT5_32_32_32_32_SNORM = 129,
-	VFMT5_32_32_32_32_FLOAT = 130,
-	VFMT5_32_32_32_32_UINT = 131,
-	VFMT5_32_32_32_32_SINT = 132,
-	VFMT5_32_32_32_32_FIXED = 133,
-	VFMT5_NONE = 255,
-};
-
-enum a5xx_tex_fmt {
-	TFMT5_A8_UNORM = 2,
-	TFMT5_8_UNORM = 3,
-	TFMT5_8_SNORM = 4,
-	TFMT5_8_UINT = 5,
-	TFMT5_8_SINT = 6,
-	TFMT5_4_4_4_4_UNORM = 8,
-	TFMT5_5_5_5_1_UNORM = 10,
-	TFMT5_5_6_5_UNORM = 14,
-	TFMT5_8_8_UNORM = 15,
-	TFMT5_8_8_SNORM = 16,
-	TFMT5_8_8_UINT = 17,
-	TFMT5_8_8_SINT = 18,
-	TFMT5_L8_A8_UNORM = 19,
-	TFMT5_16_UNORM = 21,
-	TFMT5_16_SNORM = 22,
-	TFMT5_16_FLOAT = 23,
-	TFMT5_16_UINT = 24,
-	TFMT5_16_SINT = 25,
-	TFMT5_8_8_8_8_UNORM = 48,
-	TFMT5_8_8_8_UNORM = 49,
-	TFMT5_8_8_8_8_SNORM = 50,
-	TFMT5_8_8_8_8_UINT = 51,
-	TFMT5_8_8_8_8_SINT = 52,
-	TFMT5_9_9_9_E5_FLOAT = 53,
-	TFMT5_10_10_10_2_UNORM = 54,
-	TFMT5_10_10_10_2_UINT = 58,
-	TFMT5_11_11_10_FLOAT = 66,
-	TFMT5_16_16_UNORM = 67,
-	TFMT5_16_16_SNORM = 68,
-	TFMT5_16_16_FLOAT = 69,
-	TFMT5_16_16_UINT = 70,
-	TFMT5_16_16_SINT = 71,
-	TFMT5_32_FLOAT = 74,
-	TFMT5_32_UINT = 75,
-	TFMT5_32_SINT = 76,
-	TFMT5_16_16_16_16_UNORM = 96,
-	TFMT5_16_16_16_16_SNORM = 97,
-	TFMT5_16_16_16_16_FLOAT = 98,
-	TFMT5_16_16_16_16_UINT = 99,
-	TFMT5_16_16_16_16_SINT = 100,
-	TFMT5_32_32_FLOAT = 103,
-	TFMT5_32_32_UINT = 104,
-	TFMT5_32_32_SINT = 105,
-	TFMT5_32_32_32_UINT = 114,
-	TFMT5_32_32_32_SINT = 115,
-	TFMT5_32_32_32_FLOAT = 116,
-	TFMT5_32_32_32_32_FLOAT = 130,
-	TFMT5_32_32_32_32_UINT = 131,
-	TFMT5_32_32_32_32_SINT = 132,
-	TFMT5_X8Z24_UNORM = 160,
-	TFMT5_ETC2_RG11_UNORM = 171,
-	TFMT5_ETC2_RG11_SNORM = 172,
-	TFMT5_ETC2_R11_UNORM = 173,
-	TFMT5_ETC2_R11_SNORM = 174,
-	TFMT5_ETC1 = 175,
-	TFMT5_ETC2_RGB8 = 176,
-	TFMT5_ETC2_RGBA8 = 177,
-	TFMT5_ETC2_RGB8A1 = 178,
-	TFMT5_DXT1 = 179,
-	TFMT5_DXT3 = 180,
-	TFMT5_DXT5 = 181,
-	TFMT5_RGTC1_UNORM = 183,
-	TFMT5_RGTC1_SNORM = 184,
-	TFMT5_RGTC2_UNORM = 187,
-	TFMT5_RGTC2_SNORM = 188,
-	TFMT5_BPTC_UFLOAT = 190,
-	TFMT5_BPTC_FLOAT = 191,
-	TFMT5_BPTC = 192,
-	TFMT5_ASTC_4x4 = 193,
-	TFMT5_ASTC_5x4 = 194,
-	TFMT5_ASTC_5x5 = 195,
-	TFMT5_ASTC_6x5 = 196,
-	TFMT5_ASTC_6x6 = 197,
-	TFMT5_ASTC_8x5 = 198,
-	TFMT5_ASTC_8x6 = 199,
-	TFMT5_ASTC_8x8 = 200,
-	TFMT5_ASTC_10x5 = 201,
-	TFMT5_ASTC_10x6 = 202,
-	TFMT5_ASTC_10x8 = 203,
-	TFMT5_ASTC_10x10 = 204,
-	TFMT5_ASTC_12x10 = 205,
-	TFMT5_ASTC_12x12 = 206,
-	TFMT5_NONE = 255,
-};
-
-enum a5xx_depth_format {
-	DEPTH5_NONE = 0,
-	DEPTH5_16 = 1,
-	DEPTH5_24_8 = 2,
-	DEPTH5_32 = 4,
-};
-
-enum a5xx_blit_buf {
-	BLIT_MRT0 = 0,
-	BLIT_MRT1 = 1,
-	BLIT_MRT2 = 2,
-	BLIT_MRT3 = 3,
-	BLIT_MRT4 = 4,
-	BLIT_MRT5 = 5,
-	BLIT_MRT6 = 6,
-	BLIT_MRT7 = 7,
-	BLIT_ZS = 8,
-	BLIT_S = 9,
-};
-
-enum a5xx_cp_perfcounter_select {
-	PERF_CP_ALWAYS_COUNT = 0,
-	PERF_CP_BUSY_GFX_CORE_IDLE = 1,
-	PERF_CP_BUSY_CYCLES = 2,
-	PERF_CP_PFP_IDLE = 3,
-	PERF_CP_PFP_BUSY_WORKING = 4,
-	PERF_CP_PFP_STALL_CYCLES_ANY = 5,
-	PERF_CP_PFP_STARVE_CYCLES_ANY = 6,
-	PERF_CP_PFP_ICACHE_MISS = 7,
-	PERF_CP_PFP_ICACHE_HIT = 8,
-	PERF_CP_PFP_MATCH_PM4_PKT_PROFILE = 9,
-	PERF_CP_ME_BUSY_WORKING = 10,
-	PERF_CP_ME_IDLE = 11,
-	PERF_CP_ME_STARVE_CYCLES_ANY = 12,
-	PERF_CP_ME_FIFO_EMPTY_PFP_IDLE = 13,
-	PERF_CP_ME_FIFO_EMPTY_PFP_BUSY = 14,
-	PERF_CP_ME_FIFO_FULL_ME_BUSY = 15,
-	PERF_CP_ME_FIFO_FULL_ME_NON_WORKING = 16,
-	PERF_CP_ME_STALL_CYCLES_ANY = 17,
-	PERF_CP_ME_ICACHE_MISS = 18,
-	PERF_CP_ME_ICACHE_HIT = 19,
-	PERF_CP_NUM_PREEMPTIONS = 20,
-	PERF_CP_PREEMPTION_REACTION_DELAY = 21,
-	PERF_CP_PREEMPTION_SWITCH_OUT_TIME = 22,
-	PERF_CP_PREEMPTION_SWITCH_IN_TIME = 23,
-	PERF_CP_DEAD_DRAWS_IN_BIN_RENDER = 24,
-	PERF_CP_PREDICATED_DRAWS_KILLED = 25,
-	PERF_CP_MODE_SWITCH = 26,
-	PERF_CP_ZPASS_DONE = 27,
-	PERF_CP_CONTEXT_DONE = 28,
-	PERF_CP_CACHE_FLUSH = 29,
-	PERF_CP_LONG_PREEMPTIONS = 30,
-};
-
-enum a5xx_rbbm_perfcounter_select {
-	PERF_RBBM_ALWAYS_COUNT = 0,
-	PERF_RBBM_ALWAYS_ON = 1,
-	PERF_RBBM_TSE_BUSY = 2,
-	PERF_RBBM_RAS_BUSY = 3,
-	PERF_RBBM_PC_DCALL_BUSY = 4,
-	PERF_RBBM_PC_VSD_BUSY = 5,
-	PERF_RBBM_STATUS_MASKED = 6,
-	PERF_RBBM_COM_BUSY = 7,
-	PERF_RBBM_DCOM_BUSY = 8,
-	PERF_RBBM_VBIF_BUSY = 9,
-	PERF_RBBM_VSC_BUSY = 10,
-	PERF_RBBM_TESS_BUSY = 11,
-	PERF_RBBM_UCHE_BUSY = 12,
-	PERF_RBBM_HLSQ_BUSY = 13,
-};
-
-enum a5xx_pc_perfcounter_select {
-	PERF_PC_BUSY_CYCLES = 0,
-	PERF_PC_WORKING_CYCLES = 1,
-	PERF_PC_STALL_CYCLES_VFD = 2,
-	PERF_PC_STALL_CYCLES_TSE = 3,
-	PERF_PC_STALL_CYCLES_VPC = 4,
-	PERF_PC_STALL_CYCLES_UCHE = 5,
-	PERF_PC_STALL_CYCLES_TESS = 6,
-	PERF_PC_STALL_CYCLES_TSE_ONLY = 7,
-	PERF_PC_STALL_CYCLES_VPC_ONLY = 8,
-	PERF_PC_PASS1_TF_STALL_CYCLES = 9,
-	PERF_PC_STARVE_CYCLES_FOR_INDEX = 10,
-	PERF_PC_STARVE_CYCLES_FOR_TESS_FACTOR = 11,
-	PERF_PC_STARVE_CYCLES_FOR_VIZ_STREAM = 12,
-	PERF_PC_STARVE_CYCLES_FOR_POSITION = 13,
-	PERF_PC_STARVE_CYCLES_DI = 14,
-	PERF_PC_VIS_STREAMS_LOADED = 15,
-	PERF_PC_INSTANCES = 16,
-	PERF_PC_VPC_PRIMITIVES = 17,
-	PERF_PC_DEAD_PRIM = 18,
-	PERF_PC_LIVE_PRIM = 19,
-	PERF_PC_VERTEX_HITS = 20,
-	PERF_PC_IA_VERTICES = 21,
-	PERF_PC_IA_PRIMITIVES = 22,
-	PERF_PC_GS_PRIMITIVES = 23,
-	PERF_PC_HS_INVOCATIONS = 24,
-	PERF_PC_DS_INVOCATIONS = 25,
-	PERF_PC_VS_INVOCATIONS = 26,
-	PERF_PC_GS_INVOCATIONS = 27,
-	PERF_PC_DS_PRIMITIVES = 28,
-	PERF_PC_VPC_POS_DATA_TRANSACTION = 29,
-	PERF_PC_3D_DRAWCALLS = 30,
-	PERF_PC_2D_DRAWCALLS = 31,
-	PERF_PC_NON_DRAWCALL_GLOBAL_EVENTS = 32,
-	PERF_TESS_BUSY_CYCLES = 33,
-	PERF_TESS_WORKING_CYCLES = 34,
-	PERF_TESS_STALL_CYCLES_PC = 35,
-	PERF_TESS_STARVE_CYCLES_PC = 36,
-};
-
-enum a5xx_vfd_perfcounter_select {
-	PERF_VFD_BUSY_CYCLES = 0,
-	PERF_VFD_STALL_CYCLES_UCHE = 1,
-	PERF_VFD_STALL_CYCLES_VPC_ALLOC = 2,
-	PERF_VFD_STALL_CYCLES_MISS_VB = 3,
-	PERF_VFD_STALL_CYCLES_MISS_Q = 4,
-	PERF_VFD_STALL_CYCLES_SP_INFO = 5,
-	PERF_VFD_STALL_CYCLES_SP_ATTR = 6,
-	PERF_VFD_STALL_CYCLES_VFDP_VB = 7,
-	PERF_VFD_STALL_CYCLES_VFDP_Q = 8,
-	PERF_VFD_DECODER_PACKER_STALL = 9,
-	PERF_VFD_STARVE_CYCLES_UCHE = 10,
-	PERF_VFD_RBUFFER_FULL = 11,
-	PERF_VFD_ATTR_INFO_FIFO_FULL = 12,
-	PERF_VFD_DECODED_ATTRIBUTE_BYTES = 13,
-	PERF_VFD_NUM_ATTRIBUTES = 14,
-	PERF_VFD_INSTRUCTIONS = 15,
-	PERF_VFD_UPPER_SHADER_FIBERS = 16,
-	PERF_VFD_LOWER_SHADER_FIBERS = 17,
-	PERF_VFD_MODE_0_FIBERS = 18,
-	PERF_VFD_MODE_1_FIBERS = 19,
-	PERF_VFD_MODE_2_FIBERS = 20,
-	PERF_VFD_MODE_3_FIBERS = 21,
-	PERF_VFD_MODE_4_FIBERS = 22,
-	PERF_VFD_TOTAL_VERTICES = 23,
-	PERF_VFD_NUM_ATTR_MISS = 24,
-	PERF_VFD_1_BURST_REQ = 25,
-	PERF_VFDP_STALL_CYCLES_VFD = 26,
-	PERF_VFDP_STALL_CYCLES_VFD_INDEX = 27,
-	PERF_VFDP_STALL_CYCLES_VFD_PROG = 28,
-	PERF_VFDP_STARVE_CYCLES_PC = 29,
-	PERF_VFDP_VS_STAGE_32_WAVES = 30,
-};
-
-enum a5xx_hlsq_perfcounter_select {
-	PERF_HLSQ_BUSY_CYCLES = 0,
-	PERF_HLSQ_STALL_CYCLES_UCHE = 1,
-	PERF_HLSQ_STALL_CYCLES_SP_STATE = 2,
-	PERF_HLSQ_STALL_CYCLES_SP_FS_STAGE = 3,
-	PERF_HLSQ_UCHE_LATENCY_CYCLES = 4,
-	PERF_HLSQ_UCHE_LATENCY_COUNT = 5,
-	PERF_HLSQ_FS_STAGE_32_WAVES = 6,
-	PERF_HLSQ_FS_STAGE_64_WAVES = 7,
-	PERF_HLSQ_QUADS = 8,
-	PERF_HLSQ_SP_STATE_COPY_TRANS_FS_STAGE = 9,
-	PERF_HLSQ_SP_STATE_COPY_TRANS_VS_STAGE = 10,
-	PERF_HLSQ_TP_STATE_COPY_TRANS_FS_STAGE = 11,
-	PERF_HLSQ_TP_STATE_COPY_TRANS_VS_STAGE = 12,
-	PERF_HLSQ_CS_INVOCATIONS = 13,
-	PERF_HLSQ_COMPUTE_DRAWCALLS = 14,
-};
-
-enum a5xx_vpc_perfcounter_select {
-	PERF_VPC_BUSY_CYCLES = 0,
-	PERF_VPC_WORKING_CYCLES = 1,
-	PERF_VPC_STALL_CYCLES_UCHE = 2,
-	PERF_VPC_STALL_CYCLES_VFD_WACK = 3,
-	PERF_VPC_STALL_CYCLES_HLSQ_PRIM_ALLOC = 4,
-	PERF_VPC_STALL_CYCLES_PC = 5,
-	PERF_VPC_STALL_CYCLES_SP_LM = 6,
-	PERF_VPC_POS_EXPORT_STALL_CYCLES = 7,
-	PERF_VPC_STARVE_CYCLES_SP = 8,
-	PERF_VPC_STARVE_CYCLES_LRZ = 9,
-	PERF_VPC_PC_PRIMITIVES = 10,
-	PERF_VPC_SP_COMPONENTS = 11,
-	PERF_VPC_SP_LM_PRIMITIVES = 12,
-	PERF_VPC_SP_LM_COMPONENTS = 13,
-	PERF_VPC_SP_LM_DWORDS = 14,
-	PERF_VPC_STREAMOUT_COMPONENTS = 15,
-	PERF_VPC_GRANT_PHASES = 16,
-};
-
-enum a5xx_tse_perfcounter_select {
-	PERF_TSE_BUSY_CYCLES = 0,
-	PERF_TSE_CLIPPING_CYCLES = 1,
-	PERF_TSE_STALL_CYCLES_RAS = 2,
-	PERF_TSE_STALL_CYCLES_LRZ_BARYPLANE = 3,
-	PERF_TSE_STALL_CYCLES_LRZ_ZPLANE = 4,
-	PERF_TSE_STARVE_CYCLES_PC = 5,
-	PERF_TSE_INPUT_PRIM = 6,
-	PERF_TSE_INPUT_NULL_PRIM = 7,
-	PERF_TSE_TRIVAL_REJ_PRIM = 8,
-	PERF_TSE_CLIPPED_PRIM = 9,
-	PERF_TSE_ZERO_AREA_PRIM = 10,
-	PERF_TSE_FACENESS_CULLED_PRIM = 11,
-	PERF_TSE_ZERO_PIXEL_PRIM = 12,
-	PERF_TSE_OUTPUT_NULL_PRIM = 13,
-	PERF_TSE_OUTPUT_VISIBLE_PRIM = 14,
-	PERF_TSE_CINVOCATION = 15,
-	PERF_TSE_CPRIMITIVES = 16,
-	PERF_TSE_2D_INPUT_PRIM = 17,
-	PERF_TSE_2D_ALIVE_CLCLES = 18,
-};
-
-enum a5xx_ras_perfcounter_select {
-	PERF_RAS_BUSY_CYCLES = 0,
-	PERF_RAS_SUPERTILE_ACTIVE_CYCLES = 1,
-	PERF_RAS_STALL_CYCLES_LRZ = 2,
-	PERF_RAS_STARVE_CYCLES_TSE = 3,
-	PERF_RAS_SUPER_TILES = 4,
-	PERF_RAS_8X4_TILES = 5,
-	PERF_RAS_MASKGEN_ACTIVE = 6,
-	PERF_RAS_FULLY_COVERED_SUPER_TILES = 7,
-	PERF_RAS_FULLY_COVERED_8X4_TILES = 8,
-	PERF_RAS_PRIM_KILLED_INVISILBE = 9,
-};
-
-enum a5xx_lrz_perfcounter_select {
-	PERF_LRZ_BUSY_CYCLES = 0,
-	PERF_LRZ_STARVE_CYCLES_RAS = 1,
-	PERF_LRZ_STALL_CYCLES_RB = 2,
-	PERF_LRZ_STALL_CYCLES_VSC = 3,
-	PERF_LRZ_STALL_CYCLES_VPC = 4,
-	PERF_LRZ_STALL_CYCLES_FLAG_PREFETCH = 5,
-	PERF_LRZ_STALL_CYCLES_UCHE = 6,
-	PERF_LRZ_LRZ_READ = 7,
-	PERF_LRZ_LRZ_WRITE = 8,
-	PERF_LRZ_READ_LATENCY = 9,
-	PERF_LRZ_MERGE_CACHE_UPDATING = 10,
-	PERF_LRZ_PRIM_KILLED_BY_MASKGEN = 11,
-	PERF_LRZ_PRIM_KILLED_BY_LRZ = 12,
-	PERF_LRZ_VISIBLE_PRIM_AFTER_LRZ = 13,
-	PERF_LRZ_FULL_8X8_TILES = 14,
-	PERF_LRZ_PARTIAL_8X8_TILES = 15,
-	PERF_LRZ_TILE_KILLED = 16,
-	PERF_LRZ_TOTAL_PIXEL = 17,
-	PERF_LRZ_VISIBLE_PIXEL_AFTER_LRZ = 18,
-};
-
-enum a5xx_uche_perfcounter_select {
-	PERF_UCHE_BUSY_CYCLES = 0,
-	PERF_UCHE_STALL_CYCLES_VBIF = 1,
-	PERF_UCHE_VBIF_LATENCY_CYCLES = 2,
-	PERF_UCHE_VBIF_LATENCY_SAMPLES = 3,
-	PERF_UCHE_VBIF_READ_BEATS_TP = 4,
-	PERF_UCHE_VBIF_READ_BEATS_VFD = 5,
-	PERF_UCHE_VBIF_READ_BEATS_HLSQ = 6,
-	PERF_UCHE_VBIF_READ_BEATS_LRZ = 7,
-	PERF_UCHE_VBIF_READ_BEATS_SP = 8,
-	PERF_UCHE_READ_REQUESTS_TP = 9,
-	PERF_UCHE_READ_REQUESTS_VFD = 10,
-	PERF_UCHE_READ_REQUESTS_HLSQ = 11,
-	PERF_UCHE_READ_REQUESTS_LRZ = 12,
-	PERF_UCHE_READ_REQUESTS_SP = 13,
-	PERF_UCHE_WRITE_REQUESTS_LRZ = 14,
-	PERF_UCHE_WRITE_REQUESTS_SP = 15,
-	PERF_UCHE_WRITE_REQUESTS_VPC = 16,
-	PERF_UCHE_WRITE_REQUESTS_VSC = 17,
-	PERF_UCHE_EVICTS = 18,
-	PERF_UCHE_BANK_REQ0 = 19,
-	PERF_UCHE_BANK_REQ1 = 20,
-	PERF_UCHE_BANK_REQ2 = 21,
-	PERF_UCHE_BANK_REQ3 = 22,
-	PERF_UCHE_BANK_REQ4 = 23,
-	PERF_UCHE_BANK_REQ5 = 24,
-	PERF_UCHE_BANK_REQ6 = 25,
-	PERF_UCHE_BANK_REQ7 = 26,
-	PERF_UCHE_VBIF_READ_BEATS_CH0 = 27,
-	PERF_UCHE_VBIF_READ_BEATS_CH1 = 28,
-	PERF_UCHE_GMEM_READ_BEATS = 29,
-	PERF_UCHE_FLAG_COUNT = 30,
-};
-
-enum a5xx_tp_perfcounter_select {
-	PERF_TP_BUSY_CYCLES = 0,
-	PERF_TP_STALL_CYCLES_UCHE = 1,
-	PERF_TP_LATENCY_CYCLES = 2,
-	PERF_TP_LATENCY_TRANS = 3,
-	PERF_TP_FLAG_CACHE_REQUEST_SAMPLES = 4,
-	PERF_TP_FLAG_CACHE_REQUEST_LATENCY = 5,
-	PERF_TP_L1_CACHELINE_REQUESTS = 6,
-	PERF_TP_L1_CACHELINE_MISSES = 7,
-	PERF_TP_SP_TP_TRANS = 8,
-	PERF_TP_TP_SP_TRANS = 9,
-	PERF_TP_OUTPUT_PIXELS = 10,
-	PERF_TP_FILTER_WORKLOAD_16BIT = 11,
-	PERF_TP_FILTER_WORKLOAD_32BIT = 12,
-	PERF_TP_QUADS_RECEIVED = 13,
-	PERF_TP_QUADS_OFFSET = 14,
-	PERF_TP_QUADS_SHADOW = 15,
-	PERF_TP_QUADS_ARRAY = 16,
-	PERF_TP_QUADS_GRADIENT = 17,
-	PERF_TP_QUADS_1D = 18,
-	PERF_TP_QUADS_2D = 19,
-	PERF_TP_QUADS_BUFFER = 20,
-	PERF_TP_QUADS_3D = 21,
-	PERF_TP_QUADS_CUBE = 22,
-	PERF_TP_STATE_CACHE_REQUESTS = 23,
-	PERF_TP_STATE_CACHE_MISSES = 24,
-	PERF_TP_DIVERGENT_QUADS_RECEIVED = 25,
-	PERF_TP_BINDLESS_STATE_CACHE_REQUESTS = 26,
-	PERF_TP_BINDLESS_STATE_CACHE_MISSES = 27,
-	PERF_TP_PRT_NON_RESIDENT_EVENTS = 28,
-	PERF_TP_OUTPUT_PIXELS_POINT = 29,
-	PERF_TP_OUTPUT_PIXELS_BILINEAR = 30,
-	PERF_TP_OUTPUT_PIXELS_MIP = 31,
-	PERF_TP_OUTPUT_PIXELS_ANISO = 32,
-	PERF_TP_OUTPUT_PIXELS_ZERO_LOD = 33,
-	PERF_TP_FLAG_CACHE_REQUESTS = 34,
-	PERF_TP_FLAG_CACHE_MISSES = 35,
-	PERF_TP_L1_5_L2_REQUESTS = 36,
-	PERF_TP_2D_OUTPUT_PIXELS = 37,
-	PERF_TP_2D_OUTPUT_PIXELS_POINT = 38,
-	PERF_TP_2D_OUTPUT_PIXELS_BILINEAR = 39,
-	PERF_TP_2D_FILTER_WORKLOAD_16BIT = 40,
-	PERF_TP_2D_FILTER_WORKLOAD_32BIT = 41,
-};
-
-enum a5xx_sp_perfcounter_select {
-	PERF_SP_BUSY_CYCLES = 0,
-	PERF_SP_ALU_WORKING_CYCLES = 1,
-	PERF_SP_EFU_WORKING_CYCLES = 2,
-	PERF_SP_STALL_CYCLES_VPC = 3,
-	PERF_SP_STALL_CYCLES_TP = 4,
-	PERF_SP_STALL_CYCLES_UCHE = 5,
-	PERF_SP_STALL_CYCLES_RB = 6,
-	PERF_SP_SCHEDULER_NON_WORKING = 7,
-	PERF_SP_WAVE_CONTEXTS = 8,
-	PERF_SP_WAVE_CONTEXT_CYCLES = 9,
-	PERF_SP_FS_STAGE_WAVE_CYCLES = 10,
-	PERF_SP_FS_STAGE_WAVE_SAMPLES = 11,
-	PERF_SP_VS_STAGE_WAVE_CYCLES = 12,
-	PERF_SP_VS_STAGE_WAVE_SAMPLES = 13,
-	PERF_SP_FS_STAGE_DURATION_CYCLES = 14,
-	PERF_SP_VS_STAGE_DURATION_CYCLES = 15,
-	PERF_SP_WAVE_CTRL_CYCLES = 16,
-	PERF_SP_WAVE_LOAD_CYCLES = 17,
-	PERF_SP_WAVE_EMIT_CYCLES = 18,
-	PERF_SP_WAVE_NOP_CYCLES = 19,
-	PERF_SP_WAVE_WAIT_CYCLES = 20,
-	PERF_SP_WAVE_FETCH_CYCLES = 21,
-	PERF_SP_WAVE_IDLE_CYCLES = 22,
-	PERF_SP_WAVE_END_CYCLES = 23,
-	PERF_SP_WAVE_LONG_SYNC_CYCLES = 24,
-	PERF_SP_WAVE_SHORT_SYNC_CYCLES = 25,
-	PERF_SP_WAVE_JOIN_CYCLES = 26,
-	PERF_SP_LM_LOAD_INSTRUCTIONS = 27,
-	PERF_SP_LM_STORE_INSTRUCTIONS = 28,
-	PERF_SP_LM_ATOMICS = 29,
-	PERF_SP_GM_LOAD_INSTRUCTIONS = 30,
-	PERF_SP_GM_STORE_INSTRUCTIONS = 31,
-	PERF_SP_GM_ATOMICS = 32,
-	PERF_SP_VS_STAGE_TEX_INSTRUCTIONS = 33,
-	PERF_SP_VS_STAGE_CFLOW_INSTRUCTIONS = 34,
-	PERF_SP_VS_STAGE_EFU_INSTRUCTIONS = 35,
-	PERF_SP_VS_STAGE_FULL_ALU_INSTRUCTIONS = 36,
-	PERF_SP_VS_STAGE_HALF_ALU_INSTRUCTIONS = 37,
-	PERF_SP_FS_STAGE_TEX_INSTRUCTIONS = 38,
-	PERF_SP_FS_STAGE_CFLOW_INSTRUCTIONS = 39,
-	PERF_SP_FS_STAGE_EFU_INSTRUCTIONS = 40,
-	PERF_SP_FS_STAGE_FULL_ALU_INSTRUCTIONS = 41,
-	PERF_SP_FS_STAGE_HALF_ALU_INSTRUCTIONS = 42,
-	PERF_SP_FS_STAGE_BARY_INSTRUCTIONS = 43,
-	PERF_SP_VS_INSTRUCTIONS = 44,
-	PERF_SP_FS_INSTRUCTIONS = 45,
-	PERF_SP_ADDR_LOCK_COUNT = 46,
-	PERF_SP_UCHE_READ_TRANS = 47,
-	PERF_SP_UCHE_WRITE_TRANS = 48,
-	PERF_SP_EXPORT_VPC_TRANS = 49,
-	PERF_SP_EXPORT_RB_TRANS = 50,
-	PERF_SP_PIXELS_KILLED = 51,
-	PERF_SP_ICL1_REQUESTS = 52,
-	PERF_SP_ICL1_MISSES = 53,
-	PERF_SP_ICL0_REQUESTS = 54,
-	PERF_SP_ICL0_MISSES = 55,
-	PERF_SP_HS_INSTRUCTIONS = 56,
-	PERF_SP_DS_INSTRUCTIONS = 57,
-	PERF_SP_GS_INSTRUCTIONS = 58,
-	PERF_SP_CS_INSTRUCTIONS = 59,
-	PERF_SP_GPR_READ = 60,
-	PERF_SP_GPR_WRITE = 61,
-	PERF_SP_LM_CH0_REQUESTS = 62,
-	PERF_SP_LM_CH1_REQUESTS = 63,
-	PERF_SP_LM_BANK_CONFLICTS = 64,
-};
-
-enum a5xx_rb_perfcounter_select {
-	PERF_RB_BUSY_CYCLES = 0,
-	PERF_RB_STALL_CYCLES_CCU = 1,
-	PERF_RB_STALL_CYCLES_HLSQ = 2,
-	PERF_RB_STALL_CYCLES_FIFO0_FULL = 3,
-	PERF_RB_STALL_CYCLES_FIFO1_FULL = 4,
-	PERF_RB_STALL_CYCLES_FIFO2_FULL = 5,
-	PERF_RB_STARVE_CYCLES_SP = 6,
-	PERF_RB_STARVE_CYCLES_LRZ_TILE = 7,
-	PERF_RB_STARVE_CYCLES_CCU = 8,
-	PERF_RB_STARVE_CYCLES_Z_PLANE = 9,
-	PERF_RB_STARVE_CYCLES_BARY_PLANE = 10,
-	PERF_RB_Z_WORKLOAD = 11,
-	PERF_RB_HLSQ_ACTIVE = 12,
-	PERF_RB_Z_READ = 13,
-	PERF_RB_Z_WRITE = 14,
-	PERF_RB_C_READ = 15,
-	PERF_RB_C_WRITE = 16,
-	PERF_RB_TOTAL_PASS = 17,
-	PERF_RB_Z_PASS = 18,
-	PERF_RB_Z_FAIL = 19,
-	PERF_RB_S_FAIL = 20,
-	PERF_RB_BLENDED_FXP_COMPONENTS = 21,
-	PERF_RB_BLENDED_FP16_COMPONENTS = 22,
-	RB_RESERVED = 23,
-	PERF_RB_2D_ALIVE_CYCLES = 24,
-	PERF_RB_2D_STALL_CYCLES_A2D = 25,
-	PERF_RB_2D_STARVE_CYCLES_SRC = 26,
-	PERF_RB_2D_STARVE_CYCLES_SP = 27,
-	PERF_RB_2D_STARVE_CYCLES_DST = 28,
-	PERF_RB_2D_VALID_PIXELS = 29,
-};
-
-enum a5xx_rb_samples_perfcounter_select {
-	TOTAL_SAMPLES = 0,
-	ZPASS_SAMPLES = 1,
-	ZFAIL_SAMPLES = 2,
-	SFAIL_SAMPLES = 3,
-};
-
-enum a5xx_vsc_perfcounter_select {
-	PERF_VSC_BUSY_CYCLES = 0,
-	PERF_VSC_WORKING_CYCLES = 1,
-	PERF_VSC_STALL_CYCLES_UCHE = 2,
-	PERF_VSC_EOT_NUM = 3,
-};
-
-enum a5xx_ccu_perfcounter_select {
-	PERF_CCU_BUSY_CYCLES = 0,
-	PERF_CCU_STALL_CYCLES_RB_DEPTH_RETURN = 1,
-	PERF_CCU_STALL_CYCLES_RB_COLOR_RETURN = 2,
-	PERF_CCU_STARVE_CYCLES_FLAG_RETURN = 3,
-	PERF_CCU_DEPTH_BLOCKS = 4,
-	PERF_CCU_COLOR_BLOCKS = 5,
-	PERF_CCU_DEPTH_BLOCK_HIT = 6,
-	PERF_CCU_COLOR_BLOCK_HIT = 7,
-	PERF_CCU_PARTIAL_BLOCK_READ = 8,
-	PERF_CCU_GMEM_READ = 9,
-	PERF_CCU_GMEM_WRITE = 10,
-	PERF_CCU_DEPTH_READ_FLAG0_COUNT = 11,
-	PERF_CCU_DEPTH_READ_FLAG1_COUNT = 12,
-	PERF_CCU_DEPTH_READ_FLAG2_COUNT = 13,
-	PERF_CCU_DEPTH_READ_FLAG3_COUNT = 14,
-	PERF_CCU_DEPTH_READ_FLAG4_COUNT = 15,
-	PERF_CCU_COLOR_READ_FLAG0_COUNT = 16,
-	PERF_CCU_COLOR_READ_FLAG1_COUNT = 17,
-	PERF_CCU_COLOR_READ_FLAG2_COUNT = 18,
-	PERF_CCU_COLOR_READ_FLAG3_COUNT = 19,
-	PERF_CCU_COLOR_READ_FLAG4_COUNT = 20,
-	PERF_CCU_2D_BUSY_CYCLES = 21,
-	PERF_CCU_2D_RD_REQ = 22,
-	PERF_CCU_2D_WR_REQ = 23,
-	PERF_CCU_2D_REORDER_STARVE_CYCLES = 24,
-	PERF_CCU_2D_PIXELS = 25,
-};
-
-enum a5xx_cmp_perfcounter_select {
-	PERF_CMPDECMP_STALL_CYCLES_VBIF = 0,
-	PERF_CMPDECMP_VBIF_LATENCY_CYCLES = 1,
-	PERF_CMPDECMP_VBIF_LATENCY_SAMPLES = 2,
-	PERF_CMPDECMP_VBIF_READ_DATA_CCU = 3,
-	PERF_CMPDECMP_VBIF_WRITE_DATA_CCU = 4,
-	PERF_CMPDECMP_VBIF_READ_REQUEST = 5,
-	PERF_CMPDECMP_VBIF_WRITE_REQUEST = 6,
-	PERF_CMPDECMP_VBIF_READ_DATA = 7,
-	PERF_CMPDECMP_VBIF_WRITE_DATA = 8,
-	PERF_CMPDECMP_FLAG_FETCH_CYCLES = 9,
-	PERF_CMPDECMP_FLAG_FETCH_SAMPLES = 10,
-	PERF_CMPDECMP_DEPTH_WRITE_FLAG1_COUNT = 11,
-	PERF_CMPDECMP_DEPTH_WRITE_FLAG2_COUNT = 12,
-	PERF_CMPDECMP_DEPTH_WRITE_FLAG3_COUNT = 13,
-	PERF_CMPDECMP_DEPTH_WRITE_FLAG4_COUNT = 14,
-	PERF_CMPDECMP_COLOR_WRITE_FLAG1_COUNT = 15,
-	PERF_CMPDECMP_COLOR_WRITE_FLAG2_COUNT = 16,
-	PERF_CMPDECMP_COLOR_WRITE_FLAG3_COUNT = 17,
-	PERF_CMPDECMP_COLOR_WRITE_FLAG4_COUNT = 18,
-	PERF_CMPDECMP_2D_STALL_CYCLES_VBIF_REQ = 19,
-	PERF_CMPDECMP_2D_STALL_CYCLES_VBIF_WR = 20,
-	PERF_CMPDECMP_2D_STALL_CYCLES_VBIF_RETURN = 21,
-	PERF_CMPDECMP_2D_RD_DATA = 22,
-	PERF_CMPDECMP_2D_WR_DATA = 23,
-};
-
-enum a5xx_vbif_perfcounter_select {
-	AXI_READ_REQUESTS_ID_0 = 0,
-	AXI_READ_REQUESTS_ID_1 = 1,
-	AXI_READ_REQUESTS_ID_2 = 2,
-	AXI_READ_REQUESTS_ID_3 = 3,
-	AXI_READ_REQUESTS_ID_4 = 4,
-	AXI_READ_REQUESTS_ID_5 = 5,
-	AXI_READ_REQUESTS_ID_6 = 6,
-	AXI_READ_REQUESTS_ID_7 = 7,
-	AXI_READ_REQUESTS_ID_8 = 8,
-	AXI_READ_REQUESTS_ID_9 = 9,
-	AXI_READ_REQUESTS_ID_10 = 10,
-	AXI_READ_REQUESTS_ID_11 = 11,
-	AXI_READ_REQUESTS_ID_12 = 12,
-	AXI_READ_REQUESTS_ID_13 = 13,
-	AXI_READ_REQUESTS_ID_14 = 14,
-	AXI_READ_REQUESTS_ID_15 = 15,
-	AXI0_READ_REQUESTS_TOTAL = 16,
-	AXI1_READ_REQUESTS_TOTAL = 17,
-	AXI2_READ_REQUESTS_TOTAL = 18,
-	AXI3_READ_REQUESTS_TOTAL = 19,
-	AXI_READ_REQUESTS_TOTAL = 20,
-	AXI_WRITE_REQUESTS_ID_0 = 21,
-	AXI_WRITE_REQUESTS_ID_1 = 22,
-	AXI_WRITE_REQUESTS_ID_2 = 23,
-	AXI_WRITE_REQUESTS_ID_3 = 24,
-	AXI_WRITE_REQUESTS_ID_4 = 25,
-	AXI_WRITE_REQUESTS_ID_5 = 26,
-	AXI_WRITE_REQUESTS_ID_6 = 27,
-	AXI_WRITE_REQUESTS_ID_7 = 28,
-	AXI_WRITE_REQUESTS_ID_8 = 29,
-	AXI_WRITE_REQUESTS_ID_9 = 30,
-	AXI_WRITE_REQUESTS_ID_10 = 31,
-	AXI_WRITE_REQUESTS_ID_11 = 32,
-	AXI_WRITE_REQUESTS_ID_12 = 33,
-	AXI_WRITE_REQUESTS_ID_13 = 34,
-	AXI_WRITE_REQUESTS_ID_14 = 35,
-	AXI_WRITE_REQUESTS_ID_15 = 36,
-	AXI0_WRITE_REQUESTS_TOTAL = 37,
-	AXI1_WRITE_REQUESTS_TOTAL = 38,
-	AXI2_WRITE_REQUESTS_TOTAL = 39,
-	AXI3_WRITE_REQUESTS_TOTAL = 40,
-	AXI_WRITE_REQUESTS_TOTAL = 41,
-	AXI_TOTAL_REQUESTS = 42,
-	AXI_READ_DATA_BEATS_ID_0 = 43,
-	AXI_READ_DATA_BEATS_ID_1 = 44,
-	AXI_READ_DATA_BEATS_ID_2 = 45,
-	AXI_READ_DATA_BEATS_ID_3 = 46,
-	AXI_READ_DATA_BEATS_ID_4 = 47,
-	AXI_READ_DATA_BEATS_ID_5 = 48,
-	AXI_READ_DATA_BEATS_ID_6 = 49,
-	AXI_READ_DATA_BEATS_ID_7 = 50,
-	AXI_READ_DATA_BEATS_ID_8 = 51,
-	AXI_READ_DATA_BEATS_ID_9 = 52,
-	AXI_READ_DATA_BEATS_ID_10 = 53,
-	AXI_READ_DATA_BEATS_ID_11 = 54,
-	AXI_READ_DATA_BEATS_ID_12 = 55,
-	AXI_READ_DATA_BEATS_ID_13 = 56,
-	AXI_READ_DATA_BEATS_ID_14 = 57,
-	AXI_READ_DATA_BEATS_ID_15 = 58,
-	AXI0_READ_DATA_BEATS_TOTAL = 59,
-	AXI1_READ_DATA_BEATS_TOTAL = 60,
-	AXI2_READ_DATA_BEATS_TOTAL = 61,
-	AXI3_READ_DATA_BEATS_TOTAL = 62,
-	AXI_READ_DATA_BEATS_TOTAL = 63,
-	AXI_WRITE_DATA_BEATS_ID_0 = 64,
-	AXI_WRITE_DATA_BEATS_ID_1 = 65,
-	AXI_WRITE_DATA_BEATS_ID_2 = 66,
-	AXI_WRITE_DATA_BEATS_ID_3 = 67,
-	AXI_WRITE_DATA_BEATS_ID_4 = 68,
-	AXI_WRITE_DATA_BEATS_ID_5 = 69,
-	AXI_WRITE_DATA_BEATS_ID_6 = 70,
-	AXI_WRITE_DATA_BEATS_ID_7 = 71,
-	AXI_WRITE_DATA_BEATS_ID_8 = 72,
-	AXI_WRITE_DATA_BEATS_ID_9 = 73,
-	AXI_WRITE_DATA_BEATS_ID_10 = 74,
-	AXI_WRITE_DATA_BEATS_ID_11 = 75,
-	AXI_WRITE_DATA_BEATS_ID_12 = 76,
-	AXI_WRITE_DATA_BEATS_ID_13 = 77,
-	AXI_WRITE_DATA_BEATS_ID_14 = 78,
-	AXI_WRITE_DATA_BEATS_ID_15 = 79,
-	AXI0_WRITE_DATA_BEATS_TOTAL = 80,
-	AXI1_WRITE_DATA_BEATS_TOTAL = 81,
-	AXI2_WRITE_DATA_BEATS_TOTAL = 82,
-	AXI3_WRITE_DATA_BEATS_TOTAL = 83,
-	AXI_WRITE_DATA_BEATS_TOTAL = 84,
-	AXI_DATA_BEATS_TOTAL = 85,
-};
-
-enum a5xx_tex_filter {
-	A5XX_TEX_NEAREST = 0,
-	A5XX_TEX_LINEAR = 1,
-	A5XX_TEX_ANISO = 2,
-};
-
-enum a5xx_tex_clamp {
-	A5XX_TEX_REPEAT = 0,
-	A5XX_TEX_CLAMP_TO_EDGE = 1,
-	A5XX_TEX_MIRROR_REPEAT = 2,
-	A5XX_TEX_CLAMP_TO_BORDER = 3,
-	A5XX_TEX_MIRROR_CLAMP = 4,
-};
-
-enum a5xx_tex_aniso {
-	A5XX_TEX_ANISO_1 = 0,
-	A5XX_TEX_ANISO_2 = 1,
-	A5XX_TEX_ANISO_4 = 2,
-	A5XX_TEX_ANISO_8 = 3,
-	A5XX_TEX_ANISO_16 = 4,
-};
-
-enum a5xx_tex_swiz {
-	A5XX_TEX_X = 0,
-	A5XX_TEX_Y = 1,
-	A5XX_TEX_Z = 2,
-	A5XX_TEX_W = 3,
-	A5XX_TEX_ZERO = 4,
-	A5XX_TEX_ONE = 5,
-};
-
-enum a5xx_tex_type {
-	A5XX_TEX_1D = 0,
-	A5XX_TEX_2D = 1,
-	A5XX_TEX_CUBE = 2,
-	A5XX_TEX_3D = 3,
-	A5XX_TEX_BUFFER = 4,
-};
-
-#define A5XX_INT0_RBBM_GPU_IDLE					0x00000001
-#define A5XX_INT0_RBBM_AHB_ERROR				0x00000002
-#define A5XX_INT0_RBBM_TRANSFER_TIMEOUT				0x00000004
-#define A5XX_INT0_RBBM_ME_MS_TIMEOUT				0x00000008
-#define A5XX_INT0_RBBM_PFP_MS_TIMEOUT				0x00000010
-#define A5XX_INT0_RBBM_ETS_MS_TIMEOUT				0x00000020
-#define A5XX_INT0_RBBM_ATB_ASYNC_OVERFLOW			0x00000040
-#define A5XX_INT0_RBBM_GPC_ERROR				0x00000080
-#define A5XX_INT0_CP_SW						0x00000100
-#define A5XX_INT0_CP_HW_ERROR					0x00000200
-#define A5XX_INT0_CP_CCU_FLUSH_DEPTH_TS				0x00000400
-#define A5XX_INT0_CP_CCU_FLUSH_COLOR_TS				0x00000800
-#define A5XX_INT0_CP_CCU_RESOLVE_TS				0x00001000
-#define A5XX_INT0_CP_IB2					0x00002000
-#define A5XX_INT0_CP_IB1					0x00004000
-#define A5XX_INT0_CP_RB						0x00008000
-#define A5XX_INT0_CP_UNUSED_1					0x00010000
-#define A5XX_INT0_CP_RB_DONE_TS					0x00020000
-#define A5XX_INT0_CP_WT_DONE_TS					0x00040000
-#define A5XX_INT0_UNKNOWN_1					0x00080000
-#define A5XX_INT0_CP_CACHE_FLUSH_TS				0x00100000
-#define A5XX_INT0_UNUSED_2					0x00200000
-#define A5XX_INT0_RBBM_ATB_BUS_OVERFLOW				0x00400000
-#define A5XX_INT0_MISC_HANG_DETECT				0x00800000
-#define A5XX_INT0_UCHE_OOB_ACCESS				0x01000000
-#define A5XX_INT0_UCHE_TRAP_INTR				0x02000000
-#define A5XX_INT0_DEBBUS_INTR_0					0x04000000
-#define A5XX_INT0_DEBBUS_INTR_1					0x08000000
-#define A5XX_INT0_GPMU_VOLTAGE_DROOP				0x10000000
-#define A5XX_INT0_GPMU_FIRMWARE					0x20000000
-#define A5XX_INT0_ISDB_CPU_IRQ					0x40000000
-#define A5XX_INT0_ISDB_UNDER_DEBUG				0x80000000
-
-#define A5XX_CP_INT_CP_OPCODE_ERROR				0x00000001
-#define A5XX_CP_INT_CP_RESERVED_BIT_ERROR			0x00000002
-#define A5XX_CP_INT_CP_HW_FAULT_ERROR				0x00000004
-#define A5XX_CP_INT_CP_DMA_ERROR				0x00000008
-#define A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR		0x00000010
-#define A5XX_CP_INT_CP_AHB_ERROR				0x00000020
-
-#define REG_A5XX_CP_RB_BASE					0x00000800
-
-#define REG_A5XX_CP_RB_BASE_HI					0x00000801
-
-#define REG_A5XX_CP_RB_CNTL					0x00000802
-
-#define REG_A5XX_CP_RB_RPTR_ADDR				0x00000804
-
-#define REG_A5XX_CP_RB_RPTR_ADDR_HI				0x00000805
-
-#define REG_A5XX_CP_RB_RPTR					0x00000806
-
-#define REG_A5XX_CP_RB_WPTR					0x00000807
-
-#define REG_A5XX_CP_PFP_STAT_ADDR				0x00000808
-
-#define REG_A5XX_CP_PFP_STAT_DATA				0x00000809
-
-#define REG_A5XX_CP_DRAW_STATE_ADDR				0x0000080b
-
-#define REG_A5XX_CP_DRAW_STATE_DATA				0x0000080c
-
-#define REG_A5XX_CP_ME_NRT_ADDR_LO				0x0000080d
-
-#define REG_A5XX_CP_ME_NRT_ADDR_HI				0x0000080e
-
-#define REG_A5XX_CP_ME_NRT_DATA					0x00000810
-
-#define REG_A5XX_CP_CRASH_SCRIPT_BASE_LO			0x00000817
-
-#define REG_A5XX_CP_CRASH_SCRIPT_BASE_HI			0x00000818
-
-#define REG_A5XX_CP_CRASH_DUMP_CNTL				0x00000819
-
-#define REG_A5XX_CP_ME_STAT_ADDR				0x0000081a
-
-#define REG_A5XX_CP_ROQ_THRESHOLDS_1				0x0000081f
-
-#define REG_A5XX_CP_ROQ_THRESHOLDS_2				0x00000820
-
-#define REG_A5XX_CP_ROQ_DBG_ADDR				0x00000821
-
-#define REG_A5XX_CP_ROQ_DBG_DATA				0x00000822
-
-#define REG_A5XX_CP_MEQ_DBG_ADDR				0x00000823
-
-#define REG_A5XX_CP_MEQ_DBG_DATA				0x00000824
-
-#define REG_A5XX_CP_MEQ_THRESHOLDS				0x00000825
-
-#define REG_A5XX_CP_MERCIU_SIZE					0x00000826
-
-#define REG_A5XX_CP_MERCIU_DBG_ADDR				0x00000827
-
-#define REG_A5XX_CP_MERCIU_DBG_DATA_1				0x00000828
-
-#define REG_A5XX_CP_MERCIU_DBG_DATA_2				0x00000829
-
-#define REG_A5XX_CP_PFP_UCODE_DBG_ADDR				0x0000082a
-
-#define REG_A5XX_CP_PFP_UCODE_DBG_DATA				0x0000082b
-
-#define REG_A5XX_CP_ME_UCODE_DBG_ADDR				0x0000082f
-
-#define REG_A5XX_CP_ME_UCODE_DBG_DATA				0x00000830
-
-#define REG_A5XX_CP_CNTL					0x00000831
-
-#define REG_A5XX_CP_PFP_ME_CNTL					0x00000832
-
-#define REG_A5XX_CP_CHICKEN_DBG					0x00000833
-
-#define REG_A5XX_CP_PFP_INSTR_BASE_LO				0x00000835
-
-#define REG_A5XX_CP_PFP_INSTR_BASE_HI				0x00000836
-
-#define REG_A5XX_CP_ME_INSTR_BASE_LO				0x00000838
-
-#define REG_A5XX_CP_ME_INSTR_BASE_HI				0x00000839
-
-#define REG_A5XX_CP_CONTEXT_SWITCH_CNTL				0x0000083b
-
-#define REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO		0x0000083c
-
-#define REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI		0x0000083d
-
-#define REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO			0x0000083e
-
-#define REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_HI			0x0000083f
-
-#define REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO			0x00000840
-
-#define REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI			0x00000841
-
-#define REG_A5XX_CP_ADDR_MODE_CNTL				0x00000860
-
-#define REG_A5XX_CP_ME_STAT_DATA				0x00000b14
-
-#define REG_A5XX_CP_WFI_PEND_CTR				0x00000b15
-
-#define REG_A5XX_CP_INTERRUPT_STATUS				0x00000b18
-
-#define REG_A5XX_CP_HW_FAULT					0x00000b1a
-
-#define REG_A5XX_CP_PROTECT_STATUS				0x00000b1c
-
-#define REG_A5XX_CP_IB1_BASE					0x00000b1f
-
-#define REG_A5XX_CP_IB1_BASE_HI					0x00000b20
-
-#define REG_A5XX_CP_IB1_BUFSZ					0x00000b21
-
-#define REG_A5XX_CP_IB2_BASE					0x00000b22
-
-#define REG_A5XX_CP_IB2_BASE_HI					0x00000b23
-
-#define REG_A5XX_CP_IB2_BUFSZ					0x00000b24
-
-#define REG_A5XX_CP_SCRATCH(i0) (0x00000b78 + 0x1*(i0))
-
-static inline uint32_t REG_A5XX_CP_SCRATCH_REG(uint32_t i0) { return 0x00000b78 + 0x1*i0; }
-
-#define REG_A5XX_CP_PROTECT(i0) (0x00000880 + 0x1*(i0))
-
-static inline uint32_t REG_A5XX_CP_PROTECT_REG(uint32_t i0) { return 0x00000880 + 0x1*i0; }
-#define A5XX_CP_PROTECT_REG_BASE_ADDR__MASK			0x0001ffff
-#define A5XX_CP_PROTECT_REG_BASE_ADDR__SHIFT			0
-static inline uint32_t A5XX_CP_PROTECT_REG_BASE_ADDR(uint32_t val)
-{
-	return ((val) << A5XX_CP_PROTECT_REG_BASE_ADDR__SHIFT) & A5XX_CP_PROTECT_REG_BASE_ADDR__MASK;
-}
-#define A5XX_CP_PROTECT_REG_MASK_LEN__MASK			0x1f000000
-#define A5XX_CP_PROTECT_REG_MASK_LEN__SHIFT			24
-static inline uint32_t A5XX_CP_PROTECT_REG_MASK_LEN(uint32_t val)
-{
-	return ((val) << A5XX_CP_PROTECT_REG_MASK_LEN__SHIFT) & A5XX_CP_PROTECT_REG_MASK_LEN__MASK;
-}
-#define A5XX_CP_PROTECT_REG_TRAP_WRITE				0x20000000
-#define A5XX_CP_PROTECT_REG_TRAP_READ				0x40000000
-
-#define REG_A5XX_CP_PROTECT_CNTL				0x000008a0
-
-#define REG_A5XX_CP_AHB_FAULT					0x00000b1b
-
-#define REG_A5XX_CP_PERFCTR_CP_SEL_0				0x00000bb0
-
-#define REG_A5XX_CP_PERFCTR_CP_SEL_1				0x00000bb1
-
-#define REG_A5XX_CP_PERFCTR_CP_SEL_2				0x00000bb2
-
-#define REG_A5XX_CP_PERFCTR_CP_SEL_3				0x00000bb3
-
-#define REG_A5XX_CP_PERFCTR_CP_SEL_4				0x00000bb4
-
-#define REG_A5XX_CP_PERFCTR_CP_SEL_5				0x00000bb5
-
-#define REG_A5XX_CP_PERFCTR_CP_SEL_6				0x00000bb6
-
-#define REG_A5XX_CP_PERFCTR_CP_SEL_7				0x00000bb7
-
-#define REG_A5XX_VSC_ADDR_MODE_CNTL				0x00000bc1
-
-#define REG_A5XX_CP_POWERCTR_CP_SEL_0				0x00000bba
-
-#define REG_A5XX_CP_POWERCTR_CP_SEL_1				0x00000bbb
-
-#define REG_A5XX_CP_POWERCTR_CP_SEL_2				0x00000bbc
-
-#define REG_A5XX_CP_POWERCTR_CP_SEL_3				0x00000bbd
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_SEL_A				0x00000004
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_SEL_B				0x00000005
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_SEL_C				0x00000006
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_SEL_D				0x00000007
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_CNTLT				0x00000008
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_CNTLM				0x00000009
-
-#define REG_A5XX_RBBM_CFG_DEBBUS_CTLTM_ENABLE_SHIFT		0x00000018
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_OPL				0x0000000a
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_OPE				0x0000000b
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_IVTL_0				0x0000000c
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_IVTL_1				0x0000000d
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_IVTL_2				0x0000000e
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_IVTL_3				0x0000000f
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_MASKL_0			0x00000010
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_MASKL_1			0x00000011
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_MASKL_2			0x00000012
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_MASKL_3			0x00000013
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_BYTEL_0			0x00000014
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_BYTEL_1			0x00000015
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_IVTE_0				0x00000016
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_IVTE_1				0x00000017
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_IVTE_2				0x00000018
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_IVTE_3				0x00000019
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_MASKE_0			0x0000001a
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_MASKE_1			0x0000001b
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_MASKE_2			0x0000001c
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_MASKE_3			0x0000001d
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_NIBBLEE			0x0000001e
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_PTRC0				0x0000001f
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_PTRC1				0x00000020
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_LOADREG			0x00000021
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_IDX				0x00000022
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_CLRC				0x00000023
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_LOADIVT			0x00000024
-
-#define REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL			0x0000002f
-
-#define REG_A5XX_RBBM_INT_CLEAR_CMD				0x00000037
-
-#define REG_A5XX_RBBM_INT_0_MASK				0x00000038
-#define A5XX_RBBM_INT_0_MASK_RBBM_GPU_IDLE			0x00000001
-#define A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR			0x00000002
-#define A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT		0x00000004
-#define A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT			0x00000008
-#define A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT		0x00000010
-#define A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT		0x00000020
-#define A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW		0x00000040
-#define A5XX_RBBM_INT_0_MASK_RBBM_GPC_ERROR			0x00000080
-#define A5XX_RBBM_INT_0_MASK_CP_SW				0x00000100
-#define A5XX_RBBM_INT_0_MASK_CP_HW_ERROR			0x00000200
-#define A5XX_RBBM_INT_0_MASK_CP_CCU_FLUSH_DEPTH_TS		0x00000400
-#define A5XX_RBBM_INT_0_MASK_CP_CCU_FLUSH_COLOR_TS		0x00000800
-#define A5XX_RBBM_INT_0_MASK_CP_CCU_RESOLVE_TS			0x00001000
-#define A5XX_RBBM_INT_0_MASK_CP_IB2				0x00002000
-#define A5XX_RBBM_INT_0_MASK_CP_IB1				0x00004000
-#define A5XX_RBBM_INT_0_MASK_CP_RB				0x00008000
-#define A5XX_RBBM_INT_0_MASK_CP_RB_DONE_TS			0x00020000
-#define A5XX_RBBM_INT_0_MASK_CP_WT_DONE_TS			0x00040000
-#define A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS			0x00100000
-#define A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW		0x00400000
-#define A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT			0x00800000
-#define A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS			0x01000000
-#define A5XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR			0x02000000
-#define A5XX_RBBM_INT_0_MASK_DEBBUS_INTR_0			0x04000000
-#define A5XX_RBBM_INT_0_MASK_DEBBUS_INTR_1			0x08000000
-#define A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP			0x10000000
-#define A5XX_RBBM_INT_0_MASK_GPMU_FIRMWARE			0x20000000
-#define A5XX_RBBM_INT_0_MASK_ISDB_CPU_IRQ			0x40000000
-#define A5XX_RBBM_INT_0_MASK_ISDB_UNDER_DEBUG			0x80000000
-
-#define REG_A5XX_RBBM_AHB_DBG_CNTL				0x0000003f
-
-#define REG_A5XX_RBBM_EXT_VBIF_DBG_CNTL				0x00000041
-
-#define REG_A5XX_RBBM_SW_RESET_CMD				0x00000043
-
-#define REG_A5XX_RBBM_BLOCK_SW_RESET_CMD			0x00000045
-
-#define REG_A5XX_RBBM_BLOCK_SW_RESET_CMD2			0x00000046
-
-#define REG_A5XX_RBBM_DBG_LO_HI_GPIO				0x00000048
-
-#define REG_A5XX_RBBM_EXT_TRACE_BUS_CNTL			0x00000049
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_TP0				0x0000004a
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_TP1				0x0000004b
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_TP2				0x0000004c
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_TP3				0x0000004d
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_TP0				0x0000004e
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_TP1				0x0000004f
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_TP2				0x00000050
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_TP3				0x00000051
-
-#define REG_A5XX_RBBM_CLOCK_CNTL3_TP0				0x00000052
-
-#define REG_A5XX_RBBM_CLOCK_CNTL3_TP1				0x00000053
-
-#define REG_A5XX_RBBM_CLOCK_CNTL3_TP2				0x00000054
-
-#define REG_A5XX_RBBM_CLOCK_CNTL3_TP3				0x00000055
-
-#define REG_A5XX_RBBM_READ_AHB_THROUGH_DBG			0x00000059
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_UCHE				0x0000005a
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_UCHE				0x0000005b
-
-#define REG_A5XX_RBBM_CLOCK_CNTL3_UCHE				0x0000005c
-
-#define REG_A5XX_RBBM_CLOCK_CNTL4_UCHE				0x0000005d
-
-#define REG_A5XX_RBBM_CLOCK_HYST_UCHE				0x0000005e
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_UCHE				0x0000005f
-
-#define REG_A5XX_RBBM_CLOCK_MODE_GPC				0x00000060
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_GPC				0x00000061
-
-#define REG_A5XX_RBBM_CLOCK_HYST_GPC				0x00000062
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM			0x00000063
-
-#define REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM			0x00000064
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM			0x00000065
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_HLSQ				0x00000066
-
-#define REG_A5XX_RBBM_CLOCK_CNTL				0x00000067
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_SP0				0x00000068
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_SP1				0x00000069
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_SP2				0x0000006a
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_SP3				0x0000006b
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_SP0				0x0000006c
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_SP1				0x0000006d
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_SP2				0x0000006e
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_SP3				0x0000006f
-
-#define REG_A5XX_RBBM_CLOCK_HYST_SP0				0x00000070
-
-#define REG_A5XX_RBBM_CLOCK_HYST_SP1				0x00000071
-
-#define REG_A5XX_RBBM_CLOCK_HYST_SP2				0x00000072
-
-#define REG_A5XX_RBBM_CLOCK_HYST_SP3				0x00000073
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_SP0				0x00000074
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_SP1				0x00000075
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_SP2				0x00000076
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_SP3				0x00000077
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_RB0				0x00000078
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_RB1				0x00000079
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_RB2				0x0000007a
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_RB3				0x0000007b
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_RB0				0x0000007c
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_RB1				0x0000007d
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_RB2				0x0000007e
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_RB3				0x0000007f
-
-#define REG_A5XX_RBBM_CLOCK_HYST_RAC				0x00000080
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_RAC				0x00000081
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_CCU0				0x00000082
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_CCU1				0x00000083
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_CCU2				0x00000084
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_CCU3				0x00000085
-
-#define REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0			0x00000086
-
-#define REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1			0x00000087
-
-#define REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2			0x00000088
-
-#define REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3			0x00000089
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_RAC				0x0000008a
-
-#define REG_A5XX_RBBM_CLOCK_CNTL2_RAC				0x0000008b
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0			0x0000008c
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1			0x0000008d
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2			0x0000008e
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3			0x0000008f
-
-#define REG_A5XX_RBBM_CLOCK_HYST_VFD				0x00000090
-
-#define REG_A5XX_RBBM_CLOCK_MODE_VFD				0x00000091
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_VFD				0x00000092
-
-#define REG_A5XX_RBBM_AHB_CNTL0					0x00000093
-
-#define REG_A5XX_RBBM_AHB_CNTL1					0x00000094
-
-#define REG_A5XX_RBBM_AHB_CNTL2					0x00000095
-
-#define REG_A5XX_RBBM_AHB_CMD					0x00000096
-
-#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11		0x0000009c
-
-#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12		0x0000009d
-
-#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13		0x0000009e
-
-#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14		0x0000009f
-
-#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15		0x000000a0
-
-#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16		0x000000a1
-
-#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17		0x000000a2
-
-#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18		0x000000a3
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_TP0				0x000000a4
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_TP1				0x000000a5
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_TP2				0x000000a6
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_TP3				0x000000a7
-
-#define REG_A5XX_RBBM_CLOCK_DELAY2_TP0				0x000000a8
-
-#define REG_A5XX_RBBM_CLOCK_DELAY2_TP1				0x000000a9
-
-#define REG_A5XX_RBBM_CLOCK_DELAY2_TP2				0x000000aa
-
-#define REG_A5XX_RBBM_CLOCK_DELAY2_TP3				0x000000ab
-
-#define REG_A5XX_RBBM_CLOCK_DELAY3_TP0				0x000000ac
-
-#define REG_A5XX_RBBM_CLOCK_DELAY3_TP1				0x000000ad
-
-#define REG_A5XX_RBBM_CLOCK_DELAY3_TP2				0x000000ae
-
-#define REG_A5XX_RBBM_CLOCK_DELAY3_TP3				0x000000af
-
-#define REG_A5XX_RBBM_CLOCK_HYST_TP0				0x000000b0
-
-#define REG_A5XX_RBBM_CLOCK_HYST_TP1				0x000000b1
-
-#define REG_A5XX_RBBM_CLOCK_HYST_TP2				0x000000b2
-
-#define REG_A5XX_RBBM_CLOCK_HYST_TP3				0x000000b3
-
-#define REG_A5XX_RBBM_CLOCK_HYST2_TP0				0x000000b4
-
-#define REG_A5XX_RBBM_CLOCK_HYST2_TP1				0x000000b5
-
-#define REG_A5XX_RBBM_CLOCK_HYST2_TP2				0x000000b6
-
-#define REG_A5XX_RBBM_CLOCK_HYST2_TP3				0x000000b7
-
-#define REG_A5XX_RBBM_CLOCK_HYST3_TP0				0x000000b8
-
-#define REG_A5XX_RBBM_CLOCK_HYST3_TP1				0x000000b9
-
-#define REG_A5XX_RBBM_CLOCK_HYST3_TP2				0x000000ba
-
-#define REG_A5XX_RBBM_CLOCK_HYST3_TP3				0x000000bb
-
-#define REG_A5XX_RBBM_CLOCK_CNTL_GPMU				0x000000c8
-
-#define REG_A5XX_RBBM_CLOCK_DELAY_GPMU				0x000000c9
-
-#define REG_A5XX_RBBM_CLOCK_HYST_GPMU				0x000000ca
-
-#define REG_A5XX_RBBM_PERFCTR_CP_0_LO				0x000003a0
-
-#define REG_A5XX_RBBM_PERFCTR_CP_0_HI				0x000003a1
-
-#define REG_A5XX_RBBM_PERFCTR_CP_1_LO				0x000003a2
-
-#define REG_A5XX_RBBM_PERFCTR_CP_1_HI				0x000003a3
-
-#define REG_A5XX_RBBM_PERFCTR_CP_2_LO				0x000003a4
-
-#define REG_A5XX_RBBM_PERFCTR_CP_2_HI				0x000003a5
-
-#define REG_A5XX_RBBM_PERFCTR_CP_3_LO				0x000003a6
-
-#define REG_A5XX_RBBM_PERFCTR_CP_3_HI				0x000003a7
-
-#define REG_A5XX_RBBM_PERFCTR_CP_4_LO				0x000003a8
-
-#define REG_A5XX_RBBM_PERFCTR_CP_4_HI				0x000003a9
-
-#define REG_A5XX_RBBM_PERFCTR_CP_5_LO				0x000003aa
-
-#define REG_A5XX_RBBM_PERFCTR_CP_5_HI				0x000003ab
-
-#define REG_A5XX_RBBM_PERFCTR_CP_6_LO				0x000003ac
-
-#define REG_A5XX_RBBM_PERFCTR_CP_6_HI				0x000003ad
-
-#define REG_A5XX_RBBM_PERFCTR_CP_7_LO				0x000003ae
-
-#define REG_A5XX_RBBM_PERFCTR_CP_7_HI				0x000003af
-
-#define REG_A5XX_RBBM_PERFCTR_RBBM_0_LO				0x000003b0
-
-#define REG_A5XX_RBBM_PERFCTR_RBBM_0_HI				0x000003b1
-
-#define REG_A5XX_RBBM_PERFCTR_RBBM_1_LO				0x000003b2
-
-#define REG_A5XX_RBBM_PERFCTR_RBBM_1_HI				0x000003b3
-
-#define REG_A5XX_RBBM_PERFCTR_RBBM_2_LO				0x000003b4
-
-#define REG_A5XX_RBBM_PERFCTR_RBBM_2_HI				0x000003b5
-
-#define REG_A5XX_RBBM_PERFCTR_RBBM_3_LO				0x000003b6
-
-#define REG_A5XX_RBBM_PERFCTR_RBBM_3_HI				0x000003b7
-
-#define REG_A5XX_RBBM_PERFCTR_PC_0_LO				0x000003b8
-
-#define REG_A5XX_RBBM_PERFCTR_PC_0_HI				0x000003b9
-
-#define REG_A5XX_RBBM_PERFCTR_PC_1_LO				0x000003ba
-
-#define REG_A5XX_RBBM_PERFCTR_PC_1_HI				0x000003bb
-
-#define REG_A5XX_RBBM_PERFCTR_PC_2_LO				0x000003bc
-
-#define REG_A5XX_RBBM_PERFCTR_PC_2_HI				0x000003bd
-
-#define REG_A5XX_RBBM_PERFCTR_PC_3_LO				0x000003be
-
-#define REG_A5XX_RBBM_PERFCTR_PC_3_HI				0x000003bf
-
-#define REG_A5XX_RBBM_PERFCTR_PC_4_LO				0x000003c0
-
-#define REG_A5XX_RBBM_PERFCTR_PC_4_HI				0x000003c1
-
-#define REG_A5XX_RBBM_PERFCTR_PC_5_LO				0x000003c2
-
-#define REG_A5XX_RBBM_PERFCTR_PC_5_HI				0x000003c3
-
-#define REG_A5XX_RBBM_PERFCTR_PC_6_LO				0x000003c4
-
-#define REG_A5XX_RBBM_PERFCTR_PC_6_HI				0x000003c5
-
-#define REG_A5XX_RBBM_PERFCTR_PC_7_LO				0x000003c6
-
-#define REG_A5XX_RBBM_PERFCTR_PC_7_HI				0x000003c7
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_0_LO				0x000003c8
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_0_HI				0x000003c9
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_1_LO				0x000003ca
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_1_HI				0x000003cb
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_2_LO				0x000003cc
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_2_HI				0x000003cd
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_3_LO				0x000003ce
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_3_HI				0x000003cf
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_4_LO				0x000003d0
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_4_HI				0x000003d1
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_5_LO				0x000003d2
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_5_HI				0x000003d3
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_6_LO				0x000003d4
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_6_HI				0x000003d5
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_7_LO				0x000003d6
-
-#define REG_A5XX_RBBM_PERFCTR_VFD_7_HI				0x000003d7
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_0_LO				0x000003d8
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_0_HI				0x000003d9
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_1_LO				0x000003da
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_1_HI				0x000003db
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_2_LO				0x000003dc
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_2_HI				0x000003dd
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_3_LO				0x000003de
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_3_HI				0x000003df
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_4_LO				0x000003e0
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_4_HI				0x000003e1
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_5_LO				0x000003e2
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_5_HI				0x000003e3
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_6_LO				0x000003e4
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_6_HI				0x000003e5
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_7_LO				0x000003e6
-
-#define REG_A5XX_RBBM_PERFCTR_HLSQ_7_HI				0x000003e7
-
-#define REG_A5XX_RBBM_PERFCTR_VPC_0_LO				0x000003e8
-
-#define REG_A5XX_RBBM_PERFCTR_VPC_0_HI				0x000003e9
-
-#define REG_A5XX_RBBM_PERFCTR_VPC_1_LO				0x000003ea
-
-#define REG_A5XX_RBBM_PERFCTR_VPC_1_HI				0x000003eb
-
-#define REG_A5XX_RBBM_PERFCTR_VPC_2_LO				0x000003ec
-
-#define REG_A5XX_RBBM_PERFCTR_VPC_2_HI				0x000003ed
-
-#define REG_A5XX_RBBM_PERFCTR_VPC_3_LO				0x000003ee
-
-#define REG_A5XX_RBBM_PERFCTR_VPC_3_HI				0x000003ef
-
-#define REG_A5XX_RBBM_PERFCTR_CCU_0_LO				0x000003f0
-
-#define REG_A5XX_RBBM_PERFCTR_CCU_0_HI				0x000003f1
-
-#define REG_A5XX_RBBM_PERFCTR_CCU_1_LO				0x000003f2
-
-#define REG_A5XX_RBBM_PERFCTR_CCU_1_HI				0x000003f3
-
-#define REG_A5XX_RBBM_PERFCTR_CCU_2_LO				0x000003f4
-
-#define REG_A5XX_RBBM_PERFCTR_CCU_2_HI				0x000003f5
-
-#define REG_A5XX_RBBM_PERFCTR_CCU_3_LO				0x000003f6
-
-#define REG_A5XX_RBBM_PERFCTR_CCU_3_HI				0x000003f7
-
-#define REG_A5XX_RBBM_PERFCTR_TSE_0_LO				0x000003f8
-
-#define REG_A5XX_RBBM_PERFCTR_TSE_0_HI				0x000003f9
-
-#define REG_A5XX_RBBM_PERFCTR_TSE_1_LO				0x000003fa
-
-#define REG_A5XX_RBBM_PERFCTR_TSE_1_HI				0x000003fb
-
-#define REG_A5XX_RBBM_PERFCTR_TSE_2_LO				0x000003fc
-
-#define REG_A5XX_RBBM_PERFCTR_TSE_2_HI				0x000003fd
-
-#define REG_A5XX_RBBM_PERFCTR_TSE_3_LO				0x000003fe
-
-#define REG_A5XX_RBBM_PERFCTR_TSE_3_HI				0x000003ff
-
-#define REG_A5XX_RBBM_PERFCTR_RAS_0_LO				0x00000400
-
-#define REG_A5XX_RBBM_PERFCTR_RAS_0_HI				0x00000401
-
-#define REG_A5XX_RBBM_PERFCTR_RAS_1_LO				0x00000402
-
-#define REG_A5XX_RBBM_PERFCTR_RAS_1_HI				0x00000403
-
-#define REG_A5XX_RBBM_PERFCTR_RAS_2_LO				0x00000404
-
-#define REG_A5XX_RBBM_PERFCTR_RAS_2_HI				0x00000405
-
-#define REG_A5XX_RBBM_PERFCTR_RAS_3_LO				0x00000406
-
-#define REG_A5XX_RBBM_PERFCTR_RAS_3_HI				0x00000407
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_0_LO				0x00000408
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_0_HI				0x00000409
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_1_LO				0x0000040a
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_1_HI				0x0000040b
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_2_LO				0x0000040c
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_2_HI				0x0000040d
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_3_LO				0x0000040e
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_3_HI				0x0000040f
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_4_LO				0x00000410
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_4_HI				0x00000411
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_5_LO				0x00000412
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_5_HI				0x00000413
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_6_LO				0x00000414
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_6_HI				0x00000415
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_7_LO				0x00000416
-
-#define REG_A5XX_RBBM_PERFCTR_UCHE_7_HI				0x00000417
-
-#define REG_A5XX_RBBM_PERFCTR_TP_0_LO				0x00000418
-
-#define REG_A5XX_RBBM_PERFCTR_TP_0_HI				0x00000419
-
-#define REG_A5XX_RBBM_PERFCTR_TP_1_LO				0x0000041a
-
-#define REG_A5XX_RBBM_PERFCTR_TP_1_HI				0x0000041b
-
-#define REG_A5XX_RBBM_PERFCTR_TP_2_LO				0x0000041c
-
-#define REG_A5XX_RBBM_PERFCTR_TP_2_HI				0x0000041d
-
-#define REG_A5XX_RBBM_PERFCTR_TP_3_LO				0x0000041e
-
-#define REG_A5XX_RBBM_PERFCTR_TP_3_HI				0x0000041f
-
-#define REG_A5XX_RBBM_PERFCTR_TP_4_LO				0x00000420
-
-#define REG_A5XX_RBBM_PERFCTR_TP_4_HI				0x00000421
-
-#define REG_A5XX_RBBM_PERFCTR_TP_5_LO				0x00000422
-
-#define REG_A5XX_RBBM_PERFCTR_TP_5_HI				0x00000423
-
-#define REG_A5XX_RBBM_PERFCTR_TP_6_LO				0x00000424
-
-#define REG_A5XX_RBBM_PERFCTR_TP_6_HI				0x00000425
-
-#define REG_A5XX_RBBM_PERFCTR_TP_7_LO				0x00000426
-
-#define REG_A5XX_RBBM_PERFCTR_TP_7_HI				0x00000427
-
-#define REG_A5XX_RBBM_PERFCTR_SP_0_LO				0x00000428
-
-#define REG_A5XX_RBBM_PERFCTR_SP_0_HI				0x00000429
-
-#define REG_A5XX_RBBM_PERFCTR_SP_1_LO				0x0000042a
-
-#define REG_A5XX_RBBM_PERFCTR_SP_1_HI				0x0000042b
-
-#define REG_A5XX_RBBM_PERFCTR_SP_2_LO				0x0000042c
-
-#define REG_A5XX_RBBM_PERFCTR_SP_2_HI				0x0000042d
-
-#define REG_A5XX_RBBM_PERFCTR_SP_3_LO				0x0000042e
-
-#define REG_A5XX_RBBM_PERFCTR_SP_3_HI				0x0000042f
-
-#define REG_A5XX_RBBM_PERFCTR_SP_4_LO				0x00000430
-
-#define REG_A5XX_RBBM_PERFCTR_SP_4_HI				0x00000431
-
-#define REG_A5XX_RBBM_PERFCTR_SP_5_LO				0x00000432
-
-#define REG_A5XX_RBBM_PERFCTR_SP_5_HI				0x00000433
-
-#define REG_A5XX_RBBM_PERFCTR_SP_6_LO				0x00000434
-
-#define REG_A5XX_RBBM_PERFCTR_SP_6_HI				0x00000435
-
-#define REG_A5XX_RBBM_PERFCTR_SP_7_LO				0x00000436
-
-#define REG_A5XX_RBBM_PERFCTR_SP_7_HI				0x00000437
-
-#define REG_A5XX_RBBM_PERFCTR_SP_8_LO				0x00000438
-
-#define REG_A5XX_RBBM_PERFCTR_SP_8_HI				0x00000439
-
-#define REG_A5XX_RBBM_PERFCTR_SP_9_LO				0x0000043a
-
-#define REG_A5XX_RBBM_PERFCTR_SP_9_HI				0x0000043b
-
-#define REG_A5XX_RBBM_PERFCTR_SP_10_LO				0x0000043c
-
-#define REG_A5XX_RBBM_PERFCTR_SP_10_HI				0x0000043d
-
-#define REG_A5XX_RBBM_PERFCTR_SP_11_LO				0x0000043e
-
-#define REG_A5XX_RBBM_PERFCTR_SP_11_HI				0x0000043f
-
-#define REG_A5XX_RBBM_PERFCTR_RB_0_LO				0x00000440
-
-#define REG_A5XX_RBBM_PERFCTR_RB_0_HI				0x00000441
-
-#define REG_A5XX_RBBM_PERFCTR_RB_1_LO				0x00000442
-
-#define REG_A5XX_RBBM_PERFCTR_RB_1_HI				0x00000443
-
-#define REG_A5XX_RBBM_PERFCTR_RB_2_LO				0x00000444
-
-#define REG_A5XX_RBBM_PERFCTR_RB_2_HI				0x00000445
-
-#define REG_A5XX_RBBM_PERFCTR_RB_3_LO				0x00000446
-
-#define REG_A5XX_RBBM_PERFCTR_RB_3_HI				0x00000447
-
-#define REG_A5XX_RBBM_PERFCTR_RB_4_LO				0x00000448
-
-#define REG_A5XX_RBBM_PERFCTR_RB_4_HI				0x00000449
-
-#define REG_A5XX_RBBM_PERFCTR_RB_5_LO				0x0000044a
-
-#define REG_A5XX_RBBM_PERFCTR_RB_5_HI				0x0000044b
-
-#define REG_A5XX_RBBM_PERFCTR_RB_6_LO				0x0000044c
-
-#define REG_A5XX_RBBM_PERFCTR_RB_6_HI				0x0000044d
-
-#define REG_A5XX_RBBM_PERFCTR_RB_7_LO				0x0000044e
-
-#define REG_A5XX_RBBM_PERFCTR_RB_7_HI				0x0000044f
-
-#define REG_A5XX_RBBM_PERFCTR_VSC_0_LO				0x00000450
-
-#define REG_A5XX_RBBM_PERFCTR_VSC_0_HI				0x00000451
-
-#define REG_A5XX_RBBM_PERFCTR_VSC_1_LO				0x00000452
-
-#define REG_A5XX_RBBM_PERFCTR_VSC_1_HI				0x00000453
-
-#define REG_A5XX_RBBM_PERFCTR_LRZ_0_LO				0x00000454
-
-#define REG_A5XX_RBBM_PERFCTR_LRZ_0_HI				0x00000455
-
-#define REG_A5XX_RBBM_PERFCTR_LRZ_1_LO				0x00000456
-
-#define REG_A5XX_RBBM_PERFCTR_LRZ_1_HI				0x00000457
-
-#define REG_A5XX_RBBM_PERFCTR_LRZ_2_LO				0x00000458
-
-#define REG_A5XX_RBBM_PERFCTR_LRZ_2_HI				0x00000459
-
-#define REG_A5XX_RBBM_PERFCTR_LRZ_3_LO				0x0000045a
-
-#define REG_A5XX_RBBM_PERFCTR_LRZ_3_HI				0x0000045b
-
-#define REG_A5XX_RBBM_PERFCTR_CMP_0_LO				0x0000045c
-
-#define REG_A5XX_RBBM_PERFCTR_CMP_0_HI				0x0000045d
-
-#define REG_A5XX_RBBM_PERFCTR_CMP_1_LO				0x0000045e
-
-#define REG_A5XX_RBBM_PERFCTR_CMP_1_HI				0x0000045f
-
-#define REG_A5XX_RBBM_PERFCTR_CMP_2_LO				0x00000460
-
-#define REG_A5XX_RBBM_PERFCTR_CMP_2_HI				0x00000461
-
-#define REG_A5XX_RBBM_PERFCTR_CMP_3_LO				0x00000462
-
-#define REG_A5XX_RBBM_PERFCTR_CMP_3_HI				0x00000463
-
-#define REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0			0x0000046b
-
-#define REG_A5XX_RBBM_PERFCTR_RBBM_SEL_1			0x0000046c
-
-#define REG_A5XX_RBBM_PERFCTR_RBBM_SEL_2			0x0000046d
-
-#define REG_A5XX_RBBM_PERFCTR_RBBM_SEL_3			0x0000046e
-
-#define REG_A5XX_RBBM_ALWAYSON_COUNTER_LO			0x000004d2
-
-#define REG_A5XX_RBBM_ALWAYSON_COUNTER_HI			0x000004d3
-
-#define REG_A5XX_RBBM_STATUS					0x000004f5
-#define A5XX_RBBM_STATUS_GPU_BUSY_IGN_AHB			0x80000000
-#define A5XX_RBBM_STATUS_GPU_BUSY_IGN_AHB_CP			0x40000000
-#define A5XX_RBBM_STATUS_HLSQ_BUSY				0x20000000
-#define A5XX_RBBM_STATUS_VSC_BUSY				0x10000000
-#define A5XX_RBBM_STATUS_TPL1_BUSY				0x08000000
-#define A5XX_RBBM_STATUS_SP_BUSY				0x04000000
-#define A5XX_RBBM_STATUS_UCHE_BUSY				0x02000000
-#define A5XX_RBBM_STATUS_VPC_BUSY				0x01000000
-#define A5XX_RBBM_STATUS_VFDP_BUSY				0x00800000
-#define A5XX_RBBM_STATUS_VFD_BUSY				0x00400000
-#define A5XX_RBBM_STATUS_TESS_BUSY				0x00200000
-#define A5XX_RBBM_STATUS_PC_VSD_BUSY				0x00100000
-#define A5XX_RBBM_STATUS_PC_DCALL_BUSY				0x00080000
-#define A5XX_RBBM_STATUS_GPMU_SLAVE_BUSY			0x00040000
-#define A5XX_RBBM_STATUS_DCOM_BUSY				0x00020000
-#define A5XX_RBBM_STATUS_COM_BUSY				0x00010000
-#define A5XX_RBBM_STATUS_LRZ_BUZY				0x00008000
-#define A5XX_RBBM_STATUS_A2D_DSP_BUSY				0x00004000
-#define A5XX_RBBM_STATUS_CCUFCHE_BUSY				0x00002000
-#define A5XX_RBBM_STATUS_RB_BUSY				0x00001000
-#define A5XX_RBBM_STATUS_RAS_BUSY				0x00000800
-#define A5XX_RBBM_STATUS_TSE_BUSY				0x00000400
-#define A5XX_RBBM_STATUS_VBIF_BUSY				0x00000200
-#define A5XX_RBBM_STATUS_GPU_BUSY_IGN_AHB_HYST			0x00000100
-#define A5XX_RBBM_STATUS_CP_BUSY_IGN_HYST			0x00000080
-#define A5XX_RBBM_STATUS_CP_BUSY				0x00000040
-#define A5XX_RBBM_STATUS_GPMU_MASTER_BUSY			0x00000020
-#define A5XX_RBBM_STATUS_CP_CRASH_BUSY				0x00000010
-#define A5XX_RBBM_STATUS_CP_ETS_BUSY				0x00000008
-#define A5XX_RBBM_STATUS_CP_PFP_BUSY				0x00000004
-#define A5XX_RBBM_STATUS_CP_ME_BUSY				0x00000002
-#define A5XX_RBBM_STATUS_HI_BUSY				0x00000001
-
-#define REG_A5XX_RBBM_STATUS3					0x00000530
-#define A5XX_RBBM_STATUS3_SMMU_STALLED_ON_FAULT			0x01000000
-
-#define REG_A5XX_RBBM_INT_0_STATUS				0x000004e1
-
-#define REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS			0x000004f0
-
-#define REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS			0x000004f1
-
-#define REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS			0x000004f3
-
-#define REG_A5XX_RBBM_AHB_ERROR_STATUS				0x000004f4
-
-#define REG_A5XX_RBBM_PERFCTR_CNTL				0x00000464
-
-#define REG_A5XX_RBBM_PERFCTR_LOAD_CMD0				0x00000465
-
-#define REG_A5XX_RBBM_PERFCTR_LOAD_CMD1				0x00000466
-
-#define REG_A5XX_RBBM_PERFCTR_LOAD_CMD2				0x00000467
-
-#define REG_A5XX_RBBM_PERFCTR_LOAD_CMD3				0x00000468
-
-#define REG_A5XX_RBBM_PERFCTR_LOAD_VALUE_LO			0x00000469
-
-#define REG_A5XX_RBBM_PERFCTR_LOAD_VALUE_HI			0x0000046a
-
-#define REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED			0x0000046f
-
-#define REG_A5XX_RBBM_AHB_ERROR					0x000004ed
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_EVENT_LOGIC			0x00000504
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_OVER				0x00000505
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_COUNT0				0x00000506
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_COUNT1				0x00000507
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_COUNT2				0x00000508
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_COUNT3				0x00000509
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_COUNT4				0x0000050a
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_COUNT5				0x0000050b
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_TRACE_ADDR			0x0000050c
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_TRACE_BUF0			0x0000050d
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_TRACE_BUF1			0x0000050e
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_TRACE_BUF2			0x0000050f
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_TRACE_BUF3			0x00000510
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_TRACE_BUF4			0x00000511
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_MISR0				0x00000512
-
-#define REG_A5XX_RBBM_CFG_DBGBUS_MISR1				0x00000513
-
-#define REG_A5XX_RBBM_ISDB_CNT					0x00000533
-
-#define REG_A5XX_RBBM_SECVID_TRUST_CONFIG			0x0000f000
-
-#define REG_A5XX_RBBM_SECVID_TRUST_CNTL				0x0000f400
-
-#define REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO		0x0000f800
-
-#define REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI		0x0000f801
-
-#define REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE			0x0000f802
-
-#define REG_A5XX_RBBM_SECVID_TSB_CNTL				0x0000f803
-
-#define REG_A5XX_RBBM_SECVID_TSB_COMP_STATUS_LO			0x0000f804
-
-#define REG_A5XX_RBBM_SECVID_TSB_COMP_STATUS_HI			0x0000f805
-
-#define REG_A5XX_RBBM_SECVID_TSB_UCHE_STATUS_LO			0x0000f806
-
-#define REG_A5XX_RBBM_SECVID_TSB_UCHE_STATUS_HI			0x0000f807
-
-#define REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL			0x0000f810
-
-#define REG_A5XX_VSC_BIN_SIZE					0x00000bc2
-#define A5XX_VSC_BIN_SIZE_WIDTH__MASK				0x000000ff
-#define A5XX_VSC_BIN_SIZE_WIDTH__SHIFT				0
-static inline uint32_t A5XX_VSC_BIN_SIZE_WIDTH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A5XX_VSC_BIN_SIZE_WIDTH__SHIFT) & A5XX_VSC_BIN_SIZE_WIDTH__MASK;
-}
-#define A5XX_VSC_BIN_SIZE_HEIGHT__MASK				0x0001fe00
-#define A5XX_VSC_BIN_SIZE_HEIGHT__SHIFT				9
-static inline uint32_t A5XX_VSC_BIN_SIZE_HEIGHT(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A5XX_VSC_BIN_SIZE_HEIGHT__SHIFT) & A5XX_VSC_BIN_SIZE_HEIGHT__MASK;
-}
-
-#define REG_A5XX_VSC_SIZE_ADDRESS_LO				0x00000bc3
-
-#define REG_A5XX_VSC_SIZE_ADDRESS_HI				0x00000bc4
-
-#define REG_A5XX_UNKNOWN_0BC5					0x00000bc5
-
-#define REG_A5XX_UNKNOWN_0BC6					0x00000bc6
-
-#define REG_A5XX_VSC_PIPE_CONFIG(i0) (0x00000bd0 + 0x1*(i0))
-
-static inline uint32_t REG_A5XX_VSC_PIPE_CONFIG_REG(uint32_t i0) { return 0x00000bd0 + 0x1*i0; }
-#define A5XX_VSC_PIPE_CONFIG_REG_X__MASK			0x000003ff
-#define A5XX_VSC_PIPE_CONFIG_REG_X__SHIFT			0
-static inline uint32_t A5XX_VSC_PIPE_CONFIG_REG_X(uint32_t val)
-{
-	return ((val) << A5XX_VSC_PIPE_CONFIG_REG_X__SHIFT) & A5XX_VSC_PIPE_CONFIG_REG_X__MASK;
-}
-#define A5XX_VSC_PIPE_CONFIG_REG_Y__MASK			0x000ffc00
-#define A5XX_VSC_PIPE_CONFIG_REG_Y__SHIFT			10
-static inline uint32_t A5XX_VSC_PIPE_CONFIG_REG_Y(uint32_t val)
-{
-	return ((val) << A5XX_VSC_PIPE_CONFIG_REG_Y__SHIFT) & A5XX_VSC_PIPE_CONFIG_REG_Y__MASK;
-}
-#define A5XX_VSC_PIPE_CONFIG_REG_W__MASK			0x00f00000
-#define A5XX_VSC_PIPE_CONFIG_REG_W__SHIFT			20
-static inline uint32_t A5XX_VSC_PIPE_CONFIG_REG_W(uint32_t val)
-{
-	return ((val) << A5XX_VSC_PIPE_CONFIG_REG_W__SHIFT) & A5XX_VSC_PIPE_CONFIG_REG_W__MASK;
-}
-#define A5XX_VSC_PIPE_CONFIG_REG_H__MASK			0x0f000000
-#define A5XX_VSC_PIPE_CONFIG_REG_H__SHIFT			24
-static inline uint32_t A5XX_VSC_PIPE_CONFIG_REG_H(uint32_t val)
-{
-	return ((val) << A5XX_VSC_PIPE_CONFIG_REG_H__SHIFT) & A5XX_VSC_PIPE_CONFIG_REG_H__MASK;
-}
-
-#define REG_A5XX_VSC_PIPE_DATA_ADDRESS(i0) (0x00000be0 + 0x2*(i0))
-
-static inline uint32_t REG_A5XX_VSC_PIPE_DATA_ADDRESS_LO(uint32_t i0) { return 0x00000be0 + 0x2*i0; }
-
-static inline uint32_t REG_A5XX_VSC_PIPE_DATA_ADDRESS_HI(uint32_t i0) { return 0x00000be1 + 0x2*i0; }
-
-#define REG_A5XX_VSC_PIPE_DATA_LENGTH(i0) (0x00000c00 + 0x1*(i0))
-
-static inline uint32_t REG_A5XX_VSC_PIPE_DATA_LENGTH_REG(uint32_t i0) { return 0x00000c00 + 0x1*i0; }
-
-#define REG_A5XX_VSC_PERFCTR_VSC_SEL_0				0x00000c60
-
-#define REG_A5XX_VSC_PERFCTR_VSC_SEL_1				0x00000c61
-
-#define REG_A5XX_VSC_RESOLVE_CNTL				0x00000cdd
-#define A5XX_VSC_RESOLVE_CNTL_WINDOW_OFFSET_DISABLE		0x80000000
-#define A5XX_VSC_RESOLVE_CNTL_X__MASK				0x00007fff
-#define A5XX_VSC_RESOLVE_CNTL_X__SHIFT				0
-static inline uint32_t A5XX_VSC_RESOLVE_CNTL_X(uint32_t val)
-{
-	return ((val) << A5XX_VSC_RESOLVE_CNTL_X__SHIFT) & A5XX_VSC_RESOLVE_CNTL_X__MASK;
-}
-#define A5XX_VSC_RESOLVE_CNTL_Y__MASK				0x7fff0000
-#define A5XX_VSC_RESOLVE_CNTL_Y__SHIFT				16
-static inline uint32_t A5XX_VSC_RESOLVE_CNTL_Y(uint32_t val)
-{
-	return ((val) << A5XX_VSC_RESOLVE_CNTL_Y__SHIFT) & A5XX_VSC_RESOLVE_CNTL_Y__MASK;
-}
-
-#define REG_A5XX_GRAS_ADDR_MODE_CNTL				0x00000c81
-
-#define REG_A5XX_GRAS_PERFCTR_TSE_SEL_0				0x00000c90
-
-#define REG_A5XX_GRAS_PERFCTR_TSE_SEL_1				0x00000c91
-
-#define REG_A5XX_GRAS_PERFCTR_TSE_SEL_2				0x00000c92
-
-#define REG_A5XX_GRAS_PERFCTR_TSE_SEL_3				0x00000c93
-
-#define REG_A5XX_GRAS_PERFCTR_RAS_SEL_0				0x00000c94
-
-#define REG_A5XX_GRAS_PERFCTR_RAS_SEL_1				0x00000c95
-
-#define REG_A5XX_GRAS_PERFCTR_RAS_SEL_2				0x00000c96
-
-#define REG_A5XX_GRAS_PERFCTR_RAS_SEL_3				0x00000c97
-
-#define REG_A5XX_GRAS_PERFCTR_LRZ_SEL_0				0x00000c98
-
-#define REG_A5XX_GRAS_PERFCTR_LRZ_SEL_1				0x00000c99
-
-#define REG_A5XX_GRAS_PERFCTR_LRZ_SEL_2				0x00000c9a
-
-#define REG_A5XX_GRAS_PERFCTR_LRZ_SEL_3				0x00000c9b
-
-#define REG_A5XX_RB_DBG_ECO_CNTL				0x00000cc4
-
-#define REG_A5XX_RB_ADDR_MODE_CNTL				0x00000cc5
-
-#define REG_A5XX_RB_MODE_CNTL					0x00000cc6
-
-#define REG_A5XX_RB_CCU_CNTL					0x00000cc7
-
-#define REG_A5XX_RB_PERFCTR_RB_SEL_0				0x00000cd0
-
-#define REG_A5XX_RB_PERFCTR_RB_SEL_1				0x00000cd1
-
-#define REG_A5XX_RB_PERFCTR_RB_SEL_2				0x00000cd2
-
-#define REG_A5XX_RB_PERFCTR_RB_SEL_3				0x00000cd3
-
-#define REG_A5XX_RB_PERFCTR_RB_SEL_4				0x00000cd4
-
-#define REG_A5XX_RB_PERFCTR_RB_SEL_5				0x00000cd5
-
-#define REG_A5XX_RB_PERFCTR_RB_SEL_6				0x00000cd6
-
-#define REG_A5XX_RB_PERFCTR_RB_SEL_7				0x00000cd7
-
-#define REG_A5XX_RB_PERFCTR_CCU_SEL_0				0x00000cd8
-
-#define REG_A5XX_RB_PERFCTR_CCU_SEL_1				0x00000cd9
-
-#define REG_A5XX_RB_PERFCTR_CCU_SEL_2				0x00000cda
-
-#define REG_A5XX_RB_PERFCTR_CCU_SEL_3				0x00000cdb
-
-#define REG_A5XX_RB_POWERCTR_RB_SEL_0				0x00000ce0
-
-#define REG_A5XX_RB_POWERCTR_RB_SEL_1				0x00000ce1
-
-#define REG_A5XX_RB_POWERCTR_RB_SEL_2				0x00000ce2
-
-#define REG_A5XX_RB_POWERCTR_RB_SEL_3				0x00000ce3
-
-#define REG_A5XX_RB_POWERCTR_CCU_SEL_0				0x00000ce4
-
-#define REG_A5XX_RB_POWERCTR_CCU_SEL_1				0x00000ce5
-
-#define REG_A5XX_RB_PERFCTR_CMP_SEL_0				0x00000cec
-
-#define REG_A5XX_RB_PERFCTR_CMP_SEL_1				0x00000ced
-
-#define REG_A5XX_RB_PERFCTR_CMP_SEL_2				0x00000cee
-
-#define REG_A5XX_RB_PERFCTR_CMP_SEL_3				0x00000cef
-
-#define REG_A5XX_PC_DBG_ECO_CNTL				0x00000d00
-#define A5XX_PC_DBG_ECO_CNTL_TWOPASSUSEWFI			0x00000100
-
-#define REG_A5XX_PC_ADDR_MODE_CNTL				0x00000d01
-
-#define REG_A5XX_PC_MODE_CNTL					0x00000d02
-
-#define REG_A5XX_PC_INDEX_BUF_LO				0x00000d04
-
-#define REG_A5XX_PC_INDEX_BUF_HI				0x00000d05
-
-#define REG_A5XX_PC_START_INDEX					0x00000d06
-
-#define REG_A5XX_PC_MAX_INDEX					0x00000d07
-
-#define REG_A5XX_PC_TESSFACTOR_ADDR_LO				0x00000d08
-
-#define REG_A5XX_PC_TESSFACTOR_ADDR_HI				0x00000d09
-
-#define REG_A5XX_PC_PERFCTR_PC_SEL_0				0x00000d10
-
-#define REG_A5XX_PC_PERFCTR_PC_SEL_1				0x00000d11
-
-#define REG_A5XX_PC_PERFCTR_PC_SEL_2				0x00000d12
-
-#define REG_A5XX_PC_PERFCTR_PC_SEL_3				0x00000d13
-
-#define REG_A5XX_PC_PERFCTR_PC_SEL_4				0x00000d14
-
-#define REG_A5XX_PC_PERFCTR_PC_SEL_5				0x00000d15
-
-#define REG_A5XX_PC_PERFCTR_PC_SEL_6				0x00000d16
-
-#define REG_A5XX_PC_PERFCTR_PC_SEL_7				0x00000d17
-
-#define REG_A5XX_HLSQ_TIMEOUT_THRESHOLD_0			0x00000e00
-
-#define REG_A5XX_HLSQ_TIMEOUT_THRESHOLD_1			0x00000e01
-
-#define REG_A5XX_HLSQ_DBG_ECO_CNTL				0x00000e04
-
-#define REG_A5XX_HLSQ_ADDR_MODE_CNTL				0x00000e05
-
-#define REG_A5XX_HLSQ_MODE_CNTL					0x00000e06
-
-#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_0			0x00000e10
-
-#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_1			0x00000e11
-
-#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_2			0x00000e12
-
-#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_3			0x00000e13
-
-#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_4			0x00000e14
-
-#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_5			0x00000e15
-
-#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_6			0x00000e16
-
-#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_7			0x00000e17
-
-#define REG_A5XX_HLSQ_SPTP_RDSEL				0x00000f08
-
-#define REG_A5XX_HLSQ_DBG_READ_SEL				0x0000bc00
-
-#define REG_A5XX_HLSQ_DBG_AHB_READ_APERTURE			0x0000a000
-
-#define REG_A5XX_VFD_ADDR_MODE_CNTL				0x00000e41
-
-#define REG_A5XX_VFD_MODE_CNTL					0x00000e42
-
-#define REG_A5XX_VFD_PERFCTR_VFD_SEL_0				0x00000e50
-
-#define REG_A5XX_VFD_PERFCTR_VFD_SEL_1				0x00000e51
-
-#define REG_A5XX_VFD_PERFCTR_VFD_SEL_2				0x00000e52
-
-#define REG_A5XX_VFD_PERFCTR_VFD_SEL_3				0x00000e53
-
-#define REG_A5XX_VFD_PERFCTR_VFD_SEL_4				0x00000e54
-
-#define REG_A5XX_VFD_PERFCTR_VFD_SEL_5				0x00000e55
-
-#define REG_A5XX_VFD_PERFCTR_VFD_SEL_6				0x00000e56
-
-#define REG_A5XX_VFD_PERFCTR_VFD_SEL_7				0x00000e57
-
-#define REG_A5XX_VPC_DBG_ECO_CNTL				0x00000e60
-#define A5XX_VPC_DBG_ECO_CNTL_ALLFLATOPTDIS			0x00000400
-
-#define REG_A5XX_VPC_ADDR_MODE_CNTL				0x00000e61
-
-#define REG_A5XX_VPC_MODE_CNTL					0x00000e62
-#define A5XX_VPC_MODE_CNTL_BINNING_PASS				0x00000001
-
-#define REG_A5XX_VPC_PERFCTR_VPC_SEL_0				0x00000e64
-
-#define REG_A5XX_VPC_PERFCTR_VPC_SEL_1				0x00000e65
-
-#define REG_A5XX_VPC_PERFCTR_VPC_SEL_2				0x00000e66
-
-#define REG_A5XX_VPC_PERFCTR_VPC_SEL_3				0x00000e67
-
-#define REG_A5XX_UCHE_ADDR_MODE_CNTL				0x00000e80
-
-#define REG_A5XX_UCHE_MODE_CNTL					0x00000e81
-
-#define REG_A5XX_UCHE_SVM_CNTL					0x00000e82
-
-#define REG_A5XX_UCHE_WRITE_THRU_BASE_LO			0x00000e87
-
-#define REG_A5XX_UCHE_WRITE_THRU_BASE_HI			0x00000e88
-
-#define REG_A5XX_UCHE_TRAP_BASE_LO				0x00000e89
-
-#define REG_A5XX_UCHE_TRAP_BASE_HI				0x00000e8a
-
-#define REG_A5XX_UCHE_GMEM_RANGE_MIN_LO				0x00000e8b
-
-#define REG_A5XX_UCHE_GMEM_RANGE_MIN_HI				0x00000e8c
-
-#define REG_A5XX_UCHE_GMEM_RANGE_MAX_LO				0x00000e8d
-
-#define REG_A5XX_UCHE_GMEM_RANGE_MAX_HI				0x00000e8e
-
-#define REG_A5XX_UCHE_DBG_ECO_CNTL_2				0x00000e8f
-
-#define REG_A5XX_UCHE_DBG_ECO_CNTL				0x00000e90
-
-#define REG_A5XX_UCHE_CACHE_INVALIDATE_MIN_LO			0x00000e91
-
-#define REG_A5XX_UCHE_CACHE_INVALIDATE_MIN_HI			0x00000e92
-
-#define REG_A5XX_UCHE_CACHE_INVALIDATE_MAX_LO			0x00000e93
-
-#define REG_A5XX_UCHE_CACHE_INVALIDATE_MAX_HI			0x00000e94
-
-#define REG_A5XX_UCHE_CACHE_INVALIDATE				0x00000e95
-
-#define REG_A5XX_UCHE_CACHE_WAYS				0x00000e96
-
-#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_0			0x00000ea0
-
-#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_1			0x00000ea1
-
-#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_2			0x00000ea2
-
-#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_3			0x00000ea3
-
-#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_4			0x00000ea4
-
-#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_5			0x00000ea5
-
-#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_6			0x00000ea6
-
-#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_7			0x00000ea7
-
-#define REG_A5XX_UCHE_POWERCTR_UCHE_SEL_0			0x00000ea8
-
-#define REG_A5XX_UCHE_POWERCTR_UCHE_SEL_1			0x00000ea9
-
-#define REG_A5XX_UCHE_POWERCTR_UCHE_SEL_2			0x00000eaa
-
-#define REG_A5XX_UCHE_POWERCTR_UCHE_SEL_3			0x00000eab
-
-#define REG_A5XX_UCHE_TRAP_LOG_LO				0x00000eb1
-
-#define REG_A5XX_UCHE_TRAP_LOG_HI				0x00000eb2
-
-#define REG_A5XX_SP_DBG_ECO_CNTL				0x00000ec0
-
-#define REG_A5XX_SP_ADDR_MODE_CNTL				0x00000ec1
-
-#define REG_A5XX_SP_MODE_CNTL					0x00000ec2
-
-#define REG_A5XX_SP_PERFCTR_SP_SEL_0				0x00000ed0
-
-#define REG_A5XX_SP_PERFCTR_SP_SEL_1				0x00000ed1
-
-#define REG_A5XX_SP_PERFCTR_SP_SEL_2				0x00000ed2
-
-#define REG_A5XX_SP_PERFCTR_SP_SEL_3				0x00000ed3
-
-#define REG_A5XX_SP_PERFCTR_SP_SEL_4				0x00000ed4
-
-#define REG_A5XX_SP_PERFCTR_SP_SEL_5				0x00000ed5
-
-#define REG_A5XX_SP_PERFCTR_SP_SEL_6				0x00000ed6
-
-#define REG_A5XX_SP_PERFCTR_SP_SEL_7				0x00000ed7
-
-#define REG_A5XX_SP_PERFCTR_SP_SEL_8				0x00000ed8
-
-#define REG_A5XX_SP_PERFCTR_SP_SEL_9				0x00000ed9
-
-#define REG_A5XX_SP_PERFCTR_SP_SEL_10				0x00000eda
-
-#define REG_A5XX_SP_PERFCTR_SP_SEL_11				0x00000edb
-
-#define REG_A5XX_SP_POWERCTR_SP_SEL_0				0x00000edc
-
-#define REG_A5XX_SP_POWERCTR_SP_SEL_1				0x00000edd
-
-#define REG_A5XX_SP_POWERCTR_SP_SEL_2				0x00000ede
-
-#define REG_A5XX_SP_POWERCTR_SP_SEL_3				0x00000edf
-
-#define REG_A5XX_TPL1_ADDR_MODE_CNTL				0x00000f01
-
-#define REG_A5XX_TPL1_MODE_CNTL					0x00000f02
-
-#define REG_A5XX_TPL1_PERFCTR_TP_SEL_0				0x00000f10
-
-#define REG_A5XX_TPL1_PERFCTR_TP_SEL_1				0x00000f11
-
-#define REG_A5XX_TPL1_PERFCTR_TP_SEL_2				0x00000f12
-
-#define REG_A5XX_TPL1_PERFCTR_TP_SEL_3				0x00000f13
-
-#define REG_A5XX_TPL1_PERFCTR_TP_SEL_4				0x00000f14
-
-#define REG_A5XX_TPL1_PERFCTR_TP_SEL_5				0x00000f15
-
-#define REG_A5XX_TPL1_PERFCTR_TP_SEL_6				0x00000f16
-
-#define REG_A5XX_TPL1_PERFCTR_TP_SEL_7				0x00000f17
-
-#define REG_A5XX_TPL1_POWERCTR_TP_SEL_0				0x00000f18
-
-#define REG_A5XX_TPL1_POWERCTR_TP_SEL_1				0x00000f19
-
-#define REG_A5XX_TPL1_POWERCTR_TP_SEL_2				0x00000f1a
-
-#define REG_A5XX_TPL1_POWERCTR_TP_SEL_3				0x00000f1b
-
-#define REG_A5XX_VBIF_VERSION					0x00003000
-
-#define REG_A5XX_VBIF_CLKON					0x00003001
-
-#define REG_A5XX_VBIF_ABIT_SORT					0x00003028
-
-#define REG_A5XX_VBIF_ABIT_SORT_CONF				0x00003029
-
-#define REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB			0x00003049
-
-#define REG_A5XX_VBIF_GATE_OFF_WRREQ_EN				0x0000302a
-
-#define REG_A5XX_VBIF_IN_RD_LIM_CONF0				0x0000302c
-
-#define REG_A5XX_VBIF_IN_RD_LIM_CONF1				0x0000302d
-
-#define REG_A5XX_VBIF_XIN_HALT_CTRL0				0x00003080
-
-#define REG_A5XX_VBIF_XIN_HALT_CTRL1				0x00003081
-
-#define REG_A5XX_VBIF_TEST_BUS_OUT_CTRL				0x00003084
-
-#define REG_A5XX_VBIF_TEST_BUS1_CTRL0				0x00003085
-
-#define REG_A5XX_VBIF_TEST_BUS1_CTRL1				0x00003086
-
-#define REG_A5XX_VBIF_TEST_BUS2_CTRL0				0x00003087
-
-#define REG_A5XX_VBIF_TEST_BUS2_CTRL1				0x00003088
-
-#define REG_A5XX_VBIF_TEST_BUS_OUT				0x0000308c
-
-#define REG_A5XX_VBIF_PERF_CNT_EN0				0x000030c0
-
-#define REG_A5XX_VBIF_PERF_CNT_EN1				0x000030c1
-
-#define REG_A5XX_VBIF_PERF_CNT_EN2				0x000030c2
-
-#define REG_A5XX_VBIF_PERF_CNT_EN3				0x000030c3
-
-#define REG_A5XX_VBIF_PERF_CNT_CLR0				0x000030c8
-
-#define REG_A5XX_VBIF_PERF_CNT_CLR1				0x000030c9
-
-#define REG_A5XX_VBIF_PERF_CNT_CLR2				0x000030ca
-
-#define REG_A5XX_VBIF_PERF_CNT_CLR3				0x000030cb
-
-#define REG_A5XX_VBIF_PERF_CNT_SEL0				0x000030d0
-
-#define REG_A5XX_VBIF_PERF_CNT_SEL1				0x000030d1
-
-#define REG_A5XX_VBIF_PERF_CNT_SEL2				0x000030d2
-
-#define REG_A5XX_VBIF_PERF_CNT_SEL3				0x000030d3
-
-#define REG_A5XX_VBIF_PERF_CNT_LOW0				0x000030d8
-
-#define REG_A5XX_VBIF_PERF_CNT_LOW1				0x000030d9
-
-#define REG_A5XX_VBIF_PERF_CNT_LOW2				0x000030da
-
-#define REG_A5XX_VBIF_PERF_CNT_LOW3				0x000030db
-
-#define REG_A5XX_VBIF_PERF_CNT_HIGH0				0x000030e0
-
-#define REG_A5XX_VBIF_PERF_CNT_HIGH1				0x000030e1
-
-#define REG_A5XX_VBIF_PERF_CNT_HIGH2				0x000030e2
-
-#define REG_A5XX_VBIF_PERF_CNT_HIGH3				0x000030e3
-
-#define REG_A5XX_VBIF_PERF_PWR_CNT_EN0				0x00003100
-
-#define REG_A5XX_VBIF_PERF_PWR_CNT_EN1				0x00003101
-
-#define REG_A5XX_VBIF_PERF_PWR_CNT_EN2				0x00003102
-
-#define REG_A5XX_VBIF_PERF_PWR_CNT_LOW0				0x00003110
-
-#define REG_A5XX_VBIF_PERF_PWR_CNT_LOW1				0x00003111
-
-#define REG_A5XX_VBIF_PERF_PWR_CNT_LOW2				0x00003112
-
-#define REG_A5XX_VBIF_PERF_PWR_CNT_HIGH0			0x00003118
-
-#define REG_A5XX_VBIF_PERF_PWR_CNT_HIGH1			0x00003119
-
-#define REG_A5XX_VBIF_PERF_PWR_CNT_HIGH2			0x0000311a
-
-#define REG_A5XX_GPMU_INST_RAM_BASE				0x00008800
-
-#define REG_A5XX_GPMU_DATA_RAM_BASE				0x00009800
-
-#define REG_A5XX_SP_POWER_COUNTER_0_LO				0x0000a840
-
-#define REG_A5XX_SP_POWER_COUNTER_0_HI				0x0000a841
-
-#define REG_A5XX_SP_POWER_COUNTER_1_LO				0x0000a842
-
-#define REG_A5XX_SP_POWER_COUNTER_1_HI				0x0000a843
-
-#define REG_A5XX_SP_POWER_COUNTER_2_LO				0x0000a844
-
-#define REG_A5XX_SP_POWER_COUNTER_2_HI				0x0000a845
-
-#define REG_A5XX_SP_POWER_COUNTER_3_LO				0x0000a846
-
-#define REG_A5XX_SP_POWER_COUNTER_3_HI				0x0000a847
-
-#define REG_A5XX_TP_POWER_COUNTER_0_LO				0x0000a848
-
-#define REG_A5XX_TP_POWER_COUNTER_0_HI				0x0000a849
-
-#define REG_A5XX_TP_POWER_COUNTER_1_LO				0x0000a84a
-
-#define REG_A5XX_TP_POWER_COUNTER_1_HI				0x0000a84b
-
-#define REG_A5XX_TP_POWER_COUNTER_2_LO				0x0000a84c
-
-#define REG_A5XX_TP_POWER_COUNTER_2_HI				0x0000a84d
-
-#define REG_A5XX_TP_POWER_COUNTER_3_LO				0x0000a84e
-
-#define REG_A5XX_TP_POWER_COUNTER_3_HI				0x0000a84f
-
-#define REG_A5XX_RB_POWER_COUNTER_0_LO				0x0000a850
-
-#define REG_A5XX_RB_POWER_COUNTER_0_HI				0x0000a851
-
-#define REG_A5XX_RB_POWER_COUNTER_1_LO				0x0000a852
-
-#define REG_A5XX_RB_POWER_COUNTER_1_HI				0x0000a853
-
-#define REG_A5XX_RB_POWER_COUNTER_2_LO				0x0000a854
-
-#define REG_A5XX_RB_POWER_COUNTER_2_HI				0x0000a855
-
-#define REG_A5XX_RB_POWER_COUNTER_3_LO				0x0000a856
-
-#define REG_A5XX_RB_POWER_COUNTER_3_HI				0x0000a857
-
-#define REG_A5XX_CCU_POWER_COUNTER_0_LO				0x0000a858
-
-#define REG_A5XX_CCU_POWER_COUNTER_0_HI				0x0000a859
-
-#define REG_A5XX_CCU_POWER_COUNTER_1_LO				0x0000a85a
-
-#define REG_A5XX_CCU_POWER_COUNTER_1_HI				0x0000a85b
-
-#define REG_A5XX_UCHE_POWER_COUNTER_0_LO			0x0000a85c
-
-#define REG_A5XX_UCHE_POWER_COUNTER_0_HI			0x0000a85d
-
-#define REG_A5XX_UCHE_POWER_COUNTER_1_LO			0x0000a85e
-
-#define REG_A5XX_UCHE_POWER_COUNTER_1_HI			0x0000a85f
-
-#define REG_A5XX_UCHE_POWER_COUNTER_2_LO			0x0000a860
-
-#define REG_A5XX_UCHE_POWER_COUNTER_2_HI			0x0000a861
-
-#define REG_A5XX_UCHE_POWER_COUNTER_3_LO			0x0000a862
-
-#define REG_A5XX_UCHE_POWER_COUNTER_3_HI			0x0000a863
-
-#define REG_A5XX_CP_POWER_COUNTER_0_LO				0x0000a864
-
-#define REG_A5XX_CP_POWER_COUNTER_0_HI				0x0000a865
-
-#define REG_A5XX_CP_POWER_COUNTER_1_LO				0x0000a866
-
-#define REG_A5XX_CP_POWER_COUNTER_1_HI				0x0000a867
-
-#define REG_A5XX_CP_POWER_COUNTER_2_LO				0x0000a868
-
-#define REG_A5XX_CP_POWER_COUNTER_2_HI				0x0000a869
-
-#define REG_A5XX_CP_POWER_COUNTER_3_LO				0x0000a86a
-
-#define REG_A5XX_CP_POWER_COUNTER_3_HI				0x0000a86b
-
-#define REG_A5XX_GPMU_POWER_COUNTER_0_LO			0x0000a86c
-
-#define REG_A5XX_GPMU_POWER_COUNTER_0_HI			0x0000a86d
-
-#define REG_A5XX_GPMU_POWER_COUNTER_1_LO			0x0000a86e
-
-#define REG_A5XX_GPMU_POWER_COUNTER_1_HI			0x0000a86f
-
-#define REG_A5XX_GPMU_POWER_COUNTER_2_LO			0x0000a870
-
-#define REG_A5XX_GPMU_POWER_COUNTER_2_HI			0x0000a871
-
-#define REG_A5XX_GPMU_POWER_COUNTER_3_LO			0x0000a872
-
-#define REG_A5XX_GPMU_POWER_COUNTER_3_HI			0x0000a873
-
-#define REG_A5XX_GPMU_POWER_COUNTER_4_LO			0x0000a874
-
-#define REG_A5XX_GPMU_POWER_COUNTER_4_HI			0x0000a875
-
-#define REG_A5XX_GPMU_POWER_COUNTER_5_LO			0x0000a876
-
-#define REG_A5XX_GPMU_POWER_COUNTER_5_HI			0x0000a877
-
-#define REG_A5XX_GPMU_POWER_COUNTER_ENABLE			0x0000a878
-
-#define REG_A5XX_GPMU_ALWAYS_ON_COUNTER_LO			0x0000a879
-
-#define REG_A5XX_GPMU_ALWAYS_ON_COUNTER_HI			0x0000a87a
-
-#define REG_A5XX_GPMU_ALWAYS_ON_COUNTER_RESET			0x0000a87b
-
-#define REG_A5XX_GPMU_POWER_COUNTER_SELECT_0			0x0000a87c
-
-#define REG_A5XX_GPMU_POWER_COUNTER_SELECT_1			0x0000a87d
-
-#define REG_A5XX_GPMU_GPMU_SP_CLOCK_CONTROL			0x0000a880
-
-#define REG_A5XX_GPMU_SP_POWER_CNTL				0x0000a881
-
-#define REG_A5XX_GPMU_RBCCU_CLOCK_CNTL				0x0000a886
-
-#define REG_A5XX_GPMU_RBCCU_POWER_CNTL				0x0000a887
-
-#define REG_A5XX_GPMU_SP_PWR_CLK_STATUS				0x0000a88b
-#define A5XX_GPMU_SP_PWR_CLK_STATUS_PWR_ON			0x00100000
-
-#define REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS			0x0000a88d
-#define A5XX_GPMU_RBCCU_PWR_CLK_STATUS_PWR_ON			0x00100000
-
-#define REG_A5XX_GPMU_PWR_COL_STAGGER_DELAY			0x0000a891
-
-#define REG_A5XX_GPMU_PWR_COL_INTER_FRAME_CTRL			0x0000a892
-
-#define REG_A5XX_GPMU_PWR_COL_INTER_FRAME_HYST			0x0000a893
-
-#define REG_A5XX_GPMU_PWR_COL_BINNING_CTRL			0x0000a894
-
-#define REG_A5XX_GPMU_CLOCK_THROTTLE_CTRL			0x0000a8a3
-
-#define REG_A5XX_GPMU_THROTTLE_UNMASK_FORCE_CTRL		0x0000a8a8
-
-#define REG_A5XX_GPMU_WFI_CONFIG				0x0000a8c1
-
-#define REG_A5XX_GPMU_RBBM_INTR_INFO				0x0000a8d6
-
-#define REG_A5XX_GPMU_CM3_SYSRESET				0x0000a8d8
-
-#define REG_A5XX_GPMU_GENERAL_0					0x0000a8e0
-
-#define REG_A5XX_GPMU_GENERAL_1					0x0000a8e1
-
-#define REG_A5XX_GPMU_TEMP_SENSOR_ID				0x0000ac00
-
-#define REG_A5XX_GPMU_TEMP_SENSOR_CONFIG			0x0000ac01
-
-#define REG_A5XX_GPMU_TEMP_VAL					0x0000ac02
-
-#define REG_A5XX_GPMU_DELTA_TEMP_THRESHOLD			0x0000ac03
-
-#define REG_A5XX_GPMU_TEMP_THRESHOLD_INTR_STATUS		0x0000ac05
-
-#define REG_A5XX_GPMU_TEMP_THRESHOLD_INTR_EN_MASK		0x0000ac06
-
-#define REG_A5XX_GPMU_LEAKAGE_TEMP_COEFF_0_1			0x0000ac40
-
-#define REG_A5XX_GPMU_LEAKAGE_TEMP_COEFF_2_3			0x0000ac41
-
-#define REG_A5XX_GPMU_LEAKAGE_VTG_COEFF_0_1			0x0000ac42
-
-#define REG_A5XX_GPMU_LEAKAGE_VTG_COEFF_2_3			0x0000ac43
-
-#define REG_A5XX_GPMU_BASE_LEAKAGE				0x0000ac46
-
-#define REG_A5XX_GPMU_GPMU_VOLTAGE				0x0000ac60
-
-#define REG_A5XX_GPMU_GPMU_VOLTAGE_INTR_STATUS			0x0000ac61
-
-#define REG_A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK			0x0000ac62
-
-#define REG_A5XX_GPMU_GPMU_PWR_THRESHOLD			0x0000ac80
-
-#define REG_A5XX_GPMU_GPMU_LLM_GLM_SLEEP_CTRL			0x0000acc4
-
-#define REG_A5XX_GPMU_GPMU_LLM_GLM_SLEEP_STATUS			0x0000acc5
-
-#define REG_A5XX_GDPM_CONFIG1					0x0000b80c
-
-#define REG_A5XX_GDPM_CONFIG2					0x0000b80d
-
-#define REG_A5XX_GDPM_INT_EN					0x0000b80f
-
-#define REG_A5XX_GDPM_INT_MASK					0x0000b811
-
-#define REG_A5XX_GPMU_BEC_ENABLE				0x0000b9a0
-
-#define REG_A5XX_GPU_CS_SENSOR_GENERAL_STATUS			0x0000c41a
-
-#define REG_A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_0		0x0000c41d
-
-#define REG_A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_2		0x0000c41f
-
-#define REG_A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_4		0x0000c421
-
-#define REG_A5XX_GPU_CS_ENABLE_REG				0x0000c520
-
-#define REG_A5XX_GPU_CS_AMP_CALIBRATION_CONTROL1		0x0000c557
-
-#define REG_A5XX_GRAS_CL_CNTL					0x0000e000
-#define A5XX_GRAS_CL_CNTL_ZERO_GB_SCALE_Z			0x00000040
-
-#define REG_A5XX_GRAS_VS_CL_CNTL				0x0000e001
-#define A5XX_GRAS_VS_CL_CNTL_CLIP_MASK__MASK			0x000000ff
-#define A5XX_GRAS_VS_CL_CNTL_CLIP_MASK__SHIFT			0
-static inline uint32_t A5XX_GRAS_VS_CL_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_VS_CL_CNTL_CLIP_MASK__SHIFT) & A5XX_GRAS_VS_CL_CNTL_CLIP_MASK__MASK;
-}
-#define A5XX_GRAS_VS_CL_CNTL_CULL_MASK__MASK			0x0000ff00
-#define A5XX_GRAS_VS_CL_CNTL_CULL_MASK__SHIFT			8
-static inline uint32_t A5XX_GRAS_VS_CL_CNTL_CULL_MASK(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_VS_CL_CNTL_CULL_MASK__SHIFT) & A5XX_GRAS_VS_CL_CNTL_CULL_MASK__MASK;
-}
-
-#define REG_A5XX_UNKNOWN_E004					0x0000e004
-
-#define REG_A5XX_GRAS_CNTL					0x0000e005
-#define A5XX_GRAS_CNTL_IJ_PERSP_PIXEL				0x00000001
-#define A5XX_GRAS_CNTL_IJ_PERSP_CENTROID			0x00000002
-#define A5XX_GRAS_CNTL_IJ_PERSP_SAMPLE				0x00000004
-#define A5XX_GRAS_CNTL_IJ_LINEAR_PIXEL				0x00000008
-#define A5XX_GRAS_CNTL_IJ_LINEAR_CENTROID			0x00000010
-#define A5XX_GRAS_CNTL_IJ_LINEAR_SAMPLE				0x00000020
-#define A5XX_GRAS_CNTL_COORD_MASK__MASK				0x000003c0
-#define A5XX_GRAS_CNTL_COORD_MASK__SHIFT			6
-static inline uint32_t A5XX_GRAS_CNTL_COORD_MASK(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_CNTL_COORD_MASK__SHIFT) & A5XX_GRAS_CNTL_COORD_MASK__MASK;
-}
-
-#define REG_A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ			0x0000e006
-#define A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__MASK		0x000003ff
-#define A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__SHIFT		0
-static inline uint32_t A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__SHIFT) & A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__MASK;
-}
-#define A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__MASK		0x000ffc00
-#define A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__SHIFT		10
-static inline uint32_t A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__SHIFT) & A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__MASK;
-}
-
-#define REG_A5XX_GRAS_CL_VPORT_XOFFSET_0			0x0000e010
-#define A5XX_GRAS_CL_VPORT_XOFFSET_0__MASK			0xffffffff
-#define A5XX_GRAS_CL_VPORT_XOFFSET_0__SHIFT			0
-static inline uint32_t A5XX_GRAS_CL_VPORT_XOFFSET_0(float val)
-{
-	return ((fui(val)) << A5XX_GRAS_CL_VPORT_XOFFSET_0__SHIFT) & A5XX_GRAS_CL_VPORT_XOFFSET_0__MASK;
-}
-
-#define REG_A5XX_GRAS_CL_VPORT_XSCALE_0				0x0000e011
-#define A5XX_GRAS_CL_VPORT_XSCALE_0__MASK			0xffffffff
-#define A5XX_GRAS_CL_VPORT_XSCALE_0__SHIFT			0
-static inline uint32_t A5XX_GRAS_CL_VPORT_XSCALE_0(float val)
-{
-	return ((fui(val)) << A5XX_GRAS_CL_VPORT_XSCALE_0__SHIFT) & A5XX_GRAS_CL_VPORT_XSCALE_0__MASK;
-}
-
-#define REG_A5XX_GRAS_CL_VPORT_YOFFSET_0			0x0000e012
-#define A5XX_GRAS_CL_VPORT_YOFFSET_0__MASK			0xffffffff
-#define A5XX_GRAS_CL_VPORT_YOFFSET_0__SHIFT			0
-static inline uint32_t A5XX_GRAS_CL_VPORT_YOFFSET_0(float val)
-{
-	return ((fui(val)) << A5XX_GRAS_CL_VPORT_YOFFSET_0__SHIFT) & A5XX_GRAS_CL_VPORT_YOFFSET_0__MASK;
-}
-
-#define REG_A5XX_GRAS_CL_VPORT_YSCALE_0				0x0000e013
-#define A5XX_GRAS_CL_VPORT_YSCALE_0__MASK			0xffffffff
-#define A5XX_GRAS_CL_VPORT_YSCALE_0__SHIFT			0
-static inline uint32_t A5XX_GRAS_CL_VPORT_YSCALE_0(float val)
-{
-	return ((fui(val)) << A5XX_GRAS_CL_VPORT_YSCALE_0__SHIFT) & A5XX_GRAS_CL_VPORT_YSCALE_0__MASK;
-}
-
-#define REG_A5XX_GRAS_CL_VPORT_ZOFFSET_0			0x0000e014
-#define A5XX_GRAS_CL_VPORT_ZOFFSET_0__MASK			0xffffffff
-#define A5XX_GRAS_CL_VPORT_ZOFFSET_0__SHIFT			0
-static inline uint32_t A5XX_GRAS_CL_VPORT_ZOFFSET_0(float val)
-{
-	return ((fui(val)) << A5XX_GRAS_CL_VPORT_ZOFFSET_0__SHIFT) & A5XX_GRAS_CL_VPORT_ZOFFSET_0__MASK;
-}
-
-#define REG_A5XX_GRAS_CL_VPORT_ZSCALE_0				0x0000e015
-#define A5XX_GRAS_CL_VPORT_ZSCALE_0__MASK			0xffffffff
-#define A5XX_GRAS_CL_VPORT_ZSCALE_0__SHIFT			0
-static inline uint32_t A5XX_GRAS_CL_VPORT_ZSCALE_0(float val)
-{
-	return ((fui(val)) << A5XX_GRAS_CL_VPORT_ZSCALE_0__SHIFT) & A5XX_GRAS_CL_VPORT_ZSCALE_0__MASK;
-}
-
-#define REG_A5XX_GRAS_SU_CNTL					0x0000e090
-#define A5XX_GRAS_SU_CNTL_CULL_FRONT				0x00000001
-#define A5XX_GRAS_SU_CNTL_CULL_BACK				0x00000002
-#define A5XX_GRAS_SU_CNTL_FRONT_CW				0x00000004
-#define A5XX_GRAS_SU_CNTL_LINEHALFWIDTH__MASK			0x000007f8
-#define A5XX_GRAS_SU_CNTL_LINEHALFWIDTH__SHIFT			3
-static inline uint32_t A5XX_GRAS_SU_CNTL_LINEHALFWIDTH(float val)
-{
-	return ((((int32_t)(val * 4.0))) << A5XX_GRAS_SU_CNTL_LINEHALFWIDTH__SHIFT) & A5XX_GRAS_SU_CNTL_LINEHALFWIDTH__MASK;
-}
-#define A5XX_GRAS_SU_CNTL_POLY_OFFSET				0x00000800
-#define A5XX_GRAS_SU_CNTL_LINE_MODE__MASK			0x00002000
-#define A5XX_GRAS_SU_CNTL_LINE_MODE__SHIFT			13
-static inline uint32_t A5XX_GRAS_SU_CNTL_LINE_MODE(enum a5xx_line_mode val)
-{
-	return ((val) << A5XX_GRAS_SU_CNTL_LINE_MODE__SHIFT) & A5XX_GRAS_SU_CNTL_LINE_MODE__MASK;
-}
-
-#define REG_A5XX_GRAS_SU_POINT_MINMAX				0x0000e091
-#define A5XX_GRAS_SU_POINT_MINMAX_MIN__MASK			0x0000ffff
-#define A5XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT			0
-static inline uint32_t A5XX_GRAS_SU_POINT_MINMAX_MIN(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A5XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT) & A5XX_GRAS_SU_POINT_MINMAX_MIN__MASK;
-}
-#define A5XX_GRAS_SU_POINT_MINMAX_MAX__MASK			0xffff0000
-#define A5XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT			16
-static inline uint32_t A5XX_GRAS_SU_POINT_MINMAX_MAX(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A5XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT) & A5XX_GRAS_SU_POINT_MINMAX_MAX__MASK;
-}
-
-#define REG_A5XX_GRAS_SU_POINT_SIZE				0x0000e092
-#define A5XX_GRAS_SU_POINT_SIZE__MASK				0xffffffff
-#define A5XX_GRAS_SU_POINT_SIZE__SHIFT				0
-static inline uint32_t A5XX_GRAS_SU_POINT_SIZE(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A5XX_GRAS_SU_POINT_SIZE__SHIFT) & A5XX_GRAS_SU_POINT_SIZE__MASK;
-}
-
-#define REG_A5XX_GRAS_SU_LAYERED				0x0000e093
-
-#define REG_A5XX_GRAS_SU_DEPTH_PLANE_CNTL			0x0000e094
-#define A5XX_GRAS_SU_DEPTH_PLANE_CNTL_FRAG_WRITES_Z		0x00000001
-#define A5XX_GRAS_SU_DEPTH_PLANE_CNTL_UNK1			0x00000002
-
-#define REG_A5XX_GRAS_SU_POLY_OFFSET_SCALE			0x0000e095
-#define A5XX_GRAS_SU_POLY_OFFSET_SCALE__MASK			0xffffffff
-#define A5XX_GRAS_SU_POLY_OFFSET_SCALE__SHIFT			0
-static inline uint32_t A5XX_GRAS_SU_POLY_OFFSET_SCALE(float val)
-{
-	return ((fui(val)) << A5XX_GRAS_SU_POLY_OFFSET_SCALE__SHIFT) & A5XX_GRAS_SU_POLY_OFFSET_SCALE__MASK;
-}
-
-#define REG_A5XX_GRAS_SU_POLY_OFFSET_OFFSET			0x0000e096
-#define A5XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK			0xffffffff
-#define A5XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT			0
-static inline uint32_t A5XX_GRAS_SU_POLY_OFFSET_OFFSET(float val)
-{
-	return ((fui(val)) << A5XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT) & A5XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK;
-}
-
-#define REG_A5XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP		0x0000e097
-#define A5XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__MASK		0xffffffff
-#define A5XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__SHIFT		0
-static inline uint32_t A5XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP(float val)
-{
-	return ((fui(val)) << A5XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__SHIFT) & A5XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__MASK;
-}
-
-#define REG_A5XX_GRAS_SU_DEPTH_BUFFER_INFO			0x0000e098
-#define A5XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK	0x00000007
-#define A5XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT	0
-static inline uint32_t A5XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT(enum a5xx_depth_format val)
-{
-	return ((val) << A5XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT) & A5XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK;
-}
-
-#define REG_A5XX_GRAS_SU_CONSERVATIVE_RAS_CNTL			0x0000e099
-
-#define REG_A5XX_GRAS_SC_CNTL					0x0000e0a0
-#define A5XX_GRAS_SC_CNTL_BINNING_PASS				0x00000001
-#define A5XX_GRAS_SC_CNTL_SAMPLES_PASSED			0x00008000
-
-#define REG_A5XX_GRAS_SC_BIN_CNTL				0x0000e0a1
-
-#define REG_A5XX_GRAS_SC_RAS_MSAA_CNTL				0x0000e0a2
-#define A5XX_GRAS_SC_RAS_MSAA_CNTL_SAMPLES__MASK		0x00000003
-#define A5XX_GRAS_SC_RAS_MSAA_CNTL_SAMPLES__SHIFT		0
-static inline uint32_t A5XX_GRAS_SC_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A5XX_GRAS_SC_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A5XX_GRAS_SC_RAS_MSAA_CNTL_SAMPLES__MASK;
-}
-
-#define REG_A5XX_GRAS_SC_DEST_MSAA_CNTL				0x0000e0a3
-#define A5XX_GRAS_SC_DEST_MSAA_CNTL_SAMPLES__MASK		0x00000003
-#define A5XX_GRAS_SC_DEST_MSAA_CNTL_SAMPLES__SHIFT		0
-static inline uint32_t A5XX_GRAS_SC_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A5XX_GRAS_SC_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A5XX_GRAS_SC_DEST_MSAA_CNTL_SAMPLES__MASK;
-}
-#define A5XX_GRAS_SC_DEST_MSAA_CNTL_MSAA_DISABLE		0x00000004
-
-#define REG_A5XX_GRAS_SC_SCREEN_SCISSOR_CNTL			0x0000e0a4
-
-#define REG_A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0			0x0000e0aa
-#define A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_WINDOW_OFFSET_DISABLE	0x80000000
-#define A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__MASK		0x00007fff
-#define A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__SHIFT		0
-static inline uint32_t A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__SHIFT) & A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__MASK;
-}
-#define A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__MASK		0x7fff0000
-#define A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__SHIFT		16
-static inline uint32_t A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__SHIFT) & A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__MASK;
-}
-
-#define REG_A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0			0x0000e0ab
-#define A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_WINDOW_OFFSET_DISABLE	0x80000000
-#define A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__MASK		0x00007fff
-#define A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__SHIFT		0
-static inline uint32_t A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__SHIFT) & A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__MASK;
-}
-#define A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__MASK		0x7fff0000
-#define A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__SHIFT		16
-static inline uint32_t A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__SHIFT) & A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__MASK;
-}
-
-#define REG_A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0			0x0000e0ca
-#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_WINDOW_OFFSET_DISABLE	0x80000000
-#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__MASK		0x00007fff
-#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__SHIFT		0
-static inline uint32_t A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__SHIFT) & A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__MASK;
-}
-#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__MASK		0x7fff0000
-#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__SHIFT		16
-static inline uint32_t A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__SHIFT) & A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__MASK;
-}
-
-#define REG_A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0			0x0000e0cb
-#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_WINDOW_OFFSET_DISABLE	0x80000000
-#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__MASK		0x00007fff
-#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__SHIFT		0
-static inline uint32_t A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__SHIFT) & A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__MASK;
-}
-#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__MASK		0x7fff0000
-#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__SHIFT		16
-static inline uint32_t A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__SHIFT) & A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__MASK;
-}
-
-#define REG_A5XX_GRAS_SC_WINDOW_SCISSOR_TL			0x0000e0ea
-#define A5XX_GRAS_SC_WINDOW_SCISSOR_TL_WINDOW_OFFSET_DISABLE	0x80000000
-#define A5XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK			0x00007fff
-#define A5XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT			0
-static inline uint32_t A5XX_GRAS_SC_WINDOW_SCISSOR_TL_X(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT) & A5XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK;
-}
-#define A5XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK			0x7fff0000
-#define A5XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT			16
-static inline uint32_t A5XX_GRAS_SC_WINDOW_SCISSOR_TL_Y(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT) & A5XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK;
-}
-
-#define REG_A5XX_GRAS_SC_WINDOW_SCISSOR_BR			0x0000e0eb
-#define A5XX_GRAS_SC_WINDOW_SCISSOR_BR_WINDOW_OFFSET_DISABLE	0x80000000
-#define A5XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK			0x00007fff
-#define A5XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT			0
-static inline uint32_t A5XX_GRAS_SC_WINDOW_SCISSOR_BR_X(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT) & A5XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK;
-}
-#define A5XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK			0x7fff0000
-#define A5XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT			16
-static inline uint32_t A5XX_GRAS_SC_WINDOW_SCISSOR_BR_Y(uint32_t val)
-{
-	return ((val) << A5XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT) & A5XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK;
-}
-
-#define REG_A5XX_GRAS_LRZ_CNTL					0x0000e100
-#define A5XX_GRAS_LRZ_CNTL_ENABLE				0x00000001
-#define A5XX_GRAS_LRZ_CNTL_LRZ_WRITE				0x00000002
-#define A5XX_GRAS_LRZ_CNTL_GREATER				0x00000004
-
-#define REG_A5XX_GRAS_LRZ_BUFFER_BASE_LO			0x0000e101
-
-#define REG_A5XX_GRAS_LRZ_BUFFER_BASE_HI			0x0000e102
-
-#define REG_A5XX_GRAS_LRZ_BUFFER_PITCH				0x0000e103
-#define A5XX_GRAS_LRZ_BUFFER_PITCH__MASK			0xffffffff
-#define A5XX_GRAS_LRZ_BUFFER_PITCH__SHIFT			0
-static inline uint32_t A5XX_GRAS_LRZ_BUFFER_PITCH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A5XX_GRAS_LRZ_BUFFER_PITCH__SHIFT) & A5XX_GRAS_LRZ_BUFFER_PITCH__MASK;
-}
-
-#define REG_A5XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_LO		0x0000e104
-
-#define REG_A5XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_HI		0x0000e105
-
-#define REG_A5XX_RB_CNTL					0x0000e140
-#define A5XX_RB_CNTL_WIDTH__MASK				0x000000ff
-#define A5XX_RB_CNTL_WIDTH__SHIFT				0
-static inline uint32_t A5XX_RB_CNTL_WIDTH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A5XX_RB_CNTL_WIDTH__SHIFT) & A5XX_RB_CNTL_WIDTH__MASK;
-}
-#define A5XX_RB_CNTL_HEIGHT__MASK				0x0001fe00
-#define A5XX_RB_CNTL_HEIGHT__SHIFT				9
-static inline uint32_t A5XX_RB_CNTL_HEIGHT(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A5XX_RB_CNTL_HEIGHT__SHIFT) & A5XX_RB_CNTL_HEIGHT__MASK;
-}
-#define A5XX_RB_CNTL_BYPASS					0x00020000
-
-#define REG_A5XX_RB_RENDER_CNTL					0x0000e141
-#define A5XX_RB_RENDER_CNTL_BINNING_PASS			0x00000001
-#define A5XX_RB_RENDER_CNTL_SAMPLES_PASSED			0x00000040
-#define A5XX_RB_RENDER_CNTL_DISABLE_COLOR_PIPE			0x00000080
-#define A5XX_RB_RENDER_CNTL_FLAG_DEPTH				0x00004000
-#define A5XX_RB_RENDER_CNTL_FLAG_DEPTH2				0x00008000
-#define A5XX_RB_RENDER_CNTL_FLAG_MRTS__MASK			0x00ff0000
-#define A5XX_RB_RENDER_CNTL_FLAG_MRTS__SHIFT			16
-static inline uint32_t A5XX_RB_RENDER_CNTL_FLAG_MRTS(uint32_t val)
-{
-	return ((val) << A5XX_RB_RENDER_CNTL_FLAG_MRTS__SHIFT) & A5XX_RB_RENDER_CNTL_FLAG_MRTS__MASK;
-}
-#define A5XX_RB_RENDER_CNTL_FLAG_MRTS2__MASK			0xff000000
-#define A5XX_RB_RENDER_CNTL_FLAG_MRTS2__SHIFT			24
-static inline uint32_t A5XX_RB_RENDER_CNTL_FLAG_MRTS2(uint32_t val)
-{
-	return ((val) << A5XX_RB_RENDER_CNTL_FLAG_MRTS2__SHIFT) & A5XX_RB_RENDER_CNTL_FLAG_MRTS2__MASK;
-}
-
-#define REG_A5XX_RB_RAS_MSAA_CNTL				0x0000e142
-#define A5XX_RB_RAS_MSAA_CNTL_SAMPLES__MASK			0x00000003
-#define A5XX_RB_RAS_MSAA_CNTL_SAMPLES__SHIFT			0
-static inline uint32_t A5XX_RB_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A5XX_RB_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A5XX_RB_RAS_MSAA_CNTL_SAMPLES__MASK;
-}
-
-#define REG_A5XX_RB_DEST_MSAA_CNTL				0x0000e143
-#define A5XX_RB_DEST_MSAA_CNTL_SAMPLES__MASK			0x00000003
-#define A5XX_RB_DEST_MSAA_CNTL_SAMPLES__SHIFT			0
-static inline uint32_t A5XX_RB_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A5XX_RB_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A5XX_RB_DEST_MSAA_CNTL_SAMPLES__MASK;
-}
-#define A5XX_RB_DEST_MSAA_CNTL_MSAA_DISABLE			0x00000004
-
-#define REG_A5XX_RB_RENDER_CONTROL0				0x0000e144
-#define A5XX_RB_RENDER_CONTROL0_IJ_PERSP_PIXEL			0x00000001
-#define A5XX_RB_RENDER_CONTROL0_IJ_PERSP_CENTROID		0x00000002
-#define A5XX_RB_RENDER_CONTROL0_IJ_PERSP_SAMPLE			0x00000004
-#define A5XX_RB_RENDER_CONTROL0_IJ_LINEAR_PIXEL			0x00000008
-#define A5XX_RB_RENDER_CONTROL0_IJ_LINEAR_CENTROID		0x00000010
-#define A5XX_RB_RENDER_CONTROL0_IJ_LINEAR_SAMPLE		0x00000020
-#define A5XX_RB_RENDER_CONTROL0_COORD_MASK__MASK		0x000003c0
-#define A5XX_RB_RENDER_CONTROL0_COORD_MASK__SHIFT		6
-static inline uint32_t A5XX_RB_RENDER_CONTROL0_COORD_MASK(uint32_t val)
-{
-	return ((val) << A5XX_RB_RENDER_CONTROL0_COORD_MASK__SHIFT) & A5XX_RB_RENDER_CONTROL0_COORD_MASK__MASK;
-}
-
-#define REG_A5XX_RB_RENDER_CONTROL1				0x0000e145
-#define A5XX_RB_RENDER_CONTROL1_SAMPLEMASK			0x00000001
-#define A5XX_RB_RENDER_CONTROL1_FACENESS			0x00000002
-#define A5XX_RB_RENDER_CONTROL1_SAMPLEID			0x00000004
-
-#define REG_A5XX_RB_FS_OUTPUT_CNTL				0x0000e146
-#define A5XX_RB_FS_OUTPUT_CNTL_MRT__MASK			0x0000000f
-#define A5XX_RB_FS_OUTPUT_CNTL_MRT__SHIFT			0
-static inline uint32_t A5XX_RB_FS_OUTPUT_CNTL_MRT(uint32_t val)
-{
-	return ((val) << A5XX_RB_FS_OUTPUT_CNTL_MRT__SHIFT) & A5XX_RB_FS_OUTPUT_CNTL_MRT__MASK;
-}
-#define A5XX_RB_FS_OUTPUT_CNTL_FRAG_WRITES_Z			0x00000020
-
-#define REG_A5XX_RB_RENDER_COMPONENTS				0x0000e147
-#define A5XX_RB_RENDER_COMPONENTS_RT0__MASK			0x0000000f
-#define A5XX_RB_RENDER_COMPONENTS_RT0__SHIFT			0
-static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT0(uint32_t val)
-{
-	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT0__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT0__MASK;
-}
-#define A5XX_RB_RENDER_COMPONENTS_RT1__MASK			0x000000f0
-#define A5XX_RB_RENDER_COMPONENTS_RT1__SHIFT			4
-static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT1(uint32_t val)
-{
-	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT1__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT1__MASK;
-}
-#define A5XX_RB_RENDER_COMPONENTS_RT2__MASK			0x00000f00
-#define A5XX_RB_RENDER_COMPONENTS_RT2__SHIFT			8
-static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT2(uint32_t val)
-{
-	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT2__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT2__MASK;
-}
-#define A5XX_RB_RENDER_COMPONENTS_RT3__MASK			0x0000f000
-#define A5XX_RB_RENDER_COMPONENTS_RT3__SHIFT			12
-static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT3(uint32_t val)
-{
-	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT3__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT3__MASK;
-}
-#define A5XX_RB_RENDER_COMPONENTS_RT4__MASK			0x000f0000
-#define A5XX_RB_RENDER_COMPONENTS_RT4__SHIFT			16
-static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT4(uint32_t val)
-{
-	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT4__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT4__MASK;
-}
-#define A5XX_RB_RENDER_COMPONENTS_RT5__MASK			0x00f00000
-#define A5XX_RB_RENDER_COMPONENTS_RT5__SHIFT			20
-static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT5(uint32_t val)
-{
-	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT5__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT5__MASK;
-}
-#define A5XX_RB_RENDER_COMPONENTS_RT6__MASK			0x0f000000
-#define A5XX_RB_RENDER_COMPONENTS_RT6__SHIFT			24
-static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT6(uint32_t val)
-{
-	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT6__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT6__MASK;
-}
-#define A5XX_RB_RENDER_COMPONENTS_RT7__MASK			0xf0000000
-#define A5XX_RB_RENDER_COMPONENTS_RT7__SHIFT			28
-static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT7(uint32_t val)
-{
-	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT7__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT7__MASK;
-}
-
-#define REG_A5XX_RB_MRT(i0) (0x0000e150 + 0x7*(i0))
-
-static inline uint32_t REG_A5XX_RB_MRT_CONTROL(uint32_t i0) { return 0x0000e150 + 0x7*i0; }
-#define A5XX_RB_MRT_CONTROL_BLEND				0x00000001
-#define A5XX_RB_MRT_CONTROL_BLEND2				0x00000002
-#define A5XX_RB_MRT_CONTROL_ROP_ENABLE				0x00000004
-#define A5XX_RB_MRT_CONTROL_ROP_CODE__MASK			0x00000078
-#define A5XX_RB_MRT_CONTROL_ROP_CODE__SHIFT			3
-static inline uint32_t A5XX_RB_MRT_CONTROL_ROP_CODE(enum a3xx_rop_code val)
-{
-	return ((val) << A5XX_RB_MRT_CONTROL_ROP_CODE__SHIFT) & A5XX_RB_MRT_CONTROL_ROP_CODE__MASK;
-}
-#define A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK		0x00000780
-#define A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT		7
-static inline uint32_t A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE(uint32_t val)
-{
-	return ((val) << A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT) & A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK;
-}
-
-static inline uint32_t REG_A5XX_RB_MRT_BLEND_CONTROL(uint32_t i0) { return 0x0000e151 + 0x7*i0; }
-#define A5XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK		0x0000001f
-#define A5XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT		0
-static inline uint32_t A5XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A5XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT) & A5XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK;
-}
-#define A5XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK	0x000000e0
-#define A5XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT	5
-static inline uint32_t A5XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
-{
-	return ((val) << A5XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT) & A5XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK;
-}
-#define A5XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK		0x00001f00
-#define A5XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT	8
-static inline uint32_t A5XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A5XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT) & A5XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK;
-}
-#define A5XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK	0x001f0000
-#define A5XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT	16
-static inline uint32_t A5XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A5XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT) & A5XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK;
-}
-#define A5XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK	0x00e00000
-#define A5XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT	21
-static inline uint32_t A5XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
-{
-	return ((val) << A5XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT) & A5XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK;
-}
-#define A5XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK	0x1f000000
-#define A5XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT	24
-static inline uint32_t A5XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A5XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT) & A5XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK;
-}
-
-static inline uint32_t REG_A5XX_RB_MRT_BUF_INFO(uint32_t i0) { return 0x0000e152 + 0x7*i0; }
-#define A5XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK			0x000000ff
-#define A5XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT		0
-static inline uint32_t A5XX_RB_MRT_BUF_INFO_COLOR_FORMAT(enum a5xx_color_fmt val)
-{
-	return ((val) << A5XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT) & A5XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK;
-}
-#define A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK		0x00000300
-#define A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT		8
-static inline uint32_t A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE(enum a5xx_tile_mode val)
-{
-	return ((val) << A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT) & A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK;
-}
-#define A5XX_RB_MRT_BUF_INFO_DITHER_MODE__MASK			0x00001800
-#define A5XX_RB_MRT_BUF_INFO_DITHER_MODE__SHIFT			11
-static inline uint32_t A5XX_RB_MRT_BUF_INFO_DITHER_MODE(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A5XX_RB_MRT_BUF_INFO_DITHER_MODE__SHIFT) & A5XX_RB_MRT_BUF_INFO_DITHER_MODE__MASK;
-}
-#define A5XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK			0x00006000
-#define A5XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT			13
-static inline uint32_t A5XX_RB_MRT_BUF_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A5XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT) & A5XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK;
-}
-#define A5XX_RB_MRT_BUF_INFO_COLOR_SRGB				0x00008000
-
-static inline uint32_t REG_A5XX_RB_MRT_PITCH(uint32_t i0) { return 0x0000e153 + 0x7*i0; }
-#define A5XX_RB_MRT_PITCH__MASK					0xffffffff
-#define A5XX_RB_MRT_PITCH__SHIFT				0
-static inline uint32_t A5XX_RB_MRT_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_MRT_PITCH__SHIFT) & A5XX_RB_MRT_PITCH__MASK;
-}
-
-static inline uint32_t REG_A5XX_RB_MRT_ARRAY_PITCH(uint32_t i0) { return 0x0000e154 + 0x7*i0; }
-#define A5XX_RB_MRT_ARRAY_PITCH__MASK				0xffffffff
-#define A5XX_RB_MRT_ARRAY_PITCH__SHIFT				0
-static inline uint32_t A5XX_RB_MRT_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_MRT_ARRAY_PITCH__SHIFT) & A5XX_RB_MRT_ARRAY_PITCH__MASK;
-}
-
-static inline uint32_t REG_A5XX_RB_MRT_BASE_LO(uint32_t i0) { return 0x0000e155 + 0x7*i0; }
-
-static inline uint32_t REG_A5XX_RB_MRT_BASE_HI(uint32_t i0) { return 0x0000e156 + 0x7*i0; }
-
-#define REG_A5XX_RB_BLEND_RED					0x0000e1a0
-#define A5XX_RB_BLEND_RED_UINT__MASK				0x000000ff
-#define A5XX_RB_BLEND_RED_UINT__SHIFT				0
-static inline uint32_t A5XX_RB_BLEND_RED_UINT(uint32_t val)
-{
-	return ((val) << A5XX_RB_BLEND_RED_UINT__SHIFT) & A5XX_RB_BLEND_RED_UINT__MASK;
-}
-#define A5XX_RB_BLEND_RED_SINT__MASK				0x0000ff00
-#define A5XX_RB_BLEND_RED_SINT__SHIFT				8
-static inline uint32_t A5XX_RB_BLEND_RED_SINT(uint32_t val)
-{
-	return ((val) << A5XX_RB_BLEND_RED_SINT__SHIFT) & A5XX_RB_BLEND_RED_SINT__MASK;
-}
-#define A5XX_RB_BLEND_RED_FLOAT__MASK				0xffff0000
-#define A5XX_RB_BLEND_RED_FLOAT__SHIFT				16
-static inline uint32_t A5XX_RB_BLEND_RED_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A5XX_RB_BLEND_RED_FLOAT__SHIFT) & A5XX_RB_BLEND_RED_FLOAT__MASK;
-}
-
-#define REG_A5XX_RB_BLEND_RED_F32				0x0000e1a1
-#define A5XX_RB_BLEND_RED_F32__MASK				0xffffffff
-#define A5XX_RB_BLEND_RED_F32__SHIFT				0
-static inline uint32_t A5XX_RB_BLEND_RED_F32(float val)
-{
-	return ((fui(val)) << A5XX_RB_BLEND_RED_F32__SHIFT) & A5XX_RB_BLEND_RED_F32__MASK;
-}
-
-#define REG_A5XX_RB_BLEND_GREEN					0x0000e1a2
-#define A5XX_RB_BLEND_GREEN_UINT__MASK				0x000000ff
-#define A5XX_RB_BLEND_GREEN_UINT__SHIFT				0
-static inline uint32_t A5XX_RB_BLEND_GREEN_UINT(uint32_t val)
-{
-	return ((val) << A5XX_RB_BLEND_GREEN_UINT__SHIFT) & A5XX_RB_BLEND_GREEN_UINT__MASK;
-}
-#define A5XX_RB_BLEND_GREEN_SINT__MASK				0x0000ff00
-#define A5XX_RB_BLEND_GREEN_SINT__SHIFT				8
-static inline uint32_t A5XX_RB_BLEND_GREEN_SINT(uint32_t val)
-{
-	return ((val) << A5XX_RB_BLEND_GREEN_SINT__SHIFT) & A5XX_RB_BLEND_GREEN_SINT__MASK;
-}
-#define A5XX_RB_BLEND_GREEN_FLOAT__MASK				0xffff0000
-#define A5XX_RB_BLEND_GREEN_FLOAT__SHIFT			16
-static inline uint32_t A5XX_RB_BLEND_GREEN_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A5XX_RB_BLEND_GREEN_FLOAT__SHIFT) & A5XX_RB_BLEND_GREEN_FLOAT__MASK;
-}
-
-#define REG_A5XX_RB_BLEND_GREEN_F32				0x0000e1a3
-#define A5XX_RB_BLEND_GREEN_F32__MASK				0xffffffff
-#define A5XX_RB_BLEND_GREEN_F32__SHIFT				0
-static inline uint32_t A5XX_RB_BLEND_GREEN_F32(float val)
-{
-	return ((fui(val)) << A5XX_RB_BLEND_GREEN_F32__SHIFT) & A5XX_RB_BLEND_GREEN_F32__MASK;
-}
-
-#define REG_A5XX_RB_BLEND_BLUE					0x0000e1a4
-#define A5XX_RB_BLEND_BLUE_UINT__MASK				0x000000ff
-#define A5XX_RB_BLEND_BLUE_UINT__SHIFT				0
-static inline uint32_t A5XX_RB_BLEND_BLUE_UINT(uint32_t val)
-{
-	return ((val) << A5XX_RB_BLEND_BLUE_UINT__SHIFT) & A5XX_RB_BLEND_BLUE_UINT__MASK;
-}
-#define A5XX_RB_BLEND_BLUE_SINT__MASK				0x0000ff00
-#define A5XX_RB_BLEND_BLUE_SINT__SHIFT				8
-static inline uint32_t A5XX_RB_BLEND_BLUE_SINT(uint32_t val)
-{
-	return ((val) << A5XX_RB_BLEND_BLUE_SINT__SHIFT) & A5XX_RB_BLEND_BLUE_SINT__MASK;
-}
-#define A5XX_RB_BLEND_BLUE_FLOAT__MASK				0xffff0000
-#define A5XX_RB_BLEND_BLUE_FLOAT__SHIFT				16
-static inline uint32_t A5XX_RB_BLEND_BLUE_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A5XX_RB_BLEND_BLUE_FLOAT__SHIFT) & A5XX_RB_BLEND_BLUE_FLOAT__MASK;
-}
-
-#define REG_A5XX_RB_BLEND_BLUE_F32				0x0000e1a5
-#define A5XX_RB_BLEND_BLUE_F32__MASK				0xffffffff
-#define A5XX_RB_BLEND_BLUE_F32__SHIFT				0
-static inline uint32_t A5XX_RB_BLEND_BLUE_F32(float val)
-{
-	return ((fui(val)) << A5XX_RB_BLEND_BLUE_F32__SHIFT) & A5XX_RB_BLEND_BLUE_F32__MASK;
-}
-
-#define REG_A5XX_RB_BLEND_ALPHA					0x0000e1a6
-#define A5XX_RB_BLEND_ALPHA_UINT__MASK				0x000000ff
-#define A5XX_RB_BLEND_ALPHA_UINT__SHIFT				0
-static inline uint32_t A5XX_RB_BLEND_ALPHA_UINT(uint32_t val)
-{
-	return ((val) << A5XX_RB_BLEND_ALPHA_UINT__SHIFT) & A5XX_RB_BLEND_ALPHA_UINT__MASK;
-}
-#define A5XX_RB_BLEND_ALPHA_SINT__MASK				0x0000ff00
-#define A5XX_RB_BLEND_ALPHA_SINT__SHIFT				8
-static inline uint32_t A5XX_RB_BLEND_ALPHA_SINT(uint32_t val)
-{
-	return ((val) << A5XX_RB_BLEND_ALPHA_SINT__SHIFT) & A5XX_RB_BLEND_ALPHA_SINT__MASK;
-}
-#define A5XX_RB_BLEND_ALPHA_FLOAT__MASK				0xffff0000
-#define A5XX_RB_BLEND_ALPHA_FLOAT__SHIFT			16
-static inline uint32_t A5XX_RB_BLEND_ALPHA_FLOAT(float val)
-{
-	return ((_mesa_float_to_half(val)) << A5XX_RB_BLEND_ALPHA_FLOAT__SHIFT) & A5XX_RB_BLEND_ALPHA_FLOAT__MASK;
-}
-
-#define REG_A5XX_RB_BLEND_ALPHA_F32				0x0000e1a7
-#define A5XX_RB_BLEND_ALPHA_F32__MASK				0xffffffff
-#define A5XX_RB_BLEND_ALPHA_F32__SHIFT				0
-static inline uint32_t A5XX_RB_BLEND_ALPHA_F32(float val)
-{
-	return ((fui(val)) << A5XX_RB_BLEND_ALPHA_F32__SHIFT) & A5XX_RB_BLEND_ALPHA_F32__MASK;
-}
-
-#define REG_A5XX_RB_ALPHA_CONTROL				0x0000e1a8
-#define A5XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK			0x000000ff
-#define A5XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT			0
-static inline uint32_t A5XX_RB_ALPHA_CONTROL_ALPHA_REF(uint32_t val)
-{
-	return ((val) << A5XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT) & A5XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK;
-}
-#define A5XX_RB_ALPHA_CONTROL_ALPHA_TEST			0x00000100
-#define A5XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__MASK		0x00000e00
-#define A5XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__SHIFT		9
-static inline uint32_t A5XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A5XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__SHIFT) & A5XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__MASK;
-}
-
-#define REG_A5XX_RB_BLEND_CNTL					0x0000e1a9
-#define A5XX_RB_BLEND_CNTL_ENABLE_BLEND__MASK			0x000000ff
-#define A5XX_RB_BLEND_CNTL_ENABLE_BLEND__SHIFT			0
-static inline uint32_t A5XX_RB_BLEND_CNTL_ENABLE_BLEND(uint32_t val)
-{
-	return ((val) << A5XX_RB_BLEND_CNTL_ENABLE_BLEND__SHIFT) & A5XX_RB_BLEND_CNTL_ENABLE_BLEND__MASK;
-}
-#define A5XX_RB_BLEND_CNTL_INDEPENDENT_BLEND			0x00000100
-#define A5XX_RB_BLEND_CNTL_ALPHA_TO_COVERAGE			0x00000400
-#define A5XX_RB_BLEND_CNTL_SAMPLE_MASK__MASK			0xffff0000
-#define A5XX_RB_BLEND_CNTL_SAMPLE_MASK__SHIFT			16
-static inline uint32_t A5XX_RB_BLEND_CNTL_SAMPLE_MASK(uint32_t val)
-{
-	return ((val) << A5XX_RB_BLEND_CNTL_SAMPLE_MASK__SHIFT) & A5XX_RB_BLEND_CNTL_SAMPLE_MASK__MASK;
-}
-
-#define REG_A5XX_RB_DEPTH_PLANE_CNTL				0x0000e1b0
-#define A5XX_RB_DEPTH_PLANE_CNTL_FRAG_WRITES_Z			0x00000001
-#define A5XX_RB_DEPTH_PLANE_CNTL_UNK1				0x00000002
-
-#define REG_A5XX_RB_DEPTH_CNTL					0x0000e1b1
-#define A5XX_RB_DEPTH_CNTL_Z_TEST_ENABLE			0x00000001
-#define A5XX_RB_DEPTH_CNTL_Z_WRITE_ENABLE			0x00000002
-#define A5XX_RB_DEPTH_CNTL_ZFUNC__MASK				0x0000001c
-#define A5XX_RB_DEPTH_CNTL_ZFUNC__SHIFT				2
-static inline uint32_t A5XX_RB_DEPTH_CNTL_ZFUNC(enum adreno_compare_func val)
-{
-	return ((val) << A5XX_RB_DEPTH_CNTL_ZFUNC__SHIFT) & A5XX_RB_DEPTH_CNTL_ZFUNC__MASK;
-}
-#define A5XX_RB_DEPTH_CNTL_Z_READ_ENABLE			0x00000040
-
-#define REG_A5XX_RB_DEPTH_BUFFER_INFO				0x0000e1b2
-#define A5XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK		0x00000007
-#define A5XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT		0
-static inline uint32_t A5XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT(enum a5xx_depth_format val)
-{
-	return ((val) << A5XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT) & A5XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK;
-}
-
-#define REG_A5XX_RB_DEPTH_BUFFER_BASE_LO			0x0000e1b3
-
-#define REG_A5XX_RB_DEPTH_BUFFER_BASE_HI			0x0000e1b4
-
-#define REG_A5XX_RB_DEPTH_BUFFER_PITCH				0x0000e1b5
-#define A5XX_RB_DEPTH_BUFFER_PITCH__MASK			0xffffffff
-#define A5XX_RB_DEPTH_BUFFER_PITCH__SHIFT			0
-static inline uint32_t A5XX_RB_DEPTH_BUFFER_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_DEPTH_BUFFER_PITCH__SHIFT) & A5XX_RB_DEPTH_BUFFER_PITCH__MASK;
-}
-
-#define REG_A5XX_RB_DEPTH_BUFFER_ARRAY_PITCH			0x0000e1b6
-#define A5XX_RB_DEPTH_BUFFER_ARRAY_PITCH__MASK			0xffffffff
-#define A5XX_RB_DEPTH_BUFFER_ARRAY_PITCH__SHIFT			0
-static inline uint32_t A5XX_RB_DEPTH_BUFFER_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_DEPTH_BUFFER_ARRAY_PITCH__SHIFT) & A5XX_RB_DEPTH_BUFFER_ARRAY_PITCH__MASK;
-}
-
-#define REG_A5XX_RB_STENCIL_CONTROL				0x0000e1c0
-#define A5XX_RB_STENCIL_CONTROL_STENCIL_ENABLE			0x00000001
-#define A5XX_RB_STENCIL_CONTROL_STENCIL_ENABLE_BF		0x00000002
-#define A5XX_RB_STENCIL_CONTROL_STENCIL_READ			0x00000004
-#define A5XX_RB_STENCIL_CONTROL_FUNC__MASK			0x00000700
-#define A5XX_RB_STENCIL_CONTROL_FUNC__SHIFT			8
-static inline uint32_t A5XX_RB_STENCIL_CONTROL_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A5XX_RB_STENCIL_CONTROL_FUNC__SHIFT) & A5XX_RB_STENCIL_CONTROL_FUNC__MASK;
-}
-#define A5XX_RB_STENCIL_CONTROL_FAIL__MASK			0x00003800
-#define A5XX_RB_STENCIL_CONTROL_FAIL__SHIFT			11
-static inline uint32_t A5XX_RB_STENCIL_CONTROL_FAIL(enum adreno_stencil_op val)
-{
-	return ((val) << A5XX_RB_STENCIL_CONTROL_FAIL__SHIFT) & A5XX_RB_STENCIL_CONTROL_FAIL__MASK;
-}
-#define A5XX_RB_STENCIL_CONTROL_ZPASS__MASK			0x0001c000
-#define A5XX_RB_STENCIL_CONTROL_ZPASS__SHIFT			14
-static inline uint32_t A5XX_RB_STENCIL_CONTROL_ZPASS(enum adreno_stencil_op val)
-{
-	return ((val) << A5XX_RB_STENCIL_CONTROL_ZPASS__SHIFT) & A5XX_RB_STENCIL_CONTROL_ZPASS__MASK;
-}
-#define A5XX_RB_STENCIL_CONTROL_ZFAIL__MASK			0x000e0000
-#define A5XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT			17
-static inline uint32_t A5XX_RB_STENCIL_CONTROL_ZFAIL(enum adreno_stencil_op val)
-{
-	return ((val) << A5XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT) & A5XX_RB_STENCIL_CONTROL_ZFAIL__MASK;
-}
-#define A5XX_RB_STENCIL_CONTROL_FUNC_BF__MASK			0x00700000
-#define A5XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT			20
-static inline uint32_t A5XX_RB_STENCIL_CONTROL_FUNC_BF(enum adreno_compare_func val)
-{
-	return ((val) << A5XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT) & A5XX_RB_STENCIL_CONTROL_FUNC_BF__MASK;
-}
-#define A5XX_RB_STENCIL_CONTROL_FAIL_BF__MASK			0x03800000
-#define A5XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT			23
-static inline uint32_t A5XX_RB_STENCIL_CONTROL_FAIL_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A5XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT) & A5XX_RB_STENCIL_CONTROL_FAIL_BF__MASK;
-}
-#define A5XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK			0x1c000000
-#define A5XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT			26
-static inline uint32_t A5XX_RB_STENCIL_CONTROL_ZPASS_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A5XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT) & A5XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK;
-}
-#define A5XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK			0xe0000000
-#define A5XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT			29
-static inline uint32_t A5XX_RB_STENCIL_CONTROL_ZFAIL_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A5XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT) & A5XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK;
-}
-
-#define REG_A5XX_RB_STENCIL_INFO				0x0000e1c1
-#define A5XX_RB_STENCIL_INFO_SEPARATE_STENCIL			0x00000001
-
-#define REG_A5XX_RB_STENCIL_BASE_LO				0x0000e1c2
-
-#define REG_A5XX_RB_STENCIL_BASE_HI				0x0000e1c3
-
-#define REG_A5XX_RB_STENCIL_PITCH				0x0000e1c4
-#define A5XX_RB_STENCIL_PITCH__MASK				0xffffffff
-#define A5XX_RB_STENCIL_PITCH__SHIFT				0
-static inline uint32_t A5XX_RB_STENCIL_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_STENCIL_PITCH__SHIFT) & A5XX_RB_STENCIL_PITCH__MASK;
-}
-
-#define REG_A5XX_RB_STENCIL_ARRAY_PITCH				0x0000e1c5
-#define A5XX_RB_STENCIL_ARRAY_PITCH__MASK			0xffffffff
-#define A5XX_RB_STENCIL_ARRAY_PITCH__SHIFT			0
-static inline uint32_t A5XX_RB_STENCIL_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_STENCIL_ARRAY_PITCH__SHIFT) & A5XX_RB_STENCIL_ARRAY_PITCH__MASK;
-}
-
-#define REG_A5XX_RB_STENCILREFMASK				0x0000e1c6
-#define A5XX_RB_STENCILREFMASK_STENCILREF__MASK			0x000000ff
-#define A5XX_RB_STENCILREFMASK_STENCILREF__SHIFT		0
-static inline uint32_t A5XX_RB_STENCILREFMASK_STENCILREF(uint32_t val)
-{
-	return ((val) << A5XX_RB_STENCILREFMASK_STENCILREF__SHIFT) & A5XX_RB_STENCILREFMASK_STENCILREF__MASK;
-}
-#define A5XX_RB_STENCILREFMASK_STENCILMASK__MASK		0x0000ff00
-#define A5XX_RB_STENCILREFMASK_STENCILMASK__SHIFT		8
-static inline uint32_t A5XX_RB_STENCILREFMASK_STENCILMASK(uint32_t val)
-{
-	return ((val) << A5XX_RB_STENCILREFMASK_STENCILMASK__SHIFT) & A5XX_RB_STENCILREFMASK_STENCILMASK__MASK;
-}
-#define A5XX_RB_STENCILREFMASK_STENCILWRITEMASK__MASK		0x00ff0000
-#define A5XX_RB_STENCILREFMASK_STENCILWRITEMASK__SHIFT		16
-static inline uint32_t A5XX_RB_STENCILREFMASK_STENCILWRITEMASK(uint32_t val)
-{
-	return ((val) << A5XX_RB_STENCILREFMASK_STENCILWRITEMASK__SHIFT) & A5XX_RB_STENCILREFMASK_STENCILWRITEMASK__MASK;
-}
-
-#define REG_A5XX_RB_STENCILREFMASK_BF				0x0000e1c7
-#define A5XX_RB_STENCILREFMASK_BF_STENCILREF__MASK		0x000000ff
-#define A5XX_RB_STENCILREFMASK_BF_STENCILREF__SHIFT		0
-static inline uint32_t A5XX_RB_STENCILREFMASK_BF_STENCILREF(uint32_t val)
-{
-	return ((val) << A5XX_RB_STENCILREFMASK_BF_STENCILREF__SHIFT) & A5XX_RB_STENCILREFMASK_BF_STENCILREF__MASK;
-}
-#define A5XX_RB_STENCILREFMASK_BF_STENCILMASK__MASK		0x0000ff00
-#define A5XX_RB_STENCILREFMASK_BF_STENCILMASK__SHIFT		8
-static inline uint32_t A5XX_RB_STENCILREFMASK_BF_STENCILMASK(uint32_t val)
-{
-	return ((val) << A5XX_RB_STENCILREFMASK_BF_STENCILMASK__SHIFT) & A5XX_RB_STENCILREFMASK_BF_STENCILMASK__MASK;
-}
-#define A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__MASK	0x00ff0000
-#define A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__SHIFT	16
-static inline uint32_t A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK(uint32_t val)
-{
-	return ((val) << A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__SHIFT) & A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__MASK;
-}
-
-#define REG_A5XX_RB_WINDOW_OFFSET				0x0000e1d0
-#define A5XX_RB_WINDOW_OFFSET_WINDOW_OFFSET_DISABLE		0x80000000
-#define A5XX_RB_WINDOW_OFFSET_X__MASK				0x00007fff
-#define A5XX_RB_WINDOW_OFFSET_X__SHIFT				0
-static inline uint32_t A5XX_RB_WINDOW_OFFSET_X(uint32_t val)
-{
-	return ((val) << A5XX_RB_WINDOW_OFFSET_X__SHIFT) & A5XX_RB_WINDOW_OFFSET_X__MASK;
-}
-#define A5XX_RB_WINDOW_OFFSET_Y__MASK				0x7fff0000
-#define A5XX_RB_WINDOW_OFFSET_Y__SHIFT				16
-static inline uint32_t A5XX_RB_WINDOW_OFFSET_Y(uint32_t val)
-{
-	return ((val) << A5XX_RB_WINDOW_OFFSET_Y__SHIFT) & A5XX_RB_WINDOW_OFFSET_Y__MASK;
-}
-
-#define REG_A5XX_RB_SAMPLE_COUNT_CONTROL			0x0000e1d1
-#define A5XX_RB_SAMPLE_COUNT_CONTROL_COPY			0x00000002
-
-#define REG_A5XX_RB_BLIT_CNTL					0x0000e210
-#define A5XX_RB_BLIT_CNTL_BUF__MASK				0x0000000f
-#define A5XX_RB_BLIT_CNTL_BUF__SHIFT				0
-static inline uint32_t A5XX_RB_BLIT_CNTL_BUF(enum a5xx_blit_buf val)
-{
-	return ((val) << A5XX_RB_BLIT_CNTL_BUF__SHIFT) & A5XX_RB_BLIT_CNTL_BUF__MASK;
-}
-
-#define REG_A5XX_RB_RESOLVE_CNTL_1				0x0000e211
-#define A5XX_RB_RESOLVE_CNTL_1_WINDOW_OFFSET_DISABLE		0x80000000
-#define A5XX_RB_RESOLVE_CNTL_1_X__MASK				0x00007fff
-#define A5XX_RB_RESOLVE_CNTL_1_X__SHIFT				0
-static inline uint32_t A5XX_RB_RESOLVE_CNTL_1_X(uint32_t val)
-{
-	return ((val) << A5XX_RB_RESOLVE_CNTL_1_X__SHIFT) & A5XX_RB_RESOLVE_CNTL_1_X__MASK;
-}
-#define A5XX_RB_RESOLVE_CNTL_1_Y__MASK				0x7fff0000
-#define A5XX_RB_RESOLVE_CNTL_1_Y__SHIFT				16
-static inline uint32_t A5XX_RB_RESOLVE_CNTL_1_Y(uint32_t val)
-{
-	return ((val) << A5XX_RB_RESOLVE_CNTL_1_Y__SHIFT) & A5XX_RB_RESOLVE_CNTL_1_Y__MASK;
-}
-
-#define REG_A5XX_RB_RESOLVE_CNTL_2				0x0000e212
-#define A5XX_RB_RESOLVE_CNTL_2_WINDOW_OFFSET_DISABLE		0x80000000
-#define A5XX_RB_RESOLVE_CNTL_2_X__MASK				0x00007fff
-#define A5XX_RB_RESOLVE_CNTL_2_X__SHIFT				0
-static inline uint32_t A5XX_RB_RESOLVE_CNTL_2_X(uint32_t val)
-{
-	return ((val) << A5XX_RB_RESOLVE_CNTL_2_X__SHIFT) & A5XX_RB_RESOLVE_CNTL_2_X__MASK;
-}
-#define A5XX_RB_RESOLVE_CNTL_2_Y__MASK				0x7fff0000
-#define A5XX_RB_RESOLVE_CNTL_2_Y__SHIFT				16
-static inline uint32_t A5XX_RB_RESOLVE_CNTL_2_Y(uint32_t val)
-{
-	return ((val) << A5XX_RB_RESOLVE_CNTL_2_Y__SHIFT) & A5XX_RB_RESOLVE_CNTL_2_Y__MASK;
-}
-
-#define REG_A5XX_RB_RESOLVE_CNTL_3				0x0000e213
-#define A5XX_RB_RESOLVE_CNTL_3_TILED				0x00000001
-
-#define REG_A5XX_RB_BLIT_DST_LO					0x0000e214
-
-#define REG_A5XX_RB_BLIT_DST_HI					0x0000e215
-
-#define REG_A5XX_RB_BLIT_DST_PITCH				0x0000e216
-#define A5XX_RB_BLIT_DST_PITCH__MASK				0xffffffff
-#define A5XX_RB_BLIT_DST_PITCH__SHIFT				0
-static inline uint32_t A5XX_RB_BLIT_DST_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_BLIT_DST_PITCH__SHIFT) & A5XX_RB_BLIT_DST_PITCH__MASK;
-}
-
-#define REG_A5XX_RB_BLIT_DST_ARRAY_PITCH			0x0000e217
-#define A5XX_RB_BLIT_DST_ARRAY_PITCH__MASK			0xffffffff
-#define A5XX_RB_BLIT_DST_ARRAY_PITCH__SHIFT			0
-static inline uint32_t A5XX_RB_BLIT_DST_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_BLIT_DST_ARRAY_PITCH__SHIFT) & A5XX_RB_BLIT_DST_ARRAY_PITCH__MASK;
-}
-
-#define REG_A5XX_RB_CLEAR_COLOR_DW0				0x0000e218
-
-#define REG_A5XX_RB_CLEAR_COLOR_DW1				0x0000e219
-
-#define REG_A5XX_RB_CLEAR_COLOR_DW2				0x0000e21a
-
-#define REG_A5XX_RB_CLEAR_COLOR_DW3				0x0000e21b
-
-#define REG_A5XX_RB_CLEAR_CNTL					0x0000e21c
-#define A5XX_RB_CLEAR_CNTL_FAST_CLEAR				0x00000002
-#define A5XX_RB_CLEAR_CNTL_MSAA_RESOLVE				0x00000004
-#define A5XX_RB_CLEAR_CNTL_MASK__MASK				0x000000f0
-#define A5XX_RB_CLEAR_CNTL_MASK__SHIFT				4
-static inline uint32_t A5XX_RB_CLEAR_CNTL_MASK(uint32_t val)
-{
-	return ((val) << A5XX_RB_CLEAR_CNTL_MASK__SHIFT) & A5XX_RB_CLEAR_CNTL_MASK__MASK;
-}
-
-#define REG_A5XX_RB_DEPTH_FLAG_BUFFER_BASE_LO			0x0000e240
-
-#define REG_A5XX_RB_DEPTH_FLAG_BUFFER_BASE_HI			0x0000e241
-
-#define REG_A5XX_RB_DEPTH_FLAG_BUFFER_PITCH			0x0000e242
-
-#define REG_A5XX_RB_MRT_FLAG_BUFFER(i0) (0x0000e243 + 0x4*(i0))
-
-static inline uint32_t REG_A5XX_RB_MRT_FLAG_BUFFER_ADDR_LO(uint32_t i0) { return 0x0000e243 + 0x4*i0; }
-
-static inline uint32_t REG_A5XX_RB_MRT_FLAG_BUFFER_ADDR_HI(uint32_t i0) { return 0x0000e244 + 0x4*i0; }
-
-static inline uint32_t REG_A5XX_RB_MRT_FLAG_BUFFER_PITCH(uint32_t i0) { return 0x0000e245 + 0x4*i0; }
-#define A5XX_RB_MRT_FLAG_BUFFER_PITCH__MASK			0xffffffff
-#define A5XX_RB_MRT_FLAG_BUFFER_PITCH__SHIFT			0
-static inline uint32_t A5XX_RB_MRT_FLAG_BUFFER_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_MRT_FLAG_BUFFER_PITCH__SHIFT) & A5XX_RB_MRT_FLAG_BUFFER_PITCH__MASK;
-}
-
-static inline uint32_t REG_A5XX_RB_MRT_FLAG_BUFFER_ARRAY_PITCH(uint32_t i0) { return 0x0000e246 + 0x4*i0; }
-#define A5XX_RB_MRT_FLAG_BUFFER_ARRAY_PITCH__MASK		0xffffffff
-#define A5XX_RB_MRT_FLAG_BUFFER_ARRAY_PITCH__SHIFT		0
-static inline uint32_t A5XX_RB_MRT_FLAG_BUFFER_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_MRT_FLAG_BUFFER_ARRAY_PITCH__SHIFT) & A5XX_RB_MRT_FLAG_BUFFER_ARRAY_PITCH__MASK;
-}
-
-#define REG_A5XX_RB_BLIT_FLAG_DST_LO				0x0000e263
-
-#define REG_A5XX_RB_BLIT_FLAG_DST_HI				0x0000e264
-
-#define REG_A5XX_RB_BLIT_FLAG_DST_PITCH				0x0000e265
-#define A5XX_RB_BLIT_FLAG_DST_PITCH__MASK			0xffffffff
-#define A5XX_RB_BLIT_FLAG_DST_PITCH__SHIFT			0
-static inline uint32_t A5XX_RB_BLIT_FLAG_DST_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_BLIT_FLAG_DST_PITCH__SHIFT) & A5XX_RB_BLIT_FLAG_DST_PITCH__MASK;
-}
-
-#define REG_A5XX_RB_BLIT_FLAG_DST_ARRAY_PITCH			0x0000e266
-#define A5XX_RB_BLIT_FLAG_DST_ARRAY_PITCH__MASK			0xffffffff
-#define A5XX_RB_BLIT_FLAG_DST_ARRAY_PITCH__SHIFT		0
-static inline uint32_t A5XX_RB_BLIT_FLAG_DST_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_BLIT_FLAG_DST_ARRAY_PITCH__SHIFT) & A5XX_RB_BLIT_FLAG_DST_ARRAY_PITCH__MASK;
-}
-
-#define REG_A5XX_RB_SAMPLE_COUNT_ADDR_LO			0x0000e267
-
-#define REG_A5XX_RB_SAMPLE_COUNT_ADDR_HI			0x0000e268
-
-#define REG_A5XX_VPC_CNTL_0					0x0000e280
-#define A5XX_VPC_CNTL_0_STRIDE_IN_VPC__MASK			0x0000007f
-#define A5XX_VPC_CNTL_0_STRIDE_IN_VPC__SHIFT			0
-static inline uint32_t A5XX_VPC_CNTL_0_STRIDE_IN_VPC(uint32_t val)
-{
-	return ((val) << A5XX_VPC_CNTL_0_STRIDE_IN_VPC__SHIFT) & A5XX_VPC_CNTL_0_STRIDE_IN_VPC__MASK;
-}
-#define A5XX_VPC_CNTL_0_VARYING					0x00000800
-
-#define REG_A5XX_VPC_VARYING_INTERP(i0) (0x0000e282 + 0x1*(i0))
-
-static inline uint32_t REG_A5XX_VPC_VARYING_INTERP_MODE(uint32_t i0) { return 0x0000e282 + 0x1*i0; }
-
-#define REG_A5XX_VPC_VARYING_PS_REPL(i0) (0x0000e28a + 0x1*(i0))
-
-static inline uint32_t REG_A5XX_VPC_VARYING_PS_REPL_MODE(uint32_t i0) { return 0x0000e28a + 0x1*i0; }
-
-#define REG_A5XX_UNKNOWN_E292					0x0000e292
-
-#define REG_A5XX_UNKNOWN_E293					0x0000e293
-
-#define REG_A5XX_VPC_VAR(i0) (0x0000e294 + 0x1*(i0))
-
-static inline uint32_t REG_A5XX_VPC_VAR_DISABLE(uint32_t i0) { return 0x0000e294 + 0x1*i0; }
-
-#define REG_A5XX_VPC_GS_SIV_CNTL				0x0000e298
-
-#define REG_A5XX_VPC_CLIP_CNTL					0x0000e29a
-#define A5XX_VPC_CLIP_CNTL_CLIP_MASK__MASK			0x000000ff
-#define A5XX_VPC_CLIP_CNTL_CLIP_MASK__SHIFT			0
-static inline uint32_t A5XX_VPC_CLIP_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A5XX_VPC_CLIP_CNTL_CLIP_MASK__SHIFT) & A5XX_VPC_CLIP_CNTL_CLIP_MASK__MASK;
-}
-#define A5XX_VPC_CLIP_CNTL_CLIP_DIST_03_LOC__MASK		0x0000ff00
-#define A5XX_VPC_CLIP_CNTL_CLIP_DIST_03_LOC__SHIFT		8
-static inline uint32_t A5XX_VPC_CLIP_CNTL_CLIP_DIST_03_LOC(uint32_t val)
-{
-	return ((val) << A5XX_VPC_CLIP_CNTL_CLIP_DIST_03_LOC__SHIFT) & A5XX_VPC_CLIP_CNTL_CLIP_DIST_03_LOC__MASK;
-}
-#define A5XX_VPC_CLIP_CNTL_CLIP_DIST_47_LOC__MASK		0x00ff0000
-#define A5XX_VPC_CLIP_CNTL_CLIP_DIST_47_LOC__SHIFT		16
-static inline uint32_t A5XX_VPC_CLIP_CNTL_CLIP_DIST_47_LOC(uint32_t val)
-{
-	return ((val) << A5XX_VPC_CLIP_CNTL_CLIP_DIST_47_LOC__SHIFT) & A5XX_VPC_CLIP_CNTL_CLIP_DIST_47_LOC__MASK;
-}
-
-#define REG_A5XX_VPC_PACK					0x0000e29d
-#define A5XX_VPC_PACK_NUMNONPOSVAR__MASK			0x000000ff
-#define A5XX_VPC_PACK_NUMNONPOSVAR__SHIFT			0
-static inline uint32_t A5XX_VPC_PACK_NUMNONPOSVAR(uint32_t val)
-{
-	return ((val) << A5XX_VPC_PACK_NUMNONPOSVAR__SHIFT) & A5XX_VPC_PACK_NUMNONPOSVAR__MASK;
-}
-#define A5XX_VPC_PACK_PSIZELOC__MASK				0x0000ff00
-#define A5XX_VPC_PACK_PSIZELOC__SHIFT				8
-static inline uint32_t A5XX_VPC_PACK_PSIZELOC(uint32_t val)
-{
-	return ((val) << A5XX_VPC_PACK_PSIZELOC__SHIFT) & A5XX_VPC_PACK_PSIZELOC__MASK;
-}
-
-#define REG_A5XX_VPC_FS_PRIMITIVEID_CNTL			0x0000e2a0
-
-#define REG_A5XX_VPC_SO_BUF_CNTL				0x0000e2a1
-#define A5XX_VPC_SO_BUF_CNTL_BUF0				0x00000001
-#define A5XX_VPC_SO_BUF_CNTL_BUF1				0x00000008
-#define A5XX_VPC_SO_BUF_CNTL_BUF2				0x00000040
-#define A5XX_VPC_SO_BUF_CNTL_BUF3				0x00000200
-#define A5XX_VPC_SO_BUF_CNTL_ENABLE				0x00008000
-
-#define REG_A5XX_VPC_SO_OVERRIDE				0x0000e2a2
-#define A5XX_VPC_SO_OVERRIDE_SO_DISABLE				0x00000001
-
-#define REG_A5XX_VPC_SO_CNTL					0x0000e2a3
-#define A5XX_VPC_SO_CNTL_ENABLE					0x00010000
-
-#define REG_A5XX_VPC_SO_PROG					0x0000e2a4
-#define A5XX_VPC_SO_PROG_A_BUF__MASK				0x00000003
-#define A5XX_VPC_SO_PROG_A_BUF__SHIFT				0
-static inline uint32_t A5XX_VPC_SO_PROG_A_BUF(uint32_t val)
-{
-	return ((val) << A5XX_VPC_SO_PROG_A_BUF__SHIFT) & A5XX_VPC_SO_PROG_A_BUF__MASK;
-}
-#define A5XX_VPC_SO_PROG_A_OFF__MASK				0x000007fc
-#define A5XX_VPC_SO_PROG_A_OFF__SHIFT				2
-static inline uint32_t A5XX_VPC_SO_PROG_A_OFF(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A5XX_VPC_SO_PROG_A_OFF__SHIFT) & A5XX_VPC_SO_PROG_A_OFF__MASK;
-}
-#define A5XX_VPC_SO_PROG_A_EN					0x00000800
-#define A5XX_VPC_SO_PROG_B_BUF__MASK				0x00003000
-#define A5XX_VPC_SO_PROG_B_BUF__SHIFT				12
-static inline uint32_t A5XX_VPC_SO_PROG_B_BUF(uint32_t val)
-{
-	return ((val) << A5XX_VPC_SO_PROG_B_BUF__SHIFT) & A5XX_VPC_SO_PROG_B_BUF__MASK;
-}
-#define A5XX_VPC_SO_PROG_B_OFF__MASK				0x007fc000
-#define A5XX_VPC_SO_PROG_B_OFF__SHIFT				14
-static inline uint32_t A5XX_VPC_SO_PROG_B_OFF(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A5XX_VPC_SO_PROG_B_OFF__SHIFT) & A5XX_VPC_SO_PROG_B_OFF__MASK;
-}
-#define A5XX_VPC_SO_PROG_B_EN					0x00800000
-
-#define REG_A5XX_VPC_SO(i0) (0x0000e2a7 + 0x7*(i0))
-
-static inline uint32_t REG_A5XX_VPC_SO_BUFFER_BASE_LO(uint32_t i0) { return 0x0000e2a7 + 0x7*i0; }
-
-static inline uint32_t REG_A5XX_VPC_SO_BUFFER_BASE_HI(uint32_t i0) { return 0x0000e2a8 + 0x7*i0; }
-
-static inline uint32_t REG_A5XX_VPC_SO_BUFFER_SIZE(uint32_t i0) { return 0x0000e2a9 + 0x7*i0; }
-
-static inline uint32_t REG_A5XX_VPC_SO_NCOMP(uint32_t i0) { return 0x0000e2aa + 0x7*i0; }
-
-static inline uint32_t REG_A5XX_VPC_SO_BUFFER_OFFSET(uint32_t i0) { return 0x0000e2ab + 0x7*i0; }
-
-static inline uint32_t REG_A5XX_VPC_SO_FLUSH_BASE_LO(uint32_t i0) { return 0x0000e2ac + 0x7*i0; }
-
-static inline uint32_t REG_A5XX_VPC_SO_FLUSH_BASE_HI(uint32_t i0) { return 0x0000e2ad + 0x7*i0; }
-
-#define REG_A5XX_PC_PRIMITIVE_CNTL				0x0000e384
-#define A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC__MASK		0x0000007f
-#define A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC__SHIFT		0
-static inline uint32_t A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC(uint32_t val)
-{
-	return ((val) << A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC__SHIFT) & A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC__MASK;
-}
-#define A5XX_PC_PRIMITIVE_CNTL_PRIMITIVE_RESTART		0x00000100
-#define A5XX_PC_PRIMITIVE_CNTL_COUNT_PRIMITIVES			0x00000200
-#define A5XX_PC_PRIMITIVE_CNTL_PROVOKING_VTX_LAST		0x00000400
-
-#define REG_A5XX_PC_PRIM_VTX_CNTL				0x0000e385
-#define A5XX_PC_PRIM_VTX_CNTL_PSIZE				0x00000800
-
-#define REG_A5XX_PC_RASTER_CNTL					0x0000e388
-#define A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE__MASK		0x00000007
-#define A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE__SHIFT		0
-static inline uint32_t A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE(enum adreno_pa_su_sc_draw val)
-{
-	return ((val) << A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE__SHIFT) & A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE__MASK;
-}
-#define A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE__MASK		0x00000038
-#define A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE__SHIFT		3
-static inline uint32_t A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE(enum adreno_pa_su_sc_draw val)
-{
-	return ((val) << A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE__SHIFT) & A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE__MASK;
-}
-#define A5XX_PC_RASTER_CNTL_POLYMODE_ENABLE			0x00000040
-
-#define REG_A5XX_PC_CLIP_CNTL					0x0000e389
-#define A5XX_PC_CLIP_CNTL_CLIP_MASK__MASK			0x000000ff
-#define A5XX_PC_CLIP_CNTL_CLIP_MASK__SHIFT			0
-static inline uint32_t A5XX_PC_CLIP_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A5XX_PC_CLIP_CNTL_CLIP_MASK__SHIFT) & A5XX_PC_CLIP_CNTL_CLIP_MASK__MASK;
-}
-
-#define REG_A5XX_PC_RESTART_INDEX				0x0000e38c
-
-#define REG_A5XX_PC_GS_LAYERED					0x0000e38d
-
-#define REG_A5XX_PC_GS_PARAM					0x0000e38e
-#define A5XX_PC_GS_PARAM_MAX_VERTICES__MASK			0x000003ff
-#define A5XX_PC_GS_PARAM_MAX_VERTICES__SHIFT			0
-static inline uint32_t A5XX_PC_GS_PARAM_MAX_VERTICES(uint32_t val)
-{
-	return ((val) << A5XX_PC_GS_PARAM_MAX_VERTICES__SHIFT) & A5XX_PC_GS_PARAM_MAX_VERTICES__MASK;
-}
-#define A5XX_PC_GS_PARAM_INVOCATIONS__MASK			0x0000f800
-#define A5XX_PC_GS_PARAM_INVOCATIONS__SHIFT			11
-static inline uint32_t A5XX_PC_GS_PARAM_INVOCATIONS(uint32_t val)
-{
-	return ((val) << A5XX_PC_GS_PARAM_INVOCATIONS__SHIFT) & A5XX_PC_GS_PARAM_INVOCATIONS__MASK;
-}
-#define A5XX_PC_GS_PARAM_PRIMTYPE__MASK				0x01800000
-#define A5XX_PC_GS_PARAM_PRIMTYPE__SHIFT			23
-static inline uint32_t A5XX_PC_GS_PARAM_PRIMTYPE(enum adreno_pa_su_sc_draw val)
-{
-	return ((val) << A5XX_PC_GS_PARAM_PRIMTYPE__SHIFT) & A5XX_PC_GS_PARAM_PRIMTYPE__MASK;
-}
-
-#define REG_A5XX_PC_HS_PARAM					0x0000e38f
-#define A5XX_PC_HS_PARAM_VERTICES_OUT__MASK			0x0000003f
-#define A5XX_PC_HS_PARAM_VERTICES_OUT__SHIFT			0
-static inline uint32_t A5XX_PC_HS_PARAM_VERTICES_OUT(uint32_t val)
-{
-	return ((val) << A5XX_PC_HS_PARAM_VERTICES_OUT__SHIFT) & A5XX_PC_HS_PARAM_VERTICES_OUT__MASK;
-}
-#define A5XX_PC_HS_PARAM_SPACING__MASK				0x00600000
-#define A5XX_PC_HS_PARAM_SPACING__SHIFT				21
-static inline uint32_t A5XX_PC_HS_PARAM_SPACING(enum a4xx_tess_spacing val)
-{
-	return ((val) << A5XX_PC_HS_PARAM_SPACING__SHIFT) & A5XX_PC_HS_PARAM_SPACING__MASK;
-}
-#define A5XX_PC_HS_PARAM_CW					0x00800000
-#define A5XX_PC_HS_PARAM_CONNECTED				0x01000000
-
-#define REG_A5XX_PC_POWER_CNTL					0x0000e3b0
-
-#define REG_A5XX_VFD_CONTROL_0					0x0000e400
-#define A5XX_VFD_CONTROL_0_VTXCNT__MASK				0x0000003f
-#define A5XX_VFD_CONTROL_0_VTXCNT__SHIFT			0
-static inline uint32_t A5XX_VFD_CONTROL_0_VTXCNT(uint32_t val)
-{
-	return ((val) << A5XX_VFD_CONTROL_0_VTXCNT__SHIFT) & A5XX_VFD_CONTROL_0_VTXCNT__MASK;
-}
-
-#define REG_A5XX_VFD_CONTROL_1					0x0000e401
-#define A5XX_VFD_CONTROL_1_REGID4VTX__MASK			0x000000ff
-#define A5XX_VFD_CONTROL_1_REGID4VTX__SHIFT			0
-static inline uint32_t A5XX_VFD_CONTROL_1_REGID4VTX(uint32_t val)
-{
-	return ((val) << A5XX_VFD_CONTROL_1_REGID4VTX__SHIFT) & A5XX_VFD_CONTROL_1_REGID4VTX__MASK;
-}
-#define A5XX_VFD_CONTROL_1_REGID4INST__MASK			0x0000ff00
-#define A5XX_VFD_CONTROL_1_REGID4INST__SHIFT			8
-static inline uint32_t A5XX_VFD_CONTROL_1_REGID4INST(uint32_t val)
-{
-	return ((val) << A5XX_VFD_CONTROL_1_REGID4INST__SHIFT) & A5XX_VFD_CONTROL_1_REGID4INST__MASK;
-}
-#define A5XX_VFD_CONTROL_1_REGID4PRIMID__MASK			0x00ff0000
-#define A5XX_VFD_CONTROL_1_REGID4PRIMID__SHIFT			16
-static inline uint32_t A5XX_VFD_CONTROL_1_REGID4PRIMID(uint32_t val)
-{
-	return ((val) << A5XX_VFD_CONTROL_1_REGID4PRIMID__SHIFT) & A5XX_VFD_CONTROL_1_REGID4PRIMID__MASK;
-}
-
-#define REG_A5XX_VFD_CONTROL_2					0x0000e402
-#define A5XX_VFD_CONTROL_2_REGID_PATCHID__MASK			0x000000ff
-#define A5XX_VFD_CONTROL_2_REGID_PATCHID__SHIFT			0
-static inline uint32_t A5XX_VFD_CONTROL_2_REGID_PATCHID(uint32_t val)
-{
-	return ((val) << A5XX_VFD_CONTROL_2_REGID_PATCHID__SHIFT) & A5XX_VFD_CONTROL_2_REGID_PATCHID__MASK;
-}
-
-#define REG_A5XX_VFD_CONTROL_3					0x0000e403
-#define A5XX_VFD_CONTROL_3_REGID_PATCHID__MASK			0x0000ff00
-#define A5XX_VFD_CONTROL_3_REGID_PATCHID__SHIFT			8
-static inline uint32_t A5XX_VFD_CONTROL_3_REGID_PATCHID(uint32_t val)
-{
-	return ((val) << A5XX_VFD_CONTROL_3_REGID_PATCHID__SHIFT) & A5XX_VFD_CONTROL_3_REGID_PATCHID__MASK;
-}
-#define A5XX_VFD_CONTROL_3_REGID_TESSX__MASK			0x00ff0000
-#define A5XX_VFD_CONTROL_3_REGID_TESSX__SHIFT			16
-static inline uint32_t A5XX_VFD_CONTROL_3_REGID_TESSX(uint32_t val)
-{
-	return ((val) << A5XX_VFD_CONTROL_3_REGID_TESSX__SHIFT) & A5XX_VFD_CONTROL_3_REGID_TESSX__MASK;
-}
-#define A5XX_VFD_CONTROL_3_REGID_TESSY__MASK			0xff000000
-#define A5XX_VFD_CONTROL_3_REGID_TESSY__SHIFT			24
-static inline uint32_t A5XX_VFD_CONTROL_3_REGID_TESSY(uint32_t val)
-{
-	return ((val) << A5XX_VFD_CONTROL_3_REGID_TESSY__SHIFT) & A5XX_VFD_CONTROL_3_REGID_TESSY__MASK;
-}
-
-#define REG_A5XX_VFD_CONTROL_4					0x0000e404
-
-#define REG_A5XX_VFD_CONTROL_5					0x0000e405
-
-#define REG_A5XX_VFD_INDEX_OFFSET				0x0000e408
-
-#define REG_A5XX_VFD_INSTANCE_START_OFFSET			0x0000e409
-
-#define REG_A5XX_VFD_FETCH(i0) (0x0000e40a + 0x4*(i0))
-
-static inline uint32_t REG_A5XX_VFD_FETCH_BASE_LO(uint32_t i0) { return 0x0000e40a + 0x4*i0; }
-
-static inline uint32_t REG_A5XX_VFD_FETCH_BASE_HI(uint32_t i0) { return 0x0000e40b + 0x4*i0; }
-
-static inline uint32_t REG_A5XX_VFD_FETCH_SIZE(uint32_t i0) { return 0x0000e40c + 0x4*i0; }
-
-static inline uint32_t REG_A5XX_VFD_FETCH_STRIDE(uint32_t i0) { return 0x0000e40d + 0x4*i0; }
-
-#define REG_A5XX_VFD_DECODE(i0) (0x0000e48a + 0x2*(i0))
-
-static inline uint32_t REG_A5XX_VFD_DECODE_INSTR(uint32_t i0) { return 0x0000e48a + 0x2*i0; }
-#define A5XX_VFD_DECODE_INSTR_IDX__MASK				0x0000001f
-#define A5XX_VFD_DECODE_INSTR_IDX__SHIFT			0
-static inline uint32_t A5XX_VFD_DECODE_INSTR_IDX(uint32_t val)
-{
-	return ((val) << A5XX_VFD_DECODE_INSTR_IDX__SHIFT) & A5XX_VFD_DECODE_INSTR_IDX__MASK;
-}
-#define A5XX_VFD_DECODE_INSTR_INSTANCED				0x00020000
-#define A5XX_VFD_DECODE_INSTR_FORMAT__MASK			0x0ff00000
-#define A5XX_VFD_DECODE_INSTR_FORMAT__SHIFT			20
-static inline uint32_t A5XX_VFD_DECODE_INSTR_FORMAT(enum a5xx_vtx_fmt val)
-{
-	return ((val) << A5XX_VFD_DECODE_INSTR_FORMAT__SHIFT) & A5XX_VFD_DECODE_INSTR_FORMAT__MASK;
-}
-#define A5XX_VFD_DECODE_INSTR_SWAP__MASK			0x30000000
-#define A5XX_VFD_DECODE_INSTR_SWAP__SHIFT			28
-static inline uint32_t A5XX_VFD_DECODE_INSTR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A5XX_VFD_DECODE_INSTR_SWAP__SHIFT) & A5XX_VFD_DECODE_INSTR_SWAP__MASK;
-}
-#define A5XX_VFD_DECODE_INSTR_UNK30				0x40000000
-#define A5XX_VFD_DECODE_INSTR_FLOAT				0x80000000
-
-static inline uint32_t REG_A5XX_VFD_DECODE_STEP_RATE(uint32_t i0) { return 0x0000e48b + 0x2*i0; }
-
-#define REG_A5XX_VFD_DEST_CNTL(i0) (0x0000e4ca + 0x1*(i0))
-
-static inline uint32_t REG_A5XX_VFD_DEST_CNTL_INSTR(uint32_t i0) { return 0x0000e4ca + 0x1*i0; }
-#define A5XX_VFD_DEST_CNTL_INSTR_WRITEMASK__MASK		0x0000000f
-#define A5XX_VFD_DEST_CNTL_INSTR_WRITEMASK__SHIFT		0
-static inline uint32_t A5XX_VFD_DEST_CNTL_INSTR_WRITEMASK(uint32_t val)
-{
-	return ((val) << A5XX_VFD_DEST_CNTL_INSTR_WRITEMASK__SHIFT) & A5XX_VFD_DEST_CNTL_INSTR_WRITEMASK__MASK;
-}
-#define A5XX_VFD_DEST_CNTL_INSTR_REGID__MASK			0x00000ff0
-#define A5XX_VFD_DEST_CNTL_INSTR_REGID__SHIFT			4
-static inline uint32_t A5XX_VFD_DEST_CNTL_INSTR_REGID(uint32_t val)
-{
-	return ((val) << A5XX_VFD_DEST_CNTL_INSTR_REGID__SHIFT) & A5XX_VFD_DEST_CNTL_INSTR_REGID__MASK;
-}
-
-#define REG_A5XX_VFD_POWER_CNTL					0x0000e4f0
-
-#define REG_A5XX_SP_SP_CNTL					0x0000e580
-
-#define REG_A5XX_SP_VS_CONFIG					0x0000e584
-#define A5XX_SP_VS_CONFIG_ENABLED				0x00000001
-#define A5XX_SP_VS_CONFIG_CONSTOBJECTOFFSET__MASK		0x000000fe
-#define A5XX_SP_VS_CONFIG_CONSTOBJECTOFFSET__SHIFT		1
-static inline uint32_t A5XX_SP_VS_CONFIG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_CONFIG_CONSTOBJECTOFFSET__SHIFT) & A5XX_SP_VS_CONFIG_CONSTOBJECTOFFSET__MASK;
-}
-#define A5XX_SP_VS_CONFIG_SHADEROBJOFFSET__MASK			0x00007f00
-#define A5XX_SP_VS_CONFIG_SHADEROBJOFFSET__SHIFT		8
-static inline uint32_t A5XX_SP_VS_CONFIG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_CONFIG_SHADEROBJOFFSET__SHIFT) & A5XX_SP_VS_CONFIG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A5XX_SP_FS_CONFIG					0x0000e585
-#define A5XX_SP_FS_CONFIG_ENABLED				0x00000001
-#define A5XX_SP_FS_CONFIG_CONSTOBJECTOFFSET__MASK		0x000000fe
-#define A5XX_SP_FS_CONFIG_CONSTOBJECTOFFSET__SHIFT		1
-static inline uint32_t A5XX_SP_FS_CONFIG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_SP_FS_CONFIG_CONSTOBJECTOFFSET__SHIFT) & A5XX_SP_FS_CONFIG_CONSTOBJECTOFFSET__MASK;
-}
-#define A5XX_SP_FS_CONFIG_SHADEROBJOFFSET__MASK			0x00007f00
-#define A5XX_SP_FS_CONFIG_SHADEROBJOFFSET__SHIFT		8
-static inline uint32_t A5XX_SP_FS_CONFIG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_SP_FS_CONFIG_SHADEROBJOFFSET__SHIFT) & A5XX_SP_FS_CONFIG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A5XX_SP_HS_CONFIG					0x0000e586
-#define A5XX_SP_HS_CONFIG_ENABLED				0x00000001
-#define A5XX_SP_HS_CONFIG_CONSTOBJECTOFFSET__MASK		0x000000fe
-#define A5XX_SP_HS_CONFIG_CONSTOBJECTOFFSET__SHIFT		1
-static inline uint32_t A5XX_SP_HS_CONFIG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_SP_HS_CONFIG_CONSTOBJECTOFFSET__SHIFT) & A5XX_SP_HS_CONFIG_CONSTOBJECTOFFSET__MASK;
-}
-#define A5XX_SP_HS_CONFIG_SHADEROBJOFFSET__MASK			0x00007f00
-#define A5XX_SP_HS_CONFIG_SHADEROBJOFFSET__SHIFT		8
-static inline uint32_t A5XX_SP_HS_CONFIG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_SP_HS_CONFIG_SHADEROBJOFFSET__SHIFT) & A5XX_SP_HS_CONFIG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A5XX_SP_DS_CONFIG					0x0000e587
-#define A5XX_SP_DS_CONFIG_ENABLED				0x00000001
-#define A5XX_SP_DS_CONFIG_CONSTOBJECTOFFSET__MASK		0x000000fe
-#define A5XX_SP_DS_CONFIG_CONSTOBJECTOFFSET__SHIFT		1
-static inline uint32_t A5XX_SP_DS_CONFIG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_SP_DS_CONFIG_CONSTOBJECTOFFSET__SHIFT) & A5XX_SP_DS_CONFIG_CONSTOBJECTOFFSET__MASK;
-}
-#define A5XX_SP_DS_CONFIG_SHADEROBJOFFSET__MASK			0x00007f00
-#define A5XX_SP_DS_CONFIG_SHADEROBJOFFSET__SHIFT		8
-static inline uint32_t A5XX_SP_DS_CONFIG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_SP_DS_CONFIG_SHADEROBJOFFSET__SHIFT) & A5XX_SP_DS_CONFIG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A5XX_SP_GS_CONFIG					0x0000e588
-#define A5XX_SP_GS_CONFIG_ENABLED				0x00000001
-#define A5XX_SP_GS_CONFIG_CONSTOBJECTOFFSET__MASK		0x000000fe
-#define A5XX_SP_GS_CONFIG_CONSTOBJECTOFFSET__SHIFT		1
-static inline uint32_t A5XX_SP_GS_CONFIG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_SP_GS_CONFIG_CONSTOBJECTOFFSET__SHIFT) & A5XX_SP_GS_CONFIG_CONSTOBJECTOFFSET__MASK;
-}
-#define A5XX_SP_GS_CONFIG_SHADEROBJOFFSET__MASK			0x00007f00
-#define A5XX_SP_GS_CONFIG_SHADEROBJOFFSET__SHIFT		8
-static inline uint32_t A5XX_SP_GS_CONFIG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_SP_GS_CONFIG_SHADEROBJOFFSET__SHIFT) & A5XX_SP_GS_CONFIG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A5XX_SP_CS_CONFIG					0x0000e589
-#define A5XX_SP_CS_CONFIG_ENABLED				0x00000001
-#define A5XX_SP_CS_CONFIG_CONSTOBJECTOFFSET__MASK		0x000000fe
-#define A5XX_SP_CS_CONFIG_CONSTOBJECTOFFSET__SHIFT		1
-static inline uint32_t A5XX_SP_CS_CONFIG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_SP_CS_CONFIG_CONSTOBJECTOFFSET__SHIFT) & A5XX_SP_CS_CONFIG_CONSTOBJECTOFFSET__MASK;
-}
-#define A5XX_SP_CS_CONFIG_SHADEROBJOFFSET__MASK			0x00007f00
-#define A5XX_SP_CS_CONFIG_SHADEROBJOFFSET__SHIFT		8
-static inline uint32_t A5XX_SP_CS_CONFIG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_SP_CS_CONFIG_SHADEROBJOFFSET__SHIFT) & A5XX_SP_CS_CONFIG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A5XX_SP_VS_CONFIG_MAX_CONST				0x0000e58a
-
-#define REG_A5XX_SP_FS_CONFIG_MAX_CONST				0x0000e58b
-
-#define REG_A5XX_SP_VS_CTRL_REG0				0x0000e590
-#define A5XX_SP_VS_CTRL_REG0_BUFFER				0x00000004
-#define A5XX_SP_VS_CTRL_REG0_THREADSIZE__MASK			0x00000008
-#define A5XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT			3
-static inline uint32_t A5XX_SP_VS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A5XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT) & A5XX_SP_VS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A5XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
-#define A5XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
-static inline uint32_t A5XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A5XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
-#define A5XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
-static inline uint32_t A5XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A5XX_SP_VS_CTRL_REG0_VARYING				0x00010000
-#define A5XX_SP_VS_CTRL_REG0_PIXLODENABLE			0x00100000
-#define A5XX_SP_VS_CTRL_REG0_BRANCHSTACK__MASK			0xfe000000
-#define A5XX_SP_VS_CTRL_REG0_BRANCHSTACK__SHIFT			25
-static inline uint32_t A5XX_SP_VS_CTRL_REG0_BRANCHSTACK(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_VS_CTRL_REG0_BRANCHSTACK__MASK;
-}
-
-#define REG_A5XX_SP_PRIMITIVE_CNTL				0x0000e592
-#define A5XX_SP_PRIMITIVE_CNTL_VSOUT__MASK			0x0000001f
-#define A5XX_SP_PRIMITIVE_CNTL_VSOUT__SHIFT			0
-static inline uint32_t A5XX_SP_PRIMITIVE_CNTL_VSOUT(uint32_t val)
-{
-	return ((val) << A5XX_SP_PRIMITIVE_CNTL_VSOUT__SHIFT) & A5XX_SP_PRIMITIVE_CNTL_VSOUT__MASK;
-}
-
-#define REG_A5XX_SP_VS_OUT(i0) (0x0000e593 + 0x1*(i0))
-
-static inline uint32_t REG_A5XX_SP_VS_OUT_REG(uint32_t i0) { return 0x0000e593 + 0x1*i0; }
-#define A5XX_SP_VS_OUT_REG_A_REGID__MASK			0x000000ff
-#define A5XX_SP_VS_OUT_REG_A_REGID__SHIFT			0
-static inline uint32_t A5XX_SP_VS_OUT_REG_A_REGID(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_OUT_REG_A_REGID__SHIFT) & A5XX_SP_VS_OUT_REG_A_REGID__MASK;
-}
-#define A5XX_SP_VS_OUT_REG_A_COMPMASK__MASK			0x00000f00
-#define A5XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT			8
-static inline uint32_t A5XX_SP_VS_OUT_REG_A_COMPMASK(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT) & A5XX_SP_VS_OUT_REG_A_COMPMASK__MASK;
-}
-#define A5XX_SP_VS_OUT_REG_B_REGID__MASK			0x00ff0000
-#define A5XX_SP_VS_OUT_REG_B_REGID__SHIFT			16
-static inline uint32_t A5XX_SP_VS_OUT_REG_B_REGID(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_OUT_REG_B_REGID__SHIFT) & A5XX_SP_VS_OUT_REG_B_REGID__MASK;
-}
-#define A5XX_SP_VS_OUT_REG_B_COMPMASK__MASK			0x0f000000
-#define A5XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT			24
-static inline uint32_t A5XX_SP_VS_OUT_REG_B_COMPMASK(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT) & A5XX_SP_VS_OUT_REG_B_COMPMASK__MASK;
-}
-
-#define REG_A5XX_SP_VS_VPC_DST(i0) (0x0000e5a3 + 0x1*(i0))
-
-static inline uint32_t REG_A5XX_SP_VS_VPC_DST_REG(uint32_t i0) { return 0x0000e5a3 + 0x1*i0; }
-#define A5XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK			0x000000ff
-#define A5XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT			0
-static inline uint32_t A5XX_SP_VS_VPC_DST_REG_OUTLOC0(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT) & A5XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK;
-}
-#define A5XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK			0x0000ff00
-#define A5XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT			8
-static inline uint32_t A5XX_SP_VS_VPC_DST_REG_OUTLOC1(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT) & A5XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK;
-}
-#define A5XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK			0x00ff0000
-#define A5XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT			16
-static inline uint32_t A5XX_SP_VS_VPC_DST_REG_OUTLOC2(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT) & A5XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK;
-}
-#define A5XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK			0xff000000
-#define A5XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT			24
-static inline uint32_t A5XX_SP_VS_VPC_DST_REG_OUTLOC3(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT) & A5XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK;
-}
-
-#define REG_A5XX_UNKNOWN_E5AB					0x0000e5ab
-
-#define REG_A5XX_SP_VS_OBJ_START_LO				0x0000e5ac
-
-#define REG_A5XX_SP_VS_OBJ_START_HI				0x0000e5ad
-
-#define REG_A5XX_SP_VS_PVT_MEM_PARAM				0x0000e5ae
-#define A5XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK		0x000000ff
-#define A5XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT		0
-static inline uint32_t A5XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x1ff));
-	return (((val >> 9)) << A5XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT) & A5XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK;
-}
-#define A5XX_SP_VS_PVT_MEM_PARAM_HWSTACKOFFSET__MASK		0x00ffff00
-#define A5XX_SP_VS_PVT_MEM_PARAM_HWSTACKOFFSET__SHIFT		8
-static inline uint32_t A5XX_SP_VS_PVT_MEM_PARAM_HWSTACKOFFSET(uint32_t val)
-{
-	assert(!(val & 0x7ff));
-	return (((val >> 11)) << A5XX_SP_VS_PVT_MEM_PARAM_HWSTACKOFFSET__SHIFT) & A5XX_SP_VS_PVT_MEM_PARAM_HWSTACKOFFSET__MASK;
-}
-#define A5XX_SP_VS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A5XX_SP_VS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A5XX_SP_VS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A5XX_SP_VS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT) & A5XX_SP_VS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A5XX_SP_VS_PVT_MEM_ADDR				0x0000e5af
-
-#define REG_A5XX_SP_VS_PVT_MEM_SIZE				0x0000e5b1
-#define A5XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK		0x0003ffff
-#define A5XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT		0
-static inline uint32_t A5XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A5XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT) & A5XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK;
-}
-
-#define REG_A5XX_SP_FS_CTRL_REG0				0x0000e5c0
-#define A5XX_SP_FS_CTRL_REG0_BUFFER				0x00000004
-#define A5XX_SP_FS_CTRL_REG0_THREADSIZE__MASK			0x00000008
-#define A5XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT			3
-static inline uint32_t A5XX_SP_FS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A5XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT) & A5XX_SP_FS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A5XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
-#define A5XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
-static inline uint32_t A5XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A5XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A5XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
-#define A5XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
-static inline uint32_t A5XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A5XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A5XX_SP_FS_CTRL_REG0_VARYING				0x00010000
-#define A5XX_SP_FS_CTRL_REG0_PIXLODENABLE			0x00100000
-#define A5XX_SP_FS_CTRL_REG0_BRANCHSTACK__MASK			0xfe000000
-#define A5XX_SP_FS_CTRL_REG0_BRANCHSTACK__SHIFT			25
-static inline uint32_t A5XX_SP_FS_CTRL_REG0_BRANCHSTACK(uint32_t val)
-{
-	return ((val) << A5XX_SP_FS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_FS_CTRL_REG0_BRANCHSTACK__MASK;
-}
-
-#define REG_A5XX_UNKNOWN_E5C2					0x0000e5c2
-
-#define REG_A5XX_SP_FS_OBJ_START_LO				0x0000e5c3
-
-#define REG_A5XX_SP_FS_OBJ_START_HI				0x0000e5c4
-
-#define REG_A5XX_SP_FS_PVT_MEM_PARAM				0x0000e5c5
-#define A5XX_SP_FS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK		0x000000ff
-#define A5XX_SP_FS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT		0
-static inline uint32_t A5XX_SP_FS_PVT_MEM_PARAM_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x1ff));
-	return (((val >> 9)) << A5XX_SP_FS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT) & A5XX_SP_FS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK;
-}
-#define A5XX_SP_FS_PVT_MEM_PARAM_HWSTACKOFFSET__MASK		0x00ffff00
-#define A5XX_SP_FS_PVT_MEM_PARAM_HWSTACKOFFSET__SHIFT		8
-static inline uint32_t A5XX_SP_FS_PVT_MEM_PARAM_HWSTACKOFFSET(uint32_t val)
-{
-	assert(!(val & 0x7ff));
-	return (((val >> 11)) << A5XX_SP_FS_PVT_MEM_PARAM_HWSTACKOFFSET__SHIFT) & A5XX_SP_FS_PVT_MEM_PARAM_HWSTACKOFFSET__MASK;
-}
-#define A5XX_SP_FS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A5XX_SP_FS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A5XX_SP_FS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A5XX_SP_FS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT) & A5XX_SP_FS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A5XX_SP_FS_PVT_MEM_ADDR				0x0000e5c6
-
-#define REG_A5XX_SP_FS_PVT_MEM_SIZE				0x0000e5c8
-#define A5XX_SP_FS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK		0x0003ffff
-#define A5XX_SP_FS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT		0
-static inline uint32_t A5XX_SP_FS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A5XX_SP_FS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT) & A5XX_SP_FS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK;
-}
-
-#define REG_A5XX_SP_BLEND_CNTL					0x0000e5c9
-#define A5XX_SP_BLEND_CNTL_ENABLE_BLEND__MASK			0x000000ff
-#define A5XX_SP_BLEND_CNTL_ENABLE_BLEND__SHIFT			0
-static inline uint32_t A5XX_SP_BLEND_CNTL_ENABLE_BLEND(uint32_t val)
-{
-	return ((val) << A5XX_SP_BLEND_CNTL_ENABLE_BLEND__SHIFT) & A5XX_SP_BLEND_CNTL_ENABLE_BLEND__MASK;
-}
-#define A5XX_SP_BLEND_CNTL_UNK8					0x00000100
-#define A5XX_SP_BLEND_CNTL_ALPHA_TO_COVERAGE			0x00000400
-
-#define REG_A5XX_SP_FS_OUTPUT_CNTL				0x0000e5ca
-#define A5XX_SP_FS_OUTPUT_CNTL_MRT__MASK			0x0000000f
-#define A5XX_SP_FS_OUTPUT_CNTL_MRT__SHIFT			0
-static inline uint32_t A5XX_SP_FS_OUTPUT_CNTL_MRT(uint32_t val)
-{
-	return ((val) << A5XX_SP_FS_OUTPUT_CNTL_MRT__SHIFT) & A5XX_SP_FS_OUTPUT_CNTL_MRT__MASK;
-}
-#define A5XX_SP_FS_OUTPUT_CNTL_DEPTH_REGID__MASK		0x00001fe0
-#define A5XX_SP_FS_OUTPUT_CNTL_DEPTH_REGID__SHIFT		5
-static inline uint32_t A5XX_SP_FS_OUTPUT_CNTL_DEPTH_REGID(uint32_t val)
-{
-	return ((val) << A5XX_SP_FS_OUTPUT_CNTL_DEPTH_REGID__SHIFT) & A5XX_SP_FS_OUTPUT_CNTL_DEPTH_REGID__MASK;
-}
-#define A5XX_SP_FS_OUTPUT_CNTL_SAMPLEMASK_REGID__MASK		0x001fe000
-#define A5XX_SP_FS_OUTPUT_CNTL_SAMPLEMASK_REGID__SHIFT		13
-static inline uint32_t A5XX_SP_FS_OUTPUT_CNTL_SAMPLEMASK_REGID(uint32_t val)
-{
-	return ((val) << A5XX_SP_FS_OUTPUT_CNTL_SAMPLEMASK_REGID__SHIFT) & A5XX_SP_FS_OUTPUT_CNTL_SAMPLEMASK_REGID__MASK;
-}
-
-#define REG_A5XX_SP_FS_OUTPUT(i0) (0x0000e5cb + 0x1*(i0))
-
-static inline uint32_t REG_A5XX_SP_FS_OUTPUT_REG(uint32_t i0) { return 0x0000e5cb + 0x1*i0; }
-#define A5XX_SP_FS_OUTPUT_REG_REGID__MASK			0x000000ff
-#define A5XX_SP_FS_OUTPUT_REG_REGID__SHIFT			0
-static inline uint32_t A5XX_SP_FS_OUTPUT_REG_REGID(uint32_t val)
-{
-	return ((val) << A5XX_SP_FS_OUTPUT_REG_REGID__SHIFT) & A5XX_SP_FS_OUTPUT_REG_REGID__MASK;
-}
-#define A5XX_SP_FS_OUTPUT_REG_HALF_PRECISION			0x00000100
-
-#define REG_A5XX_SP_FS_MRT(i0) (0x0000e5d3 + 0x1*(i0))
-
-static inline uint32_t REG_A5XX_SP_FS_MRT_REG(uint32_t i0) { return 0x0000e5d3 + 0x1*i0; }
-#define A5XX_SP_FS_MRT_REG_COLOR_FORMAT__MASK			0x000000ff
-#define A5XX_SP_FS_MRT_REG_COLOR_FORMAT__SHIFT			0
-static inline uint32_t A5XX_SP_FS_MRT_REG_COLOR_FORMAT(enum a5xx_color_fmt val)
-{
-	return ((val) << A5XX_SP_FS_MRT_REG_COLOR_FORMAT__SHIFT) & A5XX_SP_FS_MRT_REG_COLOR_FORMAT__MASK;
-}
-#define A5XX_SP_FS_MRT_REG_COLOR_SINT				0x00000100
-#define A5XX_SP_FS_MRT_REG_COLOR_UINT				0x00000200
-#define A5XX_SP_FS_MRT_REG_COLOR_SRGB				0x00000400
-
-#define REG_A5XX_UNKNOWN_E5DB					0x0000e5db
-
-#define REG_A5XX_SP_CS_CTRL_REG0				0x0000e5f0
-#define A5XX_SP_CS_CTRL_REG0_BUFFER				0x00000004
-#define A5XX_SP_CS_CTRL_REG0_THREADSIZE__MASK			0x00000008
-#define A5XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT			3
-static inline uint32_t A5XX_SP_CS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A5XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT) & A5XX_SP_CS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A5XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
-#define A5XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
-static inline uint32_t A5XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A5XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A5XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
-#define A5XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
-static inline uint32_t A5XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A5XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A5XX_SP_CS_CTRL_REG0_VARYING				0x00010000
-#define A5XX_SP_CS_CTRL_REG0_PIXLODENABLE			0x00100000
-#define A5XX_SP_CS_CTRL_REG0_BRANCHSTACK__MASK			0xfe000000
-#define A5XX_SP_CS_CTRL_REG0_BRANCHSTACK__SHIFT			25
-static inline uint32_t A5XX_SP_CS_CTRL_REG0_BRANCHSTACK(uint32_t val)
-{
-	return ((val) << A5XX_SP_CS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_CS_CTRL_REG0_BRANCHSTACK__MASK;
-}
-
-#define REG_A5XX_UNKNOWN_E5F2					0x0000e5f2
-
-#define REG_A5XX_SP_CS_OBJ_START_LO				0x0000e5f3
-
-#define REG_A5XX_SP_CS_OBJ_START_HI				0x0000e5f4
-
-#define REG_A5XX_SP_CS_PVT_MEM_PARAM				0x0000e5f5
-#define A5XX_SP_CS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK		0x000000ff
-#define A5XX_SP_CS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT		0
-static inline uint32_t A5XX_SP_CS_PVT_MEM_PARAM_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x1ff));
-	return (((val >> 9)) << A5XX_SP_CS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT) & A5XX_SP_CS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK;
-}
-#define A5XX_SP_CS_PVT_MEM_PARAM_HWSTACKOFFSET__MASK		0x00ffff00
-#define A5XX_SP_CS_PVT_MEM_PARAM_HWSTACKOFFSET__SHIFT		8
-static inline uint32_t A5XX_SP_CS_PVT_MEM_PARAM_HWSTACKOFFSET(uint32_t val)
-{
-	assert(!(val & 0x7ff));
-	return (((val >> 11)) << A5XX_SP_CS_PVT_MEM_PARAM_HWSTACKOFFSET__SHIFT) & A5XX_SP_CS_PVT_MEM_PARAM_HWSTACKOFFSET__MASK;
-}
-#define A5XX_SP_CS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A5XX_SP_CS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A5XX_SP_CS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A5XX_SP_CS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT) & A5XX_SP_CS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A5XX_SP_CS_PVT_MEM_ADDR				0x0000e5f6
-
-#define REG_A5XX_SP_CS_PVT_MEM_SIZE				0x0000e5f8
-#define A5XX_SP_CS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK		0x0003ffff
-#define A5XX_SP_CS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT		0
-static inline uint32_t A5XX_SP_CS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A5XX_SP_CS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT) & A5XX_SP_CS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK;
-}
-
-#define REG_A5XX_SP_HS_CTRL_REG0				0x0000e600
-#define A5XX_SP_HS_CTRL_REG0_BUFFER				0x00000004
-#define A5XX_SP_HS_CTRL_REG0_THREADSIZE__MASK			0x00000008
-#define A5XX_SP_HS_CTRL_REG0_THREADSIZE__SHIFT			3
-static inline uint32_t A5XX_SP_HS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A5XX_SP_HS_CTRL_REG0_THREADSIZE__SHIFT) & A5XX_SP_HS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
-#define A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
-static inline uint32_t A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
-#define A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
-static inline uint32_t A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A5XX_SP_HS_CTRL_REG0_VARYING				0x00010000
-#define A5XX_SP_HS_CTRL_REG0_PIXLODENABLE			0x00100000
-#define A5XX_SP_HS_CTRL_REG0_BRANCHSTACK__MASK			0xfe000000
-#define A5XX_SP_HS_CTRL_REG0_BRANCHSTACK__SHIFT			25
-static inline uint32_t A5XX_SP_HS_CTRL_REG0_BRANCHSTACK(uint32_t val)
-{
-	return ((val) << A5XX_SP_HS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_HS_CTRL_REG0_BRANCHSTACK__MASK;
-}
-
-#define REG_A5XX_UNKNOWN_E602					0x0000e602
-
-#define REG_A5XX_SP_HS_OBJ_START_LO				0x0000e603
-
-#define REG_A5XX_SP_HS_OBJ_START_HI				0x0000e604
-
-#define REG_A5XX_SP_HS_PVT_MEM_PARAM				0x0000e605
-#define A5XX_SP_HS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK		0x000000ff
-#define A5XX_SP_HS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT		0
-static inline uint32_t A5XX_SP_HS_PVT_MEM_PARAM_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x1ff));
-	return (((val >> 9)) << A5XX_SP_HS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT) & A5XX_SP_HS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK;
-}
-#define A5XX_SP_HS_PVT_MEM_PARAM_HWSTACKOFFSET__MASK		0x00ffff00
-#define A5XX_SP_HS_PVT_MEM_PARAM_HWSTACKOFFSET__SHIFT		8
-static inline uint32_t A5XX_SP_HS_PVT_MEM_PARAM_HWSTACKOFFSET(uint32_t val)
-{
-	assert(!(val & 0x7ff));
-	return (((val >> 11)) << A5XX_SP_HS_PVT_MEM_PARAM_HWSTACKOFFSET__SHIFT) & A5XX_SP_HS_PVT_MEM_PARAM_HWSTACKOFFSET__MASK;
-}
-#define A5XX_SP_HS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A5XX_SP_HS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A5XX_SP_HS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A5XX_SP_HS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT) & A5XX_SP_HS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A5XX_SP_HS_PVT_MEM_ADDR				0x0000e606
-
-#define REG_A5XX_SP_HS_PVT_MEM_SIZE				0x0000e608
-#define A5XX_SP_HS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK		0x0003ffff
-#define A5XX_SP_HS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT		0
-static inline uint32_t A5XX_SP_HS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A5XX_SP_HS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT) & A5XX_SP_HS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK;
-}
-
-#define REG_A5XX_SP_DS_CTRL_REG0				0x0000e610
-#define A5XX_SP_DS_CTRL_REG0_BUFFER				0x00000004
-#define A5XX_SP_DS_CTRL_REG0_THREADSIZE__MASK			0x00000008
-#define A5XX_SP_DS_CTRL_REG0_THREADSIZE__SHIFT			3
-static inline uint32_t A5XX_SP_DS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A5XX_SP_DS_CTRL_REG0_THREADSIZE__SHIFT) & A5XX_SP_DS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
-#define A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
-static inline uint32_t A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
-#define A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
-static inline uint32_t A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A5XX_SP_DS_CTRL_REG0_VARYING				0x00010000
-#define A5XX_SP_DS_CTRL_REG0_PIXLODENABLE			0x00100000
-#define A5XX_SP_DS_CTRL_REG0_BRANCHSTACK__MASK			0xfe000000
-#define A5XX_SP_DS_CTRL_REG0_BRANCHSTACK__SHIFT			25
-static inline uint32_t A5XX_SP_DS_CTRL_REG0_BRANCHSTACK(uint32_t val)
-{
-	return ((val) << A5XX_SP_DS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_DS_CTRL_REG0_BRANCHSTACK__MASK;
-}
-
-#define REG_A5XX_UNKNOWN_E62B					0x0000e62b
-
-#define REG_A5XX_SP_DS_OBJ_START_LO				0x0000e62c
-
-#define REG_A5XX_SP_DS_OBJ_START_HI				0x0000e62d
-
-#define REG_A5XX_SP_DS_PVT_MEM_PARAM				0x0000e62e
-#define A5XX_SP_DS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK		0x000000ff
-#define A5XX_SP_DS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT		0
-static inline uint32_t A5XX_SP_DS_PVT_MEM_PARAM_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x1ff));
-	return (((val >> 9)) << A5XX_SP_DS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT) & A5XX_SP_DS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK;
-}
-#define A5XX_SP_DS_PVT_MEM_PARAM_HWSTACKOFFSET__MASK		0x00ffff00
-#define A5XX_SP_DS_PVT_MEM_PARAM_HWSTACKOFFSET__SHIFT		8
-static inline uint32_t A5XX_SP_DS_PVT_MEM_PARAM_HWSTACKOFFSET(uint32_t val)
-{
-	assert(!(val & 0x7ff));
-	return (((val >> 11)) << A5XX_SP_DS_PVT_MEM_PARAM_HWSTACKOFFSET__SHIFT) & A5XX_SP_DS_PVT_MEM_PARAM_HWSTACKOFFSET__MASK;
-}
-#define A5XX_SP_DS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A5XX_SP_DS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A5XX_SP_DS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A5XX_SP_DS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT) & A5XX_SP_DS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A5XX_SP_DS_PVT_MEM_ADDR				0x0000e62f
-
-#define REG_A5XX_SP_DS_PVT_MEM_SIZE				0x0000e631
-#define A5XX_SP_DS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK		0x0003ffff
-#define A5XX_SP_DS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT		0
-static inline uint32_t A5XX_SP_DS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A5XX_SP_DS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT) & A5XX_SP_DS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK;
-}
-
-#define REG_A5XX_SP_GS_CTRL_REG0				0x0000e640
-#define A5XX_SP_GS_CTRL_REG0_BUFFER				0x00000004
-#define A5XX_SP_GS_CTRL_REG0_THREADSIZE__MASK			0x00000008
-#define A5XX_SP_GS_CTRL_REG0_THREADSIZE__SHIFT			3
-static inline uint32_t A5XX_SP_GS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A5XX_SP_GS_CTRL_REG0_THREADSIZE__SHIFT) & A5XX_SP_GS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
-#define A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
-static inline uint32_t A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
-#define A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
-static inline uint32_t A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A5XX_SP_GS_CTRL_REG0_VARYING				0x00010000
-#define A5XX_SP_GS_CTRL_REG0_PIXLODENABLE			0x00100000
-#define A5XX_SP_GS_CTRL_REG0_BRANCHSTACK__MASK			0xfe000000
-#define A5XX_SP_GS_CTRL_REG0_BRANCHSTACK__SHIFT			25
-static inline uint32_t A5XX_SP_GS_CTRL_REG0_BRANCHSTACK(uint32_t val)
-{
-	return ((val) << A5XX_SP_GS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_GS_CTRL_REG0_BRANCHSTACK__MASK;
-}
-
-#define REG_A5XX_UNKNOWN_E65B					0x0000e65b
-
-#define REG_A5XX_SP_GS_OBJ_START_LO				0x0000e65c
-
-#define REG_A5XX_SP_GS_OBJ_START_HI				0x0000e65d
-
-#define REG_A5XX_SP_GS_PVT_MEM_PARAM				0x0000e65e
-#define A5XX_SP_GS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK		0x000000ff
-#define A5XX_SP_GS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT		0
-static inline uint32_t A5XX_SP_GS_PVT_MEM_PARAM_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x1ff));
-	return (((val >> 9)) << A5XX_SP_GS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT) & A5XX_SP_GS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK;
-}
-#define A5XX_SP_GS_PVT_MEM_PARAM_HWSTACKOFFSET__MASK		0x00ffff00
-#define A5XX_SP_GS_PVT_MEM_PARAM_HWSTACKOFFSET__SHIFT		8
-static inline uint32_t A5XX_SP_GS_PVT_MEM_PARAM_HWSTACKOFFSET(uint32_t val)
-{
-	assert(!(val & 0x7ff));
-	return (((val >> 11)) << A5XX_SP_GS_PVT_MEM_PARAM_HWSTACKOFFSET__SHIFT) & A5XX_SP_GS_PVT_MEM_PARAM_HWSTACKOFFSET__MASK;
-}
-#define A5XX_SP_GS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A5XX_SP_GS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A5XX_SP_GS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A5XX_SP_GS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT) & A5XX_SP_GS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A5XX_SP_GS_PVT_MEM_ADDR				0x0000e65f
-
-#define REG_A5XX_SP_GS_PVT_MEM_SIZE				0x0000e661
-#define A5XX_SP_GS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK		0x0003ffff
-#define A5XX_SP_GS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT		0
-static inline uint32_t A5XX_SP_GS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A5XX_SP_GS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT) & A5XX_SP_GS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK;
-}
-
-#define REG_A5XX_TPL1_TP_RAS_MSAA_CNTL				0x0000e704
-#define A5XX_TPL1_TP_RAS_MSAA_CNTL_SAMPLES__MASK		0x00000003
-#define A5XX_TPL1_TP_RAS_MSAA_CNTL_SAMPLES__SHIFT		0
-static inline uint32_t A5XX_TPL1_TP_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A5XX_TPL1_TP_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A5XX_TPL1_TP_RAS_MSAA_CNTL_SAMPLES__MASK;
-}
-
-#define REG_A5XX_TPL1_TP_DEST_MSAA_CNTL				0x0000e705
-#define A5XX_TPL1_TP_DEST_MSAA_CNTL_SAMPLES__MASK		0x00000003
-#define A5XX_TPL1_TP_DEST_MSAA_CNTL_SAMPLES__SHIFT		0
-static inline uint32_t A5XX_TPL1_TP_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A5XX_TPL1_TP_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A5XX_TPL1_TP_DEST_MSAA_CNTL_SAMPLES__MASK;
-}
-#define A5XX_TPL1_TP_DEST_MSAA_CNTL_MSAA_DISABLE		0x00000004
-
-#define REG_A5XX_TPL1_TP_BORDER_COLOR_BASE_ADDR_LO		0x0000e706
-
-#define REG_A5XX_TPL1_TP_BORDER_COLOR_BASE_ADDR_HI		0x0000e707
-
-#define REG_A5XX_TPL1_VS_TEX_COUNT				0x0000e700
-
-#define REG_A5XX_TPL1_HS_TEX_COUNT				0x0000e701
-
-#define REG_A5XX_TPL1_DS_TEX_COUNT				0x0000e702
-
-#define REG_A5XX_TPL1_GS_TEX_COUNT				0x0000e703
-
-#define REG_A5XX_TPL1_VS_TEX_SAMP_LO				0x0000e722
-
-#define REG_A5XX_TPL1_VS_TEX_SAMP_HI				0x0000e723
-
-#define REG_A5XX_TPL1_HS_TEX_SAMP_LO				0x0000e724
-
-#define REG_A5XX_TPL1_HS_TEX_SAMP_HI				0x0000e725
-
-#define REG_A5XX_TPL1_DS_TEX_SAMP_LO				0x0000e726
-
-#define REG_A5XX_TPL1_DS_TEX_SAMP_HI				0x0000e727
-
-#define REG_A5XX_TPL1_GS_TEX_SAMP_LO				0x0000e728
-
-#define REG_A5XX_TPL1_GS_TEX_SAMP_HI				0x0000e729
-
-#define REG_A5XX_TPL1_VS_TEX_CONST_LO				0x0000e72a
-
-#define REG_A5XX_TPL1_VS_TEX_CONST_HI				0x0000e72b
-
-#define REG_A5XX_TPL1_HS_TEX_CONST_LO				0x0000e72c
-
-#define REG_A5XX_TPL1_HS_TEX_CONST_HI				0x0000e72d
-
-#define REG_A5XX_TPL1_DS_TEX_CONST_LO				0x0000e72e
-
-#define REG_A5XX_TPL1_DS_TEX_CONST_HI				0x0000e72f
-
-#define REG_A5XX_TPL1_GS_TEX_CONST_LO				0x0000e730
-
-#define REG_A5XX_TPL1_GS_TEX_CONST_HI				0x0000e731
-
-#define REG_A5XX_TPL1_FS_TEX_COUNT				0x0000e750
-
-#define REG_A5XX_TPL1_CS_TEX_COUNT				0x0000e751
-
-#define REG_A5XX_TPL1_FS_TEX_SAMP_LO				0x0000e75a
-
-#define REG_A5XX_TPL1_FS_TEX_SAMP_HI				0x0000e75b
-
-#define REG_A5XX_TPL1_CS_TEX_SAMP_LO				0x0000e75c
-
-#define REG_A5XX_TPL1_CS_TEX_SAMP_HI				0x0000e75d
-
-#define REG_A5XX_TPL1_FS_TEX_CONST_LO				0x0000e75e
-
-#define REG_A5XX_TPL1_FS_TEX_CONST_HI				0x0000e75f
-
-#define REG_A5XX_TPL1_CS_TEX_CONST_LO				0x0000e760
-
-#define REG_A5XX_TPL1_CS_TEX_CONST_HI				0x0000e761
-
-#define REG_A5XX_TPL1_TP_FS_ROTATION_CNTL			0x0000e764
-
-#define REG_A5XX_HLSQ_CONTROL_0_REG				0x0000e784
-#define A5XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__MASK		0x00000001
-#define A5XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__SHIFT		0
-static inline uint32_t A5XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__SHIFT) & A5XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE__MASK;
-}
-#define A5XX_HLSQ_CONTROL_0_REG_CSTHREADSIZE__MASK		0x00000004
-#define A5XX_HLSQ_CONTROL_0_REG_CSTHREADSIZE__SHIFT		2
-static inline uint32_t A5XX_HLSQ_CONTROL_0_REG_CSTHREADSIZE(enum a3xx_threadsize val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_0_REG_CSTHREADSIZE__SHIFT) & A5XX_HLSQ_CONTROL_0_REG_CSTHREADSIZE__MASK;
-}
-
-#define REG_A5XX_HLSQ_CONTROL_1_REG				0x0000e785
-#define A5XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__MASK	0x0000003f
-#define A5XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__SHIFT	0
-static inline uint32_t A5XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__SHIFT) & A5XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__MASK;
-}
-
-#define REG_A5XX_HLSQ_CONTROL_2_REG				0x0000e786
-#define A5XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK			0x000000ff
-#define A5XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT		0
-static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_FACEREGID(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK;
-}
-#define A5XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK			0x0000ff00
-#define A5XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT			8
-static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_SAMPLEID(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK;
-}
-#define A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK		0x00ff0000
-#define A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT		16
-static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK;
-}
-#define A5XX_HLSQ_CONTROL_2_REG_CENTERRHW__MASK			0xff000000
-#define A5XX_HLSQ_CONTROL_2_REG_CENTERRHW__SHIFT		24
-static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_CENTERRHW(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_2_REG_CENTERRHW__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_CENTERRHW__MASK;
-}
-
-#define REG_A5XX_HLSQ_CONTROL_3_REG				0x0000e787
-#define A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__MASK		0x000000ff
-#define A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__SHIFT		0
-static inline uint32_t A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__SHIFT) & A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__MASK;
-}
-#define A5XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__MASK		0x0000ff00
-#define A5XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__SHIFT		8
-static inline uint32_t A5XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__SHIFT) & A5XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__MASK;
-}
-#define A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__MASK		0x00ff0000
-#define A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__SHIFT	16
-static inline uint32_t A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__SHIFT) & A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__MASK;
-}
-#define A5XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__MASK	0xff000000
-#define A5XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__SHIFT	24
-static inline uint32_t A5XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__SHIFT) & A5XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__MASK;
-}
-
-#define REG_A5XX_HLSQ_CONTROL_4_REG				0x0000e788
-#define A5XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__MASK		0x000000ff
-#define A5XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__SHIFT		0
-static inline uint32_t A5XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__SHIFT) & A5XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__MASK;
-}
-#define A5XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__MASK		0x0000ff00
-#define A5XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__SHIFT		8
-static inline uint32_t A5XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__SHIFT) & A5XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__MASK;
-}
-#define A5XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK		0x00ff0000
-#define A5XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT		16
-static inline uint32_t A5XX_HLSQ_CONTROL_4_REG_XYCOORDREGID(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT) & A5XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK;
-}
-#define A5XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK		0xff000000
-#define A5XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT		24
-static inline uint32_t A5XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT) & A5XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK;
-}
-
-#define REG_A5XX_HLSQ_UPDATE_CNTL				0x0000e78a
-
-#define REG_A5XX_HLSQ_VS_CONFIG					0x0000e78b
-#define A5XX_HLSQ_VS_CONFIG_ENABLED				0x00000001
-#define A5XX_HLSQ_VS_CONFIG_CONSTOBJECTOFFSET__MASK		0x000000fe
-#define A5XX_HLSQ_VS_CONFIG_CONSTOBJECTOFFSET__SHIFT		1
-static inline uint32_t A5XX_HLSQ_VS_CONFIG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_VS_CONFIG_CONSTOBJECTOFFSET__SHIFT) & A5XX_HLSQ_VS_CONFIG_CONSTOBJECTOFFSET__MASK;
-}
-#define A5XX_HLSQ_VS_CONFIG_SHADEROBJOFFSET__MASK		0x00007f00
-#define A5XX_HLSQ_VS_CONFIG_SHADEROBJOFFSET__SHIFT		8
-static inline uint32_t A5XX_HLSQ_VS_CONFIG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_VS_CONFIG_SHADEROBJOFFSET__SHIFT) & A5XX_HLSQ_VS_CONFIG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A5XX_HLSQ_FS_CONFIG					0x0000e78c
-#define A5XX_HLSQ_FS_CONFIG_ENABLED				0x00000001
-#define A5XX_HLSQ_FS_CONFIG_CONSTOBJECTOFFSET__MASK		0x000000fe
-#define A5XX_HLSQ_FS_CONFIG_CONSTOBJECTOFFSET__SHIFT		1
-static inline uint32_t A5XX_HLSQ_FS_CONFIG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_FS_CONFIG_CONSTOBJECTOFFSET__SHIFT) & A5XX_HLSQ_FS_CONFIG_CONSTOBJECTOFFSET__MASK;
-}
-#define A5XX_HLSQ_FS_CONFIG_SHADEROBJOFFSET__MASK		0x00007f00
-#define A5XX_HLSQ_FS_CONFIG_SHADEROBJOFFSET__SHIFT		8
-static inline uint32_t A5XX_HLSQ_FS_CONFIG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_FS_CONFIG_SHADEROBJOFFSET__SHIFT) & A5XX_HLSQ_FS_CONFIG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A5XX_HLSQ_HS_CONFIG					0x0000e78d
-#define A5XX_HLSQ_HS_CONFIG_ENABLED				0x00000001
-#define A5XX_HLSQ_HS_CONFIG_CONSTOBJECTOFFSET__MASK		0x000000fe
-#define A5XX_HLSQ_HS_CONFIG_CONSTOBJECTOFFSET__SHIFT		1
-static inline uint32_t A5XX_HLSQ_HS_CONFIG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_HS_CONFIG_CONSTOBJECTOFFSET__SHIFT) & A5XX_HLSQ_HS_CONFIG_CONSTOBJECTOFFSET__MASK;
-}
-#define A5XX_HLSQ_HS_CONFIG_SHADEROBJOFFSET__MASK		0x00007f00
-#define A5XX_HLSQ_HS_CONFIG_SHADEROBJOFFSET__SHIFT		8
-static inline uint32_t A5XX_HLSQ_HS_CONFIG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_HS_CONFIG_SHADEROBJOFFSET__SHIFT) & A5XX_HLSQ_HS_CONFIG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A5XX_HLSQ_DS_CONFIG					0x0000e78e
-#define A5XX_HLSQ_DS_CONFIG_ENABLED				0x00000001
-#define A5XX_HLSQ_DS_CONFIG_CONSTOBJECTOFFSET__MASK		0x000000fe
-#define A5XX_HLSQ_DS_CONFIG_CONSTOBJECTOFFSET__SHIFT		1
-static inline uint32_t A5XX_HLSQ_DS_CONFIG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_DS_CONFIG_CONSTOBJECTOFFSET__SHIFT) & A5XX_HLSQ_DS_CONFIG_CONSTOBJECTOFFSET__MASK;
-}
-#define A5XX_HLSQ_DS_CONFIG_SHADEROBJOFFSET__MASK		0x00007f00
-#define A5XX_HLSQ_DS_CONFIG_SHADEROBJOFFSET__SHIFT		8
-static inline uint32_t A5XX_HLSQ_DS_CONFIG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_DS_CONFIG_SHADEROBJOFFSET__SHIFT) & A5XX_HLSQ_DS_CONFIG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A5XX_HLSQ_GS_CONFIG					0x0000e78f
-#define A5XX_HLSQ_GS_CONFIG_ENABLED				0x00000001
-#define A5XX_HLSQ_GS_CONFIG_CONSTOBJECTOFFSET__MASK		0x000000fe
-#define A5XX_HLSQ_GS_CONFIG_CONSTOBJECTOFFSET__SHIFT		1
-static inline uint32_t A5XX_HLSQ_GS_CONFIG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_GS_CONFIG_CONSTOBJECTOFFSET__SHIFT) & A5XX_HLSQ_GS_CONFIG_CONSTOBJECTOFFSET__MASK;
-}
-#define A5XX_HLSQ_GS_CONFIG_SHADEROBJOFFSET__MASK		0x00007f00
-#define A5XX_HLSQ_GS_CONFIG_SHADEROBJOFFSET__SHIFT		8
-static inline uint32_t A5XX_HLSQ_GS_CONFIG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_GS_CONFIG_SHADEROBJOFFSET__SHIFT) & A5XX_HLSQ_GS_CONFIG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A5XX_HLSQ_CS_CONFIG					0x0000e790
-#define A5XX_HLSQ_CS_CONFIG_ENABLED				0x00000001
-#define A5XX_HLSQ_CS_CONFIG_CONSTOBJECTOFFSET__MASK		0x000000fe
-#define A5XX_HLSQ_CS_CONFIG_CONSTOBJECTOFFSET__SHIFT		1
-static inline uint32_t A5XX_HLSQ_CS_CONFIG_CONSTOBJECTOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_CONFIG_CONSTOBJECTOFFSET__SHIFT) & A5XX_HLSQ_CS_CONFIG_CONSTOBJECTOFFSET__MASK;
-}
-#define A5XX_HLSQ_CS_CONFIG_SHADEROBJOFFSET__MASK		0x00007f00
-#define A5XX_HLSQ_CS_CONFIG_SHADEROBJOFFSET__SHIFT		8
-static inline uint32_t A5XX_HLSQ_CS_CONFIG_SHADEROBJOFFSET(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_CONFIG_SHADEROBJOFFSET__SHIFT) & A5XX_HLSQ_CS_CONFIG_SHADEROBJOFFSET__MASK;
-}
-
-#define REG_A5XX_HLSQ_VS_CNTL					0x0000e791
-#define A5XX_HLSQ_VS_CNTL_SSBO_ENABLE				0x00000001
-#define A5XX_HLSQ_VS_CNTL_INSTRLEN__MASK			0xfffffffe
-#define A5XX_HLSQ_VS_CNTL_INSTRLEN__SHIFT			1
-static inline uint32_t A5XX_HLSQ_VS_CNTL_INSTRLEN(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_VS_CNTL_INSTRLEN__SHIFT) & A5XX_HLSQ_VS_CNTL_INSTRLEN__MASK;
-}
-
-#define REG_A5XX_HLSQ_FS_CNTL					0x0000e792
-#define A5XX_HLSQ_FS_CNTL_SSBO_ENABLE				0x00000001
-#define A5XX_HLSQ_FS_CNTL_INSTRLEN__MASK			0xfffffffe
-#define A5XX_HLSQ_FS_CNTL_INSTRLEN__SHIFT			1
-static inline uint32_t A5XX_HLSQ_FS_CNTL_INSTRLEN(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_FS_CNTL_INSTRLEN__SHIFT) & A5XX_HLSQ_FS_CNTL_INSTRLEN__MASK;
-}
-
-#define REG_A5XX_HLSQ_HS_CNTL					0x0000e793
-#define A5XX_HLSQ_HS_CNTL_SSBO_ENABLE				0x00000001
-#define A5XX_HLSQ_HS_CNTL_INSTRLEN__MASK			0xfffffffe
-#define A5XX_HLSQ_HS_CNTL_INSTRLEN__SHIFT			1
-static inline uint32_t A5XX_HLSQ_HS_CNTL_INSTRLEN(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_HS_CNTL_INSTRLEN__SHIFT) & A5XX_HLSQ_HS_CNTL_INSTRLEN__MASK;
-}
-
-#define REG_A5XX_HLSQ_DS_CNTL					0x0000e794
-#define A5XX_HLSQ_DS_CNTL_SSBO_ENABLE				0x00000001
-#define A5XX_HLSQ_DS_CNTL_INSTRLEN__MASK			0xfffffffe
-#define A5XX_HLSQ_DS_CNTL_INSTRLEN__SHIFT			1
-static inline uint32_t A5XX_HLSQ_DS_CNTL_INSTRLEN(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_DS_CNTL_INSTRLEN__SHIFT) & A5XX_HLSQ_DS_CNTL_INSTRLEN__MASK;
-}
-
-#define REG_A5XX_HLSQ_GS_CNTL					0x0000e795
-#define A5XX_HLSQ_GS_CNTL_SSBO_ENABLE				0x00000001
-#define A5XX_HLSQ_GS_CNTL_INSTRLEN__MASK			0xfffffffe
-#define A5XX_HLSQ_GS_CNTL_INSTRLEN__SHIFT			1
-static inline uint32_t A5XX_HLSQ_GS_CNTL_INSTRLEN(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_GS_CNTL_INSTRLEN__SHIFT) & A5XX_HLSQ_GS_CNTL_INSTRLEN__MASK;
-}
-
-#define REG_A5XX_HLSQ_CS_CNTL					0x0000e796
-#define A5XX_HLSQ_CS_CNTL_SSBO_ENABLE				0x00000001
-#define A5XX_HLSQ_CS_CNTL_INSTRLEN__MASK			0xfffffffe
-#define A5XX_HLSQ_CS_CNTL_INSTRLEN__SHIFT			1
-static inline uint32_t A5XX_HLSQ_CS_CNTL_INSTRLEN(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_CNTL_INSTRLEN__SHIFT) & A5XX_HLSQ_CS_CNTL_INSTRLEN__MASK;
-}
-
-#define REG_A5XX_HLSQ_CS_KERNEL_GROUP_X				0x0000e7b9
-
-#define REG_A5XX_HLSQ_CS_KERNEL_GROUP_Y				0x0000e7ba
-
-#define REG_A5XX_HLSQ_CS_KERNEL_GROUP_Z				0x0000e7bb
-
-#define REG_A5XX_HLSQ_CS_NDRANGE_0				0x0000e7b0
-#define A5XX_HLSQ_CS_NDRANGE_0_KERNELDIM__MASK			0x00000003
-#define A5XX_HLSQ_CS_NDRANGE_0_KERNELDIM__SHIFT			0
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_0_KERNELDIM(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_0_KERNELDIM__SHIFT) & A5XX_HLSQ_CS_NDRANGE_0_KERNELDIM__MASK;
-}
-#define A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__MASK			0x00000ffc
-#define A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__SHIFT		2
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__SHIFT) & A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__MASK;
-}
-#define A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__MASK			0x003ff000
-#define A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__SHIFT		12
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__SHIFT) & A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__MASK;
-}
-#define A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__MASK			0xffc00000
-#define A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__SHIFT		22
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__SHIFT) & A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__MASK;
-}
-
-#define REG_A5XX_HLSQ_CS_NDRANGE_1				0x0000e7b1
-#define A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK		0xffffffff
-#define A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT		0
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT) & A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK;
-}
-
-#define REG_A5XX_HLSQ_CS_NDRANGE_2				0x0000e7b2
-#define A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK		0xffffffff
-#define A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT		0
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT) & A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK;
-}
-
-#define REG_A5XX_HLSQ_CS_NDRANGE_3				0x0000e7b3
-#define A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK		0xffffffff
-#define A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT		0
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT) & A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK;
-}
-
-#define REG_A5XX_HLSQ_CS_NDRANGE_4				0x0000e7b4
-#define A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK		0xffffffff
-#define A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT		0
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT) & A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK;
-}
-
-#define REG_A5XX_HLSQ_CS_NDRANGE_5				0x0000e7b5
-#define A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK		0xffffffff
-#define A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT		0
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT) & A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK;
-}
-
-#define REG_A5XX_HLSQ_CS_NDRANGE_6				0x0000e7b6
-#define A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK		0xffffffff
-#define A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT		0
-static inline uint32_t A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT) & A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK;
-}
-
-#define REG_A5XX_HLSQ_CS_CNTL_0					0x0000e7b7
-#define A5XX_HLSQ_CS_CNTL_0_WGIDCONSTID__MASK			0x000000ff
-#define A5XX_HLSQ_CS_CNTL_0_WGIDCONSTID__SHIFT			0
-static inline uint32_t A5XX_HLSQ_CS_CNTL_0_WGIDCONSTID(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_CNTL_0_WGIDCONSTID__SHIFT) & A5XX_HLSQ_CS_CNTL_0_WGIDCONSTID__MASK;
-}
-#define A5XX_HLSQ_CS_CNTL_0_UNK0__MASK				0x0000ff00
-#define A5XX_HLSQ_CS_CNTL_0_UNK0__SHIFT				8
-static inline uint32_t A5XX_HLSQ_CS_CNTL_0_UNK0(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_CNTL_0_UNK0__SHIFT) & A5XX_HLSQ_CS_CNTL_0_UNK0__MASK;
-}
-#define A5XX_HLSQ_CS_CNTL_0_UNK1__MASK				0x00ff0000
-#define A5XX_HLSQ_CS_CNTL_0_UNK1__SHIFT				16
-static inline uint32_t A5XX_HLSQ_CS_CNTL_0_UNK1(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_CNTL_0_UNK1__SHIFT) & A5XX_HLSQ_CS_CNTL_0_UNK1__MASK;
-}
-#define A5XX_HLSQ_CS_CNTL_0_LOCALIDREGID__MASK			0xff000000
-#define A5XX_HLSQ_CS_CNTL_0_LOCALIDREGID__SHIFT			24
-static inline uint32_t A5XX_HLSQ_CS_CNTL_0_LOCALIDREGID(uint32_t val)
-{
-	return ((val) << A5XX_HLSQ_CS_CNTL_0_LOCALIDREGID__SHIFT) & A5XX_HLSQ_CS_CNTL_0_LOCALIDREGID__MASK;
-}
-
-#define REG_A5XX_HLSQ_CS_CNTL_1					0x0000e7b8
-
-#define REG_A5XX_UNKNOWN_E7C0					0x0000e7c0
-
-#define REG_A5XX_HLSQ_VS_CONSTLEN				0x0000e7c3
-
-#define REG_A5XX_HLSQ_VS_INSTRLEN				0x0000e7c4
-
-#define REG_A5XX_UNKNOWN_E7C5					0x0000e7c5
-
-#define REG_A5XX_HLSQ_HS_CONSTLEN				0x0000e7c8
-
-#define REG_A5XX_HLSQ_HS_INSTRLEN				0x0000e7c9
-
-#define REG_A5XX_UNKNOWN_E7CA					0x0000e7ca
-
-#define REG_A5XX_HLSQ_DS_CONSTLEN				0x0000e7cd
-
-#define REG_A5XX_HLSQ_DS_INSTRLEN				0x0000e7ce
-
-#define REG_A5XX_UNKNOWN_E7CF					0x0000e7cf
-
-#define REG_A5XX_HLSQ_GS_CONSTLEN				0x0000e7d2
-
-#define REG_A5XX_HLSQ_GS_INSTRLEN				0x0000e7d3
-
-#define REG_A5XX_UNKNOWN_E7D4					0x0000e7d4
-
-#define REG_A5XX_HLSQ_FS_CONSTLEN				0x0000e7d7
-
-#define REG_A5XX_HLSQ_FS_INSTRLEN				0x0000e7d8
-
-#define REG_A5XX_UNKNOWN_E7D9					0x0000e7d9
-
-#define REG_A5XX_HLSQ_CS_CONSTLEN				0x0000e7dc
-
-#define REG_A5XX_HLSQ_CS_INSTRLEN				0x0000e7dd
-
-#define REG_A5XX_RB_2D_BLIT_CNTL				0x00002100
-
-#define REG_A5XX_RB_2D_SRC_SOLID_DW0				0x00002101
-
-#define REG_A5XX_RB_2D_SRC_SOLID_DW1				0x00002102
-
-#define REG_A5XX_RB_2D_SRC_SOLID_DW2				0x00002103
-
-#define REG_A5XX_RB_2D_SRC_SOLID_DW3				0x00002104
-
-#define REG_A5XX_RB_2D_SRC_INFO					0x00002107
-#define A5XX_RB_2D_SRC_INFO_COLOR_FORMAT__MASK			0x000000ff
-#define A5XX_RB_2D_SRC_INFO_COLOR_FORMAT__SHIFT			0
-static inline uint32_t A5XX_RB_2D_SRC_INFO_COLOR_FORMAT(enum a5xx_color_fmt val)
-{
-	return ((val) << A5XX_RB_2D_SRC_INFO_COLOR_FORMAT__SHIFT) & A5XX_RB_2D_SRC_INFO_COLOR_FORMAT__MASK;
-}
-#define A5XX_RB_2D_SRC_INFO_TILE_MODE__MASK			0x00000300
-#define A5XX_RB_2D_SRC_INFO_TILE_MODE__SHIFT			8
-static inline uint32_t A5XX_RB_2D_SRC_INFO_TILE_MODE(enum a5xx_tile_mode val)
-{
-	return ((val) << A5XX_RB_2D_SRC_INFO_TILE_MODE__SHIFT) & A5XX_RB_2D_SRC_INFO_TILE_MODE__MASK;
-}
-#define A5XX_RB_2D_SRC_INFO_COLOR_SWAP__MASK			0x00000c00
-#define A5XX_RB_2D_SRC_INFO_COLOR_SWAP__SHIFT			10
-static inline uint32_t A5XX_RB_2D_SRC_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A5XX_RB_2D_SRC_INFO_COLOR_SWAP__SHIFT) & A5XX_RB_2D_SRC_INFO_COLOR_SWAP__MASK;
-}
-#define A5XX_RB_2D_SRC_INFO_FLAGS				0x00001000
-#define A5XX_RB_2D_SRC_INFO_SRGB				0x00002000
-
-#define REG_A5XX_RB_2D_SRC_LO					0x00002108
-
-#define REG_A5XX_RB_2D_SRC_HI					0x00002109
-
-#define REG_A5XX_RB_2D_SRC_SIZE					0x0000210a
-#define A5XX_RB_2D_SRC_SIZE_PITCH__MASK				0x0000ffff
-#define A5XX_RB_2D_SRC_SIZE_PITCH__SHIFT			0
-static inline uint32_t A5XX_RB_2D_SRC_SIZE_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_2D_SRC_SIZE_PITCH__SHIFT) & A5XX_RB_2D_SRC_SIZE_PITCH__MASK;
-}
-#define A5XX_RB_2D_SRC_SIZE_ARRAY_PITCH__MASK			0xffff0000
-#define A5XX_RB_2D_SRC_SIZE_ARRAY_PITCH__SHIFT			16
-static inline uint32_t A5XX_RB_2D_SRC_SIZE_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_2D_SRC_SIZE_ARRAY_PITCH__SHIFT) & A5XX_RB_2D_SRC_SIZE_ARRAY_PITCH__MASK;
-}
-
-#define REG_A5XX_RB_2D_DST_INFO					0x00002110
-#define A5XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK			0x000000ff
-#define A5XX_RB_2D_DST_INFO_COLOR_FORMAT__SHIFT			0
-static inline uint32_t A5XX_RB_2D_DST_INFO_COLOR_FORMAT(enum a5xx_color_fmt val)
-{
-	return ((val) << A5XX_RB_2D_DST_INFO_COLOR_FORMAT__SHIFT) & A5XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK;
-}
-#define A5XX_RB_2D_DST_INFO_TILE_MODE__MASK			0x00000300
-#define A5XX_RB_2D_DST_INFO_TILE_MODE__SHIFT			8
-static inline uint32_t A5XX_RB_2D_DST_INFO_TILE_MODE(enum a5xx_tile_mode val)
-{
-	return ((val) << A5XX_RB_2D_DST_INFO_TILE_MODE__SHIFT) & A5XX_RB_2D_DST_INFO_TILE_MODE__MASK;
-}
-#define A5XX_RB_2D_DST_INFO_COLOR_SWAP__MASK			0x00000c00
-#define A5XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT			10
-static inline uint32_t A5XX_RB_2D_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A5XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT) & A5XX_RB_2D_DST_INFO_COLOR_SWAP__MASK;
-}
-#define A5XX_RB_2D_DST_INFO_FLAGS				0x00001000
-#define A5XX_RB_2D_DST_INFO_SRGB				0x00002000
-
-#define REG_A5XX_RB_2D_DST_LO					0x00002111
-
-#define REG_A5XX_RB_2D_DST_HI					0x00002112
-
-#define REG_A5XX_RB_2D_DST_SIZE					0x00002113
-#define A5XX_RB_2D_DST_SIZE_PITCH__MASK				0x0000ffff
-#define A5XX_RB_2D_DST_SIZE_PITCH__SHIFT			0
-static inline uint32_t A5XX_RB_2D_DST_SIZE_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_2D_DST_SIZE_PITCH__SHIFT) & A5XX_RB_2D_DST_SIZE_PITCH__MASK;
-}
-#define A5XX_RB_2D_DST_SIZE_ARRAY_PITCH__MASK			0xffff0000
-#define A5XX_RB_2D_DST_SIZE_ARRAY_PITCH__SHIFT			16
-static inline uint32_t A5XX_RB_2D_DST_SIZE_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_2D_DST_SIZE_ARRAY_PITCH__SHIFT) & A5XX_RB_2D_DST_SIZE_ARRAY_PITCH__MASK;
-}
-
-#define REG_A5XX_RB_2D_SRC_FLAGS_LO				0x00002140
-
-#define REG_A5XX_RB_2D_SRC_FLAGS_HI				0x00002141
-
-#define REG_A5XX_RB_2D_SRC_FLAGS_PITCH				0x00002142
-#define A5XX_RB_2D_SRC_FLAGS_PITCH__MASK			0xffffffff
-#define A5XX_RB_2D_SRC_FLAGS_PITCH__SHIFT			0
-static inline uint32_t A5XX_RB_2D_SRC_FLAGS_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_2D_SRC_FLAGS_PITCH__SHIFT) & A5XX_RB_2D_SRC_FLAGS_PITCH__MASK;
-}
-
-#define REG_A5XX_RB_2D_DST_FLAGS_LO				0x00002143
-
-#define REG_A5XX_RB_2D_DST_FLAGS_HI				0x00002144
-
-#define REG_A5XX_RB_2D_DST_FLAGS_PITCH				0x00002145
-#define A5XX_RB_2D_DST_FLAGS_PITCH__MASK			0xffffffff
-#define A5XX_RB_2D_DST_FLAGS_PITCH__SHIFT			0
-static inline uint32_t A5XX_RB_2D_DST_FLAGS_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A5XX_RB_2D_DST_FLAGS_PITCH__SHIFT) & A5XX_RB_2D_DST_FLAGS_PITCH__MASK;
-}
-
-#define REG_A5XX_GRAS_2D_BLIT_CNTL				0x00002180
-
-#define REG_A5XX_GRAS_2D_SRC_INFO				0x00002181
-#define A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__MASK		0x000000ff
-#define A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__SHIFT		0
-static inline uint32_t A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT(enum a5xx_color_fmt val)
-{
-	return ((val) << A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__SHIFT) & A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__MASK;
-}
-#define A5XX_GRAS_2D_SRC_INFO_TILE_MODE__MASK			0x00000300
-#define A5XX_GRAS_2D_SRC_INFO_TILE_MODE__SHIFT			8
-static inline uint32_t A5XX_GRAS_2D_SRC_INFO_TILE_MODE(enum a5xx_tile_mode val)
-{
-	return ((val) << A5XX_GRAS_2D_SRC_INFO_TILE_MODE__SHIFT) & A5XX_GRAS_2D_SRC_INFO_TILE_MODE__MASK;
-}
-#define A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__MASK			0x00000c00
-#define A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__SHIFT			10
-static inline uint32_t A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__SHIFT) & A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__MASK;
-}
-#define A5XX_GRAS_2D_SRC_INFO_FLAGS				0x00001000
-#define A5XX_GRAS_2D_SRC_INFO_SRGB				0x00002000
-
-#define REG_A5XX_GRAS_2D_DST_INFO				0x00002182
-#define A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__MASK		0x000000ff
-#define A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__SHIFT		0
-static inline uint32_t A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT(enum a5xx_color_fmt val)
-{
-	return ((val) << A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__SHIFT) & A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__MASK;
-}
-#define A5XX_GRAS_2D_DST_INFO_TILE_MODE__MASK			0x00000300
-#define A5XX_GRAS_2D_DST_INFO_TILE_MODE__SHIFT			8
-static inline uint32_t A5XX_GRAS_2D_DST_INFO_TILE_MODE(enum a5xx_tile_mode val)
-{
-	return ((val) << A5XX_GRAS_2D_DST_INFO_TILE_MODE__SHIFT) & A5XX_GRAS_2D_DST_INFO_TILE_MODE__MASK;
-}
-#define A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__MASK			0x00000c00
-#define A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__SHIFT			10
-static inline uint32_t A5XX_GRAS_2D_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__SHIFT) & A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__MASK;
-}
-#define A5XX_GRAS_2D_DST_INFO_FLAGS				0x00001000
-#define A5XX_GRAS_2D_DST_INFO_SRGB				0x00002000
-
-#define REG_A5XX_UNKNOWN_2184					0x00002184
-
-#define REG_A5XX_TEX_SAMP_0					0x00000000
-#define A5XX_TEX_SAMP_0_MIPFILTER_LINEAR_NEAR			0x00000001
-#define A5XX_TEX_SAMP_0_XY_MAG__MASK				0x00000006
-#define A5XX_TEX_SAMP_0_XY_MAG__SHIFT				1
-static inline uint32_t A5XX_TEX_SAMP_0_XY_MAG(enum a5xx_tex_filter val)
-{
-	return ((val) << A5XX_TEX_SAMP_0_XY_MAG__SHIFT) & A5XX_TEX_SAMP_0_XY_MAG__MASK;
-}
-#define A5XX_TEX_SAMP_0_XY_MIN__MASK				0x00000018
-#define A5XX_TEX_SAMP_0_XY_MIN__SHIFT				3
-static inline uint32_t A5XX_TEX_SAMP_0_XY_MIN(enum a5xx_tex_filter val)
-{
-	return ((val) << A5XX_TEX_SAMP_0_XY_MIN__SHIFT) & A5XX_TEX_SAMP_0_XY_MIN__MASK;
-}
-#define A5XX_TEX_SAMP_0_WRAP_S__MASK				0x000000e0
-#define A5XX_TEX_SAMP_0_WRAP_S__SHIFT				5
-static inline uint32_t A5XX_TEX_SAMP_0_WRAP_S(enum a5xx_tex_clamp val)
-{
-	return ((val) << A5XX_TEX_SAMP_0_WRAP_S__SHIFT) & A5XX_TEX_SAMP_0_WRAP_S__MASK;
-}
-#define A5XX_TEX_SAMP_0_WRAP_T__MASK				0x00000700
-#define A5XX_TEX_SAMP_0_WRAP_T__SHIFT				8
-static inline uint32_t A5XX_TEX_SAMP_0_WRAP_T(enum a5xx_tex_clamp val)
-{
-	return ((val) << A5XX_TEX_SAMP_0_WRAP_T__SHIFT) & A5XX_TEX_SAMP_0_WRAP_T__MASK;
-}
-#define A5XX_TEX_SAMP_0_WRAP_R__MASK				0x00003800
-#define A5XX_TEX_SAMP_0_WRAP_R__SHIFT				11
-static inline uint32_t A5XX_TEX_SAMP_0_WRAP_R(enum a5xx_tex_clamp val)
-{
-	return ((val) << A5XX_TEX_SAMP_0_WRAP_R__SHIFT) & A5XX_TEX_SAMP_0_WRAP_R__MASK;
-}
-#define A5XX_TEX_SAMP_0_ANISO__MASK				0x0001c000
-#define A5XX_TEX_SAMP_0_ANISO__SHIFT				14
-static inline uint32_t A5XX_TEX_SAMP_0_ANISO(enum a5xx_tex_aniso val)
-{
-	return ((val) << A5XX_TEX_SAMP_0_ANISO__SHIFT) & A5XX_TEX_SAMP_0_ANISO__MASK;
-}
-#define A5XX_TEX_SAMP_0_LOD_BIAS__MASK				0xfff80000
-#define A5XX_TEX_SAMP_0_LOD_BIAS__SHIFT				19
-static inline uint32_t A5XX_TEX_SAMP_0_LOD_BIAS(float val)
-{
-	return ((((int32_t)(val * 256.0))) << A5XX_TEX_SAMP_0_LOD_BIAS__SHIFT) & A5XX_TEX_SAMP_0_LOD_BIAS__MASK;
-}
-
-#define REG_A5XX_TEX_SAMP_1					0x00000001
-#define A5XX_TEX_SAMP_1_COMPARE_FUNC__MASK			0x0000000e
-#define A5XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT			1
-static inline uint32_t A5XX_TEX_SAMP_1_COMPARE_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A5XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT) & A5XX_TEX_SAMP_1_COMPARE_FUNC__MASK;
-}
-#define A5XX_TEX_SAMP_1_CUBEMAPSEAMLESSFILTOFF			0x00000010
-#define A5XX_TEX_SAMP_1_UNNORM_COORDS				0x00000020
-#define A5XX_TEX_SAMP_1_MIPFILTER_LINEAR_FAR			0x00000040
-#define A5XX_TEX_SAMP_1_MAX_LOD__MASK				0x000fff00
-#define A5XX_TEX_SAMP_1_MAX_LOD__SHIFT				8
-static inline uint32_t A5XX_TEX_SAMP_1_MAX_LOD(float val)
-{
-	return ((((uint32_t)(val * 256.0))) << A5XX_TEX_SAMP_1_MAX_LOD__SHIFT) & A5XX_TEX_SAMP_1_MAX_LOD__MASK;
-}
-#define A5XX_TEX_SAMP_1_MIN_LOD__MASK				0xfff00000
-#define A5XX_TEX_SAMP_1_MIN_LOD__SHIFT				20
-static inline uint32_t A5XX_TEX_SAMP_1_MIN_LOD(float val)
-{
-	return ((((uint32_t)(val * 256.0))) << A5XX_TEX_SAMP_1_MIN_LOD__SHIFT) & A5XX_TEX_SAMP_1_MIN_LOD__MASK;
-}
-
-#define REG_A5XX_TEX_SAMP_2					0x00000002
-#define A5XX_TEX_SAMP_2_BCOLOR_OFFSET__MASK			0xffffff80
-#define A5XX_TEX_SAMP_2_BCOLOR_OFFSET__SHIFT			7
-static inline uint32_t A5XX_TEX_SAMP_2_BCOLOR_OFFSET(uint32_t val)
-{
-	return ((val) << A5XX_TEX_SAMP_2_BCOLOR_OFFSET__SHIFT) & A5XX_TEX_SAMP_2_BCOLOR_OFFSET__MASK;
-}
-
-#define REG_A5XX_TEX_SAMP_3					0x00000003
-
-#define REG_A5XX_TEX_CONST_0					0x00000000
-#define A5XX_TEX_CONST_0_TILE_MODE__MASK			0x00000003
-#define A5XX_TEX_CONST_0_TILE_MODE__SHIFT			0
-static inline uint32_t A5XX_TEX_CONST_0_TILE_MODE(enum a5xx_tile_mode val)
-{
-	return ((val) << A5XX_TEX_CONST_0_TILE_MODE__SHIFT) & A5XX_TEX_CONST_0_TILE_MODE__MASK;
-}
-#define A5XX_TEX_CONST_0_SRGB					0x00000004
-#define A5XX_TEX_CONST_0_SWIZ_X__MASK				0x00000070
-#define A5XX_TEX_CONST_0_SWIZ_X__SHIFT				4
-static inline uint32_t A5XX_TEX_CONST_0_SWIZ_X(enum a5xx_tex_swiz val)
-{
-	return ((val) << A5XX_TEX_CONST_0_SWIZ_X__SHIFT) & A5XX_TEX_CONST_0_SWIZ_X__MASK;
-}
-#define A5XX_TEX_CONST_0_SWIZ_Y__MASK				0x00000380
-#define A5XX_TEX_CONST_0_SWIZ_Y__SHIFT				7
-static inline uint32_t A5XX_TEX_CONST_0_SWIZ_Y(enum a5xx_tex_swiz val)
-{
-	return ((val) << A5XX_TEX_CONST_0_SWIZ_Y__SHIFT) & A5XX_TEX_CONST_0_SWIZ_Y__MASK;
-}
-#define A5XX_TEX_CONST_0_SWIZ_Z__MASK				0x00001c00
-#define A5XX_TEX_CONST_0_SWIZ_Z__SHIFT				10
-static inline uint32_t A5XX_TEX_CONST_0_SWIZ_Z(enum a5xx_tex_swiz val)
-{
-	return ((val) << A5XX_TEX_CONST_0_SWIZ_Z__SHIFT) & A5XX_TEX_CONST_0_SWIZ_Z__MASK;
-}
-#define A5XX_TEX_CONST_0_SWIZ_W__MASK				0x0000e000
-#define A5XX_TEX_CONST_0_SWIZ_W__SHIFT				13
-static inline uint32_t A5XX_TEX_CONST_0_SWIZ_W(enum a5xx_tex_swiz val)
-{
-	return ((val) << A5XX_TEX_CONST_0_SWIZ_W__SHIFT) & A5XX_TEX_CONST_0_SWIZ_W__MASK;
-}
-#define A5XX_TEX_CONST_0_MIPLVLS__MASK				0x000f0000
-#define A5XX_TEX_CONST_0_MIPLVLS__SHIFT				16
-static inline uint32_t A5XX_TEX_CONST_0_MIPLVLS(uint32_t val)
-{
-	return ((val) << A5XX_TEX_CONST_0_MIPLVLS__SHIFT) & A5XX_TEX_CONST_0_MIPLVLS__MASK;
-}
-#define A5XX_TEX_CONST_0_SAMPLES__MASK				0x00300000
-#define A5XX_TEX_CONST_0_SAMPLES__SHIFT				20
-static inline uint32_t A5XX_TEX_CONST_0_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A5XX_TEX_CONST_0_SAMPLES__SHIFT) & A5XX_TEX_CONST_0_SAMPLES__MASK;
-}
-#define A5XX_TEX_CONST_0_FMT__MASK				0x3fc00000
-#define A5XX_TEX_CONST_0_FMT__SHIFT				22
-static inline uint32_t A5XX_TEX_CONST_0_FMT(enum a5xx_tex_fmt val)
-{
-	return ((val) << A5XX_TEX_CONST_0_FMT__SHIFT) & A5XX_TEX_CONST_0_FMT__MASK;
-}
-#define A5XX_TEX_CONST_0_SWAP__MASK				0xc0000000
-#define A5XX_TEX_CONST_0_SWAP__SHIFT				30
-static inline uint32_t A5XX_TEX_CONST_0_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A5XX_TEX_CONST_0_SWAP__SHIFT) & A5XX_TEX_CONST_0_SWAP__MASK;
-}
-
-#define REG_A5XX_TEX_CONST_1					0x00000001
-#define A5XX_TEX_CONST_1_WIDTH__MASK				0x00007fff
-#define A5XX_TEX_CONST_1_WIDTH__SHIFT				0
-static inline uint32_t A5XX_TEX_CONST_1_WIDTH(uint32_t val)
-{
-	return ((val) << A5XX_TEX_CONST_1_WIDTH__SHIFT) & A5XX_TEX_CONST_1_WIDTH__MASK;
-}
-#define A5XX_TEX_CONST_1_HEIGHT__MASK				0x3fff8000
-#define A5XX_TEX_CONST_1_HEIGHT__SHIFT				15
-static inline uint32_t A5XX_TEX_CONST_1_HEIGHT(uint32_t val)
-{
-	return ((val) << A5XX_TEX_CONST_1_HEIGHT__SHIFT) & A5XX_TEX_CONST_1_HEIGHT__MASK;
-}
-
-#define REG_A5XX_TEX_CONST_2					0x00000002
-#define A5XX_TEX_CONST_2_BUFFER					0x00000010
-#define A5XX_TEX_CONST_2_PITCHALIGN__MASK			0x0000000f
-#define A5XX_TEX_CONST_2_PITCHALIGN__SHIFT			0
-static inline uint32_t A5XX_TEX_CONST_2_PITCHALIGN(uint32_t val)
-{
-	return ((val) << A5XX_TEX_CONST_2_PITCHALIGN__SHIFT) & A5XX_TEX_CONST_2_PITCHALIGN__MASK;
-}
-#define A5XX_TEX_CONST_2_PITCH__MASK				0x1fffff80
-#define A5XX_TEX_CONST_2_PITCH__SHIFT				7
-static inline uint32_t A5XX_TEX_CONST_2_PITCH(uint32_t val)
-{
-	return ((val) << A5XX_TEX_CONST_2_PITCH__SHIFT) & A5XX_TEX_CONST_2_PITCH__MASK;
-}
-#define A5XX_TEX_CONST_2_TYPE__MASK				0xe0000000
-#define A5XX_TEX_CONST_2_TYPE__SHIFT				29
-static inline uint32_t A5XX_TEX_CONST_2_TYPE(enum a5xx_tex_type val)
-{
-	return ((val) << A5XX_TEX_CONST_2_TYPE__SHIFT) & A5XX_TEX_CONST_2_TYPE__MASK;
-}
-
-#define REG_A5XX_TEX_CONST_3					0x00000003
-#define A5XX_TEX_CONST_3_ARRAY_PITCH__MASK			0x00003fff
-#define A5XX_TEX_CONST_3_ARRAY_PITCH__SHIFT			0
-static inline uint32_t A5XX_TEX_CONST_3_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A5XX_TEX_CONST_3_ARRAY_PITCH__SHIFT) & A5XX_TEX_CONST_3_ARRAY_PITCH__MASK;
-}
-#define A5XX_TEX_CONST_3_MIN_LAYERSZ__MASK			0x07800000
-#define A5XX_TEX_CONST_3_MIN_LAYERSZ__SHIFT			23
-static inline uint32_t A5XX_TEX_CONST_3_MIN_LAYERSZ(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A5XX_TEX_CONST_3_MIN_LAYERSZ__SHIFT) & A5XX_TEX_CONST_3_MIN_LAYERSZ__MASK;
-}
-#define A5XX_TEX_CONST_3_TILE_ALL				0x08000000
-#define A5XX_TEX_CONST_3_FLAG					0x10000000
-
-#define REG_A5XX_TEX_CONST_4					0x00000004
-#define A5XX_TEX_CONST_4_BASE_LO__MASK				0xffffffe0
-#define A5XX_TEX_CONST_4_BASE_LO__SHIFT				5
-static inline uint32_t A5XX_TEX_CONST_4_BASE_LO(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A5XX_TEX_CONST_4_BASE_LO__SHIFT) & A5XX_TEX_CONST_4_BASE_LO__MASK;
-}
-
-#define REG_A5XX_TEX_CONST_5					0x00000005
-#define A5XX_TEX_CONST_5_BASE_HI__MASK				0x0001ffff
-#define A5XX_TEX_CONST_5_BASE_HI__SHIFT				0
-static inline uint32_t A5XX_TEX_CONST_5_BASE_HI(uint32_t val)
-{
-	return ((val) << A5XX_TEX_CONST_5_BASE_HI__SHIFT) & A5XX_TEX_CONST_5_BASE_HI__MASK;
-}
-#define A5XX_TEX_CONST_5_DEPTH__MASK				0x3ffe0000
-#define A5XX_TEX_CONST_5_DEPTH__SHIFT				17
-static inline uint32_t A5XX_TEX_CONST_5_DEPTH(uint32_t val)
-{
-	return ((val) << A5XX_TEX_CONST_5_DEPTH__SHIFT) & A5XX_TEX_CONST_5_DEPTH__MASK;
-}
-
-#define REG_A5XX_TEX_CONST_6					0x00000006
-
-#define REG_A5XX_TEX_CONST_7					0x00000007
-
-#define REG_A5XX_TEX_CONST_8					0x00000008
-
-#define REG_A5XX_TEX_CONST_9					0x00000009
-
-#define REG_A5XX_TEX_CONST_10					0x0000000a
-
-#define REG_A5XX_TEX_CONST_11					0x0000000b
-
-#define REG_A5XX_SSBO_0_0					0x00000000
-#define A5XX_SSBO_0_0_BASE_LO__MASK				0xffffffe0
-#define A5XX_SSBO_0_0_BASE_LO__SHIFT				5
-static inline uint32_t A5XX_SSBO_0_0_BASE_LO(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A5XX_SSBO_0_0_BASE_LO__SHIFT) & A5XX_SSBO_0_0_BASE_LO__MASK;
-}
-
-#define REG_A5XX_SSBO_0_1					0x00000001
-#define A5XX_SSBO_0_1_PITCH__MASK				0x003fffff
-#define A5XX_SSBO_0_1_PITCH__SHIFT				0
-static inline uint32_t A5XX_SSBO_0_1_PITCH(uint32_t val)
-{
-	return ((val) << A5XX_SSBO_0_1_PITCH__SHIFT) & A5XX_SSBO_0_1_PITCH__MASK;
-}
-
-#define REG_A5XX_SSBO_0_2					0x00000002
-#define A5XX_SSBO_0_2_ARRAY_PITCH__MASK				0x03fff000
-#define A5XX_SSBO_0_2_ARRAY_PITCH__SHIFT			12
-static inline uint32_t A5XX_SSBO_0_2_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A5XX_SSBO_0_2_ARRAY_PITCH__SHIFT) & A5XX_SSBO_0_2_ARRAY_PITCH__MASK;
-}
-
-#define REG_A5XX_SSBO_0_3					0x00000003
-#define A5XX_SSBO_0_3_CPP__MASK					0x0000003f
-#define A5XX_SSBO_0_3_CPP__SHIFT				0
-static inline uint32_t A5XX_SSBO_0_3_CPP(uint32_t val)
-{
-	return ((val) << A5XX_SSBO_0_3_CPP__SHIFT) & A5XX_SSBO_0_3_CPP__MASK;
-}
-
-#define REG_A5XX_SSBO_1_0					0x00000000
-#define A5XX_SSBO_1_0_FMT__MASK					0x0000ff00
-#define A5XX_SSBO_1_0_FMT__SHIFT				8
-static inline uint32_t A5XX_SSBO_1_0_FMT(enum a5xx_tex_fmt val)
-{
-	return ((val) << A5XX_SSBO_1_0_FMT__SHIFT) & A5XX_SSBO_1_0_FMT__MASK;
-}
-#define A5XX_SSBO_1_0_WIDTH__MASK				0xffff0000
-#define A5XX_SSBO_1_0_WIDTH__SHIFT				16
-static inline uint32_t A5XX_SSBO_1_0_WIDTH(uint32_t val)
-{
-	return ((val) << A5XX_SSBO_1_0_WIDTH__SHIFT) & A5XX_SSBO_1_0_WIDTH__MASK;
-}
-
-#define REG_A5XX_SSBO_1_1					0x00000001
-#define A5XX_SSBO_1_1_HEIGHT__MASK				0x0000ffff
-#define A5XX_SSBO_1_1_HEIGHT__SHIFT				0
-static inline uint32_t A5XX_SSBO_1_1_HEIGHT(uint32_t val)
-{
-	return ((val) << A5XX_SSBO_1_1_HEIGHT__SHIFT) & A5XX_SSBO_1_1_HEIGHT__MASK;
-}
-#define A5XX_SSBO_1_1_DEPTH__MASK				0xffff0000
-#define A5XX_SSBO_1_1_DEPTH__SHIFT				16
-static inline uint32_t A5XX_SSBO_1_1_DEPTH(uint32_t val)
-{
-	return ((val) << A5XX_SSBO_1_1_DEPTH__SHIFT) & A5XX_SSBO_1_1_DEPTH__MASK;
-}
-
-#define REG_A5XX_SSBO_2_0					0x00000000
-#define A5XX_SSBO_2_0_BASE_LO__MASK				0xffffffff
-#define A5XX_SSBO_2_0_BASE_LO__SHIFT				0
-static inline uint32_t A5XX_SSBO_2_0_BASE_LO(uint32_t val)
-{
-	return ((val) << A5XX_SSBO_2_0_BASE_LO__SHIFT) & A5XX_SSBO_2_0_BASE_LO__MASK;
-}
-
-#define REG_A5XX_SSBO_2_1					0x00000001
-#define A5XX_SSBO_2_1_BASE_HI__MASK				0xffffffff
-#define A5XX_SSBO_2_1_BASE_HI__SHIFT				0
-static inline uint32_t A5XX_SSBO_2_1_BASE_HI(uint32_t val)
-{
-	return ((val) << A5XX_SSBO_2_1_BASE_HI__SHIFT) & A5XX_SSBO_2_1_BASE_HI__MASK;
-}
-
-#define REG_A5XX_UBO_0						0x00000000
-#define A5XX_UBO_0_BASE_LO__MASK				0xffffffff
-#define A5XX_UBO_0_BASE_LO__SHIFT				0
-static inline uint32_t A5XX_UBO_0_BASE_LO(uint32_t val)
-{
-	return ((val) << A5XX_UBO_0_BASE_LO__SHIFT) & A5XX_UBO_0_BASE_LO__MASK;
-}
-
-#define REG_A5XX_UBO_1						0x00000001
-#define A5XX_UBO_1_BASE_HI__MASK				0x0001ffff
-#define A5XX_UBO_1_BASE_HI__SHIFT				0
-static inline uint32_t A5XX_UBO_1_BASE_HI(uint32_t val)
-{
-	return ((val) << A5XX_UBO_1_BASE_HI__SHIFT) & A5XX_UBO_1_BASE_HI__MASK;
-}
-
-#ifdef __cplusplus
-#endif
-
-#endif /* A5XX_XML */
diff --git a/drivers/gpu/drm/msm/adreno/a6xx.xml.h b/drivers/gpu/drm/msm/adreno/a6xx.xml.h
deleted file mode 100644
index 92e23bf2458d..000000000000
--- a/drivers/gpu/drm/msm/adreno/a6xx.xml.h
+++ /dev/null
@@ -1,11858 +0,0 @@
-#ifndef A6XX_XML
-#define A6XX_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng gen_header.py tool in this git repository:
-http://gitlab.freedesktop.org/mesa/mesa/
-git clone https://gitlab.freedesktop.org/mesa/mesa.git
-
-The rules-ng-ng source files this header was generated from are:
-
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx.xml          ( 243381 bytes, from Sat Feb 24 09:06:40 2024)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml  (   1572 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml (  15434 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml    (  85856 bytes, from Fri Feb 23 13:07:00 2024)
-
-Copyright (C) 2013-2024 by the following authors:
-- Rob Clark <robdclark@gmail.com> Rob Clark
-- Ilia Mirkin <imirkin@alum.mit.edu> Ilia Mirkin
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-*/
-
-#ifdef __KERNEL__
-#include <linux/bug.h>
-#define assert(x) BUG_ON(!(x))
-#else
-#include <assert.h>
-#endif
-
-#ifdef __cplusplus
-#define __struct_cast(X)
-#else
-#define __struct_cast(X) (struct X)
-#endif
-
-enum a6xx_tile_mode {
-	TILE6_LINEAR = 0,
-	TILE6_2 = 2,
-	TILE6_3 = 3,
-};
-
-enum a6xx_format {
-	FMT6_A8_UNORM = 2,
-	FMT6_8_UNORM = 3,
-	FMT6_8_SNORM = 4,
-	FMT6_8_UINT = 5,
-	FMT6_8_SINT = 6,
-	FMT6_4_4_4_4_UNORM = 8,
-	FMT6_5_5_5_1_UNORM = 10,
-	FMT6_1_5_5_5_UNORM = 12,
-	FMT6_5_6_5_UNORM = 14,
-	FMT6_8_8_UNORM = 15,
-	FMT6_8_8_SNORM = 16,
-	FMT6_8_8_UINT = 17,
-	FMT6_8_8_SINT = 18,
-	FMT6_L8_A8_UNORM = 19,
-	FMT6_16_UNORM = 21,
-	FMT6_16_SNORM = 22,
-	FMT6_16_FLOAT = 23,
-	FMT6_16_UINT = 24,
-	FMT6_16_SINT = 25,
-	FMT6_8_8_8_UNORM = 33,
-	FMT6_8_8_8_SNORM = 34,
-	FMT6_8_8_8_UINT = 35,
-	FMT6_8_8_8_SINT = 36,
-	FMT6_8_8_8_8_UNORM = 48,
-	FMT6_8_8_8_X8_UNORM = 49,
-	FMT6_8_8_8_8_SNORM = 50,
-	FMT6_8_8_8_8_UINT = 51,
-	FMT6_8_8_8_8_SINT = 52,
-	FMT6_9_9_9_E5_FLOAT = 53,
-	FMT6_10_10_10_2_UNORM = 54,
-	FMT6_10_10_10_2_UNORM_DEST = 55,
-	FMT6_10_10_10_2_SNORM = 57,
-	FMT6_10_10_10_2_UINT = 58,
-	FMT6_10_10_10_2_SINT = 59,
-	FMT6_11_11_10_FLOAT = 66,
-	FMT6_16_16_UNORM = 67,
-	FMT6_16_16_SNORM = 68,
-	FMT6_16_16_FLOAT = 69,
-	FMT6_16_16_UINT = 70,
-	FMT6_16_16_SINT = 71,
-	FMT6_32_UNORM = 72,
-	FMT6_32_SNORM = 73,
-	FMT6_32_FLOAT = 74,
-	FMT6_32_UINT = 75,
-	FMT6_32_SINT = 76,
-	FMT6_32_FIXED = 77,
-	FMT6_16_16_16_UNORM = 88,
-	FMT6_16_16_16_SNORM = 89,
-	FMT6_16_16_16_FLOAT = 90,
-	FMT6_16_16_16_UINT = 91,
-	FMT6_16_16_16_SINT = 92,
-	FMT6_16_16_16_16_UNORM = 96,
-	FMT6_16_16_16_16_SNORM = 97,
-	FMT6_16_16_16_16_FLOAT = 98,
-	FMT6_16_16_16_16_UINT = 99,
-	FMT6_16_16_16_16_SINT = 100,
-	FMT6_32_32_UNORM = 101,
-	FMT6_32_32_SNORM = 102,
-	FMT6_32_32_FLOAT = 103,
-	FMT6_32_32_UINT = 104,
-	FMT6_32_32_SINT = 105,
-	FMT6_32_32_FIXED = 106,
-	FMT6_32_32_32_UNORM = 112,
-	FMT6_32_32_32_SNORM = 113,
-	FMT6_32_32_32_UINT = 114,
-	FMT6_32_32_32_SINT = 115,
-	FMT6_32_32_32_FLOAT = 116,
-	FMT6_32_32_32_FIXED = 117,
-	FMT6_32_32_32_32_UNORM = 128,
-	FMT6_32_32_32_32_SNORM = 129,
-	FMT6_32_32_32_32_FLOAT = 130,
-	FMT6_32_32_32_32_UINT = 131,
-	FMT6_32_32_32_32_SINT = 132,
-	FMT6_32_32_32_32_FIXED = 133,
-	FMT6_G8R8B8R8_422_UNORM = 140,
-	FMT6_R8G8R8B8_422_UNORM = 141,
-	FMT6_R8_G8B8_2PLANE_420_UNORM = 142,
-	FMT6_NV21 = 143,
-	FMT6_R8_G8_B8_3PLANE_420_UNORM = 144,
-	FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8 = 145,
-	FMT6_NV12_Y = 148,
-	FMT6_NV12_UV = 149,
-	FMT6_NV12_VU = 150,
-	FMT6_NV12_4R = 151,
-	FMT6_NV12_4R_Y = 152,
-	FMT6_NV12_4R_UV = 153,
-	FMT6_P010 = 154,
-	FMT6_P010_Y = 155,
-	FMT6_P010_UV = 156,
-	FMT6_TP10 = 157,
-	FMT6_TP10_Y = 158,
-	FMT6_TP10_UV = 159,
-	FMT6_Z24_UNORM_S8_UINT = 160,
-	FMT6_ETC2_RG11_UNORM = 171,
-	FMT6_ETC2_RG11_SNORM = 172,
-	FMT6_ETC2_R11_UNORM = 173,
-	FMT6_ETC2_R11_SNORM = 174,
-	FMT6_ETC1 = 175,
-	FMT6_ETC2_RGB8 = 176,
-	FMT6_ETC2_RGBA8 = 177,
-	FMT6_ETC2_RGB8A1 = 178,
-	FMT6_DXT1 = 179,
-	FMT6_DXT3 = 180,
-	FMT6_DXT5 = 181,
-	FMT6_RGTC1_UNORM = 183,
-	FMT6_RGTC1_SNORM = 184,
-	FMT6_RGTC2_UNORM = 187,
-	FMT6_RGTC2_SNORM = 188,
-	FMT6_BPTC_UFLOAT = 190,
-	FMT6_BPTC_FLOAT = 191,
-	FMT6_BPTC = 192,
-	FMT6_ASTC_4x4 = 193,
-	FMT6_ASTC_5x4 = 194,
-	FMT6_ASTC_5x5 = 195,
-	FMT6_ASTC_6x5 = 196,
-	FMT6_ASTC_6x6 = 197,
-	FMT6_ASTC_8x5 = 198,
-	FMT6_ASTC_8x6 = 199,
-	FMT6_ASTC_8x8 = 200,
-	FMT6_ASTC_10x5 = 201,
-	FMT6_ASTC_10x6 = 202,
-	FMT6_ASTC_10x8 = 203,
-	FMT6_ASTC_10x10 = 204,
-	FMT6_ASTC_12x10 = 205,
-	FMT6_ASTC_12x12 = 206,
-	FMT6_Z24_UINT_S8_UINT = 234,
-	FMT6_NONE = 255,
-};
-
-enum a6xx_polygon_mode {
-	POLYMODE6_POINTS = 1,
-	POLYMODE6_LINES = 2,
-	POLYMODE6_TRIANGLES = 3,
-};
-
-enum a6xx_depth_format {
-	DEPTH6_NONE = 0,
-	DEPTH6_16 = 1,
-	DEPTH6_24_8 = 2,
-	DEPTH6_32 = 4,
-};
-
-enum a6xx_shader_id {
-	A6XX_TP0_TMO_DATA = 9,
-	A6XX_TP0_SMO_DATA = 10,
-	A6XX_TP0_MIPMAP_BASE_DATA = 11,
-	A6XX_TP1_TMO_DATA = 25,
-	A6XX_TP1_SMO_DATA = 26,
-	A6XX_TP1_MIPMAP_BASE_DATA = 27,
-	A6XX_SP_INST_DATA = 41,
-	A6XX_SP_LB_0_DATA = 42,
-	A6XX_SP_LB_1_DATA = 43,
-	A6XX_SP_LB_2_DATA = 44,
-	A6XX_SP_LB_3_DATA = 45,
-	A6XX_SP_LB_4_DATA = 46,
-	A6XX_SP_LB_5_DATA = 47,
-	A6XX_SP_CB_BINDLESS_DATA = 48,
-	A6XX_SP_CB_LEGACY_DATA = 49,
-	A6XX_SP_UAV_DATA = 50,
-	A6XX_SP_INST_TAG = 51,
-	A6XX_SP_CB_BINDLESS_TAG = 52,
-	A6XX_SP_TMO_UMO_TAG = 53,
-	A6XX_SP_SMO_TAG = 54,
-	A6XX_SP_STATE_DATA = 55,
-	A6XX_HLSQ_CHUNK_CVS_RAM = 73,
-	A6XX_HLSQ_CHUNK_CPS_RAM = 74,
-	A6XX_HLSQ_CHUNK_CVS_RAM_TAG = 75,
-	A6XX_HLSQ_CHUNK_CPS_RAM_TAG = 76,
-	A6XX_HLSQ_ICB_CVS_CB_BASE_TAG = 77,
-	A6XX_HLSQ_ICB_CPS_CB_BASE_TAG = 78,
-	A6XX_HLSQ_CVS_MISC_RAM = 80,
-	A6XX_HLSQ_CPS_MISC_RAM = 81,
-	A6XX_HLSQ_INST_RAM = 82,
-	A6XX_HLSQ_GFX_CVS_CONST_RAM = 83,
-	A6XX_HLSQ_GFX_CPS_CONST_RAM = 84,
-	A6XX_HLSQ_CVS_MISC_RAM_TAG = 85,
-	A6XX_HLSQ_CPS_MISC_RAM_TAG = 86,
-	A6XX_HLSQ_INST_RAM_TAG = 87,
-	A6XX_HLSQ_GFX_CVS_CONST_RAM_TAG = 88,
-	A6XX_HLSQ_GFX_CPS_CONST_RAM_TAG = 89,
-	A6XX_HLSQ_PWR_REST_RAM = 90,
-	A6XX_HLSQ_PWR_REST_TAG = 91,
-	A6XX_HLSQ_DATAPATH_META = 96,
-	A6XX_HLSQ_FRONTEND_META = 97,
-	A6XX_HLSQ_INDIRECT_META = 98,
-	A6XX_HLSQ_BACKEND_META = 99,
-	A6XX_SP_LB_6_DATA = 112,
-	A6XX_SP_LB_7_DATA = 113,
-	A6XX_HLSQ_INST_RAM_1 = 115,
-};
-
-enum a7xx_statetype_id {
-	A7XX_TP0_NCTX_REG = 0,
-	A7XX_TP0_CTX0_3D_CVS_REG = 1,
-	A7XX_TP0_CTX0_3D_CPS_REG = 2,
-	A7XX_TP0_CTX1_3D_CVS_REG = 3,
-	A7XX_TP0_CTX1_3D_CPS_REG = 4,
-	A7XX_TP0_CTX2_3D_CPS_REG = 5,
-	A7XX_TP0_CTX3_3D_CPS_REG = 6,
-	A7XX_TP0_TMO_DATA = 9,
-	A7XX_TP0_SMO_DATA = 10,
-	A7XX_TP0_MIPMAP_BASE_DATA = 11,
-	A7XX_SP_NCTX_REG = 32,
-	A7XX_SP_CTX0_3D_CVS_REG = 33,
-	A7XX_SP_CTX0_3D_CPS_REG = 34,
-	A7XX_SP_CTX1_3D_CVS_REG = 35,
-	A7XX_SP_CTX1_3D_CPS_REG = 36,
-	A7XX_SP_CTX2_3D_CPS_REG = 37,
-	A7XX_SP_CTX3_3D_CPS_REG = 38,
-	A7XX_SP_INST_DATA = 39,
-	A7XX_SP_INST_DATA_1 = 40,
-	A7XX_SP_LB_0_DATA = 41,
-	A7XX_SP_LB_1_DATA = 42,
-	A7XX_SP_LB_2_DATA = 43,
-	A7XX_SP_LB_3_DATA = 44,
-	A7XX_SP_LB_4_DATA = 45,
-	A7XX_SP_LB_5_DATA = 46,
-	A7XX_SP_LB_6_DATA = 47,
-	A7XX_SP_LB_7_DATA = 48,
-	A7XX_SP_CB_RAM = 49,
-	A7XX_SP_LB_13_DATA = 50,
-	A7XX_SP_LB_14_DATA = 51,
-	A7XX_SP_INST_TAG = 52,
-	A7XX_SP_INST_DATA_2 = 53,
-	A7XX_SP_TMO_TAG = 54,
-	A7XX_SP_SMO_TAG = 55,
-	A7XX_SP_STATE_DATA = 56,
-	A7XX_SP_HWAVE_RAM = 57,
-	A7XX_SP_L0_INST_BUF = 58,
-	A7XX_SP_LB_8_DATA = 59,
-	A7XX_SP_LB_9_DATA = 60,
-	A7XX_SP_LB_10_DATA = 61,
-	A7XX_SP_LB_11_DATA = 62,
-	A7XX_SP_LB_12_DATA = 63,
-	A7XX_HLSQ_DATAPATH_DSTR_META = 64,
-	A7XX_HLSQ_L2STC_TAG_RAM = 67,
-	A7XX_HLSQ_L2STC_INFO_CMD = 68,
-	A7XX_HLSQ_CVS_BE_CTXT_BUF_RAM_TAG = 69,
-	A7XX_HLSQ_CPS_BE_CTXT_BUF_RAM_TAG = 70,
-	A7XX_HLSQ_GFX_CVS_BE_CTXT_BUF_RAM = 71,
-	A7XX_HLSQ_GFX_CPS_BE_CTXT_BUF_RAM = 72,
-	A7XX_HLSQ_CHUNK_CVS_RAM = 73,
-	A7XX_HLSQ_CHUNK_CPS_RAM = 74,
-	A7XX_HLSQ_CHUNK_CVS_RAM_TAG = 75,
-	A7XX_HLSQ_CHUNK_CPS_RAM_TAG = 76,
-	A7XX_HLSQ_ICB_CVS_CB_BASE_TAG = 77,
-	A7XX_HLSQ_ICB_CPS_CB_BASE_TAG = 78,
-	A7XX_HLSQ_CVS_MISC_RAM = 79,
-	A7XX_HLSQ_CPS_MISC_RAM = 80,
-	A7XX_HLSQ_CPS_MISC_RAM_1 = 81,
-	A7XX_HLSQ_INST_RAM = 82,
-	A7XX_HLSQ_GFX_CVS_CONST_RAM = 83,
-	A7XX_HLSQ_GFX_CPS_CONST_RAM = 84,
-	A7XX_HLSQ_CVS_MISC_RAM_TAG = 85,
-	A7XX_HLSQ_CPS_MISC_RAM_TAG = 86,
-	A7XX_HLSQ_INST_RAM_TAG = 87,
-	A7XX_HLSQ_GFX_CVS_CONST_RAM_TAG = 88,
-	A7XX_HLSQ_GFX_CPS_CONST_RAM_TAG = 89,
-	A7XX_HLSQ_GFX_LOCAL_MISC_RAM = 90,
-	A7XX_HLSQ_GFX_LOCAL_MISC_RAM_TAG = 91,
-	A7XX_HLSQ_INST_RAM_1 = 92,
-	A7XX_HLSQ_STPROC_META = 93,
-	A7XX_HLSQ_BV_BE_META = 94,
-	A7XX_HLSQ_INST_RAM_2 = 95,
-	A7XX_HLSQ_DATAPATH_META = 96,
-	A7XX_HLSQ_FRONTEND_META = 97,
-	A7XX_HLSQ_INDIRECT_META = 98,
-	A7XX_HLSQ_BACKEND_META = 99,
-};
-
-enum a6xx_debugbus_id {
-	A6XX_DBGBUS_CP = 1,
-	A6XX_DBGBUS_RBBM = 2,
-	A6XX_DBGBUS_VBIF = 3,
-	A6XX_DBGBUS_HLSQ = 4,
-	A6XX_DBGBUS_UCHE = 5,
-	A6XX_DBGBUS_DPM = 6,
-	A6XX_DBGBUS_TESS = 7,
-	A6XX_DBGBUS_PC = 8,
-	A6XX_DBGBUS_VFDP = 9,
-	A6XX_DBGBUS_VPC = 10,
-	A6XX_DBGBUS_TSE = 11,
-	A6XX_DBGBUS_RAS = 12,
-	A6XX_DBGBUS_VSC = 13,
-	A6XX_DBGBUS_COM = 14,
-	A6XX_DBGBUS_LRZ = 16,
-	A6XX_DBGBUS_A2D = 17,
-	A6XX_DBGBUS_CCUFCHE = 18,
-	A6XX_DBGBUS_GMU_CX = 19,
-	A6XX_DBGBUS_RBP = 20,
-	A6XX_DBGBUS_DCS = 21,
-	A6XX_DBGBUS_DBGC = 22,
-	A6XX_DBGBUS_CX = 23,
-	A6XX_DBGBUS_GMU_GX = 24,
-	A6XX_DBGBUS_TPFCHE = 25,
-	A6XX_DBGBUS_GBIF_GX = 26,
-	A6XX_DBGBUS_GPC = 29,
-	A6XX_DBGBUS_LARC = 30,
-	A6XX_DBGBUS_HLSQ_SPTP = 31,
-	A6XX_DBGBUS_RB_0 = 32,
-	A6XX_DBGBUS_RB_1 = 33,
-	A6XX_DBGBUS_RB_2 = 34,
-	A6XX_DBGBUS_UCHE_WRAPPER = 36,
-	A6XX_DBGBUS_CCU_0 = 40,
-	A6XX_DBGBUS_CCU_1 = 41,
-	A6XX_DBGBUS_CCU_2 = 42,
-	A6XX_DBGBUS_VFD_0 = 56,
-	A6XX_DBGBUS_VFD_1 = 57,
-	A6XX_DBGBUS_VFD_2 = 58,
-	A6XX_DBGBUS_VFD_3 = 59,
-	A6XX_DBGBUS_VFD_4 = 60,
-	A6XX_DBGBUS_VFD_5 = 61,
-	A6XX_DBGBUS_SP_0 = 64,
-	A6XX_DBGBUS_SP_1 = 65,
-	A6XX_DBGBUS_SP_2 = 66,
-	A6XX_DBGBUS_TPL1_0 = 72,
-	A6XX_DBGBUS_TPL1_1 = 73,
-	A6XX_DBGBUS_TPL1_2 = 74,
-	A6XX_DBGBUS_TPL1_3 = 75,
-	A6XX_DBGBUS_TPL1_4 = 76,
-	A6XX_DBGBUS_TPL1_5 = 77,
-	A6XX_DBGBUS_SPTP_0 = 88,
-	A6XX_DBGBUS_SPTP_1 = 89,
-	A6XX_DBGBUS_SPTP_2 = 90,
-	A6XX_DBGBUS_SPTP_3 = 91,
-	A6XX_DBGBUS_SPTP_4 = 92,
-	A6XX_DBGBUS_SPTP_5 = 93,
-};
-
-enum a7xx_state_location {
-	A7XX_HLSQ_STATE = 0,
-	A7XX_HLSQ_DP = 1,
-	A7XX_SP_TOP = 2,
-	A7XX_USPTP = 3,
-};
-
-enum a7xx_pipe {
-	A7XX_PIPE_NONE = 0,
-	A7XX_PIPE_BR = 1,
-	A7XX_PIPE_BV = 2,
-	A7XX_PIPE_LPAC = 3,
-};
-
-enum a7xx_cluster {
-	A7XX_CLUSTER_NONE = 0,
-	A7XX_CLUSTER_FE = 1,
-	A7XX_CLUSTER_SP_VS = 2,
-	A7XX_CLUSTER_PC_VS = 3,
-	A7XX_CLUSTER_GRAS = 4,
-	A7XX_CLUSTER_SP_PS = 5,
-	A7XX_CLUSTER_VPC_PS = 6,
-	A7XX_CLUSTER_PS = 7,
-};
-
-enum a7xx_debugbus_id {
-	A7XX_DBGBUS_CP_0_0 = 1,
-	A7XX_DBGBUS_CP_0_1 = 2,
-	A7XX_DBGBUS_RBBM = 3,
-	A7XX_DBGBUS_GBIF_GX = 5,
-	A7XX_DBGBUS_GBIF_CX = 6,
-	A7XX_DBGBUS_HLSQ = 7,
-	A7XX_DBGBUS_UCHE_0 = 9,
-	A7XX_DBGBUS_UCHE_1 = 10,
-	A7XX_DBGBUS_TESS_BR = 13,
-	A7XX_DBGBUS_TESS_BV = 14,
-	A7XX_DBGBUS_PC_BR = 17,
-	A7XX_DBGBUS_PC_BV = 18,
-	A7XX_DBGBUS_VFDP_BR = 21,
-	A7XX_DBGBUS_VFDP_BV = 22,
-	A7XX_DBGBUS_VPC_BR = 25,
-	A7XX_DBGBUS_VPC_BV = 26,
-	A7XX_DBGBUS_TSE_BR = 29,
-	A7XX_DBGBUS_TSE_BV = 30,
-	A7XX_DBGBUS_RAS_BR = 33,
-	A7XX_DBGBUS_RAS_BV = 34,
-	A7XX_DBGBUS_VSC = 37,
-	A7XX_DBGBUS_COM_0 = 39,
-	A7XX_DBGBUS_LRZ_BR = 43,
-	A7XX_DBGBUS_LRZ_BV = 44,
-	A7XX_DBGBUS_UFC_0 = 47,
-	A7XX_DBGBUS_UFC_1 = 48,
-	A7XX_DBGBUS_GMU_GX = 55,
-	A7XX_DBGBUS_DBGC = 59,
-	A7XX_DBGBUS_CX = 60,
-	A7XX_DBGBUS_GMU_CX = 61,
-	A7XX_DBGBUS_GPC_BR = 62,
-	A7XX_DBGBUS_GPC_BV = 63,
-	A7XX_DBGBUS_LARC = 66,
-	A7XX_DBGBUS_HLSQ_SPTP = 68,
-	A7XX_DBGBUS_RB_0 = 70,
-	A7XX_DBGBUS_RB_1 = 71,
-	A7XX_DBGBUS_RB_2 = 72,
-	A7XX_DBGBUS_RB_3 = 73,
-	A7XX_DBGBUS_RB_4 = 74,
-	A7XX_DBGBUS_RB_5 = 75,
-	A7XX_DBGBUS_UCHE_WRAPPER = 102,
-	A7XX_DBGBUS_CCU_0 = 106,
-	A7XX_DBGBUS_CCU_1 = 107,
-	A7XX_DBGBUS_CCU_2 = 108,
-	A7XX_DBGBUS_CCU_3 = 109,
-	A7XX_DBGBUS_CCU_4 = 110,
-	A7XX_DBGBUS_CCU_5 = 111,
-	A7XX_DBGBUS_VFD_BR_0 = 138,
-	A7XX_DBGBUS_VFD_BR_1 = 139,
-	A7XX_DBGBUS_VFD_BR_2 = 140,
-	A7XX_DBGBUS_VFD_BR_3 = 141,
-	A7XX_DBGBUS_VFD_BR_4 = 142,
-	A7XX_DBGBUS_VFD_BR_5 = 143,
-	A7XX_DBGBUS_VFD_BR_6 = 144,
-	A7XX_DBGBUS_VFD_BR_7 = 145,
-	A7XX_DBGBUS_VFD_BV_0 = 202,
-	A7XX_DBGBUS_VFD_BV_1 = 203,
-	A7XX_DBGBUS_VFD_BV_2 = 204,
-	A7XX_DBGBUS_VFD_BV_3 = 205,
-	A7XX_DBGBUS_USP_0 = 234,
-	A7XX_DBGBUS_USP_1 = 235,
-	A7XX_DBGBUS_USP_2 = 236,
-	A7XX_DBGBUS_USP_3 = 237,
-	A7XX_DBGBUS_USP_4 = 238,
-	A7XX_DBGBUS_USP_5 = 239,
-	A7XX_DBGBUS_TP_0 = 266,
-	A7XX_DBGBUS_TP_1 = 267,
-	A7XX_DBGBUS_TP_2 = 268,
-	A7XX_DBGBUS_TP_3 = 269,
-	A7XX_DBGBUS_TP_4 = 270,
-	A7XX_DBGBUS_TP_5 = 271,
-	A7XX_DBGBUS_TP_6 = 272,
-	A7XX_DBGBUS_TP_7 = 273,
-	A7XX_DBGBUS_TP_8 = 274,
-	A7XX_DBGBUS_TP_9 = 275,
-	A7XX_DBGBUS_TP_10 = 276,
-	A7XX_DBGBUS_TP_11 = 277,
-	A7XX_DBGBUS_USPTP_0 = 330,
-	A7XX_DBGBUS_USPTP_1 = 331,
-	A7XX_DBGBUS_USPTP_2 = 332,
-	A7XX_DBGBUS_USPTP_3 = 333,
-	A7XX_DBGBUS_USPTP_4 = 334,
-	A7XX_DBGBUS_USPTP_5 = 335,
-	A7XX_DBGBUS_USPTP_6 = 336,
-	A7XX_DBGBUS_USPTP_7 = 337,
-	A7XX_DBGBUS_USPTP_8 = 338,
-	A7XX_DBGBUS_USPTP_9 = 339,
-	A7XX_DBGBUS_USPTP_10 = 340,
-	A7XX_DBGBUS_USPTP_11 = 341,
-	A7XX_DBGBUS_CCHE_0 = 396,
-	A7XX_DBGBUS_CCHE_1 = 397,
-	A7XX_DBGBUS_CCHE_2 = 398,
-	A7XX_DBGBUS_VPC_DSTR_0 = 408,
-	A7XX_DBGBUS_VPC_DSTR_1 = 409,
-	A7XX_DBGBUS_VPC_DSTR_2 = 410,
-	A7XX_DBGBUS_HLSQ_DP_STR_0 = 411,
-	A7XX_DBGBUS_HLSQ_DP_STR_1 = 412,
-	A7XX_DBGBUS_HLSQ_DP_STR_2 = 413,
-	A7XX_DBGBUS_HLSQ_DP_STR_3 = 414,
-	A7XX_DBGBUS_HLSQ_DP_STR_4 = 415,
-	A7XX_DBGBUS_HLSQ_DP_STR_5 = 416,
-	A7XX_DBGBUS_UFC_DSTR_0 = 443,
-	A7XX_DBGBUS_UFC_DSTR_1 = 444,
-	A7XX_DBGBUS_UFC_DSTR_2 = 445,
-	A7XX_DBGBUS_CGC_SUBCORE = 446,
-	A7XX_DBGBUS_CGC_CORE = 447,
-};
-
-enum a6xx_cp_perfcounter_select {
-	PERF_CP_ALWAYS_COUNT = 0,
-	PERF_CP_BUSY_GFX_CORE_IDLE = 1,
-	PERF_CP_BUSY_CYCLES = 2,
-	PERF_CP_NUM_PREEMPTIONS = 3,
-	PERF_CP_PREEMPTION_REACTION_DELAY = 4,
-	PERF_CP_PREEMPTION_SWITCH_OUT_TIME = 5,
-	PERF_CP_PREEMPTION_SWITCH_IN_TIME = 6,
-	PERF_CP_DEAD_DRAWS_IN_BIN_RENDER = 7,
-	PERF_CP_PREDICATED_DRAWS_KILLED = 8,
-	PERF_CP_MODE_SWITCH = 9,
-	PERF_CP_ZPASS_DONE = 10,
-	PERF_CP_CONTEXT_DONE = 11,
-	PERF_CP_CACHE_FLUSH = 12,
-	PERF_CP_LONG_PREEMPTIONS = 13,
-	PERF_CP_SQE_I_CACHE_STARVE = 14,
-	PERF_CP_SQE_IDLE = 15,
-	PERF_CP_SQE_PM4_STARVE_RB_IB = 16,
-	PERF_CP_SQE_PM4_STARVE_SDS = 17,
-	PERF_CP_SQE_MRB_STARVE = 18,
-	PERF_CP_SQE_RRB_STARVE = 19,
-	PERF_CP_SQE_VSD_STARVE = 20,
-	PERF_CP_VSD_DECODE_STARVE = 21,
-	PERF_CP_SQE_PIPE_OUT_STALL = 22,
-	PERF_CP_SQE_SYNC_STALL = 23,
-	PERF_CP_SQE_PM4_WFI_STALL = 24,
-	PERF_CP_SQE_SYS_WFI_STALL = 25,
-	PERF_CP_SQE_T4_EXEC = 26,
-	PERF_CP_SQE_LOAD_STATE_EXEC = 27,
-	PERF_CP_SQE_SAVE_SDS_STATE = 28,
-	PERF_CP_SQE_DRAW_EXEC = 29,
-	PERF_CP_SQE_CTXT_REG_BUNCH_EXEC = 30,
-	PERF_CP_SQE_EXEC_PROFILED = 31,
-	PERF_CP_MEMORY_POOL_EMPTY = 32,
-	PERF_CP_MEMORY_POOL_SYNC_STALL = 33,
-	PERF_CP_MEMORY_POOL_ABOVE_THRESH = 34,
-	PERF_CP_AHB_WR_STALL_PRE_DRAWS = 35,
-	PERF_CP_AHB_STALL_SQE_GMU = 36,
-	PERF_CP_AHB_STALL_SQE_WR_OTHER = 37,
-	PERF_CP_AHB_STALL_SQE_RD_OTHER = 38,
-	PERF_CP_CLUSTER0_EMPTY = 39,
-	PERF_CP_CLUSTER1_EMPTY = 40,
-	PERF_CP_CLUSTER2_EMPTY = 41,
-	PERF_CP_CLUSTER3_EMPTY = 42,
-	PERF_CP_CLUSTER4_EMPTY = 43,
-	PERF_CP_CLUSTER5_EMPTY = 44,
-	PERF_CP_PM4_DATA = 45,
-	PERF_CP_PM4_HEADERS = 46,
-	PERF_CP_VBIF_READ_BEATS = 47,
-	PERF_CP_VBIF_WRITE_BEATS = 48,
-	PERF_CP_SQE_INSTR_COUNTER = 49,
-};
-
-enum a6xx_rbbm_perfcounter_select {
-	PERF_RBBM_ALWAYS_COUNT = 0,
-	PERF_RBBM_ALWAYS_ON = 1,
-	PERF_RBBM_TSE_BUSY = 2,
-	PERF_RBBM_RAS_BUSY = 3,
-	PERF_RBBM_PC_DCALL_BUSY = 4,
-	PERF_RBBM_PC_VSD_BUSY = 5,
-	PERF_RBBM_STATUS_MASKED = 6,
-	PERF_RBBM_COM_BUSY = 7,
-	PERF_RBBM_DCOM_BUSY = 8,
-	PERF_RBBM_VBIF_BUSY = 9,
-	PERF_RBBM_VSC_BUSY = 10,
-	PERF_RBBM_TESS_BUSY = 11,
-	PERF_RBBM_UCHE_BUSY = 12,
-	PERF_RBBM_HLSQ_BUSY = 13,
-};
-
-enum a6xx_pc_perfcounter_select {
-	PERF_PC_BUSY_CYCLES = 0,
-	PERF_PC_WORKING_CYCLES = 1,
-	PERF_PC_STALL_CYCLES_VFD = 2,
-	PERF_PC_STALL_CYCLES_TSE = 3,
-	PERF_PC_STALL_CYCLES_VPC = 4,
-	PERF_PC_STALL_CYCLES_UCHE = 5,
-	PERF_PC_STALL_CYCLES_TESS = 6,
-	PERF_PC_STALL_CYCLES_TSE_ONLY = 7,
-	PERF_PC_STALL_CYCLES_VPC_ONLY = 8,
-	PERF_PC_PASS1_TF_STALL_CYCLES = 9,
-	PERF_PC_STARVE_CYCLES_FOR_INDEX = 10,
-	PERF_PC_STARVE_CYCLES_FOR_TESS_FACTOR = 11,
-	PERF_PC_STARVE_CYCLES_FOR_VIZ_STREAM = 12,
-	PERF_PC_STARVE_CYCLES_FOR_POSITION = 13,
-	PERF_PC_STARVE_CYCLES_DI = 14,
-	PERF_PC_VIS_STREAMS_LOADED = 15,
-	PERF_PC_INSTANCES = 16,
-	PERF_PC_VPC_PRIMITIVES = 17,
-	PERF_PC_DEAD_PRIM = 18,
-	PERF_PC_LIVE_PRIM = 19,
-	PERF_PC_VERTEX_HITS = 20,
-	PERF_PC_IA_VERTICES = 21,
-	PERF_PC_IA_PRIMITIVES = 22,
-	PERF_PC_GS_PRIMITIVES = 23,
-	PERF_PC_HS_INVOCATIONS = 24,
-	PERF_PC_DS_INVOCATIONS = 25,
-	PERF_PC_VS_INVOCATIONS = 26,
-	PERF_PC_GS_INVOCATIONS = 27,
-	PERF_PC_DS_PRIMITIVES = 28,
-	PERF_PC_VPC_POS_DATA_TRANSACTION = 29,
-	PERF_PC_3D_DRAWCALLS = 30,
-	PERF_PC_2D_DRAWCALLS = 31,
-	PERF_PC_NON_DRAWCALL_GLOBAL_EVENTS = 32,
-	PERF_TESS_BUSY_CYCLES = 33,
-	PERF_TESS_WORKING_CYCLES = 34,
-	PERF_TESS_STALL_CYCLES_PC = 35,
-	PERF_TESS_STARVE_CYCLES_PC = 36,
-	PERF_PC_TSE_TRANSACTION = 37,
-	PERF_PC_TSE_VERTEX = 38,
-	PERF_PC_TESS_PC_UV_TRANS = 39,
-	PERF_PC_TESS_PC_UV_PATCHES = 40,
-	PERF_PC_TESS_FACTOR_TRANS = 41,
-};
-
-enum a6xx_vfd_perfcounter_select {
-	PERF_VFD_BUSY_CYCLES = 0,
-	PERF_VFD_STALL_CYCLES_UCHE = 1,
-	PERF_VFD_STALL_CYCLES_VPC_ALLOC = 2,
-	PERF_VFD_STALL_CYCLES_SP_INFO = 3,
-	PERF_VFD_STALL_CYCLES_SP_ATTR = 4,
-	PERF_VFD_STARVE_CYCLES_UCHE = 5,
-	PERF_VFD_RBUFFER_FULL = 6,
-	PERF_VFD_ATTR_INFO_FIFO_FULL = 7,
-	PERF_VFD_DECODED_ATTRIBUTE_BYTES = 8,
-	PERF_VFD_NUM_ATTRIBUTES = 9,
-	PERF_VFD_UPPER_SHADER_FIBERS = 10,
-	PERF_VFD_LOWER_SHADER_FIBERS = 11,
-	PERF_VFD_MODE_0_FIBERS = 12,
-	PERF_VFD_MODE_1_FIBERS = 13,
-	PERF_VFD_MODE_2_FIBERS = 14,
-	PERF_VFD_MODE_3_FIBERS = 15,
-	PERF_VFD_MODE_4_FIBERS = 16,
-	PERF_VFD_TOTAL_VERTICES = 17,
-	PERF_VFDP_STALL_CYCLES_VFD = 18,
-	PERF_VFDP_STALL_CYCLES_VFD_INDEX = 19,
-	PERF_VFDP_STALL_CYCLES_VFD_PROG = 20,
-	PERF_VFDP_STARVE_CYCLES_PC = 21,
-	PERF_VFDP_VS_STAGE_WAVES = 22,
-};
-
-enum a6xx_hlsq_perfcounter_select {
-	PERF_HLSQ_BUSY_CYCLES = 0,
-	PERF_HLSQ_STALL_CYCLES_UCHE = 1,
-	PERF_HLSQ_STALL_CYCLES_SP_STATE = 2,
-	PERF_HLSQ_STALL_CYCLES_SP_FS_STAGE = 3,
-	PERF_HLSQ_UCHE_LATENCY_CYCLES = 4,
-	PERF_HLSQ_UCHE_LATENCY_COUNT = 5,
-	PERF_HLSQ_FS_STAGE_1X_WAVES = 6,
-	PERF_HLSQ_FS_STAGE_2X_WAVES = 7,
-	PERF_HLSQ_QUADS = 8,
-	PERF_HLSQ_CS_INVOCATIONS = 9,
-	PERF_HLSQ_COMPUTE_DRAWCALLS = 10,
-	PERF_HLSQ_FS_DATA_WAIT_PROGRAMMING = 11,
-	PERF_HLSQ_DUAL_FS_PROG_ACTIVE = 12,
-	PERF_HLSQ_DUAL_VS_PROG_ACTIVE = 13,
-	PERF_HLSQ_FS_BATCH_COUNT_ZERO = 14,
-	PERF_HLSQ_VS_BATCH_COUNT_ZERO = 15,
-	PERF_HLSQ_WAVE_PENDING_NO_QUAD = 16,
-	PERF_HLSQ_WAVE_PENDING_NO_PRIM_BASE = 17,
-	PERF_HLSQ_STALL_CYCLES_VPC = 18,
-	PERF_HLSQ_PIXELS = 19,
-	PERF_HLSQ_DRAW_MODE_SWITCH_VSFS_SYNC = 20,
-};
-
-enum a6xx_vpc_perfcounter_select {
-	PERF_VPC_BUSY_CYCLES = 0,
-	PERF_VPC_WORKING_CYCLES = 1,
-	PERF_VPC_STALL_CYCLES_UCHE = 2,
-	PERF_VPC_STALL_CYCLES_VFD_WACK = 3,
-	PERF_VPC_STALL_CYCLES_HLSQ_PRIM_ALLOC = 4,
-	PERF_VPC_STALL_CYCLES_PC = 5,
-	PERF_VPC_STALL_CYCLES_SP_LM = 6,
-	PERF_VPC_STARVE_CYCLES_SP = 7,
-	PERF_VPC_STARVE_CYCLES_LRZ = 8,
-	PERF_VPC_PC_PRIMITIVES = 9,
-	PERF_VPC_SP_COMPONENTS = 10,
-	PERF_VPC_STALL_CYCLES_VPCRAM_POS = 11,
-	PERF_VPC_LRZ_ASSIGN_PRIMITIVES = 12,
-	PERF_VPC_RB_VISIBLE_PRIMITIVES = 13,
-	PERF_VPC_LM_TRANSACTION = 14,
-	PERF_VPC_STREAMOUT_TRANSACTION = 15,
-	PERF_VPC_VS_BUSY_CYCLES = 16,
-	PERF_VPC_PS_BUSY_CYCLES = 17,
-	PERF_VPC_VS_WORKING_CYCLES = 18,
-	PERF_VPC_PS_WORKING_CYCLES = 19,
-	PERF_VPC_STARVE_CYCLES_RB = 20,
-	PERF_VPC_NUM_VPCRAM_READ_POS = 21,
-	PERF_VPC_WIT_FULL_CYCLES = 22,
-	PERF_VPC_VPCRAM_FULL_CYCLES = 23,
-	PERF_VPC_LM_FULL_WAIT_FOR_INTP_END = 24,
-	PERF_VPC_NUM_VPCRAM_WRITE = 25,
-	PERF_VPC_NUM_VPCRAM_READ_SO = 26,
-	PERF_VPC_NUM_ATTR_REQ_LM = 27,
-};
-
-enum a6xx_tse_perfcounter_select {
-	PERF_TSE_BUSY_CYCLES = 0,
-	PERF_TSE_CLIPPING_CYCLES = 1,
-	PERF_TSE_STALL_CYCLES_RAS = 2,
-	PERF_TSE_STALL_CYCLES_LRZ_BARYPLANE = 3,
-	PERF_TSE_STALL_CYCLES_LRZ_ZPLANE = 4,
-	PERF_TSE_STARVE_CYCLES_PC = 5,
-	PERF_TSE_INPUT_PRIM = 6,
-	PERF_TSE_INPUT_NULL_PRIM = 7,
-	PERF_TSE_TRIVAL_REJ_PRIM = 8,
-	PERF_TSE_CLIPPED_PRIM = 9,
-	PERF_TSE_ZERO_AREA_PRIM = 10,
-	PERF_TSE_FACENESS_CULLED_PRIM = 11,
-	PERF_TSE_ZERO_PIXEL_PRIM = 12,
-	PERF_TSE_OUTPUT_NULL_PRIM = 13,
-	PERF_TSE_OUTPUT_VISIBLE_PRIM = 14,
-	PERF_TSE_CINVOCATION = 15,
-	PERF_TSE_CPRIMITIVES = 16,
-	PERF_TSE_2D_INPUT_PRIM = 17,
-	PERF_TSE_2D_ALIVE_CYCLES = 18,
-	PERF_TSE_CLIP_PLANES = 19,
-};
-
-enum a6xx_ras_perfcounter_select {
-	PERF_RAS_BUSY_CYCLES = 0,
-	PERF_RAS_SUPERTILE_ACTIVE_CYCLES = 1,
-	PERF_RAS_STALL_CYCLES_LRZ = 2,
-	PERF_RAS_STARVE_CYCLES_TSE = 3,
-	PERF_RAS_SUPER_TILES = 4,
-	PERF_RAS_8X4_TILES = 5,
-	PERF_RAS_MASKGEN_ACTIVE = 6,
-	PERF_RAS_FULLY_COVERED_SUPER_TILES = 7,
-	PERF_RAS_FULLY_COVERED_8X4_TILES = 8,
-	PERF_RAS_PRIM_KILLED_INVISILBE = 9,
-	PERF_RAS_SUPERTILE_GEN_ACTIVE_CYCLES = 10,
-	PERF_RAS_LRZ_INTF_WORKING_CYCLES = 11,
-	PERF_RAS_BLOCKS = 12,
-};
-
-enum a6xx_uche_perfcounter_select {
-	PERF_UCHE_BUSY_CYCLES = 0,
-	PERF_UCHE_STALL_CYCLES_ARBITER = 1,
-	PERF_UCHE_VBIF_LATENCY_CYCLES = 2,
-	PERF_UCHE_VBIF_LATENCY_SAMPLES = 3,
-	PERF_UCHE_VBIF_READ_BEATS_TP = 4,
-	PERF_UCHE_VBIF_READ_BEATS_VFD = 5,
-	PERF_UCHE_VBIF_READ_BEATS_HLSQ = 6,
-	PERF_UCHE_VBIF_READ_BEATS_LRZ = 7,
-	PERF_UCHE_VBIF_READ_BEATS_SP = 8,
-	PERF_UCHE_READ_REQUESTS_TP = 9,
-	PERF_UCHE_READ_REQUESTS_VFD = 10,
-	PERF_UCHE_READ_REQUESTS_HLSQ = 11,
-	PERF_UCHE_READ_REQUESTS_LRZ = 12,
-	PERF_UCHE_READ_REQUESTS_SP = 13,
-	PERF_UCHE_WRITE_REQUESTS_LRZ = 14,
-	PERF_UCHE_WRITE_REQUESTS_SP = 15,
-	PERF_UCHE_WRITE_REQUESTS_VPC = 16,
-	PERF_UCHE_WRITE_REQUESTS_VSC = 17,
-	PERF_UCHE_EVICTS = 18,
-	PERF_UCHE_BANK_REQ0 = 19,
-	PERF_UCHE_BANK_REQ1 = 20,
-	PERF_UCHE_BANK_REQ2 = 21,
-	PERF_UCHE_BANK_REQ3 = 22,
-	PERF_UCHE_BANK_REQ4 = 23,
-	PERF_UCHE_BANK_REQ5 = 24,
-	PERF_UCHE_BANK_REQ6 = 25,
-	PERF_UCHE_BANK_REQ7 = 26,
-	PERF_UCHE_VBIF_READ_BEATS_CH0 = 27,
-	PERF_UCHE_VBIF_READ_BEATS_CH1 = 28,
-	PERF_UCHE_GMEM_READ_BEATS = 29,
-	PERF_UCHE_TPH_REF_FULL = 30,
-	PERF_UCHE_TPH_VICTIM_FULL = 31,
-	PERF_UCHE_TPH_EXT_FULL = 32,
-	PERF_UCHE_VBIF_STALL_WRITE_DATA = 33,
-	PERF_UCHE_DCMP_LATENCY_SAMPLES = 34,
-	PERF_UCHE_DCMP_LATENCY_CYCLES = 35,
-	PERF_UCHE_VBIF_READ_BEATS_PC = 36,
-	PERF_UCHE_READ_REQUESTS_PC = 37,
-	PERF_UCHE_RAM_READ_REQ = 38,
-	PERF_UCHE_RAM_WRITE_REQ = 39,
-};
-
-enum a6xx_tp_perfcounter_select {
-	PERF_TP_BUSY_CYCLES = 0,
-	PERF_TP_STALL_CYCLES_UCHE = 1,
-	PERF_TP_LATENCY_CYCLES = 2,
-	PERF_TP_LATENCY_TRANS = 3,
-	PERF_TP_FLAG_CACHE_REQUEST_SAMPLES = 4,
-	PERF_TP_FLAG_CACHE_REQUEST_LATENCY = 5,
-	PERF_TP_L1_CACHELINE_REQUESTS = 6,
-	PERF_TP_L1_CACHELINE_MISSES = 7,
-	PERF_TP_SP_TP_TRANS = 8,
-	PERF_TP_TP_SP_TRANS = 9,
-	PERF_TP_OUTPUT_PIXELS = 10,
-	PERF_TP_FILTER_WORKLOAD_16BIT = 11,
-	PERF_TP_FILTER_WORKLOAD_32BIT = 12,
-	PERF_TP_QUADS_RECEIVED = 13,
-	PERF_TP_QUADS_OFFSET = 14,
-	PERF_TP_QUADS_SHADOW = 15,
-	PERF_TP_QUADS_ARRAY = 16,
-	PERF_TP_QUADS_GRADIENT = 17,
-	PERF_TP_QUADS_1D = 18,
-	PERF_TP_QUADS_2D = 19,
-	PERF_TP_QUADS_BUFFER = 20,
-	PERF_TP_QUADS_3D = 21,
-	PERF_TP_QUADS_CUBE = 22,
-	PERF_TP_DIVERGENT_QUADS_RECEIVED = 23,
-	PERF_TP_PRT_NON_RESIDENT_EVENTS = 24,
-	PERF_TP_OUTPUT_PIXELS_POINT = 25,
-	PERF_TP_OUTPUT_PIXELS_BILINEAR = 26,
-	PERF_TP_OUTPUT_PIXELS_MIP = 27,
-	PERF_TP_OUTPUT_PIXELS_ANISO = 28,
-	PERF_TP_OUTPUT_PIXELS_ZERO_LOD = 29,
-	PERF_TP_FLAG_CACHE_REQUESTS = 30,
-	PERF_TP_FLAG_CACHE_MISSES = 31,
-	PERF_TP_L1_5_L2_REQUESTS = 32,
-	PERF_TP_2D_OUTPUT_PIXELS = 33,
-	PERF_TP_2D_OUTPUT_PIXELS_POINT = 34,
-	PERF_TP_2D_OUTPUT_PIXELS_BILINEAR = 35,
-	PERF_TP_2D_FILTER_WORKLOAD_16BIT = 36,
-	PERF_TP_2D_FILTER_WORKLOAD_32BIT = 37,
-	PERF_TP_TPA2TPC_TRANS = 38,
-	PERF_TP_L1_MISSES_ASTC_1TILE = 39,
-	PERF_TP_L1_MISSES_ASTC_2TILE = 40,
-	PERF_TP_L1_MISSES_ASTC_4TILE = 41,
-	PERF_TP_L1_5_L2_COMPRESS_REQS = 42,
-	PERF_TP_L1_5_L2_COMPRESS_MISS = 43,
-	PERF_TP_L1_BANK_CONFLICT = 44,
-	PERF_TP_L1_5_MISS_LATENCY_CYCLES = 45,
-	PERF_TP_L1_5_MISS_LATENCY_TRANS = 46,
-	PERF_TP_QUADS_CONSTANT_MULTIPLIED = 47,
-	PERF_TP_FRONTEND_WORKING_CYCLES = 48,
-	PERF_TP_L1_TAG_WORKING_CYCLES = 49,
-	PERF_TP_L1_DATA_WRITE_WORKING_CYCLES = 50,
-	PERF_TP_PRE_L1_DECOM_WORKING_CYCLES = 51,
-	PERF_TP_BACKEND_WORKING_CYCLES = 52,
-	PERF_TP_FLAG_CACHE_WORKING_CYCLES = 53,
-	PERF_TP_L1_5_CACHE_WORKING_CYCLES = 54,
-	PERF_TP_STARVE_CYCLES_SP = 55,
-	PERF_TP_STARVE_CYCLES_UCHE = 56,
-};
-
-enum a6xx_sp_perfcounter_select {
-	PERF_SP_BUSY_CYCLES = 0,
-	PERF_SP_ALU_WORKING_CYCLES = 1,
-	PERF_SP_EFU_WORKING_CYCLES = 2,
-	PERF_SP_STALL_CYCLES_VPC = 3,
-	PERF_SP_STALL_CYCLES_TP = 4,
-	PERF_SP_STALL_CYCLES_UCHE = 5,
-	PERF_SP_STALL_CYCLES_RB = 6,
-	PERF_SP_NON_EXECUTION_CYCLES = 7,
-	PERF_SP_WAVE_CONTEXTS = 8,
-	PERF_SP_WAVE_CONTEXT_CYCLES = 9,
-	PERF_SP_FS_STAGE_WAVE_CYCLES = 10,
-	PERF_SP_FS_STAGE_WAVE_SAMPLES = 11,
-	PERF_SP_VS_STAGE_WAVE_CYCLES = 12,
-	PERF_SP_VS_STAGE_WAVE_SAMPLES = 13,
-	PERF_SP_FS_STAGE_DURATION_CYCLES = 14,
-	PERF_SP_VS_STAGE_DURATION_CYCLES = 15,
-	PERF_SP_WAVE_CTRL_CYCLES = 16,
-	PERF_SP_WAVE_LOAD_CYCLES = 17,
-	PERF_SP_WAVE_EMIT_CYCLES = 18,
-	PERF_SP_WAVE_NOP_CYCLES = 19,
-	PERF_SP_WAVE_WAIT_CYCLES = 20,
-	PERF_SP_WAVE_FETCH_CYCLES = 21,
-	PERF_SP_WAVE_IDLE_CYCLES = 22,
-	PERF_SP_WAVE_END_CYCLES = 23,
-	PERF_SP_WAVE_LONG_SYNC_CYCLES = 24,
-	PERF_SP_WAVE_SHORT_SYNC_CYCLES = 25,
-	PERF_SP_WAVE_JOIN_CYCLES = 26,
-	PERF_SP_LM_LOAD_INSTRUCTIONS = 27,
-	PERF_SP_LM_STORE_INSTRUCTIONS = 28,
-	PERF_SP_LM_ATOMICS = 29,
-	PERF_SP_GM_LOAD_INSTRUCTIONS = 30,
-	PERF_SP_GM_STORE_INSTRUCTIONS = 31,
-	PERF_SP_GM_ATOMICS = 32,
-	PERF_SP_VS_STAGE_TEX_INSTRUCTIONS = 33,
-	PERF_SP_VS_STAGE_EFU_INSTRUCTIONS = 34,
-	PERF_SP_VS_STAGE_FULL_ALU_INSTRUCTIONS = 35,
-	PERF_SP_VS_STAGE_HALF_ALU_INSTRUCTIONS = 36,
-	PERF_SP_FS_STAGE_TEX_INSTRUCTIONS = 37,
-	PERF_SP_FS_STAGE_CFLOW_INSTRUCTIONS = 38,
-	PERF_SP_FS_STAGE_EFU_INSTRUCTIONS = 39,
-	PERF_SP_FS_STAGE_FULL_ALU_INSTRUCTIONS = 40,
-	PERF_SP_FS_STAGE_HALF_ALU_INSTRUCTIONS = 41,
-	PERF_SP_FS_STAGE_BARY_INSTRUCTIONS = 42,
-	PERF_SP_VS_INSTRUCTIONS = 43,
-	PERF_SP_FS_INSTRUCTIONS = 44,
-	PERF_SP_ADDR_LOCK_COUNT = 45,
-	PERF_SP_UCHE_READ_TRANS = 46,
-	PERF_SP_UCHE_WRITE_TRANS = 47,
-	PERF_SP_EXPORT_VPC_TRANS = 48,
-	PERF_SP_EXPORT_RB_TRANS = 49,
-	PERF_SP_PIXELS_KILLED = 50,
-	PERF_SP_ICL1_REQUESTS = 51,
-	PERF_SP_ICL1_MISSES = 52,
-	PERF_SP_HS_INSTRUCTIONS = 53,
-	PERF_SP_DS_INSTRUCTIONS = 54,
-	PERF_SP_GS_INSTRUCTIONS = 55,
-	PERF_SP_CS_INSTRUCTIONS = 56,
-	PERF_SP_GPR_READ = 57,
-	PERF_SP_GPR_WRITE = 58,
-	PERF_SP_FS_STAGE_HALF_EFU_INSTRUCTIONS = 59,
-	PERF_SP_VS_STAGE_HALF_EFU_INSTRUCTIONS = 60,
-	PERF_SP_LM_BANK_CONFLICTS = 61,
-	PERF_SP_TEX_CONTROL_WORKING_CYCLES = 62,
-	PERF_SP_LOAD_CONTROL_WORKING_CYCLES = 63,
-	PERF_SP_FLOW_CONTROL_WORKING_CYCLES = 64,
-	PERF_SP_LM_WORKING_CYCLES = 65,
-	PERF_SP_DISPATCHER_WORKING_CYCLES = 66,
-	PERF_SP_SEQUENCER_WORKING_CYCLES = 67,
-	PERF_SP_LOW_EFFICIENCY_STARVED_BY_TP = 68,
-	PERF_SP_STARVE_CYCLES_HLSQ = 69,
-	PERF_SP_NON_EXECUTION_LS_CYCLES = 70,
-	PERF_SP_WORKING_EU = 71,
-	PERF_SP_ANY_EU_WORKING = 72,
-	PERF_SP_WORKING_EU_FS_STAGE = 73,
-	PERF_SP_ANY_EU_WORKING_FS_STAGE = 74,
-	PERF_SP_WORKING_EU_VS_STAGE = 75,
-	PERF_SP_ANY_EU_WORKING_VS_STAGE = 76,
-	PERF_SP_WORKING_EU_CS_STAGE = 77,
-	PERF_SP_ANY_EU_WORKING_CS_STAGE = 78,
-	PERF_SP_GPR_READ_PREFETCH = 79,
-	PERF_SP_GPR_READ_CONFLICT = 80,
-	PERF_SP_GPR_WRITE_CONFLICT = 81,
-	PERF_SP_GM_LOAD_LATENCY_CYCLES = 82,
-	PERF_SP_GM_LOAD_LATENCY_SAMPLES = 83,
-	PERF_SP_EXECUTABLE_WAVES = 84,
-};
-
-enum a6xx_rb_perfcounter_select {
-	PERF_RB_BUSY_CYCLES = 0,
-	PERF_RB_STALL_CYCLES_HLSQ = 1,
-	PERF_RB_STALL_CYCLES_FIFO0_FULL = 2,
-	PERF_RB_STALL_CYCLES_FIFO1_FULL = 3,
-	PERF_RB_STALL_CYCLES_FIFO2_FULL = 4,
-	PERF_RB_STARVE_CYCLES_SP = 5,
-	PERF_RB_STARVE_CYCLES_LRZ_TILE = 6,
-	PERF_RB_STARVE_CYCLES_CCU = 7,
-	PERF_RB_STARVE_CYCLES_Z_PLANE = 8,
-	PERF_RB_STARVE_CYCLES_BARY_PLANE = 9,
-	PERF_RB_Z_WORKLOAD = 10,
-	PERF_RB_HLSQ_ACTIVE = 11,
-	PERF_RB_Z_READ = 12,
-	PERF_RB_Z_WRITE = 13,
-	PERF_RB_C_READ = 14,
-	PERF_RB_C_WRITE = 15,
-	PERF_RB_TOTAL_PASS = 16,
-	PERF_RB_Z_PASS = 17,
-	PERF_RB_Z_FAIL = 18,
-	PERF_RB_S_FAIL = 19,
-	PERF_RB_BLENDED_FXP_COMPONENTS = 20,
-	PERF_RB_BLENDED_FP16_COMPONENTS = 21,
-	PERF_RB_PS_INVOCATIONS = 22,
-	PERF_RB_2D_ALIVE_CYCLES = 23,
-	PERF_RB_2D_STALL_CYCLES_A2D = 24,
-	PERF_RB_2D_STARVE_CYCLES_SRC = 25,
-	PERF_RB_2D_STARVE_CYCLES_SP = 26,
-	PERF_RB_2D_STARVE_CYCLES_DST = 27,
-	PERF_RB_2D_VALID_PIXELS = 28,
-	PERF_RB_3D_PIXELS = 29,
-	PERF_RB_BLENDER_WORKING_CYCLES = 30,
-	PERF_RB_ZPROC_WORKING_CYCLES = 31,
-	PERF_RB_CPROC_WORKING_CYCLES = 32,
-	PERF_RB_SAMPLER_WORKING_CYCLES = 33,
-	PERF_RB_STALL_CYCLES_CCU_COLOR_READ = 34,
-	PERF_RB_STALL_CYCLES_CCU_COLOR_WRITE = 35,
-	PERF_RB_STALL_CYCLES_CCU_DEPTH_READ = 36,
-	PERF_RB_STALL_CYCLES_CCU_DEPTH_WRITE = 37,
-	PERF_RB_STALL_CYCLES_VPC = 38,
-	PERF_RB_2D_INPUT_TRANS = 39,
-	PERF_RB_2D_OUTPUT_RB_DST_TRANS = 40,
-	PERF_RB_2D_OUTPUT_RB_SRC_TRANS = 41,
-	PERF_RB_BLENDED_FP32_COMPONENTS = 42,
-	PERF_RB_COLOR_PIX_TILES = 43,
-	PERF_RB_STALL_CYCLES_CCU = 44,
-	PERF_RB_EARLY_Z_ARB3_GRANT = 45,
-	PERF_RB_LATE_Z_ARB3_GRANT = 46,
-	PERF_RB_EARLY_Z_SKIP_GRANT = 47,
-};
-
-enum a6xx_vsc_perfcounter_select {
-	PERF_VSC_BUSY_CYCLES = 0,
-	PERF_VSC_WORKING_CYCLES = 1,
-	PERF_VSC_STALL_CYCLES_UCHE = 2,
-	PERF_VSC_EOT_NUM = 3,
-	PERF_VSC_INPUT_TILES = 4,
-};
-
-enum a6xx_ccu_perfcounter_select {
-	PERF_CCU_BUSY_CYCLES = 0,
-	PERF_CCU_STALL_CYCLES_RB_DEPTH_RETURN = 1,
-	PERF_CCU_STALL_CYCLES_RB_COLOR_RETURN = 2,
-	PERF_CCU_STARVE_CYCLES_FLAG_RETURN = 3,
-	PERF_CCU_DEPTH_BLOCKS = 4,
-	PERF_CCU_COLOR_BLOCKS = 5,
-	PERF_CCU_DEPTH_BLOCK_HIT = 6,
-	PERF_CCU_COLOR_BLOCK_HIT = 7,
-	PERF_CCU_PARTIAL_BLOCK_READ = 8,
-	PERF_CCU_GMEM_READ = 9,
-	PERF_CCU_GMEM_WRITE = 10,
-	PERF_CCU_DEPTH_READ_FLAG0_COUNT = 11,
-	PERF_CCU_DEPTH_READ_FLAG1_COUNT = 12,
-	PERF_CCU_DEPTH_READ_FLAG2_COUNT = 13,
-	PERF_CCU_DEPTH_READ_FLAG3_COUNT = 14,
-	PERF_CCU_DEPTH_READ_FLAG4_COUNT = 15,
-	PERF_CCU_DEPTH_READ_FLAG5_COUNT = 16,
-	PERF_CCU_DEPTH_READ_FLAG6_COUNT = 17,
-	PERF_CCU_DEPTH_READ_FLAG8_COUNT = 18,
-	PERF_CCU_COLOR_READ_FLAG0_COUNT = 19,
-	PERF_CCU_COLOR_READ_FLAG1_COUNT = 20,
-	PERF_CCU_COLOR_READ_FLAG2_COUNT = 21,
-	PERF_CCU_COLOR_READ_FLAG3_COUNT = 22,
-	PERF_CCU_COLOR_READ_FLAG4_COUNT = 23,
-	PERF_CCU_COLOR_READ_FLAG5_COUNT = 24,
-	PERF_CCU_COLOR_READ_FLAG6_COUNT = 25,
-	PERF_CCU_COLOR_READ_FLAG8_COUNT = 26,
-	PERF_CCU_2D_RD_REQ = 27,
-	PERF_CCU_2D_WR_REQ = 28,
-};
-
-enum a6xx_lrz_perfcounter_select {
-	PERF_LRZ_BUSY_CYCLES = 0,
-	PERF_LRZ_STARVE_CYCLES_RAS = 1,
-	PERF_LRZ_STALL_CYCLES_RB = 2,
-	PERF_LRZ_STALL_CYCLES_VSC = 3,
-	PERF_LRZ_STALL_CYCLES_VPC = 4,
-	PERF_LRZ_STALL_CYCLES_FLAG_PREFETCH = 5,
-	PERF_LRZ_STALL_CYCLES_UCHE = 6,
-	PERF_LRZ_LRZ_READ = 7,
-	PERF_LRZ_LRZ_WRITE = 8,
-	PERF_LRZ_READ_LATENCY = 9,
-	PERF_LRZ_MERGE_CACHE_UPDATING = 10,
-	PERF_LRZ_PRIM_KILLED_BY_MASKGEN = 11,
-	PERF_LRZ_PRIM_KILLED_BY_LRZ = 12,
-	PERF_LRZ_VISIBLE_PRIM_AFTER_LRZ = 13,
-	PERF_LRZ_FULL_8X8_TILES = 14,
-	PERF_LRZ_PARTIAL_8X8_TILES = 15,
-	PERF_LRZ_TILE_KILLED = 16,
-	PERF_LRZ_TOTAL_PIXEL = 17,
-	PERF_LRZ_VISIBLE_PIXEL_AFTER_LRZ = 18,
-	PERF_LRZ_FULLY_COVERED_TILES = 19,
-	PERF_LRZ_PARTIAL_COVERED_TILES = 20,
-	PERF_LRZ_FEEDBACK_ACCEPT = 21,
-	PERF_LRZ_FEEDBACK_DISCARD = 22,
-	PERF_LRZ_FEEDBACK_STALL = 23,
-	PERF_LRZ_STALL_CYCLES_RB_ZPLANE = 24,
-	PERF_LRZ_STALL_CYCLES_RB_BPLANE = 25,
-	PERF_LRZ_STALL_CYCLES_VC = 26,
-	PERF_LRZ_RAS_MASK_TRANS = 27,
-};
-
-enum a6xx_cmp_perfcounter_select {
-	PERF_CMPDECMP_STALL_CYCLES_ARB = 0,
-	PERF_CMPDECMP_VBIF_LATENCY_CYCLES = 1,
-	PERF_CMPDECMP_VBIF_LATENCY_SAMPLES = 2,
-	PERF_CMPDECMP_VBIF_READ_DATA_CCU = 3,
-	PERF_CMPDECMP_VBIF_WRITE_DATA_CCU = 4,
-	PERF_CMPDECMP_VBIF_READ_REQUEST = 5,
-	PERF_CMPDECMP_VBIF_WRITE_REQUEST = 6,
-	PERF_CMPDECMP_VBIF_READ_DATA = 7,
-	PERF_CMPDECMP_VBIF_WRITE_DATA = 8,
-	PERF_CMPDECMP_FLAG_FETCH_CYCLES = 9,
-	PERF_CMPDECMP_FLAG_FETCH_SAMPLES = 10,
-	PERF_CMPDECMP_DEPTH_WRITE_FLAG1_COUNT = 11,
-	PERF_CMPDECMP_DEPTH_WRITE_FLAG2_COUNT = 12,
-	PERF_CMPDECMP_DEPTH_WRITE_FLAG3_COUNT = 13,
-	PERF_CMPDECMP_DEPTH_WRITE_FLAG4_COUNT = 14,
-	PERF_CMPDECMP_DEPTH_WRITE_FLAG5_COUNT = 15,
-	PERF_CMPDECMP_DEPTH_WRITE_FLAG6_COUNT = 16,
-	PERF_CMPDECMP_DEPTH_WRITE_FLAG8_COUNT = 17,
-	PERF_CMPDECMP_COLOR_WRITE_FLAG1_COUNT = 18,
-	PERF_CMPDECMP_COLOR_WRITE_FLAG2_COUNT = 19,
-	PERF_CMPDECMP_COLOR_WRITE_FLAG3_COUNT = 20,
-	PERF_CMPDECMP_COLOR_WRITE_FLAG4_COUNT = 21,
-	PERF_CMPDECMP_COLOR_WRITE_FLAG5_COUNT = 22,
-	PERF_CMPDECMP_COLOR_WRITE_FLAG6_COUNT = 23,
-	PERF_CMPDECMP_COLOR_WRITE_FLAG8_COUNT = 24,
-	PERF_CMPDECMP_2D_STALL_CYCLES_VBIF_REQ = 25,
-	PERF_CMPDECMP_2D_STALL_CYCLES_VBIF_WR = 26,
-	PERF_CMPDECMP_2D_STALL_CYCLES_VBIF_RETURN = 27,
-	PERF_CMPDECMP_2D_RD_DATA = 28,
-	PERF_CMPDECMP_2D_WR_DATA = 29,
-	PERF_CMPDECMP_VBIF_READ_DATA_UCHE_CH0 = 30,
-	PERF_CMPDECMP_VBIF_READ_DATA_UCHE_CH1 = 31,
-	PERF_CMPDECMP_2D_OUTPUT_TRANS = 32,
-	PERF_CMPDECMP_VBIF_WRITE_DATA_UCHE = 33,
-	PERF_CMPDECMP_DEPTH_WRITE_FLAG0_COUNT = 34,
-	PERF_CMPDECMP_COLOR_WRITE_FLAG0_COUNT = 35,
-	PERF_CMPDECMP_COLOR_WRITE_FLAGALPHA_COUNT = 36,
-	PERF_CMPDECMP_2D_BUSY_CYCLES = 37,
-	PERF_CMPDECMP_2D_REORDER_STARVE_CYCLES = 38,
-	PERF_CMPDECMP_2D_PIXELS = 39,
-};
-
-enum a6xx_2d_ifmt {
-	R2D_UNORM8 = 16,
-	R2D_INT32 = 7,
-	R2D_INT16 = 6,
-	R2D_INT8 = 5,
-	R2D_FLOAT32 = 4,
-	R2D_FLOAT16 = 3,
-	R2D_UNORM8_SRGB = 1,
-	R2D_RAW = 0,
-};
-
-enum a6xx_ztest_mode {
-	A6XX_EARLY_Z = 0,
-	A6XX_LATE_Z = 1,
-	A6XX_EARLY_LRZ_LATE_Z = 2,
-	A6XX_INVALID_ZTEST = 3,
-};
-
-enum a6xx_tess_spacing {
-	TESS_EQUAL = 0,
-	TESS_FRACTIONAL_ODD = 2,
-	TESS_FRACTIONAL_EVEN = 3,
-};
-
-enum a6xx_tess_output {
-	TESS_POINTS = 0,
-	TESS_LINES = 1,
-	TESS_CW_TRIS = 2,
-	TESS_CCW_TRIS = 3,
-};
-
-enum a6xx_sequenced_thread_dist {
-	DIST_SCREEN_COORD = 0,
-	DIST_ALL_TO_RB0 = 1,
-};
-
-enum a6xx_single_prim_mode {
-	NO_FLUSH = 0,
-	FLUSH_PER_OVERLAP_AND_OVERWRITE = 1,
-	FLUSH_PER_OVERLAP = 3,
-};
-
-enum a6xx_raster_mode {
-	TYPE_TILED = 0,
-	TYPE_WRITER = 1,
-};
-
-enum a6xx_raster_direction {
-	LR_TB = 0,
-	RL_TB = 1,
-	LR_BT = 2,
-	RB_BT = 3,
-};
-
-enum a6xx_render_mode {
-	RENDERING_PASS = 0,
-	BINNING_PASS = 1,
-};
-
-enum a6xx_buffers_location {
-	BUFFERS_IN_GMEM = 0,
-	BUFFERS_IN_SYSMEM = 3,
-};
-
-enum a6xx_lrz_dir_status {
-	LRZ_DIR_LE = 1,
-	LRZ_DIR_GE = 2,
-	LRZ_DIR_INVALID = 3,
-};
-
-enum a6xx_fragcoord_sample_mode {
-	FRAGCOORD_CENTER = 0,
-	FRAGCOORD_SAMPLE = 3,
-};
-
-enum a6xx_rotation {
-	ROTATE_0 = 0,
-	ROTATE_90 = 1,
-	ROTATE_180 = 2,
-	ROTATE_270 = 3,
-	ROTATE_HFLIP = 4,
-	ROTATE_VFLIP = 5,
-};
-
-enum a6xx_ccu_cache_size {
-	CCU_CACHE_SIZE_FULL = 0,
-	CCU_CACHE_SIZE_HALF = 1,
-	CCU_CACHE_SIZE_QUARTER = 2,
-	CCU_CACHE_SIZE_EIGHTH = 3,
-};
-
-enum a6xx_varying_interp_mode {
-	INTERP_SMOOTH = 0,
-	INTERP_FLAT = 1,
-	INTERP_ZERO = 2,
-	INTERP_ONE = 3,
-};
-
-enum a6xx_varying_ps_repl_mode {
-	PS_REPL_NONE = 0,
-	PS_REPL_S = 1,
-	PS_REPL_T = 2,
-	PS_REPL_ONE_MINUS_T = 3,
-};
-
-enum a6xx_threadsize {
-	THREAD64 = 0,
-	THREAD128 = 1,
-};
-
-enum a6xx_bindless_descriptor_size {
-	BINDLESS_DESCRIPTOR_16B = 1,
-	BINDLESS_DESCRIPTOR_64B = 3,
-};
-
-enum a6xx_isam_mode {
-	ISAMMODE_CL = 1,
-	ISAMMODE_GL = 2,
-};
-
-enum a7xx_cs_yalign {
-	CS_YALIGN_1 = 8,
-	CS_YALIGN_2 = 4,
-	CS_YALIGN_4 = 2,
-	CS_YALIGN_8 = 1,
-};
-
-enum a6xx_tex_filter {
-	A6XX_TEX_NEAREST = 0,
-	A6XX_TEX_LINEAR = 1,
-	A6XX_TEX_ANISO = 2,
-	A6XX_TEX_CUBIC = 3,
-};
-
-enum a6xx_tex_clamp {
-	A6XX_TEX_REPEAT = 0,
-	A6XX_TEX_CLAMP_TO_EDGE = 1,
-	A6XX_TEX_MIRROR_REPEAT = 2,
-	A6XX_TEX_CLAMP_TO_BORDER = 3,
-	A6XX_TEX_MIRROR_CLAMP = 4,
-};
-
-enum a6xx_tex_aniso {
-	A6XX_TEX_ANISO_1 = 0,
-	A6XX_TEX_ANISO_2 = 1,
-	A6XX_TEX_ANISO_4 = 2,
-	A6XX_TEX_ANISO_8 = 3,
-	A6XX_TEX_ANISO_16 = 4,
-};
-
-enum a6xx_reduction_mode {
-	A6XX_REDUCTION_MODE_AVERAGE = 0,
-	A6XX_REDUCTION_MODE_MIN = 1,
-	A6XX_REDUCTION_MODE_MAX = 2,
-};
-
-enum a6xx_tex_swiz {
-	A6XX_TEX_X = 0,
-	A6XX_TEX_Y = 1,
-	A6XX_TEX_Z = 2,
-	A6XX_TEX_W = 3,
-	A6XX_TEX_ZERO = 4,
-	A6XX_TEX_ONE = 5,
-};
-
-enum a6xx_tex_type {
-	A6XX_TEX_1D = 0,
-	A6XX_TEX_2D = 1,
-	A6XX_TEX_CUBE = 2,
-	A6XX_TEX_3D = 3,
-	A6XX_TEX_BUFFER = 4,
-};
-
-#define A6XX_RBBM_INT_0_MASK_RBBM_GPU_IDLE			0x00000001
-#define A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR			0x00000002
-#define A6XX_RBBM_INT_0_MASK_CP_IPC_INTR_0			0x00000010
-#define A6XX_RBBM_INT_0_MASK_CP_IPC_INTR_1			0x00000020
-#define A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW	0x00000040
-#define A6XX_RBBM_INT_0_MASK_RBBM_GPC_ERROR			0x00000080
-#define A6XX_RBBM_INT_0_MASK_CP_SW				0x00000100
-#define A6XX_RBBM_INT_0_MASK_CP_HW_ERROR			0x00000200
-#define A6XX_RBBM_INT_0_MASK_CP_CCU_FLUSH_DEPTH_TS		0x00000400
-#define A6XX_RBBM_INT_0_MASK_CP_CCU_FLUSH_COLOR_TS		0x00000800
-#define A6XX_RBBM_INT_0_MASK_CP_CCU_RESOLVE_TS			0x00001000
-#define A6XX_RBBM_INT_0_MASK_CP_IB2				0x00002000
-#define A6XX_RBBM_INT_0_MASK_CP_IB1				0x00004000
-#define A6XX_RBBM_INT_0_MASK_CP_RB				0x00008000
-#define A6XX_RBBM_INT_0_MASK_PM4CPINTERRUPT			0x00008000
-#define A6XX_RBBM_INT_0_MASK_PM4CPINTERRUPTLPAC			0x00010000
-#define A6XX_RBBM_INT_0_MASK_CP_RB_DONE_TS			0x00020000
-#define A6XX_RBBM_INT_0_MASK_CP_WT_DONE_TS			0x00040000
-#define A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS			0x00100000
-#define A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS_LPAC		0x00200000
-#define A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW		0x00400000
-#define A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT			0x00800000
-#define A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS			0x01000000
-#define A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR			0x02000000
-#define A6XX_RBBM_INT_0_MASK_DEBBUS_INTR_0			0x04000000
-#define A6XX_RBBM_INT_0_MASK_DEBBUS_INTR_1			0x08000000
-#define A6XX_RBBM_INT_0_MASK_TSBWRITEERROR			0x10000000
-#define A6XX_RBBM_INT_0_MASK_ISDB_CPU_IRQ			0x40000000
-#define A6XX_RBBM_INT_0_MASK_ISDB_UNDER_DEBUG			0x80000000
-
-#define A6XX_CP_INT_CP_OPCODE_ERROR				0x00000001
-#define A6XX_CP_INT_CP_UCODE_ERROR				0x00000002
-#define A6XX_CP_INT_CP_HW_FAULT_ERROR				0x00000004
-#define A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR		0x00000010
-#define A6XX_CP_INT_CP_AHB_ERROR				0x00000020
-#define A6XX_CP_INT_CP_VSD_PARITY_ERROR				0x00000040
-#define A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR			0x00000080
-#define A6XX_CP_INT_CP_OPCODE_ERROR_LPAC			0x00000100
-#define A6XX_CP_INT_CP_UCODE_ERROR_LPAC				0x00000200
-#define A6XX_CP_INT_CP_HW_FAULT_ERROR_LPAC			0x00000400
-#define A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR_LPAC		0x00000800
-#define A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR_LPAC			0x00001000
-#define A6XX_CP_INT_CP_OPCODE_ERROR_BV				0x00002000
-#define A6XX_CP_INT_CP_UCODE_ERROR_BV				0x00004000
-#define A6XX_CP_INT_CP_HW_FAULT_ERROR_BV			0x00008000
-#define A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR_BV		0x00010000
-#define A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR_BV			0x00020000
-
-#define REG_A6XX_CP_RB_BASE					0x00000800
-
-#define REG_A6XX_CP_RB_CNTL					0x00000802
-
-#define REG_A6XX_CP_RB_RPTR_ADDR				0x00000804
-
-#define REG_A6XX_CP_RB_RPTR					0x00000806
-
-#define REG_A6XX_CP_RB_WPTR					0x00000807
-
-#define REG_A6XX_CP_SQE_CNTL					0x00000808
-
-#define REG_A6XX_CP_CP2GMU_STATUS				0x00000812
-#define A6XX_CP_CP2GMU_STATUS_IFPC				0x00000001
-
-#define REG_A6XX_CP_HW_FAULT					0x00000821
-
-#define REG_A6XX_CP_INTERRUPT_STATUS				0x00000823
-#define REG_A6XX_CP_PROTECT_STATUS				0x00000824
-
-#define REG_A6XX_CP_STATUS_1					0x00000825
-
-#define REG_A6XX_CP_SQE_INSTR_BASE				0x00000830
-
-#define REG_A6XX_CP_MISC_CNTL					0x00000840
-
-#define REG_A6XX_CP_APRIV_CNTL					0x00000844
-#define A6XX_CP_APRIV_CNTL_CDWRITE				0x00000040
-#define A6XX_CP_APRIV_CNTL_CDREAD				0x00000020
-#define A6XX_CP_APRIV_CNTL_RBRPWB				0x00000008
-#define A6XX_CP_APRIV_CNTL_RBPRIVLEVEL				0x00000004
-#define A6XX_CP_APRIV_CNTL_RBFETCH				0x00000002
-#define A6XX_CP_APRIV_CNTL_ICACHE				0x00000001
-
-#define REG_A6XX_CP_PREEMPT_THRESHOLD				0x000008c0
-
-#define REG_A6XX_CP_ROQ_THRESHOLDS_1				0x000008c1
-#define A6XX_CP_ROQ_THRESHOLDS_1_MRB_START__MASK		0x000000ff
-#define A6XX_CP_ROQ_THRESHOLDS_1_MRB_START__SHIFT		0
-static inline uint32_t A6XX_CP_ROQ_THRESHOLDS_1_MRB_START(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_CP_ROQ_THRESHOLDS_1_MRB_START__SHIFT) & A6XX_CP_ROQ_THRESHOLDS_1_MRB_START__MASK;
-}
-#define A6XX_CP_ROQ_THRESHOLDS_1_VSD_START__MASK		0x0000ff00
-#define A6XX_CP_ROQ_THRESHOLDS_1_VSD_START__SHIFT		8
-static inline uint32_t A6XX_CP_ROQ_THRESHOLDS_1_VSD_START(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_CP_ROQ_THRESHOLDS_1_VSD_START__SHIFT) & A6XX_CP_ROQ_THRESHOLDS_1_VSD_START__MASK;
-}
-#define A6XX_CP_ROQ_THRESHOLDS_1_IB1_START__MASK		0x00ff0000
-#define A6XX_CP_ROQ_THRESHOLDS_1_IB1_START__SHIFT		16
-static inline uint32_t A6XX_CP_ROQ_THRESHOLDS_1_IB1_START(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_CP_ROQ_THRESHOLDS_1_IB1_START__SHIFT) & A6XX_CP_ROQ_THRESHOLDS_1_IB1_START__MASK;
-}
-#define A6XX_CP_ROQ_THRESHOLDS_1_IB2_START__MASK		0xff000000
-#define A6XX_CP_ROQ_THRESHOLDS_1_IB2_START__SHIFT		24
-static inline uint32_t A6XX_CP_ROQ_THRESHOLDS_1_IB2_START(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_CP_ROQ_THRESHOLDS_1_IB2_START__SHIFT) & A6XX_CP_ROQ_THRESHOLDS_1_IB2_START__MASK;
-}
-
-#define REG_A6XX_CP_ROQ_THRESHOLDS_2				0x000008c2
-#define A6XX_CP_ROQ_THRESHOLDS_2_SDS_START__MASK		0x000001ff
-#define A6XX_CP_ROQ_THRESHOLDS_2_SDS_START__SHIFT		0
-static inline uint32_t A6XX_CP_ROQ_THRESHOLDS_2_SDS_START(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_CP_ROQ_THRESHOLDS_2_SDS_START__SHIFT) & A6XX_CP_ROQ_THRESHOLDS_2_SDS_START__MASK;
-}
-#define A6XX_CP_ROQ_THRESHOLDS_2_ROQ_SIZE__MASK			0xffff0000
-#define A6XX_CP_ROQ_THRESHOLDS_2_ROQ_SIZE__SHIFT		16
-static inline uint32_t A6XX_CP_ROQ_THRESHOLDS_2_ROQ_SIZE(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_CP_ROQ_THRESHOLDS_2_ROQ_SIZE__SHIFT) & A6XX_CP_ROQ_THRESHOLDS_2_ROQ_SIZE__MASK;
-}
-
-#define REG_A6XX_CP_MEM_POOL_SIZE				0x000008c3
-
-#define REG_A6XX_CP_CHICKEN_DBG					0x00000841
-
-#define REG_A6XX_CP_ADDR_MODE_CNTL				0x00000842
-
-#define REG_A6XX_CP_DBG_ECO_CNTL				0x00000843
-
-#define REG_A6XX_CP_PROTECT_CNTL				0x0000084f
-#define A6XX_CP_PROTECT_CNTL_LAST_SPAN_INF_RANGE		0x00000008
-#define A6XX_CP_PROTECT_CNTL_ACCESS_FAULT_ON_VIOL_EN		0x00000002
-#define A6XX_CP_PROTECT_CNTL_ACCESS_PROT_EN			0x00000001
-
-#define REG_A6XX_CP_SCRATCH(i0) (0x00000883 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_CP_SCRATCH_REG(uint32_t i0) { return 0x00000883 + 0x1*i0; }
-
-#define REG_A6XX_CP_PROTECT(i0) (0x00000850 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_CP_PROTECT_REG(uint32_t i0) { return 0x00000850 + 0x1*i0; }
-#define A6XX_CP_PROTECT_REG_BASE_ADDR__MASK			0x0003ffff
-#define A6XX_CP_PROTECT_REG_BASE_ADDR__SHIFT			0
-static inline uint32_t A6XX_CP_PROTECT_REG_BASE_ADDR(uint32_t val)
-{
-	return ((val) << A6XX_CP_PROTECT_REG_BASE_ADDR__SHIFT) & A6XX_CP_PROTECT_REG_BASE_ADDR__MASK;
-}
-#define A6XX_CP_PROTECT_REG_MASK_LEN__MASK			0x7ffc0000
-#define A6XX_CP_PROTECT_REG_MASK_LEN__SHIFT			18
-static inline uint32_t A6XX_CP_PROTECT_REG_MASK_LEN(uint32_t val)
-{
-	return ((val) << A6XX_CP_PROTECT_REG_MASK_LEN__SHIFT) & A6XX_CP_PROTECT_REG_MASK_LEN__MASK;
-}
-#define A6XX_CP_PROTECT_REG_READ				0x80000000
-
-#define REG_A6XX_CP_CONTEXT_SWITCH_CNTL				0x000008a0
-
-#define REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO			0x000008a1
-
-#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR	0x000008a3
-
-#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR	0x000008a5
-
-#define REG_A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR	0x000008a7
-
-#define REG_A7XX_CP_CONTEXT_SWITCH_LEVEL_STATUS			0x000008ab
-
-#define REG_A6XX_CP_PERFCTR_CP_SEL(i0) (0x000008d0 + 0x1*(i0))
-
-#define REG_A7XX_CP_BV_PERFCTR_CP_SEL(i0) (0x000008e0 + 0x1*(i0))
-
-#define REG_A6XX_CP_CRASH_SCRIPT_BASE				0x00000900
-
-#define REG_A6XX_CP_CRASH_DUMP_CNTL				0x00000902
-
-#define REG_A6XX_CP_CRASH_DUMP_STATUS				0x00000903
-
-#define REG_A6XX_CP_SQE_STAT_ADDR				0x00000908
-
-#define REG_A6XX_CP_SQE_STAT_DATA				0x00000909
-
-#define REG_A6XX_CP_DRAW_STATE_ADDR				0x0000090a
-
-#define REG_A6XX_CP_DRAW_STATE_DATA				0x0000090b
-
-#define REG_A6XX_CP_ROQ_DBG_ADDR				0x0000090c
-
-#define REG_A6XX_CP_ROQ_DBG_DATA				0x0000090d
-
-#define REG_A6XX_CP_MEM_POOL_DBG_ADDR				0x0000090e
-
-#define REG_A6XX_CP_MEM_POOL_DBG_DATA				0x0000090f
-
-#define REG_A6XX_CP_SQE_UCODE_DBG_ADDR				0x00000910
-
-#define REG_A6XX_CP_SQE_UCODE_DBG_DATA				0x00000911
-
-#define REG_A6XX_CP_IB1_BASE					0x00000928
-
-#define REG_A6XX_CP_IB1_REM_SIZE				0x0000092a
-
-#define REG_A6XX_CP_IB2_BASE					0x0000092b
-
-#define REG_A6XX_CP_IB2_REM_SIZE				0x0000092d
-
-#define REG_A6XX_CP_SDS_BASE					0x0000092e
-
-#define REG_A6XX_CP_SDS_REM_SIZE				0x00000930
-
-#define REG_A6XX_CP_MRB_BASE					0x00000931
-
-#define REG_A6XX_CP_MRB_REM_SIZE				0x00000933
-
-#define REG_A6XX_CP_VSD_BASE					0x00000934
-
-#define REG_A6XX_CP_ROQ_RB_STAT					0x00000939
-#define A6XX_CP_ROQ_RB_STAT_RPTR__MASK				0x000003ff
-#define A6XX_CP_ROQ_RB_STAT_RPTR__SHIFT				0
-static inline uint32_t A6XX_CP_ROQ_RB_STAT_RPTR(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_RB_STAT_RPTR__SHIFT) & A6XX_CP_ROQ_RB_STAT_RPTR__MASK;
-}
-#define A6XX_CP_ROQ_RB_STAT_WPTR__MASK				0x03ff0000
-#define A6XX_CP_ROQ_RB_STAT_WPTR__SHIFT				16
-static inline uint32_t A6XX_CP_ROQ_RB_STAT_WPTR(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_RB_STAT_WPTR__SHIFT) & A6XX_CP_ROQ_RB_STAT_WPTR__MASK;
-}
-
-#define REG_A6XX_CP_ROQ_IB1_STAT				0x0000093a
-#define A6XX_CP_ROQ_IB1_STAT_RPTR__MASK				0x000003ff
-#define A6XX_CP_ROQ_IB1_STAT_RPTR__SHIFT			0
-static inline uint32_t A6XX_CP_ROQ_IB1_STAT_RPTR(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_IB1_STAT_RPTR__SHIFT) & A6XX_CP_ROQ_IB1_STAT_RPTR__MASK;
-}
-#define A6XX_CP_ROQ_IB1_STAT_WPTR__MASK				0x03ff0000
-#define A6XX_CP_ROQ_IB1_STAT_WPTR__SHIFT			16
-static inline uint32_t A6XX_CP_ROQ_IB1_STAT_WPTR(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_IB1_STAT_WPTR__SHIFT) & A6XX_CP_ROQ_IB1_STAT_WPTR__MASK;
-}
-
-#define REG_A6XX_CP_ROQ_IB2_STAT				0x0000093b
-#define A6XX_CP_ROQ_IB2_STAT_RPTR__MASK				0x000003ff
-#define A6XX_CP_ROQ_IB2_STAT_RPTR__SHIFT			0
-static inline uint32_t A6XX_CP_ROQ_IB2_STAT_RPTR(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_IB2_STAT_RPTR__SHIFT) & A6XX_CP_ROQ_IB2_STAT_RPTR__MASK;
-}
-#define A6XX_CP_ROQ_IB2_STAT_WPTR__MASK				0x03ff0000
-#define A6XX_CP_ROQ_IB2_STAT_WPTR__SHIFT			16
-static inline uint32_t A6XX_CP_ROQ_IB2_STAT_WPTR(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_IB2_STAT_WPTR__SHIFT) & A6XX_CP_ROQ_IB2_STAT_WPTR__MASK;
-}
-
-#define REG_A6XX_CP_ROQ_SDS_STAT				0x0000093c
-#define A6XX_CP_ROQ_SDS_STAT_RPTR__MASK				0x000003ff
-#define A6XX_CP_ROQ_SDS_STAT_RPTR__SHIFT			0
-static inline uint32_t A6XX_CP_ROQ_SDS_STAT_RPTR(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_SDS_STAT_RPTR__SHIFT) & A6XX_CP_ROQ_SDS_STAT_RPTR__MASK;
-}
-#define A6XX_CP_ROQ_SDS_STAT_WPTR__MASK				0x03ff0000
-#define A6XX_CP_ROQ_SDS_STAT_WPTR__SHIFT			16
-static inline uint32_t A6XX_CP_ROQ_SDS_STAT_WPTR(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_SDS_STAT_WPTR__SHIFT) & A6XX_CP_ROQ_SDS_STAT_WPTR__MASK;
-}
-
-#define REG_A6XX_CP_ROQ_MRB_STAT				0x0000093d
-#define A6XX_CP_ROQ_MRB_STAT_RPTR__MASK				0x000003ff
-#define A6XX_CP_ROQ_MRB_STAT_RPTR__SHIFT			0
-static inline uint32_t A6XX_CP_ROQ_MRB_STAT_RPTR(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_MRB_STAT_RPTR__SHIFT) & A6XX_CP_ROQ_MRB_STAT_RPTR__MASK;
-}
-#define A6XX_CP_ROQ_MRB_STAT_WPTR__MASK				0x03ff0000
-#define A6XX_CP_ROQ_MRB_STAT_WPTR__SHIFT			16
-static inline uint32_t A6XX_CP_ROQ_MRB_STAT_WPTR(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_MRB_STAT_WPTR__SHIFT) & A6XX_CP_ROQ_MRB_STAT_WPTR__MASK;
-}
-
-#define REG_A6XX_CP_ROQ_VSD_STAT				0x0000093e
-#define A6XX_CP_ROQ_VSD_STAT_RPTR__MASK				0x000003ff
-#define A6XX_CP_ROQ_VSD_STAT_RPTR__SHIFT			0
-static inline uint32_t A6XX_CP_ROQ_VSD_STAT_RPTR(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_VSD_STAT_RPTR__SHIFT) & A6XX_CP_ROQ_VSD_STAT_RPTR__MASK;
-}
-#define A6XX_CP_ROQ_VSD_STAT_WPTR__MASK				0x03ff0000
-#define A6XX_CP_ROQ_VSD_STAT_WPTR__SHIFT			16
-static inline uint32_t A6XX_CP_ROQ_VSD_STAT_WPTR(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_VSD_STAT_WPTR__SHIFT) & A6XX_CP_ROQ_VSD_STAT_WPTR__MASK;
-}
-
-#define REG_A6XX_CP_IB1_DWORDS					0x00000943
-
-#define REG_A6XX_CP_IB2_DWORDS					0x00000944
-
-#define REG_A6XX_CP_SDS_DWORDS					0x00000945
-
-#define REG_A6XX_CP_MRB_DWORDS					0x00000946
-
-#define REG_A6XX_CP_VSD_DWORDS					0x00000947
-
-#define REG_A6XX_CP_ROQ_AVAIL_RB				0x00000948
-#define A6XX_CP_ROQ_AVAIL_RB_REM__MASK				0xffff0000
-#define A6XX_CP_ROQ_AVAIL_RB_REM__SHIFT				16
-static inline uint32_t A6XX_CP_ROQ_AVAIL_RB_REM(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_AVAIL_RB_REM__SHIFT) & A6XX_CP_ROQ_AVAIL_RB_REM__MASK;
-}
-
-#define REG_A6XX_CP_ROQ_AVAIL_IB1				0x00000949
-#define A6XX_CP_ROQ_AVAIL_IB1_REM__MASK				0xffff0000
-#define A6XX_CP_ROQ_AVAIL_IB1_REM__SHIFT			16
-static inline uint32_t A6XX_CP_ROQ_AVAIL_IB1_REM(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_AVAIL_IB1_REM__SHIFT) & A6XX_CP_ROQ_AVAIL_IB1_REM__MASK;
-}
-
-#define REG_A6XX_CP_ROQ_AVAIL_IB2				0x0000094a
-#define A6XX_CP_ROQ_AVAIL_IB2_REM__MASK				0xffff0000
-#define A6XX_CP_ROQ_AVAIL_IB2_REM__SHIFT			16
-static inline uint32_t A6XX_CP_ROQ_AVAIL_IB2_REM(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_AVAIL_IB2_REM__SHIFT) & A6XX_CP_ROQ_AVAIL_IB2_REM__MASK;
-}
-
-#define REG_A6XX_CP_ROQ_AVAIL_SDS				0x0000094b
-#define A6XX_CP_ROQ_AVAIL_SDS_REM__MASK				0xffff0000
-#define A6XX_CP_ROQ_AVAIL_SDS_REM__SHIFT			16
-static inline uint32_t A6XX_CP_ROQ_AVAIL_SDS_REM(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_AVAIL_SDS_REM__SHIFT) & A6XX_CP_ROQ_AVAIL_SDS_REM__MASK;
-}
-
-#define REG_A6XX_CP_ROQ_AVAIL_MRB				0x0000094c
-#define A6XX_CP_ROQ_AVAIL_MRB_REM__MASK				0xffff0000
-#define A6XX_CP_ROQ_AVAIL_MRB_REM__SHIFT			16
-static inline uint32_t A6XX_CP_ROQ_AVAIL_MRB_REM(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_AVAIL_MRB_REM__SHIFT) & A6XX_CP_ROQ_AVAIL_MRB_REM__MASK;
-}
-
-#define REG_A6XX_CP_ROQ_AVAIL_VSD				0x0000094d
-#define A6XX_CP_ROQ_AVAIL_VSD_REM__MASK				0xffff0000
-#define A6XX_CP_ROQ_AVAIL_VSD_REM__SHIFT			16
-static inline uint32_t A6XX_CP_ROQ_AVAIL_VSD_REM(uint32_t val)
-{
-	return ((val) << A6XX_CP_ROQ_AVAIL_VSD_REM__SHIFT) & A6XX_CP_ROQ_AVAIL_VSD_REM__MASK;
-}
-
-#define REG_A6XX_CP_ALWAYS_ON_COUNTER				0x00000980
-
-#define REG_A6XX_CP_AHB_CNTL					0x0000098d
-
-#define REG_A6XX_CP_APERTURE_CNTL_HOST				0x00000a00
-
-#define REG_A7XX_CP_APERTURE_CNTL_HOST				0x00000a00
-#define A7XX_CP_APERTURE_CNTL_HOST_PIPE__MASK			0x00003000
-#define A7XX_CP_APERTURE_CNTL_HOST_PIPE__SHIFT			12
-static inline uint32_t A7XX_CP_APERTURE_CNTL_HOST_PIPE(enum a7xx_pipe val)
-{
-	return ((val) << A7XX_CP_APERTURE_CNTL_HOST_PIPE__SHIFT) & A7XX_CP_APERTURE_CNTL_HOST_PIPE__MASK;
-}
-#define A7XX_CP_APERTURE_CNTL_HOST_CLUSTER__MASK		0x00000700
-#define A7XX_CP_APERTURE_CNTL_HOST_CLUSTER__SHIFT		8
-static inline uint32_t A7XX_CP_APERTURE_CNTL_HOST_CLUSTER(enum a7xx_cluster val)
-{
-	return ((val) << A7XX_CP_APERTURE_CNTL_HOST_CLUSTER__SHIFT) & A7XX_CP_APERTURE_CNTL_HOST_CLUSTER__MASK;
-}
-#define A7XX_CP_APERTURE_CNTL_HOST_CONTEXT__MASK		0x00000030
-#define A7XX_CP_APERTURE_CNTL_HOST_CONTEXT__SHIFT		4
-static inline uint32_t A7XX_CP_APERTURE_CNTL_HOST_CONTEXT(uint32_t val)
-{
-	return ((val) << A7XX_CP_APERTURE_CNTL_HOST_CONTEXT__SHIFT) & A7XX_CP_APERTURE_CNTL_HOST_CONTEXT__MASK;
-}
-
-#define REG_A6XX_CP_APERTURE_CNTL_CD				0x00000a03
-
-#define REG_A7XX_CP_APERTURE_CNTL_CD				0x00000a03
-#define A7XX_CP_APERTURE_CNTL_CD_PIPE__MASK			0x00003000
-#define A7XX_CP_APERTURE_CNTL_CD_PIPE__SHIFT			12
-static inline uint32_t A7XX_CP_APERTURE_CNTL_CD_PIPE(enum a7xx_pipe val)
-{
-	return ((val) << A7XX_CP_APERTURE_CNTL_CD_PIPE__SHIFT) & A7XX_CP_APERTURE_CNTL_CD_PIPE__MASK;
-}
-#define A7XX_CP_APERTURE_CNTL_CD_CLUSTER__MASK			0x00000700
-#define A7XX_CP_APERTURE_CNTL_CD_CLUSTER__SHIFT			8
-static inline uint32_t A7XX_CP_APERTURE_CNTL_CD_CLUSTER(enum a7xx_cluster val)
-{
-	return ((val) << A7XX_CP_APERTURE_CNTL_CD_CLUSTER__SHIFT) & A7XX_CP_APERTURE_CNTL_CD_CLUSTER__MASK;
-}
-#define A7XX_CP_APERTURE_CNTL_CD_CONTEXT__MASK			0x00000030
-#define A7XX_CP_APERTURE_CNTL_CD_CONTEXT__SHIFT			4
-static inline uint32_t A7XX_CP_APERTURE_CNTL_CD_CONTEXT(uint32_t val)
-{
-	return ((val) << A7XX_CP_APERTURE_CNTL_CD_CONTEXT__SHIFT) & A7XX_CP_APERTURE_CNTL_CD_CONTEXT__MASK;
-}
-
-#define REG_A7XX_CP_BV_PROTECT_STATUS				0x00000a61
-
-#define REG_A7XX_CP_BV_HW_FAULT					0x00000a64
-
-#define REG_A7XX_CP_BV_DRAW_STATE_ADDR				0x00000a81
-
-#define REG_A7XX_CP_BV_DRAW_STATE_DATA				0x00000a82
-
-#define REG_A7XX_CP_BV_ROQ_DBG_ADDR				0x00000a83
-
-#define REG_A7XX_CP_BV_ROQ_DBG_DATA				0x00000a84
-
-#define REG_A7XX_CP_BV_SQE_UCODE_DBG_ADDR			0x00000a85
-
-#define REG_A7XX_CP_BV_SQE_UCODE_DBG_DATA			0x00000a86
-
-#define REG_A7XX_CP_BV_SQE_STAT_ADDR				0x00000a87
-
-#define REG_A7XX_CP_BV_SQE_STAT_DATA				0x00000a88
-
-#define REG_A7XX_CP_BV_MEM_POOL_DBG_ADDR			0x00000a96
-
-#define REG_A7XX_CP_BV_MEM_POOL_DBG_DATA			0x00000a97
-
-#define REG_A7XX_CP_BV_RB_RPTR_ADDR				0x00000a98
-
-#define REG_A7XX_CP_RESOURCE_TBL_DBG_ADDR			0x00000a9a
-
-#define REG_A7XX_CP_RESOURCE_TBL_DBG_DATA			0x00000a9b
-
-#define REG_A7XX_CP_BV_APRIV_CNTL				0x00000ad0
-
-#define REG_A7XX_CP_BV_CHICKEN_DBG				0x00000ada
-
-#define REG_A7XX_CP_LPAC_DRAW_STATE_ADDR			0x00000b0a
-
-#define REG_A7XX_CP_LPAC_DRAW_STATE_DATA			0x00000b0b
-
-#define REG_A7XX_CP_LPAC_ROQ_DBG_ADDR				0x00000b0c
-
-#define REG_A7XX_CP_SQE_AC_UCODE_DBG_ADDR			0x00000b27
-
-#define REG_A7XX_CP_SQE_AC_UCODE_DBG_DATA			0x00000b28
-
-#define REG_A7XX_CP_SQE_AC_STAT_ADDR				0x00000b29
-
-#define REG_A7XX_CP_SQE_AC_STAT_DATA				0x00000b2a
-
-#define REG_A7XX_CP_LPAC_APRIV_CNTL				0x00000b31
-
-#define REG_A6XX_CP_LPAC_PROG_FIFO_SIZE				0x00000b34
-
-#define REG_A7XX_CP_LPAC_ROQ_DBG_DATA				0x00000b35
-
-#define REG_A7XX_CP_LPAC_FIFO_DBG_DATA				0x00000b36
-
-#define REG_A7XX_CP_LPAC_FIFO_DBG_ADDR				0x00000b40
-
-#define REG_A6XX_CP_LPAC_SQE_INSTR_BASE				0x00000b82
-
-#define REG_A6XX_VSC_ADDR_MODE_CNTL				0x00000c01
-
-#define REG_A6XX_RBBM_GPR0_CNTL					0x00000018
-
-#define REG_A6XX_RBBM_INT_0_STATUS				0x00000201
-#define REG_A6XX_RBBM_STATUS					0x00000210
-#define A6XX_RBBM_STATUS_GPU_BUSY_IGN_AHB			0x00800000
-#define A6XX_RBBM_STATUS_GPU_BUSY_IGN_AHB_CP			0x00400000
-#define A6XX_RBBM_STATUS_HLSQ_BUSY				0x00200000
-#define A6XX_RBBM_STATUS_VSC_BUSY				0x00100000
-#define A6XX_RBBM_STATUS_TPL1_BUSY				0x00080000
-#define A6XX_RBBM_STATUS_SP_BUSY				0x00040000
-#define A6XX_RBBM_STATUS_UCHE_BUSY				0x00020000
-#define A6XX_RBBM_STATUS_VPC_BUSY				0x00010000
-#define A6XX_RBBM_STATUS_VFD_BUSY				0x00008000
-#define A6XX_RBBM_STATUS_TESS_BUSY				0x00004000
-#define A6XX_RBBM_STATUS_PC_VSD_BUSY				0x00002000
-#define A6XX_RBBM_STATUS_PC_DCALL_BUSY				0x00001000
-#define A6XX_RBBM_STATUS_COM_DCOM_BUSY				0x00000800
-#define A6XX_RBBM_STATUS_LRZ_BUSY				0x00000400
-#define A6XX_RBBM_STATUS_A2D_BUSY				0x00000200
-#define A6XX_RBBM_STATUS_CCU_BUSY				0x00000100
-#define A6XX_RBBM_STATUS_RB_BUSY				0x00000080
-#define A6XX_RBBM_STATUS_RAS_BUSY				0x00000040
-#define A6XX_RBBM_STATUS_TSE_BUSY				0x00000020
-#define A6XX_RBBM_STATUS_VBIF_BUSY				0x00000010
-#define A6XX_RBBM_STATUS_GFX_DBGC_BUSY				0x00000008
-#define A6XX_RBBM_STATUS_CP_BUSY				0x00000004
-#define A6XX_RBBM_STATUS_CP_AHB_BUSY_CP_MASTER			0x00000002
-#define A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER			0x00000001
-
-#define REG_A6XX_RBBM_STATUS1					0x00000211
-
-#define REG_A6XX_RBBM_STATUS2					0x00000212
-
-#define REG_A6XX_RBBM_STATUS3					0x00000213
-#define A6XX_RBBM_STATUS3_SMMU_STALLED_ON_FAULT			0x01000000
-
-#define REG_A6XX_RBBM_VBIF_GX_RESET_STATUS			0x00000215
-
-#define REG_A7XX_RBBM_CLOCK_MODE_CP				0x00000260
-
-#define REG_A7XX_RBBM_CLOCK_MODE_BV_LRZ				0x00000284
-
-#define REG_A7XX_RBBM_CLOCK_MODE_BV_GRAS			0x00000285
-
-#define REG_A7XX_RBBM_CLOCK_MODE2_GRAS				0x00000286
-
-#define REG_A7XX_RBBM_CLOCK_MODE_BV_VFD				0x00000287
-
-#define REG_A7XX_RBBM_CLOCK_MODE_BV_GPC				0x00000288
-
-#define REG_A6XX_RBBM_PERFCTR_CP(i0) (0x00000400 + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_RBBM(i0) (0x0000041c + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_PC(i0) (0x00000424 + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_VFD(i0) (0x00000434 + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_HLSQ(i0) (0x00000444 + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_VPC(i0) (0x00000450 + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_CCU(i0) (0x0000045c + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_TSE(i0) (0x00000466 + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_RAS(i0) (0x0000046e + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_UCHE(i0) (0x00000476 + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_TP(i0) (0x0000048e + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_SP(i0) (0x000004a6 + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_RB(i0) (0x000004d6 + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_VSC(i0) (0x000004e6 + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_LRZ(i0) (0x000004ea + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_CMP(i0) (0x000004f2 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_CP(i0) (0x00000300 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_RBBM(i0) (0x0000031c + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_PC(i0) (0x00000324 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_VFD(i0) (0x00000334 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_HLSQ(i0) (0x00000344 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_VPC(i0) (0x00000350 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_CCU(i0) (0x0000035c + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_TSE(i0) (0x00000366 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_RAS(i0) (0x0000036e + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_UCHE(i0) (0x00000376 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_TP(i0) (0x0000038e + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_SP(i0) (0x000003a6 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_RB(i0) (0x000003d6 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_VSC(i0) (0x000003e6 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_LRZ(i0) (0x000003ea + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_CMP(i0) (0x000003f2 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_UFC(i0) (0x000003fa + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR2_HLSQ(i0) (0x00000410 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR2_CP(i0) (0x0000041c + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR2_SP(i0) (0x0000042a + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR2_TP(i0) (0x00000442 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR2_UFC(i0) (0x0000044e + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_BV_PC(i0) (0x00000460 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_BV_VFD(i0) (0x00000470 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_BV_VPC(i0) (0x00000480 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_BV_TSE(i0) (0x0000048c + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_BV_RAS(i0) (0x00000494 + 0x2*(i0))
-
-#define REG_A7XX_RBBM_PERFCTR_BV_LRZ(i0) (0x0000049c + 0x2*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_CNTL				0x00000500
-
-#define REG_A6XX_RBBM_PERFCTR_LOAD_CMD0				0x00000501
-
-#define REG_A6XX_RBBM_PERFCTR_LOAD_CMD1				0x00000502
-
-#define REG_A6XX_RBBM_PERFCTR_LOAD_CMD2				0x00000503
-
-#define REG_A6XX_RBBM_PERFCTR_LOAD_CMD3				0x00000504
-
-#define REG_A6XX_RBBM_PERFCTR_LOAD_VALUE_LO			0x00000505
-
-#define REG_A6XX_RBBM_PERFCTR_LOAD_VALUE_HI			0x00000506
-
-#define REG_A6XX_RBBM_PERFCTR_RBBM_SEL(i0) (0x00000507 + 0x1*(i0))
-
-#define REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED			0x0000050b
-
-#define REG_A6XX_RBBM_PERFCTR_SRAM_INIT_CMD			0x0000050e
-
-#define REG_A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS			0x0000050f
-
-#define REG_A6XX_RBBM_ISDB_CNT					0x00000533
-
-#define REG_A7XX_RBBM_NC_MODE_CNTL				0x00000534
-
-#define REG_A7XX_RBBM_SNAPSHOT_STATUS				0x00000535
-
-#define REG_A6XX_RBBM_PRIMCTR_0_LO				0x00000540
-
-#define REG_A6XX_RBBM_PRIMCTR_0_HI				0x00000541
-
-#define REG_A6XX_RBBM_PRIMCTR_1_LO				0x00000542
-
-#define REG_A6XX_RBBM_PRIMCTR_1_HI				0x00000543
-
-#define REG_A6XX_RBBM_PRIMCTR_2_LO				0x00000544
-
-#define REG_A6XX_RBBM_PRIMCTR_2_HI				0x00000545
-
-#define REG_A6XX_RBBM_PRIMCTR_3_LO				0x00000546
-
-#define REG_A6XX_RBBM_PRIMCTR_3_HI				0x00000547
-
-#define REG_A6XX_RBBM_PRIMCTR_4_LO				0x00000548
-
-#define REG_A6XX_RBBM_PRIMCTR_4_HI				0x00000549
-
-#define REG_A6XX_RBBM_PRIMCTR_5_LO				0x0000054a
-
-#define REG_A6XX_RBBM_PRIMCTR_5_HI				0x0000054b
-
-#define REG_A6XX_RBBM_PRIMCTR_6_LO				0x0000054c
-
-#define REG_A6XX_RBBM_PRIMCTR_6_HI				0x0000054d
-
-#define REG_A6XX_RBBM_PRIMCTR_7_LO				0x0000054e
-
-#define REG_A6XX_RBBM_PRIMCTR_7_HI				0x0000054f
-
-#define REG_A6XX_RBBM_PRIMCTR_8_LO				0x00000550
-
-#define REG_A6XX_RBBM_PRIMCTR_8_HI				0x00000551
-
-#define REG_A6XX_RBBM_PRIMCTR_9_LO				0x00000552
-
-#define REG_A6XX_RBBM_PRIMCTR_9_HI				0x00000553
-
-#define REG_A6XX_RBBM_PRIMCTR_10_LO				0x00000554
-
-#define REG_A6XX_RBBM_PRIMCTR_10_HI				0x00000555
-
-#define REG_A6XX_RBBM_SECVID_TRUST_CNTL				0x0000f400
-
-#define REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE			0x0000f800
-
-#define REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE			0x0000f802
-
-#define REG_A6XX_RBBM_SECVID_TSB_CNTL				0x0000f803
-
-#define REG_A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL			0x0000f810
-
-#define REG_A7XX_RBBM_SECVID_TSB_STATUS				0x0000fc00
-
-#define REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL			0x00000010
-
-#define REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL			0x00000011
-
-#define REG_A6XX_RBBM_GBIF_HALT					0x00000016
-
-#define REG_A6XX_RBBM_GBIF_HALT_ACK				0x00000017
-
-#define REG_A6XX_RBBM_WAIT_FOR_GPU_IDLE_CMD			0x0000001c
-#define A6XX_RBBM_WAIT_FOR_GPU_IDLE_CMD_WAIT_GPU_IDLE		0x00000001
-
-#define REG_A7XX_RBBM_GBIF_HALT					0x00000016
-
-#define REG_A7XX_RBBM_GBIF_HALT_ACK				0x00000017
-
-#define REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL			0x0000001f
-
-#define REG_A6XX_RBBM_INT_CLEAR_CMD				0x00000037
-#define REG_A6XX_RBBM_INT_0_MASK				0x00000038
-#define REG_A7XX_RBBM_INT_2_MASK				0x0000003a
-
-#define REG_A6XX_RBBM_SP_HYST_CNT				0x00000042
-
-#define REG_A6XX_RBBM_SW_RESET_CMD				0x00000043
-
-#define REG_A6XX_RBBM_RAC_THRESHOLD_CNT				0x00000044
-
-#define REG_A6XX_RBBM_BLOCK_SW_RESET_CMD			0x00000045
-
-#define REG_A6XX_RBBM_BLOCK_SW_RESET_CMD2			0x00000046
-
-#define REG_A7XX_RBBM_CLOCK_CNTL_GLOBAL				0x000000ad
-
-#define REG_A6XX_RBBM_CLOCK_CNTL				0x000000ae
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_SP0				0x000000b0
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_SP1				0x000000b1
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_SP2				0x000000b2
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_SP3				0x000000b3
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_SP0				0x000000b4
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_SP1				0x000000b5
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_SP2				0x000000b6
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_SP3				0x000000b7
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_SP0				0x000000b8
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_SP1				0x000000b9
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_SP2				0x000000ba
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_SP3				0x000000bb
-
-#define REG_A6XX_RBBM_CLOCK_HYST_SP0				0x000000bc
-
-#define REG_A6XX_RBBM_CLOCK_HYST_SP1				0x000000bd
-
-#define REG_A6XX_RBBM_CLOCK_HYST_SP2				0x000000be
-
-#define REG_A6XX_RBBM_CLOCK_HYST_SP3				0x000000bf
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_TP0				0x000000c0
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_TP1				0x000000c1
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_TP2				0x000000c2
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_TP3				0x000000c3
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_TP0				0x000000c4
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_TP1				0x000000c5
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_TP2				0x000000c6
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_TP3				0x000000c7
-
-#define REG_A6XX_RBBM_CLOCK_CNTL3_TP0				0x000000c8
-
-#define REG_A6XX_RBBM_CLOCK_CNTL3_TP1				0x000000c9
-
-#define REG_A6XX_RBBM_CLOCK_CNTL3_TP2				0x000000ca
-
-#define REG_A6XX_RBBM_CLOCK_CNTL3_TP3				0x000000cb
-
-#define REG_A6XX_RBBM_CLOCK_CNTL4_TP0				0x000000cc
-
-#define REG_A6XX_RBBM_CLOCK_CNTL4_TP1				0x000000cd
-
-#define REG_A6XX_RBBM_CLOCK_CNTL4_TP2				0x000000ce
-
-#define REG_A6XX_RBBM_CLOCK_CNTL4_TP3				0x000000cf
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_TP0				0x000000d0
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_TP1				0x000000d1
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_TP2				0x000000d2
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_TP3				0x000000d3
-
-#define REG_A6XX_RBBM_CLOCK_DELAY2_TP0				0x000000d4
-
-#define REG_A6XX_RBBM_CLOCK_DELAY2_TP1				0x000000d5
-
-#define REG_A6XX_RBBM_CLOCK_DELAY2_TP2				0x000000d6
-
-#define REG_A6XX_RBBM_CLOCK_DELAY2_TP3				0x000000d7
-
-#define REG_A6XX_RBBM_CLOCK_DELAY3_TP0				0x000000d8
-
-#define REG_A6XX_RBBM_CLOCK_DELAY3_TP1				0x000000d9
-
-#define REG_A6XX_RBBM_CLOCK_DELAY3_TP2				0x000000da
-
-#define REG_A6XX_RBBM_CLOCK_DELAY3_TP3				0x000000db
-
-#define REG_A6XX_RBBM_CLOCK_DELAY4_TP0				0x000000dc
-
-#define REG_A6XX_RBBM_CLOCK_DELAY4_TP1				0x000000dd
-
-#define REG_A6XX_RBBM_CLOCK_DELAY4_TP2				0x000000de
-
-#define REG_A6XX_RBBM_CLOCK_DELAY4_TP3				0x000000df
-
-#define REG_A6XX_RBBM_CLOCK_HYST_TP0				0x000000e0
-
-#define REG_A6XX_RBBM_CLOCK_HYST_TP1				0x000000e1
-
-#define REG_A6XX_RBBM_CLOCK_HYST_TP2				0x000000e2
-
-#define REG_A6XX_RBBM_CLOCK_HYST_TP3				0x000000e3
-
-#define REG_A6XX_RBBM_CLOCK_HYST2_TP0				0x000000e4
-
-#define REG_A6XX_RBBM_CLOCK_HYST2_TP1				0x000000e5
-
-#define REG_A6XX_RBBM_CLOCK_HYST2_TP2				0x000000e6
-
-#define REG_A6XX_RBBM_CLOCK_HYST2_TP3				0x000000e7
-
-#define REG_A6XX_RBBM_CLOCK_HYST3_TP0				0x000000e8
-
-#define REG_A6XX_RBBM_CLOCK_HYST3_TP1				0x000000e9
-
-#define REG_A6XX_RBBM_CLOCK_HYST3_TP2				0x000000ea
-
-#define REG_A6XX_RBBM_CLOCK_HYST3_TP3				0x000000eb
-
-#define REG_A6XX_RBBM_CLOCK_HYST4_TP0				0x000000ec
-
-#define REG_A6XX_RBBM_CLOCK_HYST4_TP1				0x000000ed
-
-#define REG_A6XX_RBBM_CLOCK_HYST4_TP2				0x000000ee
-
-#define REG_A6XX_RBBM_CLOCK_HYST4_TP3				0x000000ef
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_RB0				0x000000f0
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_RB1				0x000000f1
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_RB2				0x000000f2
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_RB3				0x000000f3
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_RB0				0x000000f4
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_RB1				0x000000f5
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_RB2				0x000000f6
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_RB3				0x000000f7
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_CCU0				0x000000f8
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_CCU1				0x000000f9
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_CCU2				0x000000fa
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_CCU3				0x000000fb
-
-#define REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0			0x00000100
-
-#define REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1			0x00000101
-
-#define REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2			0x00000102
-
-#define REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3			0x00000103
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_RAC				0x00000104
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_RAC				0x00000105
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_RAC				0x00000106
-
-#define REG_A6XX_RBBM_CLOCK_HYST_RAC				0x00000107
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM			0x00000108
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM			0x00000109
-
-#define REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM			0x0000010a
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_UCHE				0x0000010b
-
-#define REG_A6XX_RBBM_CLOCK_CNTL2_UCHE				0x0000010c
-
-#define REG_A6XX_RBBM_CLOCK_CNTL3_UCHE				0x0000010d
-
-#define REG_A6XX_RBBM_CLOCK_CNTL4_UCHE				0x0000010e
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_UCHE				0x0000010f
-
-#define REG_A6XX_RBBM_CLOCK_HYST_UCHE				0x00000110
-
-#define REG_A6XX_RBBM_CLOCK_MODE_VFD				0x00000111
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_VFD				0x00000112
-
-#define REG_A6XX_RBBM_CLOCK_HYST_VFD				0x00000113
-
-#define REG_A6XX_RBBM_CLOCK_MODE_GPC				0x00000114
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_GPC				0x00000115
-
-#define REG_A6XX_RBBM_CLOCK_HYST_GPC				0x00000116
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2			0x00000117
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX				0x00000118
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX			0x00000119
-
-#define REG_A6XX_RBBM_CLOCK_HYST_GMU_GX				0x0000011a
-
-#define REG_A6XX_RBBM_CLOCK_MODE_HLSQ				0x0000011b
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_HLSQ				0x0000011c
-
-#define REG_A6XX_RBBM_CLOCK_HYST_HLSQ				0x0000011d
-
-#define REG_A7XX_RBBM_CGC_GLOBAL_LOAD_CMD			0x0000011e
-
-#define REG_A7XX_RBBM_CGC_P2S_TRIG_CMD				0x0000011f
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE			0x00000120
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE			0x00000121
-
-#define REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE			0x00000122
-
-#define REG_A7XX_RBBM_CGC_P2S_STATUS				0x00000122
-#define A7XX_RBBM_CGC_P2S_STATUS_TXDONE				0x00000001
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_FCHE				0x00000123
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_FCHE				0x00000124
-
-#define REG_A6XX_RBBM_CLOCK_HYST_FCHE				0x00000125
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_MHUB				0x00000126
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_MHUB				0x00000127
-
-#define REG_A6XX_RBBM_CLOCK_HYST_MHUB				0x00000128
-
-#define REG_A6XX_RBBM_CLOCK_DELAY_GLC				0x00000129
-
-#define REG_A6XX_RBBM_CLOCK_HYST_GLC				0x0000012a
-
-#define REG_A6XX_RBBM_CLOCK_CNTL_GLC				0x0000012b
-
-#define REG_A7XX_RBBM_CLOCK_HYST2_VFD				0x0000012f
-
-#define REG_A6XX_RBBM_LPAC_GBIF_CLIENT_QOS_CNTL			0x000005ff
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_SEL_A				0x00000600
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_SEL_B				0x00000601
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_SEL_C				0x00000602
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_SEL_D				0x00000603
-#define A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__MASK		0x000000ff
-#define A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__SHIFT		0
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__SHIFT) & A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__MASK		0x0000ff00
-#define A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__SHIFT		8
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__SHIFT) & A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__MASK;
-}
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_CNTLT				0x00000604
-#define A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__MASK		0x0000003f
-#define A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__SHIFT		0
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__SHIFT) & A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU__MASK			0x00007000
-#define A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU__SHIFT			12
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU__SHIFT) & A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT__MASK			0xf0000000
-#define A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT__SHIFT			28
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT__SHIFT) & A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT__MASK;
-}
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_CNTLM				0x00000605
-#define A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__MASK			0x0f000000
-#define A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__SHIFT		24
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__SHIFT) & A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__MASK;
-}
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_IVTL_0				0x00000608
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_IVTL_1				0x00000609
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_IVTL_2				0x0000060a
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_IVTL_3				0x0000060b
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_MASKL_0			0x0000060c
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_MASKL_1			0x0000060d
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_MASKL_2			0x0000060e
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_MASKL_3			0x0000060f
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_BYTEL_0			0x00000610
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__MASK		0x0000000f
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__SHIFT		0
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__MASK		0x000000f0
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__SHIFT		4
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__MASK		0x00000f00
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__SHIFT		8
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__MASK		0x0000f000
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__SHIFT		12
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__MASK		0x000f0000
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__SHIFT		16
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__MASK		0x00f00000
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__SHIFT		20
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__MASK		0x0f000000
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__SHIFT		24
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__MASK		0xf0000000
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__SHIFT		28
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__MASK;
-}
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_BYTEL_1			0x00000611
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__MASK		0x0000000f
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__SHIFT		0
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__MASK		0x000000f0
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__SHIFT		4
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__MASK		0x00000f00
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__SHIFT		8
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__MASK		0x0000f000
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__SHIFT		12
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__MASK		0x000f0000
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__SHIFT		16
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__MASK		0x00f00000
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__SHIFT		20
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__MASK		0x0f000000
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__SHIFT		24
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__MASK;
-}
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__MASK		0xf0000000
-#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__SHIFT		28
-static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15(uint32_t val)
-{
-	return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__MASK;
-}
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_TRACE_BUF1			0x0000062f
-
-#define REG_A6XX_DBGC_CFG_DBGBUS_TRACE_BUF2			0x00000630
-
-#define REG_A6XX_VSC_PERFCTR_VSC_SEL(i0) (0x00000cd8 + 0x1*(i0))
-
-#define REG_A7XX_VSC_UNKNOWN_0CD8				0x00000cd8
-#define A7XX_VSC_UNKNOWN_0CD8_BINNING				0x00000001
-
-#define REG_A6XX_HLSQ_DBG_AHB_READ_APERTURE			0x0000c800
-
-#define REG_A6XX_HLSQ_DBG_READ_SEL				0x0000d000
-
-#define REG_A6XX_UCHE_ADDR_MODE_CNTL				0x00000e00
-
-#define REG_A6XX_UCHE_MODE_CNTL					0x00000e01
-
-#define REG_A6XX_UCHE_WRITE_RANGE_MAX				0x00000e05
-
-#define REG_A6XX_UCHE_WRITE_THRU_BASE				0x00000e07
-
-#define REG_A6XX_UCHE_TRAP_BASE					0x00000e09
-
-#define REG_A6XX_UCHE_GMEM_RANGE_MIN				0x00000e0b
-
-#define REG_A6XX_UCHE_GMEM_RANGE_MAX				0x00000e0d
-
-#define REG_A6XX_UCHE_CACHE_WAYS				0x00000e17
-
-#define REG_A6XX_UCHE_FILTER_CNTL				0x00000e18
-
-#define REG_A6XX_UCHE_CLIENT_PF					0x00000e19
-#define A6XX_UCHE_CLIENT_PF_PERFSEL__MASK			0x000000ff
-#define A6XX_UCHE_CLIENT_PF_PERFSEL__SHIFT			0
-static inline uint32_t A6XX_UCHE_CLIENT_PF_PERFSEL(uint32_t val)
-{
-	return ((val) << A6XX_UCHE_CLIENT_PF_PERFSEL__SHIFT) & A6XX_UCHE_CLIENT_PF_PERFSEL__MASK;
-}
-
-#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL(i0) (0x00000e1c + 0x1*(i0))
-
-#define REG_A6XX_UCHE_GBIF_GX_CONFIG				0x00000e3a
-
-#define REG_A6XX_UCHE_CMDQ_CONFIG				0x00000e3c
-
-#define REG_A6XX_VBIF_VERSION					0x00003000
-
-#define REG_A6XX_VBIF_CLKON					0x00003001
-#define A6XX_VBIF_CLKON_FORCE_ON_TESTBUS			0x00000002
-
-#define REG_A6XX_VBIF_GATE_OFF_WRREQ_EN				0x0000302a
-
-#define REG_A6XX_VBIF_XIN_HALT_CTRL0				0x00003080
-
-#define REG_A6XX_VBIF_XIN_HALT_CTRL1				0x00003081
-
-#define REG_A6XX_VBIF_TEST_BUS_OUT_CTRL				0x00003084
-
-#define REG_A6XX_VBIF_TEST_BUS1_CTRL0				0x00003085
-
-#define REG_A6XX_VBIF_TEST_BUS1_CTRL1				0x00003086
-#define A6XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL__MASK		0x0000000f
-#define A6XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL__SHIFT		0
-static inline uint32_t A6XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL(uint32_t val)
-{
-	return ((val) << A6XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL__SHIFT) & A6XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL__MASK;
-}
-
-#define REG_A6XX_VBIF_TEST_BUS2_CTRL0				0x00003087
-
-#define REG_A6XX_VBIF_TEST_BUS2_CTRL1				0x00003088
-#define A6XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL__MASK		0x000001ff
-#define A6XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL__SHIFT		0
-static inline uint32_t A6XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL(uint32_t val)
-{
-	return ((val) << A6XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL__SHIFT) & A6XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL__MASK;
-}
-
-#define REG_A6XX_VBIF_TEST_BUS_OUT				0x0000308c
-
-#define REG_A6XX_VBIF_PERF_CNT_SEL0				0x000030d0
-
-#define REG_A6XX_VBIF_PERF_CNT_SEL1				0x000030d1
-
-#define REG_A6XX_VBIF_PERF_CNT_SEL2				0x000030d2
-
-#define REG_A6XX_VBIF_PERF_CNT_SEL3				0x000030d3
-
-#define REG_A6XX_VBIF_PERF_CNT_LOW0				0x000030d8
-
-#define REG_A6XX_VBIF_PERF_CNT_LOW1				0x000030d9
-
-#define REG_A6XX_VBIF_PERF_CNT_LOW2				0x000030da
-
-#define REG_A6XX_VBIF_PERF_CNT_LOW3				0x000030db
-
-#define REG_A6XX_VBIF_PERF_CNT_HIGH0				0x000030e0
-
-#define REG_A6XX_VBIF_PERF_CNT_HIGH1				0x000030e1
-
-#define REG_A6XX_VBIF_PERF_CNT_HIGH2				0x000030e2
-
-#define REG_A6XX_VBIF_PERF_CNT_HIGH3				0x000030e3
-
-#define REG_A6XX_VBIF_PERF_PWR_CNT_EN0				0x00003100
-
-#define REG_A6XX_VBIF_PERF_PWR_CNT_EN1				0x00003101
-
-#define REG_A6XX_VBIF_PERF_PWR_CNT_EN2				0x00003102
-
-#define REG_A6XX_VBIF_PERF_PWR_CNT_LOW0				0x00003110
-
-#define REG_A6XX_VBIF_PERF_PWR_CNT_LOW1				0x00003111
-
-#define REG_A6XX_VBIF_PERF_PWR_CNT_LOW2				0x00003112
-
-#define REG_A6XX_VBIF_PERF_PWR_CNT_HIGH0			0x00003118
-
-#define REG_A6XX_VBIF_PERF_PWR_CNT_HIGH1			0x00003119
-
-#define REG_A6XX_VBIF_PERF_PWR_CNT_HIGH2			0x0000311a
-
-#define REG_A6XX_GBIF_SCACHE_CNTL0				0x00003c01
-
-#define REG_A6XX_GBIF_SCACHE_CNTL1				0x00003c02
-
-#define REG_A6XX_GBIF_QSB_SIDE0					0x00003c03
-
-#define REG_A6XX_GBIF_QSB_SIDE1					0x00003c04
-
-#define REG_A6XX_GBIF_QSB_SIDE2					0x00003c05
-
-#define REG_A6XX_GBIF_QSB_SIDE3					0x00003c06
-
-#define REG_A6XX_GBIF_HALT					0x00003c45
-
-#define REG_A6XX_GBIF_HALT_ACK					0x00003c46
-
-#define REG_A6XX_GBIF_PERF_PWR_CNT_EN				0x00003cc0
-
-#define REG_A6XX_GBIF_PERF_PWR_CNT_CLR				0x00003cc1
-
-#define REG_A6XX_GBIF_PERF_CNT_SEL				0x00003cc2
-
-#define REG_A6XX_GBIF_PERF_PWR_CNT_SEL				0x00003cc3
-
-#define REG_A6XX_GBIF_PERF_CNT_LOW0				0x00003cc4
-
-#define REG_A6XX_GBIF_PERF_CNT_LOW1				0x00003cc5
-
-#define REG_A6XX_GBIF_PERF_CNT_LOW2				0x00003cc6
-
-#define REG_A6XX_GBIF_PERF_CNT_LOW3				0x00003cc7
-
-#define REG_A6XX_GBIF_PERF_CNT_HIGH0				0x00003cc8
-
-#define REG_A6XX_GBIF_PERF_CNT_HIGH1				0x00003cc9
-
-#define REG_A6XX_GBIF_PERF_CNT_HIGH2				0x00003cca
-
-#define REG_A6XX_GBIF_PERF_CNT_HIGH3				0x00003ccb
-
-#define REG_A6XX_GBIF_PWR_CNT_LOW0				0x00003ccc
-
-#define REG_A6XX_GBIF_PWR_CNT_LOW1				0x00003ccd
-
-#define REG_A6XX_GBIF_PWR_CNT_LOW2				0x00003cce
-
-#define REG_A6XX_GBIF_PWR_CNT_HIGH0				0x00003ccf
-
-#define REG_A6XX_GBIF_PWR_CNT_HIGH1				0x00003cd0
-
-#define REG_A6XX_GBIF_PWR_CNT_HIGH2				0x00003cd1
-
-#define REG_A6XX_VSC_DBG_ECO_CNTL				0x00000c00
-
-#define REG_A6XX_VSC_BIN_SIZE					0x00000c02
-#define A6XX_VSC_BIN_SIZE_WIDTH__MASK				0x000000ff
-#define A6XX_VSC_BIN_SIZE_WIDTH__SHIFT				0
-static inline uint32_t A6XX_VSC_BIN_SIZE_WIDTH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A6XX_VSC_BIN_SIZE_WIDTH__SHIFT) & A6XX_VSC_BIN_SIZE_WIDTH__MASK;
-}
-#define A6XX_VSC_BIN_SIZE_HEIGHT__MASK				0x0001ff00
-#define A6XX_VSC_BIN_SIZE_HEIGHT__SHIFT				8
-static inline uint32_t A6XX_VSC_BIN_SIZE_HEIGHT(uint32_t val)
-{
-	assert(!(val & 0xf));
-	return (((val >> 4)) << A6XX_VSC_BIN_SIZE_HEIGHT__SHIFT) & A6XX_VSC_BIN_SIZE_HEIGHT__MASK;
-}
-
-#define REG_A6XX_VSC_DRAW_STRM_SIZE_ADDRESS			0x00000c03
-
-#define REG_A6XX_VSC_BIN_COUNT					0x00000c06
-#define A6XX_VSC_BIN_COUNT_NX__MASK				0x000007fe
-#define A6XX_VSC_BIN_COUNT_NX__SHIFT				1
-static inline uint32_t A6XX_VSC_BIN_COUNT_NX(uint32_t val)
-{
-	return ((val) << A6XX_VSC_BIN_COUNT_NX__SHIFT) & A6XX_VSC_BIN_COUNT_NX__MASK;
-}
-#define A6XX_VSC_BIN_COUNT_NY__MASK				0x001ff800
-#define A6XX_VSC_BIN_COUNT_NY__SHIFT				11
-static inline uint32_t A6XX_VSC_BIN_COUNT_NY(uint32_t val)
-{
-	return ((val) << A6XX_VSC_BIN_COUNT_NY__SHIFT) & A6XX_VSC_BIN_COUNT_NY__MASK;
-}
-
-#define REG_A6XX_VSC_PIPE_CONFIG(i0) (0x00000c10 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_VSC_PIPE_CONFIG_REG(uint32_t i0) { return 0x00000c10 + 0x1*i0; }
-#define A6XX_VSC_PIPE_CONFIG_REG_X__MASK			0x000003ff
-#define A6XX_VSC_PIPE_CONFIG_REG_X__SHIFT			0
-static inline uint32_t A6XX_VSC_PIPE_CONFIG_REG_X(uint32_t val)
-{
-	return ((val) << A6XX_VSC_PIPE_CONFIG_REG_X__SHIFT) & A6XX_VSC_PIPE_CONFIG_REG_X__MASK;
-}
-#define A6XX_VSC_PIPE_CONFIG_REG_Y__MASK			0x000ffc00
-#define A6XX_VSC_PIPE_CONFIG_REG_Y__SHIFT			10
-static inline uint32_t A6XX_VSC_PIPE_CONFIG_REG_Y(uint32_t val)
-{
-	return ((val) << A6XX_VSC_PIPE_CONFIG_REG_Y__SHIFT) & A6XX_VSC_PIPE_CONFIG_REG_Y__MASK;
-}
-#define A6XX_VSC_PIPE_CONFIG_REG_W__MASK			0x03f00000
-#define A6XX_VSC_PIPE_CONFIG_REG_W__SHIFT			20
-static inline uint32_t A6XX_VSC_PIPE_CONFIG_REG_W(uint32_t val)
-{
-	return ((val) << A6XX_VSC_PIPE_CONFIG_REG_W__SHIFT) & A6XX_VSC_PIPE_CONFIG_REG_W__MASK;
-}
-#define A6XX_VSC_PIPE_CONFIG_REG_H__MASK			0xfc000000
-#define A6XX_VSC_PIPE_CONFIG_REG_H__SHIFT			26
-static inline uint32_t A6XX_VSC_PIPE_CONFIG_REG_H(uint32_t val)
-{
-	return ((val) << A6XX_VSC_PIPE_CONFIG_REG_H__SHIFT) & A6XX_VSC_PIPE_CONFIG_REG_H__MASK;
-}
-
-#define REG_A6XX_VSC_PRIM_STRM_ADDRESS				0x00000c30
-
-#define REG_A6XX_VSC_PRIM_STRM_PITCH				0x00000c32
-
-#define REG_A6XX_VSC_PRIM_STRM_LIMIT				0x00000c33
-
-#define REG_A6XX_VSC_DRAW_STRM_ADDRESS				0x00000c34
-
-#define REG_A6XX_VSC_DRAW_STRM_PITCH				0x00000c36
-
-#define REG_A6XX_VSC_DRAW_STRM_LIMIT				0x00000c37
-
-#define REG_A6XX_VSC_STATE(i0) (0x00000c38 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_VSC_STATE_REG(uint32_t i0) { return 0x00000c38 + 0x1*i0; }
-
-#define REG_A6XX_VSC_PRIM_STRM_SIZE(i0) (0x00000c58 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_VSC_PRIM_STRM_SIZE_REG(uint32_t i0) { return 0x00000c58 + 0x1*i0; }
-
-#define REG_A6XX_VSC_DRAW_STRM_SIZE(i0) (0x00000c78 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_VSC_DRAW_STRM_SIZE_REG(uint32_t i0) { return 0x00000c78 + 0x1*i0; }
-
-#define REG_A7XX_UCHE_UNKNOWN_0E10				0x00000e10
-
-#define REG_A7XX_UCHE_UNKNOWN_0E11				0x00000e11
-
-#define REG_A6XX_UCHE_UNKNOWN_0E12				0x00000e12
-
-#define REG_A6XX_GRAS_CL_CNTL					0x00008000
-#define A6XX_GRAS_CL_CNTL_CLIP_DISABLE				0x00000001
-#define A6XX_GRAS_CL_CNTL_ZNEAR_CLIP_DISABLE			0x00000002
-#define A6XX_GRAS_CL_CNTL_ZFAR_CLIP_DISABLE			0x00000004
-#define A6XX_GRAS_CL_CNTL_Z_CLAMP_ENABLE			0x00000020
-#define A6XX_GRAS_CL_CNTL_ZERO_GB_SCALE_Z			0x00000040
-#define A6XX_GRAS_CL_CNTL_VP_CLIP_CODE_IGNORE			0x00000080
-#define A6XX_GRAS_CL_CNTL_VP_XFORM_DISABLE			0x00000100
-#define A6XX_GRAS_CL_CNTL_PERSP_DIVISION_DISABLE		0x00000200
-
-#define REG_A6XX_GRAS_VS_CL_CNTL				0x00008001
-#define A6XX_GRAS_VS_CL_CNTL_CLIP_MASK__MASK			0x000000ff
-#define A6XX_GRAS_VS_CL_CNTL_CLIP_MASK__SHIFT			0
-static inline uint32_t A6XX_GRAS_VS_CL_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_VS_CL_CNTL_CLIP_MASK__SHIFT) & A6XX_GRAS_VS_CL_CNTL_CLIP_MASK__MASK;
-}
-#define A6XX_GRAS_VS_CL_CNTL_CULL_MASK__MASK			0x0000ff00
-#define A6XX_GRAS_VS_CL_CNTL_CULL_MASK__SHIFT			8
-static inline uint32_t A6XX_GRAS_VS_CL_CNTL_CULL_MASK(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_VS_CL_CNTL_CULL_MASK__SHIFT) & A6XX_GRAS_VS_CL_CNTL_CULL_MASK__MASK;
-}
-
-#define REG_A6XX_GRAS_DS_CL_CNTL				0x00008002
-#define A6XX_GRAS_DS_CL_CNTL_CLIP_MASK__MASK			0x000000ff
-#define A6XX_GRAS_DS_CL_CNTL_CLIP_MASK__SHIFT			0
-static inline uint32_t A6XX_GRAS_DS_CL_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_DS_CL_CNTL_CLIP_MASK__SHIFT) & A6XX_GRAS_DS_CL_CNTL_CLIP_MASK__MASK;
-}
-#define A6XX_GRAS_DS_CL_CNTL_CULL_MASK__MASK			0x0000ff00
-#define A6XX_GRAS_DS_CL_CNTL_CULL_MASK__SHIFT			8
-static inline uint32_t A6XX_GRAS_DS_CL_CNTL_CULL_MASK(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_DS_CL_CNTL_CULL_MASK__SHIFT) & A6XX_GRAS_DS_CL_CNTL_CULL_MASK__MASK;
-}
-
-#define REG_A6XX_GRAS_GS_CL_CNTL				0x00008003
-#define A6XX_GRAS_GS_CL_CNTL_CLIP_MASK__MASK			0x000000ff
-#define A6XX_GRAS_GS_CL_CNTL_CLIP_MASK__SHIFT			0
-static inline uint32_t A6XX_GRAS_GS_CL_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_GS_CL_CNTL_CLIP_MASK__SHIFT) & A6XX_GRAS_GS_CL_CNTL_CLIP_MASK__MASK;
-}
-#define A6XX_GRAS_GS_CL_CNTL_CULL_MASK__MASK			0x0000ff00
-#define A6XX_GRAS_GS_CL_CNTL_CULL_MASK__SHIFT			8
-static inline uint32_t A6XX_GRAS_GS_CL_CNTL_CULL_MASK(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_GS_CL_CNTL_CULL_MASK__SHIFT) & A6XX_GRAS_GS_CL_CNTL_CULL_MASK__MASK;
-}
-
-#define REG_A6XX_GRAS_MAX_LAYER_INDEX				0x00008004
-
-#define REG_A6XX_GRAS_CNTL					0x00008005
-#define A6XX_GRAS_CNTL_IJ_PERSP_PIXEL				0x00000001
-#define A6XX_GRAS_CNTL_IJ_PERSP_CENTROID			0x00000002
-#define A6XX_GRAS_CNTL_IJ_PERSP_SAMPLE				0x00000004
-#define A6XX_GRAS_CNTL_IJ_LINEAR_PIXEL				0x00000008
-#define A6XX_GRAS_CNTL_IJ_LINEAR_CENTROID			0x00000010
-#define A6XX_GRAS_CNTL_IJ_LINEAR_SAMPLE				0x00000020
-#define A6XX_GRAS_CNTL_COORD_MASK__MASK				0x000003c0
-#define A6XX_GRAS_CNTL_COORD_MASK__SHIFT			6
-static inline uint32_t A6XX_GRAS_CNTL_COORD_MASK(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_CNTL_COORD_MASK__SHIFT) & A6XX_GRAS_CNTL_COORD_MASK__MASK;
-}
-#define A6XX_GRAS_CNTL_UNK10					0x00000400
-#define A6XX_GRAS_CNTL_UNK11					0x00000800
-
-#define REG_A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ			0x00008006
-#define A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__MASK		0x000001ff
-#define A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__SHIFT		0
-static inline uint32_t A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__SHIFT) & A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__MASK;
-}
-#define A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__MASK		0x0007fc00
-#define A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__SHIFT		10
-static inline uint32_t A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__SHIFT) & A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__MASK;
-}
-
-#define REG_A7XX_GRAS_UNKNOWN_8007				0x00008007
-
-#define REG_A7XX_GRAS_UNKNOWN_8008				0x00008008
-
-#define REG_A7XX_GRAS_UNKNOWN_8009				0x00008009
-
-#define REG_A7XX_GRAS_UNKNOWN_800A				0x0000800a
-
-#define REG_A7XX_GRAS_UNKNOWN_800B				0x0000800b
-
-#define REG_A7XX_GRAS_UNKNOWN_800C				0x0000800c
-
-#define REG_A6XX_GRAS_CL_VPORT(i0) (0x00008010 + 0x6*(i0))
-
-static inline uint32_t REG_A6XX_GRAS_CL_VPORT_XOFFSET(uint32_t i0) { return 0x00008010 + 0x6*i0; }
-#define A6XX_GRAS_CL_VPORT_XOFFSET__MASK			0xffffffff
-#define A6XX_GRAS_CL_VPORT_XOFFSET__SHIFT			0
-static inline uint32_t A6XX_GRAS_CL_VPORT_XOFFSET(float val)
-{
-	return ((fui(val)) << A6XX_GRAS_CL_VPORT_XOFFSET__SHIFT) & A6XX_GRAS_CL_VPORT_XOFFSET__MASK;
-}
-
-static inline uint32_t REG_A6XX_GRAS_CL_VPORT_XSCALE(uint32_t i0) { return 0x00008011 + 0x6*i0; }
-#define A6XX_GRAS_CL_VPORT_XSCALE__MASK				0xffffffff
-#define A6XX_GRAS_CL_VPORT_XSCALE__SHIFT			0
-static inline uint32_t A6XX_GRAS_CL_VPORT_XSCALE(float val)
-{
-	return ((fui(val)) << A6XX_GRAS_CL_VPORT_XSCALE__SHIFT) & A6XX_GRAS_CL_VPORT_XSCALE__MASK;
-}
-
-static inline uint32_t REG_A6XX_GRAS_CL_VPORT_YOFFSET(uint32_t i0) { return 0x00008012 + 0x6*i0; }
-#define A6XX_GRAS_CL_VPORT_YOFFSET__MASK			0xffffffff
-#define A6XX_GRAS_CL_VPORT_YOFFSET__SHIFT			0
-static inline uint32_t A6XX_GRAS_CL_VPORT_YOFFSET(float val)
-{
-	return ((fui(val)) << A6XX_GRAS_CL_VPORT_YOFFSET__SHIFT) & A6XX_GRAS_CL_VPORT_YOFFSET__MASK;
-}
-
-static inline uint32_t REG_A6XX_GRAS_CL_VPORT_YSCALE(uint32_t i0) { return 0x00008013 + 0x6*i0; }
-#define A6XX_GRAS_CL_VPORT_YSCALE__MASK				0xffffffff
-#define A6XX_GRAS_CL_VPORT_YSCALE__SHIFT			0
-static inline uint32_t A6XX_GRAS_CL_VPORT_YSCALE(float val)
-{
-	return ((fui(val)) << A6XX_GRAS_CL_VPORT_YSCALE__SHIFT) & A6XX_GRAS_CL_VPORT_YSCALE__MASK;
-}
-
-static inline uint32_t REG_A6XX_GRAS_CL_VPORT_ZOFFSET(uint32_t i0) { return 0x00008014 + 0x6*i0; }
-#define A6XX_GRAS_CL_VPORT_ZOFFSET__MASK			0xffffffff
-#define A6XX_GRAS_CL_VPORT_ZOFFSET__SHIFT			0
-static inline uint32_t A6XX_GRAS_CL_VPORT_ZOFFSET(float val)
-{
-	return ((fui(val)) << A6XX_GRAS_CL_VPORT_ZOFFSET__SHIFT) & A6XX_GRAS_CL_VPORT_ZOFFSET__MASK;
-}
-
-static inline uint32_t REG_A6XX_GRAS_CL_VPORT_ZSCALE(uint32_t i0) { return 0x00008015 + 0x6*i0; }
-#define A6XX_GRAS_CL_VPORT_ZSCALE__MASK				0xffffffff
-#define A6XX_GRAS_CL_VPORT_ZSCALE__SHIFT			0
-static inline uint32_t A6XX_GRAS_CL_VPORT_ZSCALE(float val)
-{
-	return ((fui(val)) << A6XX_GRAS_CL_VPORT_ZSCALE__SHIFT) & A6XX_GRAS_CL_VPORT_ZSCALE__MASK;
-}
-
-#define REG_A6XX_GRAS_CL_Z_CLAMP(i0) (0x00008070 + 0x2*(i0))
-
-static inline uint32_t REG_A6XX_GRAS_CL_Z_CLAMP_MIN(uint32_t i0) { return 0x00008070 + 0x2*i0; }
-#define A6XX_GRAS_CL_Z_CLAMP_MIN__MASK				0xffffffff
-#define A6XX_GRAS_CL_Z_CLAMP_MIN__SHIFT				0
-static inline uint32_t A6XX_GRAS_CL_Z_CLAMP_MIN(float val)
-{
-	return ((fui(val)) << A6XX_GRAS_CL_Z_CLAMP_MIN__SHIFT) & A6XX_GRAS_CL_Z_CLAMP_MIN__MASK;
-}
-
-static inline uint32_t REG_A6XX_GRAS_CL_Z_CLAMP_MAX(uint32_t i0) { return 0x00008071 + 0x2*i0; }
-#define A6XX_GRAS_CL_Z_CLAMP_MAX__MASK				0xffffffff
-#define A6XX_GRAS_CL_Z_CLAMP_MAX__SHIFT				0
-static inline uint32_t A6XX_GRAS_CL_Z_CLAMP_MAX(float val)
-{
-	return ((fui(val)) << A6XX_GRAS_CL_Z_CLAMP_MAX__SHIFT) & A6XX_GRAS_CL_Z_CLAMP_MAX__MASK;
-}
-
-#define REG_A6XX_GRAS_SU_CNTL					0x00008090
-#define A6XX_GRAS_SU_CNTL_CULL_FRONT				0x00000001
-#define A6XX_GRAS_SU_CNTL_CULL_BACK				0x00000002
-#define A6XX_GRAS_SU_CNTL_FRONT_CW				0x00000004
-#define A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__MASK			0x000007f8
-#define A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__SHIFT			3
-static inline uint32_t A6XX_GRAS_SU_CNTL_LINEHALFWIDTH(float val)
-{
-	return ((((int32_t)(val * 4.0))) << A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__SHIFT) & A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__MASK;
-}
-#define A6XX_GRAS_SU_CNTL_POLY_OFFSET				0x00000800
-#define A6XX_GRAS_SU_CNTL_UNK12					0x00001000
-#define A6XX_GRAS_SU_CNTL_LINE_MODE__MASK			0x00002000
-#define A6XX_GRAS_SU_CNTL_LINE_MODE__SHIFT			13
-static inline uint32_t A6XX_GRAS_SU_CNTL_LINE_MODE(enum a5xx_line_mode val)
-{
-	return ((val) << A6XX_GRAS_SU_CNTL_LINE_MODE__SHIFT) & A6XX_GRAS_SU_CNTL_LINE_MODE__MASK;
-}
-#define A6XX_GRAS_SU_CNTL_UNK15__MASK				0x00018000
-#define A6XX_GRAS_SU_CNTL_UNK15__SHIFT				15
-static inline uint32_t A6XX_GRAS_SU_CNTL_UNK15(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SU_CNTL_UNK15__SHIFT) & A6XX_GRAS_SU_CNTL_UNK15__MASK;
-}
-#define A6XX_GRAS_SU_CNTL_MULTIVIEW_ENABLE			0x00020000
-#define A6XX_GRAS_SU_CNTL_RENDERTARGETINDEXINCR			0x00040000
-#define A6XX_GRAS_SU_CNTL_VIEWPORTINDEXINCR			0x00080000
-#define A6XX_GRAS_SU_CNTL_UNK20__MASK				0x00700000
-#define A6XX_GRAS_SU_CNTL_UNK20__SHIFT				20
-static inline uint32_t A6XX_GRAS_SU_CNTL_UNK20(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SU_CNTL_UNK20__SHIFT) & A6XX_GRAS_SU_CNTL_UNK20__MASK;
-}
-
-#define REG_A6XX_GRAS_SU_POINT_MINMAX				0x00008091
-#define A6XX_GRAS_SU_POINT_MINMAX_MIN__MASK			0x0000ffff
-#define A6XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT			0
-static inline uint32_t A6XX_GRAS_SU_POINT_MINMAX_MIN(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A6XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT) & A6XX_GRAS_SU_POINT_MINMAX_MIN__MASK;
-}
-#define A6XX_GRAS_SU_POINT_MINMAX_MAX__MASK			0xffff0000
-#define A6XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT			16
-static inline uint32_t A6XX_GRAS_SU_POINT_MINMAX_MAX(float val)
-{
-	return ((((uint32_t)(val * 16.0))) << A6XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT) & A6XX_GRAS_SU_POINT_MINMAX_MAX__MASK;
-}
-
-#define REG_A6XX_GRAS_SU_POINT_SIZE				0x00008092
-#define A6XX_GRAS_SU_POINT_SIZE__MASK				0x0000ffff
-#define A6XX_GRAS_SU_POINT_SIZE__SHIFT				0
-static inline uint32_t A6XX_GRAS_SU_POINT_SIZE(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SU_POINT_SIZE__SHIFT) & A6XX_GRAS_SU_POINT_SIZE__MASK;
-}
-
-#define REG_A6XX_GRAS_SU_DEPTH_PLANE_CNTL			0x00008094
-#define A6XX_GRAS_SU_DEPTH_PLANE_CNTL_Z_MODE__MASK		0x00000003
-#define A6XX_GRAS_SU_DEPTH_PLANE_CNTL_Z_MODE__SHIFT		0
-static inline uint32_t A6XX_GRAS_SU_DEPTH_PLANE_CNTL_Z_MODE(enum a6xx_ztest_mode val)
-{
-	return ((val) << A6XX_GRAS_SU_DEPTH_PLANE_CNTL_Z_MODE__SHIFT) & A6XX_GRAS_SU_DEPTH_PLANE_CNTL_Z_MODE__MASK;
-}
-
-#define REG_A6XX_GRAS_SU_POLY_OFFSET_SCALE			0x00008095
-#define A6XX_GRAS_SU_POLY_OFFSET_SCALE__MASK			0xffffffff
-#define A6XX_GRAS_SU_POLY_OFFSET_SCALE__SHIFT			0
-static inline uint32_t A6XX_GRAS_SU_POLY_OFFSET_SCALE(float val)
-{
-	return ((fui(val)) << A6XX_GRAS_SU_POLY_OFFSET_SCALE__SHIFT) & A6XX_GRAS_SU_POLY_OFFSET_SCALE__MASK;
-}
-
-#define REG_A6XX_GRAS_SU_POLY_OFFSET_OFFSET			0x00008096
-#define A6XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK			0xffffffff
-#define A6XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT			0
-static inline uint32_t A6XX_GRAS_SU_POLY_OFFSET_OFFSET(float val)
-{
-	return ((fui(val)) << A6XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT) & A6XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK;
-}
-
-#define REG_A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP		0x00008097
-#define A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__MASK		0xffffffff
-#define A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__SHIFT		0
-static inline uint32_t A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP(float val)
-{
-	return ((fui(val)) << A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__SHIFT) & A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__MASK;
-}
-
-#define REG_A6XX_GRAS_SU_DEPTH_BUFFER_INFO			0x00008098
-#define A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK	0x00000007
-#define A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT	0
-static inline uint32_t A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT(enum a6xx_depth_format val)
-{
-	return ((val) << A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT) & A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK;
-}
-#define A6XX_GRAS_SU_DEPTH_BUFFER_INFO_UNK3			0x00000008
-
-#define REG_A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL			0x00008099
-#define A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL_CONSERVATIVERASEN	0x00000001
-#define A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL_SHIFTAMOUNT__MASK	0x00000006
-#define A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL_SHIFTAMOUNT__SHIFT	1
-static inline uint32_t A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL_SHIFTAMOUNT(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL_SHIFTAMOUNT__SHIFT) & A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL_SHIFTAMOUNT__MASK;
-}
-#define A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL_INNERCONSERVATIVERASEN	0x00000008
-#define A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL_UNK4__MASK		0x00000030
-#define A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL_UNK4__SHIFT		4
-static inline uint32_t A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL_UNK4(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL_UNK4__SHIFT) & A6XX_GRAS_SU_CONSERVATIVE_RAS_CNTL_UNK4__MASK;
-}
-
-#define REG_A6XX_GRAS_SU_PATH_RENDERING_CNTL			0x0000809a
-#define A6XX_GRAS_SU_PATH_RENDERING_CNTL_UNK0			0x00000001
-#define A6XX_GRAS_SU_PATH_RENDERING_CNTL_LINELENGTHEN		0x00000002
-
-#define REG_A6XX_GRAS_VS_LAYER_CNTL				0x0000809b
-#define A6XX_GRAS_VS_LAYER_CNTL_WRITES_LAYER			0x00000001
-#define A6XX_GRAS_VS_LAYER_CNTL_WRITES_VIEW			0x00000002
-
-#define REG_A6XX_GRAS_GS_LAYER_CNTL				0x0000809c
-#define A6XX_GRAS_GS_LAYER_CNTL_WRITES_LAYER			0x00000001
-#define A6XX_GRAS_GS_LAYER_CNTL_WRITES_VIEW			0x00000002
-
-#define REG_A6XX_GRAS_DS_LAYER_CNTL				0x0000809d
-#define A6XX_GRAS_DS_LAYER_CNTL_WRITES_LAYER			0x00000001
-#define A6XX_GRAS_DS_LAYER_CNTL_WRITES_VIEW			0x00000002
-
-#define REG_A6XX_GRAS_SC_CNTL					0x000080a0
-#define A6XX_GRAS_SC_CNTL_CCUSINGLECACHELINESIZE__MASK		0x00000007
-#define A6XX_GRAS_SC_CNTL_CCUSINGLECACHELINESIZE__SHIFT		0
-static inline uint32_t A6XX_GRAS_SC_CNTL_CCUSINGLECACHELINESIZE(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_CNTL_CCUSINGLECACHELINESIZE__SHIFT) & A6XX_GRAS_SC_CNTL_CCUSINGLECACHELINESIZE__MASK;
-}
-#define A6XX_GRAS_SC_CNTL_SINGLE_PRIM_MODE__MASK		0x00000018
-#define A6XX_GRAS_SC_CNTL_SINGLE_PRIM_MODE__SHIFT		3
-static inline uint32_t A6XX_GRAS_SC_CNTL_SINGLE_PRIM_MODE(enum a6xx_single_prim_mode val)
-{
-	return ((val) << A6XX_GRAS_SC_CNTL_SINGLE_PRIM_MODE__SHIFT) & A6XX_GRAS_SC_CNTL_SINGLE_PRIM_MODE__MASK;
-}
-#define A6XX_GRAS_SC_CNTL_RASTER_MODE__MASK			0x00000020
-#define A6XX_GRAS_SC_CNTL_RASTER_MODE__SHIFT			5
-static inline uint32_t A6XX_GRAS_SC_CNTL_RASTER_MODE(enum a6xx_raster_mode val)
-{
-	return ((val) << A6XX_GRAS_SC_CNTL_RASTER_MODE__SHIFT) & A6XX_GRAS_SC_CNTL_RASTER_MODE__MASK;
-}
-#define A6XX_GRAS_SC_CNTL_RASTER_DIRECTION__MASK		0x000000c0
-#define A6XX_GRAS_SC_CNTL_RASTER_DIRECTION__SHIFT		6
-static inline uint32_t A6XX_GRAS_SC_CNTL_RASTER_DIRECTION(enum a6xx_raster_direction val)
-{
-	return ((val) << A6XX_GRAS_SC_CNTL_RASTER_DIRECTION__SHIFT) & A6XX_GRAS_SC_CNTL_RASTER_DIRECTION__MASK;
-}
-#define A6XX_GRAS_SC_CNTL_SEQUENCED_THREAD_DISTRIBUTION__MASK	0x00000100
-#define A6XX_GRAS_SC_CNTL_SEQUENCED_THREAD_DISTRIBUTION__SHIFT	8
-static inline uint32_t A6XX_GRAS_SC_CNTL_SEQUENCED_THREAD_DISTRIBUTION(enum a6xx_sequenced_thread_dist val)
-{
-	return ((val) << A6XX_GRAS_SC_CNTL_SEQUENCED_THREAD_DISTRIBUTION__SHIFT) & A6XX_GRAS_SC_CNTL_SEQUENCED_THREAD_DISTRIBUTION__MASK;
-}
-#define A6XX_GRAS_SC_CNTL_UNK9					0x00000200
-#define A6XX_GRAS_SC_CNTL_ROTATION__MASK			0x00000c00
-#define A6XX_GRAS_SC_CNTL_ROTATION__SHIFT			10
-static inline uint32_t A6XX_GRAS_SC_CNTL_ROTATION(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_CNTL_ROTATION__SHIFT) & A6XX_GRAS_SC_CNTL_ROTATION__MASK;
-}
-#define A6XX_GRAS_SC_CNTL_EARLYVIZOUTEN				0x00001000
-
-#define REG_A6XX_GRAS_BIN_CONTROL				0x000080a1
-#define A6XX_GRAS_BIN_CONTROL_BINW__MASK			0x0000003f
-#define A6XX_GRAS_BIN_CONTROL_BINW__SHIFT			0
-static inline uint32_t A6XX_GRAS_BIN_CONTROL_BINW(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A6XX_GRAS_BIN_CONTROL_BINW__SHIFT) & A6XX_GRAS_BIN_CONTROL_BINW__MASK;
-}
-#define A6XX_GRAS_BIN_CONTROL_BINH__MASK			0x00007f00
-#define A6XX_GRAS_BIN_CONTROL_BINH__SHIFT			8
-static inline uint32_t A6XX_GRAS_BIN_CONTROL_BINH(uint32_t val)
-{
-	assert(!(val & 0xf));
-	return (((val >> 4)) << A6XX_GRAS_BIN_CONTROL_BINH__SHIFT) & A6XX_GRAS_BIN_CONTROL_BINH__MASK;
-}
-#define A6XX_GRAS_BIN_CONTROL_RENDER_MODE__MASK			0x001c0000
-#define A6XX_GRAS_BIN_CONTROL_RENDER_MODE__SHIFT		18
-static inline uint32_t A6XX_GRAS_BIN_CONTROL_RENDER_MODE(enum a6xx_render_mode val)
-{
-	return ((val) << A6XX_GRAS_BIN_CONTROL_RENDER_MODE__SHIFT) & A6XX_GRAS_BIN_CONTROL_RENDER_MODE__MASK;
-}
-#define A6XX_GRAS_BIN_CONTROL_FORCE_LRZ_WRITE_DIS		0x00200000
-#define A6XX_GRAS_BIN_CONTROL_BUFFERS_LOCATION__MASK		0x00c00000
-#define A6XX_GRAS_BIN_CONTROL_BUFFERS_LOCATION__SHIFT		22
-static inline uint32_t A6XX_GRAS_BIN_CONTROL_BUFFERS_LOCATION(enum a6xx_buffers_location val)
-{
-	return ((val) << A6XX_GRAS_BIN_CONTROL_BUFFERS_LOCATION__SHIFT) & A6XX_GRAS_BIN_CONTROL_BUFFERS_LOCATION__MASK;
-}
-#define A6XX_GRAS_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK__MASK	0x07000000
-#define A6XX_GRAS_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK__SHIFT	24
-static inline uint32_t A6XX_GRAS_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK__SHIFT) & A6XX_GRAS_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK__MASK;
-}
-#define A6XX_GRAS_BIN_CONTROL_UNK27				0x08000000
-
-#define REG_A6XX_GRAS_RAS_MSAA_CNTL				0x000080a2
-#define A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES__MASK			0x00000003
-#define A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES__SHIFT			0
-static inline uint32_t A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES__MASK;
-}
-#define A6XX_GRAS_RAS_MSAA_CNTL_UNK2				0x00000004
-#define A6XX_GRAS_RAS_MSAA_CNTL_UNK3				0x00000008
-
-#define REG_A6XX_GRAS_DEST_MSAA_CNTL				0x000080a3
-#define A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES__MASK			0x00000003
-#define A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES__SHIFT			0
-static inline uint32_t A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES__MASK;
-}
-#define A6XX_GRAS_DEST_MSAA_CNTL_MSAA_DISABLE			0x00000004
-
-#define REG_A6XX_GRAS_SAMPLE_CONFIG				0x000080a4
-#define A6XX_GRAS_SAMPLE_CONFIG_UNK0				0x00000001
-#define A6XX_GRAS_SAMPLE_CONFIG_LOCATION_ENABLE			0x00000002
-
-#define REG_A6XX_GRAS_SAMPLE_LOCATION_0				0x000080a5
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_0_X__MASK		0x0000000f
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_0_X__SHIFT		0
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_0_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_0_X__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_0_X__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_0_Y__MASK		0x000000f0
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_0_Y__SHIFT		4
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_0_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_0_Y__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_0_Y__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_1_X__MASK		0x00000f00
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_1_X__SHIFT		8
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_1_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_1_X__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_1_X__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_1_Y__MASK		0x0000f000
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_1_Y__SHIFT		12
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_1_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_1_Y__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_1_Y__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_2_X__MASK		0x000f0000
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_2_X__SHIFT		16
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_2_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_2_X__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_2_X__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_2_Y__MASK		0x00f00000
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_2_Y__SHIFT		20
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_2_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_2_Y__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_2_Y__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_3_X__MASK		0x0f000000
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_3_X__SHIFT		24
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_3_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_3_X__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_3_X__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_3_Y__MASK		0xf0000000
-#define A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_3_Y__SHIFT		28
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_3_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_3_Y__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_0_SAMPLE_3_Y__MASK;
-}
-
-#define REG_A6XX_GRAS_SAMPLE_LOCATION_1				0x000080a6
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_0_X__MASK		0x0000000f
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_0_X__SHIFT		0
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_0_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_0_X__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_0_X__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_0_Y__MASK		0x000000f0
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_0_Y__SHIFT		4
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_0_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_0_Y__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_0_Y__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_1_X__MASK		0x00000f00
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_1_X__SHIFT		8
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_1_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_1_X__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_1_X__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_1_Y__MASK		0x0000f000
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_1_Y__SHIFT		12
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_1_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_1_Y__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_1_Y__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_2_X__MASK		0x000f0000
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_2_X__SHIFT		16
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_2_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_2_X__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_2_X__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_2_Y__MASK		0x00f00000
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_2_Y__SHIFT		20
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_2_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_2_Y__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_2_Y__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_3_X__MASK		0x0f000000
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_3_X__SHIFT		24
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_3_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_3_X__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_3_X__MASK;
-}
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_3_Y__MASK		0xf0000000
-#define A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_3_Y__SHIFT		28
-static inline uint32_t A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_3_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_3_Y__SHIFT) & A6XX_GRAS_SAMPLE_LOCATION_1_SAMPLE_3_Y__MASK;
-}
-
-#define REG_A7XX_GRAS_UNKNOWN_80A7				0x000080a7
-
-#define REG_A6XX_GRAS_UNKNOWN_80AF				0x000080af
-
-#define REG_A6XX_GRAS_SC_SCREEN_SCISSOR(i0) (0x000080b0 + 0x2*(i0))
-
-static inline uint32_t REG_A6XX_GRAS_SC_SCREEN_SCISSOR_TL(uint32_t i0) { return 0x000080b0 + 0x2*i0; }
-#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_X__MASK			0x0000ffff
-#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_X__SHIFT			0
-static inline uint32_t A6XX_GRAS_SC_SCREEN_SCISSOR_TL_X(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_SCREEN_SCISSOR_TL_X__SHIFT) & A6XX_GRAS_SC_SCREEN_SCISSOR_TL_X__MASK;
-}
-#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_Y__MASK			0xffff0000
-#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_Y__SHIFT			16
-static inline uint32_t A6XX_GRAS_SC_SCREEN_SCISSOR_TL_Y(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_SCREEN_SCISSOR_TL_Y__SHIFT) & A6XX_GRAS_SC_SCREEN_SCISSOR_TL_Y__MASK;
-}
-
-static inline uint32_t REG_A6XX_GRAS_SC_SCREEN_SCISSOR_BR(uint32_t i0) { return 0x000080b1 + 0x2*i0; }
-#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_X__MASK			0x0000ffff
-#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_X__SHIFT			0
-static inline uint32_t A6XX_GRAS_SC_SCREEN_SCISSOR_BR_X(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_SCREEN_SCISSOR_BR_X__SHIFT) & A6XX_GRAS_SC_SCREEN_SCISSOR_BR_X__MASK;
-}
-#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_Y__MASK			0xffff0000
-#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_Y__SHIFT			16
-static inline uint32_t A6XX_GRAS_SC_SCREEN_SCISSOR_BR_Y(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_SCREEN_SCISSOR_BR_Y__SHIFT) & A6XX_GRAS_SC_SCREEN_SCISSOR_BR_Y__MASK;
-}
-
-#define REG_A6XX_GRAS_SC_VIEWPORT_SCISSOR(i0) (0x000080d0 + 0x2*(i0))
-
-static inline uint32_t REG_A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL(uint32_t i0) { return 0x000080d0 + 0x2*i0; }
-#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_X__MASK		0x0000ffff
-#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_X__SHIFT		0
-static inline uint32_t A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_X(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_X__SHIFT) & A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_X__MASK;
-}
-#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_Y__MASK		0xffff0000
-#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_Y__SHIFT		16
-static inline uint32_t A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_Y(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_Y__SHIFT) & A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_Y__MASK;
-}
-
-static inline uint32_t REG_A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR(uint32_t i0) { return 0x000080d1 + 0x2*i0; }
-#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_X__MASK		0x0000ffff
-#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_X__SHIFT		0
-static inline uint32_t A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_X(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_X__SHIFT) & A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_X__MASK;
-}
-#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_Y__MASK		0xffff0000
-#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_Y__SHIFT		16
-static inline uint32_t A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_Y(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_Y__SHIFT) & A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_Y__MASK;
-}
-
-#define REG_A6XX_GRAS_SC_WINDOW_SCISSOR_TL			0x000080f0
-#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK			0x00003fff
-#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT			0
-static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT) & A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK;
-}
-#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK			0x3fff0000
-#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT			16
-static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT) & A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK;
-}
-
-#define REG_A6XX_GRAS_SC_WINDOW_SCISSOR_BR			0x000080f1
-#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK			0x00003fff
-#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT			0
-static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT) & A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK;
-}
-#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK			0x3fff0000
-#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT			16
-static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT) & A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK;
-}
-
-#define REG_A7XX_GRAS_UNKNOWN_80F4				0x000080f4
-
-#define REG_A7XX_GRAS_UNKNOWN_80F5				0x000080f5
-
-#define REG_A7XX_GRAS_UNKNOWN_80F6				0x000080f6
-
-#define REG_A7XX_GRAS_UNKNOWN_80F8				0x000080f8
-
-#define REG_A7XX_GRAS_UNKNOWN_80F9				0x000080f9
-
-#define REG_A7XX_GRAS_UNKNOWN_80FA				0x000080fa
-
-#define REG_A6XX_GRAS_LRZ_CNTL					0x00008100
-#define A6XX_GRAS_LRZ_CNTL_ENABLE				0x00000001
-#define A6XX_GRAS_LRZ_CNTL_LRZ_WRITE				0x00000002
-#define A6XX_GRAS_LRZ_CNTL_GREATER				0x00000004
-#define A6XX_GRAS_LRZ_CNTL_FC_ENABLE				0x00000008
-#define A6XX_GRAS_LRZ_CNTL_Z_TEST_ENABLE			0x00000010
-#define A6XX_GRAS_LRZ_CNTL_Z_BOUNDS_ENABLE			0x00000020
-#define A6XX_GRAS_LRZ_CNTL_DIR__MASK				0x000000c0
-#define A6XX_GRAS_LRZ_CNTL_DIR__SHIFT				6
-static inline uint32_t A6XX_GRAS_LRZ_CNTL_DIR(enum a6xx_lrz_dir_status val)
-{
-	return ((val) << A6XX_GRAS_LRZ_CNTL_DIR__SHIFT) & A6XX_GRAS_LRZ_CNTL_DIR__MASK;
-}
-#define A6XX_GRAS_LRZ_CNTL_DIR_WRITE				0x00000100
-#define A6XX_GRAS_LRZ_CNTL_DISABLE_ON_WRONG_DIR			0x00000200
-#define A6XX_GRAS_LRZ_CNTL_Z_FUNC__MASK				0x00003800
-#define A6XX_GRAS_LRZ_CNTL_Z_FUNC__SHIFT			11
-static inline uint32_t A6XX_GRAS_LRZ_CNTL_Z_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A6XX_GRAS_LRZ_CNTL_Z_FUNC__SHIFT) & A6XX_GRAS_LRZ_CNTL_Z_FUNC__MASK;
-}
-
-#define REG_A6XX_GRAS_LRZ_PS_INPUT_CNTL				0x00008101
-#define A6XX_GRAS_LRZ_PS_INPUT_CNTL_SAMPLEID			0x00000001
-#define A6XX_GRAS_LRZ_PS_INPUT_CNTL_FRAGCOORDSAMPLEMODE__MASK	0x00000006
-#define A6XX_GRAS_LRZ_PS_INPUT_CNTL_FRAGCOORDSAMPLEMODE__SHIFT	1
-static inline uint32_t A6XX_GRAS_LRZ_PS_INPUT_CNTL_FRAGCOORDSAMPLEMODE(enum a6xx_fragcoord_sample_mode val)
-{
-	return ((val) << A6XX_GRAS_LRZ_PS_INPUT_CNTL_FRAGCOORDSAMPLEMODE__SHIFT) & A6XX_GRAS_LRZ_PS_INPUT_CNTL_FRAGCOORDSAMPLEMODE__MASK;
-}
-
-#define REG_A6XX_GRAS_LRZ_MRT_BUF_INFO_0			0x00008102
-#define A6XX_GRAS_LRZ_MRT_BUF_INFO_0_COLOR_FORMAT__MASK		0x000000ff
-#define A6XX_GRAS_LRZ_MRT_BUF_INFO_0_COLOR_FORMAT__SHIFT	0
-static inline uint32_t A6XX_GRAS_LRZ_MRT_BUF_INFO_0_COLOR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A6XX_GRAS_LRZ_MRT_BUF_INFO_0_COLOR_FORMAT__SHIFT) & A6XX_GRAS_LRZ_MRT_BUF_INFO_0_COLOR_FORMAT__MASK;
-}
-
-#define REG_A6XX_GRAS_LRZ_BUFFER_BASE				0x00008103
-
-#define REG_A6XX_GRAS_LRZ_BUFFER_PITCH				0x00008105
-#define A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH__MASK			0x000000ff
-#define A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH__SHIFT			0
-static inline uint32_t A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH__SHIFT) & A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH__MASK;
-}
-#define A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH__MASK		0x1ffffc00
-#define A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH__SHIFT		10
-static inline uint32_t A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0xf));
-	return (((val >> 4)) << A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH__SHIFT) & A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH__MASK;
-}
-
-#define REG_A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE		0x00008106
-
-#define REG_A6XX_GRAS_SAMPLE_CNTL				0x00008109
-#define A6XX_GRAS_SAMPLE_CNTL_PER_SAMP_MODE			0x00000001
-
-#define REG_A6XX_GRAS_LRZ_DEPTH_VIEW				0x0000810a
-#define A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_LAYER__MASK		0x000007ff
-#define A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_LAYER__SHIFT		0
-static inline uint32_t A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_LAYER(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_LAYER__SHIFT) & A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_LAYER__MASK;
-}
-#define A6XX_GRAS_LRZ_DEPTH_VIEW_LAYER_COUNT__MASK		0x07ff0000
-#define A6XX_GRAS_LRZ_DEPTH_VIEW_LAYER_COUNT__SHIFT		16
-static inline uint32_t A6XX_GRAS_LRZ_DEPTH_VIEW_LAYER_COUNT(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_LRZ_DEPTH_VIEW_LAYER_COUNT__SHIFT) & A6XX_GRAS_LRZ_DEPTH_VIEW_LAYER_COUNT__MASK;
-}
-#define A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_MIP_LEVEL__MASK		0xf0000000
-#define A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_MIP_LEVEL__SHIFT		28
-static inline uint32_t A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_MIP_LEVEL(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_MIP_LEVEL__SHIFT) & A6XX_GRAS_LRZ_DEPTH_VIEW_BASE_MIP_LEVEL__MASK;
-}
-
-#define REG_A7XX_GRAS_UNKNOWN_810B				0x0000810b
-
-#define REG_A6XX_GRAS_UNKNOWN_8110				0x00008110
-
-#define REG_A7XX_GRAS_LRZ_CLEAR_DEPTH_F32			0x00008111
-#define A7XX_GRAS_LRZ_CLEAR_DEPTH_F32__MASK			0xffffffff
-#define A7XX_GRAS_LRZ_CLEAR_DEPTH_F32__SHIFT			0
-static inline uint32_t A7XX_GRAS_LRZ_CLEAR_DEPTH_F32(float val)
-{
-	return ((fui(val)) << A7XX_GRAS_LRZ_CLEAR_DEPTH_F32__SHIFT) & A7XX_GRAS_LRZ_CLEAR_DEPTH_F32__MASK;
-}
-
-#define REG_A7XX_GRAS_UNKNOWN_8113				0x00008113
-
-#define REG_A7XX_GRAS_UNKNOWN_8120				0x00008120
-
-#define REG_A7XX_GRAS_UNKNOWN_8121				0x00008121
-
-#define REG_A6XX_GRAS_2D_BLIT_CNTL				0x00008400
-#define A6XX_GRAS_2D_BLIT_CNTL_ROTATE__MASK			0x00000007
-#define A6XX_GRAS_2D_BLIT_CNTL_ROTATE__SHIFT			0
-static inline uint32_t A6XX_GRAS_2D_BLIT_CNTL_ROTATE(enum a6xx_rotation val)
-{
-	return ((val) << A6XX_GRAS_2D_BLIT_CNTL_ROTATE__SHIFT) & A6XX_GRAS_2D_BLIT_CNTL_ROTATE__MASK;
-}
-#define A6XX_GRAS_2D_BLIT_CNTL_OVERWRITEEN			0x00000008
-#define A6XX_GRAS_2D_BLIT_CNTL_UNK4__MASK			0x00000070
-#define A6XX_GRAS_2D_BLIT_CNTL_UNK4__SHIFT			4
-static inline uint32_t A6XX_GRAS_2D_BLIT_CNTL_UNK4(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_BLIT_CNTL_UNK4__SHIFT) & A6XX_GRAS_2D_BLIT_CNTL_UNK4__MASK;
-}
-#define A6XX_GRAS_2D_BLIT_CNTL_SOLID_COLOR			0x00000080
-#define A6XX_GRAS_2D_BLIT_CNTL_COLOR_FORMAT__MASK		0x0000ff00
-#define A6XX_GRAS_2D_BLIT_CNTL_COLOR_FORMAT__SHIFT		8
-static inline uint32_t A6XX_GRAS_2D_BLIT_CNTL_COLOR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A6XX_GRAS_2D_BLIT_CNTL_COLOR_FORMAT__SHIFT) & A6XX_GRAS_2D_BLIT_CNTL_COLOR_FORMAT__MASK;
-}
-#define A6XX_GRAS_2D_BLIT_CNTL_SCISSOR				0x00010000
-#define A6XX_GRAS_2D_BLIT_CNTL_UNK17__MASK			0x00060000
-#define A6XX_GRAS_2D_BLIT_CNTL_UNK17__SHIFT			17
-static inline uint32_t A6XX_GRAS_2D_BLIT_CNTL_UNK17(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_BLIT_CNTL_UNK17__SHIFT) & A6XX_GRAS_2D_BLIT_CNTL_UNK17__MASK;
-}
-#define A6XX_GRAS_2D_BLIT_CNTL_D24S8				0x00080000
-#define A6XX_GRAS_2D_BLIT_CNTL_MASK__MASK			0x00f00000
-#define A6XX_GRAS_2D_BLIT_CNTL_MASK__SHIFT			20
-static inline uint32_t A6XX_GRAS_2D_BLIT_CNTL_MASK(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_BLIT_CNTL_MASK__SHIFT) & A6XX_GRAS_2D_BLIT_CNTL_MASK__MASK;
-}
-#define A6XX_GRAS_2D_BLIT_CNTL_IFMT__MASK			0x1f000000
-#define A6XX_GRAS_2D_BLIT_CNTL_IFMT__SHIFT			24
-static inline uint32_t A6XX_GRAS_2D_BLIT_CNTL_IFMT(enum a6xx_2d_ifmt val)
-{
-	return ((val) << A6XX_GRAS_2D_BLIT_CNTL_IFMT__SHIFT) & A6XX_GRAS_2D_BLIT_CNTL_IFMT__MASK;
-}
-#define A6XX_GRAS_2D_BLIT_CNTL_RASTER_MODE__MASK		0x20000000
-#define A6XX_GRAS_2D_BLIT_CNTL_RASTER_MODE__SHIFT		29
-static inline uint32_t A6XX_GRAS_2D_BLIT_CNTL_RASTER_MODE(enum a6xx_raster_mode val)
-{
-	return ((val) << A6XX_GRAS_2D_BLIT_CNTL_RASTER_MODE__SHIFT) & A6XX_GRAS_2D_BLIT_CNTL_RASTER_MODE__MASK;
-}
-#define A6XX_GRAS_2D_BLIT_CNTL_UNK30				0x40000000
-
-#define REG_A6XX_GRAS_2D_SRC_TL_X				0x00008401
-#define A6XX_GRAS_2D_SRC_TL_X__MASK				0x01ffff00
-#define A6XX_GRAS_2D_SRC_TL_X__SHIFT				8
-static inline uint32_t A6XX_GRAS_2D_SRC_TL_X(int32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_SRC_TL_X__SHIFT) & A6XX_GRAS_2D_SRC_TL_X__MASK;
-}
-
-#define REG_A6XX_GRAS_2D_SRC_BR_X				0x00008402
-#define A6XX_GRAS_2D_SRC_BR_X__MASK				0x01ffff00
-#define A6XX_GRAS_2D_SRC_BR_X__SHIFT				8
-static inline uint32_t A6XX_GRAS_2D_SRC_BR_X(int32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_SRC_BR_X__SHIFT) & A6XX_GRAS_2D_SRC_BR_X__MASK;
-}
-
-#define REG_A6XX_GRAS_2D_SRC_TL_Y				0x00008403
-#define A6XX_GRAS_2D_SRC_TL_Y__MASK				0x01ffff00
-#define A6XX_GRAS_2D_SRC_TL_Y__SHIFT				8
-static inline uint32_t A6XX_GRAS_2D_SRC_TL_Y(int32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_SRC_TL_Y__SHIFT) & A6XX_GRAS_2D_SRC_TL_Y__MASK;
-}
-
-#define REG_A6XX_GRAS_2D_SRC_BR_Y				0x00008404
-#define A6XX_GRAS_2D_SRC_BR_Y__MASK				0x01ffff00
-#define A6XX_GRAS_2D_SRC_BR_Y__SHIFT				8
-static inline uint32_t A6XX_GRAS_2D_SRC_BR_Y(int32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_SRC_BR_Y__SHIFT) & A6XX_GRAS_2D_SRC_BR_Y__MASK;
-}
-
-#define REG_A6XX_GRAS_2D_DST_TL					0x00008405
-#define A6XX_GRAS_2D_DST_TL_X__MASK				0x00003fff
-#define A6XX_GRAS_2D_DST_TL_X__SHIFT				0
-static inline uint32_t A6XX_GRAS_2D_DST_TL_X(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_DST_TL_X__SHIFT) & A6XX_GRAS_2D_DST_TL_X__MASK;
-}
-#define A6XX_GRAS_2D_DST_TL_Y__MASK				0x3fff0000
-#define A6XX_GRAS_2D_DST_TL_Y__SHIFT				16
-static inline uint32_t A6XX_GRAS_2D_DST_TL_Y(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_DST_TL_Y__SHIFT) & A6XX_GRAS_2D_DST_TL_Y__MASK;
-}
-
-#define REG_A6XX_GRAS_2D_DST_BR					0x00008406
-#define A6XX_GRAS_2D_DST_BR_X__MASK				0x00003fff
-#define A6XX_GRAS_2D_DST_BR_X__SHIFT				0
-static inline uint32_t A6XX_GRAS_2D_DST_BR_X(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_DST_BR_X__SHIFT) & A6XX_GRAS_2D_DST_BR_X__MASK;
-}
-#define A6XX_GRAS_2D_DST_BR_Y__MASK				0x3fff0000
-#define A6XX_GRAS_2D_DST_BR_Y__SHIFT				16
-static inline uint32_t A6XX_GRAS_2D_DST_BR_Y(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_DST_BR_Y__SHIFT) & A6XX_GRAS_2D_DST_BR_Y__MASK;
-}
-
-#define REG_A6XX_GRAS_2D_UNKNOWN_8407				0x00008407
-
-#define REG_A6XX_GRAS_2D_UNKNOWN_8408				0x00008408
-
-#define REG_A6XX_GRAS_2D_UNKNOWN_8409				0x00008409
-
-#define REG_A6XX_GRAS_2D_RESOLVE_CNTL_1				0x0000840a
-#define A6XX_GRAS_2D_RESOLVE_CNTL_1_X__MASK			0x00003fff
-#define A6XX_GRAS_2D_RESOLVE_CNTL_1_X__SHIFT			0
-static inline uint32_t A6XX_GRAS_2D_RESOLVE_CNTL_1_X(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_RESOLVE_CNTL_1_X__SHIFT) & A6XX_GRAS_2D_RESOLVE_CNTL_1_X__MASK;
-}
-#define A6XX_GRAS_2D_RESOLVE_CNTL_1_Y__MASK			0x3fff0000
-#define A6XX_GRAS_2D_RESOLVE_CNTL_1_Y__SHIFT			16
-static inline uint32_t A6XX_GRAS_2D_RESOLVE_CNTL_1_Y(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_RESOLVE_CNTL_1_Y__SHIFT) & A6XX_GRAS_2D_RESOLVE_CNTL_1_Y__MASK;
-}
-
-#define REG_A6XX_GRAS_2D_RESOLVE_CNTL_2				0x0000840b
-#define A6XX_GRAS_2D_RESOLVE_CNTL_2_X__MASK			0x00003fff
-#define A6XX_GRAS_2D_RESOLVE_CNTL_2_X__SHIFT			0
-static inline uint32_t A6XX_GRAS_2D_RESOLVE_CNTL_2_X(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_RESOLVE_CNTL_2_X__SHIFT) & A6XX_GRAS_2D_RESOLVE_CNTL_2_X__MASK;
-}
-#define A6XX_GRAS_2D_RESOLVE_CNTL_2_Y__MASK			0x3fff0000
-#define A6XX_GRAS_2D_RESOLVE_CNTL_2_Y__SHIFT			16
-static inline uint32_t A6XX_GRAS_2D_RESOLVE_CNTL_2_Y(uint32_t val)
-{
-	return ((val) << A6XX_GRAS_2D_RESOLVE_CNTL_2_Y__SHIFT) & A6XX_GRAS_2D_RESOLVE_CNTL_2_Y__MASK;
-}
-
-#define REG_A6XX_GRAS_DBG_ECO_CNTL				0x00008600
-#define A6XX_GRAS_DBG_ECO_CNTL_UNK7				0x00000080
-#define A6XX_GRAS_DBG_ECO_CNTL_LRZCACHELOCKDIS			0x00000800
-
-#define REG_A6XX_GRAS_ADDR_MODE_CNTL				0x00008601
-
-#define REG_A7XX_GRAS_NC_MODE_CNTL				0x00008602
-
-#define REG_A6XX_GRAS_PERFCTR_TSE_SEL(i0) (0x00008610 + 0x1*(i0))
-
-#define REG_A6XX_GRAS_PERFCTR_RAS_SEL(i0) (0x00008614 + 0x1*(i0))
-
-#define REG_A6XX_GRAS_PERFCTR_LRZ_SEL(i0) (0x00008618 + 0x1*(i0))
-
-#define REG_A6XX_RB_BIN_CONTROL					0x00008800
-#define A6XX_RB_BIN_CONTROL_BINW__MASK				0x0000003f
-#define A6XX_RB_BIN_CONTROL_BINW__SHIFT				0
-static inline uint32_t A6XX_RB_BIN_CONTROL_BINW(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A6XX_RB_BIN_CONTROL_BINW__SHIFT) & A6XX_RB_BIN_CONTROL_BINW__MASK;
-}
-#define A6XX_RB_BIN_CONTROL_BINH__MASK				0x00007f00
-#define A6XX_RB_BIN_CONTROL_BINH__SHIFT				8
-static inline uint32_t A6XX_RB_BIN_CONTROL_BINH(uint32_t val)
-{
-	assert(!(val & 0xf));
-	return (((val >> 4)) << A6XX_RB_BIN_CONTROL_BINH__SHIFT) & A6XX_RB_BIN_CONTROL_BINH__MASK;
-}
-#define A6XX_RB_BIN_CONTROL_RENDER_MODE__MASK			0x001c0000
-#define A6XX_RB_BIN_CONTROL_RENDER_MODE__SHIFT			18
-static inline uint32_t A6XX_RB_BIN_CONTROL_RENDER_MODE(enum a6xx_render_mode val)
-{
-	return ((val) << A6XX_RB_BIN_CONTROL_RENDER_MODE__SHIFT) & A6XX_RB_BIN_CONTROL_RENDER_MODE__MASK;
-}
-#define A6XX_RB_BIN_CONTROL_FORCE_LRZ_WRITE_DIS			0x00200000
-#define A6XX_RB_BIN_CONTROL_BUFFERS_LOCATION__MASK		0x00c00000
-#define A6XX_RB_BIN_CONTROL_BUFFERS_LOCATION__SHIFT		22
-static inline uint32_t A6XX_RB_BIN_CONTROL_BUFFERS_LOCATION(enum a6xx_buffers_location val)
-{
-	return ((val) << A6XX_RB_BIN_CONTROL_BUFFERS_LOCATION__SHIFT) & A6XX_RB_BIN_CONTROL_BUFFERS_LOCATION__MASK;
-}
-#define A6XX_RB_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK__MASK	0x07000000
-#define A6XX_RB_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK__SHIFT	24
-static inline uint32_t A6XX_RB_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK(uint32_t val)
-{
-	return ((val) << A6XX_RB_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK__SHIFT) & A6XX_RB_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK__MASK;
-}
-
-#define REG_A7XX_RB_BIN_CONTROL					0x00008800
-#define A7XX_RB_BIN_CONTROL_BINW__MASK				0x0000003f
-#define A7XX_RB_BIN_CONTROL_BINW__SHIFT				0
-static inline uint32_t A7XX_RB_BIN_CONTROL_BINW(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A7XX_RB_BIN_CONTROL_BINW__SHIFT) & A7XX_RB_BIN_CONTROL_BINW__MASK;
-}
-#define A7XX_RB_BIN_CONTROL_BINH__MASK				0x00007f00
-#define A7XX_RB_BIN_CONTROL_BINH__SHIFT				8
-static inline uint32_t A7XX_RB_BIN_CONTROL_BINH(uint32_t val)
-{
-	assert(!(val & 0xf));
-	return (((val >> 4)) << A7XX_RB_BIN_CONTROL_BINH__SHIFT) & A7XX_RB_BIN_CONTROL_BINH__MASK;
-}
-#define A7XX_RB_BIN_CONTROL_RENDER_MODE__MASK			0x001c0000
-#define A7XX_RB_BIN_CONTROL_RENDER_MODE__SHIFT			18
-static inline uint32_t A7XX_RB_BIN_CONTROL_RENDER_MODE(enum a6xx_render_mode val)
-{
-	return ((val) << A7XX_RB_BIN_CONTROL_RENDER_MODE__SHIFT) & A7XX_RB_BIN_CONTROL_RENDER_MODE__MASK;
-}
-#define A7XX_RB_BIN_CONTROL_FORCE_LRZ_WRITE_DIS			0x00200000
-#define A7XX_RB_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK__MASK	0x07000000
-#define A7XX_RB_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK__SHIFT	24
-static inline uint32_t A7XX_RB_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK(uint32_t val)
-{
-	return ((val) << A7XX_RB_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK__SHIFT) & A7XX_RB_BIN_CONTROL_LRZ_FEEDBACK_ZMODE_MASK__MASK;
-}
-
-#define REG_A6XX_RB_RENDER_CNTL					0x00008801
-#define A6XX_RB_RENDER_CNTL_CCUSINGLECACHELINESIZE__MASK	0x00000038
-#define A6XX_RB_RENDER_CNTL_CCUSINGLECACHELINESIZE__SHIFT	3
-static inline uint32_t A6XX_RB_RENDER_CNTL_CCUSINGLECACHELINESIZE(uint32_t val)
-{
-	return ((val) << A6XX_RB_RENDER_CNTL_CCUSINGLECACHELINESIZE__SHIFT) & A6XX_RB_RENDER_CNTL_CCUSINGLECACHELINESIZE__MASK;
-}
-#define A6XX_RB_RENDER_CNTL_EARLYVIZOUTEN			0x00000040
-#define A6XX_RB_RENDER_CNTL_BINNING				0x00000080
-#define A6XX_RB_RENDER_CNTL_UNK8__MASK				0x00000700
-#define A6XX_RB_RENDER_CNTL_UNK8__SHIFT				8
-static inline uint32_t A6XX_RB_RENDER_CNTL_UNK8(uint32_t val)
-{
-	return ((val) << A6XX_RB_RENDER_CNTL_UNK8__SHIFT) & A6XX_RB_RENDER_CNTL_UNK8__MASK;
-}
-#define A6XX_RB_RENDER_CNTL_RASTER_MODE__MASK			0x00000100
-#define A6XX_RB_RENDER_CNTL_RASTER_MODE__SHIFT			8
-static inline uint32_t A6XX_RB_RENDER_CNTL_RASTER_MODE(enum a6xx_raster_mode val)
-{
-	return ((val) << A6XX_RB_RENDER_CNTL_RASTER_MODE__SHIFT) & A6XX_RB_RENDER_CNTL_RASTER_MODE__MASK;
-}
-#define A6XX_RB_RENDER_CNTL_RASTER_DIRECTION__MASK		0x00000600
-#define A6XX_RB_RENDER_CNTL_RASTER_DIRECTION__SHIFT		9
-static inline uint32_t A6XX_RB_RENDER_CNTL_RASTER_DIRECTION(enum a6xx_raster_direction val)
-{
-	return ((val) << A6XX_RB_RENDER_CNTL_RASTER_DIRECTION__SHIFT) & A6XX_RB_RENDER_CNTL_RASTER_DIRECTION__MASK;
-}
-#define A6XX_RB_RENDER_CNTL_CONSERVATIVERASEN			0x00000800
-#define A6XX_RB_RENDER_CNTL_INNERCONSERVATIVERASEN		0x00001000
-#define A6XX_RB_RENDER_CNTL_FLAG_DEPTH				0x00004000
-#define A6XX_RB_RENDER_CNTL_FLAG_MRTS__MASK			0x00ff0000
-#define A6XX_RB_RENDER_CNTL_FLAG_MRTS__SHIFT			16
-static inline uint32_t A6XX_RB_RENDER_CNTL_FLAG_MRTS(uint32_t val)
-{
-	return ((val) << A6XX_RB_RENDER_CNTL_FLAG_MRTS__SHIFT) & A6XX_RB_RENDER_CNTL_FLAG_MRTS__MASK;
-}
-
-#define REG_A7XX_RB_RENDER_CNTL					0x00008801
-#define A7XX_RB_RENDER_CNTL_EARLYVIZOUTEN			0x00000040
-#define A7XX_RB_RENDER_CNTL_BINNING				0x00000080
-#define A7XX_RB_RENDER_CNTL_RASTER_MODE__MASK			0x00000100
-#define A7XX_RB_RENDER_CNTL_RASTER_MODE__SHIFT			8
-static inline uint32_t A7XX_RB_RENDER_CNTL_RASTER_MODE(enum a6xx_raster_mode val)
-{
-	return ((val) << A7XX_RB_RENDER_CNTL_RASTER_MODE__SHIFT) & A7XX_RB_RENDER_CNTL_RASTER_MODE__MASK;
-}
-#define A7XX_RB_RENDER_CNTL_RASTER_DIRECTION__MASK		0x00000600
-#define A7XX_RB_RENDER_CNTL_RASTER_DIRECTION__SHIFT		9
-static inline uint32_t A7XX_RB_RENDER_CNTL_RASTER_DIRECTION(enum a6xx_raster_direction val)
-{
-	return ((val) << A7XX_RB_RENDER_CNTL_RASTER_DIRECTION__SHIFT) & A7XX_RB_RENDER_CNTL_RASTER_DIRECTION__MASK;
-}
-#define A7XX_RB_RENDER_CNTL_CONSERVATIVERASEN			0x00000800
-#define A7XX_RB_RENDER_CNTL_INNERCONSERVATIVERASEN		0x00001000
-
-#define REG_A7XX_GRAS_SU_RENDER_CNTL				0x00008116
-#define A7XX_GRAS_SU_RENDER_CNTL_BINNING			0x00000080
-
-#define REG_A6XX_RB_RAS_MSAA_CNTL				0x00008802
-#define A6XX_RB_RAS_MSAA_CNTL_SAMPLES__MASK			0x00000003
-#define A6XX_RB_RAS_MSAA_CNTL_SAMPLES__SHIFT			0
-static inline uint32_t A6XX_RB_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A6XX_RB_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_RB_RAS_MSAA_CNTL_SAMPLES__MASK;
-}
-#define A6XX_RB_RAS_MSAA_CNTL_UNK2				0x00000004
-#define A6XX_RB_RAS_MSAA_CNTL_UNK3				0x00000008
-
-#define REG_A6XX_RB_DEST_MSAA_CNTL				0x00008803
-#define A6XX_RB_DEST_MSAA_CNTL_SAMPLES__MASK			0x00000003
-#define A6XX_RB_DEST_MSAA_CNTL_SAMPLES__SHIFT			0
-static inline uint32_t A6XX_RB_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A6XX_RB_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_RB_DEST_MSAA_CNTL_SAMPLES__MASK;
-}
-#define A6XX_RB_DEST_MSAA_CNTL_MSAA_DISABLE			0x00000004
-
-#define REG_A6XX_RB_SAMPLE_CONFIG				0x00008804
-#define A6XX_RB_SAMPLE_CONFIG_UNK0				0x00000001
-#define A6XX_RB_SAMPLE_CONFIG_LOCATION_ENABLE			0x00000002
-
-#define REG_A6XX_RB_SAMPLE_LOCATION_0				0x00008805
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_0_X__MASK		0x0000000f
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_0_X__SHIFT		0
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_0_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_0_X__SHIFT) & A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_0_X__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_0_Y__MASK		0x000000f0
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_0_Y__SHIFT		4
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_0_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_0_Y__SHIFT) & A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_0_Y__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_1_X__MASK		0x00000f00
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_1_X__SHIFT		8
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_1_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_1_X__SHIFT) & A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_1_X__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_1_Y__MASK		0x0000f000
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_1_Y__SHIFT		12
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_1_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_1_Y__SHIFT) & A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_1_Y__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_2_X__MASK		0x000f0000
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_2_X__SHIFT		16
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_2_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_2_X__SHIFT) & A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_2_X__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_2_Y__MASK		0x00f00000
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_2_Y__SHIFT		20
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_2_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_2_Y__SHIFT) & A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_2_Y__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_3_X__MASK		0x0f000000
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_3_X__SHIFT		24
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_3_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_3_X__SHIFT) & A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_3_X__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_3_Y__MASK		0xf0000000
-#define A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_3_Y__SHIFT		28
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_3_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_3_Y__SHIFT) & A6XX_RB_SAMPLE_LOCATION_0_SAMPLE_3_Y__MASK;
-}
-
-#define REG_A6XX_RB_SAMPLE_LOCATION_1				0x00008806
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_0_X__MASK		0x0000000f
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_0_X__SHIFT		0
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_0_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_0_X__SHIFT) & A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_0_X__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_0_Y__MASK		0x000000f0
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_0_Y__SHIFT		4
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_0_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_0_Y__SHIFT) & A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_0_Y__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_1_X__MASK		0x00000f00
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_1_X__SHIFT		8
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_1_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_1_X__SHIFT) & A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_1_X__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_1_Y__MASK		0x0000f000
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_1_Y__SHIFT		12
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_1_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_1_Y__SHIFT) & A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_1_Y__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_2_X__MASK		0x000f0000
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_2_X__SHIFT		16
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_2_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_2_X__SHIFT) & A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_2_X__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_2_Y__MASK		0x00f00000
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_2_Y__SHIFT		20
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_2_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_2_Y__SHIFT) & A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_2_Y__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_3_X__MASK		0x0f000000
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_3_X__SHIFT		24
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_3_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_3_X__SHIFT) & A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_3_X__MASK;
-}
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_3_Y__MASK		0xf0000000
-#define A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_3_Y__SHIFT		28
-static inline uint32_t A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_3_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_3_Y__SHIFT) & A6XX_RB_SAMPLE_LOCATION_1_SAMPLE_3_Y__MASK;
-}
-
-#define REG_A6XX_RB_RENDER_CONTROL0				0x00008809
-#define A6XX_RB_RENDER_CONTROL0_IJ_PERSP_PIXEL			0x00000001
-#define A6XX_RB_RENDER_CONTROL0_IJ_PERSP_CENTROID		0x00000002
-#define A6XX_RB_RENDER_CONTROL0_IJ_PERSP_SAMPLE			0x00000004
-#define A6XX_RB_RENDER_CONTROL0_IJ_LINEAR_PIXEL			0x00000008
-#define A6XX_RB_RENDER_CONTROL0_IJ_LINEAR_CENTROID		0x00000010
-#define A6XX_RB_RENDER_CONTROL0_IJ_LINEAR_SAMPLE		0x00000020
-#define A6XX_RB_RENDER_CONTROL0_COORD_MASK__MASK		0x000003c0
-#define A6XX_RB_RENDER_CONTROL0_COORD_MASK__SHIFT		6
-static inline uint32_t A6XX_RB_RENDER_CONTROL0_COORD_MASK(uint32_t val)
-{
-	return ((val) << A6XX_RB_RENDER_CONTROL0_COORD_MASK__SHIFT) & A6XX_RB_RENDER_CONTROL0_COORD_MASK__MASK;
-}
-#define A6XX_RB_RENDER_CONTROL0_UNK10				0x00000400
-
-#define REG_A6XX_RB_RENDER_CONTROL1				0x0000880a
-#define A6XX_RB_RENDER_CONTROL1_SAMPLEMASK			0x00000001
-#define A6XX_RB_RENDER_CONTROL1_POSTDEPTHCOVERAGE		0x00000002
-#define A6XX_RB_RENDER_CONTROL1_FACENESS			0x00000004
-#define A6XX_RB_RENDER_CONTROL1_SAMPLEID			0x00000008
-#define A6XX_RB_RENDER_CONTROL1_FRAGCOORDSAMPLEMODE__MASK	0x00000030
-#define A6XX_RB_RENDER_CONTROL1_FRAGCOORDSAMPLEMODE__SHIFT	4
-static inline uint32_t A6XX_RB_RENDER_CONTROL1_FRAGCOORDSAMPLEMODE(enum a6xx_fragcoord_sample_mode val)
-{
-	return ((val) << A6XX_RB_RENDER_CONTROL1_FRAGCOORDSAMPLEMODE__SHIFT) & A6XX_RB_RENDER_CONTROL1_FRAGCOORDSAMPLEMODE__MASK;
-}
-#define A6XX_RB_RENDER_CONTROL1_CENTERRHW			0x00000040
-#define A6XX_RB_RENDER_CONTROL1_LINELENGTHEN			0x00000080
-#define A6XX_RB_RENDER_CONTROL1_FOVEATION			0x00000100
-
-#define REG_A6XX_RB_FS_OUTPUT_CNTL0				0x0000880b
-#define A6XX_RB_FS_OUTPUT_CNTL0_DUAL_COLOR_IN_ENABLE		0x00000001
-#define A6XX_RB_FS_OUTPUT_CNTL0_FRAG_WRITES_Z			0x00000002
-#define A6XX_RB_FS_OUTPUT_CNTL0_FRAG_WRITES_SAMPMASK		0x00000004
-#define A6XX_RB_FS_OUTPUT_CNTL0_FRAG_WRITES_STENCILREF		0x00000008
-
-#define REG_A6XX_RB_FS_OUTPUT_CNTL1				0x0000880c
-#define A6XX_RB_FS_OUTPUT_CNTL1_MRT__MASK			0x0000000f
-#define A6XX_RB_FS_OUTPUT_CNTL1_MRT__SHIFT			0
-static inline uint32_t A6XX_RB_FS_OUTPUT_CNTL1_MRT(uint32_t val)
-{
-	return ((val) << A6XX_RB_FS_OUTPUT_CNTL1_MRT__SHIFT) & A6XX_RB_FS_OUTPUT_CNTL1_MRT__MASK;
-}
-
-#define REG_A6XX_RB_RENDER_COMPONENTS				0x0000880d
-#define A6XX_RB_RENDER_COMPONENTS_RT0__MASK			0x0000000f
-#define A6XX_RB_RENDER_COMPONENTS_RT0__SHIFT			0
-static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT0(uint32_t val)
-{
-	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT0__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT0__MASK;
-}
-#define A6XX_RB_RENDER_COMPONENTS_RT1__MASK			0x000000f0
-#define A6XX_RB_RENDER_COMPONENTS_RT1__SHIFT			4
-static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT1(uint32_t val)
-{
-	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT1__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT1__MASK;
-}
-#define A6XX_RB_RENDER_COMPONENTS_RT2__MASK			0x00000f00
-#define A6XX_RB_RENDER_COMPONENTS_RT2__SHIFT			8
-static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT2(uint32_t val)
-{
-	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT2__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT2__MASK;
-}
-#define A6XX_RB_RENDER_COMPONENTS_RT3__MASK			0x0000f000
-#define A6XX_RB_RENDER_COMPONENTS_RT3__SHIFT			12
-static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT3(uint32_t val)
-{
-	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT3__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT3__MASK;
-}
-#define A6XX_RB_RENDER_COMPONENTS_RT4__MASK			0x000f0000
-#define A6XX_RB_RENDER_COMPONENTS_RT4__SHIFT			16
-static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT4(uint32_t val)
-{
-	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT4__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT4__MASK;
-}
-#define A6XX_RB_RENDER_COMPONENTS_RT5__MASK			0x00f00000
-#define A6XX_RB_RENDER_COMPONENTS_RT5__SHIFT			20
-static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT5(uint32_t val)
-{
-	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT5__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT5__MASK;
-}
-#define A6XX_RB_RENDER_COMPONENTS_RT6__MASK			0x0f000000
-#define A6XX_RB_RENDER_COMPONENTS_RT6__SHIFT			24
-static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT6(uint32_t val)
-{
-	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT6__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT6__MASK;
-}
-#define A6XX_RB_RENDER_COMPONENTS_RT7__MASK			0xf0000000
-#define A6XX_RB_RENDER_COMPONENTS_RT7__SHIFT			28
-static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT7(uint32_t val)
-{
-	return ((val) << A6XX_RB_RENDER_COMPONENTS_RT7__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT7__MASK;
-}
-
-#define REG_A6XX_RB_DITHER_CNTL					0x0000880e
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0__MASK		0x00000003
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0__SHIFT		0
-static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0__MASK;
-}
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1__MASK		0x0000000c
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1__SHIFT		2
-static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1__MASK;
-}
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2__MASK		0x00000030
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2__SHIFT		4
-static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2__MASK;
-}
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3__MASK		0x000000c0
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3__SHIFT		6
-static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3__MASK;
-}
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4__MASK		0x00000300
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4__SHIFT		8
-static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4__MASK;
-}
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5__MASK		0x00000c00
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5__SHIFT		10
-static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5__MASK;
-}
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6__MASK		0x00003000
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6__SHIFT		12
-static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6__MASK;
-}
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7__MASK		0x0000c000
-#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7__SHIFT		14
-static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7(enum adreno_rb_dither_mode val)
-{
-	return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7__MASK;
-}
-
-#define REG_A6XX_RB_SRGB_CNTL					0x0000880f
-#define A6XX_RB_SRGB_CNTL_SRGB_MRT0				0x00000001
-#define A6XX_RB_SRGB_CNTL_SRGB_MRT1				0x00000002
-#define A6XX_RB_SRGB_CNTL_SRGB_MRT2				0x00000004
-#define A6XX_RB_SRGB_CNTL_SRGB_MRT3				0x00000008
-#define A6XX_RB_SRGB_CNTL_SRGB_MRT4				0x00000010
-#define A6XX_RB_SRGB_CNTL_SRGB_MRT5				0x00000020
-#define A6XX_RB_SRGB_CNTL_SRGB_MRT6				0x00000040
-#define A6XX_RB_SRGB_CNTL_SRGB_MRT7				0x00000080
-
-#define REG_A6XX_RB_SAMPLE_CNTL					0x00008810
-#define A6XX_RB_SAMPLE_CNTL_PER_SAMP_MODE			0x00000001
-
-#define REG_A6XX_RB_UNKNOWN_8811				0x00008811
-
-#define REG_A7XX_RB_UNKNOWN_8812				0x00008812
-
-#define REG_A6XX_RB_UNKNOWN_8818				0x00008818
-
-#define REG_A6XX_RB_UNKNOWN_8819				0x00008819
-
-#define REG_A6XX_RB_UNKNOWN_881A				0x0000881a
-
-#define REG_A6XX_RB_UNKNOWN_881B				0x0000881b
-
-#define REG_A6XX_RB_UNKNOWN_881C				0x0000881c
-
-#define REG_A6XX_RB_UNKNOWN_881D				0x0000881d
-
-#define REG_A6XX_RB_UNKNOWN_881E				0x0000881e
-
-#define REG_A6XX_RB_MRT(i0) (0x00008820 + 0x8*(i0))
-
-static inline uint32_t REG_A6XX_RB_MRT_CONTROL(uint32_t i0) { return 0x00008820 + 0x8*i0; }
-#define A6XX_RB_MRT_CONTROL_BLEND				0x00000001
-#define A6XX_RB_MRT_CONTROL_BLEND2				0x00000002
-#define A6XX_RB_MRT_CONTROL_ROP_ENABLE				0x00000004
-#define A6XX_RB_MRT_CONTROL_ROP_CODE__MASK			0x00000078
-#define A6XX_RB_MRT_CONTROL_ROP_CODE__SHIFT			3
-static inline uint32_t A6XX_RB_MRT_CONTROL_ROP_CODE(enum a3xx_rop_code val)
-{
-	return ((val) << A6XX_RB_MRT_CONTROL_ROP_CODE__SHIFT) & A6XX_RB_MRT_CONTROL_ROP_CODE__MASK;
-}
-#define A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK		0x00000780
-#define A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT		7
-static inline uint32_t A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE(uint32_t val)
-{
-	return ((val) << A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT) & A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK;
-}
-
-static inline uint32_t REG_A6XX_RB_MRT_BLEND_CONTROL(uint32_t i0) { return 0x00008821 + 0x8*i0; }
-#define A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK		0x0000001f
-#define A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT		0
-static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK;
-}
-#define A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK	0x000000e0
-#define A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT	5
-static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
-{
-	return ((val) << A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK;
-}
-#define A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK		0x00001f00
-#define A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT	8
-static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK;
-}
-#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK	0x001f0000
-#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT	16
-static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK;
-}
-#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK	0x00e00000
-#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT	21
-static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
-{
-	return ((val) << A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK;
-}
-#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK	0x1f000000
-#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT	24
-static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR(enum adreno_rb_blend_factor val)
-{
-	return ((val) << A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK;
-}
-
-static inline uint32_t REG_A6XX_RB_MRT_BUF_INFO(uint32_t i0) { return 0x00008822 + 0x8*i0; }
-#define A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK			0x000000ff
-#define A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT		0
-static inline uint32_t A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT) & A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK;
-}
-#define A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK		0x00000300
-#define A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT		8
-static inline uint32_t A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE(enum a6xx_tile_mode val)
-{
-	return ((val) << A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT) & A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK;
-}
-#define A6XX_RB_MRT_BUF_INFO_UNK10				0x00000400
-#define A6XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK			0x00006000
-#define A6XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT			13
-static inline uint32_t A6XX_RB_MRT_BUF_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A6XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT) & A6XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK;
-}
-
-static inline uint32_t REG_A7XX_RB_MRT_BUF_INFO(uint32_t i0) { return 0x00008822 + 0x8*i0; }
-#define A7XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK			0x000000ff
-#define A7XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT		0
-static inline uint32_t A7XX_RB_MRT_BUF_INFO_COLOR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A7XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT) & A7XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK;
-}
-#define A7XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK		0x00000300
-#define A7XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT		8
-static inline uint32_t A7XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE(enum a6xx_tile_mode val)
-{
-	return ((val) << A7XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT) & A7XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK;
-}
-#define A7XX_RB_MRT_BUF_INFO_UNK10				0x00000400
-#define A7XX_RB_MRT_BUF_INFO_LOSSLESSCOMPEN			0x00000800
-#define A7XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK			0x00006000
-#define A7XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT			13
-static inline uint32_t A7XX_RB_MRT_BUF_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A7XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT) & A7XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK;
-}
-
-static inline uint32_t REG_A6XX_RB_MRT_PITCH(uint32_t i0) { return 0x00008823 + 0x8*i0; }
-#define A6XX_RB_MRT_PITCH__MASK					0xffffffff
-#define A6XX_RB_MRT_PITCH__SHIFT				0
-static inline uint32_t A6XX_RB_MRT_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_MRT_PITCH__SHIFT) & A6XX_RB_MRT_PITCH__MASK;
-}
-
-static inline uint32_t REG_A6XX_RB_MRT_ARRAY_PITCH(uint32_t i0) { return 0x00008824 + 0x8*i0; }
-#define A6XX_RB_MRT_ARRAY_PITCH__MASK				0xffffffff
-#define A6XX_RB_MRT_ARRAY_PITCH__SHIFT				0
-static inline uint32_t A6XX_RB_MRT_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_MRT_ARRAY_PITCH__SHIFT) & A6XX_RB_MRT_ARRAY_PITCH__MASK;
-}
-
-static inline uint32_t REG_A6XX_RB_MRT_BASE(uint32_t i0) { return 0x00008825 + 0x8*i0; }
-
-static inline uint32_t REG_A6XX_RB_MRT_BASE_GMEM(uint32_t i0) { return 0x00008827 + 0x8*i0; }
-
-#define REG_A6XX_RB_BLEND_RED_F32				0x00008860
-#define A6XX_RB_BLEND_RED_F32__MASK				0xffffffff
-#define A6XX_RB_BLEND_RED_F32__SHIFT				0
-static inline uint32_t A6XX_RB_BLEND_RED_F32(float val)
-{
-	return ((fui(val)) << A6XX_RB_BLEND_RED_F32__SHIFT) & A6XX_RB_BLEND_RED_F32__MASK;
-}
-
-#define REG_A6XX_RB_BLEND_GREEN_F32				0x00008861
-#define A6XX_RB_BLEND_GREEN_F32__MASK				0xffffffff
-#define A6XX_RB_BLEND_GREEN_F32__SHIFT				0
-static inline uint32_t A6XX_RB_BLEND_GREEN_F32(float val)
-{
-	return ((fui(val)) << A6XX_RB_BLEND_GREEN_F32__SHIFT) & A6XX_RB_BLEND_GREEN_F32__MASK;
-}
-
-#define REG_A6XX_RB_BLEND_BLUE_F32				0x00008862
-#define A6XX_RB_BLEND_BLUE_F32__MASK				0xffffffff
-#define A6XX_RB_BLEND_BLUE_F32__SHIFT				0
-static inline uint32_t A6XX_RB_BLEND_BLUE_F32(float val)
-{
-	return ((fui(val)) << A6XX_RB_BLEND_BLUE_F32__SHIFT) & A6XX_RB_BLEND_BLUE_F32__MASK;
-}
-
-#define REG_A6XX_RB_BLEND_ALPHA_F32				0x00008863
-#define A6XX_RB_BLEND_ALPHA_F32__MASK				0xffffffff
-#define A6XX_RB_BLEND_ALPHA_F32__SHIFT				0
-static inline uint32_t A6XX_RB_BLEND_ALPHA_F32(float val)
-{
-	return ((fui(val)) << A6XX_RB_BLEND_ALPHA_F32__SHIFT) & A6XX_RB_BLEND_ALPHA_F32__MASK;
-}
-
-#define REG_A6XX_RB_ALPHA_CONTROL				0x00008864
-#define A6XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK			0x000000ff
-#define A6XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT			0
-static inline uint32_t A6XX_RB_ALPHA_CONTROL_ALPHA_REF(uint32_t val)
-{
-	return ((val) << A6XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT) & A6XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK;
-}
-#define A6XX_RB_ALPHA_CONTROL_ALPHA_TEST			0x00000100
-#define A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__MASK		0x00000e00
-#define A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__SHIFT		9
-static inline uint32_t A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__SHIFT) & A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__MASK;
-}
-
-#define REG_A6XX_RB_BLEND_CNTL					0x00008865
-#define A6XX_RB_BLEND_CNTL_ENABLE_BLEND__MASK			0x000000ff
-#define A6XX_RB_BLEND_CNTL_ENABLE_BLEND__SHIFT			0
-static inline uint32_t A6XX_RB_BLEND_CNTL_ENABLE_BLEND(uint32_t val)
-{
-	return ((val) << A6XX_RB_BLEND_CNTL_ENABLE_BLEND__SHIFT) & A6XX_RB_BLEND_CNTL_ENABLE_BLEND__MASK;
-}
-#define A6XX_RB_BLEND_CNTL_INDEPENDENT_BLEND			0x00000100
-#define A6XX_RB_BLEND_CNTL_DUAL_COLOR_IN_ENABLE			0x00000200
-#define A6XX_RB_BLEND_CNTL_ALPHA_TO_COVERAGE			0x00000400
-#define A6XX_RB_BLEND_CNTL_ALPHA_TO_ONE				0x00000800
-#define A6XX_RB_BLEND_CNTL_SAMPLE_MASK__MASK			0xffff0000
-#define A6XX_RB_BLEND_CNTL_SAMPLE_MASK__SHIFT			16
-static inline uint32_t A6XX_RB_BLEND_CNTL_SAMPLE_MASK(uint32_t val)
-{
-	return ((val) << A6XX_RB_BLEND_CNTL_SAMPLE_MASK__SHIFT) & A6XX_RB_BLEND_CNTL_SAMPLE_MASK__MASK;
-}
-
-#define REG_A6XX_RB_DEPTH_PLANE_CNTL				0x00008870
-#define A6XX_RB_DEPTH_PLANE_CNTL_Z_MODE__MASK			0x00000003
-#define A6XX_RB_DEPTH_PLANE_CNTL_Z_MODE__SHIFT			0
-static inline uint32_t A6XX_RB_DEPTH_PLANE_CNTL_Z_MODE(enum a6xx_ztest_mode val)
-{
-	return ((val) << A6XX_RB_DEPTH_PLANE_CNTL_Z_MODE__SHIFT) & A6XX_RB_DEPTH_PLANE_CNTL_Z_MODE__MASK;
-}
-
-#define REG_A6XX_RB_DEPTH_CNTL					0x00008871
-#define A6XX_RB_DEPTH_CNTL_Z_TEST_ENABLE			0x00000001
-#define A6XX_RB_DEPTH_CNTL_Z_WRITE_ENABLE			0x00000002
-#define A6XX_RB_DEPTH_CNTL_ZFUNC__MASK				0x0000001c
-#define A6XX_RB_DEPTH_CNTL_ZFUNC__SHIFT				2
-static inline uint32_t A6XX_RB_DEPTH_CNTL_ZFUNC(enum adreno_compare_func val)
-{
-	return ((val) << A6XX_RB_DEPTH_CNTL_ZFUNC__SHIFT) & A6XX_RB_DEPTH_CNTL_ZFUNC__MASK;
-}
-#define A6XX_RB_DEPTH_CNTL_Z_CLAMP_ENABLE			0x00000020
-#define A6XX_RB_DEPTH_CNTL_Z_READ_ENABLE			0x00000040
-#define A6XX_RB_DEPTH_CNTL_Z_BOUNDS_ENABLE			0x00000080
-
-#define REG_A6XX_GRAS_SU_DEPTH_CNTL				0x00008114
-#define A6XX_GRAS_SU_DEPTH_CNTL_Z_TEST_ENABLE			0x00000001
-
-#define REG_A6XX_RB_DEPTH_BUFFER_INFO				0x00008872
-#define A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK		0x00000007
-#define A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT		0
-static inline uint32_t A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT(enum a6xx_depth_format val)
-{
-	return ((val) << A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT) & A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK;
-}
-#define A6XX_RB_DEPTH_BUFFER_INFO_UNK3__MASK			0x00000018
-#define A6XX_RB_DEPTH_BUFFER_INFO_UNK3__SHIFT			3
-static inline uint32_t A6XX_RB_DEPTH_BUFFER_INFO_UNK3(uint32_t val)
-{
-	return ((val) << A6XX_RB_DEPTH_BUFFER_INFO_UNK3__SHIFT) & A6XX_RB_DEPTH_BUFFER_INFO_UNK3__MASK;
-}
-
-#define REG_A7XX_RB_DEPTH_BUFFER_INFO				0x00008872
-#define A7XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK		0x00000007
-#define A7XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT		0
-static inline uint32_t A7XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT(enum a6xx_depth_format val)
-{
-	return ((val) << A7XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT) & A7XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK;
-}
-#define A7XX_RB_DEPTH_BUFFER_INFO_UNK3__MASK			0x00000018
-#define A7XX_RB_DEPTH_BUFFER_INFO_UNK3__SHIFT			3
-static inline uint32_t A7XX_RB_DEPTH_BUFFER_INFO_UNK3(uint32_t val)
-{
-	return ((val) << A7XX_RB_DEPTH_BUFFER_INFO_UNK3__SHIFT) & A7XX_RB_DEPTH_BUFFER_INFO_UNK3__MASK;
-}
-#define A7XX_RB_DEPTH_BUFFER_INFO_TILEMODE__MASK		0x00000060
-#define A7XX_RB_DEPTH_BUFFER_INFO_TILEMODE__SHIFT		5
-static inline uint32_t A7XX_RB_DEPTH_BUFFER_INFO_TILEMODE(enum a6xx_tile_mode val)
-{
-	return ((val) << A7XX_RB_DEPTH_BUFFER_INFO_TILEMODE__SHIFT) & A7XX_RB_DEPTH_BUFFER_INFO_TILEMODE__MASK;
-}
-#define A7XX_RB_DEPTH_BUFFER_INFO_LOSSLESSCOMPEN		0x00000080
-
-#define REG_A6XX_RB_DEPTH_BUFFER_PITCH				0x00008873
-#define A6XX_RB_DEPTH_BUFFER_PITCH__MASK			0x00003fff
-#define A6XX_RB_DEPTH_BUFFER_PITCH__SHIFT			0
-static inline uint32_t A6XX_RB_DEPTH_BUFFER_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_DEPTH_BUFFER_PITCH__SHIFT) & A6XX_RB_DEPTH_BUFFER_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH			0x00008874
-#define A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH__MASK			0x0fffffff
-#define A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH__SHIFT			0
-static inline uint32_t A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH__SHIFT) & A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_DEPTH_BUFFER_BASE				0x00008875
-
-#define REG_A6XX_RB_DEPTH_BUFFER_BASE_GMEM			0x00008877
-
-#define REG_A6XX_RB_Z_BOUNDS_MIN				0x00008878
-#define A6XX_RB_Z_BOUNDS_MIN__MASK				0xffffffff
-#define A6XX_RB_Z_BOUNDS_MIN__SHIFT				0
-static inline uint32_t A6XX_RB_Z_BOUNDS_MIN(float val)
-{
-	return ((fui(val)) << A6XX_RB_Z_BOUNDS_MIN__SHIFT) & A6XX_RB_Z_BOUNDS_MIN__MASK;
-}
-
-#define REG_A6XX_RB_Z_BOUNDS_MAX				0x00008879
-#define A6XX_RB_Z_BOUNDS_MAX__MASK				0xffffffff
-#define A6XX_RB_Z_BOUNDS_MAX__SHIFT				0
-static inline uint32_t A6XX_RB_Z_BOUNDS_MAX(float val)
-{
-	return ((fui(val)) << A6XX_RB_Z_BOUNDS_MAX__SHIFT) & A6XX_RB_Z_BOUNDS_MAX__MASK;
-}
-
-#define REG_A6XX_RB_STENCIL_CONTROL				0x00008880
-#define A6XX_RB_STENCIL_CONTROL_STENCIL_ENABLE			0x00000001
-#define A6XX_RB_STENCIL_CONTROL_STENCIL_ENABLE_BF		0x00000002
-#define A6XX_RB_STENCIL_CONTROL_STENCIL_READ			0x00000004
-#define A6XX_RB_STENCIL_CONTROL_FUNC__MASK			0x00000700
-#define A6XX_RB_STENCIL_CONTROL_FUNC__SHIFT			8
-static inline uint32_t A6XX_RB_STENCIL_CONTROL_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A6XX_RB_STENCIL_CONTROL_FUNC__SHIFT) & A6XX_RB_STENCIL_CONTROL_FUNC__MASK;
-}
-#define A6XX_RB_STENCIL_CONTROL_FAIL__MASK			0x00003800
-#define A6XX_RB_STENCIL_CONTROL_FAIL__SHIFT			11
-static inline uint32_t A6XX_RB_STENCIL_CONTROL_FAIL(enum adreno_stencil_op val)
-{
-	return ((val) << A6XX_RB_STENCIL_CONTROL_FAIL__SHIFT) & A6XX_RB_STENCIL_CONTROL_FAIL__MASK;
-}
-#define A6XX_RB_STENCIL_CONTROL_ZPASS__MASK			0x0001c000
-#define A6XX_RB_STENCIL_CONTROL_ZPASS__SHIFT			14
-static inline uint32_t A6XX_RB_STENCIL_CONTROL_ZPASS(enum adreno_stencil_op val)
-{
-	return ((val) << A6XX_RB_STENCIL_CONTROL_ZPASS__SHIFT) & A6XX_RB_STENCIL_CONTROL_ZPASS__MASK;
-}
-#define A6XX_RB_STENCIL_CONTROL_ZFAIL__MASK			0x000e0000
-#define A6XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT			17
-static inline uint32_t A6XX_RB_STENCIL_CONTROL_ZFAIL(enum adreno_stencil_op val)
-{
-	return ((val) << A6XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT) & A6XX_RB_STENCIL_CONTROL_ZFAIL__MASK;
-}
-#define A6XX_RB_STENCIL_CONTROL_FUNC_BF__MASK			0x00700000
-#define A6XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT			20
-static inline uint32_t A6XX_RB_STENCIL_CONTROL_FUNC_BF(enum adreno_compare_func val)
-{
-	return ((val) << A6XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT) & A6XX_RB_STENCIL_CONTROL_FUNC_BF__MASK;
-}
-#define A6XX_RB_STENCIL_CONTROL_FAIL_BF__MASK			0x03800000
-#define A6XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT			23
-static inline uint32_t A6XX_RB_STENCIL_CONTROL_FAIL_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A6XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT) & A6XX_RB_STENCIL_CONTROL_FAIL_BF__MASK;
-}
-#define A6XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK			0x1c000000
-#define A6XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT			26
-static inline uint32_t A6XX_RB_STENCIL_CONTROL_ZPASS_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A6XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT) & A6XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK;
-}
-#define A6XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK			0xe0000000
-#define A6XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT			29
-static inline uint32_t A6XX_RB_STENCIL_CONTROL_ZFAIL_BF(enum adreno_stencil_op val)
-{
-	return ((val) << A6XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT) & A6XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK;
-}
-
-#define REG_A6XX_GRAS_SU_STENCIL_CNTL				0x00008115
-#define A6XX_GRAS_SU_STENCIL_CNTL_STENCIL_ENABLE		0x00000001
-
-#define REG_A6XX_RB_STENCIL_INFO				0x00008881
-#define A6XX_RB_STENCIL_INFO_SEPARATE_STENCIL			0x00000001
-#define A6XX_RB_STENCIL_INFO_UNK1				0x00000002
-
-#define REG_A7XX_RB_STENCIL_INFO				0x00008881
-#define A7XX_RB_STENCIL_INFO_SEPARATE_STENCIL			0x00000001
-#define A7XX_RB_STENCIL_INFO_UNK1				0x00000002
-#define A7XX_RB_STENCIL_INFO_TILEMODE__MASK			0x0000000c
-#define A7XX_RB_STENCIL_INFO_TILEMODE__SHIFT			2
-static inline uint32_t A7XX_RB_STENCIL_INFO_TILEMODE(enum a6xx_tile_mode val)
-{
-	return ((val) << A7XX_RB_STENCIL_INFO_TILEMODE__SHIFT) & A7XX_RB_STENCIL_INFO_TILEMODE__MASK;
-}
-
-#define REG_A6XX_RB_STENCIL_BUFFER_PITCH			0x00008882
-#define A6XX_RB_STENCIL_BUFFER_PITCH__MASK			0x00000fff
-#define A6XX_RB_STENCIL_BUFFER_PITCH__SHIFT			0
-static inline uint32_t A6XX_RB_STENCIL_BUFFER_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_STENCIL_BUFFER_PITCH__SHIFT) & A6XX_RB_STENCIL_BUFFER_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH			0x00008883
-#define A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH__MASK		0x00ffffff
-#define A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH__SHIFT		0
-static inline uint32_t A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH__SHIFT) & A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_STENCIL_BUFFER_BASE				0x00008884
-
-#define REG_A6XX_RB_STENCIL_BUFFER_BASE_GMEM			0x00008886
-
-#define REG_A6XX_RB_STENCILREF					0x00008887
-#define A6XX_RB_STENCILREF_REF__MASK				0x000000ff
-#define A6XX_RB_STENCILREF_REF__SHIFT				0
-static inline uint32_t A6XX_RB_STENCILREF_REF(uint32_t val)
-{
-	return ((val) << A6XX_RB_STENCILREF_REF__SHIFT) & A6XX_RB_STENCILREF_REF__MASK;
-}
-#define A6XX_RB_STENCILREF_BFREF__MASK				0x0000ff00
-#define A6XX_RB_STENCILREF_BFREF__SHIFT				8
-static inline uint32_t A6XX_RB_STENCILREF_BFREF(uint32_t val)
-{
-	return ((val) << A6XX_RB_STENCILREF_BFREF__SHIFT) & A6XX_RB_STENCILREF_BFREF__MASK;
-}
-
-#define REG_A6XX_RB_STENCILMASK					0x00008888
-#define A6XX_RB_STENCILMASK_MASK__MASK				0x000000ff
-#define A6XX_RB_STENCILMASK_MASK__SHIFT				0
-static inline uint32_t A6XX_RB_STENCILMASK_MASK(uint32_t val)
-{
-	return ((val) << A6XX_RB_STENCILMASK_MASK__SHIFT) & A6XX_RB_STENCILMASK_MASK__MASK;
-}
-#define A6XX_RB_STENCILMASK_BFMASK__MASK			0x0000ff00
-#define A6XX_RB_STENCILMASK_BFMASK__SHIFT			8
-static inline uint32_t A6XX_RB_STENCILMASK_BFMASK(uint32_t val)
-{
-	return ((val) << A6XX_RB_STENCILMASK_BFMASK__SHIFT) & A6XX_RB_STENCILMASK_BFMASK__MASK;
-}
-
-#define REG_A6XX_RB_STENCILWRMASK				0x00008889
-#define A6XX_RB_STENCILWRMASK_WRMASK__MASK			0x000000ff
-#define A6XX_RB_STENCILWRMASK_WRMASK__SHIFT			0
-static inline uint32_t A6XX_RB_STENCILWRMASK_WRMASK(uint32_t val)
-{
-	return ((val) << A6XX_RB_STENCILWRMASK_WRMASK__SHIFT) & A6XX_RB_STENCILWRMASK_WRMASK__MASK;
-}
-#define A6XX_RB_STENCILWRMASK_BFWRMASK__MASK			0x0000ff00
-#define A6XX_RB_STENCILWRMASK_BFWRMASK__SHIFT			8
-static inline uint32_t A6XX_RB_STENCILWRMASK_BFWRMASK(uint32_t val)
-{
-	return ((val) << A6XX_RB_STENCILWRMASK_BFWRMASK__SHIFT) & A6XX_RB_STENCILWRMASK_BFWRMASK__MASK;
-}
-
-#define REG_A6XX_RB_WINDOW_OFFSET				0x00008890
-#define A6XX_RB_WINDOW_OFFSET_X__MASK				0x00003fff
-#define A6XX_RB_WINDOW_OFFSET_X__SHIFT				0
-static inline uint32_t A6XX_RB_WINDOW_OFFSET_X(uint32_t val)
-{
-	return ((val) << A6XX_RB_WINDOW_OFFSET_X__SHIFT) & A6XX_RB_WINDOW_OFFSET_X__MASK;
-}
-#define A6XX_RB_WINDOW_OFFSET_Y__MASK				0x3fff0000
-#define A6XX_RB_WINDOW_OFFSET_Y__SHIFT				16
-static inline uint32_t A6XX_RB_WINDOW_OFFSET_Y(uint32_t val)
-{
-	return ((val) << A6XX_RB_WINDOW_OFFSET_Y__SHIFT) & A6XX_RB_WINDOW_OFFSET_Y__MASK;
-}
-
-#define REG_A6XX_RB_SAMPLE_COUNT_CONTROL			0x00008891
-#define A6XX_RB_SAMPLE_COUNT_CONTROL_DISABLE			0x00000001
-#define A6XX_RB_SAMPLE_COUNT_CONTROL_COPY			0x00000002
-
-#define REG_A6XX_RB_LRZ_CNTL					0x00008898
-#define A6XX_RB_LRZ_CNTL_ENABLE					0x00000001
-
-#define REG_A7XX_RB_UNKNOWN_8899				0x00008899
-
-#define REG_A6XX_RB_Z_CLAMP_MIN					0x000088c0
-#define A6XX_RB_Z_CLAMP_MIN__MASK				0xffffffff
-#define A6XX_RB_Z_CLAMP_MIN__SHIFT				0
-static inline uint32_t A6XX_RB_Z_CLAMP_MIN(float val)
-{
-	return ((fui(val)) << A6XX_RB_Z_CLAMP_MIN__SHIFT) & A6XX_RB_Z_CLAMP_MIN__MASK;
-}
-
-#define REG_A6XX_RB_Z_CLAMP_MAX					0x000088c1
-#define A6XX_RB_Z_CLAMP_MAX__MASK				0xffffffff
-#define A6XX_RB_Z_CLAMP_MAX__SHIFT				0
-static inline uint32_t A6XX_RB_Z_CLAMP_MAX(float val)
-{
-	return ((fui(val)) << A6XX_RB_Z_CLAMP_MAX__SHIFT) & A6XX_RB_Z_CLAMP_MAX__MASK;
-}
-
-#define REG_A6XX_RB_UNKNOWN_88D0				0x000088d0
-#define A6XX_RB_UNKNOWN_88D0_UNK0__MASK				0x00001fff
-#define A6XX_RB_UNKNOWN_88D0_UNK0__SHIFT			0
-static inline uint32_t A6XX_RB_UNKNOWN_88D0_UNK0(uint32_t val)
-{
-	return ((val) << A6XX_RB_UNKNOWN_88D0_UNK0__SHIFT) & A6XX_RB_UNKNOWN_88D0_UNK0__MASK;
-}
-#define A6XX_RB_UNKNOWN_88D0_UNK16__MASK			0x07ff0000
-#define A6XX_RB_UNKNOWN_88D0_UNK16__SHIFT			16
-static inline uint32_t A6XX_RB_UNKNOWN_88D0_UNK16(uint32_t val)
-{
-	return ((val) << A6XX_RB_UNKNOWN_88D0_UNK16__SHIFT) & A6XX_RB_UNKNOWN_88D0_UNK16__MASK;
-}
-
-#define REG_A6XX_RB_BLIT_SCISSOR_TL				0x000088d1
-#define A6XX_RB_BLIT_SCISSOR_TL_X__MASK				0x00003fff
-#define A6XX_RB_BLIT_SCISSOR_TL_X__SHIFT			0
-static inline uint32_t A6XX_RB_BLIT_SCISSOR_TL_X(uint32_t val)
-{
-	return ((val) << A6XX_RB_BLIT_SCISSOR_TL_X__SHIFT) & A6XX_RB_BLIT_SCISSOR_TL_X__MASK;
-}
-#define A6XX_RB_BLIT_SCISSOR_TL_Y__MASK				0x3fff0000
-#define A6XX_RB_BLIT_SCISSOR_TL_Y__SHIFT			16
-static inline uint32_t A6XX_RB_BLIT_SCISSOR_TL_Y(uint32_t val)
-{
-	return ((val) << A6XX_RB_BLIT_SCISSOR_TL_Y__SHIFT) & A6XX_RB_BLIT_SCISSOR_TL_Y__MASK;
-}
-
-#define REG_A6XX_RB_BLIT_SCISSOR_BR				0x000088d2
-#define A6XX_RB_BLIT_SCISSOR_BR_X__MASK				0x00003fff
-#define A6XX_RB_BLIT_SCISSOR_BR_X__SHIFT			0
-static inline uint32_t A6XX_RB_BLIT_SCISSOR_BR_X(uint32_t val)
-{
-	return ((val) << A6XX_RB_BLIT_SCISSOR_BR_X__SHIFT) & A6XX_RB_BLIT_SCISSOR_BR_X__MASK;
-}
-#define A6XX_RB_BLIT_SCISSOR_BR_Y__MASK				0x3fff0000
-#define A6XX_RB_BLIT_SCISSOR_BR_Y__SHIFT			16
-static inline uint32_t A6XX_RB_BLIT_SCISSOR_BR_Y(uint32_t val)
-{
-	return ((val) << A6XX_RB_BLIT_SCISSOR_BR_Y__SHIFT) & A6XX_RB_BLIT_SCISSOR_BR_Y__MASK;
-}
-
-#define REG_A6XX_RB_BIN_CONTROL2				0x000088d3
-#define A6XX_RB_BIN_CONTROL2_BINW__MASK				0x0000003f
-#define A6XX_RB_BIN_CONTROL2_BINW__SHIFT			0
-static inline uint32_t A6XX_RB_BIN_CONTROL2_BINW(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A6XX_RB_BIN_CONTROL2_BINW__SHIFT) & A6XX_RB_BIN_CONTROL2_BINW__MASK;
-}
-#define A6XX_RB_BIN_CONTROL2_BINH__MASK				0x00007f00
-#define A6XX_RB_BIN_CONTROL2_BINH__SHIFT			8
-static inline uint32_t A6XX_RB_BIN_CONTROL2_BINH(uint32_t val)
-{
-	assert(!(val & 0xf));
-	return (((val >> 4)) << A6XX_RB_BIN_CONTROL2_BINH__SHIFT) & A6XX_RB_BIN_CONTROL2_BINH__MASK;
-}
-
-#define REG_A6XX_RB_WINDOW_OFFSET2				0x000088d4
-#define A6XX_RB_WINDOW_OFFSET2_X__MASK				0x00003fff
-#define A6XX_RB_WINDOW_OFFSET2_X__SHIFT				0
-static inline uint32_t A6XX_RB_WINDOW_OFFSET2_X(uint32_t val)
-{
-	return ((val) << A6XX_RB_WINDOW_OFFSET2_X__SHIFT) & A6XX_RB_WINDOW_OFFSET2_X__MASK;
-}
-#define A6XX_RB_WINDOW_OFFSET2_Y__MASK				0x3fff0000
-#define A6XX_RB_WINDOW_OFFSET2_Y__SHIFT				16
-static inline uint32_t A6XX_RB_WINDOW_OFFSET2_Y(uint32_t val)
-{
-	return ((val) << A6XX_RB_WINDOW_OFFSET2_Y__SHIFT) & A6XX_RB_WINDOW_OFFSET2_Y__MASK;
-}
-
-#define REG_A6XX_RB_BLIT_GMEM_MSAA_CNTL				0x000088d5
-#define A6XX_RB_BLIT_GMEM_MSAA_CNTL_SAMPLES__MASK		0x00000018
-#define A6XX_RB_BLIT_GMEM_MSAA_CNTL_SAMPLES__SHIFT		3
-static inline uint32_t A6XX_RB_BLIT_GMEM_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A6XX_RB_BLIT_GMEM_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_RB_BLIT_GMEM_MSAA_CNTL_SAMPLES__MASK;
-}
-
-#define REG_A6XX_RB_BLIT_BASE_GMEM				0x000088d6
-
-#define REG_A6XX_RB_BLIT_DST_INFO				0x000088d7
-#define A6XX_RB_BLIT_DST_INFO_TILE_MODE__MASK			0x00000003
-#define A6XX_RB_BLIT_DST_INFO_TILE_MODE__SHIFT			0
-static inline uint32_t A6XX_RB_BLIT_DST_INFO_TILE_MODE(enum a6xx_tile_mode val)
-{
-	return ((val) << A6XX_RB_BLIT_DST_INFO_TILE_MODE__SHIFT) & A6XX_RB_BLIT_DST_INFO_TILE_MODE__MASK;
-}
-#define A6XX_RB_BLIT_DST_INFO_FLAGS				0x00000004
-#define A6XX_RB_BLIT_DST_INFO_SAMPLES__MASK			0x00000018
-#define A6XX_RB_BLIT_DST_INFO_SAMPLES__SHIFT			3
-static inline uint32_t A6XX_RB_BLIT_DST_INFO_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A6XX_RB_BLIT_DST_INFO_SAMPLES__SHIFT) & A6XX_RB_BLIT_DST_INFO_SAMPLES__MASK;
-}
-#define A6XX_RB_BLIT_DST_INFO_COLOR_SWAP__MASK			0x00000060
-#define A6XX_RB_BLIT_DST_INFO_COLOR_SWAP__SHIFT			5
-static inline uint32_t A6XX_RB_BLIT_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A6XX_RB_BLIT_DST_INFO_COLOR_SWAP__SHIFT) & A6XX_RB_BLIT_DST_INFO_COLOR_SWAP__MASK;
-}
-#define A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT__MASK		0x00007f80
-#define A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT__SHIFT		7
-static inline uint32_t A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT__SHIFT) & A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT__MASK;
-}
-#define A6XX_RB_BLIT_DST_INFO_UNK15				0x00008000
-
-#define REG_A6XX_RB_BLIT_DST					0x000088d8
-
-#define REG_A6XX_RB_BLIT_DST_PITCH				0x000088da
-#define A6XX_RB_BLIT_DST_PITCH__MASK				0x0000ffff
-#define A6XX_RB_BLIT_DST_PITCH__SHIFT				0
-static inline uint32_t A6XX_RB_BLIT_DST_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_BLIT_DST_PITCH__SHIFT) & A6XX_RB_BLIT_DST_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_BLIT_DST_ARRAY_PITCH			0x000088db
-#define A6XX_RB_BLIT_DST_ARRAY_PITCH__MASK			0x1fffffff
-#define A6XX_RB_BLIT_DST_ARRAY_PITCH__SHIFT			0
-static inline uint32_t A6XX_RB_BLIT_DST_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_BLIT_DST_ARRAY_PITCH__SHIFT) & A6XX_RB_BLIT_DST_ARRAY_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_BLIT_FLAG_DST				0x000088dc
-
-#define REG_A6XX_RB_BLIT_FLAG_DST_PITCH				0x000088de
-#define A6XX_RB_BLIT_FLAG_DST_PITCH_PITCH__MASK			0x000007ff
-#define A6XX_RB_BLIT_FLAG_DST_PITCH_PITCH__SHIFT		0
-static inline uint32_t A6XX_RB_BLIT_FLAG_DST_PITCH_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_BLIT_FLAG_DST_PITCH_PITCH__SHIFT) & A6XX_RB_BLIT_FLAG_DST_PITCH_PITCH__MASK;
-}
-#define A6XX_RB_BLIT_FLAG_DST_PITCH_ARRAY_PITCH__MASK		0x0ffff800
-#define A6XX_RB_BLIT_FLAG_DST_PITCH_ARRAY_PITCH__SHIFT		11
-static inline uint32_t A6XX_RB_BLIT_FLAG_DST_PITCH_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x7f));
-	return (((val >> 7)) << A6XX_RB_BLIT_FLAG_DST_PITCH_ARRAY_PITCH__SHIFT) & A6XX_RB_BLIT_FLAG_DST_PITCH_ARRAY_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_BLIT_CLEAR_COLOR_DW0			0x000088df
-
-#define REG_A6XX_RB_BLIT_CLEAR_COLOR_DW1			0x000088e0
-
-#define REG_A6XX_RB_BLIT_CLEAR_COLOR_DW2			0x000088e1
-
-#define REG_A6XX_RB_BLIT_CLEAR_COLOR_DW3			0x000088e2
-
-#define REG_A6XX_RB_BLIT_INFO					0x000088e3
-#define A6XX_RB_BLIT_INFO_UNK0					0x00000001
-#define A6XX_RB_BLIT_INFO_GMEM					0x00000002
-#define A6XX_RB_BLIT_INFO_SAMPLE_0				0x00000004
-#define A6XX_RB_BLIT_INFO_DEPTH					0x00000008
-#define A6XX_RB_BLIT_INFO_CLEAR_MASK__MASK			0x000000f0
-#define A6XX_RB_BLIT_INFO_CLEAR_MASK__SHIFT			4
-static inline uint32_t A6XX_RB_BLIT_INFO_CLEAR_MASK(uint32_t val)
-{
-	return ((val) << A6XX_RB_BLIT_INFO_CLEAR_MASK__SHIFT) & A6XX_RB_BLIT_INFO_CLEAR_MASK__MASK;
-}
-#define A6XX_RB_BLIT_INFO_LAST__MASK				0x00000300
-#define A6XX_RB_BLIT_INFO_LAST__SHIFT				8
-static inline uint32_t A6XX_RB_BLIT_INFO_LAST(uint32_t val)
-{
-	return ((val) << A6XX_RB_BLIT_INFO_LAST__SHIFT) & A6XX_RB_BLIT_INFO_LAST__MASK;
-}
-#define A6XX_RB_BLIT_INFO_BUFFER_ID__MASK			0x0000f000
-#define A6XX_RB_BLIT_INFO_BUFFER_ID__SHIFT			12
-static inline uint32_t A6XX_RB_BLIT_INFO_BUFFER_ID(uint32_t val)
-{
-	return ((val) << A6XX_RB_BLIT_INFO_BUFFER_ID__SHIFT) & A6XX_RB_BLIT_INFO_BUFFER_ID__MASK;
-}
-
-#define REG_A7XX_RB_UNKNOWN_88E4				0x000088e4
-#define A7XX_RB_UNKNOWN_88E4_UNK0				0x00000001
-
-#define REG_A7XX_RB_CCU_CNTL2					0x000088e5
-#define A7XX_RB_CCU_CNTL2_DEPTH_OFFSET_HI__MASK			0x00000001
-#define A7XX_RB_CCU_CNTL2_DEPTH_OFFSET_HI__SHIFT		0
-static inline uint32_t A7XX_RB_CCU_CNTL2_DEPTH_OFFSET_HI(uint32_t val)
-{
-	return ((val) << A7XX_RB_CCU_CNTL2_DEPTH_OFFSET_HI__SHIFT) & A7XX_RB_CCU_CNTL2_DEPTH_OFFSET_HI__MASK;
-}
-#define A7XX_RB_CCU_CNTL2_COLOR_OFFSET_HI__MASK			0x00000004
-#define A7XX_RB_CCU_CNTL2_COLOR_OFFSET_HI__SHIFT		2
-static inline uint32_t A7XX_RB_CCU_CNTL2_COLOR_OFFSET_HI(uint32_t val)
-{
-	return ((val) << A7XX_RB_CCU_CNTL2_COLOR_OFFSET_HI__SHIFT) & A7XX_RB_CCU_CNTL2_COLOR_OFFSET_HI__MASK;
-}
-#define A7XX_RB_CCU_CNTL2_DEPTH_CACHE_SIZE__MASK		0x00000c00
-#define A7XX_RB_CCU_CNTL2_DEPTH_CACHE_SIZE__SHIFT		10
-static inline uint32_t A7XX_RB_CCU_CNTL2_DEPTH_CACHE_SIZE(enum a6xx_ccu_cache_size val)
-{
-	return ((val) << A7XX_RB_CCU_CNTL2_DEPTH_CACHE_SIZE__SHIFT) & A7XX_RB_CCU_CNTL2_DEPTH_CACHE_SIZE__MASK;
-}
-#define A7XX_RB_CCU_CNTL2_DEPTH_OFFSET__MASK			0x001ff000
-#define A7XX_RB_CCU_CNTL2_DEPTH_OFFSET__SHIFT			12
-static inline uint32_t A7XX_RB_CCU_CNTL2_DEPTH_OFFSET(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A7XX_RB_CCU_CNTL2_DEPTH_OFFSET__SHIFT) & A7XX_RB_CCU_CNTL2_DEPTH_OFFSET__MASK;
-}
-#define A7XX_RB_CCU_CNTL2_COLOR_CACHE_SIZE__MASK		0x00600000
-#define A7XX_RB_CCU_CNTL2_COLOR_CACHE_SIZE__SHIFT		21
-static inline uint32_t A7XX_RB_CCU_CNTL2_COLOR_CACHE_SIZE(enum a6xx_ccu_cache_size val)
-{
-	return ((val) << A7XX_RB_CCU_CNTL2_COLOR_CACHE_SIZE__SHIFT) & A7XX_RB_CCU_CNTL2_COLOR_CACHE_SIZE__MASK;
-}
-#define A7XX_RB_CCU_CNTL2_COLOR_OFFSET__MASK			0xff800000
-#define A7XX_RB_CCU_CNTL2_COLOR_OFFSET__SHIFT			23
-static inline uint32_t A7XX_RB_CCU_CNTL2_COLOR_OFFSET(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A7XX_RB_CCU_CNTL2_COLOR_OFFSET__SHIFT) & A7XX_RB_CCU_CNTL2_COLOR_OFFSET__MASK;
-}
-
-#define REG_A6XX_RB_UNKNOWN_88F0				0x000088f0
-
-#define REG_A6XX_RB_UNK_FLAG_BUFFER_BASE			0x000088f1
-
-#define REG_A6XX_RB_UNK_FLAG_BUFFER_PITCH			0x000088f3
-#define A6XX_RB_UNK_FLAG_BUFFER_PITCH_PITCH__MASK		0x000007ff
-#define A6XX_RB_UNK_FLAG_BUFFER_PITCH_PITCH__SHIFT		0
-static inline uint32_t A6XX_RB_UNK_FLAG_BUFFER_PITCH_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_UNK_FLAG_BUFFER_PITCH_PITCH__SHIFT) & A6XX_RB_UNK_FLAG_BUFFER_PITCH_PITCH__MASK;
-}
-#define A6XX_RB_UNK_FLAG_BUFFER_PITCH_ARRAY_PITCH__MASK		0x00fff800
-#define A6XX_RB_UNK_FLAG_BUFFER_PITCH_ARRAY_PITCH__SHIFT	11
-static inline uint32_t A6XX_RB_UNK_FLAG_BUFFER_PITCH_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x7f));
-	return (((val >> 7)) << A6XX_RB_UNK_FLAG_BUFFER_PITCH_ARRAY_PITCH__SHIFT) & A6XX_RB_UNK_FLAG_BUFFER_PITCH_ARRAY_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_UNKNOWN_88F4				0x000088f4
-
-#define REG_A7XX_RB_UNKNOWN_88F5				0x000088f5
-
-#define REG_A6XX_RB_DEPTH_FLAG_BUFFER_BASE			0x00008900
-
-#define REG_A6XX_RB_DEPTH_FLAG_BUFFER_PITCH			0x00008902
-#define A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_PITCH__MASK		0x0000007f
-#define A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_PITCH__SHIFT		0
-static inline uint32_t A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_PITCH__SHIFT) & A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_PITCH__MASK;
-}
-#define A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_UNK8__MASK		0x00000700
-#define A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_UNK8__SHIFT		8
-static inline uint32_t A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_UNK8(uint32_t val)
-{
-	return ((val) << A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_UNK8__SHIFT) & A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_UNK8__MASK;
-}
-#define A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_ARRAY_PITCH__MASK	0x0ffff800
-#define A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_ARRAY_PITCH__SHIFT	11
-static inline uint32_t A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x7f));
-	return (((val >> 7)) << A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_ARRAY_PITCH__SHIFT) & A6XX_RB_DEPTH_FLAG_BUFFER_PITCH_ARRAY_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_MRT_FLAG_BUFFER(i0) (0x00008903 + 0x3*(i0))
-
-static inline uint32_t REG_A6XX_RB_MRT_FLAG_BUFFER_ADDR(uint32_t i0) { return 0x00008903 + 0x3*i0; }
-
-static inline uint32_t REG_A6XX_RB_MRT_FLAG_BUFFER_PITCH(uint32_t i0) { return 0x00008905 + 0x3*i0; }
-#define A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH__MASK		0x000007ff
-#define A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH__SHIFT		0
-static inline uint32_t A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH__SHIFT) & A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH__MASK;
-}
-#define A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH__MASK		0x1ffff800
-#define A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH__SHIFT	11
-static inline uint32_t A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0x7f));
-	return (((val >> 7)) << A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH__SHIFT) & A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_SAMPLE_COUNT_ADDR				0x00008927
-
-#define REG_A6XX_RB_UNKNOWN_8A00				0x00008a00
-
-#define REG_A6XX_RB_UNKNOWN_8A10				0x00008a10
-
-#define REG_A6XX_RB_UNKNOWN_8A20				0x00008a20
-
-#define REG_A6XX_RB_UNKNOWN_8A30				0x00008a30
-
-#define REG_A6XX_RB_2D_BLIT_CNTL				0x00008c00
-#define A6XX_RB_2D_BLIT_CNTL_ROTATE__MASK			0x00000007
-#define A6XX_RB_2D_BLIT_CNTL_ROTATE__SHIFT			0
-static inline uint32_t A6XX_RB_2D_BLIT_CNTL_ROTATE(enum a6xx_rotation val)
-{
-	return ((val) << A6XX_RB_2D_BLIT_CNTL_ROTATE__SHIFT) & A6XX_RB_2D_BLIT_CNTL_ROTATE__MASK;
-}
-#define A6XX_RB_2D_BLIT_CNTL_OVERWRITEEN			0x00000008
-#define A6XX_RB_2D_BLIT_CNTL_UNK4__MASK				0x00000070
-#define A6XX_RB_2D_BLIT_CNTL_UNK4__SHIFT			4
-static inline uint32_t A6XX_RB_2D_BLIT_CNTL_UNK4(uint32_t val)
-{
-	return ((val) << A6XX_RB_2D_BLIT_CNTL_UNK4__SHIFT) & A6XX_RB_2D_BLIT_CNTL_UNK4__MASK;
-}
-#define A6XX_RB_2D_BLIT_CNTL_SOLID_COLOR			0x00000080
-#define A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT__MASK			0x0000ff00
-#define A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT__SHIFT		8
-static inline uint32_t A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT__SHIFT) & A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT__MASK;
-}
-#define A6XX_RB_2D_BLIT_CNTL_SCISSOR				0x00010000
-#define A6XX_RB_2D_BLIT_CNTL_UNK17__MASK			0x00060000
-#define A6XX_RB_2D_BLIT_CNTL_UNK17__SHIFT			17
-static inline uint32_t A6XX_RB_2D_BLIT_CNTL_UNK17(uint32_t val)
-{
-	return ((val) << A6XX_RB_2D_BLIT_CNTL_UNK17__SHIFT) & A6XX_RB_2D_BLIT_CNTL_UNK17__MASK;
-}
-#define A6XX_RB_2D_BLIT_CNTL_D24S8				0x00080000
-#define A6XX_RB_2D_BLIT_CNTL_MASK__MASK				0x00f00000
-#define A6XX_RB_2D_BLIT_CNTL_MASK__SHIFT			20
-static inline uint32_t A6XX_RB_2D_BLIT_CNTL_MASK(uint32_t val)
-{
-	return ((val) << A6XX_RB_2D_BLIT_CNTL_MASK__SHIFT) & A6XX_RB_2D_BLIT_CNTL_MASK__MASK;
-}
-#define A6XX_RB_2D_BLIT_CNTL_IFMT__MASK				0x1f000000
-#define A6XX_RB_2D_BLIT_CNTL_IFMT__SHIFT			24
-static inline uint32_t A6XX_RB_2D_BLIT_CNTL_IFMT(enum a6xx_2d_ifmt val)
-{
-	return ((val) << A6XX_RB_2D_BLIT_CNTL_IFMT__SHIFT) & A6XX_RB_2D_BLIT_CNTL_IFMT__MASK;
-}
-#define A6XX_RB_2D_BLIT_CNTL_RASTER_MODE__MASK			0x20000000
-#define A6XX_RB_2D_BLIT_CNTL_RASTER_MODE__SHIFT			29
-static inline uint32_t A6XX_RB_2D_BLIT_CNTL_RASTER_MODE(enum a6xx_raster_mode val)
-{
-	return ((val) << A6XX_RB_2D_BLIT_CNTL_RASTER_MODE__SHIFT) & A6XX_RB_2D_BLIT_CNTL_RASTER_MODE__MASK;
-}
-#define A6XX_RB_2D_BLIT_CNTL_UNK30				0x40000000
-
-#define REG_A6XX_RB_2D_UNKNOWN_8C01				0x00008c01
-
-#define REG_A6XX_RB_2D_DST_INFO					0x00008c17
-#define A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK			0x000000ff
-#define A6XX_RB_2D_DST_INFO_COLOR_FORMAT__SHIFT			0
-static inline uint32_t A6XX_RB_2D_DST_INFO_COLOR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A6XX_RB_2D_DST_INFO_COLOR_FORMAT__SHIFT) & A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK;
-}
-#define A6XX_RB_2D_DST_INFO_TILE_MODE__MASK			0x00000300
-#define A6XX_RB_2D_DST_INFO_TILE_MODE__SHIFT			8
-static inline uint32_t A6XX_RB_2D_DST_INFO_TILE_MODE(enum a6xx_tile_mode val)
-{
-	return ((val) << A6XX_RB_2D_DST_INFO_TILE_MODE__SHIFT) & A6XX_RB_2D_DST_INFO_TILE_MODE__MASK;
-}
-#define A6XX_RB_2D_DST_INFO_COLOR_SWAP__MASK			0x00000c00
-#define A6XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT			10
-static inline uint32_t A6XX_RB_2D_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A6XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT) & A6XX_RB_2D_DST_INFO_COLOR_SWAP__MASK;
-}
-#define A6XX_RB_2D_DST_INFO_FLAGS				0x00001000
-#define A6XX_RB_2D_DST_INFO_SRGB				0x00002000
-#define A6XX_RB_2D_DST_INFO_SAMPLES__MASK			0x0000c000
-#define A6XX_RB_2D_DST_INFO_SAMPLES__SHIFT			14
-static inline uint32_t A6XX_RB_2D_DST_INFO_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A6XX_RB_2D_DST_INFO_SAMPLES__SHIFT) & A6XX_RB_2D_DST_INFO_SAMPLES__MASK;
-}
-#define A6XX_RB_2D_DST_INFO_FILTER				0x00010000
-#define A6XX_RB_2D_DST_INFO_UNK17				0x00020000
-#define A6XX_RB_2D_DST_INFO_SAMPLES_AVERAGE			0x00040000
-#define A6XX_RB_2D_DST_INFO_UNK19				0x00080000
-#define A6XX_RB_2D_DST_INFO_UNK20				0x00100000
-#define A6XX_RB_2D_DST_INFO_UNK21				0x00200000
-#define A6XX_RB_2D_DST_INFO_UNK22				0x00400000
-#define A6XX_RB_2D_DST_INFO_UNK23__MASK				0x07800000
-#define A6XX_RB_2D_DST_INFO_UNK23__SHIFT			23
-static inline uint32_t A6XX_RB_2D_DST_INFO_UNK23(uint32_t val)
-{
-	return ((val) << A6XX_RB_2D_DST_INFO_UNK23__SHIFT) & A6XX_RB_2D_DST_INFO_UNK23__MASK;
-}
-#define A6XX_RB_2D_DST_INFO_UNK28				0x10000000
-
-#define REG_A6XX_RB_2D_DST					0x00008c18
-
-#define REG_A6XX_RB_2D_DST_PITCH				0x00008c1a
-#define A6XX_RB_2D_DST_PITCH__MASK				0x0000ffff
-#define A6XX_RB_2D_DST_PITCH__SHIFT				0
-static inline uint32_t A6XX_RB_2D_DST_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_2D_DST_PITCH__SHIFT) & A6XX_RB_2D_DST_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_2D_DST_PLANE1				0x00008c1b
-
-#define REG_A6XX_RB_2D_DST_PLANE_PITCH				0x00008c1d
-#define A6XX_RB_2D_DST_PLANE_PITCH__MASK			0x0000ffff
-#define A6XX_RB_2D_DST_PLANE_PITCH__SHIFT			0
-static inline uint32_t A6XX_RB_2D_DST_PLANE_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_2D_DST_PLANE_PITCH__SHIFT) & A6XX_RB_2D_DST_PLANE_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_2D_DST_PLANE2				0x00008c1e
-
-#define REG_A6XX_RB_2D_DST_FLAGS				0x00008c20
-
-#define REG_A6XX_RB_2D_DST_FLAGS_PITCH				0x00008c22
-#define A6XX_RB_2D_DST_FLAGS_PITCH__MASK			0x000000ff
-#define A6XX_RB_2D_DST_FLAGS_PITCH__SHIFT			0
-static inline uint32_t A6XX_RB_2D_DST_FLAGS_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_2D_DST_FLAGS_PITCH__SHIFT) & A6XX_RB_2D_DST_FLAGS_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_2D_DST_FLAGS_PLANE				0x00008c23
-
-#define REG_A6XX_RB_2D_DST_FLAGS_PLANE_PITCH			0x00008c25
-#define A6XX_RB_2D_DST_FLAGS_PLANE_PITCH__MASK			0x000000ff
-#define A6XX_RB_2D_DST_FLAGS_PLANE_PITCH__SHIFT			0
-static inline uint32_t A6XX_RB_2D_DST_FLAGS_PLANE_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_RB_2D_DST_FLAGS_PLANE_PITCH__SHIFT) & A6XX_RB_2D_DST_FLAGS_PLANE_PITCH__MASK;
-}
-
-#define REG_A6XX_RB_2D_SRC_SOLID_C0				0x00008c2c
-
-#define REG_A6XX_RB_2D_SRC_SOLID_C1				0x00008c2d
-
-#define REG_A6XX_RB_2D_SRC_SOLID_C2				0x00008c2e
-
-#define REG_A6XX_RB_2D_SRC_SOLID_C3				0x00008c2f
-
-#define REG_A6XX_RB_UNKNOWN_8E01				0x00008e01
-
-#define REG_A6XX_RB_DBG_ECO_CNTL				0x00008e04
-
-#define REG_A6XX_RB_ADDR_MODE_CNTL				0x00008e05
-
-#define REG_A7XX_RB_UNKNOWN_8E06				0x00008e06
-
-#define REG_A6XX_RB_CCU_CNTL					0x00008e07
-#define A6XX_RB_CCU_CNTL_GMEM_FAST_CLEAR_DISABLE		0x00000001
-#define A6XX_RB_CCU_CNTL_CONCURRENT_RESOLVE			0x00000004
-#define A6XX_RB_CCU_CNTL_DEPTH_OFFSET_HI__MASK			0x00000080
-#define A6XX_RB_CCU_CNTL_DEPTH_OFFSET_HI__SHIFT			7
-static inline uint32_t A6XX_RB_CCU_CNTL_DEPTH_OFFSET_HI(uint32_t val)
-{
-	return ((val) << A6XX_RB_CCU_CNTL_DEPTH_OFFSET_HI__SHIFT) & A6XX_RB_CCU_CNTL_DEPTH_OFFSET_HI__MASK;
-}
-#define A6XX_RB_CCU_CNTL_COLOR_OFFSET_HI__MASK			0x00000200
-#define A6XX_RB_CCU_CNTL_COLOR_OFFSET_HI__SHIFT			9
-static inline uint32_t A6XX_RB_CCU_CNTL_COLOR_OFFSET_HI(uint32_t val)
-{
-	return ((val) << A6XX_RB_CCU_CNTL_COLOR_OFFSET_HI__SHIFT) & A6XX_RB_CCU_CNTL_COLOR_OFFSET_HI__MASK;
-}
-#define A6XX_RB_CCU_CNTL_DEPTH_CACHE_SIZE__MASK			0x00000c00
-#define A6XX_RB_CCU_CNTL_DEPTH_CACHE_SIZE__SHIFT		10
-static inline uint32_t A6XX_RB_CCU_CNTL_DEPTH_CACHE_SIZE(enum a6xx_ccu_cache_size val)
-{
-	return ((val) << A6XX_RB_CCU_CNTL_DEPTH_CACHE_SIZE__SHIFT) & A6XX_RB_CCU_CNTL_DEPTH_CACHE_SIZE__MASK;
-}
-#define A6XX_RB_CCU_CNTL_DEPTH_OFFSET__MASK			0x001ff000
-#define A6XX_RB_CCU_CNTL_DEPTH_OFFSET__SHIFT			12
-static inline uint32_t A6XX_RB_CCU_CNTL_DEPTH_OFFSET(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A6XX_RB_CCU_CNTL_DEPTH_OFFSET__SHIFT) & A6XX_RB_CCU_CNTL_DEPTH_OFFSET__MASK;
-}
-#define A6XX_RB_CCU_CNTL_COLOR_CACHE_SIZE__MASK			0x00600000
-#define A6XX_RB_CCU_CNTL_COLOR_CACHE_SIZE__SHIFT		21
-static inline uint32_t A6XX_RB_CCU_CNTL_COLOR_CACHE_SIZE(enum a6xx_ccu_cache_size val)
-{
-	return ((val) << A6XX_RB_CCU_CNTL_COLOR_CACHE_SIZE__SHIFT) & A6XX_RB_CCU_CNTL_COLOR_CACHE_SIZE__MASK;
-}
-#define A6XX_RB_CCU_CNTL_COLOR_OFFSET__MASK			0xff800000
-#define A6XX_RB_CCU_CNTL_COLOR_OFFSET__SHIFT			23
-static inline uint32_t A6XX_RB_CCU_CNTL_COLOR_OFFSET(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A6XX_RB_CCU_CNTL_COLOR_OFFSET__SHIFT) & A6XX_RB_CCU_CNTL_COLOR_OFFSET__MASK;
-}
-
-#define REG_A7XX_RB_CCU_CNTL					0x00008e07
-#define A7XX_RB_CCU_CNTL_GMEM_FAST_CLEAR_DISABLE		0x00000001
-#define A7XX_RB_CCU_CNTL_CONCURRENT_RESOLVE			0x00000004
-
-#define REG_A6XX_RB_NC_MODE_CNTL				0x00008e08
-#define A6XX_RB_NC_MODE_CNTL_MODE				0x00000001
-#define A6XX_RB_NC_MODE_CNTL_LOWER_BIT__MASK			0x00000006
-#define A6XX_RB_NC_MODE_CNTL_LOWER_BIT__SHIFT			1
-static inline uint32_t A6XX_RB_NC_MODE_CNTL_LOWER_BIT(uint32_t val)
-{
-	return ((val) << A6XX_RB_NC_MODE_CNTL_LOWER_BIT__SHIFT) & A6XX_RB_NC_MODE_CNTL_LOWER_BIT__MASK;
-}
-#define A6XX_RB_NC_MODE_CNTL_MIN_ACCESS_LENGTH			0x00000008
-#define A6XX_RB_NC_MODE_CNTL_AMSBC				0x00000010
-#define A6XX_RB_NC_MODE_CNTL_UPPER_BIT__MASK			0x00000400
-#define A6XX_RB_NC_MODE_CNTL_UPPER_BIT__SHIFT			10
-static inline uint32_t A6XX_RB_NC_MODE_CNTL_UPPER_BIT(uint32_t val)
-{
-	return ((val) << A6XX_RB_NC_MODE_CNTL_UPPER_BIT__SHIFT) & A6XX_RB_NC_MODE_CNTL_UPPER_BIT__MASK;
-}
-#define A6XX_RB_NC_MODE_CNTL_RGB565_PREDICATOR			0x00000800
-#define A6XX_RB_NC_MODE_CNTL_UNK12__MASK			0x00003000
-#define A6XX_RB_NC_MODE_CNTL_UNK12__SHIFT			12
-static inline uint32_t A6XX_RB_NC_MODE_CNTL_UNK12(uint32_t val)
-{
-	return ((val) << A6XX_RB_NC_MODE_CNTL_UNK12__SHIFT) & A6XX_RB_NC_MODE_CNTL_UNK12__MASK;
-}
-
-#define REG_A7XX_RB_UNKNOWN_8E09				0x00008e09
-
-#define REG_A6XX_RB_PERFCTR_RB_SEL(i0) (0x00008e10 + 0x1*(i0))
-
-#define REG_A6XX_RB_PERFCTR_CCU_SEL(i0) (0x00008e18 + 0x1*(i0))
-
-#define REG_A6XX_RB_UNKNOWN_8E28				0x00008e28
-
-#define REG_A6XX_RB_PERFCTR_CMP_SEL(i0) (0x00008e2c + 0x1*(i0))
-
-#define REG_A7XX_RB_PERFCTR_UFC_SEL(i0) (0x00008e30 + 0x1*(i0))
-
-#define REG_A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_HOST			0x00008e3b
-
-#define REG_A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_CD			0x00008e3d
-
-#define REG_A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE		0x00008e50
-
-#define REG_A6XX_RB_UNKNOWN_8E51				0x00008e51
-
-#define REG_A7XX_RB_UNKNOWN_8E79				0x00008e79
-
-#define REG_A6XX_VPC_GS_PARAM					0x00009100
-#define A6XX_VPC_GS_PARAM_LINELENGTHLOC__MASK			0x000000ff
-#define A6XX_VPC_GS_PARAM_LINELENGTHLOC__SHIFT			0
-static inline uint32_t A6XX_VPC_GS_PARAM_LINELENGTHLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_PARAM_LINELENGTHLOC__SHIFT) & A6XX_VPC_GS_PARAM_LINELENGTHLOC__MASK;
-}
-
-#define REG_A6XX_VPC_VS_CLIP_CNTL				0x00009101
-#define A6XX_VPC_VS_CLIP_CNTL_CLIP_MASK__MASK			0x000000ff
-#define A6XX_VPC_VS_CLIP_CNTL_CLIP_MASK__SHIFT			0
-static inline uint32_t A6XX_VPC_VS_CLIP_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_CLIP_CNTL_CLIP_MASK__SHIFT) & A6XX_VPC_VS_CLIP_CNTL_CLIP_MASK__MASK;
-}
-#define A6XX_VPC_VS_CLIP_CNTL_CLIP_DIST_03_LOC__MASK		0x0000ff00
-#define A6XX_VPC_VS_CLIP_CNTL_CLIP_DIST_03_LOC__SHIFT		8
-static inline uint32_t A6XX_VPC_VS_CLIP_CNTL_CLIP_DIST_03_LOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_CLIP_CNTL_CLIP_DIST_03_LOC__SHIFT) & A6XX_VPC_VS_CLIP_CNTL_CLIP_DIST_03_LOC__MASK;
-}
-#define A6XX_VPC_VS_CLIP_CNTL_CLIP_DIST_47_LOC__MASK		0x00ff0000
-#define A6XX_VPC_VS_CLIP_CNTL_CLIP_DIST_47_LOC__SHIFT		16
-static inline uint32_t A6XX_VPC_VS_CLIP_CNTL_CLIP_DIST_47_LOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_CLIP_CNTL_CLIP_DIST_47_LOC__SHIFT) & A6XX_VPC_VS_CLIP_CNTL_CLIP_DIST_47_LOC__MASK;
-}
-
-#define REG_A6XX_VPC_GS_CLIP_CNTL				0x00009102
-#define A6XX_VPC_GS_CLIP_CNTL_CLIP_MASK__MASK			0x000000ff
-#define A6XX_VPC_GS_CLIP_CNTL_CLIP_MASK__SHIFT			0
-static inline uint32_t A6XX_VPC_GS_CLIP_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_CLIP_CNTL_CLIP_MASK__SHIFT) & A6XX_VPC_GS_CLIP_CNTL_CLIP_MASK__MASK;
-}
-#define A6XX_VPC_GS_CLIP_CNTL_CLIP_DIST_03_LOC__MASK		0x0000ff00
-#define A6XX_VPC_GS_CLIP_CNTL_CLIP_DIST_03_LOC__SHIFT		8
-static inline uint32_t A6XX_VPC_GS_CLIP_CNTL_CLIP_DIST_03_LOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_CLIP_CNTL_CLIP_DIST_03_LOC__SHIFT) & A6XX_VPC_GS_CLIP_CNTL_CLIP_DIST_03_LOC__MASK;
-}
-#define A6XX_VPC_GS_CLIP_CNTL_CLIP_DIST_47_LOC__MASK		0x00ff0000
-#define A6XX_VPC_GS_CLIP_CNTL_CLIP_DIST_47_LOC__SHIFT		16
-static inline uint32_t A6XX_VPC_GS_CLIP_CNTL_CLIP_DIST_47_LOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_CLIP_CNTL_CLIP_DIST_47_LOC__SHIFT) & A6XX_VPC_GS_CLIP_CNTL_CLIP_DIST_47_LOC__MASK;
-}
-
-#define REG_A6XX_VPC_DS_CLIP_CNTL				0x00009103
-#define A6XX_VPC_DS_CLIP_CNTL_CLIP_MASK__MASK			0x000000ff
-#define A6XX_VPC_DS_CLIP_CNTL_CLIP_MASK__SHIFT			0
-static inline uint32_t A6XX_VPC_DS_CLIP_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_CLIP_CNTL_CLIP_MASK__SHIFT) & A6XX_VPC_DS_CLIP_CNTL_CLIP_MASK__MASK;
-}
-#define A6XX_VPC_DS_CLIP_CNTL_CLIP_DIST_03_LOC__MASK		0x0000ff00
-#define A6XX_VPC_DS_CLIP_CNTL_CLIP_DIST_03_LOC__SHIFT		8
-static inline uint32_t A6XX_VPC_DS_CLIP_CNTL_CLIP_DIST_03_LOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_CLIP_CNTL_CLIP_DIST_03_LOC__SHIFT) & A6XX_VPC_DS_CLIP_CNTL_CLIP_DIST_03_LOC__MASK;
-}
-#define A6XX_VPC_DS_CLIP_CNTL_CLIP_DIST_47_LOC__MASK		0x00ff0000
-#define A6XX_VPC_DS_CLIP_CNTL_CLIP_DIST_47_LOC__SHIFT		16
-static inline uint32_t A6XX_VPC_DS_CLIP_CNTL_CLIP_DIST_47_LOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_CLIP_CNTL_CLIP_DIST_47_LOC__SHIFT) & A6XX_VPC_DS_CLIP_CNTL_CLIP_DIST_47_LOC__MASK;
-}
-
-#define REG_A6XX_VPC_VS_CLIP_CNTL_V2				0x00009311
-#define A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_MASK__MASK		0x000000ff
-#define A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_MASK__SHIFT		0
-static inline uint32_t A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_MASK__SHIFT) & A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_MASK__MASK;
-}
-#define A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_DIST_03_LOC__MASK		0x0000ff00
-#define A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_DIST_03_LOC__SHIFT	8
-static inline uint32_t A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_DIST_03_LOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_DIST_03_LOC__SHIFT) & A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_DIST_03_LOC__MASK;
-}
-#define A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_DIST_47_LOC__MASK		0x00ff0000
-#define A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_DIST_47_LOC__SHIFT	16
-static inline uint32_t A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_DIST_47_LOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_DIST_47_LOC__SHIFT) & A6XX_VPC_VS_CLIP_CNTL_V2_CLIP_DIST_47_LOC__MASK;
-}
-
-#define REG_A6XX_VPC_GS_CLIP_CNTL_V2				0x00009312
-#define A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_MASK__MASK		0x000000ff
-#define A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_MASK__SHIFT		0
-static inline uint32_t A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_MASK__SHIFT) & A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_MASK__MASK;
-}
-#define A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_DIST_03_LOC__MASK		0x0000ff00
-#define A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_DIST_03_LOC__SHIFT	8
-static inline uint32_t A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_DIST_03_LOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_DIST_03_LOC__SHIFT) & A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_DIST_03_LOC__MASK;
-}
-#define A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_DIST_47_LOC__MASK		0x00ff0000
-#define A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_DIST_47_LOC__SHIFT	16
-static inline uint32_t A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_DIST_47_LOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_DIST_47_LOC__SHIFT) & A6XX_VPC_GS_CLIP_CNTL_V2_CLIP_DIST_47_LOC__MASK;
-}
-
-#define REG_A6XX_VPC_DS_CLIP_CNTL_V2				0x00009313
-#define A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_MASK__MASK		0x000000ff
-#define A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_MASK__SHIFT		0
-static inline uint32_t A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_MASK__SHIFT) & A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_MASK__MASK;
-}
-#define A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_DIST_03_LOC__MASK		0x0000ff00
-#define A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_DIST_03_LOC__SHIFT	8
-static inline uint32_t A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_DIST_03_LOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_DIST_03_LOC__SHIFT) & A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_DIST_03_LOC__MASK;
-}
-#define A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_DIST_47_LOC__MASK		0x00ff0000
-#define A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_DIST_47_LOC__SHIFT	16
-static inline uint32_t A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_DIST_47_LOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_DIST_47_LOC__SHIFT) & A6XX_VPC_DS_CLIP_CNTL_V2_CLIP_DIST_47_LOC__MASK;
-}
-
-#define REG_A6XX_VPC_VS_LAYER_CNTL				0x00009104
-#define A6XX_VPC_VS_LAYER_CNTL_LAYERLOC__MASK			0x000000ff
-#define A6XX_VPC_VS_LAYER_CNTL_LAYERLOC__SHIFT			0
-static inline uint32_t A6XX_VPC_VS_LAYER_CNTL_LAYERLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_LAYER_CNTL_LAYERLOC__SHIFT) & A6XX_VPC_VS_LAYER_CNTL_LAYERLOC__MASK;
-}
-#define A6XX_VPC_VS_LAYER_CNTL_VIEWLOC__MASK			0x0000ff00
-#define A6XX_VPC_VS_LAYER_CNTL_VIEWLOC__SHIFT			8
-static inline uint32_t A6XX_VPC_VS_LAYER_CNTL_VIEWLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_LAYER_CNTL_VIEWLOC__SHIFT) & A6XX_VPC_VS_LAYER_CNTL_VIEWLOC__MASK;
-}
-#define A6XX_VPC_VS_LAYER_CNTL_SHADINGRATELOC__MASK		0x00ff0000
-#define A6XX_VPC_VS_LAYER_CNTL_SHADINGRATELOC__SHIFT		16
-static inline uint32_t A6XX_VPC_VS_LAYER_CNTL_SHADINGRATELOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_LAYER_CNTL_SHADINGRATELOC__SHIFT) & A6XX_VPC_VS_LAYER_CNTL_SHADINGRATELOC__MASK;
-}
-
-#define REG_A6XX_VPC_GS_LAYER_CNTL				0x00009105
-#define A6XX_VPC_GS_LAYER_CNTL_LAYERLOC__MASK			0x000000ff
-#define A6XX_VPC_GS_LAYER_CNTL_LAYERLOC__SHIFT			0
-static inline uint32_t A6XX_VPC_GS_LAYER_CNTL_LAYERLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_LAYER_CNTL_LAYERLOC__SHIFT) & A6XX_VPC_GS_LAYER_CNTL_LAYERLOC__MASK;
-}
-#define A6XX_VPC_GS_LAYER_CNTL_VIEWLOC__MASK			0x0000ff00
-#define A6XX_VPC_GS_LAYER_CNTL_VIEWLOC__SHIFT			8
-static inline uint32_t A6XX_VPC_GS_LAYER_CNTL_VIEWLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_LAYER_CNTL_VIEWLOC__SHIFT) & A6XX_VPC_GS_LAYER_CNTL_VIEWLOC__MASK;
-}
-#define A6XX_VPC_GS_LAYER_CNTL_SHADINGRATELOC__MASK		0x00ff0000
-#define A6XX_VPC_GS_LAYER_CNTL_SHADINGRATELOC__SHIFT		16
-static inline uint32_t A6XX_VPC_GS_LAYER_CNTL_SHADINGRATELOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_LAYER_CNTL_SHADINGRATELOC__SHIFT) & A6XX_VPC_GS_LAYER_CNTL_SHADINGRATELOC__MASK;
-}
-
-#define REG_A6XX_VPC_DS_LAYER_CNTL				0x00009106
-#define A6XX_VPC_DS_LAYER_CNTL_LAYERLOC__MASK			0x000000ff
-#define A6XX_VPC_DS_LAYER_CNTL_LAYERLOC__SHIFT			0
-static inline uint32_t A6XX_VPC_DS_LAYER_CNTL_LAYERLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_LAYER_CNTL_LAYERLOC__SHIFT) & A6XX_VPC_DS_LAYER_CNTL_LAYERLOC__MASK;
-}
-#define A6XX_VPC_DS_LAYER_CNTL_VIEWLOC__MASK			0x0000ff00
-#define A6XX_VPC_DS_LAYER_CNTL_VIEWLOC__SHIFT			8
-static inline uint32_t A6XX_VPC_DS_LAYER_CNTL_VIEWLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_LAYER_CNTL_VIEWLOC__SHIFT) & A6XX_VPC_DS_LAYER_CNTL_VIEWLOC__MASK;
-}
-#define A6XX_VPC_DS_LAYER_CNTL_SHADINGRATELOC__MASK		0x00ff0000
-#define A6XX_VPC_DS_LAYER_CNTL_SHADINGRATELOC__SHIFT		16
-static inline uint32_t A6XX_VPC_DS_LAYER_CNTL_SHADINGRATELOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_LAYER_CNTL_SHADINGRATELOC__SHIFT) & A6XX_VPC_DS_LAYER_CNTL_SHADINGRATELOC__MASK;
-}
-
-#define REG_A6XX_VPC_VS_LAYER_CNTL_V2				0x00009314
-#define A6XX_VPC_VS_LAYER_CNTL_V2_LAYERLOC__MASK		0x000000ff
-#define A6XX_VPC_VS_LAYER_CNTL_V2_LAYERLOC__SHIFT		0
-static inline uint32_t A6XX_VPC_VS_LAYER_CNTL_V2_LAYERLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_LAYER_CNTL_V2_LAYERLOC__SHIFT) & A6XX_VPC_VS_LAYER_CNTL_V2_LAYERLOC__MASK;
-}
-#define A6XX_VPC_VS_LAYER_CNTL_V2_VIEWLOC__MASK			0x0000ff00
-#define A6XX_VPC_VS_LAYER_CNTL_V2_VIEWLOC__SHIFT		8
-static inline uint32_t A6XX_VPC_VS_LAYER_CNTL_V2_VIEWLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_LAYER_CNTL_V2_VIEWLOC__SHIFT) & A6XX_VPC_VS_LAYER_CNTL_V2_VIEWLOC__MASK;
-}
-#define A6XX_VPC_VS_LAYER_CNTL_V2_SHADINGRATELOC__MASK		0x00ff0000
-#define A6XX_VPC_VS_LAYER_CNTL_V2_SHADINGRATELOC__SHIFT		16
-static inline uint32_t A6XX_VPC_VS_LAYER_CNTL_V2_SHADINGRATELOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_LAYER_CNTL_V2_SHADINGRATELOC__SHIFT) & A6XX_VPC_VS_LAYER_CNTL_V2_SHADINGRATELOC__MASK;
-}
-
-#define REG_A6XX_VPC_GS_LAYER_CNTL_V2				0x00009315
-#define A6XX_VPC_GS_LAYER_CNTL_V2_LAYERLOC__MASK		0x000000ff
-#define A6XX_VPC_GS_LAYER_CNTL_V2_LAYERLOC__SHIFT		0
-static inline uint32_t A6XX_VPC_GS_LAYER_CNTL_V2_LAYERLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_LAYER_CNTL_V2_LAYERLOC__SHIFT) & A6XX_VPC_GS_LAYER_CNTL_V2_LAYERLOC__MASK;
-}
-#define A6XX_VPC_GS_LAYER_CNTL_V2_VIEWLOC__MASK			0x0000ff00
-#define A6XX_VPC_GS_LAYER_CNTL_V2_VIEWLOC__SHIFT		8
-static inline uint32_t A6XX_VPC_GS_LAYER_CNTL_V2_VIEWLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_LAYER_CNTL_V2_VIEWLOC__SHIFT) & A6XX_VPC_GS_LAYER_CNTL_V2_VIEWLOC__MASK;
-}
-#define A6XX_VPC_GS_LAYER_CNTL_V2_SHADINGRATELOC__MASK		0x00ff0000
-#define A6XX_VPC_GS_LAYER_CNTL_V2_SHADINGRATELOC__SHIFT		16
-static inline uint32_t A6XX_VPC_GS_LAYER_CNTL_V2_SHADINGRATELOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_LAYER_CNTL_V2_SHADINGRATELOC__SHIFT) & A6XX_VPC_GS_LAYER_CNTL_V2_SHADINGRATELOC__MASK;
-}
-
-#define REG_A6XX_VPC_DS_LAYER_CNTL_V2				0x00009316
-#define A6XX_VPC_DS_LAYER_CNTL_V2_LAYERLOC__MASK		0x000000ff
-#define A6XX_VPC_DS_LAYER_CNTL_V2_LAYERLOC__SHIFT		0
-static inline uint32_t A6XX_VPC_DS_LAYER_CNTL_V2_LAYERLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_LAYER_CNTL_V2_LAYERLOC__SHIFT) & A6XX_VPC_DS_LAYER_CNTL_V2_LAYERLOC__MASK;
-}
-#define A6XX_VPC_DS_LAYER_CNTL_V2_VIEWLOC__MASK			0x0000ff00
-#define A6XX_VPC_DS_LAYER_CNTL_V2_VIEWLOC__SHIFT		8
-static inline uint32_t A6XX_VPC_DS_LAYER_CNTL_V2_VIEWLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_LAYER_CNTL_V2_VIEWLOC__SHIFT) & A6XX_VPC_DS_LAYER_CNTL_V2_VIEWLOC__MASK;
-}
-#define A6XX_VPC_DS_LAYER_CNTL_V2_SHADINGRATELOC__MASK		0x00ff0000
-#define A6XX_VPC_DS_LAYER_CNTL_V2_SHADINGRATELOC__SHIFT		16
-static inline uint32_t A6XX_VPC_DS_LAYER_CNTL_V2_SHADINGRATELOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_LAYER_CNTL_V2_SHADINGRATELOC__SHIFT) & A6XX_VPC_DS_LAYER_CNTL_V2_SHADINGRATELOC__MASK;
-}
-
-#define REG_A6XX_VPC_UNKNOWN_9107				0x00009107
-#define A6XX_VPC_UNKNOWN_9107_RASTER_DISCARD			0x00000001
-#define A6XX_VPC_UNKNOWN_9107_UNK2				0x00000004
-
-#define REG_A6XX_VPC_POLYGON_MODE				0x00009108
-#define A6XX_VPC_POLYGON_MODE_MODE__MASK			0x00000003
-#define A6XX_VPC_POLYGON_MODE_MODE__SHIFT			0
-static inline uint32_t A6XX_VPC_POLYGON_MODE_MODE(enum a6xx_polygon_mode val)
-{
-	return ((val) << A6XX_VPC_POLYGON_MODE_MODE__SHIFT) & A6XX_VPC_POLYGON_MODE_MODE__MASK;
-}
-
-#define REG_A7XX_VPC_PRIMITIVE_CNTL_0				0x00009109
-#define A7XX_VPC_PRIMITIVE_CNTL_0_PRIMITIVE_RESTART		0x00000001
-#define A7XX_VPC_PRIMITIVE_CNTL_0_PROVOKING_VTX_LAST		0x00000002
-#define A7XX_VPC_PRIMITIVE_CNTL_0_D3D_VERTEX_ORDERING		0x00000004
-#define A7XX_VPC_PRIMITIVE_CNTL_0_UNK3				0x00000008
-
-#define REG_A7XX_VPC_PRIMITIVE_CNTL_5				0x0000910a
-#define A7XX_VPC_PRIMITIVE_CNTL_5_GS_VERTICES_OUT__MASK		0x000000ff
-#define A7XX_VPC_PRIMITIVE_CNTL_5_GS_VERTICES_OUT__SHIFT	0
-static inline uint32_t A7XX_VPC_PRIMITIVE_CNTL_5_GS_VERTICES_OUT(uint32_t val)
-{
-	return ((val) << A7XX_VPC_PRIMITIVE_CNTL_5_GS_VERTICES_OUT__SHIFT) & A7XX_VPC_PRIMITIVE_CNTL_5_GS_VERTICES_OUT__MASK;
-}
-#define A7XX_VPC_PRIMITIVE_CNTL_5_GS_INVOCATIONS__MASK		0x00007c00
-#define A7XX_VPC_PRIMITIVE_CNTL_5_GS_INVOCATIONS__SHIFT		10
-static inline uint32_t A7XX_VPC_PRIMITIVE_CNTL_5_GS_INVOCATIONS(uint32_t val)
-{
-	return ((val) << A7XX_VPC_PRIMITIVE_CNTL_5_GS_INVOCATIONS__SHIFT) & A7XX_VPC_PRIMITIVE_CNTL_5_GS_INVOCATIONS__MASK;
-}
-#define A7XX_VPC_PRIMITIVE_CNTL_5_LINELENGTHEN			0x00008000
-#define A7XX_VPC_PRIMITIVE_CNTL_5_GS_OUTPUT__MASK		0x00030000
-#define A7XX_VPC_PRIMITIVE_CNTL_5_GS_OUTPUT__SHIFT		16
-static inline uint32_t A7XX_VPC_PRIMITIVE_CNTL_5_GS_OUTPUT(enum a6xx_tess_output val)
-{
-	return ((val) << A7XX_VPC_PRIMITIVE_CNTL_5_GS_OUTPUT__SHIFT) & A7XX_VPC_PRIMITIVE_CNTL_5_GS_OUTPUT__MASK;
-}
-#define A7XX_VPC_PRIMITIVE_CNTL_5_UNK18				0x00040000
-
-#define REG_A7XX_VPC_MULTIVIEW_MASK				0x0000910b
-
-#define REG_A7XX_VPC_MULTIVIEW_CNTL				0x0000910c
-#define A7XX_VPC_MULTIVIEW_CNTL_ENABLE				0x00000001
-#define A7XX_VPC_MULTIVIEW_CNTL_DISABLEMULTIPOS			0x00000002
-#define A7XX_VPC_MULTIVIEW_CNTL_VIEWS__MASK			0x0000007c
-#define A7XX_VPC_MULTIVIEW_CNTL_VIEWS__SHIFT			2
-static inline uint32_t A7XX_VPC_MULTIVIEW_CNTL_VIEWS(uint32_t val)
-{
-	return ((val) << A7XX_VPC_MULTIVIEW_CNTL_VIEWS__SHIFT) & A7XX_VPC_MULTIVIEW_CNTL_VIEWS__MASK;
-}
-
-#define REG_A6XX_VPC_VARYING_INTERP(i0) (0x00009200 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_VPC_VARYING_INTERP_MODE(uint32_t i0) { return 0x00009200 + 0x1*i0; }
-
-#define REG_A6XX_VPC_VARYING_PS_REPL(i0) (0x00009208 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_VPC_VARYING_PS_REPL_MODE(uint32_t i0) { return 0x00009208 + 0x1*i0; }
-
-#define REG_A6XX_VPC_UNKNOWN_9210				0x00009210
-
-#define REG_A6XX_VPC_UNKNOWN_9211				0x00009211
-
-#define REG_A6XX_VPC_VAR(i0) (0x00009212 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_VPC_VAR_DISABLE(uint32_t i0) { return 0x00009212 + 0x1*i0; }
-
-#define REG_A6XX_VPC_SO_CNTL					0x00009216
-#define A6XX_VPC_SO_CNTL_ADDR__MASK				0x000000ff
-#define A6XX_VPC_SO_CNTL_ADDR__SHIFT				0
-static inline uint32_t A6XX_VPC_SO_CNTL_ADDR(uint32_t val)
-{
-	return ((val) << A6XX_VPC_SO_CNTL_ADDR__SHIFT) & A6XX_VPC_SO_CNTL_ADDR__MASK;
-}
-#define A6XX_VPC_SO_CNTL_RESET					0x00010000
-
-#define REG_A6XX_VPC_SO_PROG					0x00009217
-#define A6XX_VPC_SO_PROG_A_BUF__MASK				0x00000003
-#define A6XX_VPC_SO_PROG_A_BUF__SHIFT				0
-static inline uint32_t A6XX_VPC_SO_PROG_A_BUF(uint32_t val)
-{
-	return ((val) << A6XX_VPC_SO_PROG_A_BUF__SHIFT) & A6XX_VPC_SO_PROG_A_BUF__MASK;
-}
-#define A6XX_VPC_SO_PROG_A_OFF__MASK				0x000007fc
-#define A6XX_VPC_SO_PROG_A_OFF__SHIFT				2
-static inline uint32_t A6XX_VPC_SO_PROG_A_OFF(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_VPC_SO_PROG_A_OFF__SHIFT) & A6XX_VPC_SO_PROG_A_OFF__MASK;
-}
-#define A6XX_VPC_SO_PROG_A_EN					0x00000800
-#define A6XX_VPC_SO_PROG_B_BUF__MASK				0x00003000
-#define A6XX_VPC_SO_PROG_B_BUF__SHIFT				12
-static inline uint32_t A6XX_VPC_SO_PROG_B_BUF(uint32_t val)
-{
-	return ((val) << A6XX_VPC_SO_PROG_B_BUF__SHIFT) & A6XX_VPC_SO_PROG_B_BUF__MASK;
-}
-#define A6XX_VPC_SO_PROG_B_OFF__MASK				0x007fc000
-#define A6XX_VPC_SO_PROG_B_OFF__SHIFT				14
-static inline uint32_t A6XX_VPC_SO_PROG_B_OFF(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_VPC_SO_PROG_B_OFF__SHIFT) & A6XX_VPC_SO_PROG_B_OFF__MASK;
-}
-#define A6XX_VPC_SO_PROG_B_EN					0x00800000
-
-#define REG_A6XX_VPC_SO_STREAM_COUNTS				0x00009218
-
-#define REG_A6XX_VPC_SO(i0) (0x0000921a + 0x7*(i0))
-
-static inline uint32_t REG_A6XX_VPC_SO_BUFFER_BASE(uint32_t i0) { return 0x0000921a + 0x7*i0; }
-
-static inline uint32_t REG_A6XX_VPC_SO_BUFFER_SIZE(uint32_t i0) { return 0x0000921c + 0x7*i0; }
-
-static inline uint32_t REG_A6XX_VPC_SO_BUFFER_STRIDE(uint32_t i0) { return 0x0000921d + 0x7*i0; }
-
-static inline uint32_t REG_A6XX_VPC_SO_BUFFER_OFFSET(uint32_t i0) { return 0x0000921e + 0x7*i0; }
-
-static inline uint32_t REG_A6XX_VPC_SO_FLUSH_BASE(uint32_t i0) { return 0x0000921f + 0x7*i0; }
-
-#define REG_A6XX_VPC_POINT_COORD_INVERT				0x00009236
-#define A6XX_VPC_POINT_COORD_INVERT_INVERT			0x00000001
-
-#define REG_A6XX_VPC_UNKNOWN_9300				0x00009300
-
-#define REG_A6XX_VPC_VS_PACK					0x00009301
-#define A6XX_VPC_VS_PACK_STRIDE_IN_VPC__MASK			0x000000ff
-#define A6XX_VPC_VS_PACK_STRIDE_IN_VPC__SHIFT			0
-static inline uint32_t A6XX_VPC_VS_PACK_STRIDE_IN_VPC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_PACK_STRIDE_IN_VPC__SHIFT) & A6XX_VPC_VS_PACK_STRIDE_IN_VPC__MASK;
-}
-#define A6XX_VPC_VS_PACK_POSITIONLOC__MASK			0x0000ff00
-#define A6XX_VPC_VS_PACK_POSITIONLOC__SHIFT			8
-static inline uint32_t A6XX_VPC_VS_PACK_POSITIONLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_PACK_POSITIONLOC__SHIFT) & A6XX_VPC_VS_PACK_POSITIONLOC__MASK;
-}
-#define A6XX_VPC_VS_PACK_PSIZELOC__MASK				0x00ff0000
-#define A6XX_VPC_VS_PACK_PSIZELOC__SHIFT			16
-static inline uint32_t A6XX_VPC_VS_PACK_PSIZELOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_PACK_PSIZELOC__SHIFT) & A6XX_VPC_VS_PACK_PSIZELOC__MASK;
-}
-#define A6XX_VPC_VS_PACK_EXTRAPOS__MASK				0x0f000000
-#define A6XX_VPC_VS_PACK_EXTRAPOS__SHIFT			24
-static inline uint32_t A6XX_VPC_VS_PACK_EXTRAPOS(uint32_t val)
-{
-	return ((val) << A6XX_VPC_VS_PACK_EXTRAPOS__SHIFT) & A6XX_VPC_VS_PACK_EXTRAPOS__MASK;
-}
-
-#define REG_A6XX_VPC_GS_PACK					0x00009302
-#define A6XX_VPC_GS_PACK_STRIDE_IN_VPC__MASK			0x000000ff
-#define A6XX_VPC_GS_PACK_STRIDE_IN_VPC__SHIFT			0
-static inline uint32_t A6XX_VPC_GS_PACK_STRIDE_IN_VPC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_PACK_STRIDE_IN_VPC__SHIFT) & A6XX_VPC_GS_PACK_STRIDE_IN_VPC__MASK;
-}
-#define A6XX_VPC_GS_PACK_POSITIONLOC__MASK			0x0000ff00
-#define A6XX_VPC_GS_PACK_POSITIONLOC__SHIFT			8
-static inline uint32_t A6XX_VPC_GS_PACK_POSITIONLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_PACK_POSITIONLOC__SHIFT) & A6XX_VPC_GS_PACK_POSITIONLOC__MASK;
-}
-#define A6XX_VPC_GS_PACK_PSIZELOC__MASK				0x00ff0000
-#define A6XX_VPC_GS_PACK_PSIZELOC__SHIFT			16
-static inline uint32_t A6XX_VPC_GS_PACK_PSIZELOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_PACK_PSIZELOC__SHIFT) & A6XX_VPC_GS_PACK_PSIZELOC__MASK;
-}
-#define A6XX_VPC_GS_PACK_EXTRAPOS__MASK				0x0f000000
-#define A6XX_VPC_GS_PACK_EXTRAPOS__SHIFT			24
-static inline uint32_t A6XX_VPC_GS_PACK_EXTRAPOS(uint32_t val)
-{
-	return ((val) << A6XX_VPC_GS_PACK_EXTRAPOS__SHIFT) & A6XX_VPC_GS_PACK_EXTRAPOS__MASK;
-}
-
-#define REG_A6XX_VPC_DS_PACK					0x00009303
-#define A6XX_VPC_DS_PACK_STRIDE_IN_VPC__MASK			0x000000ff
-#define A6XX_VPC_DS_PACK_STRIDE_IN_VPC__SHIFT			0
-static inline uint32_t A6XX_VPC_DS_PACK_STRIDE_IN_VPC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_PACK_STRIDE_IN_VPC__SHIFT) & A6XX_VPC_DS_PACK_STRIDE_IN_VPC__MASK;
-}
-#define A6XX_VPC_DS_PACK_POSITIONLOC__MASK			0x0000ff00
-#define A6XX_VPC_DS_PACK_POSITIONLOC__SHIFT			8
-static inline uint32_t A6XX_VPC_DS_PACK_POSITIONLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_PACK_POSITIONLOC__SHIFT) & A6XX_VPC_DS_PACK_POSITIONLOC__MASK;
-}
-#define A6XX_VPC_DS_PACK_PSIZELOC__MASK				0x00ff0000
-#define A6XX_VPC_DS_PACK_PSIZELOC__SHIFT			16
-static inline uint32_t A6XX_VPC_DS_PACK_PSIZELOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_PACK_PSIZELOC__SHIFT) & A6XX_VPC_DS_PACK_PSIZELOC__MASK;
-}
-#define A6XX_VPC_DS_PACK_EXTRAPOS__MASK				0x0f000000
-#define A6XX_VPC_DS_PACK_EXTRAPOS__SHIFT			24
-static inline uint32_t A6XX_VPC_DS_PACK_EXTRAPOS(uint32_t val)
-{
-	return ((val) << A6XX_VPC_DS_PACK_EXTRAPOS__SHIFT) & A6XX_VPC_DS_PACK_EXTRAPOS__MASK;
-}
-
-#define REG_A6XX_VPC_CNTL_0					0x00009304
-#define A6XX_VPC_CNTL_0_NUMNONPOSVAR__MASK			0x000000ff
-#define A6XX_VPC_CNTL_0_NUMNONPOSVAR__SHIFT			0
-static inline uint32_t A6XX_VPC_CNTL_0_NUMNONPOSVAR(uint32_t val)
-{
-	return ((val) << A6XX_VPC_CNTL_0_NUMNONPOSVAR__SHIFT) & A6XX_VPC_CNTL_0_NUMNONPOSVAR__MASK;
-}
-#define A6XX_VPC_CNTL_0_PRIMIDLOC__MASK				0x0000ff00
-#define A6XX_VPC_CNTL_0_PRIMIDLOC__SHIFT			8
-static inline uint32_t A6XX_VPC_CNTL_0_PRIMIDLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_CNTL_0_PRIMIDLOC__SHIFT) & A6XX_VPC_CNTL_0_PRIMIDLOC__MASK;
-}
-#define A6XX_VPC_CNTL_0_VARYING					0x00010000
-#define A6XX_VPC_CNTL_0_VIEWIDLOC__MASK				0xff000000
-#define A6XX_VPC_CNTL_0_VIEWIDLOC__SHIFT			24
-static inline uint32_t A6XX_VPC_CNTL_0_VIEWIDLOC(uint32_t val)
-{
-	return ((val) << A6XX_VPC_CNTL_0_VIEWIDLOC__SHIFT) & A6XX_VPC_CNTL_0_VIEWIDLOC__MASK;
-}
-
-#define REG_A6XX_VPC_SO_STREAM_CNTL				0x00009305
-#define A6XX_VPC_SO_STREAM_CNTL_BUF0_STREAM__MASK		0x00000007
-#define A6XX_VPC_SO_STREAM_CNTL_BUF0_STREAM__SHIFT		0
-static inline uint32_t A6XX_VPC_SO_STREAM_CNTL_BUF0_STREAM(uint32_t val)
-{
-	return ((val) << A6XX_VPC_SO_STREAM_CNTL_BUF0_STREAM__SHIFT) & A6XX_VPC_SO_STREAM_CNTL_BUF0_STREAM__MASK;
-}
-#define A6XX_VPC_SO_STREAM_CNTL_BUF1_STREAM__MASK		0x00000038
-#define A6XX_VPC_SO_STREAM_CNTL_BUF1_STREAM__SHIFT		3
-static inline uint32_t A6XX_VPC_SO_STREAM_CNTL_BUF1_STREAM(uint32_t val)
-{
-	return ((val) << A6XX_VPC_SO_STREAM_CNTL_BUF1_STREAM__SHIFT) & A6XX_VPC_SO_STREAM_CNTL_BUF1_STREAM__MASK;
-}
-#define A6XX_VPC_SO_STREAM_CNTL_BUF2_STREAM__MASK		0x000001c0
-#define A6XX_VPC_SO_STREAM_CNTL_BUF2_STREAM__SHIFT		6
-static inline uint32_t A6XX_VPC_SO_STREAM_CNTL_BUF2_STREAM(uint32_t val)
-{
-	return ((val) << A6XX_VPC_SO_STREAM_CNTL_BUF2_STREAM__SHIFT) & A6XX_VPC_SO_STREAM_CNTL_BUF2_STREAM__MASK;
-}
-#define A6XX_VPC_SO_STREAM_CNTL_BUF3_STREAM__MASK		0x00000e00
-#define A6XX_VPC_SO_STREAM_CNTL_BUF3_STREAM__SHIFT		9
-static inline uint32_t A6XX_VPC_SO_STREAM_CNTL_BUF3_STREAM(uint32_t val)
-{
-	return ((val) << A6XX_VPC_SO_STREAM_CNTL_BUF3_STREAM__SHIFT) & A6XX_VPC_SO_STREAM_CNTL_BUF3_STREAM__MASK;
-}
-#define A6XX_VPC_SO_STREAM_CNTL_STREAM_ENABLE__MASK		0x00078000
-#define A6XX_VPC_SO_STREAM_CNTL_STREAM_ENABLE__SHIFT		15
-static inline uint32_t A6XX_VPC_SO_STREAM_CNTL_STREAM_ENABLE(uint32_t val)
-{
-	return ((val) << A6XX_VPC_SO_STREAM_CNTL_STREAM_ENABLE__SHIFT) & A6XX_VPC_SO_STREAM_CNTL_STREAM_ENABLE__MASK;
-}
-
-#define REG_A6XX_VPC_SO_DISABLE					0x00009306
-#define A6XX_VPC_SO_DISABLE_DISABLE				0x00000001
-
-#define REG_A7XX_VPC_POLYGON_MODE2				0x00009307
-#define A7XX_VPC_POLYGON_MODE2_MODE__MASK			0x00000003
-#define A7XX_VPC_POLYGON_MODE2_MODE__SHIFT			0
-static inline uint32_t A7XX_VPC_POLYGON_MODE2_MODE(enum a6xx_polygon_mode val)
-{
-	return ((val) << A7XX_VPC_POLYGON_MODE2_MODE__SHIFT) & A7XX_VPC_POLYGON_MODE2_MODE__MASK;
-}
-
-#define REG_A7XX_VPC_ATTR_BUF_SIZE_GMEM				0x00009308
-#define A7XX_VPC_ATTR_BUF_SIZE_GMEM_SIZE_GMEM__MASK		0xffffffff
-#define A7XX_VPC_ATTR_BUF_SIZE_GMEM_SIZE_GMEM__SHIFT		0
-static inline uint32_t A7XX_VPC_ATTR_BUF_SIZE_GMEM_SIZE_GMEM(uint32_t val)
-{
-	return ((val) << A7XX_VPC_ATTR_BUF_SIZE_GMEM_SIZE_GMEM__SHIFT) & A7XX_VPC_ATTR_BUF_SIZE_GMEM_SIZE_GMEM__MASK;
-}
-
-#define REG_A7XX_VPC_ATTR_BUF_BASE_GMEM				0x00009309
-#define A7XX_VPC_ATTR_BUF_BASE_GMEM_BASE_GMEM__MASK		0xffffffff
-#define A7XX_VPC_ATTR_BUF_BASE_GMEM_BASE_GMEM__SHIFT		0
-static inline uint32_t A7XX_VPC_ATTR_BUF_BASE_GMEM_BASE_GMEM(uint32_t val)
-{
-	return ((val) << A7XX_VPC_ATTR_BUF_BASE_GMEM_BASE_GMEM__SHIFT) & A7XX_VPC_ATTR_BUF_BASE_GMEM_BASE_GMEM__MASK;
-}
-
-#define REG_A7XX_PC_ATTR_BUF_SIZE_GMEM				0x00009b09
-#define A7XX_PC_ATTR_BUF_SIZE_GMEM_SIZE_GMEM__MASK		0xffffffff
-#define A7XX_PC_ATTR_BUF_SIZE_GMEM_SIZE_GMEM__SHIFT		0
-static inline uint32_t A7XX_PC_ATTR_BUF_SIZE_GMEM_SIZE_GMEM(uint32_t val)
-{
-	return ((val) << A7XX_PC_ATTR_BUF_SIZE_GMEM_SIZE_GMEM__SHIFT) & A7XX_PC_ATTR_BUF_SIZE_GMEM_SIZE_GMEM__MASK;
-}
-
-#define REG_A6XX_VPC_DBG_ECO_CNTL				0x00009600
-
-#define REG_A6XX_VPC_ADDR_MODE_CNTL				0x00009601
-
-#define REG_A6XX_VPC_UNKNOWN_9602				0x00009602
-
-#define REG_A6XX_VPC_UNKNOWN_9603				0x00009603
-
-#define REG_A6XX_VPC_PERFCTR_VPC_SEL(i0) (0x00009604 + 0x1*(i0))
-
-#define REG_A7XX_VPC_PERFCTR_VPC_SEL(i0) (0x0000960b + 0x1*(i0))
-
-#define REG_A6XX_PC_TESS_NUM_VERTEX				0x00009800
-
-#define REG_A6XX_PC_HS_INPUT_SIZE				0x00009801
-#define A6XX_PC_HS_INPUT_SIZE_SIZE__MASK			0x000007ff
-#define A6XX_PC_HS_INPUT_SIZE_SIZE__SHIFT			0
-static inline uint32_t A6XX_PC_HS_INPUT_SIZE_SIZE(uint32_t val)
-{
-	return ((val) << A6XX_PC_HS_INPUT_SIZE_SIZE__SHIFT) & A6XX_PC_HS_INPUT_SIZE_SIZE__MASK;
-}
-#define A6XX_PC_HS_INPUT_SIZE_UNK13				0x00002000
-
-#define REG_A6XX_PC_TESS_CNTL					0x00009802
-#define A6XX_PC_TESS_CNTL_SPACING__MASK				0x00000003
-#define A6XX_PC_TESS_CNTL_SPACING__SHIFT			0
-static inline uint32_t A6XX_PC_TESS_CNTL_SPACING(enum a6xx_tess_spacing val)
-{
-	return ((val) << A6XX_PC_TESS_CNTL_SPACING__SHIFT) & A6XX_PC_TESS_CNTL_SPACING__MASK;
-}
-#define A6XX_PC_TESS_CNTL_OUTPUT__MASK				0x0000000c
-#define A6XX_PC_TESS_CNTL_OUTPUT__SHIFT				2
-static inline uint32_t A6XX_PC_TESS_CNTL_OUTPUT(enum a6xx_tess_output val)
-{
-	return ((val) << A6XX_PC_TESS_CNTL_OUTPUT__SHIFT) & A6XX_PC_TESS_CNTL_OUTPUT__MASK;
-}
-
-#define REG_A6XX_PC_RESTART_INDEX				0x00009803
-
-#define REG_A6XX_PC_MODE_CNTL					0x00009804
-
-#define REG_A6XX_PC_POWER_CNTL					0x00009805
-
-#define REG_A6XX_PC_PS_CNTL					0x00009806
-#define A6XX_PC_PS_CNTL_PRIMITIVEIDEN				0x00000001
-
-#define REG_A6XX_PC_SO_STREAM_CNTL				0x00009808
-#define A6XX_PC_SO_STREAM_CNTL_STREAM_ENABLE__MASK		0x00078000
-#define A6XX_PC_SO_STREAM_CNTL_STREAM_ENABLE__SHIFT		15
-static inline uint32_t A6XX_PC_SO_STREAM_CNTL_STREAM_ENABLE(uint32_t val)
-{
-	return ((val) << A6XX_PC_SO_STREAM_CNTL_STREAM_ENABLE__SHIFT) & A6XX_PC_SO_STREAM_CNTL_STREAM_ENABLE__MASK;
-}
-
-#define REG_A6XX_PC_DGEN_SU_CONSERVATIVE_RAS_CNTL		0x0000980a
-#define A6XX_PC_DGEN_SU_CONSERVATIVE_RAS_CNTL_CONSERVATIVERASEN	0x00000001
-
-#define REG_A6XX_PC_DRAW_CMD					0x00009840
-#define A6XX_PC_DRAW_CMD_STATE_ID__MASK				0x000000ff
-#define A6XX_PC_DRAW_CMD_STATE_ID__SHIFT			0
-static inline uint32_t A6XX_PC_DRAW_CMD_STATE_ID(uint32_t val)
-{
-	return ((val) << A6XX_PC_DRAW_CMD_STATE_ID__SHIFT) & A6XX_PC_DRAW_CMD_STATE_ID__MASK;
-}
-
-#define REG_A6XX_PC_DISPATCH_CMD				0x00009841
-#define A6XX_PC_DISPATCH_CMD_STATE_ID__MASK			0x000000ff
-#define A6XX_PC_DISPATCH_CMD_STATE_ID__SHIFT			0
-static inline uint32_t A6XX_PC_DISPATCH_CMD_STATE_ID(uint32_t val)
-{
-	return ((val) << A6XX_PC_DISPATCH_CMD_STATE_ID__SHIFT) & A6XX_PC_DISPATCH_CMD_STATE_ID__MASK;
-}
-
-#define REG_A6XX_PC_EVENT_CMD					0x00009842
-#define A6XX_PC_EVENT_CMD_STATE_ID__MASK			0x00ff0000
-#define A6XX_PC_EVENT_CMD_STATE_ID__SHIFT			16
-static inline uint32_t A6XX_PC_EVENT_CMD_STATE_ID(uint32_t val)
-{
-	return ((val) << A6XX_PC_EVENT_CMD_STATE_ID__SHIFT) & A6XX_PC_EVENT_CMD_STATE_ID__MASK;
-}
-#define A6XX_PC_EVENT_CMD_EVENT__MASK				0x0000007f
-#define A6XX_PC_EVENT_CMD_EVENT__SHIFT				0
-static inline uint32_t A6XX_PC_EVENT_CMD_EVENT(enum vgt_event_type val)
-{
-	return ((val) << A6XX_PC_EVENT_CMD_EVENT__SHIFT) & A6XX_PC_EVENT_CMD_EVENT__MASK;
-}
-
-#define REG_A6XX_PC_MARKER					0x00009880
-
-#define REG_A6XX_PC_POLYGON_MODE				0x00009981
-#define A6XX_PC_POLYGON_MODE_MODE__MASK				0x00000003
-#define A6XX_PC_POLYGON_MODE_MODE__SHIFT			0
-static inline uint32_t A6XX_PC_POLYGON_MODE_MODE(enum a6xx_polygon_mode val)
-{
-	return ((val) << A6XX_PC_POLYGON_MODE_MODE__SHIFT) & A6XX_PC_POLYGON_MODE_MODE__MASK;
-}
-
-#define REG_A7XX_PC_POLYGON_MODE				0x00009809
-#define A7XX_PC_POLYGON_MODE_MODE__MASK				0x00000003
-#define A7XX_PC_POLYGON_MODE_MODE__SHIFT			0
-static inline uint32_t A7XX_PC_POLYGON_MODE_MODE(enum a6xx_polygon_mode val)
-{
-	return ((val) << A7XX_PC_POLYGON_MODE_MODE__SHIFT) & A7XX_PC_POLYGON_MODE_MODE__MASK;
-}
-
-#define REG_A6XX_PC_RASTER_CNTL					0x00009980
-#define A6XX_PC_RASTER_CNTL_STREAM__MASK			0x00000003
-#define A6XX_PC_RASTER_CNTL_STREAM__SHIFT			0
-static inline uint32_t A6XX_PC_RASTER_CNTL_STREAM(uint32_t val)
-{
-	return ((val) << A6XX_PC_RASTER_CNTL_STREAM__SHIFT) & A6XX_PC_RASTER_CNTL_STREAM__MASK;
-}
-#define A6XX_PC_RASTER_CNTL_DISCARD				0x00000004
-
-#define REG_A7XX_PC_RASTER_CNTL					0x00009107
-#define A7XX_PC_RASTER_CNTL_STREAM__MASK			0x00000003
-#define A7XX_PC_RASTER_CNTL_STREAM__SHIFT			0
-static inline uint32_t A7XX_PC_RASTER_CNTL_STREAM(uint32_t val)
-{
-	return ((val) << A7XX_PC_RASTER_CNTL_STREAM__SHIFT) & A7XX_PC_RASTER_CNTL_STREAM__MASK;
-}
-#define A7XX_PC_RASTER_CNTL_DISCARD				0x00000004
-
-#define REG_A7XX_PC_RASTER_CNTL_V2				0x00009317
-#define A7XX_PC_RASTER_CNTL_V2_STREAM__MASK			0x00000003
-#define A7XX_PC_RASTER_CNTL_V2_STREAM__SHIFT			0
-static inline uint32_t A7XX_PC_RASTER_CNTL_V2_STREAM(uint32_t val)
-{
-	return ((val) << A7XX_PC_RASTER_CNTL_V2_STREAM__SHIFT) & A7XX_PC_RASTER_CNTL_V2_STREAM__MASK;
-}
-#define A7XX_PC_RASTER_CNTL_V2_DISCARD				0x00000004
-
-#define REG_A6XX_PC_PRIMITIVE_CNTL_0				0x00009b00
-#define A6XX_PC_PRIMITIVE_CNTL_0_PRIMITIVE_RESTART		0x00000001
-#define A6XX_PC_PRIMITIVE_CNTL_0_PROVOKING_VTX_LAST		0x00000002
-#define A6XX_PC_PRIMITIVE_CNTL_0_D3D_VERTEX_ORDERING		0x00000004
-#define A6XX_PC_PRIMITIVE_CNTL_0_UNK3				0x00000008
-
-#define REG_A6XX_PC_VS_OUT_CNTL					0x00009b01
-#define A6XX_PC_VS_OUT_CNTL_STRIDE_IN_VPC__MASK			0x000000ff
-#define A6XX_PC_VS_OUT_CNTL_STRIDE_IN_VPC__SHIFT		0
-static inline uint32_t A6XX_PC_VS_OUT_CNTL_STRIDE_IN_VPC(uint32_t val)
-{
-	return ((val) << A6XX_PC_VS_OUT_CNTL_STRIDE_IN_VPC__SHIFT) & A6XX_PC_VS_OUT_CNTL_STRIDE_IN_VPC__MASK;
-}
-#define A6XX_PC_VS_OUT_CNTL_PSIZE				0x00000100
-#define A6XX_PC_VS_OUT_CNTL_LAYER				0x00000200
-#define A6XX_PC_VS_OUT_CNTL_VIEW				0x00000400
-#define A6XX_PC_VS_OUT_CNTL_PRIMITIVE_ID			0x00000800
-#define A6XX_PC_VS_OUT_CNTL_CLIP_MASK__MASK			0x00ff0000
-#define A6XX_PC_VS_OUT_CNTL_CLIP_MASK__SHIFT			16
-static inline uint32_t A6XX_PC_VS_OUT_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_PC_VS_OUT_CNTL_CLIP_MASK__SHIFT) & A6XX_PC_VS_OUT_CNTL_CLIP_MASK__MASK;
-}
-#define A6XX_PC_VS_OUT_CNTL_SHADINGRATE				0x01000000
-
-#define REG_A6XX_PC_GS_OUT_CNTL					0x00009b02
-#define A6XX_PC_GS_OUT_CNTL_STRIDE_IN_VPC__MASK			0x000000ff
-#define A6XX_PC_GS_OUT_CNTL_STRIDE_IN_VPC__SHIFT		0
-static inline uint32_t A6XX_PC_GS_OUT_CNTL_STRIDE_IN_VPC(uint32_t val)
-{
-	return ((val) << A6XX_PC_GS_OUT_CNTL_STRIDE_IN_VPC__SHIFT) & A6XX_PC_GS_OUT_CNTL_STRIDE_IN_VPC__MASK;
-}
-#define A6XX_PC_GS_OUT_CNTL_PSIZE				0x00000100
-#define A6XX_PC_GS_OUT_CNTL_LAYER				0x00000200
-#define A6XX_PC_GS_OUT_CNTL_VIEW				0x00000400
-#define A6XX_PC_GS_OUT_CNTL_PRIMITIVE_ID			0x00000800
-#define A6XX_PC_GS_OUT_CNTL_CLIP_MASK__MASK			0x00ff0000
-#define A6XX_PC_GS_OUT_CNTL_CLIP_MASK__SHIFT			16
-static inline uint32_t A6XX_PC_GS_OUT_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_PC_GS_OUT_CNTL_CLIP_MASK__SHIFT) & A6XX_PC_GS_OUT_CNTL_CLIP_MASK__MASK;
-}
-#define A6XX_PC_GS_OUT_CNTL_SHADINGRATE				0x01000000
-
-#define REG_A6XX_PC_HS_OUT_CNTL					0x00009b03
-#define A6XX_PC_HS_OUT_CNTL_STRIDE_IN_VPC__MASK			0x000000ff
-#define A6XX_PC_HS_OUT_CNTL_STRIDE_IN_VPC__SHIFT		0
-static inline uint32_t A6XX_PC_HS_OUT_CNTL_STRIDE_IN_VPC(uint32_t val)
-{
-	return ((val) << A6XX_PC_HS_OUT_CNTL_STRIDE_IN_VPC__SHIFT) & A6XX_PC_HS_OUT_CNTL_STRIDE_IN_VPC__MASK;
-}
-#define A6XX_PC_HS_OUT_CNTL_PSIZE				0x00000100
-#define A6XX_PC_HS_OUT_CNTL_LAYER				0x00000200
-#define A6XX_PC_HS_OUT_CNTL_VIEW				0x00000400
-#define A6XX_PC_HS_OUT_CNTL_PRIMITIVE_ID			0x00000800
-#define A6XX_PC_HS_OUT_CNTL_CLIP_MASK__MASK			0x00ff0000
-#define A6XX_PC_HS_OUT_CNTL_CLIP_MASK__SHIFT			16
-static inline uint32_t A6XX_PC_HS_OUT_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_PC_HS_OUT_CNTL_CLIP_MASK__SHIFT) & A6XX_PC_HS_OUT_CNTL_CLIP_MASK__MASK;
-}
-#define A6XX_PC_HS_OUT_CNTL_SHADINGRATE				0x01000000
-
-#define REG_A6XX_PC_DS_OUT_CNTL					0x00009b04
-#define A6XX_PC_DS_OUT_CNTL_STRIDE_IN_VPC__MASK			0x000000ff
-#define A6XX_PC_DS_OUT_CNTL_STRIDE_IN_VPC__SHIFT		0
-static inline uint32_t A6XX_PC_DS_OUT_CNTL_STRIDE_IN_VPC(uint32_t val)
-{
-	return ((val) << A6XX_PC_DS_OUT_CNTL_STRIDE_IN_VPC__SHIFT) & A6XX_PC_DS_OUT_CNTL_STRIDE_IN_VPC__MASK;
-}
-#define A6XX_PC_DS_OUT_CNTL_PSIZE				0x00000100
-#define A6XX_PC_DS_OUT_CNTL_LAYER				0x00000200
-#define A6XX_PC_DS_OUT_CNTL_VIEW				0x00000400
-#define A6XX_PC_DS_OUT_CNTL_PRIMITIVE_ID			0x00000800
-#define A6XX_PC_DS_OUT_CNTL_CLIP_MASK__MASK			0x00ff0000
-#define A6XX_PC_DS_OUT_CNTL_CLIP_MASK__SHIFT			16
-static inline uint32_t A6XX_PC_DS_OUT_CNTL_CLIP_MASK(uint32_t val)
-{
-	return ((val) << A6XX_PC_DS_OUT_CNTL_CLIP_MASK__SHIFT) & A6XX_PC_DS_OUT_CNTL_CLIP_MASK__MASK;
-}
-#define A6XX_PC_DS_OUT_CNTL_SHADINGRATE				0x01000000
-
-#define REG_A6XX_PC_PRIMITIVE_CNTL_5				0x00009b05
-#define A6XX_PC_PRIMITIVE_CNTL_5_GS_VERTICES_OUT__MASK		0x000000ff
-#define A6XX_PC_PRIMITIVE_CNTL_5_GS_VERTICES_OUT__SHIFT		0
-static inline uint32_t A6XX_PC_PRIMITIVE_CNTL_5_GS_VERTICES_OUT(uint32_t val)
-{
-	return ((val) << A6XX_PC_PRIMITIVE_CNTL_5_GS_VERTICES_OUT__SHIFT) & A6XX_PC_PRIMITIVE_CNTL_5_GS_VERTICES_OUT__MASK;
-}
-#define A6XX_PC_PRIMITIVE_CNTL_5_GS_INVOCATIONS__MASK		0x00007c00
-#define A6XX_PC_PRIMITIVE_CNTL_5_GS_INVOCATIONS__SHIFT		10
-static inline uint32_t A6XX_PC_PRIMITIVE_CNTL_5_GS_INVOCATIONS(uint32_t val)
-{
-	return ((val) << A6XX_PC_PRIMITIVE_CNTL_5_GS_INVOCATIONS__SHIFT) & A6XX_PC_PRIMITIVE_CNTL_5_GS_INVOCATIONS__MASK;
-}
-#define A6XX_PC_PRIMITIVE_CNTL_5_LINELENGTHEN			0x00008000
-#define A6XX_PC_PRIMITIVE_CNTL_5_GS_OUTPUT__MASK		0x00030000
-#define A6XX_PC_PRIMITIVE_CNTL_5_GS_OUTPUT__SHIFT		16
-static inline uint32_t A6XX_PC_PRIMITIVE_CNTL_5_GS_OUTPUT(enum a6xx_tess_output val)
-{
-	return ((val) << A6XX_PC_PRIMITIVE_CNTL_5_GS_OUTPUT__SHIFT) & A6XX_PC_PRIMITIVE_CNTL_5_GS_OUTPUT__MASK;
-}
-#define A6XX_PC_PRIMITIVE_CNTL_5_UNK18				0x00040000
-
-#define REG_A6XX_PC_PRIMITIVE_CNTL_6				0x00009b06
-#define A6XX_PC_PRIMITIVE_CNTL_6_STRIDE_IN_VPC__MASK		0x000007ff
-#define A6XX_PC_PRIMITIVE_CNTL_6_STRIDE_IN_VPC__SHIFT		0
-static inline uint32_t A6XX_PC_PRIMITIVE_CNTL_6_STRIDE_IN_VPC(uint32_t val)
-{
-	return ((val) << A6XX_PC_PRIMITIVE_CNTL_6_STRIDE_IN_VPC__SHIFT) & A6XX_PC_PRIMITIVE_CNTL_6_STRIDE_IN_VPC__MASK;
-}
-
-#define REG_A6XX_PC_MULTIVIEW_CNTL				0x00009b07
-#define A6XX_PC_MULTIVIEW_CNTL_ENABLE				0x00000001
-#define A6XX_PC_MULTIVIEW_CNTL_DISABLEMULTIPOS			0x00000002
-#define A6XX_PC_MULTIVIEW_CNTL_VIEWS__MASK			0x0000007c
-#define A6XX_PC_MULTIVIEW_CNTL_VIEWS__SHIFT			2
-static inline uint32_t A6XX_PC_MULTIVIEW_CNTL_VIEWS(uint32_t val)
-{
-	return ((val) << A6XX_PC_MULTIVIEW_CNTL_VIEWS__SHIFT) & A6XX_PC_MULTIVIEW_CNTL_VIEWS__MASK;
-}
-
-#define REG_A6XX_PC_MULTIVIEW_MASK				0x00009b08
-
-#define REG_A6XX_PC_2D_EVENT_CMD				0x00009c00
-#define A6XX_PC_2D_EVENT_CMD_EVENT__MASK			0x0000007f
-#define A6XX_PC_2D_EVENT_CMD_EVENT__SHIFT			0
-static inline uint32_t A6XX_PC_2D_EVENT_CMD_EVENT(enum vgt_event_type val)
-{
-	return ((val) << A6XX_PC_2D_EVENT_CMD_EVENT__SHIFT) & A6XX_PC_2D_EVENT_CMD_EVENT__MASK;
-}
-#define A6XX_PC_2D_EVENT_CMD_STATE_ID__MASK			0x0000ff00
-#define A6XX_PC_2D_EVENT_CMD_STATE_ID__SHIFT			8
-static inline uint32_t A6XX_PC_2D_EVENT_CMD_STATE_ID(uint32_t val)
-{
-	return ((val) << A6XX_PC_2D_EVENT_CMD_STATE_ID__SHIFT) & A6XX_PC_2D_EVENT_CMD_STATE_ID__MASK;
-}
-
-#define REG_A6XX_PC_DBG_ECO_CNTL				0x00009e00
-
-#define REG_A6XX_PC_ADDR_MODE_CNTL				0x00009e01
-
-#define REG_A6XX_PC_DRAW_INDX_BASE				0x00009e04
-
-#define REG_A6XX_PC_DRAW_FIRST_INDX				0x00009e06
-
-#define REG_A6XX_PC_DRAW_MAX_INDICES				0x00009e07
-
-#define REG_A6XX_PC_TESSFACTOR_ADDR				0x00009e08
-
-#define REG_A7XX_PC_TESSFACTOR_ADDR				0x00009810
-
-#define REG_A6XX_PC_DRAW_INITIATOR				0x00009e0b
-#define A6XX_PC_DRAW_INITIATOR_PRIM_TYPE__MASK			0x0000003f
-#define A6XX_PC_DRAW_INITIATOR_PRIM_TYPE__SHIFT			0
-static inline uint32_t A6XX_PC_DRAW_INITIATOR_PRIM_TYPE(enum pc_di_primtype val)
-{
-	return ((val) << A6XX_PC_DRAW_INITIATOR_PRIM_TYPE__SHIFT) & A6XX_PC_DRAW_INITIATOR_PRIM_TYPE__MASK;
-}
-#define A6XX_PC_DRAW_INITIATOR_SOURCE_SELECT__MASK		0x000000c0
-#define A6XX_PC_DRAW_INITIATOR_SOURCE_SELECT__SHIFT		6
-static inline uint32_t A6XX_PC_DRAW_INITIATOR_SOURCE_SELECT(enum pc_di_src_sel val)
-{
-	return ((val) << A6XX_PC_DRAW_INITIATOR_SOURCE_SELECT__SHIFT) & A6XX_PC_DRAW_INITIATOR_SOURCE_SELECT__MASK;
-}
-#define A6XX_PC_DRAW_INITIATOR_VIS_CULL__MASK			0x00000300
-#define A6XX_PC_DRAW_INITIATOR_VIS_CULL__SHIFT			8
-static inline uint32_t A6XX_PC_DRAW_INITIATOR_VIS_CULL(enum pc_di_vis_cull_mode val)
-{
-	return ((val) << A6XX_PC_DRAW_INITIATOR_VIS_CULL__SHIFT) & A6XX_PC_DRAW_INITIATOR_VIS_CULL__MASK;
-}
-#define A6XX_PC_DRAW_INITIATOR_INDEX_SIZE__MASK			0x00000c00
-#define A6XX_PC_DRAW_INITIATOR_INDEX_SIZE__SHIFT		10
-static inline uint32_t A6XX_PC_DRAW_INITIATOR_INDEX_SIZE(enum a4xx_index_size val)
-{
-	return ((val) << A6XX_PC_DRAW_INITIATOR_INDEX_SIZE__SHIFT) & A6XX_PC_DRAW_INITIATOR_INDEX_SIZE__MASK;
-}
-#define A6XX_PC_DRAW_INITIATOR_PATCH_TYPE__MASK			0x00003000
-#define A6XX_PC_DRAW_INITIATOR_PATCH_TYPE__SHIFT		12
-static inline uint32_t A6XX_PC_DRAW_INITIATOR_PATCH_TYPE(enum a6xx_patch_type val)
-{
-	return ((val) << A6XX_PC_DRAW_INITIATOR_PATCH_TYPE__SHIFT) & A6XX_PC_DRAW_INITIATOR_PATCH_TYPE__MASK;
-}
-#define A6XX_PC_DRAW_INITIATOR_GS_ENABLE			0x00010000
-#define A6XX_PC_DRAW_INITIATOR_TESS_ENABLE			0x00020000
-
-#define REG_A6XX_PC_DRAW_NUM_INSTANCES				0x00009e0c
-
-#define REG_A6XX_PC_DRAW_NUM_INDICES				0x00009e0d
-
-#define REG_A6XX_PC_VSTREAM_CONTROL				0x00009e11
-#define A6XX_PC_VSTREAM_CONTROL_UNK0__MASK			0x0000ffff
-#define A6XX_PC_VSTREAM_CONTROL_UNK0__SHIFT			0
-static inline uint32_t A6XX_PC_VSTREAM_CONTROL_UNK0(uint32_t val)
-{
-	return ((val) << A6XX_PC_VSTREAM_CONTROL_UNK0__SHIFT) & A6XX_PC_VSTREAM_CONTROL_UNK0__MASK;
-}
-#define A6XX_PC_VSTREAM_CONTROL_VSC_SIZE__MASK			0x003f0000
-#define A6XX_PC_VSTREAM_CONTROL_VSC_SIZE__SHIFT			16
-static inline uint32_t A6XX_PC_VSTREAM_CONTROL_VSC_SIZE(uint32_t val)
-{
-	return ((val) << A6XX_PC_VSTREAM_CONTROL_VSC_SIZE__SHIFT) & A6XX_PC_VSTREAM_CONTROL_VSC_SIZE__MASK;
-}
-#define A6XX_PC_VSTREAM_CONTROL_VSC_N__MASK			0x07c00000
-#define A6XX_PC_VSTREAM_CONTROL_VSC_N__SHIFT			22
-static inline uint32_t A6XX_PC_VSTREAM_CONTROL_VSC_N(uint32_t val)
-{
-	return ((val) << A6XX_PC_VSTREAM_CONTROL_VSC_N__SHIFT) & A6XX_PC_VSTREAM_CONTROL_VSC_N__MASK;
-}
-
-#define REG_A6XX_PC_BIN_PRIM_STRM				0x00009e12
-
-#define REG_A6XX_PC_BIN_DRAW_STRM				0x00009e14
-
-#define REG_A6XX_PC_VISIBILITY_OVERRIDE				0x00009e1c
-#define A6XX_PC_VISIBILITY_OVERRIDE_OVERRIDE			0x00000001
-
-#define REG_A7XX_PC_UNKNOWN_9E24				0x00009e24
-
-#define REG_A6XX_PC_PERFCTR_PC_SEL(i0) (0x00009e34 + 0x1*(i0))
-
-#define REG_A7XX_PC_PERFCTR_PC_SEL(i0) (0x00009e42 + 0x1*(i0))
-
-#define REG_A6XX_PC_UNKNOWN_9E72				0x00009e72
-
-#define REG_A6XX_VFD_CONTROL_0					0x0000a000
-#define A6XX_VFD_CONTROL_0_FETCH_CNT__MASK			0x0000003f
-#define A6XX_VFD_CONTROL_0_FETCH_CNT__SHIFT			0
-static inline uint32_t A6XX_VFD_CONTROL_0_FETCH_CNT(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_0_FETCH_CNT__SHIFT) & A6XX_VFD_CONTROL_0_FETCH_CNT__MASK;
-}
-#define A6XX_VFD_CONTROL_0_DECODE_CNT__MASK			0x00003f00
-#define A6XX_VFD_CONTROL_0_DECODE_CNT__SHIFT			8
-static inline uint32_t A6XX_VFD_CONTROL_0_DECODE_CNT(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_0_DECODE_CNT__SHIFT) & A6XX_VFD_CONTROL_0_DECODE_CNT__MASK;
-}
-
-#define REG_A6XX_VFD_CONTROL_1					0x0000a001
-#define A6XX_VFD_CONTROL_1_REGID4VTX__MASK			0x000000ff
-#define A6XX_VFD_CONTROL_1_REGID4VTX__SHIFT			0
-static inline uint32_t A6XX_VFD_CONTROL_1_REGID4VTX(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_1_REGID4VTX__SHIFT) & A6XX_VFD_CONTROL_1_REGID4VTX__MASK;
-}
-#define A6XX_VFD_CONTROL_1_REGID4INST__MASK			0x0000ff00
-#define A6XX_VFD_CONTROL_1_REGID4INST__SHIFT			8
-static inline uint32_t A6XX_VFD_CONTROL_1_REGID4INST(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_1_REGID4INST__SHIFT) & A6XX_VFD_CONTROL_1_REGID4INST__MASK;
-}
-#define A6XX_VFD_CONTROL_1_REGID4PRIMID__MASK			0x00ff0000
-#define A6XX_VFD_CONTROL_1_REGID4PRIMID__SHIFT			16
-static inline uint32_t A6XX_VFD_CONTROL_1_REGID4PRIMID(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_1_REGID4PRIMID__SHIFT) & A6XX_VFD_CONTROL_1_REGID4PRIMID__MASK;
-}
-#define A6XX_VFD_CONTROL_1_REGID4VIEWID__MASK			0xff000000
-#define A6XX_VFD_CONTROL_1_REGID4VIEWID__SHIFT			24
-static inline uint32_t A6XX_VFD_CONTROL_1_REGID4VIEWID(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_1_REGID4VIEWID__SHIFT) & A6XX_VFD_CONTROL_1_REGID4VIEWID__MASK;
-}
-
-#define REG_A6XX_VFD_CONTROL_2					0x0000a002
-#define A6XX_VFD_CONTROL_2_REGID_HSRELPATCHID__MASK		0x000000ff
-#define A6XX_VFD_CONTROL_2_REGID_HSRELPATCHID__SHIFT		0
-static inline uint32_t A6XX_VFD_CONTROL_2_REGID_HSRELPATCHID(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_2_REGID_HSRELPATCHID__SHIFT) & A6XX_VFD_CONTROL_2_REGID_HSRELPATCHID__MASK;
-}
-#define A6XX_VFD_CONTROL_2_REGID_INVOCATIONID__MASK		0x0000ff00
-#define A6XX_VFD_CONTROL_2_REGID_INVOCATIONID__SHIFT		8
-static inline uint32_t A6XX_VFD_CONTROL_2_REGID_INVOCATIONID(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_2_REGID_INVOCATIONID__SHIFT) & A6XX_VFD_CONTROL_2_REGID_INVOCATIONID__MASK;
-}
-
-#define REG_A6XX_VFD_CONTROL_3					0x0000a003
-#define A6XX_VFD_CONTROL_3_REGID_DSPRIMID__MASK			0x000000ff
-#define A6XX_VFD_CONTROL_3_REGID_DSPRIMID__SHIFT		0
-static inline uint32_t A6XX_VFD_CONTROL_3_REGID_DSPRIMID(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_3_REGID_DSPRIMID__SHIFT) & A6XX_VFD_CONTROL_3_REGID_DSPRIMID__MASK;
-}
-#define A6XX_VFD_CONTROL_3_REGID_DSRELPATCHID__MASK		0x0000ff00
-#define A6XX_VFD_CONTROL_3_REGID_DSRELPATCHID__SHIFT		8
-static inline uint32_t A6XX_VFD_CONTROL_3_REGID_DSRELPATCHID(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_3_REGID_DSRELPATCHID__SHIFT) & A6XX_VFD_CONTROL_3_REGID_DSRELPATCHID__MASK;
-}
-#define A6XX_VFD_CONTROL_3_REGID_TESSX__MASK			0x00ff0000
-#define A6XX_VFD_CONTROL_3_REGID_TESSX__SHIFT			16
-static inline uint32_t A6XX_VFD_CONTROL_3_REGID_TESSX(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_3_REGID_TESSX__SHIFT) & A6XX_VFD_CONTROL_3_REGID_TESSX__MASK;
-}
-#define A6XX_VFD_CONTROL_3_REGID_TESSY__MASK			0xff000000
-#define A6XX_VFD_CONTROL_3_REGID_TESSY__SHIFT			24
-static inline uint32_t A6XX_VFD_CONTROL_3_REGID_TESSY(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_3_REGID_TESSY__SHIFT) & A6XX_VFD_CONTROL_3_REGID_TESSY__MASK;
-}
-
-#define REG_A6XX_VFD_CONTROL_4					0x0000a004
-#define A6XX_VFD_CONTROL_4_UNK0__MASK				0x000000ff
-#define A6XX_VFD_CONTROL_4_UNK0__SHIFT				0
-static inline uint32_t A6XX_VFD_CONTROL_4_UNK0(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_4_UNK0__SHIFT) & A6XX_VFD_CONTROL_4_UNK0__MASK;
-}
-
-#define REG_A6XX_VFD_CONTROL_5					0x0000a005
-#define A6XX_VFD_CONTROL_5_REGID_GSHEADER__MASK			0x000000ff
-#define A6XX_VFD_CONTROL_5_REGID_GSHEADER__SHIFT		0
-static inline uint32_t A6XX_VFD_CONTROL_5_REGID_GSHEADER(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_5_REGID_GSHEADER__SHIFT) & A6XX_VFD_CONTROL_5_REGID_GSHEADER__MASK;
-}
-#define A6XX_VFD_CONTROL_5_UNK8__MASK				0x0000ff00
-#define A6XX_VFD_CONTROL_5_UNK8__SHIFT				8
-static inline uint32_t A6XX_VFD_CONTROL_5_UNK8(uint32_t val)
-{
-	return ((val) << A6XX_VFD_CONTROL_5_UNK8__SHIFT) & A6XX_VFD_CONTROL_5_UNK8__MASK;
-}
-
-#define REG_A6XX_VFD_CONTROL_6					0x0000a006
-#define A6XX_VFD_CONTROL_6_PRIMID4PSEN				0x00000001
-
-#define REG_A6XX_VFD_MODE_CNTL					0x0000a007
-#define A6XX_VFD_MODE_CNTL_RENDER_MODE__MASK			0x00000007
-#define A6XX_VFD_MODE_CNTL_RENDER_MODE__SHIFT			0
-static inline uint32_t A6XX_VFD_MODE_CNTL_RENDER_MODE(enum a6xx_render_mode val)
-{
-	return ((val) << A6XX_VFD_MODE_CNTL_RENDER_MODE__SHIFT) & A6XX_VFD_MODE_CNTL_RENDER_MODE__MASK;
-}
-
-#define REG_A6XX_VFD_MULTIVIEW_CNTL				0x0000a008
-#define A6XX_VFD_MULTIVIEW_CNTL_ENABLE				0x00000001
-#define A6XX_VFD_MULTIVIEW_CNTL_DISABLEMULTIPOS			0x00000002
-#define A6XX_VFD_MULTIVIEW_CNTL_VIEWS__MASK			0x0000007c
-#define A6XX_VFD_MULTIVIEW_CNTL_VIEWS__SHIFT			2
-static inline uint32_t A6XX_VFD_MULTIVIEW_CNTL_VIEWS(uint32_t val)
-{
-	return ((val) << A6XX_VFD_MULTIVIEW_CNTL_VIEWS__SHIFT) & A6XX_VFD_MULTIVIEW_CNTL_VIEWS__MASK;
-}
-
-#define REG_A6XX_VFD_ADD_OFFSET					0x0000a009
-#define A6XX_VFD_ADD_OFFSET_VERTEX				0x00000001
-#define A6XX_VFD_ADD_OFFSET_INSTANCE				0x00000002
-
-#define REG_A6XX_VFD_INDEX_OFFSET				0x0000a00e
-
-#define REG_A6XX_VFD_INSTANCE_START_OFFSET			0x0000a00f
-
-#define REG_A6XX_VFD_FETCH(i0) (0x0000a010 + 0x4*(i0))
-
-static inline uint32_t REG_A6XX_VFD_FETCH_BASE(uint32_t i0) { return 0x0000a010 + 0x4*i0; }
-
-static inline uint32_t REG_A6XX_VFD_FETCH_SIZE(uint32_t i0) { return 0x0000a012 + 0x4*i0; }
-
-static inline uint32_t REG_A6XX_VFD_FETCH_STRIDE(uint32_t i0) { return 0x0000a013 + 0x4*i0; }
-
-#define REG_A6XX_VFD_DECODE(i0) (0x0000a090 + 0x2*(i0))
-
-static inline uint32_t REG_A6XX_VFD_DECODE_INSTR(uint32_t i0) { return 0x0000a090 + 0x2*i0; }
-#define A6XX_VFD_DECODE_INSTR_IDX__MASK				0x0000001f
-#define A6XX_VFD_DECODE_INSTR_IDX__SHIFT			0
-static inline uint32_t A6XX_VFD_DECODE_INSTR_IDX(uint32_t val)
-{
-	return ((val) << A6XX_VFD_DECODE_INSTR_IDX__SHIFT) & A6XX_VFD_DECODE_INSTR_IDX__MASK;
-}
-#define A6XX_VFD_DECODE_INSTR_OFFSET__MASK			0x0001ffe0
-#define A6XX_VFD_DECODE_INSTR_OFFSET__SHIFT			5
-static inline uint32_t A6XX_VFD_DECODE_INSTR_OFFSET(uint32_t val)
-{
-	return ((val) << A6XX_VFD_DECODE_INSTR_OFFSET__SHIFT) & A6XX_VFD_DECODE_INSTR_OFFSET__MASK;
-}
-#define A6XX_VFD_DECODE_INSTR_INSTANCED				0x00020000
-#define A6XX_VFD_DECODE_INSTR_FORMAT__MASK			0x0ff00000
-#define A6XX_VFD_DECODE_INSTR_FORMAT__SHIFT			20
-static inline uint32_t A6XX_VFD_DECODE_INSTR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A6XX_VFD_DECODE_INSTR_FORMAT__SHIFT) & A6XX_VFD_DECODE_INSTR_FORMAT__MASK;
-}
-#define A6XX_VFD_DECODE_INSTR_SWAP__MASK			0x30000000
-#define A6XX_VFD_DECODE_INSTR_SWAP__SHIFT			28
-static inline uint32_t A6XX_VFD_DECODE_INSTR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A6XX_VFD_DECODE_INSTR_SWAP__SHIFT) & A6XX_VFD_DECODE_INSTR_SWAP__MASK;
-}
-#define A6XX_VFD_DECODE_INSTR_UNK30				0x40000000
-#define A6XX_VFD_DECODE_INSTR_FLOAT				0x80000000
-
-static inline uint32_t REG_A6XX_VFD_DECODE_STEP_RATE(uint32_t i0) { return 0x0000a091 + 0x2*i0; }
-
-#define REG_A6XX_VFD_DEST_CNTL(i0) (0x0000a0d0 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_VFD_DEST_CNTL_INSTR(uint32_t i0) { return 0x0000a0d0 + 0x1*i0; }
-#define A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK__MASK		0x0000000f
-#define A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK__SHIFT		0
-static inline uint32_t A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK(uint32_t val)
-{
-	return ((val) << A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK__SHIFT) & A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK__MASK;
-}
-#define A6XX_VFD_DEST_CNTL_INSTR_REGID__MASK			0x00000ff0
-#define A6XX_VFD_DEST_CNTL_INSTR_REGID__SHIFT			4
-static inline uint32_t A6XX_VFD_DEST_CNTL_INSTR_REGID(uint32_t val)
-{
-	return ((val) << A6XX_VFD_DEST_CNTL_INSTR_REGID__SHIFT) & A6XX_VFD_DEST_CNTL_INSTR_REGID__MASK;
-}
-
-#define REG_A6XX_VFD_POWER_CNTL					0x0000a0f8
-
-#define REG_A7XX_VFD_UNKNOWN_A600				0x0000a600
-
-#define REG_A6XX_VFD_ADDR_MODE_CNTL				0x0000a601
-
-#define REG_A6XX_VFD_PERFCTR_VFD_SEL(i0) (0x0000a610 + 0x1*(i0))
-
-#define REG_A7XX_VFD_PERFCTR_VFD_SEL(i0) (0x0000a610 + 0x1*(i0))
-
-#define REG_A6XX_SP_VS_CTRL_REG0				0x0000a800
-#define A6XX_SP_VS_CTRL_REG0_THREADMODE__MASK			0x00000001
-#define A6XX_SP_VS_CTRL_REG0_THREADMODE__SHIFT			0
-static inline uint32_t A6XX_SP_VS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
-{
-	return ((val) << A6XX_SP_VS_CTRL_REG0_THREADMODE__SHIFT) & A6XX_SP_VS_CTRL_REG0_THREADMODE__MASK;
-}
-#define A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x0000007e
-#define A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		1
-static inline uint32_t A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x00001f80
-#define A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		7
-static inline uint32_t A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A6XX_SP_VS_CTRL_REG0_UNK13				0x00002000
-#define A6XX_SP_VS_CTRL_REG0_BRANCHSTACK__MASK			0x000fc000
-#define A6XX_SP_VS_CTRL_REG0_BRANCHSTACK__SHIFT			14
-static inline uint32_t A6XX_SP_VS_CTRL_REG0_BRANCHSTACK(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_VS_CTRL_REG0_BRANCHSTACK__MASK;
-}
-#define A6XX_SP_VS_CTRL_REG0_MERGEDREGS				0x00100000
-#define A6XX_SP_VS_CTRL_REG0_EARLYPREAMBLE			0x00200000
-
-#define REG_A6XX_SP_VS_BRANCH_COND				0x0000a801
-
-#define REG_A6XX_SP_VS_PRIMITIVE_CNTL				0x0000a802
-#define A6XX_SP_VS_PRIMITIVE_CNTL_OUT__MASK			0x0000003f
-#define A6XX_SP_VS_PRIMITIVE_CNTL_OUT__SHIFT			0
-static inline uint32_t A6XX_SP_VS_PRIMITIVE_CNTL_OUT(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_PRIMITIVE_CNTL_OUT__SHIFT) & A6XX_SP_VS_PRIMITIVE_CNTL_OUT__MASK;
-}
-#define A6XX_SP_VS_PRIMITIVE_CNTL_FLAGS_REGID__MASK		0x00003fc0
-#define A6XX_SP_VS_PRIMITIVE_CNTL_FLAGS_REGID__SHIFT		6
-static inline uint32_t A6XX_SP_VS_PRIMITIVE_CNTL_FLAGS_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_PRIMITIVE_CNTL_FLAGS_REGID__SHIFT) & A6XX_SP_VS_PRIMITIVE_CNTL_FLAGS_REGID__MASK;
-}
-
-#define REG_A6XX_SP_VS_OUT(i0) (0x0000a803 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_SP_VS_OUT_REG(uint32_t i0) { return 0x0000a803 + 0x1*i0; }
-#define A6XX_SP_VS_OUT_REG_A_REGID__MASK			0x000000ff
-#define A6XX_SP_VS_OUT_REG_A_REGID__SHIFT			0
-static inline uint32_t A6XX_SP_VS_OUT_REG_A_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_OUT_REG_A_REGID__SHIFT) & A6XX_SP_VS_OUT_REG_A_REGID__MASK;
-}
-#define A6XX_SP_VS_OUT_REG_A_COMPMASK__MASK			0x00000f00
-#define A6XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT			8
-static inline uint32_t A6XX_SP_VS_OUT_REG_A_COMPMASK(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT) & A6XX_SP_VS_OUT_REG_A_COMPMASK__MASK;
-}
-#define A6XX_SP_VS_OUT_REG_B_REGID__MASK			0x00ff0000
-#define A6XX_SP_VS_OUT_REG_B_REGID__SHIFT			16
-static inline uint32_t A6XX_SP_VS_OUT_REG_B_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_OUT_REG_B_REGID__SHIFT) & A6XX_SP_VS_OUT_REG_B_REGID__MASK;
-}
-#define A6XX_SP_VS_OUT_REG_B_COMPMASK__MASK			0x0f000000
-#define A6XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT			24
-static inline uint32_t A6XX_SP_VS_OUT_REG_B_COMPMASK(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT) & A6XX_SP_VS_OUT_REG_B_COMPMASK__MASK;
-}
-
-#define REG_A6XX_SP_VS_VPC_DST(i0) (0x0000a813 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_SP_VS_VPC_DST_REG(uint32_t i0) { return 0x0000a813 + 0x1*i0; }
-#define A6XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK			0x000000ff
-#define A6XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT			0
-static inline uint32_t A6XX_SP_VS_VPC_DST_REG_OUTLOC0(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT) & A6XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK;
-}
-#define A6XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK			0x0000ff00
-#define A6XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT			8
-static inline uint32_t A6XX_SP_VS_VPC_DST_REG_OUTLOC1(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT) & A6XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK;
-}
-#define A6XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK			0x00ff0000
-#define A6XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT			16
-static inline uint32_t A6XX_SP_VS_VPC_DST_REG_OUTLOC2(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT) & A6XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK;
-}
-#define A6XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK			0xff000000
-#define A6XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT			24
-static inline uint32_t A6XX_SP_VS_VPC_DST_REG_OUTLOC3(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT) & A6XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK;
-}
-
-#define REG_A6XX_SP_VS_OBJ_FIRST_EXEC_OFFSET			0x0000a81b
-
-#define REG_A6XX_SP_VS_OBJ_START				0x0000a81c
-
-#define REG_A6XX_SP_VS_PVT_MEM_PARAM				0x0000a81e
-#define A6XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK		0x000000ff
-#define A6XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT		0
-static inline uint32_t A6XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x1ff));
-	return (((val >> 9)) << A6XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT) & A6XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK;
-}
-#define A6XX_SP_VS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A6XX_SP_VS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A6XX_SP_VS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT) & A6XX_SP_VS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A6XX_SP_VS_PVT_MEM_ADDR				0x0000a81f
-
-#define REG_A6XX_SP_VS_PVT_MEM_SIZE				0x0000a821
-#define A6XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK		0x0003ffff
-#define A6XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT		0
-static inline uint32_t A6XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A6XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT) & A6XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK;
-}
-#define A6XX_SP_VS_PVT_MEM_SIZE_PERWAVEMEMLAYOUT		0x80000000
-
-#define REG_A6XX_SP_VS_TEX_COUNT				0x0000a822
-
-#define REG_A6XX_SP_VS_CONFIG					0x0000a823
-#define A6XX_SP_VS_CONFIG_BINDLESS_TEX				0x00000001
-#define A6XX_SP_VS_CONFIG_BINDLESS_SAMP				0x00000002
-#define A6XX_SP_VS_CONFIG_BINDLESS_IBO				0x00000004
-#define A6XX_SP_VS_CONFIG_BINDLESS_UBO				0x00000008
-#define A6XX_SP_VS_CONFIG_ENABLED				0x00000100
-#define A6XX_SP_VS_CONFIG_NTEX__MASK				0x0001fe00
-#define A6XX_SP_VS_CONFIG_NTEX__SHIFT				9
-static inline uint32_t A6XX_SP_VS_CONFIG_NTEX(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_CONFIG_NTEX__SHIFT) & A6XX_SP_VS_CONFIG_NTEX__MASK;
-}
-#define A6XX_SP_VS_CONFIG_NSAMP__MASK				0x003e0000
-#define A6XX_SP_VS_CONFIG_NSAMP__SHIFT				17
-static inline uint32_t A6XX_SP_VS_CONFIG_NSAMP(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_CONFIG_NSAMP__SHIFT) & A6XX_SP_VS_CONFIG_NSAMP__MASK;
-}
-#define A6XX_SP_VS_CONFIG_NIBO__MASK				0x1fc00000
-#define A6XX_SP_VS_CONFIG_NIBO__SHIFT				22
-static inline uint32_t A6XX_SP_VS_CONFIG_NIBO(uint32_t val)
-{
-	return ((val) << A6XX_SP_VS_CONFIG_NIBO__SHIFT) & A6XX_SP_VS_CONFIG_NIBO__MASK;
-}
-
-#define REG_A6XX_SP_VS_INSTRLEN					0x0000a824
-
-#define REG_A6XX_SP_VS_PVT_MEM_HW_STACK_OFFSET			0x0000a825
-#define A6XX_SP_VS_PVT_MEM_HW_STACK_OFFSET_OFFSET__MASK		0x0007ffff
-#define A6XX_SP_VS_PVT_MEM_HW_STACK_OFFSET_OFFSET__SHIFT	0
-static inline uint32_t A6XX_SP_VS_PVT_MEM_HW_STACK_OFFSET_OFFSET(uint32_t val)
-{
-	assert(!(val & 0x7ff));
-	return (((val >> 11)) << A6XX_SP_VS_PVT_MEM_HW_STACK_OFFSET_OFFSET__SHIFT) & A6XX_SP_VS_PVT_MEM_HW_STACK_OFFSET_OFFSET__MASK;
-}
-
-#define REG_A7XX_SP_VS_VGPR_CONFIG				0x0000a82d
-
-#define REG_A6XX_SP_HS_CTRL_REG0				0x0000a830
-#define A6XX_SP_HS_CTRL_REG0_THREADMODE__MASK			0x00000001
-#define A6XX_SP_HS_CTRL_REG0_THREADMODE__SHIFT			0
-static inline uint32_t A6XX_SP_HS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
-{
-	return ((val) << A6XX_SP_HS_CTRL_REG0_THREADMODE__SHIFT) & A6XX_SP_HS_CTRL_REG0_THREADMODE__MASK;
-}
-#define A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x0000007e
-#define A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		1
-static inline uint32_t A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x00001f80
-#define A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		7
-static inline uint32_t A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A6XX_SP_HS_CTRL_REG0_UNK13				0x00002000
-#define A6XX_SP_HS_CTRL_REG0_BRANCHSTACK__MASK			0x000fc000
-#define A6XX_SP_HS_CTRL_REG0_BRANCHSTACK__SHIFT			14
-static inline uint32_t A6XX_SP_HS_CTRL_REG0_BRANCHSTACK(uint32_t val)
-{
-	return ((val) << A6XX_SP_HS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_HS_CTRL_REG0_BRANCHSTACK__MASK;
-}
-#define A6XX_SP_HS_CTRL_REG0_EARLYPREAMBLE			0x00100000
-
-#define REG_A6XX_SP_HS_WAVE_INPUT_SIZE				0x0000a831
-
-#define REG_A6XX_SP_HS_BRANCH_COND				0x0000a832
-
-#define REG_A6XX_SP_HS_OBJ_FIRST_EXEC_OFFSET			0x0000a833
-
-#define REG_A6XX_SP_HS_OBJ_START				0x0000a834
-
-#define REG_A6XX_SP_HS_PVT_MEM_PARAM				0x0000a836
-#define A6XX_SP_HS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK		0x000000ff
-#define A6XX_SP_HS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT		0
-static inline uint32_t A6XX_SP_HS_PVT_MEM_PARAM_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x1ff));
-	return (((val >> 9)) << A6XX_SP_HS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT) & A6XX_SP_HS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK;
-}
-#define A6XX_SP_HS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A6XX_SP_HS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A6XX_SP_HS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A6XX_SP_HS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT) & A6XX_SP_HS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A6XX_SP_HS_PVT_MEM_ADDR				0x0000a837
-
-#define REG_A6XX_SP_HS_PVT_MEM_SIZE				0x0000a839
-#define A6XX_SP_HS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK		0x0003ffff
-#define A6XX_SP_HS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT		0
-static inline uint32_t A6XX_SP_HS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A6XX_SP_HS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT) & A6XX_SP_HS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK;
-}
-#define A6XX_SP_HS_PVT_MEM_SIZE_PERWAVEMEMLAYOUT		0x80000000
-
-#define REG_A6XX_SP_HS_TEX_COUNT				0x0000a83a
-
-#define REG_A6XX_SP_HS_CONFIG					0x0000a83b
-#define A6XX_SP_HS_CONFIG_BINDLESS_TEX				0x00000001
-#define A6XX_SP_HS_CONFIG_BINDLESS_SAMP				0x00000002
-#define A6XX_SP_HS_CONFIG_BINDLESS_IBO				0x00000004
-#define A6XX_SP_HS_CONFIG_BINDLESS_UBO				0x00000008
-#define A6XX_SP_HS_CONFIG_ENABLED				0x00000100
-#define A6XX_SP_HS_CONFIG_NTEX__MASK				0x0001fe00
-#define A6XX_SP_HS_CONFIG_NTEX__SHIFT				9
-static inline uint32_t A6XX_SP_HS_CONFIG_NTEX(uint32_t val)
-{
-	return ((val) << A6XX_SP_HS_CONFIG_NTEX__SHIFT) & A6XX_SP_HS_CONFIG_NTEX__MASK;
-}
-#define A6XX_SP_HS_CONFIG_NSAMP__MASK				0x003e0000
-#define A6XX_SP_HS_CONFIG_NSAMP__SHIFT				17
-static inline uint32_t A6XX_SP_HS_CONFIG_NSAMP(uint32_t val)
-{
-	return ((val) << A6XX_SP_HS_CONFIG_NSAMP__SHIFT) & A6XX_SP_HS_CONFIG_NSAMP__MASK;
-}
-#define A6XX_SP_HS_CONFIG_NIBO__MASK				0x1fc00000
-#define A6XX_SP_HS_CONFIG_NIBO__SHIFT				22
-static inline uint32_t A6XX_SP_HS_CONFIG_NIBO(uint32_t val)
-{
-	return ((val) << A6XX_SP_HS_CONFIG_NIBO__SHIFT) & A6XX_SP_HS_CONFIG_NIBO__MASK;
-}
-
-#define REG_A6XX_SP_HS_INSTRLEN					0x0000a83c
-
-#define REG_A6XX_SP_HS_PVT_MEM_HW_STACK_OFFSET			0x0000a83d
-#define A6XX_SP_HS_PVT_MEM_HW_STACK_OFFSET_OFFSET__MASK		0x0007ffff
-#define A6XX_SP_HS_PVT_MEM_HW_STACK_OFFSET_OFFSET__SHIFT	0
-static inline uint32_t A6XX_SP_HS_PVT_MEM_HW_STACK_OFFSET_OFFSET(uint32_t val)
-{
-	assert(!(val & 0x7ff));
-	return (((val >> 11)) << A6XX_SP_HS_PVT_MEM_HW_STACK_OFFSET_OFFSET__SHIFT) & A6XX_SP_HS_PVT_MEM_HW_STACK_OFFSET_OFFSET__MASK;
-}
-
-#define REG_A7XX_SP_HS_VGPR_CONFIG				0x0000a82f
-
-#define REG_A6XX_SP_DS_CTRL_REG0				0x0000a840
-#define A6XX_SP_DS_CTRL_REG0_THREADMODE__MASK			0x00000001
-#define A6XX_SP_DS_CTRL_REG0_THREADMODE__SHIFT			0
-static inline uint32_t A6XX_SP_DS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
-{
-	return ((val) << A6XX_SP_DS_CTRL_REG0_THREADMODE__SHIFT) & A6XX_SP_DS_CTRL_REG0_THREADMODE__MASK;
-}
-#define A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x0000007e
-#define A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		1
-static inline uint32_t A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x00001f80
-#define A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		7
-static inline uint32_t A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A6XX_SP_DS_CTRL_REG0_UNK13				0x00002000
-#define A6XX_SP_DS_CTRL_REG0_BRANCHSTACK__MASK			0x000fc000
-#define A6XX_SP_DS_CTRL_REG0_BRANCHSTACK__SHIFT			14
-static inline uint32_t A6XX_SP_DS_CTRL_REG0_BRANCHSTACK(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_DS_CTRL_REG0_BRANCHSTACK__MASK;
-}
-#define A6XX_SP_DS_CTRL_REG0_EARLYPREAMBLE			0x00100000
-
-#define REG_A6XX_SP_DS_BRANCH_COND				0x0000a841
-
-#define REG_A6XX_SP_DS_PRIMITIVE_CNTL				0x0000a842
-#define A6XX_SP_DS_PRIMITIVE_CNTL_OUT__MASK			0x0000003f
-#define A6XX_SP_DS_PRIMITIVE_CNTL_OUT__SHIFT			0
-static inline uint32_t A6XX_SP_DS_PRIMITIVE_CNTL_OUT(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_PRIMITIVE_CNTL_OUT__SHIFT) & A6XX_SP_DS_PRIMITIVE_CNTL_OUT__MASK;
-}
-#define A6XX_SP_DS_PRIMITIVE_CNTL_FLAGS_REGID__MASK		0x00003fc0
-#define A6XX_SP_DS_PRIMITIVE_CNTL_FLAGS_REGID__SHIFT		6
-static inline uint32_t A6XX_SP_DS_PRIMITIVE_CNTL_FLAGS_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_PRIMITIVE_CNTL_FLAGS_REGID__SHIFT) & A6XX_SP_DS_PRIMITIVE_CNTL_FLAGS_REGID__MASK;
-}
-
-#define REG_A6XX_SP_DS_OUT(i0) (0x0000a843 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_SP_DS_OUT_REG(uint32_t i0) { return 0x0000a843 + 0x1*i0; }
-#define A6XX_SP_DS_OUT_REG_A_REGID__MASK			0x000000ff
-#define A6XX_SP_DS_OUT_REG_A_REGID__SHIFT			0
-static inline uint32_t A6XX_SP_DS_OUT_REG_A_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_OUT_REG_A_REGID__SHIFT) & A6XX_SP_DS_OUT_REG_A_REGID__MASK;
-}
-#define A6XX_SP_DS_OUT_REG_A_COMPMASK__MASK			0x00000f00
-#define A6XX_SP_DS_OUT_REG_A_COMPMASK__SHIFT			8
-static inline uint32_t A6XX_SP_DS_OUT_REG_A_COMPMASK(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_OUT_REG_A_COMPMASK__SHIFT) & A6XX_SP_DS_OUT_REG_A_COMPMASK__MASK;
-}
-#define A6XX_SP_DS_OUT_REG_B_REGID__MASK			0x00ff0000
-#define A6XX_SP_DS_OUT_REG_B_REGID__SHIFT			16
-static inline uint32_t A6XX_SP_DS_OUT_REG_B_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_OUT_REG_B_REGID__SHIFT) & A6XX_SP_DS_OUT_REG_B_REGID__MASK;
-}
-#define A6XX_SP_DS_OUT_REG_B_COMPMASK__MASK			0x0f000000
-#define A6XX_SP_DS_OUT_REG_B_COMPMASK__SHIFT			24
-static inline uint32_t A6XX_SP_DS_OUT_REG_B_COMPMASK(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_OUT_REG_B_COMPMASK__SHIFT) & A6XX_SP_DS_OUT_REG_B_COMPMASK__MASK;
-}
-
-#define REG_A6XX_SP_DS_VPC_DST(i0) (0x0000a853 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_SP_DS_VPC_DST_REG(uint32_t i0) { return 0x0000a853 + 0x1*i0; }
-#define A6XX_SP_DS_VPC_DST_REG_OUTLOC0__MASK			0x000000ff
-#define A6XX_SP_DS_VPC_DST_REG_OUTLOC0__SHIFT			0
-static inline uint32_t A6XX_SP_DS_VPC_DST_REG_OUTLOC0(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_VPC_DST_REG_OUTLOC0__SHIFT) & A6XX_SP_DS_VPC_DST_REG_OUTLOC0__MASK;
-}
-#define A6XX_SP_DS_VPC_DST_REG_OUTLOC1__MASK			0x0000ff00
-#define A6XX_SP_DS_VPC_DST_REG_OUTLOC1__SHIFT			8
-static inline uint32_t A6XX_SP_DS_VPC_DST_REG_OUTLOC1(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_VPC_DST_REG_OUTLOC1__SHIFT) & A6XX_SP_DS_VPC_DST_REG_OUTLOC1__MASK;
-}
-#define A6XX_SP_DS_VPC_DST_REG_OUTLOC2__MASK			0x00ff0000
-#define A6XX_SP_DS_VPC_DST_REG_OUTLOC2__SHIFT			16
-static inline uint32_t A6XX_SP_DS_VPC_DST_REG_OUTLOC2(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_VPC_DST_REG_OUTLOC2__SHIFT) & A6XX_SP_DS_VPC_DST_REG_OUTLOC2__MASK;
-}
-#define A6XX_SP_DS_VPC_DST_REG_OUTLOC3__MASK			0xff000000
-#define A6XX_SP_DS_VPC_DST_REG_OUTLOC3__SHIFT			24
-static inline uint32_t A6XX_SP_DS_VPC_DST_REG_OUTLOC3(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_VPC_DST_REG_OUTLOC3__SHIFT) & A6XX_SP_DS_VPC_DST_REG_OUTLOC3__MASK;
-}
-
-#define REG_A6XX_SP_DS_OBJ_FIRST_EXEC_OFFSET			0x0000a85b
-
-#define REG_A6XX_SP_DS_OBJ_START				0x0000a85c
-
-#define REG_A6XX_SP_DS_PVT_MEM_PARAM				0x0000a85e
-#define A6XX_SP_DS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK		0x000000ff
-#define A6XX_SP_DS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT		0
-static inline uint32_t A6XX_SP_DS_PVT_MEM_PARAM_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x1ff));
-	return (((val >> 9)) << A6XX_SP_DS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT) & A6XX_SP_DS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK;
-}
-#define A6XX_SP_DS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A6XX_SP_DS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A6XX_SP_DS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT) & A6XX_SP_DS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A6XX_SP_DS_PVT_MEM_ADDR				0x0000a85f
-
-#define REG_A6XX_SP_DS_PVT_MEM_SIZE				0x0000a861
-#define A6XX_SP_DS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK		0x0003ffff
-#define A6XX_SP_DS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT		0
-static inline uint32_t A6XX_SP_DS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A6XX_SP_DS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT) & A6XX_SP_DS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK;
-}
-#define A6XX_SP_DS_PVT_MEM_SIZE_PERWAVEMEMLAYOUT		0x80000000
-
-#define REG_A6XX_SP_DS_TEX_COUNT				0x0000a862
-
-#define REG_A6XX_SP_DS_CONFIG					0x0000a863
-#define A6XX_SP_DS_CONFIG_BINDLESS_TEX				0x00000001
-#define A6XX_SP_DS_CONFIG_BINDLESS_SAMP				0x00000002
-#define A6XX_SP_DS_CONFIG_BINDLESS_IBO				0x00000004
-#define A6XX_SP_DS_CONFIG_BINDLESS_UBO				0x00000008
-#define A6XX_SP_DS_CONFIG_ENABLED				0x00000100
-#define A6XX_SP_DS_CONFIG_NTEX__MASK				0x0001fe00
-#define A6XX_SP_DS_CONFIG_NTEX__SHIFT				9
-static inline uint32_t A6XX_SP_DS_CONFIG_NTEX(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_CONFIG_NTEX__SHIFT) & A6XX_SP_DS_CONFIG_NTEX__MASK;
-}
-#define A6XX_SP_DS_CONFIG_NSAMP__MASK				0x003e0000
-#define A6XX_SP_DS_CONFIG_NSAMP__SHIFT				17
-static inline uint32_t A6XX_SP_DS_CONFIG_NSAMP(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_CONFIG_NSAMP__SHIFT) & A6XX_SP_DS_CONFIG_NSAMP__MASK;
-}
-#define A6XX_SP_DS_CONFIG_NIBO__MASK				0x1fc00000
-#define A6XX_SP_DS_CONFIG_NIBO__SHIFT				22
-static inline uint32_t A6XX_SP_DS_CONFIG_NIBO(uint32_t val)
-{
-	return ((val) << A6XX_SP_DS_CONFIG_NIBO__SHIFT) & A6XX_SP_DS_CONFIG_NIBO__MASK;
-}
-
-#define REG_A6XX_SP_DS_INSTRLEN					0x0000a864
-
-#define REG_A6XX_SP_DS_PVT_MEM_HW_STACK_OFFSET			0x0000a865
-#define A6XX_SP_DS_PVT_MEM_HW_STACK_OFFSET_OFFSET__MASK		0x0007ffff
-#define A6XX_SP_DS_PVT_MEM_HW_STACK_OFFSET_OFFSET__SHIFT	0
-static inline uint32_t A6XX_SP_DS_PVT_MEM_HW_STACK_OFFSET_OFFSET(uint32_t val)
-{
-	assert(!(val & 0x7ff));
-	return (((val >> 11)) << A6XX_SP_DS_PVT_MEM_HW_STACK_OFFSET_OFFSET__SHIFT) & A6XX_SP_DS_PVT_MEM_HW_STACK_OFFSET_OFFSET__MASK;
-}
-
-#define REG_A7XX_SP_DS_VGPR_CONFIG				0x0000a868
-
-#define REG_A6XX_SP_GS_CTRL_REG0				0x0000a870
-#define A6XX_SP_GS_CTRL_REG0_THREADMODE__MASK			0x00000001
-#define A6XX_SP_GS_CTRL_REG0_THREADMODE__SHIFT			0
-static inline uint32_t A6XX_SP_GS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
-{
-	return ((val) << A6XX_SP_GS_CTRL_REG0_THREADMODE__SHIFT) & A6XX_SP_GS_CTRL_REG0_THREADMODE__MASK;
-}
-#define A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x0000007e
-#define A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		1
-static inline uint32_t A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x00001f80
-#define A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		7
-static inline uint32_t A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A6XX_SP_GS_CTRL_REG0_UNK13				0x00002000
-#define A6XX_SP_GS_CTRL_REG0_BRANCHSTACK__MASK			0x000fc000
-#define A6XX_SP_GS_CTRL_REG0_BRANCHSTACK__SHIFT			14
-static inline uint32_t A6XX_SP_GS_CTRL_REG0_BRANCHSTACK(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_GS_CTRL_REG0_BRANCHSTACK__MASK;
-}
-#define A6XX_SP_GS_CTRL_REG0_EARLYPREAMBLE			0x00100000
-
-#define REG_A6XX_SP_GS_PRIM_SIZE				0x0000a871
-
-#define REG_A6XX_SP_GS_BRANCH_COND				0x0000a872
-
-#define REG_A6XX_SP_GS_PRIMITIVE_CNTL				0x0000a873
-#define A6XX_SP_GS_PRIMITIVE_CNTL_OUT__MASK			0x0000003f
-#define A6XX_SP_GS_PRIMITIVE_CNTL_OUT__SHIFT			0
-static inline uint32_t A6XX_SP_GS_PRIMITIVE_CNTL_OUT(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_PRIMITIVE_CNTL_OUT__SHIFT) & A6XX_SP_GS_PRIMITIVE_CNTL_OUT__MASK;
-}
-#define A6XX_SP_GS_PRIMITIVE_CNTL_FLAGS_REGID__MASK		0x00003fc0
-#define A6XX_SP_GS_PRIMITIVE_CNTL_FLAGS_REGID__SHIFT		6
-static inline uint32_t A6XX_SP_GS_PRIMITIVE_CNTL_FLAGS_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_PRIMITIVE_CNTL_FLAGS_REGID__SHIFT) & A6XX_SP_GS_PRIMITIVE_CNTL_FLAGS_REGID__MASK;
-}
-
-#define REG_A6XX_SP_GS_OUT(i0) (0x0000a874 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_SP_GS_OUT_REG(uint32_t i0) { return 0x0000a874 + 0x1*i0; }
-#define A6XX_SP_GS_OUT_REG_A_REGID__MASK			0x000000ff
-#define A6XX_SP_GS_OUT_REG_A_REGID__SHIFT			0
-static inline uint32_t A6XX_SP_GS_OUT_REG_A_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_OUT_REG_A_REGID__SHIFT) & A6XX_SP_GS_OUT_REG_A_REGID__MASK;
-}
-#define A6XX_SP_GS_OUT_REG_A_COMPMASK__MASK			0x00000f00
-#define A6XX_SP_GS_OUT_REG_A_COMPMASK__SHIFT			8
-static inline uint32_t A6XX_SP_GS_OUT_REG_A_COMPMASK(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_OUT_REG_A_COMPMASK__SHIFT) & A6XX_SP_GS_OUT_REG_A_COMPMASK__MASK;
-}
-#define A6XX_SP_GS_OUT_REG_B_REGID__MASK			0x00ff0000
-#define A6XX_SP_GS_OUT_REG_B_REGID__SHIFT			16
-static inline uint32_t A6XX_SP_GS_OUT_REG_B_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_OUT_REG_B_REGID__SHIFT) & A6XX_SP_GS_OUT_REG_B_REGID__MASK;
-}
-#define A6XX_SP_GS_OUT_REG_B_COMPMASK__MASK			0x0f000000
-#define A6XX_SP_GS_OUT_REG_B_COMPMASK__SHIFT			24
-static inline uint32_t A6XX_SP_GS_OUT_REG_B_COMPMASK(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_OUT_REG_B_COMPMASK__SHIFT) & A6XX_SP_GS_OUT_REG_B_COMPMASK__MASK;
-}
-
-#define REG_A6XX_SP_GS_VPC_DST(i0) (0x0000a884 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_SP_GS_VPC_DST_REG(uint32_t i0) { return 0x0000a884 + 0x1*i0; }
-#define A6XX_SP_GS_VPC_DST_REG_OUTLOC0__MASK			0x000000ff
-#define A6XX_SP_GS_VPC_DST_REG_OUTLOC0__SHIFT			0
-static inline uint32_t A6XX_SP_GS_VPC_DST_REG_OUTLOC0(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_VPC_DST_REG_OUTLOC0__SHIFT) & A6XX_SP_GS_VPC_DST_REG_OUTLOC0__MASK;
-}
-#define A6XX_SP_GS_VPC_DST_REG_OUTLOC1__MASK			0x0000ff00
-#define A6XX_SP_GS_VPC_DST_REG_OUTLOC1__SHIFT			8
-static inline uint32_t A6XX_SP_GS_VPC_DST_REG_OUTLOC1(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_VPC_DST_REG_OUTLOC1__SHIFT) & A6XX_SP_GS_VPC_DST_REG_OUTLOC1__MASK;
-}
-#define A6XX_SP_GS_VPC_DST_REG_OUTLOC2__MASK			0x00ff0000
-#define A6XX_SP_GS_VPC_DST_REG_OUTLOC2__SHIFT			16
-static inline uint32_t A6XX_SP_GS_VPC_DST_REG_OUTLOC2(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_VPC_DST_REG_OUTLOC2__SHIFT) & A6XX_SP_GS_VPC_DST_REG_OUTLOC2__MASK;
-}
-#define A6XX_SP_GS_VPC_DST_REG_OUTLOC3__MASK			0xff000000
-#define A6XX_SP_GS_VPC_DST_REG_OUTLOC3__SHIFT			24
-static inline uint32_t A6XX_SP_GS_VPC_DST_REG_OUTLOC3(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_VPC_DST_REG_OUTLOC3__SHIFT) & A6XX_SP_GS_VPC_DST_REG_OUTLOC3__MASK;
-}
-
-#define REG_A6XX_SP_GS_OBJ_FIRST_EXEC_OFFSET			0x0000a88c
-
-#define REG_A6XX_SP_GS_OBJ_START				0x0000a88d
-
-#define REG_A6XX_SP_GS_PVT_MEM_PARAM				0x0000a88f
-#define A6XX_SP_GS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK		0x000000ff
-#define A6XX_SP_GS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT		0
-static inline uint32_t A6XX_SP_GS_PVT_MEM_PARAM_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x1ff));
-	return (((val >> 9)) << A6XX_SP_GS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT) & A6XX_SP_GS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK;
-}
-#define A6XX_SP_GS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A6XX_SP_GS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A6XX_SP_GS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT) & A6XX_SP_GS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A6XX_SP_GS_PVT_MEM_ADDR				0x0000a890
-
-#define REG_A6XX_SP_GS_PVT_MEM_SIZE				0x0000a892
-#define A6XX_SP_GS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK		0x0003ffff
-#define A6XX_SP_GS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT		0
-static inline uint32_t A6XX_SP_GS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A6XX_SP_GS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT) & A6XX_SP_GS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK;
-}
-#define A6XX_SP_GS_PVT_MEM_SIZE_PERWAVEMEMLAYOUT		0x80000000
-
-#define REG_A6XX_SP_GS_TEX_COUNT				0x0000a893
-
-#define REG_A6XX_SP_GS_CONFIG					0x0000a894
-#define A6XX_SP_GS_CONFIG_BINDLESS_TEX				0x00000001
-#define A6XX_SP_GS_CONFIG_BINDLESS_SAMP				0x00000002
-#define A6XX_SP_GS_CONFIG_BINDLESS_IBO				0x00000004
-#define A6XX_SP_GS_CONFIG_BINDLESS_UBO				0x00000008
-#define A6XX_SP_GS_CONFIG_ENABLED				0x00000100
-#define A6XX_SP_GS_CONFIG_NTEX__MASK				0x0001fe00
-#define A6XX_SP_GS_CONFIG_NTEX__SHIFT				9
-static inline uint32_t A6XX_SP_GS_CONFIG_NTEX(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_CONFIG_NTEX__SHIFT) & A6XX_SP_GS_CONFIG_NTEX__MASK;
-}
-#define A6XX_SP_GS_CONFIG_NSAMP__MASK				0x003e0000
-#define A6XX_SP_GS_CONFIG_NSAMP__SHIFT				17
-static inline uint32_t A6XX_SP_GS_CONFIG_NSAMP(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_CONFIG_NSAMP__SHIFT) & A6XX_SP_GS_CONFIG_NSAMP__MASK;
-}
-#define A6XX_SP_GS_CONFIG_NIBO__MASK				0x1fc00000
-#define A6XX_SP_GS_CONFIG_NIBO__SHIFT				22
-static inline uint32_t A6XX_SP_GS_CONFIG_NIBO(uint32_t val)
-{
-	return ((val) << A6XX_SP_GS_CONFIG_NIBO__SHIFT) & A6XX_SP_GS_CONFIG_NIBO__MASK;
-}
-
-#define REG_A6XX_SP_GS_INSTRLEN					0x0000a895
-
-#define REG_A6XX_SP_GS_PVT_MEM_HW_STACK_OFFSET			0x0000a896
-#define A6XX_SP_GS_PVT_MEM_HW_STACK_OFFSET_OFFSET__MASK		0x0007ffff
-#define A6XX_SP_GS_PVT_MEM_HW_STACK_OFFSET_OFFSET__SHIFT	0
-static inline uint32_t A6XX_SP_GS_PVT_MEM_HW_STACK_OFFSET_OFFSET(uint32_t val)
-{
-	assert(!(val & 0x7ff));
-	return (((val >> 11)) << A6XX_SP_GS_PVT_MEM_HW_STACK_OFFSET_OFFSET__SHIFT) & A6XX_SP_GS_PVT_MEM_HW_STACK_OFFSET_OFFSET__MASK;
-}
-
-#define REG_A7XX_SP_GS_VGPR_CONFIG				0x0000a899
-
-#define REG_A6XX_SP_VS_TEX_SAMP					0x0000a8a0
-
-#define REG_A6XX_SP_HS_TEX_SAMP					0x0000a8a2
-
-#define REG_A6XX_SP_DS_TEX_SAMP					0x0000a8a4
-
-#define REG_A6XX_SP_GS_TEX_SAMP					0x0000a8a6
-
-#define REG_A6XX_SP_VS_TEX_CONST				0x0000a8a8
-
-#define REG_A6XX_SP_HS_TEX_CONST				0x0000a8aa
-
-#define REG_A6XX_SP_DS_TEX_CONST				0x0000a8ac
-
-#define REG_A6XX_SP_GS_TEX_CONST				0x0000a8ae
-
-#define REG_A6XX_SP_FS_CTRL_REG0				0x0000a980
-#define A6XX_SP_FS_CTRL_REG0_THREADMODE__MASK			0x00000001
-#define A6XX_SP_FS_CTRL_REG0_THREADMODE__SHIFT			0
-static inline uint32_t A6XX_SP_FS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
-{
-	return ((val) << A6XX_SP_FS_CTRL_REG0_THREADMODE__SHIFT) & A6XX_SP_FS_CTRL_REG0_THREADMODE__MASK;
-}
-#define A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x0000007e
-#define A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		1
-static inline uint32_t A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x00001f80
-#define A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		7
-static inline uint32_t A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A6XX_SP_FS_CTRL_REG0_UNK13				0x00002000
-#define A6XX_SP_FS_CTRL_REG0_BRANCHSTACK__MASK			0x000fc000
-#define A6XX_SP_FS_CTRL_REG0_BRANCHSTACK__SHIFT			14
-static inline uint32_t A6XX_SP_FS_CTRL_REG0_BRANCHSTACK(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_FS_CTRL_REG0_BRANCHSTACK__MASK;
-}
-#define A6XX_SP_FS_CTRL_REG0_THREADSIZE__MASK			0x00100000
-#define A6XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT			20
-static inline uint32_t A6XX_SP_FS_CTRL_REG0_THREADSIZE(enum a6xx_threadsize val)
-{
-	return ((val) << A6XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_FS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A6XX_SP_FS_CTRL_REG0_UNK21				0x00200000
-#define A6XX_SP_FS_CTRL_REG0_VARYING				0x00400000
-#define A6XX_SP_FS_CTRL_REG0_LODPIXMASK				0x00800000
-#define A6XX_SP_FS_CTRL_REG0_UNK24				0x01000000
-#define A6XX_SP_FS_CTRL_REG0_UNK25				0x02000000
-#define A6XX_SP_FS_CTRL_REG0_PIXLODENABLE			0x04000000
-#define A6XX_SP_FS_CTRL_REG0_UNK27				0x08000000
-#define A6XX_SP_FS_CTRL_REG0_EARLYPREAMBLE			0x10000000
-#define A6XX_SP_FS_CTRL_REG0_MERGEDREGS				0x80000000
-
-#define REG_A6XX_SP_FS_BRANCH_COND				0x0000a981
-
-#define REG_A6XX_SP_FS_OBJ_FIRST_EXEC_OFFSET			0x0000a982
-
-#define REG_A6XX_SP_FS_OBJ_START				0x0000a983
-
-#define REG_A6XX_SP_FS_PVT_MEM_PARAM				0x0000a985
-#define A6XX_SP_FS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK		0x000000ff
-#define A6XX_SP_FS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT		0
-static inline uint32_t A6XX_SP_FS_PVT_MEM_PARAM_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x1ff));
-	return (((val >> 9)) << A6XX_SP_FS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT) & A6XX_SP_FS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK;
-}
-#define A6XX_SP_FS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A6XX_SP_FS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A6XX_SP_FS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT) & A6XX_SP_FS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A6XX_SP_FS_PVT_MEM_ADDR				0x0000a986
-
-#define REG_A6XX_SP_FS_PVT_MEM_SIZE				0x0000a988
-#define A6XX_SP_FS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK		0x0003ffff
-#define A6XX_SP_FS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT		0
-static inline uint32_t A6XX_SP_FS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A6XX_SP_FS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT) & A6XX_SP_FS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK;
-}
-#define A6XX_SP_FS_PVT_MEM_SIZE_PERWAVEMEMLAYOUT		0x80000000
-
-#define REG_A6XX_SP_BLEND_CNTL					0x0000a989
-#define A6XX_SP_BLEND_CNTL_ENABLE_BLEND__MASK			0x000000ff
-#define A6XX_SP_BLEND_CNTL_ENABLE_BLEND__SHIFT			0
-static inline uint32_t A6XX_SP_BLEND_CNTL_ENABLE_BLEND(uint32_t val)
-{
-	return ((val) << A6XX_SP_BLEND_CNTL_ENABLE_BLEND__SHIFT) & A6XX_SP_BLEND_CNTL_ENABLE_BLEND__MASK;
-}
-#define A6XX_SP_BLEND_CNTL_UNK8					0x00000100
-#define A6XX_SP_BLEND_CNTL_DUAL_COLOR_IN_ENABLE			0x00000200
-#define A6XX_SP_BLEND_CNTL_ALPHA_TO_COVERAGE			0x00000400
-
-#define REG_A6XX_SP_SRGB_CNTL					0x0000a98a
-#define A6XX_SP_SRGB_CNTL_SRGB_MRT0				0x00000001
-#define A6XX_SP_SRGB_CNTL_SRGB_MRT1				0x00000002
-#define A6XX_SP_SRGB_CNTL_SRGB_MRT2				0x00000004
-#define A6XX_SP_SRGB_CNTL_SRGB_MRT3				0x00000008
-#define A6XX_SP_SRGB_CNTL_SRGB_MRT4				0x00000010
-#define A6XX_SP_SRGB_CNTL_SRGB_MRT5				0x00000020
-#define A6XX_SP_SRGB_CNTL_SRGB_MRT6				0x00000040
-#define A6XX_SP_SRGB_CNTL_SRGB_MRT7				0x00000080
-
-#define REG_A6XX_SP_FS_RENDER_COMPONENTS			0x0000a98b
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT0__MASK			0x0000000f
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT0__SHIFT			0
-static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT0(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT0__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT0__MASK;
-}
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT1__MASK			0x000000f0
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT1__SHIFT			4
-static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT1(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT1__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT1__MASK;
-}
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT2__MASK			0x00000f00
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT2__SHIFT			8
-static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT2(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT2__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT2__MASK;
-}
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT3__MASK			0x0000f000
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT3__SHIFT			12
-static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT3(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT3__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT3__MASK;
-}
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT4__MASK			0x000f0000
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT4__SHIFT			16
-static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT4(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT4__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT4__MASK;
-}
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT5__MASK			0x00f00000
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT5__SHIFT			20
-static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT5(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT5__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT5__MASK;
-}
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT6__MASK			0x0f000000
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT6__SHIFT			24
-static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT6(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT6__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT6__MASK;
-}
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT7__MASK			0xf0000000
-#define A6XX_SP_FS_RENDER_COMPONENTS_RT7__SHIFT			28
-static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT7(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT7__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT7__MASK;
-}
-
-#define REG_A6XX_SP_FS_OUTPUT_CNTL0				0x0000a98c
-#define A6XX_SP_FS_OUTPUT_CNTL0_DUAL_COLOR_IN_ENABLE		0x00000001
-#define A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID__MASK		0x0000ff00
-#define A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID__SHIFT		8
-static inline uint32_t A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID__SHIFT) & A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID__MASK;
-}
-#define A6XX_SP_FS_OUTPUT_CNTL0_SAMPMASK_REGID__MASK		0x00ff0000
-#define A6XX_SP_FS_OUTPUT_CNTL0_SAMPMASK_REGID__SHIFT		16
-static inline uint32_t A6XX_SP_FS_OUTPUT_CNTL0_SAMPMASK_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_OUTPUT_CNTL0_SAMPMASK_REGID__SHIFT) & A6XX_SP_FS_OUTPUT_CNTL0_SAMPMASK_REGID__MASK;
-}
-#define A6XX_SP_FS_OUTPUT_CNTL0_STENCILREF_REGID__MASK		0xff000000
-#define A6XX_SP_FS_OUTPUT_CNTL0_STENCILREF_REGID__SHIFT		24
-static inline uint32_t A6XX_SP_FS_OUTPUT_CNTL0_STENCILREF_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_OUTPUT_CNTL0_STENCILREF_REGID__SHIFT) & A6XX_SP_FS_OUTPUT_CNTL0_STENCILREF_REGID__MASK;
-}
-
-#define REG_A6XX_SP_FS_OUTPUT_CNTL1				0x0000a98d
-#define A6XX_SP_FS_OUTPUT_CNTL1_MRT__MASK			0x0000000f
-#define A6XX_SP_FS_OUTPUT_CNTL1_MRT__SHIFT			0
-static inline uint32_t A6XX_SP_FS_OUTPUT_CNTL1_MRT(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_OUTPUT_CNTL1_MRT__SHIFT) & A6XX_SP_FS_OUTPUT_CNTL1_MRT__MASK;
-}
-
-#define REG_A6XX_SP_FS_OUTPUT(i0) (0x0000a98e + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_SP_FS_OUTPUT_REG(uint32_t i0) { return 0x0000a98e + 0x1*i0; }
-#define A6XX_SP_FS_OUTPUT_REG_REGID__MASK			0x000000ff
-#define A6XX_SP_FS_OUTPUT_REG_REGID__SHIFT			0
-static inline uint32_t A6XX_SP_FS_OUTPUT_REG_REGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_OUTPUT_REG_REGID__SHIFT) & A6XX_SP_FS_OUTPUT_REG_REGID__MASK;
-}
-#define A6XX_SP_FS_OUTPUT_REG_HALF_PRECISION			0x00000100
-
-#define REG_A6XX_SP_FS_MRT(i0) (0x0000a996 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_SP_FS_MRT_REG(uint32_t i0) { return 0x0000a996 + 0x1*i0; }
-#define A6XX_SP_FS_MRT_REG_COLOR_FORMAT__MASK			0x000000ff
-#define A6XX_SP_FS_MRT_REG_COLOR_FORMAT__SHIFT			0
-static inline uint32_t A6XX_SP_FS_MRT_REG_COLOR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A6XX_SP_FS_MRT_REG_COLOR_FORMAT__SHIFT) & A6XX_SP_FS_MRT_REG_COLOR_FORMAT__MASK;
-}
-#define A6XX_SP_FS_MRT_REG_COLOR_SINT				0x00000100
-#define A6XX_SP_FS_MRT_REG_COLOR_UINT				0x00000200
-#define A6XX_SP_FS_MRT_REG_UNK10				0x00000400
-
-#define REG_A6XX_SP_FS_PREFETCH_CNTL				0x0000a99e
-#define A6XX_SP_FS_PREFETCH_CNTL_COUNT__MASK			0x00000007
-#define A6XX_SP_FS_PREFETCH_CNTL_COUNT__SHIFT			0
-static inline uint32_t A6XX_SP_FS_PREFETCH_CNTL_COUNT(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_PREFETCH_CNTL_COUNT__SHIFT) & A6XX_SP_FS_PREFETCH_CNTL_COUNT__MASK;
-}
-#define A6XX_SP_FS_PREFETCH_CNTL_IJ_WRITE_DISABLE		0x00000008
-#define A6XX_SP_FS_PREFETCH_CNTL_ENDOFQUAD			0x00000010
-#define A6XX_SP_FS_PREFETCH_CNTL_WRITE_COLOR_TO_OUTPUT		0x00000020
-#define A6XX_SP_FS_PREFETCH_CNTL_CONSTSLOTID__MASK		0x00007fc0
-#define A6XX_SP_FS_PREFETCH_CNTL_CONSTSLOTID__SHIFT		6
-static inline uint32_t A6XX_SP_FS_PREFETCH_CNTL_CONSTSLOTID(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_PREFETCH_CNTL_CONSTSLOTID__SHIFT) & A6XX_SP_FS_PREFETCH_CNTL_CONSTSLOTID__MASK;
-}
-#define A6XX_SP_FS_PREFETCH_CNTL_CONSTSLOTID4COORD__MASK	0x01ff0000
-#define A6XX_SP_FS_PREFETCH_CNTL_CONSTSLOTID4COORD__SHIFT	16
-static inline uint32_t A6XX_SP_FS_PREFETCH_CNTL_CONSTSLOTID4COORD(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_PREFETCH_CNTL_CONSTSLOTID4COORD__SHIFT) & A6XX_SP_FS_PREFETCH_CNTL_CONSTSLOTID4COORD__MASK;
-}
-
-#define REG_A6XX_SP_FS_PREFETCH(i0) (0x0000a99f + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_SP_FS_PREFETCH_CMD(uint32_t i0) { return 0x0000a99f + 0x1*i0; }
-#define A6XX_SP_FS_PREFETCH_CMD_SRC__MASK			0x0000007f
-#define A6XX_SP_FS_PREFETCH_CMD_SRC__SHIFT			0
-static inline uint32_t A6XX_SP_FS_PREFETCH_CMD_SRC(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_PREFETCH_CMD_SRC__SHIFT) & A6XX_SP_FS_PREFETCH_CMD_SRC__MASK;
-}
-#define A6XX_SP_FS_PREFETCH_CMD_SAMP_ID__MASK			0x00000780
-#define A6XX_SP_FS_PREFETCH_CMD_SAMP_ID__SHIFT			7
-static inline uint32_t A6XX_SP_FS_PREFETCH_CMD_SAMP_ID(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_PREFETCH_CMD_SAMP_ID__SHIFT) & A6XX_SP_FS_PREFETCH_CMD_SAMP_ID__MASK;
-}
-#define A6XX_SP_FS_PREFETCH_CMD_TEX_ID__MASK			0x0000f800
-#define A6XX_SP_FS_PREFETCH_CMD_TEX_ID__SHIFT			11
-static inline uint32_t A6XX_SP_FS_PREFETCH_CMD_TEX_ID(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_PREFETCH_CMD_TEX_ID__SHIFT) & A6XX_SP_FS_PREFETCH_CMD_TEX_ID__MASK;
-}
-#define A6XX_SP_FS_PREFETCH_CMD_DST__MASK			0x003f0000
-#define A6XX_SP_FS_PREFETCH_CMD_DST__SHIFT			16
-static inline uint32_t A6XX_SP_FS_PREFETCH_CMD_DST(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_PREFETCH_CMD_DST__SHIFT) & A6XX_SP_FS_PREFETCH_CMD_DST__MASK;
-}
-#define A6XX_SP_FS_PREFETCH_CMD_WRMASK__MASK			0x03c00000
-#define A6XX_SP_FS_PREFETCH_CMD_WRMASK__SHIFT			22
-static inline uint32_t A6XX_SP_FS_PREFETCH_CMD_WRMASK(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_PREFETCH_CMD_WRMASK__SHIFT) & A6XX_SP_FS_PREFETCH_CMD_WRMASK__MASK;
-}
-#define A6XX_SP_FS_PREFETCH_CMD_HALF				0x04000000
-#define A6XX_SP_FS_PREFETCH_CMD_UNK27				0x08000000
-#define A6XX_SP_FS_PREFETCH_CMD_BINDLESS			0x10000000
-#define A6XX_SP_FS_PREFETCH_CMD_CMD__MASK			0xe0000000
-#define A6XX_SP_FS_PREFETCH_CMD_CMD__SHIFT			29
-static inline uint32_t A6XX_SP_FS_PREFETCH_CMD_CMD(enum a6xx_tex_prefetch_cmd val)
-{
-	return ((val) << A6XX_SP_FS_PREFETCH_CMD_CMD__SHIFT) & A6XX_SP_FS_PREFETCH_CMD_CMD__MASK;
-}
-
-#define REG_A7XX_SP_FS_PREFETCH(i0) (0x0000a99f + 0x1*(i0))
-
-static inline uint32_t REG_A7XX_SP_FS_PREFETCH_CMD(uint32_t i0) { return 0x0000a99f + 0x1*i0; }
-#define A7XX_SP_FS_PREFETCH_CMD_SRC__MASK			0x0000007f
-#define A7XX_SP_FS_PREFETCH_CMD_SRC__SHIFT			0
-static inline uint32_t A7XX_SP_FS_PREFETCH_CMD_SRC(uint32_t val)
-{
-	return ((val) << A7XX_SP_FS_PREFETCH_CMD_SRC__SHIFT) & A7XX_SP_FS_PREFETCH_CMD_SRC__MASK;
-}
-#define A7XX_SP_FS_PREFETCH_CMD_SAMP_ID__MASK			0x00000380
-#define A7XX_SP_FS_PREFETCH_CMD_SAMP_ID__SHIFT			7
-static inline uint32_t A7XX_SP_FS_PREFETCH_CMD_SAMP_ID(uint32_t val)
-{
-	return ((val) << A7XX_SP_FS_PREFETCH_CMD_SAMP_ID__SHIFT) & A7XX_SP_FS_PREFETCH_CMD_SAMP_ID__MASK;
-}
-#define A7XX_SP_FS_PREFETCH_CMD_TEX_ID__MASK			0x00001c00
-#define A7XX_SP_FS_PREFETCH_CMD_TEX_ID__SHIFT			10
-static inline uint32_t A7XX_SP_FS_PREFETCH_CMD_TEX_ID(uint32_t val)
-{
-	return ((val) << A7XX_SP_FS_PREFETCH_CMD_TEX_ID__SHIFT) & A7XX_SP_FS_PREFETCH_CMD_TEX_ID__MASK;
-}
-#define A7XX_SP_FS_PREFETCH_CMD_DST__MASK			0x0007e000
-#define A7XX_SP_FS_PREFETCH_CMD_DST__SHIFT			13
-static inline uint32_t A7XX_SP_FS_PREFETCH_CMD_DST(uint32_t val)
-{
-	return ((val) << A7XX_SP_FS_PREFETCH_CMD_DST__SHIFT) & A7XX_SP_FS_PREFETCH_CMD_DST__MASK;
-}
-#define A7XX_SP_FS_PREFETCH_CMD_WRMASK__MASK			0x00780000
-#define A7XX_SP_FS_PREFETCH_CMD_WRMASK__SHIFT			19
-static inline uint32_t A7XX_SP_FS_PREFETCH_CMD_WRMASK(uint32_t val)
-{
-	return ((val) << A7XX_SP_FS_PREFETCH_CMD_WRMASK__SHIFT) & A7XX_SP_FS_PREFETCH_CMD_WRMASK__MASK;
-}
-#define A7XX_SP_FS_PREFETCH_CMD_HALF				0x00800000
-#define A7XX_SP_FS_PREFETCH_CMD_BINDLESS			0x02000000
-#define A7XX_SP_FS_PREFETCH_CMD_CMD__MASK			0x3c000000
-#define A7XX_SP_FS_PREFETCH_CMD_CMD__SHIFT			26
-static inline uint32_t A7XX_SP_FS_PREFETCH_CMD_CMD(enum a6xx_tex_prefetch_cmd val)
-{
-	return ((val) << A7XX_SP_FS_PREFETCH_CMD_CMD__SHIFT) & A7XX_SP_FS_PREFETCH_CMD_CMD__MASK;
-}
-
-#define REG_A6XX_SP_FS_BINDLESS_PREFETCH(i0) (0x0000a9a3 + 0x1*(i0))
-
-static inline uint32_t REG_A6XX_SP_FS_BINDLESS_PREFETCH_CMD(uint32_t i0) { return 0x0000a9a3 + 0x1*i0; }
-#define A6XX_SP_FS_BINDLESS_PREFETCH_CMD_SAMP_ID__MASK		0x0000ffff
-#define A6XX_SP_FS_BINDLESS_PREFETCH_CMD_SAMP_ID__SHIFT		0
-static inline uint32_t A6XX_SP_FS_BINDLESS_PREFETCH_CMD_SAMP_ID(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_BINDLESS_PREFETCH_CMD_SAMP_ID__SHIFT) & A6XX_SP_FS_BINDLESS_PREFETCH_CMD_SAMP_ID__MASK;
-}
-#define A6XX_SP_FS_BINDLESS_PREFETCH_CMD_TEX_ID__MASK		0xffff0000
-#define A6XX_SP_FS_BINDLESS_PREFETCH_CMD_TEX_ID__SHIFT		16
-static inline uint32_t A6XX_SP_FS_BINDLESS_PREFETCH_CMD_TEX_ID(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_BINDLESS_PREFETCH_CMD_TEX_ID__SHIFT) & A6XX_SP_FS_BINDLESS_PREFETCH_CMD_TEX_ID__MASK;
-}
-
-#define REG_A6XX_SP_FS_TEX_COUNT				0x0000a9a7
-
-#define REG_A6XX_SP_UNKNOWN_A9A8				0x0000a9a8
-
-#define REG_A6XX_SP_FS_PVT_MEM_HW_STACK_OFFSET			0x0000a9a9
-#define A6XX_SP_FS_PVT_MEM_HW_STACK_OFFSET_OFFSET__MASK		0x0007ffff
-#define A6XX_SP_FS_PVT_MEM_HW_STACK_OFFSET_OFFSET__SHIFT	0
-static inline uint32_t A6XX_SP_FS_PVT_MEM_HW_STACK_OFFSET_OFFSET(uint32_t val)
-{
-	assert(!(val & 0x7ff));
-	return (((val >> 11)) << A6XX_SP_FS_PVT_MEM_HW_STACK_OFFSET_OFFSET__SHIFT) & A6XX_SP_FS_PVT_MEM_HW_STACK_OFFSET_OFFSET__MASK;
-}
-
-#define REG_A6XX_SP_CS_CTRL_REG0				0x0000a9b0
-#define A6XX_SP_CS_CTRL_REG0_THREADMODE__MASK			0x00000001
-#define A6XX_SP_CS_CTRL_REG0_THREADMODE__SHIFT			0
-static inline uint32_t A6XX_SP_CS_CTRL_REG0_THREADMODE(enum a3xx_threadmode val)
-{
-	return ((val) << A6XX_SP_CS_CTRL_REG0_THREADMODE__SHIFT) & A6XX_SP_CS_CTRL_REG0_THREADMODE__MASK;
-}
-#define A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x0000007e
-#define A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		1
-static inline uint32_t A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
-}
-#define A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x00001f80
-#define A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		7
-static inline uint32_t A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
-}
-#define A6XX_SP_CS_CTRL_REG0_UNK13				0x00002000
-#define A6XX_SP_CS_CTRL_REG0_BRANCHSTACK__MASK			0x000fc000
-#define A6XX_SP_CS_CTRL_REG0_BRANCHSTACK__SHIFT			14
-static inline uint32_t A6XX_SP_CS_CTRL_REG0_BRANCHSTACK(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_CS_CTRL_REG0_BRANCHSTACK__MASK;
-}
-#define A6XX_SP_CS_CTRL_REG0_THREADSIZE__MASK			0x00100000
-#define A6XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT			20
-static inline uint32_t A6XX_SP_CS_CTRL_REG0_THREADSIZE(enum a6xx_threadsize val)
-{
-	return ((val) << A6XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_CS_CTRL_REG0_THREADSIZE__MASK;
-}
-#define A6XX_SP_CS_CTRL_REG0_UNK21				0x00200000
-#define A6XX_SP_CS_CTRL_REG0_UNK22				0x00400000
-#define A6XX_SP_CS_CTRL_REG0_EARLYPREAMBLE			0x00800000
-#define A6XX_SP_CS_CTRL_REG0_MERGEDREGS				0x80000000
-
-#define REG_A6XX_SP_CS_UNKNOWN_A9B1				0x0000a9b1
-#define A6XX_SP_CS_UNKNOWN_A9B1_SHARED_SIZE__MASK		0x0000001f
-#define A6XX_SP_CS_UNKNOWN_A9B1_SHARED_SIZE__SHIFT		0
-static inline uint32_t A6XX_SP_CS_UNKNOWN_A9B1_SHARED_SIZE(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_UNKNOWN_A9B1_SHARED_SIZE__SHIFT) & A6XX_SP_CS_UNKNOWN_A9B1_SHARED_SIZE__MASK;
-}
-#define A6XX_SP_CS_UNKNOWN_A9B1_UNK5				0x00000020
-#define A6XX_SP_CS_UNKNOWN_A9B1_UNK6				0x00000040
-
-#define REG_A6XX_SP_CS_BRANCH_COND				0x0000a9b2
-
-#define REG_A6XX_SP_CS_OBJ_FIRST_EXEC_OFFSET			0x0000a9b3
-
-#define REG_A6XX_SP_CS_OBJ_START				0x0000a9b4
-
-#define REG_A6XX_SP_CS_PVT_MEM_PARAM				0x0000a9b6
-#define A6XX_SP_CS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK		0x000000ff
-#define A6XX_SP_CS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT		0
-static inline uint32_t A6XX_SP_CS_PVT_MEM_PARAM_MEMSIZEPERITEM(uint32_t val)
-{
-	assert(!(val & 0x1ff));
-	return (((val >> 9)) << A6XX_SP_CS_PVT_MEM_PARAM_MEMSIZEPERITEM__SHIFT) & A6XX_SP_CS_PVT_MEM_PARAM_MEMSIZEPERITEM__MASK;
-}
-#define A6XX_SP_CS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK	0xff000000
-#define A6XX_SP_CS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT	24
-static inline uint32_t A6XX_SP_CS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__SHIFT) & A6XX_SP_CS_PVT_MEM_PARAM_HWSTACKSIZEPERTHREAD__MASK;
-}
-
-#define REG_A6XX_SP_CS_PVT_MEM_ADDR				0x0000a9b7
-
-#define REG_A6XX_SP_CS_PVT_MEM_SIZE				0x0000a9b9
-#define A6XX_SP_CS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK		0x0003ffff
-#define A6XX_SP_CS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT		0
-static inline uint32_t A6XX_SP_CS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A6XX_SP_CS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__SHIFT) & A6XX_SP_CS_PVT_MEM_SIZE_TOTALPVTMEMSIZE__MASK;
-}
-#define A6XX_SP_CS_PVT_MEM_SIZE_PERWAVEMEMLAYOUT		0x80000000
-
-#define REG_A6XX_SP_CS_TEX_COUNT				0x0000a9ba
-
-#define REG_A6XX_SP_CS_CONFIG					0x0000a9bb
-#define A6XX_SP_CS_CONFIG_BINDLESS_TEX				0x00000001
-#define A6XX_SP_CS_CONFIG_BINDLESS_SAMP				0x00000002
-#define A6XX_SP_CS_CONFIG_BINDLESS_IBO				0x00000004
-#define A6XX_SP_CS_CONFIG_BINDLESS_UBO				0x00000008
-#define A6XX_SP_CS_CONFIG_ENABLED				0x00000100
-#define A6XX_SP_CS_CONFIG_NTEX__MASK				0x0001fe00
-#define A6XX_SP_CS_CONFIG_NTEX__SHIFT				9
-static inline uint32_t A6XX_SP_CS_CONFIG_NTEX(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_CONFIG_NTEX__SHIFT) & A6XX_SP_CS_CONFIG_NTEX__MASK;
-}
-#define A6XX_SP_CS_CONFIG_NSAMP__MASK				0x003e0000
-#define A6XX_SP_CS_CONFIG_NSAMP__SHIFT				17
-static inline uint32_t A6XX_SP_CS_CONFIG_NSAMP(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_CONFIG_NSAMP__SHIFT) & A6XX_SP_CS_CONFIG_NSAMP__MASK;
-}
-#define A6XX_SP_CS_CONFIG_NIBO__MASK				0x1fc00000
-#define A6XX_SP_CS_CONFIG_NIBO__SHIFT				22
-static inline uint32_t A6XX_SP_CS_CONFIG_NIBO(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_CONFIG_NIBO__SHIFT) & A6XX_SP_CS_CONFIG_NIBO__MASK;
-}
-
-#define REG_A6XX_SP_CS_INSTRLEN					0x0000a9bc
-
-#define REG_A6XX_SP_CS_PVT_MEM_HW_STACK_OFFSET			0x0000a9bd
-#define A6XX_SP_CS_PVT_MEM_HW_STACK_OFFSET_OFFSET__MASK		0x0007ffff
-#define A6XX_SP_CS_PVT_MEM_HW_STACK_OFFSET_OFFSET__SHIFT	0
-static inline uint32_t A6XX_SP_CS_PVT_MEM_HW_STACK_OFFSET_OFFSET(uint32_t val)
-{
-	assert(!(val & 0x7ff));
-	return (((val >> 11)) << A6XX_SP_CS_PVT_MEM_HW_STACK_OFFSET_OFFSET__SHIFT) & A6XX_SP_CS_PVT_MEM_HW_STACK_OFFSET_OFFSET__MASK;
-}
-
-#define REG_A7XX_SP_CS_UNKNOWN_A9BE				0x0000a9be
-
-#define REG_A7XX_SP_CS_VGPR_CONFIG				0x0000a9c5
-
-#define REG_A6XX_SP_CS_CNTL_0					0x0000a9c2
-#define A6XX_SP_CS_CNTL_0_WGIDCONSTID__MASK			0x000000ff
-#define A6XX_SP_CS_CNTL_0_WGIDCONSTID__SHIFT			0
-static inline uint32_t A6XX_SP_CS_CNTL_0_WGIDCONSTID(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_CNTL_0_WGIDCONSTID__SHIFT) & A6XX_SP_CS_CNTL_0_WGIDCONSTID__MASK;
-}
-#define A6XX_SP_CS_CNTL_0_WGSIZECONSTID__MASK			0x0000ff00
-#define A6XX_SP_CS_CNTL_0_WGSIZECONSTID__SHIFT			8
-static inline uint32_t A6XX_SP_CS_CNTL_0_WGSIZECONSTID(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_CNTL_0_WGSIZECONSTID__SHIFT) & A6XX_SP_CS_CNTL_0_WGSIZECONSTID__MASK;
-}
-#define A6XX_SP_CS_CNTL_0_WGOFFSETCONSTID__MASK			0x00ff0000
-#define A6XX_SP_CS_CNTL_0_WGOFFSETCONSTID__SHIFT		16
-static inline uint32_t A6XX_SP_CS_CNTL_0_WGOFFSETCONSTID(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_CNTL_0_WGOFFSETCONSTID__SHIFT) & A6XX_SP_CS_CNTL_0_WGOFFSETCONSTID__MASK;
-}
-#define A6XX_SP_CS_CNTL_0_LOCALIDREGID__MASK			0xff000000
-#define A6XX_SP_CS_CNTL_0_LOCALIDREGID__SHIFT			24
-static inline uint32_t A6XX_SP_CS_CNTL_0_LOCALIDREGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_CNTL_0_LOCALIDREGID__SHIFT) & A6XX_SP_CS_CNTL_0_LOCALIDREGID__MASK;
-}
-
-#define REG_A6XX_SP_CS_CNTL_1					0x0000a9c3
-#define A6XX_SP_CS_CNTL_1_LINEARLOCALIDREGID__MASK		0x000000ff
-#define A6XX_SP_CS_CNTL_1_LINEARLOCALIDREGID__SHIFT		0
-static inline uint32_t A6XX_SP_CS_CNTL_1_LINEARLOCALIDREGID(uint32_t val)
-{
-	return ((val) << A6XX_SP_CS_CNTL_1_LINEARLOCALIDREGID__SHIFT) & A6XX_SP_CS_CNTL_1_LINEARLOCALIDREGID__MASK;
-}
-#define A6XX_SP_CS_CNTL_1_SINGLE_SP_CORE			0x00000100
-#define A6XX_SP_CS_CNTL_1_THREADSIZE__MASK			0x00000200
-#define A6XX_SP_CS_CNTL_1_THREADSIZE__SHIFT			9
-static inline uint32_t A6XX_SP_CS_CNTL_1_THREADSIZE(enum a6xx_threadsize val)
-{
-	return ((val) << A6XX_SP_CS_CNTL_1_THREADSIZE__SHIFT) & A6XX_SP_CS_CNTL_1_THREADSIZE__MASK;
-}
-#define A6XX_SP_CS_CNTL_1_THREADSIZE_SCALAR			0x00000400
-
-#define REG_A7XX_SP_CS_CNTL_1					0x0000a9c3
-#define A7XX_SP_CS_CNTL_1_LINEARLOCALIDREGID__MASK		0x000000ff
-#define A7XX_SP_CS_CNTL_1_LINEARLOCALIDREGID__SHIFT		0
-static inline uint32_t A7XX_SP_CS_CNTL_1_LINEARLOCALIDREGID(uint32_t val)
-{
-	return ((val) << A7XX_SP_CS_CNTL_1_LINEARLOCALIDREGID__SHIFT) & A7XX_SP_CS_CNTL_1_LINEARLOCALIDREGID__MASK;
-}
-#define A7XX_SP_CS_CNTL_1_THREADSIZE__MASK			0x00000100
-#define A7XX_SP_CS_CNTL_1_THREADSIZE__SHIFT			8
-static inline uint32_t A7XX_SP_CS_CNTL_1_THREADSIZE(enum a6xx_threadsize val)
-{
-	return ((val) << A7XX_SP_CS_CNTL_1_THREADSIZE__SHIFT) & A7XX_SP_CS_CNTL_1_THREADSIZE__MASK;
-}
-#define A7XX_SP_CS_CNTL_1_THREADSIZE_SCALAR			0x00000200
-#define A7XX_SP_CS_CNTL_1_UNK15					0x00008000
-
-#define REG_A6XX_SP_FS_TEX_SAMP					0x0000a9e0
-
-#define REG_A6XX_SP_CS_TEX_SAMP					0x0000a9e2
-
-#define REG_A6XX_SP_FS_TEX_CONST				0x0000a9e4
-
-#define REG_A6XX_SP_CS_TEX_CONST				0x0000a9e6
-
-#define REG_A6XX_SP_CS_BINDLESS_BASE(i0) (0x0000a9e8 + 0x2*(i0))
-
-static inline uint32_t REG_A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR(uint32_t i0) { return 0x0000a9e8 + 0x2*i0; }
-#define A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK	0x00000003
-#define A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT	0
-static inline uint32_t A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE(enum a6xx_bindless_descriptor_size val)
-{
-	return ((val) << A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT) & A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK;
-}
-#define A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK		0xfffffffffffffffc
-#define A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT		2
-static inline uint32_t A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR(uint64_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT) & A6XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK;
-}
-
-#define REG_A7XX_SP_CS_BINDLESS_BASE(i0) (0x0000a9e8 + 0x2*(i0))
-
-static inline uint32_t REG_A7XX_SP_CS_BINDLESS_BASE_DESCRIPTOR(uint32_t i0) { return 0x0000a9e8 + 0x2*i0; }
-#define A7XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK	0x00000003
-#define A7XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT	0
-static inline uint32_t A7XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE(enum a6xx_bindless_descriptor_size val)
-{
-	return ((val) << A7XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT) & A7XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK;
-}
-#define A7XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK		0xfffffffffffffffc
-#define A7XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT		2
-static inline uint32_t A7XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR(uint64_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A7XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT) & A7XX_SP_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK;
-}
-
-#define REG_A6XX_SP_CS_IBO					0x0000a9f2
-
-#define REG_A6XX_SP_CS_IBO_COUNT				0x0000aa00
-
-#define REG_A7XX_SP_FS_VGPR_CONFIG				0x0000aa01
-
-#define REG_A7XX_SP_PS_ALIASED_COMPONENTS_CONTROL		0x0000aa02
-#define A7XX_SP_PS_ALIASED_COMPONENTS_CONTROL_ENABLED		0x00000001
-
-#define REG_A7XX_SP_PS_ALIASED_COMPONENTS			0x0000aa03
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT0__MASK			0x0000000f
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT0__SHIFT		0
-static inline uint32_t A7XX_SP_PS_ALIASED_COMPONENTS_RT0(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_ALIASED_COMPONENTS_RT0__SHIFT) & A7XX_SP_PS_ALIASED_COMPONENTS_RT0__MASK;
-}
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT1__MASK			0x000000f0
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT1__SHIFT		4
-static inline uint32_t A7XX_SP_PS_ALIASED_COMPONENTS_RT1(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_ALIASED_COMPONENTS_RT1__SHIFT) & A7XX_SP_PS_ALIASED_COMPONENTS_RT1__MASK;
-}
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT2__MASK			0x00000f00
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT2__SHIFT		8
-static inline uint32_t A7XX_SP_PS_ALIASED_COMPONENTS_RT2(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_ALIASED_COMPONENTS_RT2__SHIFT) & A7XX_SP_PS_ALIASED_COMPONENTS_RT2__MASK;
-}
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT3__MASK			0x0000f000
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT3__SHIFT		12
-static inline uint32_t A7XX_SP_PS_ALIASED_COMPONENTS_RT3(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_ALIASED_COMPONENTS_RT3__SHIFT) & A7XX_SP_PS_ALIASED_COMPONENTS_RT3__MASK;
-}
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT4__MASK			0x000f0000
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT4__SHIFT		16
-static inline uint32_t A7XX_SP_PS_ALIASED_COMPONENTS_RT4(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_ALIASED_COMPONENTS_RT4__SHIFT) & A7XX_SP_PS_ALIASED_COMPONENTS_RT4__MASK;
-}
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT5__MASK			0x00f00000
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT5__SHIFT		20
-static inline uint32_t A7XX_SP_PS_ALIASED_COMPONENTS_RT5(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_ALIASED_COMPONENTS_RT5__SHIFT) & A7XX_SP_PS_ALIASED_COMPONENTS_RT5__MASK;
-}
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT6__MASK			0x0f000000
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT6__SHIFT		24
-static inline uint32_t A7XX_SP_PS_ALIASED_COMPONENTS_RT6(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_ALIASED_COMPONENTS_RT6__SHIFT) & A7XX_SP_PS_ALIASED_COMPONENTS_RT6__MASK;
-}
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT7__MASK			0xf0000000
-#define A7XX_SP_PS_ALIASED_COMPONENTS_RT7__SHIFT		28
-static inline uint32_t A7XX_SP_PS_ALIASED_COMPONENTS_RT7(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_ALIASED_COMPONENTS_RT7__SHIFT) & A7XX_SP_PS_ALIASED_COMPONENTS_RT7__MASK;
-}
-
-#define REG_A6XX_SP_UNKNOWN_AAF2				0x0000aaf2
-
-#define REG_A6XX_SP_MODE_CONTROL				0x0000ab00
-#define A6XX_SP_MODE_CONTROL_CONSTANT_DEMOTION_ENABLE		0x00000001
-#define A6XX_SP_MODE_CONTROL_ISAMMODE__MASK			0x00000006
-#define A6XX_SP_MODE_CONTROL_ISAMMODE__SHIFT			1
-static inline uint32_t A6XX_SP_MODE_CONTROL_ISAMMODE(enum a6xx_isam_mode val)
-{
-	return ((val) << A6XX_SP_MODE_CONTROL_ISAMMODE__SHIFT) & A6XX_SP_MODE_CONTROL_ISAMMODE__MASK;
-}
-#define A6XX_SP_MODE_CONTROL_SHARED_CONSTS_ENABLE		0x00000008
-
-#define REG_A7XX_SP_UNKNOWN_AB01				0x0000ab01
-
-#define REG_A7XX_SP_UNKNOWN_AB02				0x0000ab02
-
-#define REG_A6XX_SP_FS_CONFIG					0x0000ab04
-#define A6XX_SP_FS_CONFIG_BINDLESS_TEX				0x00000001
-#define A6XX_SP_FS_CONFIG_BINDLESS_SAMP				0x00000002
-#define A6XX_SP_FS_CONFIG_BINDLESS_IBO				0x00000004
-#define A6XX_SP_FS_CONFIG_BINDLESS_UBO				0x00000008
-#define A6XX_SP_FS_CONFIG_ENABLED				0x00000100
-#define A6XX_SP_FS_CONFIG_NTEX__MASK				0x0001fe00
-#define A6XX_SP_FS_CONFIG_NTEX__SHIFT				9
-static inline uint32_t A6XX_SP_FS_CONFIG_NTEX(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_CONFIG_NTEX__SHIFT) & A6XX_SP_FS_CONFIG_NTEX__MASK;
-}
-#define A6XX_SP_FS_CONFIG_NSAMP__MASK				0x003e0000
-#define A6XX_SP_FS_CONFIG_NSAMP__SHIFT				17
-static inline uint32_t A6XX_SP_FS_CONFIG_NSAMP(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_CONFIG_NSAMP__SHIFT) & A6XX_SP_FS_CONFIG_NSAMP__MASK;
-}
-#define A6XX_SP_FS_CONFIG_NIBO__MASK				0x1fc00000
-#define A6XX_SP_FS_CONFIG_NIBO__SHIFT				22
-static inline uint32_t A6XX_SP_FS_CONFIG_NIBO(uint32_t val)
-{
-	return ((val) << A6XX_SP_FS_CONFIG_NIBO__SHIFT) & A6XX_SP_FS_CONFIG_NIBO__MASK;
-}
-
-#define REG_A6XX_SP_FS_INSTRLEN					0x0000ab05
-
-#define REG_A6XX_SP_BINDLESS_BASE(i0) (0x0000ab10 + 0x2*(i0))
-
-static inline uint32_t REG_A6XX_SP_BINDLESS_BASE_DESCRIPTOR(uint32_t i0) { return 0x0000ab10 + 0x2*i0; }
-#define A6XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK	0x00000003
-#define A6XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT	0
-static inline uint32_t A6XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE(enum a6xx_bindless_descriptor_size val)
-{
-	return ((val) << A6XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT) & A6XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK;
-}
-#define A6XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK		0xfffffffffffffffc
-#define A6XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT		2
-static inline uint32_t A6XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR(uint64_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT) & A6XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK;
-}
-
-#define REG_A7XX_SP_BINDLESS_BASE(i0) (0x0000ab0a + 0x2*(i0))
-
-static inline uint32_t REG_A7XX_SP_BINDLESS_BASE_DESCRIPTOR(uint32_t i0) { return 0x0000ab0a + 0x2*i0; }
-#define A7XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK	0x00000003
-#define A7XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT	0
-static inline uint32_t A7XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE(enum a6xx_bindless_descriptor_size val)
-{
-	return ((val) << A7XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT) & A7XX_SP_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK;
-}
-#define A7XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK		0xfffffffffffffffc
-#define A7XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT		2
-static inline uint32_t A7XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR(uint64_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A7XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT) & A7XX_SP_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK;
-}
-
-#define REG_A6XX_SP_IBO						0x0000ab1a
-
-#define REG_A6XX_SP_IBO_COUNT					0x0000ab20
-
-#define REG_A7XX_SP_UNKNOWN_AB22				0x0000ab22
-
-#define REG_A6XX_SP_2D_DST_FORMAT				0x0000acc0
-#define A6XX_SP_2D_DST_FORMAT_NORM				0x00000001
-#define A6XX_SP_2D_DST_FORMAT_SINT				0x00000002
-#define A6XX_SP_2D_DST_FORMAT_UINT				0x00000004
-#define A6XX_SP_2D_DST_FORMAT_COLOR_FORMAT__MASK		0x000007f8
-#define A6XX_SP_2D_DST_FORMAT_COLOR_FORMAT__SHIFT		3
-static inline uint32_t A6XX_SP_2D_DST_FORMAT_COLOR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A6XX_SP_2D_DST_FORMAT_COLOR_FORMAT__SHIFT) & A6XX_SP_2D_DST_FORMAT_COLOR_FORMAT__MASK;
-}
-#define A6XX_SP_2D_DST_FORMAT_SRGB				0x00000800
-#define A6XX_SP_2D_DST_FORMAT_MASK__MASK			0x0000f000
-#define A6XX_SP_2D_DST_FORMAT_MASK__SHIFT			12
-static inline uint32_t A6XX_SP_2D_DST_FORMAT_MASK(uint32_t val)
-{
-	return ((val) << A6XX_SP_2D_DST_FORMAT_MASK__SHIFT) & A6XX_SP_2D_DST_FORMAT_MASK__MASK;
-}
-
-#define REG_A7XX_SP_2D_DST_FORMAT				0x0000a9bf
-#define A7XX_SP_2D_DST_FORMAT_NORM				0x00000001
-#define A7XX_SP_2D_DST_FORMAT_SINT				0x00000002
-#define A7XX_SP_2D_DST_FORMAT_UINT				0x00000004
-#define A7XX_SP_2D_DST_FORMAT_COLOR_FORMAT__MASK		0x000007f8
-#define A7XX_SP_2D_DST_FORMAT_COLOR_FORMAT__SHIFT		3
-static inline uint32_t A7XX_SP_2D_DST_FORMAT_COLOR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A7XX_SP_2D_DST_FORMAT_COLOR_FORMAT__SHIFT) & A7XX_SP_2D_DST_FORMAT_COLOR_FORMAT__MASK;
-}
-#define A7XX_SP_2D_DST_FORMAT_SRGB				0x00000800
-#define A7XX_SP_2D_DST_FORMAT_MASK__MASK			0x0000f000
-#define A7XX_SP_2D_DST_FORMAT_MASK__SHIFT			12
-static inline uint32_t A7XX_SP_2D_DST_FORMAT_MASK(uint32_t val)
-{
-	return ((val) << A7XX_SP_2D_DST_FORMAT_MASK__SHIFT) & A7XX_SP_2D_DST_FORMAT_MASK__MASK;
-}
-
-#define REG_A6XX_SP_DBG_ECO_CNTL				0x0000ae00
-
-#define REG_A6XX_SP_ADDR_MODE_CNTL				0x0000ae01
-
-#define REG_A6XX_SP_NC_MODE_CNTL				0x0000ae02
-
-#define REG_A6XX_SP_CHICKEN_BITS				0x0000ae03
-
-#define REG_A6XX_SP_FLOAT_CNTL					0x0000ae04
-#define A6XX_SP_FLOAT_CNTL_F16_NO_INF				0x00000008
-
-#define REG_A7XX_SP_UNKNOWN_AE06				0x0000ae06
-
-#define REG_A7XX_SP_UNKNOWN_AE08				0x0000ae08
-
-#define REG_A7XX_SP_UNKNOWN_AE09				0x0000ae09
-
-#define REG_A7XX_SP_UNKNOWN_AE0A				0x0000ae0a
-
-#define REG_A6XX_SP_PERFCTR_ENABLE				0x0000ae0f
-#define A6XX_SP_PERFCTR_ENABLE_VS				0x00000001
-#define A6XX_SP_PERFCTR_ENABLE_HS				0x00000002
-#define A6XX_SP_PERFCTR_ENABLE_DS				0x00000004
-#define A6XX_SP_PERFCTR_ENABLE_GS				0x00000008
-#define A6XX_SP_PERFCTR_ENABLE_FS				0x00000010
-#define A6XX_SP_PERFCTR_ENABLE_CS				0x00000020
-
-#define REG_A6XX_SP_PERFCTR_SP_SEL(i0) (0x0000ae10 + 0x1*(i0))
-
-#define REG_A7XX_SP_PERFCTR_HLSQ_SEL(i0) (0x0000ae60 + 0x1*(i0))
-
-#define REG_A7XX_SP_UNKNOWN_AE6A				0x0000ae6a
-
-#define REG_A7XX_SP_UNKNOWN_AE6B				0x0000ae6b
-
-#define REG_A7XX_SP_UNKNOWN_AE6C				0x0000ae6c
-
-#define REG_A7XX_SP_READ_SEL					0x0000ae6d
-#define A7XX_SP_READ_SEL_LOCATION__MASK				0x000c0000
-#define A7XX_SP_READ_SEL_LOCATION__SHIFT			18
-static inline uint32_t A7XX_SP_READ_SEL_LOCATION(enum a7xx_state_location val)
-{
-	return ((val) << A7XX_SP_READ_SEL_LOCATION__SHIFT) & A7XX_SP_READ_SEL_LOCATION__MASK;
-}
-#define A7XX_SP_READ_SEL_PIPE__MASK				0x00030000
-#define A7XX_SP_READ_SEL_PIPE__SHIFT				16
-static inline uint32_t A7XX_SP_READ_SEL_PIPE(enum a7xx_pipe val)
-{
-	return ((val) << A7XX_SP_READ_SEL_PIPE__SHIFT) & A7XX_SP_READ_SEL_PIPE__MASK;
-}
-#define A7XX_SP_READ_SEL_STATETYPE__MASK			0x0000ff00
-#define A7XX_SP_READ_SEL_STATETYPE__SHIFT			8
-static inline uint32_t A7XX_SP_READ_SEL_STATETYPE(enum a7xx_statetype_id val)
-{
-	return ((val) << A7XX_SP_READ_SEL_STATETYPE__SHIFT) & A7XX_SP_READ_SEL_STATETYPE__MASK;
-}
-#define A7XX_SP_READ_SEL_USPTP__MASK				0x000000f0
-#define A7XX_SP_READ_SEL_USPTP__SHIFT				4
-static inline uint32_t A7XX_SP_READ_SEL_USPTP(uint32_t val)
-{
-	return ((val) << A7XX_SP_READ_SEL_USPTP__SHIFT) & A7XX_SP_READ_SEL_USPTP__MASK;
-}
-#define A7XX_SP_READ_SEL_SPTP__MASK				0x0000000f
-#define A7XX_SP_READ_SEL_SPTP__SHIFT				0
-static inline uint32_t A7XX_SP_READ_SEL_SPTP(uint32_t val)
-{
-	return ((val) << A7XX_SP_READ_SEL_SPTP__SHIFT) & A7XX_SP_READ_SEL_SPTP__MASK;
-}
-
-#define REG_A7XX_SP_DBG_CNTL					0x0000ae71
-
-#define REG_A7XX_SP_UNKNOWN_AE73				0x0000ae73
-
-#define REG_A7XX_SP_PERFCTR_SP_SEL(i0) (0x0000ae80 + 0x1*(i0))
-
-#define REG_A6XX_SP_CONTEXT_SWITCH_GFX_PREEMPTION_SAFE_MODE	0x0000be22
-
-#define REG_A6XX_SP_PS_TP_BORDER_COLOR_BASE_ADDR		0x0000b180
-
-#define REG_A6XX_SP_UNKNOWN_B182				0x0000b182
-
-#define REG_A6XX_SP_UNKNOWN_B183				0x0000b183
-
-#define REG_A6XX_SP_UNKNOWN_B190				0x0000b190
-
-#define REG_A6XX_SP_UNKNOWN_B191				0x0000b191
-
-#define REG_A6XX_SP_TP_RAS_MSAA_CNTL				0x0000b300
-#define A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES__MASK			0x00000003
-#define A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES__SHIFT			0
-static inline uint32_t A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES__MASK;
-}
-#define A6XX_SP_TP_RAS_MSAA_CNTL_UNK2__MASK			0x0000000c
-#define A6XX_SP_TP_RAS_MSAA_CNTL_UNK2__SHIFT			2
-static inline uint32_t A6XX_SP_TP_RAS_MSAA_CNTL_UNK2(uint32_t val)
-{
-	return ((val) << A6XX_SP_TP_RAS_MSAA_CNTL_UNK2__SHIFT) & A6XX_SP_TP_RAS_MSAA_CNTL_UNK2__MASK;
-}
-
-#define REG_A6XX_SP_TP_DEST_MSAA_CNTL				0x0000b301
-#define A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES__MASK			0x00000003
-#define A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES__SHIFT		0
-static inline uint32_t A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES__MASK;
-}
-#define A6XX_SP_TP_DEST_MSAA_CNTL_MSAA_DISABLE			0x00000004
-
-#define REG_A6XX_SP_TP_BORDER_COLOR_BASE_ADDR			0x0000b302
-
-#define REG_A6XX_SP_TP_SAMPLE_CONFIG				0x0000b304
-#define A6XX_SP_TP_SAMPLE_CONFIG_UNK0				0x00000001
-#define A6XX_SP_TP_SAMPLE_CONFIG_LOCATION_ENABLE		0x00000002
-
-#define REG_A6XX_SP_TP_SAMPLE_LOCATION_0			0x0000b305
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_0_X__MASK		0x0000000f
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_0_X__SHIFT		0
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_0_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_0_X__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_0_X__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_0_Y__MASK		0x000000f0
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_0_Y__SHIFT		4
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_0_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_0_Y__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_0_Y__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_1_X__MASK		0x00000f00
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_1_X__SHIFT		8
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_1_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_1_X__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_1_X__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_1_Y__MASK		0x0000f000
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_1_Y__SHIFT		12
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_1_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_1_Y__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_1_Y__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_2_X__MASK		0x000f0000
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_2_X__SHIFT		16
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_2_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_2_X__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_2_X__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_2_Y__MASK		0x00f00000
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_2_Y__SHIFT		20
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_2_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_2_Y__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_2_Y__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_3_X__MASK		0x0f000000
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_3_X__SHIFT		24
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_3_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_3_X__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_3_X__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_3_Y__MASK		0xf0000000
-#define A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_3_Y__SHIFT		28
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_3_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_3_Y__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_0_SAMPLE_3_Y__MASK;
-}
-
-#define REG_A6XX_SP_TP_SAMPLE_LOCATION_1			0x0000b306
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_0_X__MASK		0x0000000f
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_0_X__SHIFT		0
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_0_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_0_X__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_0_X__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_0_Y__MASK		0x000000f0
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_0_Y__SHIFT		4
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_0_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_0_Y__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_0_Y__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_1_X__MASK		0x00000f00
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_1_X__SHIFT		8
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_1_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_1_X__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_1_X__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_1_Y__MASK		0x0000f000
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_1_Y__SHIFT		12
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_1_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_1_Y__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_1_Y__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_2_X__MASK		0x000f0000
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_2_X__SHIFT		16
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_2_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_2_X__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_2_X__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_2_Y__MASK		0x00f00000
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_2_Y__SHIFT		20
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_2_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_2_Y__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_2_Y__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_3_X__MASK		0x0f000000
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_3_X__SHIFT		24
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_3_X(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_3_X__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_3_X__MASK;
-}
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_3_Y__MASK		0xf0000000
-#define A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_3_Y__SHIFT		28
-static inline uint32_t A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_3_Y(float val)
-{
-	return ((((int32_t)(val * 16.0))) << A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_3_Y__SHIFT) & A6XX_SP_TP_SAMPLE_LOCATION_1_SAMPLE_3_Y__MASK;
-}
-
-#define REG_A6XX_SP_TP_WINDOW_OFFSET				0x0000b307
-#define A6XX_SP_TP_WINDOW_OFFSET_X__MASK			0x00003fff
-#define A6XX_SP_TP_WINDOW_OFFSET_X__SHIFT			0
-static inline uint32_t A6XX_SP_TP_WINDOW_OFFSET_X(uint32_t val)
-{
-	return ((val) << A6XX_SP_TP_WINDOW_OFFSET_X__SHIFT) & A6XX_SP_TP_WINDOW_OFFSET_X__MASK;
-}
-#define A6XX_SP_TP_WINDOW_OFFSET_Y__MASK			0x3fff0000
-#define A6XX_SP_TP_WINDOW_OFFSET_Y__SHIFT			16
-static inline uint32_t A6XX_SP_TP_WINDOW_OFFSET_Y(uint32_t val)
-{
-	return ((val) << A6XX_SP_TP_WINDOW_OFFSET_Y__SHIFT) & A6XX_SP_TP_WINDOW_OFFSET_Y__MASK;
-}
-
-#define REG_A6XX_SP_TP_MODE_CNTL				0x0000b309
-#define A6XX_SP_TP_MODE_CNTL_ISAMMODE__MASK			0x00000003
-#define A6XX_SP_TP_MODE_CNTL_ISAMMODE__SHIFT			0
-static inline uint32_t A6XX_SP_TP_MODE_CNTL_ISAMMODE(enum a6xx_isam_mode val)
-{
-	return ((val) << A6XX_SP_TP_MODE_CNTL_ISAMMODE__SHIFT) & A6XX_SP_TP_MODE_CNTL_ISAMMODE__MASK;
-}
-#define A6XX_SP_TP_MODE_CNTL_UNK3__MASK				0x000000fc
-#define A6XX_SP_TP_MODE_CNTL_UNK3__SHIFT			2
-static inline uint32_t A6XX_SP_TP_MODE_CNTL_UNK3(uint32_t val)
-{
-	return ((val) << A6XX_SP_TP_MODE_CNTL_UNK3__SHIFT) & A6XX_SP_TP_MODE_CNTL_UNK3__MASK;
-}
-
-#define REG_A7XX_SP_UNKNOWN_B310				0x0000b310
-
-#define REG_A6XX_SP_PS_2D_SRC_INFO				0x0000b4c0
-#define A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK		0x000000ff
-#define A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__SHIFT		0
-static inline uint32_t A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__SHIFT) & A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK;
-}
-#define A6XX_SP_PS_2D_SRC_INFO_TILE_MODE__MASK			0x00000300
-#define A6XX_SP_PS_2D_SRC_INFO_TILE_MODE__SHIFT			8
-static inline uint32_t A6XX_SP_PS_2D_SRC_INFO_TILE_MODE(enum a6xx_tile_mode val)
-{
-	return ((val) << A6XX_SP_PS_2D_SRC_INFO_TILE_MODE__SHIFT) & A6XX_SP_PS_2D_SRC_INFO_TILE_MODE__MASK;
-}
-#define A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__MASK			0x00000c00
-#define A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__SHIFT		10
-static inline uint32_t A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__SHIFT) & A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__MASK;
-}
-#define A6XX_SP_PS_2D_SRC_INFO_FLAGS				0x00001000
-#define A6XX_SP_PS_2D_SRC_INFO_SRGB				0x00002000
-#define A6XX_SP_PS_2D_SRC_INFO_SAMPLES__MASK			0x0000c000
-#define A6XX_SP_PS_2D_SRC_INFO_SAMPLES__SHIFT			14
-static inline uint32_t A6XX_SP_PS_2D_SRC_INFO_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A6XX_SP_PS_2D_SRC_INFO_SAMPLES__SHIFT) & A6XX_SP_PS_2D_SRC_INFO_SAMPLES__MASK;
-}
-#define A6XX_SP_PS_2D_SRC_INFO_FILTER				0x00010000
-#define A6XX_SP_PS_2D_SRC_INFO_UNK17				0x00020000
-#define A6XX_SP_PS_2D_SRC_INFO_SAMPLES_AVERAGE			0x00040000
-#define A6XX_SP_PS_2D_SRC_INFO_UNK19				0x00080000
-#define A6XX_SP_PS_2D_SRC_INFO_UNK20				0x00100000
-#define A6XX_SP_PS_2D_SRC_INFO_UNK21				0x00200000
-#define A6XX_SP_PS_2D_SRC_INFO_UNK22				0x00400000
-#define A6XX_SP_PS_2D_SRC_INFO_UNK23__MASK			0x07800000
-#define A6XX_SP_PS_2D_SRC_INFO_UNK23__SHIFT			23
-static inline uint32_t A6XX_SP_PS_2D_SRC_INFO_UNK23(uint32_t val)
-{
-	return ((val) << A6XX_SP_PS_2D_SRC_INFO_UNK23__SHIFT) & A6XX_SP_PS_2D_SRC_INFO_UNK23__MASK;
-}
-#define A6XX_SP_PS_2D_SRC_INFO_UNK28				0x10000000
-
-#define REG_A6XX_SP_PS_2D_SRC_SIZE				0x0000b4c1
-#define A6XX_SP_PS_2D_SRC_SIZE_WIDTH__MASK			0x00007fff
-#define A6XX_SP_PS_2D_SRC_SIZE_WIDTH__SHIFT			0
-static inline uint32_t A6XX_SP_PS_2D_SRC_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << A6XX_SP_PS_2D_SRC_SIZE_WIDTH__SHIFT) & A6XX_SP_PS_2D_SRC_SIZE_WIDTH__MASK;
-}
-#define A6XX_SP_PS_2D_SRC_SIZE_HEIGHT__MASK			0x3fff8000
-#define A6XX_SP_PS_2D_SRC_SIZE_HEIGHT__SHIFT			15
-static inline uint32_t A6XX_SP_PS_2D_SRC_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << A6XX_SP_PS_2D_SRC_SIZE_HEIGHT__SHIFT) & A6XX_SP_PS_2D_SRC_SIZE_HEIGHT__MASK;
-}
-
-#define REG_A6XX_SP_PS_2D_SRC					0x0000b4c2
-
-#define REG_A6XX_SP_PS_2D_SRC_PITCH				0x0000b4c4
-#define A6XX_SP_PS_2D_SRC_PITCH_UNK0__MASK			0x000001ff
-#define A6XX_SP_PS_2D_SRC_PITCH_UNK0__SHIFT			0
-static inline uint32_t A6XX_SP_PS_2D_SRC_PITCH_UNK0(uint32_t val)
-{
-	return ((val) << A6XX_SP_PS_2D_SRC_PITCH_UNK0__SHIFT) & A6XX_SP_PS_2D_SRC_PITCH_UNK0__MASK;
-}
-#define A6XX_SP_PS_2D_SRC_PITCH_PITCH__MASK			0x00fffe00
-#define A6XX_SP_PS_2D_SRC_PITCH_PITCH__SHIFT			9
-static inline uint32_t A6XX_SP_PS_2D_SRC_PITCH_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_SP_PS_2D_SRC_PITCH_PITCH__SHIFT) & A6XX_SP_PS_2D_SRC_PITCH_PITCH__MASK;
-}
-
-#define REG_A7XX_SP_PS_2D_SRC_INFO				0x0000b2c0
-#define A7XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK		0x000000ff
-#define A7XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__SHIFT		0
-static inline uint32_t A7XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(enum a6xx_format val)
-{
-	return ((val) << A7XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__SHIFT) & A7XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK;
-}
-#define A7XX_SP_PS_2D_SRC_INFO_TILE_MODE__MASK			0x00000300
-#define A7XX_SP_PS_2D_SRC_INFO_TILE_MODE__SHIFT			8
-static inline uint32_t A7XX_SP_PS_2D_SRC_INFO_TILE_MODE(enum a6xx_tile_mode val)
-{
-	return ((val) << A7XX_SP_PS_2D_SRC_INFO_TILE_MODE__SHIFT) & A7XX_SP_PS_2D_SRC_INFO_TILE_MODE__MASK;
-}
-#define A7XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__MASK			0x00000c00
-#define A7XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__SHIFT		10
-static inline uint32_t A7XX_SP_PS_2D_SRC_INFO_COLOR_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A7XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__SHIFT) & A7XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__MASK;
-}
-#define A7XX_SP_PS_2D_SRC_INFO_FLAGS				0x00001000
-#define A7XX_SP_PS_2D_SRC_INFO_SRGB				0x00002000
-#define A7XX_SP_PS_2D_SRC_INFO_SAMPLES__MASK			0x0000c000
-#define A7XX_SP_PS_2D_SRC_INFO_SAMPLES__SHIFT			14
-static inline uint32_t A7XX_SP_PS_2D_SRC_INFO_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A7XX_SP_PS_2D_SRC_INFO_SAMPLES__SHIFT) & A7XX_SP_PS_2D_SRC_INFO_SAMPLES__MASK;
-}
-#define A7XX_SP_PS_2D_SRC_INFO_FILTER				0x00010000
-#define A7XX_SP_PS_2D_SRC_INFO_UNK17				0x00020000
-#define A7XX_SP_PS_2D_SRC_INFO_SAMPLES_AVERAGE			0x00040000
-#define A7XX_SP_PS_2D_SRC_INFO_UNK19				0x00080000
-#define A7XX_SP_PS_2D_SRC_INFO_UNK20				0x00100000
-#define A7XX_SP_PS_2D_SRC_INFO_UNK21				0x00200000
-#define A7XX_SP_PS_2D_SRC_INFO_UNK22				0x00400000
-#define A7XX_SP_PS_2D_SRC_INFO_UNK23__MASK			0x07800000
-#define A7XX_SP_PS_2D_SRC_INFO_UNK23__SHIFT			23
-static inline uint32_t A7XX_SP_PS_2D_SRC_INFO_UNK23(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_2D_SRC_INFO_UNK23__SHIFT) & A7XX_SP_PS_2D_SRC_INFO_UNK23__MASK;
-}
-#define A7XX_SP_PS_2D_SRC_INFO_UNK28				0x10000000
-
-#define REG_A7XX_SP_PS_2D_SRC_SIZE				0x0000b2c1
-#define A7XX_SP_PS_2D_SRC_SIZE_WIDTH__MASK			0x00007fff
-#define A7XX_SP_PS_2D_SRC_SIZE_WIDTH__SHIFT			0
-static inline uint32_t A7XX_SP_PS_2D_SRC_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_2D_SRC_SIZE_WIDTH__SHIFT) & A7XX_SP_PS_2D_SRC_SIZE_WIDTH__MASK;
-}
-#define A7XX_SP_PS_2D_SRC_SIZE_HEIGHT__MASK			0x3fff8000
-#define A7XX_SP_PS_2D_SRC_SIZE_HEIGHT__SHIFT			15
-static inline uint32_t A7XX_SP_PS_2D_SRC_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_2D_SRC_SIZE_HEIGHT__SHIFT) & A7XX_SP_PS_2D_SRC_SIZE_HEIGHT__MASK;
-}
-
-#define REG_A7XX_SP_PS_2D_SRC					0x0000b2c2
-
-#define REG_A7XX_SP_PS_2D_SRC_PITCH				0x0000b2c4
-#define A7XX_SP_PS_2D_SRC_PITCH_UNK0__MASK			0x000001ff
-#define A7XX_SP_PS_2D_SRC_PITCH_UNK0__SHIFT			0
-static inline uint32_t A7XX_SP_PS_2D_SRC_PITCH_UNK0(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_2D_SRC_PITCH_UNK0__SHIFT) & A7XX_SP_PS_2D_SRC_PITCH_UNK0__MASK;
-}
-#define A7XX_SP_PS_2D_SRC_PITCH_PITCH__MASK			0x00fffe00
-#define A7XX_SP_PS_2D_SRC_PITCH_PITCH__SHIFT			9
-static inline uint32_t A7XX_SP_PS_2D_SRC_PITCH_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A7XX_SP_PS_2D_SRC_PITCH_PITCH__SHIFT) & A7XX_SP_PS_2D_SRC_PITCH_PITCH__MASK;
-}
-
-#define REG_A6XX_SP_PS_2D_SRC_PLANE1				0x0000b4c5
-
-#define REG_A6XX_SP_PS_2D_SRC_PLANE_PITCH			0x0000b4c7
-#define A6XX_SP_PS_2D_SRC_PLANE_PITCH__MASK			0x00000fff
-#define A6XX_SP_PS_2D_SRC_PLANE_PITCH__SHIFT			0
-static inline uint32_t A6XX_SP_PS_2D_SRC_PLANE_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_SP_PS_2D_SRC_PLANE_PITCH__SHIFT) & A6XX_SP_PS_2D_SRC_PLANE_PITCH__MASK;
-}
-
-#define REG_A6XX_SP_PS_2D_SRC_PLANE2				0x0000b4c8
-
-#define REG_A7XX_SP_PS_2D_SRC_PLANE1				0x0000b2c5
-
-#define REG_A7XX_SP_PS_2D_SRC_PLANE_PITCH			0x0000b2c7
-#define A7XX_SP_PS_2D_SRC_PLANE_PITCH__MASK			0x00000fff
-#define A7XX_SP_PS_2D_SRC_PLANE_PITCH__SHIFT			0
-static inline uint32_t A7XX_SP_PS_2D_SRC_PLANE_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A7XX_SP_PS_2D_SRC_PLANE_PITCH__SHIFT) & A7XX_SP_PS_2D_SRC_PLANE_PITCH__MASK;
-}
-
-#define REG_A7XX_SP_PS_2D_SRC_PLANE2				0x0000b2c8
-
-#define REG_A6XX_SP_PS_2D_SRC_FLAGS				0x0000b4ca
-
-#define REG_A6XX_SP_PS_2D_SRC_FLAGS_PITCH			0x0000b4cc
-#define A6XX_SP_PS_2D_SRC_FLAGS_PITCH__MASK			0x000000ff
-#define A6XX_SP_PS_2D_SRC_FLAGS_PITCH__SHIFT			0
-static inline uint32_t A6XX_SP_PS_2D_SRC_FLAGS_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_SP_PS_2D_SRC_FLAGS_PITCH__SHIFT) & A6XX_SP_PS_2D_SRC_FLAGS_PITCH__MASK;
-}
-
-#define REG_A7XX_SP_PS_2D_SRC_FLAGS				0x0000b2ca
-
-#define REG_A7XX_SP_PS_2D_SRC_FLAGS_PITCH			0x0000b2cc
-#define A7XX_SP_PS_2D_SRC_FLAGS_PITCH__MASK			0x000000ff
-#define A7XX_SP_PS_2D_SRC_FLAGS_PITCH__SHIFT			0
-static inline uint32_t A7XX_SP_PS_2D_SRC_FLAGS_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A7XX_SP_PS_2D_SRC_FLAGS_PITCH__SHIFT) & A7XX_SP_PS_2D_SRC_FLAGS_PITCH__MASK;
-}
-
-#define REG_A6XX_SP_PS_UNKNOWN_B4CD				0x0000b4cd
-
-#define REG_A6XX_SP_PS_UNKNOWN_B4CE				0x0000b4ce
-
-#define REG_A6XX_SP_PS_UNKNOWN_B4CF				0x0000b4cf
-
-#define REG_A6XX_SP_PS_UNKNOWN_B4D0				0x0000b4d0
-
-#define REG_A6XX_SP_WINDOW_OFFSET				0x0000b4d1
-#define A6XX_SP_WINDOW_OFFSET_X__MASK				0x00003fff
-#define A6XX_SP_WINDOW_OFFSET_X__SHIFT				0
-static inline uint32_t A6XX_SP_WINDOW_OFFSET_X(uint32_t val)
-{
-	return ((val) << A6XX_SP_WINDOW_OFFSET_X__SHIFT) & A6XX_SP_WINDOW_OFFSET_X__MASK;
-}
-#define A6XX_SP_WINDOW_OFFSET_Y__MASK				0x3fff0000
-#define A6XX_SP_WINDOW_OFFSET_Y__SHIFT				16
-static inline uint32_t A6XX_SP_WINDOW_OFFSET_Y(uint32_t val)
-{
-	return ((val) << A6XX_SP_WINDOW_OFFSET_Y__SHIFT) & A6XX_SP_WINDOW_OFFSET_Y__MASK;
-}
-
-#define REG_A7XX_SP_PS_UNKNOWN_B4CD				0x0000b2cd
-
-#define REG_A7XX_SP_PS_UNKNOWN_B4CE				0x0000b2ce
-
-#define REG_A7XX_SP_PS_UNKNOWN_B4CF				0x0000b2cf
-
-#define REG_A7XX_SP_PS_UNKNOWN_B4D0				0x0000b2d0
-
-#define REG_A7XX_SP_PS_2D_WINDOW_OFFSET				0x0000b2d1
-#define A7XX_SP_PS_2D_WINDOW_OFFSET_X__MASK			0x00003fff
-#define A7XX_SP_PS_2D_WINDOW_OFFSET_X__SHIFT			0
-static inline uint32_t A7XX_SP_PS_2D_WINDOW_OFFSET_X(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_2D_WINDOW_OFFSET_X__SHIFT) & A7XX_SP_PS_2D_WINDOW_OFFSET_X__MASK;
-}
-#define A7XX_SP_PS_2D_WINDOW_OFFSET_Y__MASK			0x3fff0000
-#define A7XX_SP_PS_2D_WINDOW_OFFSET_Y__SHIFT			16
-static inline uint32_t A7XX_SP_PS_2D_WINDOW_OFFSET_Y(uint32_t val)
-{
-	return ((val) << A7XX_SP_PS_2D_WINDOW_OFFSET_Y__SHIFT) & A7XX_SP_PS_2D_WINDOW_OFFSET_Y__MASK;
-}
-
-#define REG_A7XX_SP_PS_UNKNOWN_B2D2				0x0000b2d2
-
-#define REG_A7XX_SP_WINDOW_OFFSET				0x0000ab21
-#define A7XX_SP_WINDOW_OFFSET_X__MASK				0x00003fff
-#define A7XX_SP_WINDOW_OFFSET_X__SHIFT				0
-static inline uint32_t A7XX_SP_WINDOW_OFFSET_X(uint32_t val)
-{
-	return ((val) << A7XX_SP_WINDOW_OFFSET_X__SHIFT) & A7XX_SP_WINDOW_OFFSET_X__MASK;
-}
-#define A7XX_SP_WINDOW_OFFSET_Y__MASK				0x3fff0000
-#define A7XX_SP_WINDOW_OFFSET_Y__SHIFT				16
-static inline uint32_t A7XX_SP_WINDOW_OFFSET_Y(uint32_t val)
-{
-	return ((val) << A7XX_SP_WINDOW_OFFSET_Y__SHIFT) & A7XX_SP_WINDOW_OFFSET_Y__MASK;
-}
-
-#define REG_A6XX_TPL1_DBG_ECO_CNTL				0x0000b600
-
-#define REG_A6XX_TPL1_ADDR_MODE_CNTL				0x0000b601
-
-#define REG_A6XX_TPL1_UNKNOWN_B602				0x0000b602
-
-#define REG_A6XX_TPL1_NC_MODE_CNTL				0x0000b604
-#define A6XX_TPL1_NC_MODE_CNTL_MODE				0x00000001
-#define A6XX_TPL1_NC_MODE_CNTL_LOWER_BIT__MASK			0x00000006
-#define A6XX_TPL1_NC_MODE_CNTL_LOWER_BIT__SHIFT			1
-static inline uint32_t A6XX_TPL1_NC_MODE_CNTL_LOWER_BIT(uint32_t val)
-{
-	return ((val) << A6XX_TPL1_NC_MODE_CNTL_LOWER_BIT__SHIFT) & A6XX_TPL1_NC_MODE_CNTL_LOWER_BIT__MASK;
-}
-#define A6XX_TPL1_NC_MODE_CNTL_MIN_ACCESS_LENGTH		0x00000008
-#define A6XX_TPL1_NC_MODE_CNTL_UPPER_BIT__MASK			0x00000010
-#define A6XX_TPL1_NC_MODE_CNTL_UPPER_BIT__SHIFT			4
-static inline uint32_t A6XX_TPL1_NC_MODE_CNTL_UPPER_BIT(uint32_t val)
-{
-	return ((val) << A6XX_TPL1_NC_MODE_CNTL_UPPER_BIT__SHIFT) & A6XX_TPL1_NC_MODE_CNTL_UPPER_BIT__MASK;
-}
-#define A6XX_TPL1_NC_MODE_CNTL_UNK6__MASK			0x000000c0
-#define A6XX_TPL1_NC_MODE_CNTL_UNK6__SHIFT			6
-static inline uint32_t A6XX_TPL1_NC_MODE_CNTL_UNK6(uint32_t val)
-{
-	return ((val) << A6XX_TPL1_NC_MODE_CNTL_UNK6__SHIFT) & A6XX_TPL1_NC_MODE_CNTL_UNK6__MASK;
-}
-
-#define REG_A6XX_TPL1_UNKNOWN_B605				0x0000b605
-
-#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0			0x0000b608
-
-#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1			0x0000b609
-
-#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2			0x0000b60a
-
-#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3			0x0000b60b
-
-#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4			0x0000b60c
-
-#define REG_A7XX_TPL1_BICUBIC_WEIGHTS_TABLE_0			0x0000b608
-
-#define REG_A7XX_TPL1_BICUBIC_WEIGHTS_TABLE_1			0x0000b609
-
-#define REG_A7XX_TPL1_BICUBIC_WEIGHTS_TABLE_2			0x0000b60a
-
-#define REG_A7XX_TPL1_BICUBIC_WEIGHTS_TABLE_3			0x0000b60b
-
-#define REG_A7XX_TPL1_BICUBIC_WEIGHTS_TABLE_4			0x0000b60c
-
-#define REG_A6XX_TPL1_PERFCTR_TP_SEL(i0) (0x0000b610 + 0x1*(i0))
-
-#define REG_A6XX_HLSQ_VS_CNTL					0x0000b800
-#define A6XX_HLSQ_VS_CNTL_CONSTLEN__MASK			0x000000ff
-#define A6XX_HLSQ_VS_CNTL_CONSTLEN__SHIFT			0
-static inline uint32_t A6XX_HLSQ_VS_CNTL_CONSTLEN(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_HLSQ_VS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_VS_CNTL_CONSTLEN__MASK;
-}
-#define A6XX_HLSQ_VS_CNTL_ENABLED				0x00000100
-#define A6XX_HLSQ_VS_CNTL_READ_IMM_SHARED_CONSTS		0x00000200
-
-#define REG_A6XX_HLSQ_HS_CNTL					0x0000b801
-#define A6XX_HLSQ_HS_CNTL_CONSTLEN__MASK			0x000000ff
-#define A6XX_HLSQ_HS_CNTL_CONSTLEN__SHIFT			0
-static inline uint32_t A6XX_HLSQ_HS_CNTL_CONSTLEN(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_HLSQ_HS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_HS_CNTL_CONSTLEN__MASK;
-}
-#define A6XX_HLSQ_HS_CNTL_ENABLED				0x00000100
-#define A6XX_HLSQ_HS_CNTL_READ_IMM_SHARED_CONSTS		0x00000200
-
-#define REG_A6XX_HLSQ_DS_CNTL					0x0000b802
-#define A6XX_HLSQ_DS_CNTL_CONSTLEN__MASK			0x000000ff
-#define A6XX_HLSQ_DS_CNTL_CONSTLEN__SHIFT			0
-static inline uint32_t A6XX_HLSQ_DS_CNTL_CONSTLEN(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_HLSQ_DS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_DS_CNTL_CONSTLEN__MASK;
-}
-#define A6XX_HLSQ_DS_CNTL_ENABLED				0x00000100
-#define A6XX_HLSQ_DS_CNTL_READ_IMM_SHARED_CONSTS		0x00000200
-
-#define REG_A6XX_HLSQ_GS_CNTL					0x0000b803
-#define A6XX_HLSQ_GS_CNTL_CONSTLEN__MASK			0x000000ff
-#define A6XX_HLSQ_GS_CNTL_CONSTLEN__SHIFT			0
-static inline uint32_t A6XX_HLSQ_GS_CNTL_CONSTLEN(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_HLSQ_GS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_GS_CNTL_CONSTLEN__MASK;
-}
-#define A6XX_HLSQ_GS_CNTL_ENABLED				0x00000100
-#define A6XX_HLSQ_GS_CNTL_READ_IMM_SHARED_CONSTS		0x00000200
-
-#define REG_A7XX_HLSQ_VS_CNTL					0x0000a827
-#define A7XX_HLSQ_VS_CNTL_CONSTLEN__MASK			0x000000ff
-#define A7XX_HLSQ_VS_CNTL_CONSTLEN__SHIFT			0
-static inline uint32_t A7XX_HLSQ_VS_CNTL_CONSTLEN(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A7XX_HLSQ_VS_CNTL_CONSTLEN__SHIFT) & A7XX_HLSQ_VS_CNTL_CONSTLEN__MASK;
-}
-#define A7XX_HLSQ_VS_CNTL_ENABLED				0x00000100
-#define A7XX_HLSQ_VS_CNTL_READ_IMM_SHARED_CONSTS		0x00000200
-
-#define REG_A7XX_HLSQ_HS_CNTL					0x0000a83f
-#define A7XX_HLSQ_HS_CNTL_CONSTLEN__MASK			0x000000ff
-#define A7XX_HLSQ_HS_CNTL_CONSTLEN__SHIFT			0
-static inline uint32_t A7XX_HLSQ_HS_CNTL_CONSTLEN(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A7XX_HLSQ_HS_CNTL_CONSTLEN__SHIFT) & A7XX_HLSQ_HS_CNTL_CONSTLEN__MASK;
-}
-#define A7XX_HLSQ_HS_CNTL_ENABLED				0x00000100
-#define A7XX_HLSQ_HS_CNTL_READ_IMM_SHARED_CONSTS		0x00000200
-
-#define REG_A7XX_HLSQ_DS_CNTL					0x0000a867
-#define A7XX_HLSQ_DS_CNTL_CONSTLEN__MASK			0x000000ff
-#define A7XX_HLSQ_DS_CNTL_CONSTLEN__SHIFT			0
-static inline uint32_t A7XX_HLSQ_DS_CNTL_CONSTLEN(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A7XX_HLSQ_DS_CNTL_CONSTLEN__SHIFT) & A7XX_HLSQ_DS_CNTL_CONSTLEN__MASK;
-}
-#define A7XX_HLSQ_DS_CNTL_ENABLED				0x00000100
-#define A7XX_HLSQ_DS_CNTL_READ_IMM_SHARED_CONSTS		0x00000200
-
-#define REG_A7XX_HLSQ_GS_CNTL					0x0000a898
-#define A7XX_HLSQ_GS_CNTL_CONSTLEN__MASK			0x000000ff
-#define A7XX_HLSQ_GS_CNTL_CONSTLEN__SHIFT			0
-static inline uint32_t A7XX_HLSQ_GS_CNTL_CONSTLEN(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A7XX_HLSQ_GS_CNTL_CONSTLEN__SHIFT) & A7XX_HLSQ_GS_CNTL_CONSTLEN__MASK;
-}
-#define A7XX_HLSQ_GS_CNTL_ENABLED				0x00000100
-#define A7XX_HLSQ_GS_CNTL_READ_IMM_SHARED_CONSTS		0x00000200
-
-#define REG_A7XX_HLSQ_FS_UNKNOWN_A9AA				0x0000a9aa
-#define A7XX_HLSQ_FS_UNKNOWN_A9AA_CONSTS_LOAD_DISABLE		0x00000001
-
-#define REG_A7XX_HLSQ_UNKNOWN_A9AC				0x0000a9ac
-
-#define REG_A7XX_HLSQ_UNKNOWN_A9AD				0x0000a9ad
-
-#define REG_A7XX_HLSQ_UNKNOWN_A9AE				0x0000a9ae
-#define A7XX_HLSQ_UNKNOWN_A9AE_SYSVAL_REGS_COUNT__MASK		0x000000ff
-#define A7XX_HLSQ_UNKNOWN_A9AE_SYSVAL_REGS_COUNT__SHIFT		0
-static inline uint32_t A7XX_HLSQ_UNKNOWN_A9AE_SYSVAL_REGS_COUNT(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_UNKNOWN_A9AE_SYSVAL_REGS_COUNT__SHIFT) & A7XX_HLSQ_UNKNOWN_A9AE_SYSVAL_REGS_COUNT__MASK;
-}
-#define A7XX_HLSQ_UNKNOWN_A9AE_UNK8				0x00000100
-#define A7XX_HLSQ_UNKNOWN_A9AE_UNK9				0x00000200
-
-#define REG_A6XX_HLSQ_LOAD_STATE_GEOM_CMD			0x0000b820
-
-#define REG_A6XX_HLSQ_LOAD_STATE_GEOM_EXT_SRC_ADDR		0x0000b821
-
-#define REG_A6XX_HLSQ_LOAD_STATE_GEOM_DATA			0x0000b823
-
-#define REG_A6XX_HLSQ_FS_CNTL_0					0x0000b980
-#define A6XX_HLSQ_FS_CNTL_0_THREADSIZE__MASK			0x00000001
-#define A6XX_HLSQ_FS_CNTL_0_THREADSIZE__SHIFT			0
-static inline uint32_t A6XX_HLSQ_FS_CNTL_0_THREADSIZE(enum a6xx_threadsize val)
-{
-	return ((val) << A6XX_HLSQ_FS_CNTL_0_THREADSIZE__SHIFT) & A6XX_HLSQ_FS_CNTL_0_THREADSIZE__MASK;
-}
-#define A6XX_HLSQ_FS_CNTL_0_VARYINGS				0x00000002
-#define A6XX_HLSQ_FS_CNTL_0_UNK2__MASK				0x00000ffc
-#define A6XX_HLSQ_FS_CNTL_0_UNK2__SHIFT				2
-static inline uint32_t A6XX_HLSQ_FS_CNTL_0_UNK2(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_FS_CNTL_0_UNK2__SHIFT) & A6XX_HLSQ_FS_CNTL_0_UNK2__MASK;
-}
-
-#define REG_A6XX_HLSQ_UNKNOWN_B981				0x0000b981
-
-#define REG_A6XX_HLSQ_CONTROL_1_REG				0x0000b982
-#define A6XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__MASK	0x00000007
-#define A6XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__SHIFT	0
-static inline uint32_t A6XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__SHIFT) & A6XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__MASK;
-}
-
-#define REG_A6XX_HLSQ_CONTROL_2_REG				0x0000b983
-#define A6XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK			0x000000ff
-#define A6XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT		0
-static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_FACEREGID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK;
-}
-#define A6XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK			0x0000ff00
-#define A6XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT			8
-static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_SAMPLEID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK;
-}
-#define A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK		0x00ff0000
-#define A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT		16
-static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK;
-}
-#define A6XX_HLSQ_CONTROL_2_REG_CENTERRHW__MASK			0xff000000
-#define A6XX_HLSQ_CONTROL_2_REG_CENTERRHW__SHIFT		24
-static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_CENTERRHW(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_2_REG_CENTERRHW__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_CENTERRHW__MASK;
-}
-
-#define REG_A6XX_HLSQ_CONTROL_3_REG				0x0000b984
-#define A6XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__MASK		0x000000ff
-#define A6XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__SHIFT		0
-static inline uint32_t A6XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__SHIFT) & A6XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__MASK;
-}
-#define A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__MASK		0x0000ff00
-#define A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__SHIFT		8
-static inline uint32_t A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__SHIFT) & A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__MASK;
-}
-#define A6XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__MASK		0x00ff0000
-#define A6XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__SHIFT	16
-static inline uint32_t A6XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__SHIFT) & A6XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__MASK;
-}
-#define A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__MASK	0xff000000
-#define A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__SHIFT	24
-static inline uint32_t A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__SHIFT) & A6XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__MASK;
-}
-
-#define REG_A6XX_HLSQ_CONTROL_4_REG				0x0000b985
-#define A6XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__MASK		0x000000ff
-#define A6XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__SHIFT		0
-static inline uint32_t A6XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__SHIFT) & A6XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__MASK;
-}
-#define A6XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__MASK		0x0000ff00
-#define A6XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__SHIFT		8
-static inline uint32_t A6XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__SHIFT) & A6XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__MASK;
-}
-#define A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK		0x00ff0000
-#define A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT		16
-static inline uint32_t A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT) & A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK;
-}
-#define A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK		0xff000000
-#define A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT		24
-static inline uint32_t A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT) & A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK;
-}
-
-#define REG_A6XX_HLSQ_CONTROL_5_REG				0x0000b986
-#define A6XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__MASK		0x000000ff
-#define A6XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__SHIFT		0
-static inline uint32_t A6XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__SHIFT) & A6XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__MASK;
-}
-#define A6XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__MASK	0x0000ff00
-#define A6XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__SHIFT	8
-static inline uint32_t A6XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__SHIFT) & A6XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__MASK;
-}
-
-#define REG_A6XX_HLSQ_CS_CNTL					0x0000b987
-#define A6XX_HLSQ_CS_CNTL_CONSTLEN__MASK			0x000000ff
-#define A6XX_HLSQ_CS_CNTL_CONSTLEN__SHIFT			0
-static inline uint32_t A6XX_HLSQ_CS_CNTL_CONSTLEN(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_HLSQ_CS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_CS_CNTL_CONSTLEN__MASK;
-}
-#define A6XX_HLSQ_CS_CNTL_ENABLED				0x00000100
-#define A6XX_HLSQ_CS_CNTL_READ_IMM_SHARED_CONSTS		0x00000200
-
-#define REG_A7XX_HLSQ_FS_CNTL_0					0x0000a9c6
-#define A7XX_HLSQ_FS_CNTL_0_THREADSIZE__MASK			0x00000001
-#define A7XX_HLSQ_FS_CNTL_0_THREADSIZE__SHIFT			0
-static inline uint32_t A7XX_HLSQ_FS_CNTL_0_THREADSIZE(enum a6xx_threadsize val)
-{
-	return ((val) << A7XX_HLSQ_FS_CNTL_0_THREADSIZE__SHIFT) & A7XX_HLSQ_FS_CNTL_0_THREADSIZE__MASK;
-}
-#define A7XX_HLSQ_FS_CNTL_0_VARYINGS				0x00000002
-#define A7XX_HLSQ_FS_CNTL_0_UNK2__MASK				0x00000ffc
-#define A7XX_HLSQ_FS_CNTL_0_UNK2__SHIFT				2
-static inline uint32_t A7XX_HLSQ_FS_CNTL_0_UNK2(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_FS_CNTL_0_UNK2__SHIFT) & A7XX_HLSQ_FS_CNTL_0_UNK2__MASK;
-}
-
-#define REG_A7XX_HLSQ_CONTROL_1_REG				0x0000a9c7
-#define A7XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__MASK	0x00000007
-#define A7XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__SHIFT	0
-static inline uint32_t A7XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__SHIFT) & A7XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__MASK;
-}
-
-#define REG_A7XX_HLSQ_CONTROL_2_REG				0x0000a9c8
-#define A7XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK			0x000000ff
-#define A7XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT		0
-static inline uint32_t A7XX_HLSQ_CONTROL_2_REG_FACEREGID(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT) & A7XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK;
-}
-#define A7XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK			0x0000ff00
-#define A7XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT			8
-static inline uint32_t A7XX_HLSQ_CONTROL_2_REG_SAMPLEID(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT) & A7XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK;
-}
-#define A7XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK		0x00ff0000
-#define A7XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT		16
-static inline uint32_t A7XX_HLSQ_CONTROL_2_REG_SAMPLEMASK(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT) & A7XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK;
-}
-#define A7XX_HLSQ_CONTROL_2_REG_CENTERRHW__MASK			0xff000000
-#define A7XX_HLSQ_CONTROL_2_REG_CENTERRHW__SHIFT		24
-static inline uint32_t A7XX_HLSQ_CONTROL_2_REG_CENTERRHW(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_2_REG_CENTERRHW__SHIFT) & A7XX_HLSQ_CONTROL_2_REG_CENTERRHW__MASK;
-}
-
-#define REG_A7XX_HLSQ_CONTROL_3_REG				0x0000a9c9
-#define A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__MASK		0x000000ff
-#define A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__SHIFT		0
-static inline uint32_t A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__SHIFT) & A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL__MASK;
-}
-#define A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__MASK		0x0000ff00
-#define A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__SHIFT		8
-static inline uint32_t A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__SHIFT) & A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL__MASK;
-}
-#define A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__MASK		0x00ff0000
-#define A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__SHIFT	16
-static inline uint32_t A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__SHIFT) & A7XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID__MASK;
-}
-#define A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__MASK	0xff000000
-#define A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__SHIFT	24
-static inline uint32_t A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__SHIFT) & A7XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_CENTROID__MASK;
-}
-
-#define REG_A7XX_HLSQ_CONTROL_4_REG				0x0000a9ca
-#define A7XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__MASK		0x000000ff
-#define A7XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__SHIFT		0
-static inline uint32_t A7XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__SHIFT) & A7XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE__MASK;
-}
-#define A7XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__MASK		0x0000ff00
-#define A7XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__SHIFT		8
-static inline uint32_t A7XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__SHIFT) & A7XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE__MASK;
-}
-#define A7XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK		0x00ff0000
-#define A7XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT		16
-static inline uint32_t A7XX_HLSQ_CONTROL_4_REG_XYCOORDREGID(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT) & A7XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK;
-}
-#define A7XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK		0xff000000
-#define A7XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT		24
-static inline uint32_t A7XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT) & A7XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK;
-}
-
-#define REG_A7XX_HLSQ_CONTROL_5_REG				0x0000a9cb
-#define A7XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__MASK		0x000000ff
-#define A7XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__SHIFT		0
-static inline uint32_t A7XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__SHIFT) & A7XX_HLSQ_CONTROL_5_REG_LINELENGTHREGID__MASK;
-}
-#define A7XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__MASK	0x0000ff00
-#define A7XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__SHIFT	8
-static inline uint32_t A7XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__SHIFT) & A7XX_HLSQ_CONTROL_5_REG_FOVEATIONQUALITYREGID__MASK;
-}
-
-#define REG_A7XX_HLSQ_CS_CNTL					0x0000a9cd
-#define A7XX_HLSQ_CS_CNTL_CONSTLEN__MASK			0x000000ff
-#define A7XX_HLSQ_CS_CNTL_CONSTLEN__SHIFT			0
-static inline uint32_t A7XX_HLSQ_CS_CNTL_CONSTLEN(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A7XX_HLSQ_CS_CNTL_CONSTLEN__SHIFT) & A7XX_HLSQ_CS_CNTL_CONSTLEN__MASK;
-}
-#define A7XX_HLSQ_CS_CNTL_ENABLED				0x00000100
-#define A7XX_HLSQ_CS_CNTL_READ_IMM_SHARED_CONSTS		0x00000200
-
-#define REG_A6XX_HLSQ_CS_NDRANGE_0				0x0000b990
-#define A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM__MASK			0x00000003
-#define A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM__SHIFT			0
-static inline uint32_t A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM__SHIFT) & A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM__MASK;
-}
-#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__MASK			0x00000ffc
-#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__SHIFT		2
-static inline uint32_t A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__SHIFT) & A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__MASK;
-}
-#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__MASK			0x003ff000
-#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__SHIFT		12
-static inline uint32_t A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__SHIFT) & A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__MASK;
-}
-#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__MASK			0xffc00000
-#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__SHIFT		22
-static inline uint32_t A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__SHIFT) & A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__MASK;
-}
-
-#define REG_A6XX_HLSQ_CS_NDRANGE_1				0x0000b991
-#define A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK		0xffffffff
-#define A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT		0
-static inline uint32_t A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT) & A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK;
-}
-
-#define REG_A6XX_HLSQ_CS_NDRANGE_2				0x0000b992
-#define A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK		0xffffffff
-#define A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT		0
-static inline uint32_t A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT) & A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK;
-}
-
-#define REG_A6XX_HLSQ_CS_NDRANGE_3				0x0000b993
-#define A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK		0xffffffff
-#define A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT		0
-static inline uint32_t A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT) & A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK;
-}
-
-#define REG_A6XX_HLSQ_CS_NDRANGE_4				0x0000b994
-#define A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK		0xffffffff
-#define A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT		0
-static inline uint32_t A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT) & A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK;
-}
-
-#define REG_A6XX_HLSQ_CS_NDRANGE_5				0x0000b995
-#define A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK		0xffffffff
-#define A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT		0
-static inline uint32_t A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT) & A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK;
-}
-
-#define REG_A6XX_HLSQ_CS_NDRANGE_6				0x0000b996
-#define A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK		0xffffffff
-#define A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT		0
-static inline uint32_t A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT) & A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK;
-}
-
-#define REG_A6XX_HLSQ_CS_CNTL_0					0x0000b997
-#define A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID__MASK			0x000000ff
-#define A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID__SHIFT			0
-static inline uint32_t A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID__SHIFT) & A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID__MASK;
-}
-#define A6XX_HLSQ_CS_CNTL_0_WGSIZECONSTID__MASK			0x0000ff00
-#define A6XX_HLSQ_CS_CNTL_0_WGSIZECONSTID__SHIFT		8
-static inline uint32_t A6XX_HLSQ_CS_CNTL_0_WGSIZECONSTID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_CNTL_0_WGSIZECONSTID__SHIFT) & A6XX_HLSQ_CS_CNTL_0_WGSIZECONSTID__MASK;
-}
-#define A6XX_HLSQ_CS_CNTL_0_WGOFFSETCONSTID__MASK		0x00ff0000
-#define A6XX_HLSQ_CS_CNTL_0_WGOFFSETCONSTID__SHIFT		16
-static inline uint32_t A6XX_HLSQ_CS_CNTL_0_WGOFFSETCONSTID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_CNTL_0_WGOFFSETCONSTID__SHIFT) & A6XX_HLSQ_CS_CNTL_0_WGOFFSETCONSTID__MASK;
-}
-#define A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID__MASK			0xff000000
-#define A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID__SHIFT			24
-static inline uint32_t A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID__SHIFT) & A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID__MASK;
-}
-
-#define REG_A6XX_HLSQ_CS_CNTL_1					0x0000b998
-#define A6XX_HLSQ_CS_CNTL_1_LINEARLOCALIDREGID__MASK		0x000000ff
-#define A6XX_HLSQ_CS_CNTL_1_LINEARLOCALIDREGID__SHIFT		0
-static inline uint32_t A6XX_HLSQ_CS_CNTL_1_LINEARLOCALIDREGID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_CNTL_1_LINEARLOCALIDREGID__SHIFT) & A6XX_HLSQ_CS_CNTL_1_LINEARLOCALIDREGID__MASK;
-}
-#define A6XX_HLSQ_CS_CNTL_1_SINGLE_SP_CORE			0x00000100
-#define A6XX_HLSQ_CS_CNTL_1_THREADSIZE__MASK			0x00000200
-#define A6XX_HLSQ_CS_CNTL_1_THREADSIZE__SHIFT			9
-static inline uint32_t A6XX_HLSQ_CS_CNTL_1_THREADSIZE(enum a6xx_threadsize val)
-{
-	return ((val) << A6XX_HLSQ_CS_CNTL_1_THREADSIZE__SHIFT) & A6XX_HLSQ_CS_CNTL_1_THREADSIZE__MASK;
-}
-#define A6XX_HLSQ_CS_CNTL_1_THREADSIZE_SCALAR			0x00000400
-
-#define REG_A6XX_HLSQ_CS_KERNEL_GROUP_X				0x0000b999
-
-#define REG_A6XX_HLSQ_CS_KERNEL_GROUP_Y				0x0000b99a
-
-#define REG_A6XX_HLSQ_CS_KERNEL_GROUP_Z				0x0000b99b
-
-#define REG_A7XX_HLSQ_CS_NDRANGE_0				0x0000a9d4
-#define A7XX_HLSQ_CS_NDRANGE_0_KERNELDIM__MASK			0x00000003
-#define A7XX_HLSQ_CS_NDRANGE_0_KERNELDIM__SHIFT			0
-static inline uint32_t A7XX_HLSQ_CS_NDRANGE_0_KERNELDIM(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_NDRANGE_0_KERNELDIM__SHIFT) & A7XX_HLSQ_CS_NDRANGE_0_KERNELDIM__MASK;
-}
-#define A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__MASK			0x00000ffc
-#define A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__SHIFT		2
-static inline uint32_t A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__SHIFT) & A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__MASK;
-}
-#define A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__MASK			0x003ff000
-#define A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__SHIFT		12
-static inline uint32_t A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__SHIFT) & A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__MASK;
-}
-#define A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__MASK			0xffc00000
-#define A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__SHIFT		22
-static inline uint32_t A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__SHIFT) & A7XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__MASK;
-}
-
-#define REG_A7XX_HLSQ_CS_NDRANGE_1				0x0000a9d5
-#define A7XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK		0xffffffff
-#define A7XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT		0
-static inline uint32_t A7XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT) & A7XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK;
-}
-
-#define REG_A7XX_HLSQ_CS_NDRANGE_2				0x0000a9d6
-#define A7XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK		0xffffffff
-#define A7XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT		0
-static inline uint32_t A7XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT) & A7XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK;
-}
-
-#define REG_A7XX_HLSQ_CS_NDRANGE_3				0x0000a9d7
-#define A7XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK		0xffffffff
-#define A7XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT		0
-static inline uint32_t A7XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT) & A7XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK;
-}
-
-#define REG_A7XX_HLSQ_CS_NDRANGE_4				0x0000a9d8
-#define A7XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK		0xffffffff
-#define A7XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT		0
-static inline uint32_t A7XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT) & A7XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK;
-}
-
-#define REG_A7XX_HLSQ_CS_NDRANGE_5				0x0000a9d9
-#define A7XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK		0xffffffff
-#define A7XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT		0
-static inline uint32_t A7XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT) & A7XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK;
-}
-
-#define REG_A7XX_HLSQ_CS_NDRANGE_6				0x0000a9da
-#define A7XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK		0xffffffff
-#define A7XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT		0
-static inline uint32_t A7XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT) & A7XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK;
-}
-
-#define REG_A7XX_HLSQ_CS_KERNEL_GROUP_X				0x0000a9dc
-
-#define REG_A7XX_HLSQ_CS_KERNEL_GROUP_Y				0x0000a9dd
-
-#define REG_A7XX_HLSQ_CS_KERNEL_GROUP_Z				0x0000a9de
-
-#define REG_A7XX_HLSQ_CS_CNTL_1					0x0000a9db
-#define A7XX_HLSQ_CS_CNTL_1_LINEARLOCALIDREGID__MASK		0x000000ff
-#define A7XX_HLSQ_CS_CNTL_1_LINEARLOCALIDREGID__SHIFT		0
-static inline uint32_t A7XX_HLSQ_CS_CNTL_1_LINEARLOCALIDREGID(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_CNTL_1_LINEARLOCALIDREGID__SHIFT) & A7XX_HLSQ_CS_CNTL_1_LINEARLOCALIDREGID__MASK;
-}
-#define A7XX_HLSQ_CS_CNTL_1_THREADSIZE__MASK			0x00000200
-#define A7XX_HLSQ_CS_CNTL_1_THREADSIZE__SHIFT			9
-static inline uint32_t A7XX_HLSQ_CS_CNTL_1_THREADSIZE(enum a6xx_threadsize val)
-{
-	return ((val) << A7XX_HLSQ_CS_CNTL_1_THREADSIZE__SHIFT) & A7XX_HLSQ_CS_CNTL_1_THREADSIZE__MASK;
-}
-#define A7XX_HLSQ_CS_CNTL_1_UNK11				0x00000800
-#define A7XX_HLSQ_CS_CNTL_1_UNK22				0x00400000
-#define A7XX_HLSQ_CS_CNTL_1_UNK26				0x04000000
-#define A7XX_HLSQ_CS_CNTL_1_YALIGN__MASK			0x78000000
-#define A7XX_HLSQ_CS_CNTL_1_YALIGN__SHIFT			27
-static inline uint32_t A7XX_HLSQ_CS_CNTL_1_YALIGN(enum a7xx_cs_yalign val)
-{
-	return ((val) << A7XX_HLSQ_CS_CNTL_1_YALIGN__SHIFT) & A7XX_HLSQ_CS_CNTL_1_YALIGN__MASK;
-}
-
-#define REG_A7XX_HLSQ_CS_LOCAL_SIZE				0x0000a9df
-#define A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEX__MASK		0x00000ffc
-#define A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEX__SHIFT		2
-static inline uint32_t A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEX(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEX__SHIFT) & A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEX__MASK;
-}
-#define A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEY__MASK		0x003ff000
-#define A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEY__SHIFT		12
-static inline uint32_t A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEY(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEY__SHIFT) & A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEY__MASK;
-}
-#define A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEZ__MASK		0xffc00000
-#define A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEZ__SHIFT		22
-static inline uint32_t A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEZ(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEZ__SHIFT) & A7XX_HLSQ_CS_LOCAL_SIZE_LOCALSIZEZ__MASK;
-}
-
-#define REG_A6XX_HLSQ_LOAD_STATE_FRAG_CMD			0x0000b9a0
-
-#define REG_A6XX_HLSQ_LOAD_STATE_FRAG_EXT_SRC_ADDR		0x0000b9a1
-
-#define REG_A6XX_HLSQ_LOAD_STATE_FRAG_DATA			0x0000b9a3
-
-#define REG_A6XX_HLSQ_CS_BINDLESS_BASE(i0) (0x0000b9c0 + 0x2*(i0))
-
-static inline uint32_t REG_A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR(uint32_t i0) { return 0x0000b9c0 + 0x2*i0; }
-#define A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK	0x00000003
-#define A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT	0
-static inline uint32_t A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE(enum a6xx_bindless_descriptor_size val)
-{
-	return ((val) << A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT) & A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK;
-}
-#define A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK	0xfffffffffffffffc
-#define A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT	2
-static inline uint32_t A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_ADDR(uint64_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT) & A6XX_HLSQ_CS_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK;
-}
-
-#define REG_A6XX_HLSQ_CS_UNKNOWN_B9D0				0x0000b9d0
-#define A6XX_HLSQ_CS_UNKNOWN_B9D0_SHARED_SIZE__MASK		0x0000001f
-#define A6XX_HLSQ_CS_UNKNOWN_B9D0_SHARED_SIZE__SHIFT		0
-static inline uint32_t A6XX_HLSQ_CS_UNKNOWN_B9D0_SHARED_SIZE(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_CS_UNKNOWN_B9D0_SHARED_SIZE__SHIFT) & A6XX_HLSQ_CS_UNKNOWN_B9D0_SHARED_SIZE__MASK;
-}
-#define A6XX_HLSQ_CS_UNKNOWN_B9D0_UNK5				0x00000020
-#define A6XX_HLSQ_CS_UNKNOWN_B9D0_UNK6				0x00000040
-
-#define REG_A6XX_HLSQ_DRAW_CMD					0x0000bb00
-#define A6XX_HLSQ_DRAW_CMD_STATE_ID__MASK			0x000000ff
-#define A6XX_HLSQ_DRAW_CMD_STATE_ID__SHIFT			0
-static inline uint32_t A6XX_HLSQ_DRAW_CMD_STATE_ID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_DRAW_CMD_STATE_ID__SHIFT) & A6XX_HLSQ_DRAW_CMD_STATE_ID__MASK;
-}
-
-#define REG_A6XX_HLSQ_DISPATCH_CMD				0x0000bb01
-#define A6XX_HLSQ_DISPATCH_CMD_STATE_ID__MASK			0x000000ff
-#define A6XX_HLSQ_DISPATCH_CMD_STATE_ID__SHIFT			0
-static inline uint32_t A6XX_HLSQ_DISPATCH_CMD_STATE_ID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_DISPATCH_CMD_STATE_ID__SHIFT) & A6XX_HLSQ_DISPATCH_CMD_STATE_ID__MASK;
-}
-
-#define REG_A6XX_HLSQ_EVENT_CMD					0x0000bb02
-#define A6XX_HLSQ_EVENT_CMD_STATE_ID__MASK			0x00ff0000
-#define A6XX_HLSQ_EVENT_CMD_STATE_ID__SHIFT			16
-static inline uint32_t A6XX_HLSQ_EVENT_CMD_STATE_ID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_EVENT_CMD_STATE_ID__SHIFT) & A6XX_HLSQ_EVENT_CMD_STATE_ID__MASK;
-}
-#define A6XX_HLSQ_EVENT_CMD_EVENT__MASK				0x0000007f
-#define A6XX_HLSQ_EVENT_CMD_EVENT__SHIFT			0
-static inline uint32_t A6XX_HLSQ_EVENT_CMD_EVENT(enum vgt_event_type val)
-{
-	return ((val) << A6XX_HLSQ_EVENT_CMD_EVENT__SHIFT) & A6XX_HLSQ_EVENT_CMD_EVENT__MASK;
-}
-
-#define REG_A6XX_HLSQ_INVALIDATE_CMD				0x0000bb08
-#define A6XX_HLSQ_INVALIDATE_CMD_VS_STATE			0x00000001
-#define A6XX_HLSQ_INVALIDATE_CMD_HS_STATE			0x00000002
-#define A6XX_HLSQ_INVALIDATE_CMD_DS_STATE			0x00000004
-#define A6XX_HLSQ_INVALIDATE_CMD_GS_STATE			0x00000008
-#define A6XX_HLSQ_INVALIDATE_CMD_FS_STATE			0x00000010
-#define A6XX_HLSQ_INVALIDATE_CMD_CS_STATE			0x00000020
-#define A6XX_HLSQ_INVALIDATE_CMD_CS_IBO				0x00000040
-#define A6XX_HLSQ_INVALIDATE_CMD_GFX_IBO			0x00000080
-#define A6XX_HLSQ_INVALIDATE_CMD_CS_SHARED_CONST		0x00080000
-#define A6XX_HLSQ_INVALIDATE_CMD_GFX_SHARED_CONST		0x00000100
-#define A6XX_HLSQ_INVALIDATE_CMD_CS_BINDLESS__MASK		0x00003e00
-#define A6XX_HLSQ_INVALIDATE_CMD_CS_BINDLESS__SHIFT		9
-static inline uint32_t A6XX_HLSQ_INVALIDATE_CMD_CS_BINDLESS(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_INVALIDATE_CMD_CS_BINDLESS__SHIFT) & A6XX_HLSQ_INVALIDATE_CMD_CS_BINDLESS__MASK;
-}
-#define A6XX_HLSQ_INVALIDATE_CMD_GFX_BINDLESS__MASK		0x0007c000
-#define A6XX_HLSQ_INVALIDATE_CMD_GFX_BINDLESS__SHIFT		14
-static inline uint32_t A6XX_HLSQ_INVALIDATE_CMD_GFX_BINDLESS(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_INVALIDATE_CMD_GFX_BINDLESS__SHIFT) & A6XX_HLSQ_INVALIDATE_CMD_GFX_BINDLESS__MASK;
-}
-
-#define REG_A7XX_HLSQ_INVALIDATE_CMD				0x0000ab1f
-#define A7XX_HLSQ_INVALIDATE_CMD_VS_STATE			0x00000001
-#define A7XX_HLSQ_INVALIDATE_CMD_HS_STATE			0x00000002
-#define A7XX_HLSQ_INVALIDATE_CMD_DS_STATE			0x00000004
-#define A7XX_HLSQ_INVALIDATE_CMD_GS_STATE			0x00000008
-#define A7XX_HLSQ_INVALIDATE_CMD_FS_STATE			0x00000010
-#define A7XX_HLSQ_INVALIDATE_CMD_CS_STATE			0x00000020
-#define A7XX_HLSQ_INVALIDATE_CMD_CS_IBO				0x00000040
-#define A7XX_HLSQ_INVALIDATE_CMD_GFX_IBO			0x00000080
-#define A7XX_HLSQ_INVALIDATE_CMD_CS_BINDLESS__MASK		0x0001fe00
-#define A7XX_HLSQ_INVALIDATE_CMD_CS_BINDLESS__SHIFT		9
-static inline uint32_t A7XX_HLSQ_INVALIDATE_CMD_CS_BINDLESS(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_INVALIDATE_CMD_CS_BINDLESS__SHIFT) & A7XX_HLSQ_INVALIDATE_CMD_CS_BINDLESS__MASK;
-}
-#define A7XX_HLSQ_INVALIDATE_CMD_GFX_BINDLESS__MASK		0x01fe0000
-#define A7XX_HLSQ_INVALIDATE_CMD_GFX_BINDLESS__SHIFT		17
-static inline uint32_t A7XX_HLSQ_INVALIDATE_CMD_GFX_BINDLESS(uint32_t val)
-{
-	return ((val) << A7XX_HLSQ_INVALIDATE_CMD_GFX_BINDLESS__SHIFT) & A7XX_HLSQ_INVALIDATE_CMD_GFX_BINDLESS__MASK;
-}
-
-#define REG_A6XX_HLSQ_FS_CNTL					0x0000bb10
-#define A6XX_HLSQ_FS_CNTL_CONSTLEN__MASK			0x000000ff
-#define A6XX_HLSQ_FS_CNTL_CONSTLEN__SHIFT			0
-static inline uint32_t A6XX_HLSQ_FS_CNTL_CONSTLEN(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_HLSQ_FS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_FS_CNTL_CONSTLEN__MASK;
-}
-#define A6XX_HLSQ_FS_CNTL_ENABLED				0x00000100
-#define A6XX_HLSQ_FS_CNTL_READ_IMM_SHARED_CONSTS		0x00000200
-
-#define REG_A7XX_HLSQ_FS_CNTL					0x0000ab03
-#define A7XX_HLSQ_FS_CNTL_CONSTLEN__MASK			0x000000ff
-#define A7XX_HLSQ_FS_CNTL_CONSTLEN__SHIFT			0
-static inline uint32_t A7XX_HLSQ_FS_CNTL_CONSTLEN(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A7XX_HLSQ_FS_CNTL_CONSTLEN__SHIFT) & A7XX_HLSQ_FS_CNTL_CONSTLEN__MASK;
-}
-#define A7XX_HLSQ_FS_CNTL_ENABLED				0x00000100
-#define A7XX_HLSQ_FS_CNTL_READ_IMM_SHARED_CONSTS		0x00000200
-
-#define REG_A7XX_HLSQ_SHARED_CONSTS_IMM(i0) (0x0000ab40 + 0x1*(i0))
-
-#define REG_A6XX_HLSQ_SHARED_CONSTS				0x0000bb11
-#define A6XX_HLSQ_SHARED_CONSTS_ENABLE				0x00000001
-
-#define REG_A6XX_HLSQ_BINDLESS_BASE(i0) (0x0000bb20 + 0x2*(i0))
-
-static inline uint32_t REG_A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR(uint32_t i0) { return 0x0000bb20 + 0x2*i0; }
-#define A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK	0x00000003
-#define A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT	0
-static inline uint32_t A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE(enum a6xx_bindless_descriptor_size val)
-{
-	return ((val) << A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__SHIFT) & A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_DESC_SIZE__MASK;
-}
-#define A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK		0xfffffffffffffffc
-#define A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT		2
-static inline uint32_t A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_ADDR(uint64_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_ADDR__SHIFT) & A6XX_HLSQ_BINDLESS_BASE_DESCRIPTOR_ADDR__MASK;
-}
-
-#define REG_A6XX_HLSQ_2D_EVENT_CMD				0x0000bd80
-#define A6XX_HLSQ_2D_EVENT_CMD_STATE_ID__MASK			0x0000ff00
-#define A6XX_HLSQ_2D_EVENT_CMD_STATE_ID__SHIFT			8
-static inline uint32_t A6XX_HLSQ_2D_EVENT_CMD_STATE_ID(uint32_t val)
-{
-	return ((val) << A6XX_HLSQ_2D_EVENT_CMD_STATE_ID__SHIFT) & A6XX_HLSQ_2D_EVENT_CMD_STATE_ID__MASK;
-}
-#define A6XX_HLSQ_2D_EVENT_CMD_EVENT__MASK			0x0000007f
-#define A6XX_HLSQ_2D_EVENT_CMD_EVENT__SHIFT			0
-static inline uint32_t A6XX_HLSQ_2D_EVENT_CMD_EVENT(enum vgt_event_type val)
-{
-	return ((val) << A6XX_HLSQ_2D_EVENT_CMD_EVENT__SHIFT) & A6XX_HLSQ_2D_EVENT_CMD_EVENT__MASK;
-}
-
-#define REG_A6XX_HLSQ_UNKNOWN_BE00				0x0000be00
-
-#define REG_A6XX_HLSQ_UNKNOWN_BE01				0x0000be01
-
-#define REG_A6XX_HLSQ_DBG_ECO_CNTL				0x0000be04
-
-#define REG_A6XX_HLSQ_ADDR_MODE_CNTL				0x0000be05
-
-#define REG_A6XX_HLSQ_UNKNOWN_BE08				0x0000be08
-
-#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL(i0) (0x0000be10 + 0x1*(i0))
-
-#define REG_A6XX_HLSQ_CONTEXT_SWITCH_GFX_PREEMPTION_SAFE_MODE	0x0000be22
-
-#define REG_A7XX_SP_AHB_READ_APERTURE				0x0000c000
-
-#define REG_A7XX_SP_UNKNOWN_0CE2				0x00000ce2
-
-#define REG_A7XX_SP_UNKNOWN_0CE4				0x00000ce4
-
-#define REG_A7XX_SP_UNKNOWN_0CE6				0x00000ce6
-
-#define REG_A6XX_CP_EVENT_START					0x0000d600
-#define A6XX_CP_EVENT_START_STATE_ID__MASK			0x000000ff
-#define A6XX_CP_EVENT_START_STATE_ID__SHIFT			0
-static inline uint32_t A6XX_CP_EVENT_START_STATE_ID(uint32_t val)
-{
-	return ((val) << A6XX_CP_EVENT_START_STATE_ID__SHIFT) & A6XX_CP_EVENT_START_STATE_ID__MASK;
-}
-
-#define REG_A6XX_CP_EVENT_END					0x0000d601
-#define A6XX_CP_EVENT_END_STATE_ID__MASK			0x000000ff
-#define A6XX_CP_EVENT_END_STATE_ID__SHIFT			0
-static inline uint32_t A6XX_CP_EVENT_END_STATE_ID(uint32_t val)
-{
-	return ((val) << A6XX_CP_EVENT_END_STATE_ID__SHIFT) & A6XX_CP_EVENT_END_STATE_ID__MASK;
-}
-
-#define REG_A6XX_CP_2D_EVENT_START				0x0000d700
-#define A6XX_CP_2D_EVENT_START_STATE_ID__MASK			0x000000ff
-#define A6XX_CP_2D_EVENT_START_STATE_ID__SHIFT			0
-static inline uint32_t A6XX_CP_2D_EVENT_START_STATE_ID(uint32_t val)
-{
-	return ((val) << A6XX_CP_2D_EVENT_START_STATE_ID__SHIFT) & A6XX_CP_2D_EVENT_START_STATE_ID__MASK;
-}
-
-#define REG_A6XX_CP_2D_EVENT_END				0x0000d701
-#define A6XX_CP_2D_EVENT_END_STATE_ID__MASK			0x000000ff
-#define A6XX_CP_2D_EVENT_END_STATE_ID__SHIFT			0
-static inline uint32_t A6XX_CP_2D_EVENT_END_STATE_ID(uint32_t val)
-{
-	return ((val) << A6XX_CP_2D_EVENT_END_STATE_ID__SHIFT) & A6XX_CP_2D_EVENT_END_STATE_ID__MASK;
-}
-
-#define REG_A6XX_TEX_SAMP_0					0x00000000
-#define A6XX_TEX_SAMP_0_MIPFILTER_LINEAR_NEAR			0x00000001
-#define A6XX_TEX_SAMP_0_XY_MAG__MASK				0x00000006
-#define A6XX_TEX_SAMP_0_XY_MAG__SHIFT				1
-static inline uint32_t A6XX_TEX_SAMP_0_XY_MAG(enum a6xx_tex_filter val)
-{
-	return ((val) << A6XX_TEX_SAMP_0_XY_MAG__SHIFT) & A6XX_TEX_SAMP_0_XY_MAG__MASK;
-}
-#define A6XX_TEX_SAMP_0_XY_MIN__MASK				0x00000018
-#define A6XX_TEX_SAMP_0_XY_MIN__SHIFT				3
-static inline uint32_t A6XX_TEX_SAMP_0_XY_MIN(enum a6xx_tex_filter val)
-{
-	return ((val) << A6XX_TEX_SAMP_0_XY_MIN__SHIFT) & A6XX_TEX_SAMP_0_XY_MIN__MASK;
-}
-#define A6XX_TEX_SAMP_0_WRAP_S__MASK				0x000000e0
-#define A6XX_TEX_SAMP_0_WRAP_S__SHIFT				5
-static inline uint32_t A6XX_TEX_SAMP_0_WRAP_S(enum a6xx_tex_clamp val)
-{
-	return ((val) << A6XX_TEX_SAMP_0_WRAP_S__SHIFT) & A6XX_TEX_SAMP_0_WRAP_S__MASK;
-}
-#define A6XX_TEX_SAMP_0_WRAP_T__MASK				0x00000700
-#define A6XX_TEX_SAMP_0_WRAP_T__SHIFT				8
-static inline uint32_t A6XX_TEX_SAMP_0_WRAP_T(enum a6xx_tex_clamp val)
-{
-	return ((val) << A6XX_TEX_SAMP_0_WRAP_T__SHIFT) & A6XX_TEX_SAMP_0_WRAP_T__MASK;
-}
-#define A6XX_TEX_SAMP_0_WRAP_R__MASK				0x00003800
-#define A6XX_TEX_SAMP_0_WRAP_R__SHIFT				11
-static inline uint32_t A6XX_TEX_SAMP_0_WRAP_R(enum a6xx_tex_clamp val)
-{
-	return ((val) << A6XX_TEX_SAMP_0_WRAP_R__SHIFT) & A6XX_TEX_SAMP_0_WRAP_R__MASK;
-}
-#define A6XX_TEX_SAMP_0_ANISO__MASK				0x0001c000
-#define A6XX_TEX_SAMP_0_ANISO__SHIFT				14
-static inline uint32_t A6XX_TEX_SAMP_0_ANISO(enum a6xx_tex_aniso val)
-{
-	return ((val) << A6XX_TEX_SAMP_0_ANISO__SHIFT) & A6XX_TEX_SAMP_0_ANISO__MASK;
-}
-#define A6XX_TEX_SAMP_0_LOD_BIAS__MASK				0xfff80000
-#define A6XX_TEX_SAMP_0_LOD_BIAS__SHIFT				19
-static inline uint32_t A6XX_TEX_SAMP_0_LOD_BIAS(float val)
-{
-	return ((((int32_t)(val * 256.0))) << A6XX_TEX_SAMP_0_LOD_BIAS__SHIFT) & A6XX_TEX_SAMP_0_LOD_BIAS__MASK;
-}
-
-#define REG_A6XX_TEX_SAMP_1					0x00000001
-#define A6XX_TEX_SAMP_1_CLAMPENABLE				0x00000001
-#define A6XX_TEX_SAMP_1_COMPARE_FUNC__MASK			0x0000000e
-#define A6XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT			1
-static inline uint32_t A6XX_TEX_SAMP_1_COMPARE_FUNC(enum adreno_compare_func val)
-{
-	return ((val) << A6XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT) & A6XX_TEX_SAMP_1_COMPARE_FUNC__MASK;
-}
-#define A6XX_TEX_SAMP_1_CUBEMAPSEAMLESSFILTOFF			0x00000010
-#define A6XX_TEX_SAMP_1_UNNORM_COORDS				0x00000020
-#define A6XX_TEX_SAMP_1_MIPFILTER_LINEAR_FAR			0x00000040
-#define A6XX_TEX_SAMP_1_MAX_LOD__MASK				0x000fff00
-#define A6XX_TEX_SAMP_1_MAX_LOD__SHIFT				8
-static inline uint32_t A6XX_TEX_SAMP_1_MAX_LOD(float val)
-{
-	return ((((uint32_t)(val * 256.0))) << A6XX_TEX_SAMP_1_MAX_LOD__SHIFT) & A6XX_TEX_SAMP_1_MAX_LOD__MASK;
-}
-#define A6XX_TEX_SAMP_1_MIN_LOD__MASK				0xfff00000
-#define A6XX_TEX_SAMP_1_MIN_LOD__SHIFT				20
-static inline uint32_t A6XX_TEX_SAMP_1_MIN_LOD(float val)
-{
-	return ((((uint32_t)(val * 256.0))) << A6XX_TEX_SAMP_1_MIN_LOD__SHIFT) & A6XX_TEX_SAMP_1_MIN_LOD__MASK;
-}
-
-#define REG_A6XX_TEX_SAMP_2					0x00000002
-#define A6XX_TEX_SAMP_2_REDUCTION_MODE__MASK			0x00000003
-#define A6XX_TEX_SAMP_2_REDUCTION_MODE__SHIFT			0
-static inline uint32_t A6XX_TEX_SAMP_2_REDUCTION_MODE(enum a6xx_reduction_mode val)
-{
-	return ((val) << A6XX_TEX_SAMP_2_REDUCTION_MODE__SHIFT) & A6XX_TEX_SAMP_2_REDUCTION_MODE__MASK;
-}
-#define A6XX_TEX_SAMP_2_CHROMA_LINEAR				0x00000020
-#define A6XX_TEX_SAMP_2_BCOLOR__MASK				0xffffff80
-#define A6XX_TEX_SAMP_2_BCOLOR__SHIFT				7
-static inline uint32_t A6XX_TEX_SAMP_2_BCOLOR(uint32_t val)
-{
-	return ((val) << A6XX_TEX_SAMP_2_BCOLOR__SHIFT) & A6XX_TEX_SAMP_2_BCOLOR__MASK;
-}
-
-#define REG_A6XX_TEX_SAMP_3					0x00000003
-
-#define REG_A6XX_TEX_CONST_0					0x00000000
-#define A6XX_TEX_CONST_0_TILE_MODE__MASK			0x00000003
-#define A6XX_TEX_CONST_0_TILE_MODE__SHIFT			0
-static inline uint32_t A6XX_TEX_CONST_0_TILE_MODE(enum a6xx_tile_mode val)
-{
-	return ((val) << A6XX_TEX_CONST_0_TILE_MODE__SHIFT) & A6XX_TEX_CONST_0_TILE_MODE__MASK;
-}
-#define A6XX_TEX_CONST_0_SRGB					0x00000004
-#define A6XX_TEX_CONST_0_SWIZ_X__MASK				0x00000070
-#define A6XX_TEX_CONST_0_SWIZ_X__SHIFT				4
-static inline uint32_t A6XX_TEX_CONST_0_SWIZ_X(enum a6xx_tex_swiz val)
-{
-	return ((val) << A6XX_TEX_CONST_0_SWIZ_X__SHIFT) & A6XX_TEX_CONST_0_SWIZ_X__MASK;
-}
-#define A6XX_TEX_CONST_0_SWIZ_Y__MASK				0x00000380
-#define A6XX_TEX_CONST_0_SWIZ_Y__SHIFT				7
-static inline uint32_t A6XX_TEX_CONST_0_SWIZ_Y(enum a6xx_tex_swiz val)
-{
-	return ((val) << A6XX_TEX_CONST_0_SWIZ_Y__SHIFT) & A6XX_TEX_CONST_0_SWIZ_Y__MASK;
-}
-#define A6XX_TEX_CONST_0_SWIZ_Z__MASK				0x00001c00
-#define A6XX_TEX_CONST_0_SWIZ_Z__SHIFT				10
-static inline uint32_t A6XX_TEX_CONST_0_SWIZ_Z(enum a6xx_tex_swiz val)
-{
-	return ((val) << A6XX_TEX_CONST_0_SWIZ_Z__SHIFT) & A6XX_TEX_CONST_0_SWIZ_Z__MASK;
-}
-#define A6XX_TEX_CONST_0_SWIZ_W__MASK				0x0000e000
-#define A6XX_TEX_CONST_0_SWIZ_W__SHIFT				13
-static inline uint32_t A6XX_TEX_CONST_0_SWIZ_W(enum a6xx_tex_swiz val)
-{
-	return ((val) << A6XX_TEX_CONST_0_SWIZ_W__SHIFT) & A6XX_TEX_CONST_0_SWIZ_W__MASK;
-}
-#define A6XX_TEX_CONST_0_MIPLVLS__MASK				0x000f0000
-#define A6XX_TEX_CONST_0_MIPLVLS__SHIFT				16
-static inline uint32_t A6XX_TEX_CONST_0_MIPLVLS(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_0_MIPLVLS__SHIFT) & A6XX_TEX_CONST_0_MIPLVLS__MASK;
-}
-#define A6XX_TEX_CONST_0_CHROMA_MIDPOINT_X			0x00010000
-#define A6XX_TEX_CONST_0_CHROMA_MIDPOINT_Y			0x00040000
-#define A6XX_TEX_CONST_0_SAMPLES__MASK				0x00300000
-#define A6XX_TEX_CONST_0_SAMPLES__SHIFT				20
-static inline uint32_t A6XX_TEX_CONST_0_SAMPLES(enum a3xx_msaa_samples val)
-{
-	return ((val) << A6XX_TEX_CONST_0_SAMPLES__SHIFT) & A6XX_TEX_CONST_0_SAMPLES__MASK;
-}
-#define A6XX_TEX_CONST_0_FMT__MASK				0x3fc00000
-#define A6XX_TEX_CONST_0_FMT__SHIFT				22
-static inline uint32_t A6XX_TEX_CONST_0_FMT(enum a6xx_format val)
-{
-	return ((val) << A6XX_TEX_CONST_0_FMT__SHIFT) & A6XX_TEX_CONST_0_FMT__MASK;
-}
-#define A6XX_TEX_CONST_0_SWAP__MASK				0xc0000000
-#define A6XX_TEX_CONST_0_SWAP__SHIFT				30
-static inline uint32_t A6XX_TEX_CONST_0_SWAP(enum a3xx_color_swap val)
-{
-	return ((val) << A6XX_TEX_CONST_0_SWAP__SHIFT) & A6XX_TEX_CONST_0_SWAP__MASK;
-}
-
-#define REG_A6XX_TEX_CONST_1					0x00000001
-#define A6XX_TEX_CONST_1_WIDTH__MASK				0x00007fff
-#define A6XX_TEX_CONST_1_WIDTH__SHIFT				0
-static inline uint32_t A6XX_TEX_CONST_1_WIDTH(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_1_WIDTH__SHIFT) & A6XX_TEX_CONST_1_WIDTH__MASK;
-}
-#define A6XX_TEX_CONST_1_HEIGHT__MASK				0x3fff8000
-#define A6XX_TEX_CONST_1_HEIGHT__SHIFT				15
-static inline uint32_t A6XX_TEX_CONST_1_HEIGHT(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_1_HEIGHT__SHIFT) & A6XX_TEX_CONST_1_HEIGHT__MASK;
-}
-
-#define REG_A6XX_TEX_CONST_2					0x00000002
-#define A6XX_TEX_CONST_2_STRUCTSIZETEXELS__MASK			0x0000fff0
-#define A6XX_TEX_CONST_2_STRUCTSIZETEXELS__SHIFT		4
-static inline uint32_t A6XX_TEX_CONST_2_STRUCTSIZETEXELS(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_2_STRUCTSIZETEXELS__SHIFT) & A6XX_TEX_CONST_2_STRUCTSIZETEXELS__MASK;
-}
-#define A6XX_TEX_CONST_2_STARTOFFSETTEXELS__MASK		0x003f0000
-#define A6XX_TEX_CONST_2_STARTOFFSETTEXELS__SHIFT		16
-static inline uint32_t A6XX_TEX_CONST_2_STARTOFFSETTEXELS(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_2_STARTOFFSETTEXELS__SHIFT) & A6XX_TEX_CONST_2_STARTOFFSETTEXELS__MASK;
-}
-#define A6XX_TEX_CONST_2_PITCHALIGN__MASK			0x0000000f
-#define A6XX_TEX_CONST_2_PITCHALIGN__SHIFT			0
-static inline uint32_t A6XX_TEX_CONST_2_PITCHALIGN(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_2_PITCHALIGN__SHIFT) & A6XX_TEX_CONST_2_PITCHALIGN__MASK;
-}
-#define A6XX_TEX_CONST_2_PITCH__MASK				0x1fffff80
-#define A6XX_TEX_CONST_2_PITCH__SHIFT				7
-static inline uint32_t A6XX_TEX_CONST_2_PITCH(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_2_PITCH__SHIFT) & A6XX_TEX_CONST_2_PITCH__MASK;
-}
-#define A6XX_TEX_CONST_2_TYPE__MASK				0xe0000000
-#define A6XX_TEX_CONST_2_TYPE__SHIFT				29
-static inline uint32_t A6XX_TEX_CONST_2_TYPE(enum a6xx_tex_type val)
-{
-	return ((val) << A6XX_TEX_CONST_2_TYPE__SHIFT) & A6XX_TEX_CONST_2_TYPE__MASK;
-}
-
-#define REG_A6XX_TEX_CONST_3					0x00000003
-#define A6XX_TEX_CONST_3_ARRAY_PITCH__MASK			0x007fffff
-#define A6XX_TEX_CONST_3_ARRAY_PITCH__SHIFT			0
-static inline uint32_t A6XX_TEX_CONST_3_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A6XX_TEX_CONST_3_ARRAY_PITCH__SHIFT) & A6XX_TEX_CONST_3_ARRAY_PITCH__MASK;
-}
-#define A6XX_TEX_CONST_3_MIN_LAYERSZ__MASK			0x07800000
-#define A6XX_TEX_CONST_3_MIN_LAYERSZ__SHIFT			23
-static inline uint32_t A6XX_TEX_CONST_3_MIN_LAYERSZ(uint32_t val)
-{
-	assert(!(val & 0xfff));
-	return (((val >> 12)) << A6XX_TEX_CONST_3_MIN_LAYERSZ__SHIFT) & A6XX_TEX_CONST_3_MIN_LAYERSZ__MASK;
-}
-#define A6XX_TEX_CONST_3_TILE_ALL				0x08000000
-#define A6XX_TEX_CONST_3_FLAG					0x10000000
-
-#define REG_A6XX_TEX_CONST_4					0x00000004
-#define A6XX_TEX_CONST_4_BASE_LO__MASK				0xffffffe0
-#define A6XX_TEX_CONST_4_BASE_LO__SHIFT				5
-static inline uint32_t A6XX_TEX_CONST_4_BASE_LO(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A6XX_TEX_CONST_4_BASE_LO__SHIFT) & A6XX_TEX_CONST_4_BASE_LO__MASK;
-}
-
-#define REG_A6XX_TEX_CONST_5					0x00000005
-#define A6XX_TEX_CONST_5_BASE_HI__MASK				0x0001ffff
-#define A6XX_TEX_CONST_5_BASE_HI__SHIFT				0
-static inline uint32_t A6XX_TEX_CONST_5_BASE_HI(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_5_BASE_HI__SHIFT) & A6XX_TEX_CONST_5_BASE_HI__MASK;
-}
-#define A6XX_TEX_CONST_5_DEPTH__MASK				0x3ffe0000
-#define A6XX_TEX_CONST_5_DEPTH__SHIFT				17
-static inline uint32_t A6XX_TEX_CONST_5_DEPTH(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_5_DEPTH__SHIFT) & A6XX_TEX_CONST_5_DEPTH__MASK;
-}
-
-#define REG_A6XX_TEX_CONST_6					0x00000006
-#define A6XX_TEX_CONST_6_MIN_LOD_CLAMP__MASK			0x00000fff
-#define A6XX_TEX_CONST_6_MIN_LOD_CLAMP__SHIFT			0
-static inline uint32_t A6XX_TEX_CONST_6_MIN_LOD_CLAMP(float val)
-{
-	return ((((uint32_t)(val * 256.0))) << A6XX_TEX_CONST_6_MIN_LOD_CLAMP__SHIFT) & A6XX_TEX_CONST_6_MIN_LOD_CLAMP__MASK;
-}
-#define A6XX_TEX_CONST_6_PLANE_PITCH__MASK			0xffffff00
-#define A6XX_TEX_CONST_6_PLANE_PITCH__SHIFT			8
-static inline uint32_t A6XX_TEX_CONST_6_PLANE_PITCH(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_6_PLANE_PITCH__SHIFT) & A6XX_TEX_CONST_6_PLANE_PITCH__MASK;
-}
-
-#define REG_A6XX_TEX_CONST_7					0x00000007
-#define A6XX_TEX_CONST_7_FLAG_LO__MASK				0xffffffe0
-#define A6XX_TEX_CONST_7_FLAG_LO__SHIFT				5
-static inline uint32_t A6XX_TEX_CONST_7_FLAG_LO(uint32_t val)
-{
-	assert(!(val & 0x1f));
-	return (((val >> 5)) << A6XX_TEX_CONST_7_FLAG_LO__SHIFT) & A6XX_TEX_CONST_7_FLAG_LO__MASK;
-}
-
-#define REG_A6XX_TEX_CONST_8					0x00000008
-#define A6XX_TEX_CONST_8_FLAG_HI__MASK				0x0001ffff
-#define A6XX_TEX_CONST_8_FLAG_HI__SHIFT				0
-static inline uint32_t A6XX_TEX_CONST_8_FLAG_HI(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_8_FLAG_HI__SHIFT) & A6XX_TEX_CONST_8_FLAG_HI__MASK;
-}
-
-#define REG_A6XX_TEX_CONST_9					0x00000009
-#define A6XX_TEX_CONST_9_FLAG_BUFFER_ARRAY_PITCH__MASK		0x0001ffff
-#define A6XX_TEX_CONST_9_FLAG_BUFFER_ARRAY_PITCH__SHIFT		0
-static inline uint32_t A6XX_TEX_CONST_9_FLAG_BUFFER_ARRAY_PITCH(uint32_t val)
-{
-	assert(!(val & 0xf));
-	return (((val >> 4)) << A6XX_TEX_CONST_9_FLAG_BUFFER_ARRAY_PITCH__SHIFT) & A6XX_TEX_CONST_9_FLAG_BUFFER_ARRAY_PITCH__MASK;
-}
-
-#define REG_A6XX_TEX_CONST_10					0x0000000a
-#define A6XX_TEX_CONST_10_FLAG_BUFFER_PITCH__MASK		0x0000007f
-#define A6XX_TEX_CONST_10_FLAG_BUFFER_PITCH__SHIFT		0
-static inline uint32_t A6XX_TEX_CONST_10_FLAG_BUFFER_PITCH(uint32_t val)
-{
-	assert(!(val & 0x3f));
-	return (((val >> 6)) << A6XX_TEX_CONST_10_FLAG_BUFFER_PITCH__SHIFT) & A6XX_TEX_CONST_10_FLAG_BUFFER_PITCH__MASK;
-}
-#define A6XX_TEX_CONST_10_FLAG_BUFFER_LOGW__MASK		0x00000f00
-#define A6XX_TEX_CONST_10_FLAG_BUFFER_LOGW__SHIFT		8
-static inline uint32_t A6XX_TEX_CONST_10_FLAG_BUFFER_LOGW(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_10_FLAG_BUFFER_LOGW__SHIFT) & A6XX_TEX_CONST_10_FLAG_BUFFER_LOGW__MASK;
-}
-#define A6XX_TEX_CONST_10_FLAG_BUFFER_LOGH__MASK		0x0000f000
-#define A6XX_TEX_CONST_10_FLAG_BUFFER_LOGH__SHIFT		12
-static inline uint32_t A6XX_TEX_CONST_10_FLAG_BUFFER_LOGH(uint32_t val)
-{
-	return ((val) << A6XX_TEX_CONST_10_FLAG_BUFFER_LOGH__SHIFT) & A6XX_TEX_CONST_10_FLAG_BUFFER_LOGH__MASK;
-}
-
-#define REG_A6XX_TEX_CONST_11					0x0000000b
-
-#define REG_A6XX_TEX_CONST_12					0x0000000c
-
-#define REG_A6XX_TEX_CONST_13					0x0000000d
-
-#define REG_A6XX_TEX_CONST_14					0x0000000e
-
-#define REG_A6XX_TEX_CONST_15					0x0000000f
-
-#define REG_A6XX_UBO_0						0x00000000
-#define A6XX_UBO_0_BASE_LO__MASK				0xffffffff
-#define A6XX_UBO_0_BASE_LO__SHIFT				0
-static inline uint32_t A6XX_UBO_0_BASE_LO(uint32_t val)
-{
-	return ((val) << A6XX_UBO_0_BASE_LO__SHIFT) & A6XX_UBO_0_BASE_LO__MASK;
-}
-
-#define REG_A6XX_UBO_1						0x00000001
-#define A6XX_UBO_1_BASE_HI__MASK				0x0001ffff
-#define A6XX_UBO_1_BASE_HI__SHIFT				0
-static inline uint32_t A6XX_UBO_1_BASE_HI(uint32_t val)
-{
-	return ((val) << A6XX_UBO_1_BASE_HI__SHIFT) & A6XX_UBO_1_BASE_HI__MASK;
-}
-#define A6XX_UBO_1_SIZE__MASK					0xfffe0000
-#define A6XX_UBO_1_SIZE__SHIFT					17
-static inline uint32_t A6XX_UBO_1_SIZE(uint32_t val)
-{
-	return ((val) << A6XX_UBO_1_SIZE__SHIFT) & A6XX_UBO_1_SIZE__MASK;
-}
-
-#define REG_A6XX_PDC_GPU_ENABLE_PDC				0x00001140
-
-#define REG_A6XX_PDC_GPU_SEQ_START_ADDR				0x00001148
-
-#define REG_A6XX_PDC_GPU_TCS0_CONTROL				0x00001540
-
-#define REG_A6XX_PDC_GPU_TCS0_CMD_ENABLE_BANK			0x00001541
-
-#define REG_A6XX_PDC_GPU_TCS0_CMD_WAIT_FOR_CMPL_BANK		0x00001542
-
-#define REG_A6XX_PDC_GPU_TCS0_CMD0_MSGID			0x00001543
-
-#define REG_A6XX_PDC_GPU_TCS0_CMD0_ADDR				0x00001544
-
-#define REG_A6XX_PDC_GPU_TCS0_CMD0_DATA				0x00001545
-
-#define REG_A6XX_PDC_GPU_TCS1_CONTROL				0x00001572
-
-#define REG_A6XX_PDC_GPU_TCS1_CMD_ENABLE_BANK			0x00001573
-
-#define REG_A6XX_PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK		0x00001574
-
-#define REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID			0x00001575
-
-#define REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR				0x00001576
-
-#define REG_A6XX_PDC_GPU_TCS1_CMD0_DATA				0x00001577
-
-#define REG_A6XX_PDC_GPU_TCS2_CONTROL				0x000015a4
-
-#define REG_A6XX_PDC_GPU_TCS2_CMD_ENABLE_BANK			0x000015a5
-
-#define REG_A6XX_PDC_GPU_TCS2_CMD_WAIT_FOR_CMPL_BANK		0x000015a6
-
-#define REG_A6XX_PDC_GPU_TCS2_CMD0_MSGID			0x000015a7
-
-#define REG_A6XX_PDC_GPU_TCS2_CMD0_ADDR				0x000015a8
-
-#define REG_A6XX_PDC_GPU_TCS2_CMD0_DATA				0x000015a9
-
-#define REG_A6XX_PDC_GPU_TCS3_CONTROL				0x000015d6
-
-#define REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK			0x000015d7
-
-#define REG_A6XX_PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK		0x000015d8
-
-#define REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID			0x000015d9
-
-#define REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR				0x000015da
-
-#define REG_A6XX_PDC_GPU_TCS3_CMD0_DATA				0x000015db
-
-#define REG_A6XX_PDC_GPU_SEQ_MEM_0				0x00000000
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_SEL_A			0x00000000
-#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_A_PING_INDEX__MASK		0x000000ff
-#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_A_PING_INDEX__SHIFT		0
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_SEL_A_PING_INDEX(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_SEL_A_PING_INDEX__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_SEL_A_PING_INDEX__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_A_PING_BLK_SEL__MASK	0x0000ff00
-#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_A_PING_BLK_SEL__SHIFT	8
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_SEL_A_PING_BLK_SEL(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_SEL_A_PING_BLK_SEL__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_SEL_A_PING_BLK_SEL__MASK;
-}
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_SEL_B			0x00000001
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_SEL_C			0x00000002
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_SEL_D			0x00000003
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_CNTLT			0x00000004
-#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__MASK		0x0000003f
-#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__SHIFT		0
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU__MASK		0x00007000
-#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU__SHIFT		12
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT__MASK		0xf0000000
-#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT__SHIFT		28
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT__MASK;
-}
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_CNTLM			0x00000005
-#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__MASK		0x0f000000
-#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__SHIFT		24
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__MASK;
-}
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_IVTL_0			0x00000008
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_IVTL_1			0x00000009
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_IVTL_2			0x0000000a
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_IVTL_3			0x0000000b
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_MASKL_0			0x0000000c
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_MASKL_1			0x0000000d
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_MASKL_2			0x0000000e
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_MASKL_3			0x0000000f
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0			0x00000010
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__MASK		0x0000000f
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__SHIFT		0
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__MASK		0x000000f0
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__SHIFT		4
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__MASK		0x00000f00
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__SHIFT		8
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__MASK		0x0000f000
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__SHIFT		12
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__MASK		0x000f0000
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__SHIFT		16
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__MASK		0x00f00000
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__SHIFT		20
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__MASK		0x0f000000
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__SHIFT		24
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__MASK		0xf0000000
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__SHIFT		28
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__MASK;
-}
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1			0x00000011
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__MASK		0x0000000f
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__SHIFT		0
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__MASK		0x000000f0
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__SHIFT		4
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__MASK		0x00000f00
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__SHIFT		8
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__MASK		0x0000f000
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__SHIFT		12
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__MASK		0x000f0000
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__SHIFT		16
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__MASK		0x00f00000
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__SHIFT		20
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__MASK		0x0f000000
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__SHIFT		24
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__MASK;
-}
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__MASK		0xf0000000
-#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__SHIFT		28
-static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15(uint32_t val)
-{
-	return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__MASK;
-}
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_TRACE_BUF1			0x0000002f
-
-#define REG_A6XX_CX_DBGC_CFG_DBGBUS_TRACE_BUF2			0x00000030
-
-#define REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_0			0x00000001
-
-#define REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_1			0x00000002
-
-#define REG_A7XX_CX_MISC_TCM_RET_CNTL				0x00000039
-
-#ifdef __cplusplus
-template<chip CHIP> constexpr inline uint16_t CMD_REGS[] = {};
-template<chip CHIP> constexpr inline uint16_t RP_BLIT_REGS[] = {};
-template<> constexpr inline uint16_t CMD_REGS<A6XX>[] = {
-	0xc03,
-	0xc04,
-	0xc30,
-	0xc31,
-	0xc32,
-	0xc33,
-	0xc34,
-	0xc35,
-	0xc36,
-	0xc37,
-	0xe12,
-	0xe17,
-	0xe19,
-	0x8099,
-	0x80af,
-	0x810a,
-	0x8110,
-	0x8600,
-	0x880e,
-	0x8811,
-	0x8818,
-	0x8819,
-	0x881a,
-	0x881b,
-	0x881c,
-	0x881d,
-	0x881e,
-	0x8864,
-	0x8891,
-	0x88f0,
-	0x8927,
-	0x8928,
-	0x8e01,
-	0x8e04,
-	0x8e07,
-	0x9210,
-	0x9211,
-	0x9218,
-	0x9219,
-	0x921a,
-	0x921b,
-	0x921c,
-	0x921d,
-	0x921e,
-	0x921f,
-	0x9220,
-	0x9221,
-	0x9222,
-	0x9223,
-	0x9224,
-	0x9225,
-	0x9226,
-	0x9227,
-	0x9228,
-	0x9229,
-	0x922a,
-	0x922b,
-	0x922c,
-	0x922d,
-	0x922e,
-	0x922f,
-	0x9230,
-	0x9231,
-	0x9232,
-	0x9233,
-	0x9234,
-	0x9235,
-	0x9236,
-	0x9300,
-	0x9600,
-	0x9601,
-	0x9602,
-	0x9e08,
-	0x9e09,
-	0x9e72,
-	0xa007,
-	0xa009,
-	0xa8a0,
-	0xa8a1,
-	0xa8a2,
-	0xa8a3,
-	0xa8a4,
-	0xa8a5,
-	0xa8a6,
-	0xa8a7,
-	0xa8a8,
-	0xa8a9,
-	0xa8aa,
-	0xa8ab,
-	0xa8ac,
-	0xa8ad,
-	0xa8ae,
-	0xa8af,
-	0xa9a8,
-	0xa9b0,
-	0xa9b1,
-	0xa9b2,
-	0xa9b3,
-	0xa9b4,
-	0xa9b5,
-	0xa9b6,
-	0xa9b7,
-	0xa9b8,
-	0xa9b9,
-	0xa9ba,
-	0xa9bb,
-	0xa9bc,
-	0xa9bd,
-	0xa9c2,
-	0xa9c3,
-	0xa9e2,
-	0xa9e3,
-	0xa9e6,
-	0xa9e7,
-	0xa9e8,
-	0xa9e9,
-	0xa9ea,
-	0xa9eb,
-	0xa9ec,
-	0xa9ed,
-	0xa9ee,
-	0xa9ef,
-	0xa9f0,
-	0xa9f1,
-	0xaaf2,
-	0xab1a,
-	0xab1b,
-	0xab20,
-	0xae00,
-	0xae03,
-	0xae04,
-	0xae0f,
-	0xb180,
-	0xb181,
-	0xb182,
-	0xb183,
-	0xb302,
-	0xb303,
-	0xb309,
-	0xb600,
-	0xb602,
-	0xb605,
-	0xb987,
-	0xb9d0,
-	0xbb08,
-	0xbb11,
-	0xbb20,
-	0xbb21,
-	0xbb22,
-	0xbb23,
-	0xbb24,
-	0xbb25,
-	0xbb26,
-	0xbb27,
-	0xbb28,
-	0xbb29,
-	0xbe00,
-	0xbe01,
-	0xbe04,
-};
-template<> constexpr inline uint16_t CMD_REGS<A7XX>[] = {
-	0xc03,
-	0xc04,
-	0xc30,
-	0xc31,
-	0xc32,
-	0xc33,
-	0xc34,
-	0xc35,
-	0xc36,
-	0xc37,
-	0xce2,
-	0xce3,
-	0xce4,
-	0xce5,
-	0xce6,
-	0xce7,
-	0xe10,
-	0xe11,
-	0xe12,
-	0xe17,
-	0xe19,
-	0x8008,
-	0x8009,
-	0x800a,
-	0x800b,
-	0x800c,
-	0x8099,
-	0x80a7,
-	0x80af,
-	0x80f4,
-	0x80f5,
-	0x80f5,
-	0x80f6,
-	0x80f6,
-	0x80f7,
-	0x80f8,
-	0x80f9,
-	0x80f9,
-	0x80fa,
-	0x80fa,
-	0x80fb,
-	0x810a,
-	0x810b,
-	0x8110,
-	0x8120,
-	0x8121,
-	0x8600,
-	0x880e,
-	0x8811,
-	0x8818,
-	0x8819,
-	0x881a,
-	0x881b,
-	0x881c,
-	0x881d,
-	0x881e,
-	0x8864,
-	0x8891,
-	0x8899,
-	0x88e5,
-	0x88f0,
-	0x8927,
-	0x8928,
-	0x8e01,
-	0x8e04,
-	0x8e06,
-	0x8e07,
-	0x8e09,
-	0x8e79,
-	0x9218,
-	0x9219,
-	0x921a,
-	0x921b,
-	0x921c,
-	0x921d,
-	0x921e,
-	0x921f,
-	0x9220,
-	0x9221,
-	0x9222,
-	0x9223,
-	0x9224,
-	0x9225,
-	0x9226,
-	0x9227,
-	0x9228,
-	0x9229,
-	0x922a,
-	0x922b,
-	0x922c,
-	0x922d,
-	0x922e,
-	0x922f,
-	0x9230,
-	0x9231,
-	0x9232,
-	0x9233,
-	0x9234,
-	0x9235,
-	0x9236,
-	0x9300,
-	0x9600,
-	0x9601,
-	0x9602,
-	0x9810,
-	0x9811,
-	0x9e24,
-	0x9e72,
-	0xa007,
-	0xa009,
-	0xa600,
-	0xa82d,
-	0xa82f,
-	0xa868,
-	0xa899,
-	0xa8a0,
-	0xa8a1,
-	0xa8a2,
-	0xa8a3,
-	0xa8a4,
-	0xa8a5,
-	0xa8a6,
-	0xa8a7,
-	0xa8a8,
-	0xa8a9,
-	0xa8aa,
-	0xa8ab,
-	0xa8ac,
-	0xa8ad,
-	0xa8ae,
-	0xa8af,
-	0xa9a8,
-	0xa9ac,
-	0xa9ad,
-	0xa9b0,
-	0xa9b1,
-	0xa9b2,
-	0xa9b3,
-	0xa9b4,
-	0xa9b5,
-	0xa9b6,
-	0xa9b7,
-	0xa9b8,
-	0xa9b9,
-	0xa9ba,
-	0xa9bb,
-	0xa9bc,
-	0xa9bd,
-	0xa9be,
-	0xa9c2,
-	0xa9c3,
-	0xa9c5,
-	0xa9cd,
-	0xa9df,
-	0xa9e2,
-	0xa9e3,
-	0xa9e6,
-	0xa9e7,
-	0xa9e8,
-	0xa9e9,
-	0xa9ea,
-	0xa9eb,
-	0xa9ec,
-	0xa9ed,
-	0xa9ee,
-	0xa9ef,
-	0xa9f0,
-	0xa9f1,
-	0xa9f2,
-	0xa9f3,
-	0xa9f4,
-	0xa9f5,
-	0xa9f6,
-	0xa9f7,
-	0xaa01,
-	0xaa02,
-	0xaa03,
-	0xaaf2,
-	0xab01,
-	0xab02,
-	0xab1a,
-	0xab1b,
-	0xab1f,
-	0xab20,
-	0xab22,
-	0xae00,
-	0xae03,
-	0xae04,
-	0xae06,
-	0xae08,
-	0xae09,
-	0xae0a,
-	0xae0f,
-	0xae6a,
-	0xae6b,
-	0xae6c,
-	0xae73,
-	0xb180,
-	0xb181,
-	0xb182,
-	0xb183,
-	0xb302,
-	0xb303,
-	0xb309,
-	0xb310,
-	0xb600,
-	0xb602,
-	0xb608,
-	0xb609,
-	0xb60a,
-	0xb60b,
-	0xb60c,
-};
-template<> constexpr inline uint16_t RP_BLIT_REGS<A6XX>[] = {
-	0xc02,
-	0xc06,
-	0xc10,
-	0xc11,
-	0xc12,
-	0xc13,
-	0xc14,
-	0xc15,
-	0xc16,
-	0xc17,
-	0xc18,
-	0xc19,
-	0xc1a,
-	0xc1b,
-	0xc1c,
-	0xc1d,
-	0xc1e,
-	0xc1f,
-	0xc20,
-	0xc21,
-	0xc22,
-	0xc23,
-	0xc24,
-	0xc25,
-	0xc26,
-	0xc27,
-	0xc28,
-	0xc29,
-	0xc2a,
-	0xc2b,
-	0xc2c,
-	0xc2d,
-	0xc2e,
-	0xc2f,
-	0xc38,
-	0xc39,
-	0xc3a,
-	0xc3b,
-	0xc3c,
-	0xc3d,
-	0xc3e,
-	0xc3f,
-	0xc40,
-	0xc41,
-	0xc42,
-	0xc43,
-	0xc44,
-	0xc45,
-	0xc46,
-	0xc47,
-	0xc48,
-	0xc49,
-	0xc4a,
-	0xc4b,
-	0xc4c,
-	0xc4d,
-	0xc4e,
-	0xc4f,
-	0xc50,
-	0xc51,
-	0xc52,
-	0xc53,
-	0xc54,
-	0xc55,
-	0xc56,
-	0xc57,
-	0xc58,
-	0xc59,
-	0xc5a,
-	0xc5b,
-	0xc5c,
-	0xc5d,
-	0xc5e,
-	0xc5f,
-	0xc60,
-	0xc61,
-	0xc62,
-	0xc63,
-	0xc64,
-	0xc65,
-	0xc66,
-	0xc67,
-	0xc68,
-	0xc69,
-	0xc6a,
-	0xc6b,
-	0xc6c,
-	0xc6d,
-	0xc6e,
-	0xc6f,
-	0xc70,
-	0xc71,
-	0xc72,
-	0xc73,
-	0xc74,
-	0xc75,
-	0xc76,
-	0xc77,
-	0xc78,
-	0xc79,
-	0xc7a,
-	0xc7b,
-	0xc7c,
-	0xc7d,
-	0xc7e,
-	0xc7f,
-	0xc80,
-	0xc81,
-	0xc82,
-	0xc83,
-	0xc84,
-	0xc85,
-	0xc86,
-	0xc87,
-	0xc88,
-	0xc89,
-	0xc8a,
-	0xc8b,
-	0xc8c,
-	0xc8d,
-	0xc8e,
-	0xc8f,
-	0xc90,
-	0xc91,
-	0xc92,
-	0xc93,
-	0xc94,
-	0xc95,
-	0xc96,
-	0xc97,
-	0x8000,
-	0x8001,
-	0x8002,
-	0x8003,
-	0x8004,
-	0x8005,
-	0x8006,
-	0x8010,
-	0x8011,
-	0x8012,
-	0x8013,
-	0x8014,
-	0x8015,
-	0x8016,
-	0x8017,
-	0x8018,
-	0x8019,
-	0x801a,
-	0x801b,
-	0x801c,
-	0x801d,
-	0x801e,
-	0x801f,
-	0x8020,
-	0x8021,
-	0x8022,
-	0x8023,
-	0x8024,
-	0x8025,
-	0x8026,
-	0x8027,
-	0x8028,
-	0x8029,
-	0x802a,
-	0x802b,
-	0x802c,
-	0x802d,
-	0x802e,
-	0x802f,
-	0x8030,
-	0x8031,
-	0x8032,
-	0x8033,
-	0x8034,
-	0x8035,
-	0x8036,
-	0x8037,
-	0x8038,
-	0x8039,
-	0x803a,
-	0x803b,
-	0x803c,
-	0x803d,
-	0x803e,
-	0x803f,
-	0x8040,
-	0x8041,
-	0x8042,
-	0x8043,
-	0x8044,
-	0x8045,
-	0x8046,
-	0x8047,
-	0x8048,
-	0x8049,
-	0x804a,
-	0x804b,
-	0x804c,
-	0x804d,
-	0x804e,
-	0x804f,
-	0x8050,
-	0x8051,
-	0x8052,
-	0x8053,
-	0x8054,
-	0x8055,
-	0x8056,
-	0x8057,
-	0x8058,
-	0x8059,
-	0x805a,
-	0x805b,
-	0x805c,
-	0x805d,
-	0x805e,
-	0x805f,
-	0x8060,
-	0x8061,
-	0x8062,
-	0x8063,
-	0x8064,
-	0x8065,
-	0x8066,
-	0x8067,
-	0x8068,
-	0x8069,
-	0x806a,
-	0x806b,
-	0x806c,
-	0x806d,
-	0x806e,
-	0x806f,
-	0x8070,
-	0x8071,
-	0x8072,
-	0x8073,
-	0x8074,
-	0x8075,
-	0x8076,
-	0x8077,
-	0x8078,
-	0x8079,
-	0x807a,
-	0x807b,
-	0x807c,
-	0x807d,
-	0x807e,
-	0x807f,
-	0x8080,
-	0x8081,
-	0x8082,
-	0x8083,
-	0x8084,
-	0x8085,
-	0x8086,
-	0x8087,
-	0x8088,
-	0x8089,
-	0x808a,
-	0x808b,
-	0x808c,
-	0x808d,
-	0x808e,
-	0x808f,
-	0x8090,
-	0x8091,
-	0x8092,
-	0x8094,
-	0x8095,
-	0x8096,
-	0x8097,
-	0x8098,
-	0x809b,
-	0x809c,
-	0x809d,
-	0x80a0,
-	0x80a1,
-	0x80a2,
-	0x80a3,
-	0x80a4,
-	0x80a5,
-	0x80a6,
-	0x80b0,
-	0x80b1,
-	0x80b2,
-	0x80b3,
-	0x80b4,
-	0x80b5,
-	0x80b6,
-	0x80b7,
-	0x80b8,
-	0x80b9,
-	0x80ba,
-	0x80bb,
-	0x80bc,
-	0x80bd,
-	0x80be,
-	0x80bf,
-	0x80c0,
-	0x80c1,
-	0x80c2,
-	0x80c3,
-	0x80c4,
-	0x80c5,
-	0x80c6,
-	0x80c7,
-	0x80c8,
-	0x80c9,
-	0x80ca,
-	0x80cb,
-	0x80cc,
-	0x80cd,
-	0x80ce,
-	0x80cf,
-	0x80d0,
-	0x80d1,
-	0x80d2,
-	0x80d3,
-	0x80d4,
-	0x80d5,
-	0x80d6,
-	0x80d7,
-	0x80d8,
-	0x80d9,
-	0x80da,
-	0x80db,
-	0x80dc,
-	0x80dd,
-	0x80de,
-	0x80df,
-	0x80e0,
-	0x80e1,
-	0x80e2,
-	0x80e3,
-	0x80e4,
-	0x80e5,
-	0x80e6,
-	0x80e7,
-	0x80e8,
-	0x80e9,
-	0x80ea,
-	0x80eb,
-	0x80ec,
-	0x80ed,
-	0x80ee,
-	0x80ef,
-	0x80f0,
-	0x80f1,
-	0x8100,
-	0x8101,
-	0x8102,
-	0x8103,
-	0x8104,
-	0x8105,
-	0x8106,
-	0x8107,
-	0x8109,
-	0x8114,
-	0x8115,
-	0x8400,
-	0x8401,
-	0x8402,
-	0x8403,
-	0x8404,
-	0x8405,
-	0x8406,
-	0x840a,
-	0x840b,
-	0x8800,
-	0x8801,
-	0x8802,
-	0x8803,
-	0x8804,
-	0x8805,
-	0x8806,
-	0x8809,
-	0x880a,
-	0x880b,
-	0x880c,
-	0x880d,
-	0x880f,
-	0x8810,
-	0x8820,
-	0x8821,
-	0x8822,
-	0x8823,
-	0x8824,
-	0x8825,
-	0x8826,
-	0x8827,
-	0x8828,
-	0x8829,
-	0x882a,
-	0x882b,
-	0x882c,
-	0x882d,
-	0x882e,
-	0x882f,
-	0x8830,
-	0x8831,
-	0x8832,
-	0x8833,
-	0x8834,
-	0x8835,
-	0x8836,
-	0x8837,
-	0x8838,
-	0x8839,
-	0x883a,
-	0x883b,
-	0x883c,
-	0x883d,
-	0x883e,
-	0x883f,
-	0x8840,
-	0x8841,
-	0x8842,
-	0x8843,
-	0x8844,
-	0x8845,
-	0x8846,
-	0x8847,
-	0x8848,
-	0x8849,
-	0x884a,
-	0x884b,
-	0x884c,
-	0x884d,
-	0x884e,
-	0x884f,
-	0x8850,
-	0x8851,
-	0x8852,
-	0x8853,
-	0x8854,
-	0x8855,
-	0x8856,
-	0x8857,
-	0x8858,
-	0x8859,
-	0x885a,
-	0x885b,
-	0x885c,
-	0x885d,
-	0x885e,
-	0x885f,
-	0x8860,
-	0x8861,
-	0x8862,
-	0x8863,
-	0x8865,
-	0x8870,
-	0x8871,
-	0x8872,
-	0x8873,
-	0x8874,
-	0x8875,
-	0x8876,
-	0x8877,
-	0x8878,
-	0x8879,
-	0x8880,
-	0x8881,
-	0x8882,
-	0x8883,
-	0x8884,
-	0x8885,
-	0x8886,
-	0x8887,
-	0x8888,
-	0x8889,
-	0x8890,
-	0x8898,
-	0x88c0,
-	0x88c1,
-	0x88d0,
-	0x88d1,
-	0x88d2,
-	0x88d3,
-	0x88d4,
-	0x88d5,
-	0x88d6,
-	0x88d7,
-	0x88d8,
-	0x88d9,
-	0x88da,
-	0x88db,
-	0x88dc,
-	0x88dd,
-	0x88de,
-	0x88df,
-	0x88e0,
-	0x88e1,
-	0x88e2,
-	0x88e3,
-	0x8900,
-	0x8901,
-	0x8902,
-	0x8903,
-	0x8904,
-	0x8905,
-	0x8906,
-	0x8907,
-	0x8908,
-	0x8909,
-	0x890a,
-	0x890b,
-	0x890c,
-	0x890d,
-	0x890e,
-	0x890f,
-	0x8910,
-	0x8911,
-	0x8912,
-	0x8913,
-	0x8914,
-	0x8915,
-	0x8916,
-	0x8917,
-	0x8918,
-	0x8919,
-	0x891a,
-	0x8a00,
-	0x8a10,
-	0x8a20,
-	0x8a30,
-	0x8c00,
-	0x8c01,
-	0x8c17,
-	0x8c18,
-	0x8c19,
-	0x8c1a,
-	0x8c1b,
-	0x8c1c,
-	0x8c1d,
-	0x8c1e,
-	0x8c1f,
-	0x8c20,
-	0x8c21,
-	0x8c22,
-	0x8c23,
-	0x8c24,
-	0x8c25,
-	0x8c2c,
-	0x8c2d,
-	0x8c2e,
-	0x8c2f,
-	0x9100,
-	0x9101,
-	0x9102,
-	0x9103,
-	0x9104,
-	0x9105,
-	0x9106,
-	0x9107,
-	0x9108,
-	0x9200,
-	0x9201,
-	0x9202,
-	0x9203,
-	0x9204,
-	0x9205,
-	0x9206,
-	0x9207,
-	0x9208,
-	0x9209,
-	0x920a,
-	0x920b,
-	0x920c,
-	0x920d,
-	0x920e,
-	0x920f,
-	0x9212,
-	0x9213,
-	0x9214,
-	0x9215,
-	0x9216,
-	0x9217,
-	0x9301,
-	0x9302,
-	0x9303,
-	0x9304,
-	0x9305,
-	0x9306,
-	0x9311,
-	0x9312,
-	0x9313,
-	0x9314,
-	0x9315,
-	0x9316,
-	0x9800,
-	0x9801,
-	0x9802,
-	0x9803,
-	0x9804,
-	0x9805,
-	0x9806,
-	0x9808,
-	0x9980,
-	0x9981,
-	0x9b00,
-	0x9b01,
-	0x9b02,
-	0x9b03,
-	0x9b04,
-	0x9b05,
-	0x9b06,
-	0x9b07,
-	0x9b08,
-	0xa000,
-	0xa001,
-	0xa002,
-	0xa003,
-	0xa004,
-	0xa005,
-	0xa006,
-	0xa008,
-	0xa00e,
-	0xa00f,
-	0xa010,
-	0xa011,
-	0xa012,
-	0xa013,
-	0xa014,
-	0xa015,
-	0xa016,
-	0xa017,
-	0xa018,
-	0xa019,
-	0xa01a,
-	0xa01b,
-	0xa01c,
-	0xa01d,
-	0xa01e,
-	0xa01f,
-	0xa020,
-	0xa021,
-	0xa022,
-	0xa023,
-	0xa024,
-	0xa025,
-	0xa026,
-	0xa027,
-	0xa028,
-	0xa029,
-	0xa02a,
-	0xa02b,
-	0xa02c,
-	0xa02d,
-	0xa02e,
-	0xa02f,
-	0xa030,
-	0xa031,
-	0xa032,
-	0xa033,
-	0xa034,
-	0xa035,
-	0xa036,
-	0xa037,
-	0xa038,
-	0xa039,
-	0xa03a,
-	0xa03b,
-	0xa03c,
-	0xa03d,
-	0xa03e,
-	0xa03f,
-	0xa040,
-	0xa041,
-	0xa042,
-	0xa043,
-	0xa044,
-	0xa045,
-	0xa046,
-	0xa047,
-	0xa048,
-	0xa049,
-	0xa04a,
-	0xa04b,
-	0xa04c,
-	0xa04d,
-	0xa04e,
-	0xa04f,
-	0xa050,
-	0xa051,
-	0xa052,
-	0xa053,
-	0xa054,
-	0xa055,
-	0xa056,
-	0xa057,
-	0xa058,
-	0xa059,
-	0xa05a,
-	0xa05b,
-	0xa05c,
-	0xa05d,
-	0xa05e,
-	0xa05f,
-	0xa060,
-	0xa061,
-	0xa062,
-	0xa063,
-	0xa064,
-	0xa065,
-	0xa066,
-	0xa067,
-	0xa068,
-	0xa069,
-	0xa06a,
-	0xa06b,
-	0xa06c,
-	0xa06d,
-	0xa06e,
-	0xa06f,
-	0xa070,
-	0xa071,
-	0xa072,
-	0xa073,
-	0xa074,
-	0xa075,
-	0xa076,
-	0xa077,
-	0xa078,
-	0xa079,
-	0xa07a,
-	0xa07b,
-	0xa07c,
-	0xa07d,
-	0xa07e,
-	0xa07f,
-	0xa080,
-	0xa081,
-	0xa082,
-	0xa083,
-	0xa084,
-	0xa085,
-	0xa086,
-	0xa087,
-	0xa088,
-	0xa089,
-	0xa08a,
-	0xa08b,
-	0xa08c,
-	0xa08d,
-	0xa08e,
-	0xa08f,
-	0xa090,
-	0xa091,
-	0xa092,
-	0xa093,
-	0xa094,
-	0xa095,
-	0xa096,
-	0xa097,
-	0xa098,
-	0xa099,
-	0xa09a,
-	0xa09b,
-	0xa09c,
-	0xa09d,
-	0xa09e,
-	0xa09f,
-	0xa0a0,
-	0xa0a1,
-	0xa0a2,
-	0xa0a3,
-	0xa0a4,
-	0xa0a5,
-	0xa0a6,
-	0xa0a7,
-	0xa0a8,
-	0xa0a9,
-	0xa0aa,
-	0xa0ab,
-	0xa0ac,
-	0xa0ad,
-	0xa0ae,
-	0xa0af,
-	0xa0b0,
-	0xa0b1,
-	0xa0b2,
-	0xa0b3,
-	0xa0b4,
-	0xa0b5,
-	0xa0b6,
-	0xa0b7,
-	0xa0b8,
-	0xa0b9,
-	0xa0ba,
-	0xa0bb,
-	0xa0bc,
-	0xa0bd,
-	0xa0be,
-	0xa0bf,
-	0xa0c0,
-	0xa0c1,
-	0xa0c2,
-	0xa0c3,
-	0xa0c4,
-	0xa0c5,
-	0xa0c6,
-	0xa0c7,
-	0xa0c8,
-	0xa0c9,
-	0xa0ca,
-	0xa0cb,
-	0xa0cc,
-	0xa0cd,
-	0xa0ce,
-	0xa0cf,
-	0xa0d0,
-	0xa0d1,
-	0xa0d2,
-	0xa0d3,
-	0xa0d4,
-	0xa0d5,
-	0xa0d6,
-	0xa0d7,
-	0xa0d8,
-	0xa0d9,
-	0xa0da,
-	0xa0db,
-	0xa0dc,
-	0xa0dd,
-	0xa0de,
-	0xa0df,
-	0xa0e0,
-	0xa0e1,
-	0xa0e2,
-	0xa0e3,
-	0xa0e4,
-	0xa0e5,
-	0xa0e6,
-	0xa0e7,
-	0xa0e8,
-	0xa0e9,
-	0xa0ea,
-	0xa0eb,
-	0xa0ec,
-	0xa0ed,
-	0xa0ee,
-	0xa0ef,
-	0xa0f8,
-	0xa800,
-	0xa802,
-	0xa803,
-	0xa804,
-	0xa805,
-	0xa806,
-	0xa807,
-	0xa808,
-	0xa809,
-	0xa80a,
-	0xa80b,
-	0xa80c,
-	0xa80d,
-	0xa80e,
-	0xa80f,
-	0xa810,
-	0xa811,
-	0xa812,
-	0xa813,
-	0xa814,
-	0xa815,
-	0xa816,
-	0xa817,
-	0xa818,
-	0xa819,
-	0xa81a,
-	0xa81b,
-	0xa81c,
-	0xa81d,
-	0xa81e,
-	0xa81f,
-	0xa820,
-	0xa821,
-	0xa822,
-	0xa823,
-	0xa824,
-	0xa825,
-	0xa830,
-	0xa831,
-	0xa832,
-	0xa833,
-	0xa834,
-	0xa835,
-	0xa836,
-	0xa837,
-	0xa838,
-	0xa839,
-	0xa83a,
-	0xa83b,
-	0xa83c,
-	0xa83d,
-	0xa840,
-	0xa842,
-	0xa843,
-	0xa844,
-	0xa845,
-	0xa846,
-	0xa847,
-	0xa848,
-	0xa849,
-	0xa84a,
-	0xa84b,
-	0xa84c,
-	0xa84d,
-	0xa84e,
-	0xa84f,
-	0xa850,
-	0xa851,
-	0xa852,
-	0xa853,
-	0xa854,
-	0xa855,
-	0xa856,
-	0xa857,
-	0xa858,
-	0xa859,
-	0xa85a,
-	0xa85b,
-	0xa85c,
-	0xa85d,
-	0xa85e,
-	0xa85f,
-	0xa860,
-	0xa861,
-	0xa862,
-	0xa863,
-	0xa864,
-	0xa865,
-	0xa870,
-	0xa871,
-	0xa872,
-	0xa873,
-	0xa874,
-	0xa875,
-	0xa876,
-	0xa877,
-	0xa878,
-	0xa879,
-	0xa87a,
-	0xa87b,
-	0xa87c,
-	0xa87d,
-	0xa87e,
-	0xa87f,
-	0xa880,
-	0xa881,
-	0xa882,
-	0xa883,
-	0xa884,
-	0xa885,
-	0xa886,
-	0xa887,
-	0xa888,
-	0xa889,
-	0xa88a,
-	0xa88b,
-	0xa88c,
-	0xa88d,
-	0xa88e,
-	0xa88f,
-	0xa890,
-	0xa891,
-	0xa892,
-	0xa893,
-	0xa894,
-	0xa895,
-	0xa896,
-	0xa980,
-	0xa982,
-	0xa983,
-	0xa984,
-	0xa985,
-	0xa986,
-	0xa987,
-	0xa988,
-	0xa989,
-	0xa98a,
-	0xa98b,
-	0xa98c,
-	0xa98d,
-	0xa98e,
-	0xa98f,
-	0xa990,
-	0xa991,
-	0xa992,
-	0xa993,
-	0xa994,
-	0xa995,
-	0xa996,
-	0xa997,
-	0xa998,
-	0xa999,
-	0xa99a,
-	0xa99b,
-	0xa99c,
-	0xa99d,
-	0xa99e,
-	0xa99f,
-	0xa9a0,
-	0xa9a1,
-	0xa9a2,
-	0xa9a3,
-	0xa9a4,
-	0xa9a5,
-	0xa9a6,
-	0xa9a7,
-	0xa9a9,
-	0xa9e0,
-	0xa9e1,
-	0xa9e4,
-	0xa9e5,
-	0xab00,
-	0xab04,
-	0xab05,
-	0xab10,
-	0xab11,
-	0xab12,
-	0xab13,
-	0xab14,
-	0xab15,
-	0xab16,
-	0xab17,
-	0xab18,
-	0xab19,
-	0xacc0,
-	0xb300,
-	0xb301,
-	0xb304,
-	0xb305,
-	0xb306,
-	0xb307,
-	0xb4c0,
-	0xb4c1,
-	0xb4c2,
-	0xb4c3,
-	0xb4c4,
-	0xb4ca,
-	0xb4cb,
-	0xb4cc,
-	0xb4d1,
-	0xb800,
-	0xb801,
-	0xb802,
-	0xb803,
-	0xb980,
-	0xb982,
-	0xb983,
-	0xb984,
-	0xb985,
-	0xb986,
-	0xb990,
-	0xb991,
-	0xb992,
-	0xb993,
-	0xb994,
-	0xb995,
-	0xb996,
-	0xb997,
-	0xb998,
-	0xb999,
-	0xb99a,
-	0xb99b,
-	0xb9c0,
-	0xb9c1,
-	0xb9c2,
-	0xb9c3,
-	0xb9c4,
-	0xb9c5,
-	0xb9c6,
-	0xb9c7,
-	0xb9c8,
-	0xb9c9,
-	0xbb10,
-};
-template<> constexpr inline uint16_t RP_BLIT_REGS<A7XX>[] = {
-	0xc02,
-	0xc06,
-	0xc10,
-	0xc11,
-	0xc12,
-	0xc13,
-	0xc14,
-	0xc15,
-	0xc16,
-	0xc17,
-	0xc18,
-	0xc19,
-	0xc1a,
-	0xc1b,
-	0xc1c,
-	0xc1d,
-	0xc1e,
-	0xc1f,
-	0xc20,
-	0xc21,
-	0xc22,
-	0xc23,
-	0xc24,
-	0xc25,
-	0xc26,
-	0xc27,
-	0xc28,
-	0xc29,
-	0xc2a,
-	0xc2b,
-	0xc2c,
-	0xc2d,
-	0xc2e,
-	0xc2f,
-	0xc38,
-	0xc39,
-	0xc3a,
-	0xc3b,
-	0xc3c,
-	0xc3d,
-	0xc3e,
-	0xc3f,
-	0xc40,
-	0xc41,
-	0xc42,
-	0xc43,
-	0xc44,
-	0xc45,
-	0xc46,
-	0xc47,
-	0xc48,
-	0xc49,
-	0xc4a,
-	0xc4b,
-	0xc4c,
-	0xc4d,
-	0xc4e,
-	0xc4f,
-	0xc50,
-	0xc51,
-	0xc52,
-	0xc53,
-	0xc54,
-	0xc55,
-	0xc56,
-	0xc57,
-	0x8000,
-	0x8001,
-	0x8002,
-	0x8003,
-	0x8004,
-	0x8005,
-	0x8006,
-	0x8007,
-	0x8010,
-	0x8011,
-	0x8012,
-	0x8013,
-	0x8014,
-	0x8015,
-	0x8016,
-	0x8017,
-	0x8018,
-	0x8019,
-	0x801a,
-	0x801b,
-	0x801c,
-	0x801d,
-	0x801e,
-	0x801f,
-	0x8020,
-	0x8021,
-	0x8022,
-	0x8023,
-	0x8024,
-	0x8025,
-	0x8026,
-	0x8027,
-	0x8028,
-	0x8029,
-	0x802a,
-	0x802b,
-	0x802c,
-	0x802d,
-	0x802e,
-	0x802f,
-	0x8030,
-	0x8031,
-	0x8032,
-	0x8033,
-	0x8034,
-	0x8035,
-	0x8036,
-	0x8037,
-	0x8038,
-	0x8039,
-	0x803a,
-	0x803b,
-	0x803c,
-	0x803d,
-	0x803e,
-	0x803f,
-	0x8040,
-	0x8041,
-	0x8042,
-	0x8043,
-	0x8044,
-	0x8045,
-	0x8046,
-	0x8047,
-	0x8048,
-	0x8049,
-	0x804a,
-	0x804b,
-	0x804c,
-	0x804d,
-	0x804e,
-	0x804f,
-	0x8050,
-	0x8051,
-	0x8052,
-	0x8053,
-	0x8054,
-	0x8055,
-	0x8056,
-	0x8057,
-	0x8058,
-	0x8059,
-	0x805a,
-	0x805b,
-	0x805c,
-	0x805d,
-	0x805e,
-	0x805f,
-	0x8060,
-	0x8061,
-	0x8062,
-	0x8063,
-	0x8064,
-	0x8065,
-	0x8066,
-	0x8067,
-	0x8068,
-	0x8069,
-	0x806a,
-	0x806b,
-	0x806c,
-	0x806d,
-	0x806e,
-	0x806f,
-	0x8070,
-	0x8071,
-	0x8072,
-	0x8073,
-	0x8074,
-	0x8075,
-	0x8076,
-	0x8077,
-	0x8078,
-	0x8079,
-	0x807a,
-	0x807b,
-	0x807c,
-	0x807d,
-	0x807e,
-	0x807f,
-	0x8080,
-	0x8081,
-	0x8082,
-	0x8083,
-	0x8084,
-	0x8085,
-	0x8086,
-	0x8087,
-	0x8088,
-	0x8089,
-	0x808a,
-	0x808b,
-	0x808c,
-	0x808d,
-	0x808e,
-	0x808f,
-	0x8090,
-	0x8091,
-	0x8092,
-	0x8094,
-	0x8095,
-	0x8096,
-	0x8097,
-	0x8098,
-	0x809b,
-	0x809c,
-	0x809d,
-	0x80a0,
-	0x80a1,
-	0x80a2,
-	0x80a3,
-	0x80a4,
-	0x80a5,
-	0x80a6,
-	0x80b0,
-	0x80b1,
-	0x80b2,
-	0x80b3,
-	0x80b4,
-	0x80b5,
-	0x80b6,
-	0x80b7,
-	0x80b8,
-	0x80b9,
-	0x80ba,
-	0x80bb,
-	0x80bc,
-	0x80bd,
-	0x80be,
-	0x80bf,
-	0x80c0,
-	0x80c1,
-	0x80c2,
-	0x80c3,
-	0x80c4,
-	0x80c5,
-	0x80c6,
-	0x80c7,
-	0x80c8,
-	0x80c9,
-	0x80ca,
-	0x80cb,
-	0x80cc,
-	0x80cd,
-	0x80ce,
-	0x80cf,
-	0x80d0,
-	0x80d1,
-	0x80d2,
-	0x80d3,
-	0x80d4,
-	0x80d5,
-	0x80d6,
-	0x80d7,
-	0x80d8,
-	0x80d9,
-	0x80da,
-	0x80db,
-	0x80dc,
-	0x80dd,
-	0x80de,
-	0x80df,
-	0x80e0,
-	0x80e1,
-	0x80e2,
-	0x80e3,
-	0x80e4,
-	0x80e5,
-	0x80e6,
-	0x80e7,
-	0x80e8,
-	0x80e9,
-	0x80ea,
-	0x80eb,
-	0x80ec,
-	0x80ed,
-	0x80ee,
-	0x80ef,
-	0x80f0,
-	0x80f1,
-	0x8100,
-	0x8101,
-	0x8102,
-	0x8103,
-	0x8104,
-	0x8105,
-	0x8106,
-	0x8107,
-	0x8109,
-	0x8113,
-	0x8114,
-	0x8115,
-	0x8116,
-	0x8400,
-	0x8401,
-	0x8402,
-	0x8403,
-	0x8404,
-	0x8405,
-	0x8406,
-	0x840a,
-	0x840b,
-	0x8800,
-	0x8801,
-	0x8802,
-	0x8803,
-	0x8804,
-	0x8805,
-	0x8806,
-	0x8809,
-	0x880a,
-	0x880b,
-	0x880c,
-	0x880d,
-	0x880f,
-	0x8810,
-	0x8812,
-	0x8820,
-	0x8821,
-	0x8822,
-	0x8823,
-	0x8824,
-	0x8825,
-	0x8826,
-	0x8827,
-	0x8828,
-	0x8829,
-	0x882a,
-	0x882b,
-	0x882c,
-	0x882d,
-	0x882e,
-	0x882f,
-	0x8830,
-	0x8831,
-	0x8832,
-	0x8833,
-	0x8834,
-	0x8835,
-	0x8836,
-	0x8837,
-	0x8838,
-	0x8839,
-	0x883a,
-	0x883b,
-	0x883c,
-	0x883d,
-	0x883e,
-	0x883f,
-	0x8840,
-	0x8841,
-	0x8842,
-	0x8843,
-	0x8844,
-	0x8845,
-	0x8846,
-	0x8847,
-	0x8848,
-	0x8849,
-	0x884a,
-	0x884b,
-	0x884c,
-	0x884d,
-	0x884e,
-	0x884f,
-	0x8850,
-	0x8851,
-	0x8852,
-	0x8853,
-	0x8854,
-	0x8855,
-	0x8856,
-	0x8857,
-	0x8858,
-	0x8859,
-	0x885a,
-	0x885b,
-	0x885c,
-	0x885d,
-	0x885e,
-	0x885f,
-	0x8860,
-	0x8861,
-	0x8862,
-	0x8863,
-	0x8865,
-	0x8870,
-	0x8871,
-	0x8872,
-	0x8873,
-	0x8874,
-	0x8875,
-	0x8876,
-	0x8877,
-	0x8878,
-	0x8879,
-	0x8880,
-	0x8881,
-	0x8882,
-	0x8883,
-	0x8884,
-	0x8885,
-	0x8886,
-	0x8887,
-	0x8888,
-	0x8889,
-	0x8890,
-	0x8898,
-	0x88c0,
-	0x88c1,
-	0x88d0,
-	0x88d1,
-	0x88d2,
-	0x88d3,
-	0x88d4,
-	0x88d5,
-	0x88d6,
-	0x88d7,
-	0x88d8,
-	0x88d9,
-	0x88da,
-	0x88db,
-	0x88dc,
-	0x88dd,
-	0x88de,
-	0x88df,
-	0x88e0,
-	0x88e1,
-	0x88e2,
-	0x88e3,
-	0x8900,
-	0x8901,
-	0x8902,
-	0x8903,
-	0x8904,
-	0x8905,
-	0x8906,
-	0x8907,
-	0x8908,
-	0x8909,
-	0x890a,
-	0x890b,
-	0x890c,
-	0x890d,
-	0x890e,
-	0x890f,
-	0x8910,
-	0x8911,
-	0x8912,
-	0x8913,
-	0x8914,
-	0x8915,
-	0x8916,
-	0x8917,
-	0x8918,
-	0x8919,
-	0x891a,
-	0x8c00,
-	0x8c01,
-	0x8c17,
-	0x8c18,
-	0x8c19,
-	0x8c1a,
-	0x8c1b,
-	0x8c1c,
-	0x8c1d,
-	0x8c1e,
-	0x8c1f,
-	0x8c20,
-	0x8c21,
-	0x8c22,
-	0x8c23,
-	0x8c24,
-	0x8c25,
-	0x8c2c,
-	0x8c2d,
-	0x8c2e,
-	0x8c2f,
-	0x9101,
-	0x9102,
-	0x9103,
-	0x9104,
-	0x9105,
-	0x9106,
-	0x9107,
-	0x9108,
-	0x9109,
-	0x910a,
-	0x910b,
-	0x910c,
-	0x9200,
-	0x9201,
-	0x9202,
-	0x9203,
-	0x9204,
-	0x9205,
-	0x9206,
-	0x9207,
-	0x9208,
-	0x9209,
-	0x920a,
-	0x920b,
-	0x920c,
-	0x920d,
-	0x920e,
-	0x920f,
-	0x9212,
-	0x9213,
-	0x9214,
-	0x9215,
-	0x9216,
-	0x9217,
-	0x9301,
-	0x9302,
-	0x9303,
-	0x9304,
-	0x9305,
-	0x9306,
-	0x9307,
-	0x9308,
-	0x9309,
-	0x9311,
-	0x9312,
-	0x9313,
-	0x9314,
-	0x9315,
-	0x9316,
-	0x9317,
-	0x9800,
-	0x9801,
-	0x9802,
-	0x9803,
-	0x9804,
-	0x9805,
-	0x9806,
-	0x9808,
-	0x9809,
-	0x9b00,
-	0x9b01,
-	0x9b02,
-	0x9b03,
-	0x9b04,
-	0x9b05,
-	0x9b07,
-	0x9b08,
-	0x9b09,
-	0xa000,
-	0xa001,
-	0xa002,
-	0xa003,
-	0xa004,
-	0xa005,
-	0xa006,
-	0xa008,
-	0xa00e,
-	0xa00f,
-	0xa010,
-	0xa011,
-	0xa012,
-	0xa013,
-	0xa014,
-	0xa015,
-	0xa016,
-	0xa017,
-	0xa018,
-	0xa019,
-	0xa01a,
-	0xa01b,
-	0xa01c,
-	0xa01d,
-	0xa01e,
-	0xa01f,
-	0xa020,
-	0xa021,
-	0xa022,
-	0xa023,
-	0xa024,
-	0xa025,
-	0xa026,
-	0xa027,
-	0xa028,
-	0xa029,
-	0xa02a,
-	0xa02b,
-	0xa02c,
-	0xa02d,
-	0xa02e,
-	0xa02f,
-	0xa030,
-	0xa031,
-	0xa032,
-	0xa033,
-	0xa034,
-	0xa035,
-	0xa036,
-	0xa037,
-	0xa038,
-	0xa039,
-	0xa03a,
-	0xa03b,
-	0xa03c,
-	0xa03d,
-	0xa03e,
-	0xa03f,
-	0xa040,
-	0xa041,
-	0xa042,
-	0xa043,
-	0xa044,
-	0xa045,
-	0xa046,
-	0xa047,
-	0xa048,
-	0xa049,
-	0xa04a,
-	0xa04b,
-	0xa04c,
-	0xa04d,
-	0xa04e,
-	0xa04f,
-	0xa050,
-	0xa051,
-	0xa052,
-	0xa053,
-	0xa054,
-	0xa055,
-	0xa056,
-	0xa057,
-	0xa058,
-	0xa059,
-	0xa05a,
-	0xa05b,
-	0xa05c,
-	0xa05d,
-	0xa05e,
-	0xa05f,
-	0xa060,
-	0xa061,
-	0xa062,
-	0xa063,
-	0xa064,
-	0xa065,
-	0xa066,
-	0xa067,
-	0xa068,
-	0xa069,
-	0xa06a,
-	0xa06b,
-	0xa06c,
-	0xa06d,
-	0xa06e,
-	0xa06f,
-	0xa070,
-	0xa071,
-	0xa072,
-	0xa073,
-	0xa074,
-	0xa075,
-	0xa076,
-	0xa077,
-	0xa078,
-	0xa079,
-	0xa07a,
-	0xa07b,
-	0xa07c,
-	0xa07d,
-	0xa07e,
-	0xa07f,
-	0xa080,
-	0xa081,
-	0xa082,
-	0xa083,
-	0xa084,
-	0xa085,
-	0xa086,
-	0xa087,
-	0xa088,
-	0xa089,
-	0xa08a,
-	0xa08b,
-	0xa08c,
-	0xa08d,
-	0xa08e,
-	0xa08f,
-	0xa090,
-	0xa091,
-	0xa092,
-	0xa093,
-	0xa094,
-	0xa095,
-	0xa096,
-	0xa097,
-	0xa098,
-	0xa099,
-	0xa09a,
-	0xa09b,
-	0xa09c,
-	0xa09d,
-	0xa09e,
-	0xa09f,
-	0xa0a0,
-	0xa0a1,
-	0xa0a2,
-	0xa0a3,
-	0xa0a4,
-	0xa0a5,
-	0xa0a6,
-	0xa0a7,
-	0xa0a8,
-	0xa0a9,
-	0xa0aa,
-	0xa0ab,
-	0xa0ac,
-	0xa0ad,
-	0xa0ae,
-	0xa0af,
-	0xa0b0,
-	0xa0b1,
-	0xa0b2,
-	0xa0b3,
-	0xa0b4,
-	0xa0b5,
-	0xa0b6,
-	0xa0b7,
-	0xa0b8,
-	0xa0b9,
-	0xa0ba,
-	0xa0bb,
-	0xa0bc,
-	0xa0bd,
-	0xa0be,
-	0xa0bf,
-	0xa0c0,
-	0xa0c1,
-	0xa0c2,
-	0xa0c3,
-	0xa0c4,
-	0xa0c5,
-	0xa0c6,
-	0xa0c7,
-	0xa0c8,
-	0xa0c9,
-	0xa0ca,
-	0xa0cb,
-	0xa0cc,
-	0xa0cd,
-	0xa0ce,
-	0xa0cf,
-	0xa0d0,
-	0xa0d1,
-	0xa0d2,
-	0xa0d3,
-	0xa0d4,
-	0xa0d5,
-	0xa0d6,
-	0xa0d7,
-	0xa0d8,
-	0xa0d9,
-	0xa0da,
-	0xa0db,
-	0xa0dc,
-	0xa0dd,
-	0xa0de,
-	0xa0df,
-	0xa0e0,
-	0xa0e1,
-	0xa0e2,
-	0xa0e3,
-	0xa0e4,
-	0xa0e5,
-	0xa0e6,
-	0xa0e7,
-	0xa0e8,
-	0xa0e9,
-	0xa0ea,
-	0xa0eb,
-	0xa0ec,
-	0xa0ed,
-	0xa0ee,
-	0xa0ef,
-	0xa0f8,
-	0xa800,
-	0xa802,
-	0xa803,
-	0xa804,
-	0xa805,
-	0xa806,
-	0xa807,
-	0xa808,
-	0xa809,
-	0xa80a,
-	0xa80b,
-	0xa80c,
-	0xa80d,
-	0xa80e,
-	0xa80f,
-	0xa810,
-	0xa811,
-	0xa812,
-	0xa813,
-	0xa814,
-	0xa815,
-	0xa816,
-	0xa817,
-	0xa818,
-	0xa819,
-	0xa81a,
-	0xa81b,
-	0xa81c,
-	0xa81d,
-	0xa81e,
-	0xa81f,
-	0xa820,
-	0xa821,
-	0xa822,
-	0xa823,
-	0xa824,
-	0xa825,
-	0xa827,
-	0xa830,
-	0xa831,
-	0xa832,
-	0xa833,
-	0xa834,
-	0xa835,
-	0xa836,
-	0xa837,
-	0xa838,
-	0xa839,
-	0xa83a,
-	0xa83b,
-	0xa83c,
-	0xa83d,
-	0xa83f,
-	0xa840,
-	0xa842,
-	0xa843,
-	0xa844,
-	0xa845,
-	0xa846,
-	0xa847,
-	0xa848,
-	0xa849,
-	0xa84a,
-	0xa84b,
-	0xa84c,
-	0xa84d,
-	0xa84e,
-	0xa84f,
-	0xa850,
-	0xa851,
-	0xa852,
-	0xa853,
-	0xa854,
-	0xa855,
-	0xa856,
-	0xa857,
-	0xa858,
-	0xa859,
-	0xa85a,
-	0xa85b,
-	0xa85c,
-	0xa85d,
-	0xa85e,
-	0xa85f,
-	0xa860,
-	0xa861,
-	0xa862,
-	0xa863,
-	0xa864,
-	0xa865,
-	0xa867,
-	0xa870,
-	0xa871,
-	0xa872,
-	0xa873,
-	0xa874,
-	0xa875,
-	0xa876,
-	0xa877,
-	0xa878,
-	0xa879,
-	0xa87a,
-	0xa87b,
-	0xa87c,
-	0xa87d,
-	0xa87e,
-	0xa87f,
-	0xa880,
-	0xa881,
-	0xa882,
-	0xa883,
-	0xa884,
-	0xa885,
-	0xa886,
-	0xa887,
-	0xa888,
-	0xa889,
-	0xa88a,
-	0xa88b,
-	0xa88c,
-	0xa88d,
-	0xa88e,
-	0xa88f,
-	0xa890,
-	0xa891,
-	0xa892,
-	0xa893,
-	0xa894,
-	0xa895,
-	0xa896,
-	0xa898,
-	0xa980,
-	0xa982,
-	0xa983,
-	0xa984,
-	0xa985,
-	0xa986,
-	0xa987,
-	0xa988,
-	0xa989,
-	0xa98a,
-	0xa98b,
-	0xa98c,
-	0xa98d,
-	0xa98e,
-	0xa98f,
-	0xa990,
-	0xa991,
-	0xa992,
-	0xa993,
-	0xa994,
-	0xa995,
-	0xa996,
-	0xa997,
-	0xa998,
-	0xa999,
-	0xa99a,
-	0xa99b,
-	0xa99c,
-	0xa99d,
-	0xa99e,
-	0xa99f,
-	0xa9a0,
-	0xa9a1,
-	0xa9a2,
-	0xa9a3,
-	0xa9a4,
-	0xa9a5,
-	0xa9a6,
-	0xa9a7,
-	0xa9a9,
-	0xa9aa,
-	0xa9ae,
-	0xa9bf,
-	0xa9c6,
-	0xa9c7,
-	0xa9c8,
-	0xa9c9,
-	0xa9ca,
-	0xa9cb,
-	0xa9d4,
-	0xa9d5,
-	0xa9d6,
-	0xa9d7,
-	0xa9d8,
-	0xa9d9,
-	0xa9da,
-	0xa9db,
-	0xa9dc,
-	0xa9dd,
-	0xa9de,
-	0xa9e0,
-	0xa9e1,
-	0xa9e4,
-	0xa9e5,
-	0xab00,
-	0xab03,
-	0xab04,
-	0xab05,
-	0xab0a,
-	0xab0b,
-	0xab0c,
-	0xab0d,
-	0xab0e,
-	0xab0f,
-	0xab10,
-	0xab11,
-	0xab12,
-	0xab13,
-	0xab14,
-	0xab15,
-	0xab16,
-	0xab17,
-	0xab18,
-	0xab19,
-	0xab21,
-	0xb2c0,
-	0xb2c2,
-	0xb2c3,
-	0xb2ca,
-	0xb2cb,
-	0xb2cc,
-	0xb2d2,
-	0xb300,
-	0xb301,
-	0xb304,
-	0xb305,
-	0xb306,
-	0xb307,
-};
-#endif
-
-#endif /* A6XX_XML */
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
index 8bea8ef26f77..0e3dfd4c2bc8 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
@@ -507,7 +507,7 @@ static void a6xx_rpmh_stop(struct a6xx_gmu *gmu)
 
 static inline void pdc_write(void __iomem *ptr, u32 offset, u32 value)
 {
-	msm_writel(value, ptr + (offset << 2));
+	writel(value, ptr + (offset << 2));
 }
 
 static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev,
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
index 592b296aab22..94b6c5cab6f4 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
@@ -103,12 +103,12 @@ struct a6xx_gmu {
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
 {
-	return msm_readl(gmu->mmio + (offset << 2));
+	return readl(gmu->mmio + (offset << 2));
 }
 
 static inline void gmu_write(struct a6xx_gmu *gmu, u32 offset, u32 value)
 {
-	msm_writel(value, gmu->mmio + (offset << 2));
+	writel(value, gmu->mmio + (offset << 2));
 }
 
 static inline void
@@ -131,8 +131,8 @@ static inline u64 gmu_read64(struct a6xx_gmu *gmu, u32 lo, u32 hi)
 {
 	u64 val;
 
-	val = (u64) msm_readl(gmu->mmio + (lo << 2));
-	val |= ((u64) msm_readl(gmu->mmio + (hi << 2)) << 32);
+	val = (u64) readl(gmu->mmio + (lo << 2));
+	val |= ((u64) readl(gmu->mmio + (hi << 2)) << 32);
 
 	return val;
 }
@@ -143,12 +143,12 @@ static inline u64 gmu_read64(struct a6xx_gmu *gmu, u32 lo, u32 hi)
 
 static inline u32 gmu_read_rscc(struct a6xx_gmu *gmu, u32 offset)
 {
-	return msm_readl(gmu->rscc + (offset << 2));
+	return readl(gmu->rscc + (offset << 2));
 }
 
 static inline void gmu_write_rscc(struct a6xx_gmu *gmu, u32 offset, u32 value)
 {
-	msm_writel(value, gmu->rscc + (offset << 2));
+	writel(value, gmu->rscc + (offset << 2));
 }
 
 #define gmu_poll_timeout_rscc(gmu, addr, val, cond, interval, timeout) \
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h
deleted file mode 100644
index 9d7f93929367..000000000000
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h
+++ /dev/null
@@ -1,422 +0,0 @@
-#ifndef A6XX_GMU_XML
-#define A6XX_GMU_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng gen_header.py tool in this git repository:
-http://gitlab.freedesktop.org/mesa/mesa/
-git clone https://gitlab.freedesktop.org/mesa/mesa.git
-
-The rules-ng-ng source files this header was generated from are:
-
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/a6xx_gmu.xml      (  11820 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml  (   1572 bytes, from Fri Jun  2 14:59:26 2023)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml (  15434 bytes, from Fri Jun  2 14:59:26 2023)
-
-Copyright (C) 2013-2024 by the following authors:
-- Rob Clark <robdclark@gmail.com> Rob Clark
-- Ilia Mirkin <imirkin@alum.mit.edu> Ilia Mirkin
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-*/
-
-#ifdef __KERNEL__
-#include <linux/bug.h>
-#define assert(x) BUG_ON(!(x))
-#else
-#include <assert.h>
-#endif
-
-#ifdef __cplusplus
-#define __struct_cast(X)
-#else
-#define __struct_cast(X) (struct X)
-#endif
-
-#define A6XX_GMU_GPU_IDLE_STATUS_BUSY_IGN_AHB			0x00800000
-#define A6XX_GMU_GPU_IDLE_STATUS_CX_GX_CPU_BUSY_IGN_AHB		0x40000000
-
-#define A6XX_GMU_OOB_BOOT_SLUMBER_SET_MASK			0x00400000
-#define A6XX_GMU_OOB_BOOT_SLUMBER_CHECK_MASK			0x40000000
-#define A6XX_GMU_OOB_BOOT_SLUMBER_CLEAR_MASK			0x40000000
-#define A6XX_GMU_OOB_DCVS_SET_MASK				0x00800000
-#define A6XX_GMU_OOB_DCVS_CHECK_MASK				0x80000000
-#define A6XX_GMU_OOB_DCVS_CLEAR_MASK				0x80000000
-#define A6XX_GMU_OOB_GPU_SET_MASK				0x00040000
-#define A6XX_GMU_OOB_GPU_CHECK_MASK				0x04000000
-#define A6XX_GMU_OOB_GPU_CLEAR_MASK				0x04000000
-#define A6XX_GMU_OOB_PERFCNTR_SET_MASK				0x00020000
-#define A6XX_GMU_OOB_PERFCNTR_CHECK_MASK			0x02000000
-#define A6XX_GMU_OOB_PERFCNTR_CLEAR_MASK			0x02000000
-
-#define A6XX_HFI_IRQ_MSGQ_MASK					0x00000001
-#define A6XX_HFI_IRQ_DSGQ_MASK					0x00000002
-#define A6XX_HFI_IRQ_BLOCKED_MSG_MASK				0x00000004
-#define A6XX_HFI_IRQ_CM3_FAULT_MASK				0x00800000
-#define A6XX_HFI_IRQ_GMU_ERR_MASK__MASK				0x007f0000
-#define A6XX_HFI_IRQ_GMU_ERR_MASK__SHIFT			16
-static inline uint32_t A6XX_HFI_IRQ_GMU_ERR_MASK(uint32_t val)
-{
-	return ((val) << A6XX_HFI_IRQ_GMU_ERR_MASK__SHIFT) & A6XX_HFI_IRQ_GMU_ERR_MASK__MASK;
-}
-#define A6XX_HFI_IRQ_OOB_MASK__MASK				0xff000000
-#define A6XX_HFI_IRQ_OOB_MASK__SHIFT				24
-static inline uint32_t A6XX_HFI_IRQ_OOB_MASK(uint32_t val)
-{
-	return ((val) << A6XX_HFI_IRQ_OOB_MASK__SHIFT) & A6XX_HFI_IRQ_OOB_MASK__MASK;
-}
-
-#define A6XX_HFI_H2F_IRQ_MASK_BIT				0x00000001
-
-#define REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL		0x00000080
-
-#define REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL			0x00000081
-
-#define REG_A6XX_GMU_CM3_ITCM_START				0x00000c00
-
-#define REG_A6XX_GMU_CM3_DTCM_START				0x00001c00
-
-#define REG_A6XX_GMU_NMI_CONTROL_STATUS				0x000023f0
-
-#define REG_A6XX_GMU_BOOT_SLUMBER_OPTION			0x000023f8
-
-#define REG_A6XX_GMU_GX_VOTE_IDX				0x000023f9
-
-#define REG_A6XX_GMU_MX_VOTE_IDX				0x000023fa
-
-#define REG_A6XX_GMU_DCVS_ACK_OPTION				0x000023fc
-
-#define REG_A6XX_GMU_DCVS_PERF_SETTING				0x000023fd
-
-#define REG_A6XX_GMU_DCVS_BW_SETTING				0x000023fe
-
-#define REG_A6XX_GMU_DCVS_RETURN				0x000023ff
-
-#define REG_A6XX_GMU_ICACHE_CONFIG				0x00004c00
-
-#define REG_A6XX_GMU_DCACHE_CONFIG				0x00004c01
-
-#define REG_A6XX_GMU_SYS_BUS_CONFIG				0x00004c0f
-
-#define REG_A6XX_GMU_CM3_SYSRESET				0x00005000
-
-#define REG_A6XX_GMU_CM3_BOOT_CONFIG				0x00005001
-
-#define REG_A6XX_GMU_CM3_FW_BUSY				0x0000501a
-
-#define REG_A6XX_GMU_CM3_FW_INIT_RESULT				0x0000501c
-
-#define REG_A6XX_GMU_CM3_CFG					0x0000502d
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE		0x00005040
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0		0x00005041
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1		0x00005042
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L		0x00005044
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H		0x00005045
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_L		0x00005046
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_H		0x00005047
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_L		0x00005048
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_H		0x00005049
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_L		0x0000504a
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_H		0x0000504b
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_L		0x0000504c
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_H		0x0000504d
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_L		0x0000504e
-
-#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_H		0x0000504f
-
-#define REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL			0x000050c0
-#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE		0x00000001
-#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_HM_POWER_COLLAPSE_ENABLE	0x00000002
-#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_SPTPRAC_POWER_CONTROL_ENABLE	0x00000004
-#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS__MASK	0x00003c00
-#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS__SHIFT	10
-static inline uint32_t A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS(uint32_t val)
-{
-	return ((val) << A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS__SHIFT) & A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS__MASK;
-}
-#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH__MASK	0xffffc000
-#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH__SHIFT	14
-static inline uint32_t A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH(uint32_t val)
-{
-	return ((val) << A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH__SHIFT) & A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH__MASK;
-}
-
-#define REG_A6XX_GMU_PWR_COL_INTER_FRAME_HYST			0x000050c1
-
-#define REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST			0x000050c2
-
-#define REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS			0x000050d0
-#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWERING_OFF	0x00000001
-#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWERING_ON	0x00000002
-#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWER_OFF	0x00000004
-#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWER_ON	0x00000008
-#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SP_CLOCK_OFF		0x00000010
-#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GMU_UP_POWER_STATE	0x00000020
-#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF	0x00000040
-#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF		0x00000080
-
-#define REG_A6XX_GMU_GPU_NAP_CTRL				0x000050e4
-#define A6XX_GMU_GPU_NAP_CTRL_HW_NAP_ENABLE			0x00000001
-#define A6XX_GMU_GPU_NAP_CTRL_SID__MASK				0x000001f0
-#define A6XX_GMU_GPU_NAP_CTRL_SID__SHIFT			4
-static inline uint32_t A6XX_GMU_GPU_NAP_CTRL_SID(uint32_t val)
-{
-	return ((val) << A6XX_GMU_GPU_NAP_CTRL_SID__SHIFT) & A6XX_GMU_GPU_NAP_CTRL_SID__MASK;
-}
-
-#define REG_A6XX_GMU_RPMH_CTRL					0x000050e8
-#define A6XX_GMU_RPMH_CTRL_RPMH_INTERFACE_ENABLE		0x00000001
-#define A6XX_GMU_RPMH_CTRL_LLC_VOTE_ENABLE			0x00000010
-#define A6XX_GMU_RPMH_CTRL_DDR_VOTE_ENABLE			0x00000100
-#define A6XX_GMU_RPMH_CTRL_MX_VOTE_ENABLE			0x00000200
-#define A6XX_GMU_RPMH_CTRL_CX_VOTE_ENABLE			0x00000400
-#define A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE			0x00000800
-#define A6XX_GMU_RPMH_CTRL_DDR_MIN_VOTE_ENABLE			0x00001000
-#define A6XX_GMU_RPMH_CTRL_MX_MIN_VOTE_ENABLE			0x00002000
-#define A6XX_GMU_RPMH_CTRL_CX_MIN_VOTE_ENABLE			0x00004000
-#define A6XX_GMU_RPMH_CTRL_GFX_MIN_VOTE_ENABLE			0x00008000
-
-#define REG_A6XX_GMU_RPMH_HYST_CTRL				0x000050e9
-
-#define REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE		0x000050ec
-
-#define REG_A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF			0x000050f0
-
-#define REG_A6XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF			0x000050f1
-
-#define REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG			0x00005100
-
-#define REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP			0x00005101
-
-#define REG_A6XX_GMU_BOOT_KMD_LM_HANDSHAKE			0x000051f0
-
-#define REG_A6XX_GMU_LLM_GLM_SLEEP_CTRL				0x00005157
-
-#define REG_A6XX_GMU_LLM_GLM_SLEEP_STATUS			0x00005158
-
-#define REG_A6XX_GMU_ALWAYS_ON_COUNTER_L			0x00005088
-
-#define REG_A6XX_GMU_ALWAYS_ON_COUNTER_H			0x00005089
-
-#define REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE			0x000050c3
-
-#define REG_A6XX_GMU_HFI_CTRL_STATUS				0x00005180
-
-#define REG_A6XX_GMU_HFI_VERSION_INFO				0x00005181
-
-#define REG_A6XX_GMU_HFI_SFR_ADDR				0x00005182
-
-#define REG_A6XX_GMU_HFI_MMAP_ADDR				0x00005183
-
-#define REG_A6XX_GMU_HFI_QTBL_INFO				0x00005184
-
-#define REG_A6XX_GMU_HFI_QTBL_ADDR				0x00005185
-
-#define REG_A6XX_GMU_HFI_CTRL_INIT				0x00005186
-
-#define REG_A6XX_GMU_GMU2HOST_INTR_SET				0x00005190
-
-#define REG_A6XX_GMU_GMU2HOST_INTR_CLR				0x00005191
-
-#define REG_A6XX_GMU_GMU2HOST_INTR_INFO				0x00005192
-#define A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ			0x00000001
-#define A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT			0x00800000
-
-#define REG_A6XX_GMU_GMU2HOST_INTR_MASK				0x00005193
-
-#define REG_A6XX_GMU_HOST2GMU_INTR_SET				0x00005194
-
-#define REG_A6XX_GMU_HOST2GMU_INTR_CLR				0x00005195
-
-#define REG_A6XX_GMU_HOST2GMU_INTR_RAW_INFO			0x00005196
-
-#define REG_A6XX_GMU_HOST2GMU_INTR_EN_0				0x00005197
-
-#define REG_A6XX_GMU_HOST2GMU_INTR_EN_1				0x00005198
-
-#define REG_A6XX_GMU_HOST2GMU_INTR_EN_2				0x00005199
-
-#define REG_A6XX_GMU_HOST2GMU_INTR_EN_3				0x0000519a
-
-#define REG_A6XX_GMU_HOST2GMU_INTR_INFO_0			0x0000519b
-
-#define REG_A6XX_GMU_HOST2GMU_INTR_INFO_1			0x0000519c
-
-#define REG_A6XX_GMU_HOST2GMU_INTR_INFO_2			0x0000519d
-
-#define REG_A6XX_GMU_HOST2GMU_INTR_INFO_3			0x0000519e
-
-#define REG_A6XX_GMU_GENERAL_0					0x000051c5
-
-#define REG_A6XX_GMU_GENERAL_1					0x000051c6
-
-#define REG_A6XX_GMU_GENERAL_6					0x000051cb
-
-#define REG_A6XX_GMU_GENERAL_7					0x000051cc
-
-#define REG_A7XX_GMU_GENERAL_8					0x000051cd
-
-#define REG_A7XX_GMU_GENERAL_9					0x000051ce
-
-#define REG_A7XX_GMU_GENERAL_10					0x000051cf
-
-#define REG_A6XX_GMU_ISENSE_CTRL				0x0000515d
-
-#define REG_A6XX_GPU_CS_ENABLE_REG				0x00008920
-
-#define REG_A6XX_GPU_GMU_CX_GMU_ISENSE_CTRL			0x0000515d
-
-#define REG_A6XX_GPU_CS_AMP_CALIBRATION_CONTROL3		0x00008578
-
-#define REG_A6XX_GPU_CS_AMP_CALIBRATION_CONTROL2		0x00008558
-
-#define REG_A6XX_GPU_CS_A_SENSOR_CTRL_0				0x00008580
-
-#define REG_A6XX_GPU_CS_A_SENSOR_CTRL_2				0x00027ada
-
-#define REG_A6XX_GPU_CS_SENSOR_GENERAL_STATUS			0x0000881a
-
-#define REG_A6XX_GPU_CS_AMP_CALIBRATION_CONTROL1		0x00008957
-
-#define REG_A6XX_GPU_CS_SENSOR_GENERAL_STATUS			0x0000881a
-
-#define REG_A6XX_GPU_CS_AMP_CALIBRATION_STATUS1_0		0x0000881d
-
-#define REG_A6XX_GPU_CS_AMP_CALIBRATION_STATUS1_2		0x0000881f
-
-#define REG_A6XX_GPU_CS_AMP_CALIBRATION_STATUS1_4		0x00008821
-
-#define REG_A6XX_GPU_CS_AMP_CALIBRATION_DONE			0x00008965
-
-#define REG_A6XX_GPU_CS_AMP_PERIOD_CTRL				0x0000896d
-
-#define REG_A6XX_GPU_CS_AMP_CALIBRATION_DONE			0x00008965
-
-#define REG_A6XX_GPU_GMU_CX_GMU_PWR_THRESHOLD			0x0000514d
-
-#define REG_A6XX_GMU_AO_INTERRUPT_EN				0x00009303
-
-#define REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR			0x00009304
-
-#define REG_A6XX_GMU_AO_HOST_INTERRUPT_STATUS			0x00009305
-#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE		0x00000001
-#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_RSCC_COMP		0x00000002
-#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_VDROOP		0x00000004
-#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR		0x00000008
-#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_DBD_WAKEUP		0x00000010
-#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR	0x00000020
-
-#define REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK			0x00009306
-
-#define REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL			0x00009309
-
-#define REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL			0x0000930a
-
-#define REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL			0x0000930b
-
-#define REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS			0x0000930c
-#define A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB	0x00800000
-
-#define REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2			0x0000930d
-
-#define REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK			0x0000930e
-
-#define REG_A6XX_GMU_AO_AHB_FENCE_CTRL				0x00009310
-
-#define REG_A6XX_GMU_AHB_FENCE_STATUS				0x00009313
-
-#define REG_A6XX_GMU_AHB_FENCE_STATUS_CLR			0x00009314
-
-#define REG_A6XX_GMU_RBBM_INT_UNMASKED_STATUS			0x00009315
-
-#define REG_A6XX_GMU_AO_SPARE_CNTL				0x00009316
-
-#define REG_A6XX_GMU_RSCC_CONTROL_REQ				0x00009307
-
-#define REG_A6XX_GMU_RSCC_CONTROL_ACK				0x00009308
-
-#define REG_A6XX_GMU_AHB_FENCE_RANGE_0				0x00009311
-
-#define REG_A6XX_GMU_AHB_FENCE_RANGE_1				0x00009312
-
-#define REG_A6XX_GPU_CC_GX_GDSCR				0x00009c03
-
-#define REG_A6XX_GPU_CC_GX_DOMAIN_MISC				0x00009d42
-
-#define REG_A6XX_GPU_CPR_FSM_CTL				0x0000c001
-
-#define REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0			0x00000004
-
-#define REG_A6XX_RSCC_PDC_SEQ_START_ADDR			0x00000008
-
-#define REG_A6XX_RSCC_PDC_MATCH_VALUE_LO			0x00000009
-
-#define REG_A6XX_RSCC_PDC_MATCH_VALUE_HI			0x0000000a
-
-#define REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0				0x0000000b
-
-#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR			0x0000000d
-
-#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA			0x0000000e
-
-#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_L_DRV0		0x00000082
-
-#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_H_DRV0		0x00000083
-
-#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_EN_DRV0			0x00000089
-
-#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_OUTPUT_DRV0		0x0000008c
-
-#define REG_A6XX_RSCC_OVERRIDE_START_ADDR			0x00000100
-
-#define REG_A6XX_RSCC_SEQ_BUSY_DRV0				0x00000101
-
-#define REG_A7XX_RSCC_SEQ_MEM_0_DRV0_A740			0x00000154
-
-#define REG_A6XX_RSCC_SEQ_MEM_0_DRV0				0x00000180
-
-#define REG_A6XX_RSCC_TCS0_DRV0_STATUS				0x00000346
-
-#define REG_A6XX_RSCC_TCS1_DRV0_STATUS				0x000003ee
-
-#define REG_A6XX_RSCC_TCS2_DRV0_STATUS				0x00000496
-
-#define REG_A6XX_RSCC_TCS3_DRV0_STATUS				0x0000053e
-
-#ifdef __cplusplus
-#endif
-
-#endif /* A6XX_GMU_XML */
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index cf0b1de1c071..973872ad0474 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -284,7 +284,7 @@ static void a7xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 
 	a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx);
 
-	get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
+	get_stats_counter(ring, REG_A7XX_RBBM_PERFCTR_CP(0),
 		rbmemptr_stats(ring, index, cpcycles_start));
 	get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
 		rbmemptr_stats(ring, index, alwayson_start));
@@ -330,7 +330,7 @@ static void a7xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 	OUT_PKT7(ring, CP_SET_MARKER, 1);
 	OUT_RING(ring, 0x00e); /* IB1LIST end */
 
-	get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
+	get_stats_counter(ring, REG_A7XX_RBBM_PERFCTR_CP(0),
 		rbmemptr_stats(ring, index, cpcycles_end));
 	get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
 		rbmemptr_stats(ring, index, alwayson_end));
@@ -1255,8 +1255,9 @@ static const u32 a730_protect[] = {
 	A6XX_PROTECT_NORDWR(0x00699, 0x01e9),
 	A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
 	A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
-	/* 0x008d0-0x008dd are unprotected on purpose for tools like perfetto */
-	A6XX_PROTECT_RDONLY(0x008de, 0x0154),
+	/* 0x008d0-0x008dd and 0x008e0-0x008e6 are unprotected on purpose for tools like perfetto */
+	A6XX_PROTECT_NORDWR(0x008de, 0x0001),
+	A6XX_PROTECT_RDONLY(0x008e7, 0x014b),
 	A6XX_PROTECT_NORDWR(0x00900, 0x004d),
 	A6XX_PROTECT_NORDWR(0x0098d, 0x00b2),
 	A6XX_PROTECT_NORDWR(0x00a41, 0x01be),
@@ -1291,8 +1292,7 @@ static const u32 a730_protect[] = {
 	A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
 	A6XX_PROTECT_NORDWR(0x1f860, 0x0000),
 	A6XX_PROTECT_NORDWR(0x1f878, 0x002a),
-	/* CP_PROTECT_REG[44, 46] are left untouched! */
-	0,
+	/* CP_PROTECT_REG[45, 46] are left untouched! */
 	0,
 	0,
 	A6XX_PROTECT_NORDWR(0x1f8c0, 0x00000),
@@ -3062,7 +3062,8 @@ struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
 
 	ret = a6xx_set_supported_hw(&pdev->dev, config->info);
 	if (ret) {
-		a6xx_destroy(&(a6xx_gpu->base.base));
+		a6xx_llc_slices_destroy(a6xx_gpu);
+		kfree(a6xx_gpu);
 		return ERR_PTR(ret);
 	}
 
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
index 34822b080759..8917032b7515 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
@@ -69,12 +69,12 @@ static inline void a6xx_llc_rmw(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 mask, u3
 
 static inline u32 a6xx_llc_read(struct a6xx_gpu *a6xx_gpu, u32 reg)
 {
-	return msm_readl(a6xx_gpu->llc_mmio + (reg << 2));
+	return readl(a6xx_gpu->llc_mmio + (reg << 2));
 }
 
 static inline void a6xx_llc_write(struct a6xx_gpu *a6xx_gpu, u32 reg, u32 value)
 {
-	msm_writel(value, a6xx_gpu->llc_mmio + (reg << 2));
+	writel(value, a6xx_gpu->llc_mmio + (reg << 2));
 }
 
 #define shadowptr(_a6xx_gpu, _ring) ((_a6xx_gpu)->shadow_iova + \
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c
index a847a0f7a73c..0a7717a4fc2f 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c
@@ -13,15 +13,18 @@
  */
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wunused-const-variable"
 
 #include "adreno_gen7_0_0_snapshot.h"
 #include "adreno_gen7_2_0_snapshot.h"
+#include "adreno_gen7_9_0_snapshot.h"
 
 #pragma GCC diagnostic pop
 
 struct a6xx_gpu_state_obj {
 	const void *handle;
 	u32 *data;
+	u32 count;	/* optional, used when count potentially read from hw */
 };
 
 struct a6xx_gpu_state {
@@ -192,10 +195,10 @@ static int debugbus_read(struct msm_gpu *gpu, u32 block, u32 offset,
 }
 
 #define cxdbg_write(ptr, offset, val) \
-	msm_writel((val), (ptr) + ((offset) << 2))
+	writel((val), (ptr) + ((offset) << 2))
 
 #define cxdbg_read(ptr, offset) \
-	msm_readl((ptr) + ((offset) << 2))
+	readl((ptr) + ((offset) << 2))
 
 /* read a value from the CX debug bus */
 static int cx_debugbus_read(void __iomem *cxdbg, u32 block, u32 offset,
@@ -384,21 +387,29 @@ static void a7xx_get_debugbus_blocks(struct msm_gpu *gpu,
 		struct a6xx_gpu_state *a6xx_state)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
-	int debugbus_blocks_count, total_debugbus_blocks;
-	const u32 *debugbus_blocks;
+	int debugbus_blocks_count, gbif_debugbus_blocks_count, total_debugbus_blocks;
+	const u32 *debugbus_blocks, *gbif_debugbus_blocks;
 	int i;
 
 	if (adreno_is_a730(adreno_gpu)) {
 		debugbus_blocks = gen7_0_0_debugbus_blocks;
 		debugbus_blocks_count = ARRAY_SIZE(gen7_0_0_debugbus_blocks);
-	} else {
-		BUG_ON(!adreno_is_a740_family(adreno_gpu));
+		gbif_debugbus_blocks = a7xx_gbif_debugbus_blocks;
+		gbif_debugbus_blocks_count = ARRAY_SIZE(a7xx_gbif_debugbus_blocks);
+	} else if (adreno_is_a740_family(adreno_gpu)) {
 		debugbus_blocks = gen7_2_0_debugbus_blocks;
 		debugbus_blocks_count = ARRAY_SIZE(gen7_2_0_debugbus_blocks);
+		gbif_debugbus_blocks = a7xx_gbif_debugbus_blocks;
+		gbif_debugbus_blocks_count = ARRAY_SIZE(a7xx_gbif_debugbus_blocks);
+	} else {
+		BUG_ON(!adreno_is_a750(adreno_gpu));
+		debugbus_blocks = gen7_9_0_debugbus_blocks;
+		debugbus_blocks_count = ARRAY_SIZE(gen7_9_0_debugbus_blocks);
+		gbif_debugbus_blocks = gen7_9_0_gbif_debugbus_blocks;
+		gbif_debugbus_blocks_count = ARRAY_SIZE(gen7_9_0_gbif_debugbus_blocks);
 	}
 
-	total_debugbus_blocks = debugbus_blocks_count +
-		ARRAY_SIZE(a7xx_gbif_debugbus_blocks);
+	total_debugbus_blocks = debugbus_blocks_count + gbif_debugbus_blocks_count;
 
 	a6xx_state->debugbus = state_kcalloc(a6xx_state, total_debugbus_blocks,
 			sizeof(*a6xx_state->debugbus));
@@ -410,9 +421,9 @@ static void a7xx_get_debugbus_blocks(struct msm_gpu *gpu,
 				&a6xx_state->debugbus[i]);
 		}
 
-		for (i = 0; i < ARRAY_SIZE(a7xx_gbif_debugbus_blocks); i++) {
+		for (i = 0; i < gbif_debugbus_blocks_count; i++) {
 			a6xx_get_debugbus_block(gpu,
-				a6xx_state, &a7xx_gbif_debugbus_blocks[i],
+				a6xx_state, &a7xx_debugbus_blocks[gbif_debugbus_blocks[i]],
 				&a6xx_state->debugbus[i + debugbus_blocks_count]);
 		}
 	}
@@ -813,10 +824,13 @@ static void a7xx_get_clusters(struct msm_gpu *gpu,
 	if (adreno_is_a730(adreno_gpu)) {
 		clusters = gen7_0_0_clusters;
 		clusters_size = ARRAY_SIZE(gen7_0_0_clusters);
-	} else {
-		BUG_ON(!adreno_is_a740_family(adreno_gpu));
+	} else if (adreno_is_a740_family(adreno_gpu)) {
 		clusters = gen7_2_0_clusters;
 		clusters_size = ARRAY_SIZE(gen7_2_0_clusters);
+	} else {
+		BUG_ON(!adreno_is_a750(adreno_gpu));
+		clusters = gen7_9_0_clusters;
+		clusters_size = ARRAY_SIZE(gen7_9_0_clusters);
 	}
 
 	a6xx_state->clusters = state_kcalloc(a6xx_state,
@@ -948,10 +962,13 @@ static void a7xx_get_shaders(struct msm_gpu *gpu,
 	if (adreno_is_a730(adreno_gpu)) {
 		shader_blocks = gen7_0_0_shader_blocks;
 		num_shader_blocks = ARRAY_SIZE(gen7_0_0_shader_blocks);
-	} else {
-		BUG_ON(!adreno_is_a740_family(adreno_gpu));
+	} else if (adreno_is_a740_family(adreno_gpu)) {
 		shader_blocks = gen7_2_0_shader_blocks;
 		num_shader_blocks = ARRAY_SIZE(gen7_2_0_shader_blocks);
+	} else {
+		BUG_ON(!adreno_is_a750(adreno_gpu));
+		shader_blocks = gen7_9_0_shader_blocks;
+		num_shader_blocks = ARRAY_SIZE(gen7_9_0_shader_blocks);
 	}
 
 	a6xx_state->shaders = state_kcalloc(a6xx_state,
@@ -1337,10 +1354,13 @@ static void a7xx_get_registers(struct msm_gpu *gpu,
 	if (adreno_is_a730(adreno_gpu)) {
 		reglist = gen7_0_0_reg_list;
 		pre_crashdumper_regs = gen7_0_0_pre_crashdumper_gpu_registers;
-	} else {
-		BUG_ON(!adreno_is_a740_family(adreno_gpu));
+	} else if (adreno_is_a740_family(adreno_gpu)) {
 		reglist = gen7_2_0_reg_list;
 		pre_crashdumper_regs = gen7_0_0_pre_crashdumper_gpu_registers;
+	} else {
+		BUG_ON(!adreno_is_a750(adreno_gpu));
+		reglist = gen7_9_0_reg_list;
+		pre_crashdumper_regs = gen7_9_0_pre_crashdumper_gpu_registers;
 	}
 
 	count = A7XX_PRE_CRASHDUMPER_SIZE + A7XX_POST_CRASHDUMPER_SIZE;
@@ -1388,7 +1408,8 @@ static void a7xx_get_post_crashdumper_registers(struct msm_gpu *gpu,
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 	const u32 *regs;
 
-	BUG_ON(!(adreno_is_a730(adreno_gpu) || adreno_is_a740_family(adreno_gpu)));
+	BUG_ON(!(adreno_is_a730(adreno_gpu) || adreno_is_a740_family(adreno_gpu) ||
+		 adreno_is_a750(adreno_gpu)));
 	regs = gen7_0_0_post_crashdumper_registers;
 
 	a7xx_get_ahb_gpu_registers(gpu,
@@ -1417,16 +1438,18 @@ static u32 a7xx_get_cp_roq_size(struct msm_gpu *gpu)
 /* Read a block of data from an indexed register pair */
 static void a6xx_get_indexed_regs(struct msm_gpu *gpu,
 		struct a6xx_gpu_state *a6xx_state,
-		struct a6xx_indexed_registers *indexed,
+		const struct a6xx_indexed_registers *indexed,
 		struct a6xx_gpu_state_obj *obj)
 {
+	u32 count = indexed->count;
 	int i;
 
 	obj->handle = (const void *) indexed;
 	if (indexed->count_fn)
-		indexed->count = indexed->count_fn(gpu);
+		count = indexed->count_fn(gpu);
 
-	obj->data = state_kcalloc(a6xx_state, indexed->count, sizeof(u32));
+	obj->data = state_kcalloc(a6xx_state, count, sizeof(u32));
+	obj->count = count;
 	if (!obj->data)
 		return;
 
@@ -1434,7 +1457,7 @@ static void a6xx_get_indexed_regs(struct msm_gpu *gpu,
 	gpu_write(gpu, indexed->addr, 0);
 
 	/* Read the data - each read increments the internal address by 1 */
-	for (i = 0; i < indexed->count; i++)
+	for (i = 0; i < count; i++)
 		obj->data[i] = gpu_read(gpu, indexed->data);
 }
 
@@ -1491,10 +1514,18 @@ static void a7xx_get_indexed_registers(struct msm_gpu *gpu,
 		struct a6xx_gpu_state *a6xx_state)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	const struct a6xx_indexed_registers *indexed_regs;
 	int i, indexed_count, mempool_count;
 
-	BUG_ON(!(adreno_is_a730(adreno_gpu) || adreno_is_a740_family(adreno_gpu)));
-	indexed_count = ARRAY_SIZE(a7xx_indexed_reglist);
+	if (adreno_is_a730(adreno_gpu) || adreno_is_a740_family(adreno_gpu)) {
+		indexed_regs = a7xx_indexed_reglist;
+		indexed_count = ARRAY_SIZE(a7xx_indexed_reglist);
+	} else {
+		BUG_ON(!adreno_is_a750(adreno_gpu));
+		indexed_regs = gen7_9_0_cp_indexed_reg_list;
+		indexed_count = ARRAY_SIZE(gen7_9_0_cp_indexed_reg_list);
+	}
+
 	mempool_count = ARRAY_SIZE(a7xx_cp_bv_mempool_indexed);
 
 	a6xx_state->indexed_regs = state_kcalloc(a6xx_state,
@@ -1507,7 +1538,7 @@ static void a7xx_get_indexed_registers(struct msm_gpu *gpu,
 
 	/* First read the common regs */
 	for (i = 0; i < indexed_count; i++)
-		a6xx_get_indexed_regs(gpu, a6xx_state, &a7xx_indexed_reglist[i],
+		a6xx_get_indexed_regs(gpu, a6xx_state, &indexed_regs[i],
 			&a6xx_state->indexed_regs[i]);
 
 	gpu_rmw(gpu, REG_A6XX_CP_CHICKEN_DBG, 0, BIT(2));
@@ -1862,9 +1893,9 @@ static void a6xx_show_indexed_regs(struct a6xx_gpu_state_obj *obj,
 		return;
 
 	print_name(p, "  - regs-name: ", indexed->name);
-	drm_printf(p, "    dwords: %d\n", indexed->count);
+	drm_printf(p, "    dwords: %d\n", obj->count);
 
-	print_ascii85(p, indexed->count << 2, obj->data);
+	print_ascii85(p, obj->count << 2, obj->data);
 }
 
 static void a6xx_show_debugbus_block(const struct a6xx_debugbus_block *block,
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h
index 5ddd32063bcc..dd4c28a8d923 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h
@@ -397,7 +397,7 @@ struct a6xx_indexed_registers {
 	u32 (*count_fn)(struct msm_gpu *gpu);
 };
 
-static struct a6xx_indexed_registers a6xx_indexed_reglist[] = {
+static const struct a6xx_indexed_registers a6xx_indexed_reglist[] = {
 	{ "CP_SQE_STAT", REG_A6XX_CP_SQE_STAT_ADDR,
 		REG_A6XX_CP_SQE_STAT_DATA, 0x33, NULL },
 	{ "CP_DRAW_STATE", REG_A6XX_CP_DRAW_STATE_ADDR,
@@ -408,7 +408,7 @@ static struct a6xx_indexed_registers a6xx_indexed_reglist[] = {
 		REG_A6XX_CP_ROQ_DBG_DATA, 0, a6xx_get_cp_roq_size},
 };
 
-static struct a6xx_indexed_registers a7xx_indexed_reglist[] = {
+static const struct a6xx_indexed_registers a7xx_indexed_reglist[] = {
 	{ "CP_SQE_STAT", REG_A6XX_CP_SQE_STAT_ADDR,
 		REG_A6XX_CP_SQE_STAT_DATA, 0x33, NULL },
 	{ "CP_DRAW_STATE", REG_A6XX_CP_DRAW_STATE_ADDR,
@@ -433,12 +433,12 @@ static struct a6xx_indexed_registers a7xx_indexed_reglist[] = {
 		REG_A6XX_CP_ROQ_DBG_DATA, 0, a7xx_get_cp_roq_size },
 };
 
-static struct a6xx_indexed_registers a6xx_cp_mempool_indexed = {
+static const struct a6xx_indexed_registers a6xx_cp_mempool_indexed = {
 	"CP_MEMPOOL", REG_A6XX_CP_MEM_POOL_DBG_ADDR,
 		REG_A6XX_CP_MEM_POOL_DBG_DATA, 0x2060, NULL,
 };
 
-static struct a6xx_indexed_registers a7xx_cp_bv_mempool_indexed[] = {
+static const struct a6xx_indexed_registers a7xx_cp_bv_mempool_indexed[] = {
 	{ "CP_MEMPOOL", REG_A6XX_CP_MEM_POOL_DBG_ADDR,
 		REG_A6XX_CP_MEM_POOL_DBG_DATA, 0x2100, NULL },
 	{ "CP_BV_MEMPOOL", REG_A7XX_CP_BV_MEM_POOL_DBG_ADDR,
@@ -517,9 +517,9 @@ static const struct a6xx_debugbus_block a650_debugbus_blocks[] = {
 	DEBUGBUS(A6XX_DBGBUS_SPTP_5, 0x100),
 };
 
-static const struct a6xx_debugbus_block a7xx_gbif_debugbus_blocks[] = {
-	DEBUGBUS(A7XX_DBGBUS_GBIF_CX, 0x100),
-	DEBUGBUS(A7XX_DBGBUS_GBIF_GX, 0x100),
+static const u32 a7xx_gbif_debugbus_blocks[] = {
+	A7XX_DBGBUS_GBIF_CX,
+	A7XX_DBGBUS_GBIF_GX,
 };
 
 static const struct a6xx_debugbus_block a7xx_cx_debugbus_blocks[] = {
diff --git a/drivers/gpu/drm/msm/adreno/adreno_common.xml.h b/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
deleted file mode 100644
index fbc27930e550..000000000000
--- a/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
+++ /dev/null
@@ -1,539 +0,0 @@
-#ifndef ADRENO_COMMON_XML
-#define ADRENO_COMMON_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng gen_header.py tool in this git repository:
-http://gitlab.freedesktop.org/mesa/mesa/
-git clone https://gitlab.freedesktop.org/mesa/mesa.git
-
-The rules-ng-ng source files this header was generated from are:
-
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml (  15434 bytes, from Fri Jun  2 14:59:26 2023)
-*/
-
-#ifdef __KERNEL__
-#include <linux/bug.h>
-#define assert(x) BUG_ON(!(x))
-#else
-#include <assert.h>
-#endif
-
-#ifdef __cplusplus
-#define __struct_cast(X)
-#else
-#define __struct_cast(X) (struct X)
-#endif
-
-enum chip {
-	A2XX = 2,
-	A3XX = 3,
-	A4XX = 4,
-	A5XX = 5,
-	A6XX = 6,
-	A7XX = 7,
-};
-
-enum adreno_pa_su_sc_draw {
-	PC_DRAW_POINTS = 0,
-	PC_DRAW_LINES = 1,
-	PC_DRAW_TRIANGLES = 2,
-};
-
-enum adreno_compare_func {
-	FUNC_NEVER = 0,
-	FUNC_LESS = 1,
-	FUNC_EQUAL = 2,
-	FUNC_LEQUAL = 3,
-	FUNC_GREATER = 4,
-	FUNC_NOTEQUAL = 5,
-	FUNC_GEQUAL = 6,
-	FUNC_ALWAYS = 7,
-};
-
-enum adreno_stencil_op {
-	STENCIL_KEEP = 0,
-	STENCIL_ZERO = 1,
-	STENCIL_REPLACE = 2,
-	STENCIL_INCR_CLAMP = 3,
-	STENCIL_DECR_CLAMP = 4,
-	STENCIL_INVERT = 5,
-	STENCIL_INCR_WRAP = 6,
-	STENCIL_DECR_WRAP = 7,
-};
-
-enum adreno_rb_blend_factor {
-	FACTOR_ZERO = 0,
-	FACTOR_ONE = 1,
-	FACTOR_SRC_COLOR = 4,
-	FACTOR_ONE_MINUS_SRC_COLOR = 5,
-	FACTOR_SRC_ALPHA = 6,
-	FACTOR_ONE_MINUS_SRC_ALPHA = 7,
-	FACTOR_DST_COLOR = 8,
-	FACTOR_ONE_MINUS_DST_COLOR = 9,
-	FACTOR_DST_ALPHA = 10,
-	FACTOR_ONE_MINUS_DST_ALPHA = 11,
-	FACTOR_CONSTANT_COLOR = 12,
-	FACTOR_ONE_MINUS_CONSTANT_COLOR = 13,
-	FACTOR_CONSTANT_ALPHA = 14,
-	FACTOR_ONE_MINUS_CONSTANT_ALPHA = 15,
-	FACTOR_SRC_ALPHA_SATURATE = 16,
-	FACTOR_SRC1_COLOR = 20,
-	FACTOR_ONE_MINUS_SRC1_COLOR = 21,
-	FACTOR_SRC1_ALPHA = 22,
-	FACTOR_ONE_MINUS_SRC1_ALPHA = 23,
-};
-
-enum adreno_rb_surface_endian {
-	ENDIAN_NONE = 0,
-	ENDIAN_8IN16 = 1,
-	ENDIAN_8IN32 = 2,
-	ENDIAN_16IN32 = 3,
-	ENDIAN_8IN64 = 4,
-	ENDIAN_8IN128 = 5,
-};
-
-enum adreno_rb_dither_mode {
-	DITHER_DISABLE = 0,
-	DITHER_ALWAYS = 1,
-	DITHER_IF_ALPHA_OFF = 2,
-};
-
-enum adreno_rb_depth_format {
-	DEPTHX_16 = 0,
-	DEPTHX_24_8 = 1,
-	DEPTHX_32 = 2,
-};
-
-enum adreno_rb_copy_control_mode {
-	RB_COPY_RESOLVE = 1,
-	RB_COPY_CLEAR = 2,
-	RB_COPY_DEPTH_STENCIL = 5,
-};
-
-enum a3xx_rop_code {
-	ROP_CLEAR = 0,
-	ROP_NOR = 1,
-	ROP_AND_INVERTED = 2,
-	ROP_COPY_INVERTED = 3,
-	ROP_AND_REVERSE = 4,
-	ROP_INVERT = 5,
-	ROP_XOR = 6,
-	ROP_NAND = 7,
-	ROP_AND = 8,
-	ROP_EQUIV = 9,
-	ROP_NOOP = 10,
-	ROP_OR_INVERTED = 11,
-	ROP_COPY = 12,
-	ROP_OR_REVERSE = 13,
-	ROP_OR = 14,
-	ROP_SET = 15,
-};
-
-enum a3xx_render_mode {
-	RB_RENDERING_PASS = 0,
-	RB_TILING_PASS = 1,
-	RB_RESOLVE_PASS = 2,
-	RB_COMPUTE_PASS = 3,
-};
-
-enum a3xx_msaa_samples {
-	MSAA_ONE = 0,
-	MSAA_TWO = 1,
-	MSAA_FOUR = 2,
-	MSAA_EIGHT = 3,
-};
-
-enum a3xx_threadmode {
-	MULTI = 0,
-	SINGLE = 1,
-};
-
-enum a3xx_instrbuffermode {
-	CACHE = 0,
-	BUFFER = 1,
-};
-
-enum a3xx_threadsize {
-	TWO_QUADS = 0,
-	FOUR_QUADS = 1,
-};
-
-enum a3xx_color_swap {
-	WZYX = 0,
-	WXYZ = 1,
-	ZYXW = 2,
-	XYZW = 3,
-};
-
-enum a3xx_rb_blend_opcode {
-	BLEND_DST_PLUS_SRC = 0,
-	BLEND_SRC_MINUS_DST = 1,
-	BLEND_DST_MINUS_SRC = 2,
-	BLEND_MIN_DST_SRC = 3,
-	BLEND_MAX_DST_SRC = 4,
-};
-
-enum a4xx_tess_spacing {
-	EQUAL_SPACING = 0,
-	ODD_SPACING = 2,
-	EVEN_SPACING = 3,
-};
-
-enum a5xx_address_mode {
-	ADDR_32B = 0,
-	ADDR_64B = 1,
-};
-
-enum a5xx_line_mode {
-	BRESENHAM = 0,
-	RECTANGULAR = 1,
-};
-
-enum a6xx_tex_prefetch_cmd {
-	TEX_PREFETCH_UNK0 = 0,
-	TEX_PREFETCH_SAM = 1,
-	TEX_PREFETCH_GATHER4R = 2,
-	TEX_PREFETCH_GATHER4G = 3,
-	TEX_PREFETCH_GATHER4B = 4,
-	TEX_PREFETCH_GATHER4A = 5,
-	TEX_PREFETCH_UNK6 = 6,
-	TEX_PREFETCH_UNK7 = 7,
-};
-
-#define REG_AXXX_CP_RB_BASE					0x000001c0
-
-#define REG_AXXX_CP_RB_CNTL					0x000001c1
-#define AXXX_CP_RB_CNTL_BUFSZ__MASK				0x0000003f
-#define AXXX_CP_RB_CNTL_BUFSZ__SHIFT				0
-static inline uint32_t AXXX_CP_RB_CNTL_BUFSZ(uint32_t val)
-{
-	return ((val) << AXXX_CP_RB_CNTL_BUFSZ__SHIFT) & AXXX_CP_RB_CNTL_BUFSZ__MASK;
-}
-#define AXXX_CP_RB_CNTL_BLKSZ__MASK				0x00003f00
-#define AXXX_CP_RB_CNTL_BLKSZ__SHIFT				8
-static inline uint32_t AXXX_CP_RB_CNTL_BLKSZ(uint32_t val)
-{
-	return ((val) << AXXX_CP_RB_CNTL_BLKSZ__SHIFT) & AXXX_CP_RB_CNTL_BLKSZ__MASK;
-}
-#define AXXX_CP_RB_CNTL_BUF_SWAP__MASK				0x00030000
-#define AXXX_CP_RB_CNTL_BUF_SWAP__SHIFT				16
-static inline uint32_t AXXX_CP_RB_CNTL_BUF_SWAP(uint32_t val)
-{
-	return ((val) << AXXX_CP_RB_CNTL_BUF_SWAP__SHIFT) & AXXX_CP_RB_CNTL_BUF_SWAP__MASK;
-}
-#define AXXX_CP_RB_CNTL_POLL_EN					0x00100000
-#define AXXX_CP_RB_CNTL_NO_UPDATE				0x08000000
-#define AXXX_CP_RB_CNTL_RPTR_WR_EN				0x80000000
-
-#define REG_AXXX_CP_RB_RPTR_ADDR				0x000001c3
-#define AXXX_CP_RB_RPTR_ADDR_SWAP__MASK				0x00000003
-#define AXXX_CP_RB_RPTR_ADDR_SWAP__SHIFT			0
-static inline uint32_t AXXX_CP_RB_RPTR_ADDR_SWAP(uint32_t val)
-{
-	return ((val) << AXXX_CP_RB_RPTR_ADDR_SWAP__SHIFT) & AXXX_CP_RB_RPTR_ADDR_SWAP__MASK;
-}
-#define AXXX_CP_RB_RPTR_ADDR_ADDR__MASK				0xfffffffc
-#define AXXX_CP_RB_RPTR_ADDR_ADDR__SHIFT			2
-static inline uint32_t AXXX_CP_RB_RPTR_ADDR_ADDR(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << AXXX_CP_RB_RPTR_ADDR_ADDR__SHIFT) & AXXX_CP_RB_RPTR_ADDR_ADDR__MASK;
-}
-
-#define REG_AXXX_CP_RB_RPTR					0x000001c4
-
-#define REG_AXXX_CP_RB_WPTR					0x000001c5
-
-#define REG_AXXX_CP_RB_WPTR_DELAY				0x000001c6
-
-#define REG_AXXX_CP_RB_RPTR_WR					0x000001c7
-
-#define REG_AXXX_CP_RB_WPTR_BASE				0x000001c8
-
-#define REG_AXXX_CP_QUEUE_THRESHOLDS				0x000001d5
-#define AXXX_CP_QUEUE_THRESHOLDS_CSQ_IB1_START__MASK		0x0000000f
-#define AXXX_CP_QUEUE_THRESHOLDS_CSQ_IB1_START__SHIFT		0
-static inline uint32_t AXXX_CP_QUEUE_THRESHOLDS_CSQ_IB1_START(uint32_t val)
-{
-	return ((val) << AXXX_CP_QUEUE_THRESHOLDS_CSQ_IB1_START__SHIFT) & AXXX_CP_QUEUE_THRESHOLDS_CSQ_IB1_START__MASK;
-}
-#define AXXX_CP_QUEUE_THRESHOLDS_CSQ_IB2_START__MASK		0x00000f00
-#define AXXX_CP_QUEUE_THRESHOLDS_CSQ_IB2_START__SHIFT		8
-static inline uint32_t AXXX_CP_QUEUE_THRESHOLDS_CSQ_IB2_START(uint32_t val)
-{
-	return ((val) << AXXX_CP_QUEUE_THRESHOLDS_CSQ_IB2_START__SHIFT) & AXXX_CP_QUEUE_THRESHOLDS_CSQ_IB2_START__MASK;
-}
-#define AXXX_CP_QUEUE_THRESHOLDS_CSQ_ST_START__MASK		0x000f0000
-#define AXXX_CP_QUEUE_THRESHOLDS_CSQ_ST_START__SHIFT		16
-static inline uint32_t AXXX_CP_QUEUE_THRESHOLDS_CSQ_ST_START(uint32_t val)
-{
-	return ((val) << AXXX_CP_QUEUE_THRESHOLDS_CSQ_ST_START__SHIFT) & AXXX_CP_QUEUE_THRESHOLDS_CSQ_ST_START__MASK;
-}
-
-#define REG_AXXX_CP_MEQ_THRESHOLDS				0x000001d6
-#define AXXX_CP_MEQ_THRESHOLDS_MEQ_END__MASK			0x001f0000
-#define AXXX_CP_MEQ_THRESHOLDS_MEQ_END__SHIFT			16
-static inline uint32_t AXXX_CP_MEQ_THRESHOLDS_MEQ_END(uint32_t val)
-{
-	return ((val) << AXXX_CP_MEQ_THRESHOLDS_MEQ_END__SHIFT) & AXXX_CP_MEQ_THRESHOLDS_MEQ_END__MASK;
-}
-#define AXXX_CP_MEQ_THRESHOLDS_ROQ_END__MASK			0x1f000000
-#define AXXX_CP_MEQ_THRESHOLDS_ROQ_END__SHIFT			24
-static inline uint32_t AXXX_CP_MEQ_THRESHOLDS_ROQ_END(uint32_t val)
-{
-	return ((val) << AXXX_CP_MEQ_THRESHOLDS_ROQ_END__SHIFT) & AXXX_CP_MEQ_THRESHOLDS_ROQ_END__MASK;
-}
-
-#define REG_AXXX_CP_CSQ_AVAIL					0x000001d7
-#define AXXX_CP_CSQ_AVAIL_RING__MASK				0x0000007f
-#define AXXX_CP_CSQ_AVAIL_RING__SHIFT				0
-static inline uint32_t AXXX_CP_CSQ_AVAIL_RING(uint32_t val)
-{
-	return ((val) << AXXX_CP_CSQ_AVAIL_RING__SHIFT) & AXXX_CP_CSQ_AVAIL_RING__MASK;
-}
-#define AXXX_CP_CSQ_AVAIL_IB1__MASK				0x00007f00
-#define AXXX_CP_CSQ_AVAIL_IB1__SHIFT				8
-static inline uint32_t AXXX_CP_CSQ_AVAIL_IB1(uint32_t val)
-{
-	return ((val) << AXXX_CP_CSQ_AVAIL_IB1__SHIFT) & AXXX_CP_CSQ_AVAIL_IB1__MASK;
-}
-#define AXXX_CP_CSQ_AVAIL_IB2__MASK				0x007f0000
-#define AXXX_CP_CSQ_AVAIL_IB2__SHIFT				16
-static inline uint32_t AXXX_CP_CSQ_AVAIL_IB2(uint32_t val)
-{
-	return ((val) << AXXX_CP_CSQ_AVAIL_IB2__SHIFT) & AXXX_CP_CSQ_AVAIL_IB2__MASK;
-}
-
-#define REG_AXXX_CP_STQ_AVAIL					0x000001d8
-#define AXXX_CP_STQ_AVAIL_ST__MASK				0x0000007f
-#define AXXX_CP_STQ_AVAIL_ST__SHIFT				0
-static inline uint32_t AXXX_CP_STQ_AVAIL_ST(uint32_t val)
-{
-	return ((val) << AXXX_CP_STQ_AVAIL_ST__SHIFT) & AXXX_CP_STQ_AVAIL_ST__MASK;
-}
-
-#define REG_AXXX_CP_MEQ_AVAIL					0x000001d9
-#define AXXX_CP_MEQ_AVAIL_MEQ__MASK				0x0000001f
-#define AXXX_CP_MEQ_AVAIL_MEQ__SHIFT				0
-static inline uint32_t AXXX_CP_MEQ_AVAIL_MEQ(uint32_t val)
-{
-	return ((val) << AXXX_CP_MEQ_AVAIL_MEQ__SHIFT) & AXXX_CP_MEQ_AVAIL_MEQ__MASK;
-}
-
-#define REG_AXXX_SCRATCH_UMSK					0x000001dc
-#define AXXX_SCRATCH_UMSK_UMSK__MASK				0x000000ff
-#define AXXX_SCRATCH_UMSK_UMSK__SHIFT				0
-static inline uint32_t AXXX_SCRATCH_UMSK_UMSK(uint32_t val)
-{
-	return ((val) << AXXX_SCRATCH_UMSK_UMSK__SHIFT) & AXXX_SCRATCH_UMSK_UMSK__MASK;
-}
-#define AXXX_SCRATCH_UMSK_SWAP__MASK				0x00030000
-#define AXXX_SCRATCH_UMSK_SWAP__SHIFT				16
-static inline uint32_t AXXX_SCRATCH_UMSK_SWAP(uint32_t val)
-{
-	return ((val) << AXXX_SCRATCH_UMSK_SWAP__SHIFT) & AXXX_SCRATCH_UMSK_SWAP__MASK;
-}
-
-#define REG_AXXX_SCRATCH_ADDR					0x000001dd
-
-#define REG_AXXX_CP_ME_RDADDR					0x000001ea
-
-#define REG_AXXX_CP_STATE_DEBUG_INDEX				0x000001ec
-
-#define REG_AXXX_CP_STATE_DEBUG_DATA				0x000001ed
-
-#define REG_AXXX_CP_INT_CNTL					0x000001f2
-#define AXXX_CP_INT_CNTL_SW_INT_MASK				0x00080000
-#define AXXX_CP_INT_CNTL_T0_PACKET_IN_IB_MASK			0x00800000
-#define AXXX_CP_INT_CNTL_OPCODE_ERROR_MASK			0x01000000
-#define AXXX_CP_INT_CNTL_PROTECTED_MODE_ERROR_MASK		0x02000000
-#define AXXX_CP_INT_CNTL_RESERVED_BIT_ERROR_MASK		0x04000000
-#define AXXX_CP_INT_CNTL_IB_ERROR_MASK				0x08000000
-#define AXXX_CP_INT_CNTL_IB2_INT_MASK				0x20000000
-#define AXXX_CP_INT_CNTL_IB1_INT_MASK				0x40000000
-#define AXXX_CP_INT_CNTL_RB_INT_MASK				0x80000000
-
-#define REG_AXXX_CP_INT_STATUS					0x000001f3
-
-#define REG_AXXX_CP_INT_ACK					0x000001f4
-
-#define REG_AXXX_CP_ME_CNTL					0x000001f6
-#define AXXX_CP_ME_CNTL_BUSY					0x20000000
-#define AXXX_CP_ME_CNTL_HALT					0x10000000
-
-#define REG_AXXX_CP_ME_STATUS					0x000001f7
-
-#define REG_AXXX_CP_ME_RAM_WADDR				0x000001f8
-
-#define REG_AXXX_CP_ME_RAM_RADDR				0x000001f9
-
-#define REG_AXXX_CP_ME_RAM_DATA					0x000001fa
-
-#define REG_AXXX_CP_DEBUG					0x000001fc
-#define AXXX_CP_DEBUG_PREDICATE_DISABLE				0x00800000
-#define AXXX_CP_DEBUG_PROG_END_PTR_ENABLE			0x01000000
-#define AXXX_CP_DEBUG_MIU_128BIT_WRITE_ENABLE			0x02000000
-#define AXXX_CP_DEBUG_PREFETCH_PASS_NOPS			0x04000000
-#define AXXX_CP_DEBUG_DYNAMIC_CLK_DISABLE			0x08000000
-#define AXXX_CP_DEBUG_PREFETCH_MATCH_DISABLE			0x10000000
-#define AXXX_CP_DEBUG_SIMPLE_ME_FLOW_CONTROL			0x40000000
-#define AXXX_CP_DEBUG_MIU_WRITE_PACK_DISABLE			0x80000000
-
-#define REG_AXXX_CP_CSQ_RB_STAT					0x000001fd
-#define AXXX_CP_CSQ_RB_STAT_RPTR__MASK				0x0000007f
-#define AXXX_CP_CSQ_RB_STAT_RPTR__SHIFT				0
-static inline uint32_t AXXX_CP_CSQ_RB_STAT_RPTR(uint32_t val)
-{
-	return ((val) << AXXX_CP_CSQ_RB_STAT_RPTR__SHIFT) & AXXX_CP_CSQ_RB_STAT_RPTR__MASK;
-}
-#define AXXX_CP_CSQ_RB_STAT_WPTR__MASK				0x007f0000
-#define AXXX_CP_CSQ_RB_STAT_WPTR__SHIFT				16
-static inline uint32_t AXXX_CP_CSQ_RB_STAT_WPTR(uint32_t val)
-{
-	return ((val) << AXXX_CP_CSQ_RB_STAT_WPTR__SHIFT) & AXXX_CP_CSQ_RB_STAT_WPTR__MASK;
-}
-
-#define REG_AXXX_CP_CSQ_IB1_STAT				0x000001fe
-#define AXXX_CP_CSQ_IB1_STAT_RPTR__MASK				0x0000007f
-#define AXXX_CP_CSQ_IB1_STAT_RPTR__SHIFT			0
-static inline uint32_t AXXX_CP_CSQ_IB1_STAT_RPTR(uint32_t val)
-{
-	return ((val) << AXXX_CP_CSQ_IB1_STAT_RPTR__SHIFT) & AXXX_CP_CSQ_IB1_STAT_RPTR__MASK;
-}
-#define AXXX_CP_CSQ_IB1_STAT_WPTR__MASK				0x007f0000
-#define AXXX_CP_CSQ_IB1_STAT_WPTR__SHIFT			16
-static inline uint32_t AXXX_CP_CSQ_IB1_STAT_WPTR(uint32_t val)
-{
-	return ((val) << AXXX_CP_CSQ_IB1_STAT_WPTR__SHIFT) & AXXX_CP_CSQ_IB1_STAT_WPTR__MASK;
-}
-
-#define REG_AXXX_CP_CSQ_IB2_STAT				0x000001ff
-#define AXXX_CP_CSQ_IB2_STAT_RPTR__MASK				0x0000007f
-#define AXXX_CP_CSQ_IB2_STAT_RPTR__SHIFT			0
-static inline uint32_t AXXX_CP_CSQ_IB2_STAT_RPTR(uint32_t val)
-{
-	return ((val) << AXXX_CP_CSQ_IB2_STAT_RPTR__SHIFT) & AXXX_CP_CSQ_IB2_STAT_RPTR__MASK;
-}
-#define AXXX_CP_CSQ_IB2_STAT_WPTR__MASK				0x007f0000
-#define AXXX_CP_CSQ_IB2_STAT_WPTR__SHIFT			16
-static inline uint32_t AXXX_CP_CSQ_IB2_STAT_WPTR(uint32_t val)
-{
-	return ((val) << AXXX_CP_CSQ_IB2_STAT_WPTR__SHIFT) & AXXX_CP_CSQ_IB2_STAT_WPTR__MASK;
-}
-
-#define REG_AXXX_CP_NON_PREFETCH_CNTRS				0x00000440
-
-#define REG_AXXX_CP_STQ_ST_STAT					0x00000443
-
-#define REG_AXXX_CP_ST_BASE					0x0000044d
-
-#define REG_AXXX_CP_ST_BUFSZ					0x0000044e
-
-#define REG_AXXX_CP_MEQ_STAT					0x0000044f
-
-#define REG_AXXX_CP_MIU_TAG_STAT				0x00000452
-
-#define REG_AXXX_CP_BIN_MASK_LO					0x00000454
-
-#define REG_AXXX_CP_BIN_MASK_HI					0x00000455
-
-#define REG_AXXX_CP_BIN_SELECT_LO				0x00000456
-
-#define REG_AXXX_CP_BIN_SELECT_HI				0x00000457
-
-#define REG_AXXX_CP_IB1_BASE					0x00000458
-
-#define REG_AXXX_CP_IB1_BUFSZ					0x00000459
-
-#define REG_AXXX_CP_IB2_BASE					0x0000045a
-
-#define REG_AXXX_CP_IB2_BUFSZ					0x0000045b
-
-#define REG_AXXX_CP_STAT					0x0000047f
-#define AXXX_CP_STAT_CP_BUSY					0x80000000
-#define AXXX_CP_STAT_VS_EVENT_FIFO_BUSY				0x40000000
-#define AXXX_CP_STAT_PS_EVENT_FIFO_BUSY				0x20000000
-#define AXXX_CP_STAT_CF_EVENT_FIFO_BUSY				0x10000000
-#define AXXX_CP_STAT_RB_EVENT_FIFO_BUSY				0x08000000
-#define AXXX_CP_STAT_ME_BUSY					0x04000000
-#define AXXX_CP_STAT_MIU_WR_C_BUSY				0x02000000
-#define AXXX_CP_STAT_CP_3D_BUSY					0x00800000
-#define AXXX_CP_STAT_CP_NRT_BUSY				0x00400000
-#define AXXX_CP_STAT_RBIU_SCRATCH_BUSY				0x00200000
-#define AXXX_CP_STAT_RCIU_ME_BUSY				0x00100000
-#define AXXX_CP_STAT_RCIU_PFP_BUSY				0x00080000
-#define AXXX_CP_STAT_MEQ_RING_BUSY				0x00040000
-#define AXXX_CP_STAT_PFP_BUSY					0x00020000
-#define AXXX_CP_STAT_ST_QUEUE_BUSY				0x00010000
-#define AXXX_CP_STAT_INDIRECT2_QUEUE_BUSY			0x00002000
-#define AXXX_CP_STAT_INDIRECTS_QUEUE_BUSY			0x00001000
-#define AXXX_CP_STAT_RING_QUEUE_BUSY				0x00000800
-#define AXXX_CP_STAT_CSF_BUSY					0x00000400
-#define AXXX_CP_STAT_CSF_ST_BUSY				0x00000200
-#define AXXX_CP_STAT_EVENT_BUSY					0x00000100
-#define AXXX_CP_STAT_CSF_INDIRECT2_BUSY				0x00000080
-#define AXXX_CP_STAT_CSF_INDIRECTS_BUSY				0x00000040
-#define AXXX_CP_STAT_CSF_RING_BUSY				0x00000020
-#define AXXX_CP_STAT_RCIU_BUSY					0x00000010
-#define AXXX_CP_STAT_RBIU_BUSY					0x00000008
-#define AXXX_CP_STAT_MIU_RD_RETURN_BUSY				0x00000004
-#define AXXX_CP_STAT_MIU_RD_REQ_BUSY				0x00000002
-#define AXXX_CP_STAT_MIU_WR_BUSY				0x00000001
-
-#define REG_AXXX_CP_SCRATCH_REG0				0x00000578
-
-#define REG_AXXX_CP_SCRATCH_REG1				0x00000579
-
-#define REG_AXXX_CP_SCRATCH_REG2				0x0000057a
-
-#define REG_AXXX_CP_SCRATCH_REG3				0x0000057b
-
-#define REG_AXXX_CP_SCRATCH_REG4				0x0000057c
-
-#define REG_AXXX_CP_SCRATCH_REG5				0x0000057d
-
-#define REG_AXXX_CP_SCRATCH_REG6				0x0000057e
-
-#define REG_AXXX_CP_SCRATCH_REG7				0x0000057f
-
-#define REG_AXXX_CP_ME_VS_EVENT_SRC				0x00000600
-
-#define REG_AXXX_CP_ME_VS_EVENT_ADDR				0x00000601
-
-#define REG_AXXX_CP_ME_VS_EVENT_DATA				0x00000602
-
-#define REG_AXXX_CP_ME_VS_EVENT_ADDR_SWM			0x00000603
-
-#define REG_AXXX_CP_ME_VS_EVENT_DATA_SWM			0x00000604
-
-#define REG_AXXX_CP_ME_PS_EVENT_SRC				0x00000605
-
-#define REG_AXXX_CP_ME_PS_EVENT_ADDR				0x00000606
-
-#define REG_AXXX_CP_ME_PS_EVENT_DATA				0x00000607
-
-#define REG_AXXX_CP_ME_PS_EVENT_ADDR_SWM			0x00000608
-
-#define REG_AXXX_CP_ME_PS_EVENT_DATA_SWM			0x00000609
-
-#define REG_AXXX_CP_ME_CF_EVENT_SRC				0x0000060a
-
-#define REG_AXXX_CP_ME_CF_EVENT_ADDR				0x0000060b
-
-#define REG_AXXX_CP_ME_CF_EVENT_DATA				0x0000060c
-
-#define REG_AXXX_CP_ME_NRT_ADDR					0x0000060d
-
-#define REG_AXXX_CP_ME_NRT_DATA					0x0000060e
-
-#define REG_AXXX_CP_ME_VS_FETCH_DONE_SRC			0x00000612
-
-#define REG_AXXX_CP_ME_VS_FETCH_DONE_ADDR			0x00000613
-
-#define REG_AXXX_CP_ME_VS_FETCH_DONE_DATA			0x00000614
-
-#ifdef __cplusplus
-#endif
-
-#endif /* ADRENO_COMMON_XML */
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gen7_9_0_snapshot.h b/drivers/gpu/drm/msm/adreno/adreno_gen7_9_0_snapshot.h
new file mode 100644
index 000000000000..260d66eccfec
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/adreno_gen7_9_0_snapshot.h
@@ -0,0 +1,1446 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+#ifndef __ADRENO_GEN7_9_0_SNAPSHOT_H
+#define __ADRENO_GEN7_9_0_SNAPSHOT_H
+
+#include "a6xx_gpu_state.h"
+
+static const u32 gen7_9_0_debugbus_blocks[] = {
+	A7XX_DBGBUS_CP_0_0,
+	A7XX_DBGBUS_CP_0_1,
+	A7XX_DBGBUS_RBBM,
+	A7XX_DBGBUS_HLSQ,
+	A7XX_DBGBUS_UCHE_0,
+	A7XX_DBGBUS_UCHE_1,
+	A7XX_DBGBUS_TESS_BR,
+	A7XX_DBGBUS_TESS_BV,
+	A7XX_DBGBUS_PC_BR,
+	A7XX_DBGBUS_PC_BV,
+	A7XX_DBGBUS_VFDP_BR,
+	A7XX_DBGBUS_VFDP_BV,
+	A7XX_DBGBUS_VPC_BR,
+	A7XX_DBGBUS_VPC_BV,
+	A7XX_DBGBUS_TSE_BR,
+	A7XX_DBGBUS_TSE_BV,
+	A7XX_DBGBUS_RAS_BR,
+	A7XX_DBGBUS_RAS_BV,
+	A7XX_DBGBUS_VSC,
+	A7XX_DBGBUS_COM_0,
+	A7XX_DBGBUS_LRZ_BR,
+	A7XX_DBGBUS_LRZ_BV,
+	A7XX_DBGBUS_UFC_0,
+	A7XX_DBGBUS_UFC_1,
+	A7XX_DBGBUS_GMU_GX,
+	A7XX_DBGBUS_DBGC,
+	A7XX_DBGBUS_GPC_BR,
+	A7XX_DBGBUS_GPC_BV,
+	A7XX_DBGBUS_LARC,
+	A7XX_DBGBUS_HLSQ_SPTP,
+	A7XX_DBGBUS_RB_0,
+	A7XX_DBGBUS_RB_1,
+	A7XX_DBGBUS_RB_2,
+	A7XX_DBGBUS_RB_3,
+	A7XX_DBGBUS_RB_4,
+	A7XX_DBGBUS_RB_5,
+	A7XX_DBGBUS_UCHE_WRAPPER,
+	A7XX_DBGBUS_CCU_0,
+	A7XX_DBGBUS_CCU_1,
+	A7XX_DBGBUS_CCU_2,
+	A7XX_DBGBUS_CCU_3,
+	A7XX_DBGBUS_CCU_4,
+	A7XX_DBGBUS_CCU_5,
+	A7XX_DBGBUS_VFD_BR_0,
+	A7XX_DBGBUS_VFD_BR_1,
+	A7XX_DBGBUS_VFD_BR_2,
+	A7XX_DBGBUS_VFD_BV_0,
+	A7XX_DBGBUS_VFD_BV_1,
+	A7XX_DBGBUS_VFD_BV_2,
+	A7XX_DBGBUS_USP_0,
+	A7XX_DBGBUS_USP_1,
+	A7XX_DBGBUS_USP_2,
+	A7XX_DBGBUS_USP_3,
+	A7XX_DBGBUS_USP_4,
+	A7XX_DBGBUS_USP_5,
+	A7XX_DBGBUS_TP_0,
+	A7XX_DBGBUS_TP_1,
+	A7XX_DBGBUS_TP_2,
+	A7XX_DBGBUS_TP_3,
+	A7XX_DBGBUS_TP_4,
+	A7XX_DBGBUS_TP_5,
+	A7XX_DBGBUS_TP_6,
+	A7XX_DBGBUS_TP_7,
+	A7XX_DBGBUS_TP_8,
+	A7XX_DBGBUS_TP_9,
+	A7XX_DBGBUS_TP_10,
+	A7XX_DBGBUS_TP_11,
+	A7XX_DBGBUS_USPTP_0,
+	A7XX_DBGBUS_USPTP_1,
+	A7XX_DBGBUS_USPTP_2,
+	A7XX_DBGBUS_USPTP_3,
+	A7XX_DBGBUS_USPTP_4,
+	A7XX_DBGBUS_USPTP_5,
+	A7XX_DBGBUS_USPTP_6,
+	A7XX_DBGBUS_USPTP_7,
+	A7XX_DBGBUS_USPTP_8,
+	A7XX_DBGBUS_USPTP_9,
+	A7XX_DBGBUS_USPTP_10,
+	A7XX_DBGBUS_USPTP_11,
+	A7XX_DBGBUS_CCHE_0,
+	A7XX_DBGBUS_CCHE_1,
+	A7XX_DBGBUS_CCHE_2,
+	A7XX_DBGBUS_VPC_DSTR_0,
+	A7XX_DBGBUS_VPC_DSTR_1,
+	A7XX_DBGBUS_VPC_DSTR_2,
+	A7XX_DBGBUS_HLSQ_DP_STR_0,
+	A7XX_DBGBUS_HLSQ_DP_STR_1,
+	A7XX_DBGBUS_HLSQ_DP_STR_2,
+	A7XX_DBGBUS_HLSQ_DP_STR_3,
+	A7XX_DBGBUS_HLSQ_DP_STR_4,
+	A7XX_DBGBUS_HLSQ_DP_STR_5,
+	A7XX_DBGBUS_UFC_DSTR_0,
+	A7XX_DBGBUS_UFC_DSTR_1,
+	A7XX_DBGBUS_UFC_DSTR_2,
+	A7XX_DBGBUS_CGC_SUBCORE,
+	A7XX_DBGBUS_CGC_CORE,
+};
+
+static const u32 gen7_9_0_gbif_debugbus_blocks[] = {
+	A7XX_DBGBUS_GBIF_GX,
+};
+
+static const u32 gen7_9_0_cx_debugbus_blocks[] = {
+	A7XX_DBGBUS_CX,
+	A7XX_DBGBUS_GMU_CX,
+	A7XX_DBGBUS_GBIF_CX,
+};
+
+static struct gen7_shader_block gen7_9_0_shader_blocks[] = {
+	{ A7XX_TP0_TMO_DATA, 0x0200, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_TP0_SMO_DATA, 0x0080, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_TP0_MIPMAP_BASE_DATA, 0x03C0, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_INST_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_INST_DATA_1, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_0_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_1_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_2_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_3_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_4_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_5_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_6_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_7_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_CB_RAM, 0x0390, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_13_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_14_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_INST_TAG, 0x00C0, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_INST_DATA_2, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_TMO_TAG, 0x0080, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_SMO_TAG, 0x0080, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_STATE_DATA, 0x0040, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_HWAVE_RAM, 0x0100, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_L0_INST_BUF, 0x0050, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_8_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_9_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_10_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_11_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_SP_LB_12_DATA, 0x0800, 6, 2, A7XX_PIPE_BR, A7XX_USPTP },
+	{ A7XX_HLSQ_DATAPATH_DSTR_META, 0x0010, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_DATAPATH_DSTR_META, 0x0010, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_L2STC_TAG_RAM, 0x0200, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_L2STC_INFO_CMD, 0x0474, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CVS_BE_CTXT_BUF_RAM_TAG, 0x0080, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CVS_BE_CTXT_BUF_RAM_TAG, 0x0080, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CPS_BE_CTXT_BUF_RAM_TAG, 0x0080, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_CVS_BE_CTXT_BUF_RAM, 0x0400, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_CVS_BE_CTXT_BUF_RAM, 0x0400, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_CPS_BE_CTXT_BUF_RAM, 0x0400, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CHUNK_CVS_RAM, 0x01C0, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CHUNK_CVS_RAM, 0x01C0, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CHUNK_CPS_RAM, 0x0300, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CHUNK_CPS_RAM, 0x0180, 1, 1, A7XX_PIPE_LPAC, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CHUNK_CVS_RAM_TAG, 0x0040, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CHUNK_CVS_RAM_TAG, 0x0040, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CHUNK_CPS_RAM_TAG, 0x0040, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CHUNK_CPS_RAM_TAG, 0x0040, 1, 1, A7XX_PIPE_LPAC, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_ICB_CVS_CB_BASE_TAG, 0x0010, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_ICB_CVS_CB_BASE_TAG, 0x0010, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_ICB_CPS_CB_BASE_TAG, 0x0010, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_ICB_CPS_CB_BASE_TAG, 0x0010, 1, 1, A7XX_PIPE_LPAC, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CVS_MISC_RAM, 0x0540, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CVS_MISC_RAM, 0x0540, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CPS_MISC_RAM, 0x0640, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CPS_MISC_RAM, 0x00B0, 1, 1, A7XX_PIPE_LPAC, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CPS_MISC_RAM_1, 0x0800, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_INST_RAM, 0x0800, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_INST_RAM, 0x0800, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_INST_RAM, 0x0200, 1, 1, A7XX_PIPE_LPAC, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_CVS_CONST_RAM, 0x0800, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_CVS_CONST_RAM, 0x0800, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_CPS_CONST_RAM, 0x0800, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_CPS_CONST_RAM, 0x0800, 1, 1, A7XX_PIPE_LPAC, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CVS_MISC_RAM_TAG, 0x0050, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CVS_MISC_RAM_TAG, 0x0050, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CPS_MISC_RAM_TAG, 0x0050, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_CPS_MISC_RAM_TAG, 0x0008, 1, 1, A7XX_PIPE_LPAC, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_INST_RAM_TAG, 0x0014, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_INST_RAM_TAG, 0x0010, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_INST_RAM_TAG, 0x0004, 1, 1, A7XX_PIPE_LPAC, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_CVS_CONST_RAM_TAG, 0x0040, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_CVS_CONST_RAM_TAG, 0x0040, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_CPS_CONST_RAM_TAG, 0x0040, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_CPS_CONST_RAM_TAG, 0x0020, 1, 1, A7XX_PIPE_LPAC, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_LOCAL_MISC_RAM, 0x03C0, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_LOCAL_MISC_RAM, 0x0280, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_LOCAL_MISC_RAM, 0x0050, 1, 1, A7XX_PIPE_LPAC, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_LOCAL_MISC_RAM_TAG, 0x0010, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_GFX_LOCAL_MISC_RAM_TAG, 0x0008, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_INST_RAM_1, 0x0800, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_STPROC_META, 0x0010, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_BV_BE_META, 0x0018, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_BV_BE_META, 0x0018, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_INST_RAM_2, 0x0800, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_DATAPATH_META, 0x0020, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_FRONTEND_META, 0x0080, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_FRONTEND_META, 0x0080, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_FRONTEND_META, 0x0080, 1, 1, A7XX_PIPE_LPAC, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_INDIRECT_META, 0x0010, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_BACKEND_META, 0x0040, 1, 1, A7XX_PIPE_BR, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_BACKEND_META, 0x0040, 1, 1, A7XX_PIPE_BV, A7XX_HLSQ_STATE },
+	{ A7XX_HLSQ_BACKEND_META, 0x0040, 1, 1, A7XX_PIPE_LPAC, A7XX_HLSQ_STATE },
+};
+
+/*
+ * Block   : ['PRE_CRASHDUMPER', 'GBIF']
+ * pairs   : 2 (Regs:5), 5 (Regs:38)
+ */
+static const u32 gen7_9_0_pre_crashdumper_gpu_registers[] = {
+	 0x00210, 0x00213, 0x00536, 0x00536, 0x03c00, 0x03c0b, 0x03c40, 0x03c42,
+	 0x03c45, 0x03c47, 0x03c49, 0x03c4a, 0x03cc0, 0x03cd1,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_pre_crashdumper_gpu_registers), 8));
+
+/*
+ * Block   : ['BROADCAST', 'CP', 'GRAS', 'GXCLKCTL']
+ * Block   : ['PC', 'RBBM', 'RDVM', 'UCHE']
+ * Block   : ['VFD', 'VPC', 'VSC']
+ * Pipeline: A7XX_PIPE_NONE
+ * pairs   : 196 (Regs:1778)
+ */
+static const u32 gen7_9_0_gpu_registers[] = {
+	 0x00000, 0x00000, 0x00002, 0x00002, 0x00011, 0x00012, 0x00016, 0x0001b,
+	 0x0001f, 0x00032, 0x00038, 0x0003c, 0x00044, 0x00044, 0x00047, 0x00047,
+	 0x00049, 0x0004a, 0x0004c, 0x0004c, 0x00056, 0x00056, 0x00073, 0x0007d,
+	 0x00090, 0x000a8, 0x000ad, 0x000ad, 0x00117, 0x00117, 0x00120, 0x00122,
+	 0x00130, 0x0013f, 0x00142, 0x0015f, 0x00162, 0x00164, 0x00166, 0x00171,
+	 0x00173, 0x00174, 0x00176, 0x0017b, 0x0017e, 0x00180, 0x00183, 0x00192,
+	 0x00195, 0x00196, 0x00199, 0x0019a, 0x0019d, 0x001a2, 0x001aa, 0x001ae,
+	 0x001b9, 0x001b9, 0x001bb, 0x001bb, 0x001be, 0x001be, 0x001c1, 0x001c2,
+	 0x001c5, 0x001c5, 0x001c7, 0x001c7, 0x001c9, 0x001c9, 0x001cb, 0x001ce,
+	 0x001d1, 0x001df, 0x001e1, 0x001e3, 0x001e5, 0x001e5, 0x001e7, 0x001e9,
+	 0x00200, 0x0020d, 0x00215, 0x00253, 0x00260, 0x00260, 0x00264, 0x00270,
+	 0x00272, 0x00274, 0x00281, 0x00281, 0x00283, 0x00283, 0x00289, 0x0028d,
+	 0x00290, 0x002a2, 0x002c0, 0x002c1, 0x00300, 0x00401, 0x00410, 0x00451,
+	 0x00460, 0x004a3, 0x004c0, 0x004d1, 0x00500, 0x00500, 0x00507, 0x0050b,
+	 0x0050f, 0x0050f, 0x00511, 0x00511, 0x00533, 0x00535, 0x00540, 0x0055b,
+	 0x00564, 0x00567, 0x00574, 0x00577, 0x00584, 0x0059b, 0x005fb, 0x005ff,
+	 0x00800, 0x00808, 0x00810, 0x00813, 0x00820, 0x00821, 0x00823, 0x00827,
+	 0x00830, 0x00834, 0x0083f, 0x00841, 0x00843, 0x00847, 0x0084f, 0x00886,
+	 0x008a0, 0x008ab, 0x008c0, 0x008c0, 0x008c4, 0x008c4, 0x008c6, 0x008c6,
+	 0x008d0, 0x008dd, 0x008e0, 0x008e6, 0x008f0, 0x008f3, 0x00900, 0x00903,
+	 0x00908, 0x00911, 0x00928, 0x0093e, 0x00942, 0x0094d, 0x00980, 0x00984,
+	 0x0098d, 0x0098f, 0x009b0, 0x009b4, 0x009c2, 0x009c9, 0x009ce, 0x009d7,
+	 0x009e0, 0x009e7, 0x00a00, 0x00a00, 0x00a02, 0x00a03, 0x00a10, 0x00a4f,
+	 0x00a61, 0x00a9f, 0x00ad0, 0x00adb, 0x00b00, 0x00b31, 0x00b35, 0x00b3c,
+	 0x00b40, 0x00b40, 0x00b70, 0x00b73, 0x00b78, 0x00b79, 0x00b7c, 0x00b7d,
+	 0x00b80, 0x00b81, 0x00b84, 0x00b85, 0x00b88, 0x00b89, 0x00b8c, 0x00b8d,
+	 0x00b90, 0x00b93, 0x00b98, 0x00b99, 0x00b9c, 0x00b9d, 0x00ba0, 0x00ba1,
+	 0x00ba4, 0x00ba5, 0x00ba8, 0x00ba9, 0x00bac, 0x00bad, 0x00bb0, 0x00bb1,
+	 0x00bb4, 0x00bb5, 0x00bb8, 0x00bb9, 0x00bbc, 0x00bbd, 0x00bc0, 0x00bc1,
+	 0x00c00, 0x00c00, 0x00c02, 0x00c04, 0x00c06, 0x00c06, 0x00c10, 0x00cd9,
+	 0x00ce0, 0x00d0c, 0x00df0, 0x00df4, 0x00e01, 0x00e02, 0x00e07, 0x00e0e,
+	 0x00e10, 0x00e13, 0x00e17, 0x00e19, 0x00e1c, 0x00e2b, 0x00e30, 0x00e32,
+	 0x00e3a, 0x00e3d, 0x00e50, 0x00e5b, 0x02840, 0x0287f, 0x0ec00, 0x0ec01,
+	 0x0ec05, 0x0ec05, 0x0ec07, 0x0ec07, 0x0ec0a, 0x0ec0a, 0x0ec12, 0x0ec12,
+	 0x0ec26, 0x0ec28, 0x0ec2b, 0x0ec2d, 0x0ec2f, 0x0ec2f, 0x0ec40, 0x0ec41,
+	 0x0ec45, 0x0ec45, 0x0ec47, 0x0ec47, 0x0ec4a, 0x0ec4a, 0x0ec52, 0x0ec52,
+	 0x0ec66, 0x0ec68, 0x0ec6b, 0x0ec6d, 0x0ec6f, 0x0ec6f, 0x0ec80, 0x0ec81,
+	 0x0ec85, 0x0ec85, 0x0ec87, 0x0ec87, 0x0ec8a, 0x0ec8a, 0x0ec92, 0x0ec92,
+	 0x0eca6, 0x0eca8, 0x0ecab, 0x0ecad, 0x0ecaf, 0x0ecaf, 0x0ecc0, 0x0ecc1,
+	 0x0ecc5, 0x0ecc5, 0x0ecc7, 0x0ecc7, 0x0ecca, 0x0ecca, 0x0ecd2, 0x0ecd2,
+	 0x0ece6, 0x0ece8, 0x0eceb, 0x0eced, 0x0ecef, 0x0ecef, 0x0ed00, 0x0ed01,
+	 0x0ed05, 0x0ed05, 0x0ed07, 0x0ed07, 0x0ed0a, 0x0ed0a, 0x0ed12, 0x0ed12,
+	 0x0ed26, 0x0ed28, 0x0ed2b, 0x0ed2d, 0x0ed2f, 0x0ed2f, 0x0ed40, 0x0ed41,
+	 0x0ed45, 0x0ed45, 0x0ed47, 0x0ed47, 0x0ed4a, 0x0ed4a, 0x0ed52, 0x0ed52,
+	 0x0ed66, 0x0ed68, 0x0ed6b, 0x0ed6d, 0x0ed6f, 0x0ed6f, 0x0ed80, 0x0ed81,
+	 0x0ed85, 0x0ed85, 0x0ed87, 0x0ed87, 0x0ed8a, 0x0ed8a, 0x0ed92, 0x0ed92,
+	 0x0eda6, 0x0eda8, 0x0edab, 0x0edad, 0x0edaf, 0x0edaf,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_gpu_registers), 8));
+
+static const u32 gen7_9_0_gxclkctl_registers[] = {
+	 0x18800, 0x18800, 0x18808, 0x1880b, 0x18820, 0x18822, 0x18830, 0x18830,
+	 0x18834, 0x1883b,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_gxclkctl_registers), 8));
+
+/*
+ * Block   : ['GMUAO', 'GMUCX', 'GMUCX_RAM']
+ * Pipeline: A7XX_PIPE_NONE
+ * pairs   : 134 (Regs:429)
+ */
+static const u32 gen7_9_0_gmu_registers[] = {
+	 0x10001, 0x10001, 0x10003, 0x10003, 0x10401, 0x10401, 0x10403, 0x10403,
+	 0x10801, 0x10801, 0x10803, 0x10803, 0x10c01, 0x10c01, 0x10c03, 0x10c03,
+	 0x11001, 0x11001, 0x11003, 0x11003, 0x11401, 0x11401, 0x11403, 0x11403,
+	 0x11801, 0x11801, 0x11803, 0x11803, 0x11c01, 0x11c01, 0x11c03, 0x11c03,
+	 0x1f400, 0x1f40b, 0x1f40f, 0x1f411, 0x1f500, 0x1f500, 0x1f507, 0x1f507,
+	 0x1f509, 0x1f50b, 0x1f700, 0x1f701, 0x1f704, 0x1f706, 0x1f708, 0x1f709,
+	 0x1f70c, 0x1f70d, 0x1f710, 0x1f711, 0x1f713, 0x1f716, 0x1f718, 0x1f71d,
+	 0x1f720, 0x1f724, 0x1f729, 0x1f729, 0x1f730, 0x1f747, 0x1f750, 0x1f756,
+	 0x1f758, 0x1f759, 0x1f75c, 0x1f75c, 0x1f760, 0x1f761, 0x1f764, 0x1f76b,
+	 0x1f770, 0x1f775, 0x1f780, 0x1f785, 0x1f790, 0x1f798, 0x1f7a0, 0x1f7a8,
+	 0x1f7b0, 0x1f7b3, 0x1f800, 0x1f804, 0x1f807, 0x1f808, 0x1f80b, 0x1f80c,
+	 0x1f80f, 0x1f80f, 0x1f811, 0x1f811, 0x1f813, 0x1f817, 0x1f819, 0x1f81c,
+	 0x1f824, 0x1f82a, 0x1f82d, 0x1f830, 0x1f840, 0x1f853, 0x1f860, 0x1f860,
+	 0x1f862, 0x1f866, 0x1f868, 0x1f869, 0x1f870, 0x1f879, 0x1f87f, 0x1f881,
+	 0x1f890, 0x1f896, 0x1f8a0, 0x1f8a2, 0x1f8a4, 0x1f8af, 0x1f8b8, 0x1f8b9,
+	 0x1f8c0, 0x1f8c1, 0x1f8c3, 0x1f8c4, 0x1f8d0, 0x1f8d0, 0x1f8ec, 0x1f8ec,
+	 0x1f8f0, 0x1f8f1, 0x1f910, 0x1f917, 0x1f920, 0x1f921, 0x1f924, 0x1f925,
+	 0x1f928, 0x1f929, 0x1f92c, 0x1f92d, 0x1f942, 0x1f944, 0x1f948, 0x1f94a,
+	 0x1f94f, 0x1f951, 0x1f954, 0x1f955, 0x1f95d, 0x1f95d, 0x1f962, 0x1f96b,
+	 0x1f970, 0x1f971, 0x1f973, 0x1f977, 0x1f97c, 0x1f97c, 0x1f980, 0x1f981,
+	 0x1f984, 0x1f986, 0x1f992, 0x1f993, 0x1f996, 0x1f99e, 0x1f9c5, 0x1f9d4,
+	 0x1f9f0, 0x1f9f1, 0x1f9f8, 0x1f9fa, 0x1f9fc, 0x1f9fc, 0x1fa00, 0x1fa03,
+	 0x20000, 0x20013, 0x20018, 0x2001a, 0x20020, 0x20021, 0x20024, 0x20025,
+	 0x2002a, 0x2002c, 0x20030, 0x20031, 0x20034, 0x20036, 0x23801, 0x23801,
+	 0x23803, 0x23803, 0x23805, 0x23805, 0x23807, 0x23807, 0x23809, 0x23809,
+	 0x2380b, 0x2380b, 0x2380d, 0x2380d, 0x2380f, 0x2380f, 0x23811, 0x23811,
+	 0x23813, 0x23813, 0x23815, 0x23815, 0x23817, 0x23817, 0x23819, 0x23819,
+	 0x2381b, 0x2381b, 0x2381d, 0x2381d, 0x2381f, 0x23820, 0x23822, 0x23822,
+	 0x23824, 0x23824, 0x23826, 0x23826, 0x23828, 0x23828, 0x2382a, 0x2382a,
+	 0x2382c, 0x2382c, 0x2382e, 0x2382e, 0x23830, 0x23830, 0x23832, 0x23832,
+	 0x23834, 0x23834, 0x23836, 0x23836, 0x23838, 0x23838, 0x2383a, 0x2383a,
+	 0x2383c, 0x2383c, 0x2383e, 0x2383e, 0x23840, 0x23847, 0x23b00, 0x23b01,
+	 0x23b03, 0x23b03, 0x23b05, 0x23b0e, 0x23b10, 0x23b13, 0x23b15, 0x23b16,
+	 0x23b28, 0x23b28, 0x23b30, 0x23b30,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_gmu_registers), 8));
+
+/*
+ * Block   : ['GMUGX']
+ * Pipeline: A7XX_PIPE_NONE
+ * pairs   : 44 (Regs:454)
+ */
+static const u32 gen7_9_0_gmugx_registers[] = {
+	 0x1a400, 0x1a41f, 0x1a440, 0x1a45f, 0x1a480, 0x1a49f, 0x1a4c0, 0x1a4df,
+	 0x1a500, 0x1a51f, 0x1a540, 0x1a55f, 0x1a580, 0x1a59f, 0x1a600, 0x1a61f,
+	 0x1a640, 0x1a65f, 0x1a780, 0x1a781, 0x1a783, 0x1a785, 0x1a787, 0x1a789,
+	 0x1a78b, 0x1a78d, 0x1a78f, 0x1a791, 0x1a793, 0x1a795, 0x1a797, 0x1a799,
+	 0x1a79b, 0x1a79d, 0x1a79f, 0x1a7a1, 0x1a7a3, 0x1a7a3, 0x1a7a8, 0x1a7b9,
+	 0x1a7c0, 0x1a7c1, 0x1a7c4, 0x1a7c5, 0x1a7c8, 0x1a7c9, 0x1a7cc, 0x1a7cd,
+	 0x1a7d0, 0x1a7d1, 0x1a7d4, 0x1a7d5, 0x1a7d8, 0x1a7d9, 0x1a7dc, 0x1a7dd,
+	 0x1a7e0, 0x1a7e1, 0x1a7fc, 0x1a7fd, 0x1a800, 0x1a808, 0x1a816, 0x1a816,
+	 0x1a81e, 0x1a81e, 0x1a826, 0x1a826, 0x1a82e, 0x1a82e, 0x1a836, 0x1a836,
+	 0x1a83e, 0x1a83e, 0x1a846, 0x1a846, 0x1a84e, 0x1a84e, 0x1a856, 0x1a856,
+	 0x1a883, 0x1a884, 0x1a890, 0x1a8b3, 0x1a900, 0x1a92b, 0x1a940, 0x1a940,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_gmugx_registers), 8));
+
+/*
+ * Block   : ['CX_MISC']
+ * Pipeline: A7XX_PIPE_NONE
+ * pairs   : 7 (Regs:56)
+ */
+static const u32 gen7_9_0_cx_misc_registers[] = {
+	 0x27800, 0x27800, 0x27810, 0x27814, 0x27820, 0x27824, 0x27828, 0x2782a,
+	 0x27832, 0x27857, 0x27880, 0x27881, 0x27c00, 0x27c01,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_cx_misc_registers), 8));
+
+/*
+ * Block   : ['DBGC']
+ * Pipeline: A7XX_PIPE_NONE
+ * pairs   : 19 (Regs:155)
+ */
+static const u32 gen7_9_0_dbgc_registers[] = {
+	 0x00600, 0x0061c, 0x0061e, 0x00634, 0x00640, 0x00643, 0x0064e, 0x00652,
+	 0x00654, 0x0065e, 0x00699, 0x00699, 0x0069b, 0x0069e, 0x006c2, 0x006e4,
+	 0x006e6, 0x006e6, 0x006e9, 0x006e9, 0x006eb, 0x006eb, 0x006f1, 0x006f4,
+	 0x00700, 0x00707, 0x00718, 0x00718, 0x00720, 0x00729, 0x00740, 0x0074a,
+	 0x00758, 0x00758, 0x00760, 0x00762,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_dbgc_registers), 8));
+
+/*
+ * Block   : ['CX_DBGC']
+ * Pipeline: A7XX_PIPE_NONE
+ * pairs   : 7 (Regs:75)
+ */
+static const u32 gen7_9_0_cx_dbgc_registers[] = {
+	 0x18400, 0x1841c, 0x1841e, 0x18434, 0x18440, 0x18443, 0x1844e, 0x18452,
+	 0x18454, 0x1845e, 0x18520, 0x18520, 0x18580, 0x18581,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_cx_dbgc_registers), 8));
+
+/*
+ * Block   : ['BROADCAST', 'CP', 'CX_DBGC', 'CX_MISC', 'DBGC', 'GBIF']
+ * Block   : ['GMUAO', 'GMUCX', 'GMUGX', 'GRAS', 'GXCLKCTL', 'PC']
+ * Block   : ['RBBM', 'RDVM', 'UCHE', 'VFD', 'VPC', 'VSC']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_NONE
+ * pairs   : 29 (Regs:573)
+ */
+static const u32 gen7_9_0_non_context_pipe_br_registers[] = {
+	 0x00887, 0x0088c, 0x08600, 0x08602, 0x08610, 0x0861b, 0x08620, 0x08620,
+	 0x08630, 0x08630, 0x08637, 0x08639, 0x08640, 0x08640, 0x09600, 0x09603,
+	 0x0960a, 0x09616, 0x09624, 0x0963a, 0x09640, 0x09640, 0x09e00, 0x09e00,
+	 0x09e02, 0x09e07, 0x09e0a, 0x09e16, 0x09e18, 0x09e1a, 0x09e1c, 0x09e1c,
+	 0x09e20, 0x09e25, 0x09e30, 0x09e31, 0x09e40, 0x09e51, 0x09e64, 0x09e6c,
+	 0x09e70, 0x09e72, 0x09e78, 0x09e79, 0x09e80, 0x09fff, 0x0a600, 0x0a600,
+	 0x0a603, 0x0a603, 0x0a610, 0x0a61f, 0x0a630, 0x0a631, 0x0a638, 0x0a63c,
+	 0x0a640, 0x0a65f,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_pipe_br_registers), 8));
+
+/*
+ * Block   : ['BROADCAST', 'CP', 'CX_DBGC', 'CX_MISC', 'DBGC', 'GBIF']
+ * Block   : ['GMUAO', 'GMUCX', 'GMUGX', 'GRAS', 'GXCLKCTL', 'PC']
+ * Block   : ['RBBM', 'RDVM', 'UCHE', 'VFD', 'VPC', 'VSC']
+ * Pipeline: A7XX_PIPE_BV
+ * Cluster : A7XX_CLUSTER_NONE
+ * pairs   : 29 (Regs:573)
+ */
+static const u32 gen7_9_0_non_context_pipe_bv_registers[] = {
+	 0x00887, 0x0088c, 0x08600, 0x08602, 0x08610, 0x0861b, 0x08620, 0x08620,
+	 0x08630, 0x08630, 0x08637, 0x08639, 0x08640, 0x08640, 0x09600, 0x09603,
+	 0x0960a, 0x09616, 0x09624, 0x0963a, 0x09640, 0x09640, 0x09e00, 0x09e00,
+	 0x09e02, 0x09e07, 0x09e0a, 0x09e16, 0x09e18, 0x09e1a, 0x09e1c, 0x09e1c,
+	 0x09e20, 0x09e25, 0x09e30, 0x09e31, 0x09e40, 0x09e51, 0x09e64, 0x09e6c,
+	 0x09e70, 0x09e72, 0x09e78, 0x09e79, 0x09e80, 0x09fff, 0x0a600, 0x0a600,
+	 0x0a603, 0x0a603, 0x0a610, 0x0a61f, 0x0a630, 0x0a631, 0x0a638, 0x0a63c,
+	 0x0a640, 0x0a65f,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_pipe_bv_registers), 8));
+
+/*
+ * Block   : ['BROADCAST', 'CP', 'CX_DBGC', 'CX_MISC', 'DBGC', 'GBIF']
+ * Block   : ['GMUAO', 'GMUCX', 'GMUGX', 'GRAS', 'GXCLKCTL', 'PC']
+ * Block   : ['RBBM', 'RDVM', 'UCHE', 'VFD', 'VPC', 'VSC']
+ * Pipeline: A7XX_PIPE_LPAC
+ * Cluster : A7XX_CLUSTER_NONE
+ * pairs   : 2 (Regs:7)
+ */
+static const u32 gen7_9_0_non_context_pipe_lpac_registers[] = {
+	 0x00887, 0x0088c, 0x00f80, 0x00f80,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_pipe_lpac_registers), 8));
+
+/*
+ * Block   : ['RB']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_NONE
+ * pairs   : 5 (Regs:37)
+ */
+static const u32 gen7_9_0_non_context_rb_pipe_br_rac_registers[] = {
+	 0x08e10, 0x08e1c, 0x08e20, 0x08e25, 0x08e51, 0x08e5a, 0x08e6a, 0x08e6d,
+	 0x08ea0, 0x08ea3,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_rb_pipe_br_rac_registers), 8));
+
+/*
+ * Block   : ['RB']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_NONE
+ * pairs   : 15 (Regs:66)
+ */
+static const u32 gen7_9_0_non_context_rb_pipe_br_rbp_registers[] = {
+	 0x08e01, 0x08e01, 0x08e04, 0x08e04, 0x08e06, 0x08e09, 0x08e0c, 0x08e0c,
+	 0x08e28, 0x08e28, 0x08e2c, 0x08e35, 0x08e3b, 0x08e40, 0x08e50, 0x08e50,
+	 0x08e5b, 0x08e5d, 0x08e5f, 0x08e5f, 0x08e61, 0x08e61, 0x08e63, 0x08e66,
+	 0x08e68, 0x08e69, 0x08e70, 0x08e7d, 0x08e80, 0x08e8f,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_rb_pipe_br_rbp_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_NONE
+ * Location: A7XX_HLSQ_STATE
+ * pairs   : 4 (Regs:28)
+ */
+static const u32 gen7_9_0_non_context_sp_pipe_br_hlsq_state_registers[] = {
+	 0x0ae52, 0x0ae52, 0x0ae60, 0x0ae67, 0x0ae69, 0x0ae75, 0x0aec0, 0x0aec5,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_sp_pipe_br_hlsq_state_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_NONE
+ * Location: A7XX_SP_TOP
+ * pairs   : 10 (Regs:61)
+ */
+static const u32 gen7_9_0_non_context_sp_pipe_br_sp_top_registers[] = {
+	 0x0ae00, 0x0ae00, 0x0ae02, 0x0ae04, 0x0ae06, 0x0ae0a, 0x0ae0c, 0x0ae0c,
+	 0x0ae0f, 0x0ae0f, 0x0ae28, 0x0ae2b, 0x0ae35, 0x0ae35, 0x0ae3a, 0x0ae3f,
+	 0x0ae50, 0x0ae52, 0x0ae80, 0x0aea3,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_sp_pipe_br_sp_top_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_NONE
+ * Location: A7XX_USPTP
+ * pairs   : 12 (Regs:62)
+ */
+static const u32 gen7_9_0_non_context_sp_pipe_br_usptp_registers[] = {
+	 0x0ae00, 0x0ae00, 0x0ae02, 0x0ae04, 0x0ae06, 0x0ae0a, 0x0ae0c, 0x0ae0c,
+	 0x0ae0f, 0x0ae0f, 0x0ae28, 0x0ae2b, 0x0ae30, 0x0ae32, 0x0ae35, 0x0ae35,
+	 0x0ae3a, 0x0ae3b, 0x0ae3e, 0x0ae3f, 0x0ae50, 0x0ae52, 0x0ae80, 0x0aea3,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_sp_pipe_br_usptp_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_NONE
+ * Location: A7XX_HLSQ_DP_STR
+ * pairs   : 2 (Regs:5)
+ */
+static const u32 gen7_9_0_non_context_sp_pipe_br_hlsq_dp_str_registers[] = {
+	 0x0ae6b, 0x0ae6c, 0x0ae73, 0x0ae75,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_sp_pipe_br_hlsq_dp_str_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_LPAC
+ * Cluster : A7XX_CLUSTER_NONE
+ * Location: A7XX_HLSQ_STATE
+ * pairs   : 1 (Regs:5)
+ */
+static const u32 gen7_9_0_non_context_sp_pipe_lpac_hlsq_state_registers[] = {
+	 0x0af88, 0x0af8c,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_sp_pipe_lpac_hlsq_state_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_LPAC
+ * Cluster : A7XX_CLUSTER_NONE
+ * Location: A7XX_SP_TOP
+ * pairs   : 1 (Regs:6)
+ */
+static const u32 gen7_9_0_non_context_sp_pipe_lpac_sp_top_registers[] = {
+	 0x0af80, 0x0af85,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_sp_pipe_lpac_sp_top_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_LPAC
+ * Cluster : A7XX_CLUSTER_NONE
+ * Location: A7XX_USPTP
+ * pairs   : 2 (Regs:9)
+ */
+static const u32 gen7_9_0_non_context_sp_pipe_lpac_usptp_registers[] = {
+	 0x0af80, 0x0af85, 0x0af90, 0x0af92,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_sp_pipe_lpac_usptp_registers), 8));
+
+/*
+ * Block   : ['TPL1']
+ * Pipeline: A7XX_PIPE_NONE
+ * Cluster : A7XX_CLUSTER_NONE
+ * Location: A7XX_USPTP
+ * pairs   : 5 (Regs:29)
+ */
+static const u32 gen7_9_0_non_context_tpl1_pipe_none_usptp_registers[] = {
+	 0x0b602, 0x0b602, 0x0b604, 0x0b604, 0x0b608, 0x0b60c, 0x0b610, 0x0b621,
+	 0x0b630, 0x0b633,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_tpl1_pipe_none_usptp_registers), 8));
+
+/*
+ * Block   : ['TPL1']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_NONE
+ * Location: A7XX_USPTP
+ * pairs   : 1 (Regs:1)
+ */
+static const u32 gen7_9_0_non_context_tpl1_pipe_br_usptp_registers[] = {
+	 0x0b600, 0x0b600,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_tpl1_pipe_br_usptp_registers), 8));
+
+/*
+ * Block   : ['TPL1']
+ * Pipeline: A7XX_PIPE_LPAC
+ * Cluster : A7XX_CLUSTER_NONE
+ * Location: A7XX_USPTP
+ * pairs   : 1 (Regs:1)
+ */
+static const u32 gen7_9_0_non_context_tpl1_pipe_lpac_usptp_registers[] = {
+	 0x0b780, 0x0b780,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_non_context_tpl1_pipe_lpac_usptp_registers), 8));
+
+/*
+ * Block   : ['GRAS']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_GRAS
+ * pairs   : 14 (Regs:293)
+ */
+static const u32 gen7_9_0_gras_pipe_br_cluster_gras_registers[] = {
+	 0x08000, 0x0800c, 0x08010, 0x08092, 0x08094, 0x08099, 0x0809b, 0x0809d,
+	 0x080a0, 0x080a7, 0x080af, 0x080f1, 0x080f4, 0x080f6, 0x080f8, 0x080fa,
+	 0x08100, 0x08107, 0x08109, 0x0810b, 0x08110, 0x08116, 0x08120, 0x0813f,
+	 0x08400, 0x08406, 0x0840a, 0x0840b,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_gras_pipe_br_cluster_gras_registers), 8));
+
+/*
+ * Block   : ['GRAS']
+ * Pipeline: A7XX_PIPE_BV
+ * Cluster : A7XX_CLUSTER_GRAS
+ * pairs   : 14 (Regs:293)
+ */
+static const u32 gen7_9_0_gras_pipe_bv_cluster_gras_registers[] = {
+	 0x08000, 0x0800c, 0x08010, 0x08092, 0x08094, 0x08099, 0x0809b, 0x0809d,
+	 0x080a0, 0x080a7, 0x080af, 0x080f1, 0x080f4, 0x080f6, 0x080f8, 0x080fa,
+	 0x08100, 0x08107, 0x08109, 0x0810b, 0x08110, 0x08116, 0x08120, 0x0813f,
+	 0x08400, 0x08406, 0x0840a, 0x0840b,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_gras_pipe_bv_cluster_gras_registers), 8));
+
+/*
+ * Block   : ['PC']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_FE
+ * pairs   : 6 (Regs:31)
+ */
+static const u32 gen7_9_0_pc_pipe_br_cluster_fe_registers[] = {
+	 0x09800, 0x09804, 0x09806, 0x0980a, 0x09810, 0x09811, 0x09884, 0x09886,
+	 0x09970, 0x09972, 0x09b00, 0x09b0c,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_pc_pipe_br_cluster_fe_registers), 8));
+
+/*
+ * Block   : ['PC']
+ * Pipeline: A7XX_PIPE_BV
+ * Cluster : A7XX_CLUSTER_FE
+ * pairs   : 6 (Regs:31)
+ */
+static const u32 gen7_9_0_pc_pipe_bv_cluster_fe_registers[] = {
+	 0x09800, 0x09804, 0x09806, 0x0980a, 0x09810, 0x09811, 0x09884, 0x09886,
+	 0x09970, 0x09972, 0x09b00, 0x09b0c,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_pc_pipe_bv_cluster_fe_registers), 8));
+
+/*
+ * Block   : ['VFD']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_FE
+ * pairs   : 2 (Regs:236)
+ */
+static const u32 gen7_9_0_vfd_pipe_br_cluster_fe_registers[] = {
+	 0x0a000, 0x0a009, 0x0a00e, 0x0a0ef,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_vfd_pipe_br_cluster_fe_registers), 8));
+
+/*
+ * Block   : ['VFD']
+ * Pipeline: A7XX_PIPE_BV
+ * Cluster : A7XX_CLUSTER_FE
+ * pairs   : 2 (Regs:236)
+ */
+static const u32 gen7_9_0_vfd_pipe_bv_cluster_fe_registers[] = {
+	 0x0a000, 0x0a009, 0x0a00e, 0x0a0ef,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_vfd_pipe_bv_cluster_fe_registers), 8));
+
+/*
+ * Block   : ['VPC']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_FE
+ * pairs   : 2 (Regs:18)
+ */
+static const u32 gen7_9_0_vpc_pipe_br_cluster_fe_registers[] = {
+	 0x09300, 0x0930a, 0x09311, 0x09317,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_vpc_pipe_br_cluster_fe_registers), 8));
+
+/*
+ * Block   : ['VPC']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_PC_VS
+ * pairs   : 3 (Regs:30)
+ */
+static const u32 gen7_9_0_vpc_pipe_br_cluster_pc_vs_registers[] = {
+	 0x09101, 0x0910c, 0x09300, 0x0930a, 0x09311, 0x09317,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_vpc_pipe_br_cluster_pc_vs_registers), 8));
+
+/*
+ * Block   : ['VPC']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_VPC_PS
+ * pairs   : 5 (Regs:76)
+ */
+static const u32 gen7_9_0_vpc_pipe_br_cluster_vpc_ps_registers[] = {
+	 0x09200, 0x0920f, 0x09212, 0x09216, 0x09218, 0x0923c, 0x09300, 0x0930a,
+	 0x09311, 0x09317,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_vpc_pipe_br_cluster_vpc_ps_registers), 8));
+
+/*
+ * Block   : ['VPC']
+ * Pipeline: A7XX_PIPE_BV
+ * Cluster : A7XX_CLUSTER_FE
+ * pairs   : 2 (Regs:18)
+ */
+static const u32 gen7_9_0_vpc_pipe_bv_cluster_fe_registers[] = {
+	 0x09300, 0x0930a, 0x09311, 0x09317,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_vpc_pipe_bv_cluster_fe_registers), 8));
+
+/*
+ * Block   : ['VPC']
+ * Pipeline: A7XX_PIPE_BV
+ * Cluster : A7XX_CLUSTER_PC_VS
+ * pairs   : 3 (Regs:30)
+ */
+static const u32 gen7_9_0_vpc_pipe_bv_cluster_pc_vs_registers[] = {
+	 0x09101, 0x0910c, 0x09300, 0x0930a, 0x09311, 0x09317,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_vpc_pipe_bv_cluster_pc_vs_registers), 8));
+
+/*
+ * Block   : ['VPC']
+ * Pipeline: A7XX_PIPE_BV
+ * Cluster : A7XX_CLUSTER_VPC_PS
+ * pairs   : 5 (Regs:76)
+ */
+static const u32 gen7_9_0_vpc_pipe_bv_cluster_vpc_ps_registers[] = {
+	 0x09200, 0x0920f, 0x09212, 0x09216, 0x09218, 0x0923c, 0x09300, 0x0930a,
+	 0x09311, 0x09317,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_vpc_pipe_bv_cluster_vpc_ps_registers), 8));
+
+/*
+ * Block   : ['RB']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_PS
+ * pairs   : 39 (Regs:133)
+ */
+static const u32 gen7_9_0_rb_pipe_br_cluster_ps_rac_registers[] = {
+	 0x08802, 0x08802, 0x08804, 0x08806, 0x08809, 0x0880a, 0x0880e, 0x08811,
+	 0x08818, 0x0881e, 0x08821, 0x08821, 0x08823, 0x08826, 0x08829, 0x08829,
+	 0x0882b, 0x0882e, 0x08831, 0x08831, 0x08833, 0x08836, 0x08839, 0x08839,
+	 0x0883b, 0x0883e, 0x08841, 0x08841, 0x08843, 0x08846, 0x08849, 0x08849,
+	 0x0884b, 0x0884e, 0x08851, 0x08851, 0x08853, 0x08856, 0x08859, 0x08859,
+	 0x0885b, 0x0885e, 0x08860, 0x08864, 0x08870, 0x08870, 0x08873, 0x08876,
+	 0x08878, 0x08879, 0x08882, 0x08885, 0x08887, 0x08889, 0x08891, 0x08891,
+	 0x08898, 0x08899, 0x088c0, 0x088c1, 0x088e5, 0x088e5, 0x088f4, 0x088f5,
+	 0x08a00, 0x08a05, 0x08a10, 0x08a15, 0x08a20, 0x08a25, 0x08a30, 0x08a35,
+	 0x08c00, 0x08c01, 0x08c18, 0x08c1f, 0x08c26, 0x08c34,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_rb_pipe_br_cluster_ps_rac_registers), 8));
+
+/*
+ * Block   : ['RB']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_PS
+ * pairs   : 34 (Regs:100)
+ */
+static const u32 gen7_9_0_rb_pipe_br_cluster_ps_rbp_registers[] = {
+	 0x08800, 0x08801, 0x08803, 0x08803, 0x0880b, 0x0880d, 0x08812, 0x08812,
+	 0x08820, 0x08820, 0x08822, 0x08822, 0x08827, 0x08828, 0x0882a, 0x0882a,
+	 0x0882f, 0x08830, 0x08832, 0x08832, 0x08837, 0x08838, 0x0883a, 0x0883a,
+	 0x0883f, 0x08840, 0x08842, 0x08842, 0x08847, 0x08848, 0x0884a, 0x0884a,
+	 0x0884f, 0x08850, 0x08852, 0x08852, 0x08857, 0x08858, 0x0885a, 0x0885a,
+	 0x0885f, 0x0885f, 0x08865, 0x08865, 0x08871, 0x08872, 0x08877, 0x08877,
+	 0x08880, 0x08881, 0x08886, 0x08886, 0x08890, 0x08890, 0x088d0, 0x088e4,
+	 0x088e8, 0x088ea, 0x088f0, 0x088f0, 0x08900, 0x0891a, 0x08927, 0x08928,
+	 0x08c17, 0x08c17, 0x08c20, 0x08c25,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_rb_pipe_br_cluster_ps_rbp_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_SP_VS
+ * Location: A7XX_HLSQ_STATE
+ * pairs   : 29 (Regs:215)
+ */
+static const u32 gen7_9_0_sp_pipe_br_cluster_sp_vs_hlsq_state_registers[] = {
+	 0x0a800, 0x0a801, 0x0a81b, 0x0a81d, 0x0a822, 0x0a822, 0x0a824, 0x0a824,
+	 0x0a827, 0x0a82a, 0x0a830, 0x0a830, 0x0a832, 0x0a835, 0x0a83a, 0x0a83a,
+	 0x0a83c, 0x0a83c, 0x0a83f, 0x0a841, 0x0a85b, 0x0a85d, 0x0a862, 0x0a862,
+	 0x0a864, 0x0a864, 0x0a867, 0x0a867, 0x0a870, 0x0a870, 0x0a872, 0x0a872,
+	 0x0a88c, 0x0a88e, 0x0a893, 0x0a893, 0x0a895, 0x0a895, 0x0a898, 0x0a898,
+	 0x0a89a, 0x0a89d, 0x0a8a0, 0x0a8af, 0x0a8c0, 0x0a8c3, 0x0a974, 0x0a977,
+	 0x0ab00, 0x0ab03, 0x0ab05, 0x0ab05, 0x0ab0a, 0x0ab1b, 0x0ab20, 0x0ab20,
+	 0x0ab40, 0x0abbf,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_br_cluster_sp_vs_hlsq_state_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_SP_VS
+ * Location: A7XX_SP_TOP
+ * pairs   : 22 (Regs:73)
+ */
+static const u32 gen7_9_0_sp_pipe_br_cluster_sp_vs_sp_top_registers[] = {
+	 0x0a800, 0x0a800, 0x0a81c, 0x0a81d, 0x0a822, 0x0a824, 0x0a82d, 0x0a82d,
+	 0x0a82f, 0x0a831, 0x0a834, 0x0a835, 0x0a83a, 0x0a83c, 0x0a840, 0x0a840,
+	 0x0a85c, 0x0a85d, 0x0a862, 0x0a864, 0x0a868, 0x0a868, 0x0a870, 0x0a871,
+	 0x0a88d, 0x0a88e, 0x0a893, 0x0a895, 0x0a899, 0x0a899, 0x0a8a0, 0x0a8af,
+	 0x0a974, 0x0a977, 0x0ab00, 0x0ab00, 0x0ab02, 0x0ab02, 0x0ab04, 0x0ab05,
+	 0x0ab0a, 0x0ab1b, 0x0ab20, 0x0ab20,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_br_cluster_sp_vs_sp_top_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_SP_VS
+ * Location: A7XX_USPTP
+ * pairs   : 16 (Regs:269)
+ */
+static const u32 gen7_9_0_sp_pipe_br_cluster_sp_vs_usptp_registers[] = {
+	 0x0a800, 0x0a81b, 0x0a81e, 0x0a821, 0x0a823, 0x0a827, 0x0a82d, 0x0a82d,
+	 0x0a82f, 0x0a833, 0x0a836, 0x0a839, 0x0a83b, 0x0a85b, 0x0a85e, 0x0a861,
+	 0x0a863, 0x0a868, 0x0a870, 0x0a88c, 0x0a88f, 0x0a892, 0x0a894, 0x0a899,
+	 0x0a8c0, 0x0a8c3, 0x0ab00, 0x0ab05, 0x0ab21, 0x0ab22, 0x0ab40, 0x0abbf,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_br_cluster_sp_vs_usptp_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_SP_PS
+ * Location: A7XX_HLSQ_STATE
+ * pairs   : 21 (Regs:334)
+ */
+static const u32 gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_state_registers[] = {
+	 0x0a980, 0x0a984, 0x0a99e, 0x0a99e, 0x0a9a7, 0x0a9a7, 0x0a9aa, 0x0a9aa,
+	 0x0a9ae, 0x0a9b0, 0x0a9b2, 0x0a9b5, 0x0a9ba, 0x0a9ba, 0x0a9bc, 0x0a9bc,
+	 0x0a9c4, 0x0a9c4, 0x0a9c6, 0x0a9c6, 0x0a9cd, 0x0a9cd, 0x0a9e0, 0x0a9fc,
+	 0x0aa00, 0x0aa00, 0x0aa30, 0x0aa31, 0x0aa40, 0x0aabf, 0x0aaf2, 0x0aaf3,
+	 0x0ab00, 0x0ab03, 0x0ab05, 0x0ab05, 0x0ab0a, 0x0ab1b, 0x0ab20, 0x0ab20,
+	 0x0ab40, 0x0abbf,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_state_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_SP_PS
+ * Location: A7XX_HLSQ_DP
+ * pairs   : 3 (Regs:19)
+ */
+static const u32 gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_dp_registers[] = {
+	 0x0a9b1, 0x0a9b1, 0x0a9c6, 0x0a9cb, 0x0a9d4, 0x0a9df,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_dp_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_SP_PS
+ * Location: A7XX_SP_TOP
+ * pairs   : 18 (Regs:77)
+ */
+static const u32 gen7_9_0_sp_pipe_br_cluster_sp_ps_sp_top_registers[] = {
+	 0x0a980, 0x0a980, 0x0a982, 0x0a984, 0x0a99e, 0x0a9a2, 0x0a9a7, 0x0a9a8,
+	 0x0a9aa, 0x0a9aa, 0x0a9ae, 0x0a9ae, 0x0a9b0, 0x0a9b1, 0x0a9b3, 0x0a9b5,
+	 0x0a9ba, 0x0a9bc, 0x0a9c5, 0x0a9c5, 0x0a9e0, 0x0a9f9, 0x0aa00, 0x0aa03,
+	 0x0aaf2, 0x0aaf3, 0x0ab00, 0x0ab00, 0x0ab02, 0x0ab02, 0x0ab04, 0x0ab05,
+	 0x0ab0a, 0x0ab1b, 0x0ab20, 0x0ab20,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_br_cluster_sp_ps_sp_top_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_SP_PS
+ * Location: A7XX_USPTP
+ * pairs   : 17 (Regs:333)
+ */
+static const u32 gen7_9_0_sp_pipe_br_cluster_sp_ps_usptp_registers[] = {
+	 0x0a980, 0x0a982, 0x0a985, 0x0a9a6, 0x0a9a8, 0x0a9a9, 0x0a9ab, 0x0a9ae,
+	 0x0a9b0, 0x0a9b3, 0x0a9b6, 0x0a9b9, 0x0a9bb, 0x0a9bf, 0x0a9c2, 0x0a9c3,
+	 0x0a9c5, 0x0a9c5, 0x0a9cd, 0x0a9cd, 0x0a9d0, 0x0a9d3, 0x0aa01, 0x0aa03,
+	 0x0aa30, 0x0aa31, 0x0aa40, 0x0aabf, 0x0ab00, 0x0ab05, 0x0ab21, 0x0ab22,
+	 0x0ab40, 0x0abbf,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_br_cluster_sp_ps_usptp_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_SP_PS
+ * Location: A7XX_HLSQ_DP_STR
+ * pairs   : 1 (Regs:6)
+ */
+static const u32 gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_dp_str_registers[] = {
+	 0x0a9c6, 0x0a9cb,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_dp_str_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BV
+ * Cluster : A7XX_CLUSTER_SP_VS
+ * Location: A7XX_HLSQ_STATE
+ * pairs   : 28 (Regs:213)
+ */
+static const u32 gen7_9_0_sp_pipe_bv_cluster_sp_vs_hlsq_state_registers[] = {
+	 0x0a800, 0x0a801, 0x0a81b, 0x0a81d, 0x0a822, 0x0a822, 0x0a824, 0x0a824,
+	 0x0a827, 0x0a82a, 0x0a830, 0x0a830, 0x0a832, 0x0a835, 0x0a83a, 0x0a83a,
+	 0x0a83c, 0x0a83c, 0x0a83f, 0x0a841, 0x0a85b, 0x0a85d, 0x0a862, 0x0a862,
+	 0x0a864, 0x0a864, 0x0a867, 0x0a867, 0x0a870, 0x0a870, 0x0a872, 0x0a872,
+	 0x0a88c, 0x0a88e, 0x0a893, 0x0a893, 0x0a895, 0x0a895, 0x0a898, 0x0a898,
+	 0x0a89a, 0x0a89d, 0x0a8a0, 0x0a8af, 0x0a8c0, 0x0a8c3, 0x0a974, 0x0a977,
+	 0x0ab00, 0x0ab02, 0x0ab0a, 0x0ab1b, 0x0ab20, 0x0ab20, 0x0ab40, 0x0abbf,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_bv_cluster_sp_vs_hlsq_state_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BV
+ * Cluster : A7XX_CLUSTER_SP_VS
+ * Location: A7XX_SP_TOP
+ * pairs   : 21 (Regs:71)
+ */
+static const u32 gen7_9_0_sp_pipe_bv_cluster_sp_vs_sp_top_registers[] = {
+	 0x0a800, 0x0a800, 0x0a81c, 0x0a81d, 0x0a822, 0x0a824, 0x0a82d, 0x0a82d,
+	 0x0a82f, 0x0a831, 0x0a834, 0x0a835, 0x0a83a, 0x0a83c, 0x0a840, 0x0a840,
+	 0x0a85c, 0x0a85d, 0x0a862, 0x0a864, 0x0a868, 0x0a868, 0x0a870, 0x0a871,
+	 0x0a88d, 0x0a88e, 0x0a893, 0x0a895, 0x0a899, 0x0a899, 0x0a8a0, 0x0a8af,
+	 0x0a974, 0x0a977, 0x0ab00, 0x0ab00, 0x0ab02, 0x0ab02, 0x0ab0a, 0x0ab1b,
+	 0x0ab20, 0x0ab20,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_bv_cluster_sp_vs_sp_top_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_BV
+ * Cluster : A7XX_CLUSTER_SP_VS
+ * Location: A7XX_USPTP
+ * pairs   : 16 (Regs:266)
+ */
+static const u32 gen7_9_0_sp_pipe_bv_cluster_sp_vs_usptp_registers[] = {
+	 0x0a800, 0x0a81b, 0x0a81e, 0x0a821, 0x0a823, 0x0a827, 0x0a82d, 0x0a82d,
+	 0x0a82f, 0x0a833, 0x0a836, 0x0a839, 0x0a83b, 0x0a85b, 0x0a85e, 0x0a861,
+	 0x0a863, 0x0a868, 0x0a870, 0x0a88c, 0x0a88f, 0x0a892, 0x0a894, 0x0a899,
+	 0x0a8c0, 0x0a8c3, 0x0ab00, 0x0ab02, 0x0ab21, 0x0ab22, 0x0ab40, 0x0abbf,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_bv_cluster_sp_vs_usptp_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_LPAC
+ * Cluster : A7XX_CLUSTER_SP_PS
+ * Location: A7XX_HLSQ_STATE
+ * pairs   : 14 (Regs:299)
+ */
+static const u32 gen7_9_0_sp_pipe_lpac_cluster_sp_ps_hlsq_state_registers[] = {
+	 0x0a9b0, 0x0a9b0, 0x0a9b2, 0x0a9b5, 0x0a9ba, 0x0a9ba, 0x0a9bc, 0x0a9bc,
+	 0x0a9c4, 0x0a9c4, 0x0a9cd, 0x0a9cd, 0x0a9e2, 0x0a9e3, 0x0a9e6, 0x0a9fc,
+	 0x0aa00, 0x0aa00, 0x0aa31, 0x0aa35, 0x0aa40, 0x0aabf, 0x0aaf3, 0x0aaf3,
+	 0x0ab00, 0x0ab01, 0x0ab40, 0x0abbf,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_lpac_cluster_sp_ps_hlsq_state_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_LPAC
+ * Cluster : A7XX_CLUSTER_SP_PS
+ * Location: A7XX_HLSQ_DP
+ * pairs   : 2 (Regs:13)
+ */
+static const u32 gen7_9_0_sp_pipe_lpac_cluster_sp_ps_hlsq_dp_registers[] = {
+	 0x0a9b1, 0x0a9b1, 0x0a9d4, 0x0a9df,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_lpac_cluster_sp_ps_hlsq_dp_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_LPAC
+ * Cluster : A7XX_CLUSTER_SP_PS
+ * Location: A7XX_SP_TOP
+ * pairs   : 9 (Regs:34)
+ */
+static const u32 gen7_9_0_sp_pipe_lpac_cluster_sp_ps_sp_top_registers[] = {
+	 0x0a9b0, 0x0a9b1, 0x0a9b3, 0x0a9b5, 0x0a9ba, 0x0a9bc, 0x0a9c5, 0x0a9c5,
+	 0x0a9e2, 0x0a9e3, 0x0a9e6, 0x0a9f9, 0x0aa00, 0x0aa00, 0x0aaf3, 0x0aaf3,
+	 0x0ab00, 0x0ab00,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_lpac_cluster_sp_ps_sp_top_registers), 8));
+
+/*
+ * Block   : ['SP']
+ * Pipeline: A7XX_PIPE_LPAC
+ * Cluster : A7XX_CLUSTER_SP_PS
+ * Location: A7XX_USPTP
+ * pairs   : 11 (Regs:279)
+ */
+static const u32 gen7_9_0_sp_pipe_lpac_cluster_sp_ps_usptp_registers[] = {
+	 0x0a9b0, 0x0a9b3, 0x0a9b6, 0x0a9b9, 0x0a9bb, 0x0a9be, 0x0a9c2, 0x0a9c3,
+	 0x0a9c5, 0x0a9c5, 0x0a9cd, 0x0a9cd, 0x0a9d0, 0x0a9d3, 0x0aa31, 0x0aa31,
+	 0x0aa40, 0x0aabf, 0x0ab00, 0x0ab01, 0x0ab40, 0x0abbf,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_sp_pipe_lpac_cluster_sp_ps_usptp_registers), 8));
+
+/*
+ * Block   : ['TPL1']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_SP_VS
+ * Location: A7XX_USPTP
+ * pairs   : 3 (Regs:10)
+ */
+static const u32 gen7_9_0_tpl1_pipe_br_cluster_sp_vs_usptp_registers[] = {
+	 0x0b300, 0x0b307, 0x0b309, 0x0b309, 0x0b310, 0x0b310,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_tpl1_pipe_br_cluster_sp_vs_usptp_registers), 8));
+
+/*
+ * Block   : ['TPL1']
+ * Pipeline: A7XX_PIPE_BR
+ * Cluster : A7XX_CLUSTER_SP_PS
+ * Location: A7XX_USPTP
+ * pairs   : 6 (Regs:42)
+ */
+static const u32 gen7_9_0_tpl1_pipe_br_cluster_sp_ps_usptp_registers[] = {
+	 0x0b180, 0x0b183, 0x0b190, 0x0b195, 0x0b2c0, 0x0b2d5, 0x0b300, 0x0b307,
+	 0x0b309, 0x0b309, 0x0b310, 0x0b310,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_tpl1_pipe_br_cluster_sp_ps_usptp_registers), 8));
+
+/*
+ * Block   : ['TPL1']
+ * Pipeline: A7XX_PIPE_BV
+ * Cluster : A7XX_CLUSTER_SP_VS
+ * Location: A7XX_USPTP
+ * pairs   : 3 (Regs:10)
+ */
+static const u32 gen7_9_0_tpl1_pipe_bv_cluster_sp_vs_usptp_registers[] = {
+	 0x0b300, 0x0b307, 0x0b309, 0x0b309, 0x0b310, 0x0b310,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_tpl1_pipe_bv_cluster_sp_vs_usptp_registers), 8));
+
+/*
+ * Block   : ['TPL1']
+ * Pipeline: A7XX_PIPE_LPAC
+ * Cluster : A7XX_CLUSTER_SP_PS
+ * Location: A7XX_USPTP
+ * pairs   : 5 (Regs:7)
+ */
+static const u32 gen7_9_0_tpl1_pipe_lpac_cluster_sp_ps_usptp_registers[] = {
+	 0x0b180, 0x0b181, 0x0b300, 0x0b301, 0x0b307, 0x0b307, 0x0b309, 0x0b309,
+	 0x0b310, 0x0b310,
+	 UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_tpl1_pipe_lpac_cluster_sp_ps_usptp_registers), 8));
+
+static const struct gen7_sel_reg gen7_9_0_rb_rac_sel = {
+	.host_reg = REG_A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_HOST,
+	.cd_reg = REG_A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_CD,
+	.val = 0,
+};
+
+static const struct gen7_sel_reg gen7_9_0_rb_rbp_sel = {
+	.host_reg = REG_A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_HOST,
+	.cd_reg = REG_A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_CD,
+	.val = 0x9,
+};
+
+static struct gen7_cluster_registers gen7_9_0_clusters[] = {
+	{ A7XX_CLUSTER_NONE, A7XX_PIPE_BR, STATE_NON_CONTEXT,
+		gen7_9_0_non_context_pipe_br_registers,  },
+	{ A7XX_CLUSTER_NONE, A7XX_PIPE_BV, STATE_NON_CONTEXT,
+		gen7_9_0_non_context_pipe_bv_registers,  },
+	{ A7XX_CLUSTER_NONE, A7XX_PIPE_LPAC, STATE_NON_CONTEXT,
+		gen7_9_0_non_context_pipe_lpac_registers,  },
+	{ A7XX_CLUSTER_NONE, A7XX_PIPE_BR, STATE_NON_CONTEXT,
+		gen7_9_0_non_context_rb_pipe_br_rac_registers, &gen7_9_0_rb_rac_sel, },
+	{ A7XX_CLUSTER_NONE, A7XX_PIPE_BR, STATE_NON_CONTEXT,
+		gen7_9_0_non_context_rb_pipe_br_rbp_registers, &gen7_9_0_rb_rbp_sel, },
+	{ A7XX_CLUSTER_PS, A7XX_PIPE_BR, STATE_FORCE_CTXT_0,
+		gen7_9_0_rb_pipe_br_cluster_ps_rac_registers, &gen7_9_0_rb_rac_sel, },
+	{ A7XX_CLUSTER_PS, A7XX_PIPE_BR, STATE_FORCE_CTXT_1,
+		gen7_9_0_rb_pipe_br_cluster_ps_rac_registers, &gen7_9_0_rb_rac_sel, },
+	{ A7XX_CLUSTER_PS, A7XX_PIPE_BR, STATE_FORCE_CTXT_0,
+		gen7_9_0_rb_pipe_br_cluster_ps_rbp_registers, &gen7_9_0_rb_rbp_sel, },
+	{ A7XX_CLUSTER_PS, A7XX_PIPE_BR, STATE_FORCE_CTXT_1,
+		gen7_9_0_rb_pipe_br_cluster_ps_rbp_registers, &gen7_9_0_rb_rbp_sel, },
+	{ A7XX_CLUSTER_GRAS, A7XX_PIPE_BR, STATE_FORCE_CTXT_0,
+		gen7_9_0_gras_pipe_br_cluster_gras_registers,  },
+	{ A7XX_CLUSTER_GRAS, A7XX_PIPE_BR, STATE_FORCE_CTXT_1,
+		gen7_9_0_gras_pipe_br_cluster_gras_registers,  },
+	{ A7XX_CLUSTER_GRAS, A7XX_PIPE_BV, STATE_FORCE_CTXT_0,
+		gen7_9_0_gras_pipe_bv_cluster_gras_registers,  },
+	{ A7XX_CLUSTER_GRAS, A7XX_PIPE_BV, STATE_FORCE_CTXT_1,
+		gen7_9_0_gras_pipe_bv_cluster_gras_registers,  },
+	{ A7XX_CLUSTER_FE, A7XX_PIPE_BR, STATE_FORCE_CTXT_0,
+		gen7_9_0_pc_pipe_br_cluster_fe_registers,  },
+	{ A7XX_CLUSTER_FE, A7XX_PIPE_BR, STATE_FORCE_CTXT_1,
+		gen7_9_0_pc_pipe_br_cluster_fe_registers,  },
+	{ A7XX_CLUSTER_FE, A7XX_PIPE_BV, STATE_FORCE_CTXT_0,
+		gen7_9_0_pc_pipe_bv_cluster_fe_registers,  },
+	{ A7XX_CLUSTER_FE, A7XX_PIPE_BV, STATE_FORCE_CTXT_1,
+		gen7_9_0_pc_pipe_bv_cluster_fe_registers,  },
+	{ A7XX_CLUSTER_FE, A7XX_PIPE_BR, STATE_FORCE_CTXT_0,
+		gen7_9_0_vfd_pipe_br_cluster_fe_registers,  },
+	{ A7XX_CLUSTER_FE, A7XX_PIPE_BR, STATE_FORCE_CTXT_1,
+		gen7_9_0_vfd_pipe_br_cluster_fe_registers,  },
+	{ A7XX_CLUSTER_FE, A7XX_PIPE_BV, STATE_FORCE_CTXT_0,
+		gen7_9_0_vfd_pipe_bv_cluster_fe_registers,  },
+	{ A7XX_CLUSTER_FE, A7XX_PIPE_BV, STATE_FORCE_CTXT_1,
+		gen7_9_0_vfd_pipe_bv_cluster_fe_registers,  },
+	{ A7XX_CLUSTER_FE, A7XX_PIPE_BR, STATE_FORCE_CTXT_0,
+		gen7_9_0_vpc_pipe_br_cluster_fe_registers,  },
+	{ A7XX_CLUSTER_FE, A7XX_PIPE_BR, STATE_FORCE_CTXT_1,
+		gen7_9_0_vpc_pipe_br_cluster_fe_registers,  },
+	{ A7XX_CLUSTER_PC_VS, A7XX_PIPE_BR, STATE_FORCE_CTXT_0,
+		gen7_9_0_vpc_pipe_br_cluster_pc_vs_registers,  },
+	{ A7XX_CLUSTER_PC_VS, A7XX_PIPE_BR, STATE_FORCE_CTXT_1,
+		gen7_9_0_vpc_pipe_br_cluster_pc_vs_registers,  },
+	{ A7XX_CLUSTER_VPC_PS, A7XX_PIPE_BR, STATE_FORCE_CTXT_0,
+		gen7_9_0_vpc_pipe_br_cluster_vpc_ps_registers,  },
+	{ A7XX_CLUSTER_VPC_PS, A7XX_PIPE_BR, STATE_FORCE_CTXT_1,
+		gen7_9_0_vpc_pipe_br_cluster_vpc_ps_registers,  },
+	{ A7XX_CLUSTER_FE, A7XX_PIPE_BV, STATE_FORCE_CTXT_0,
+		gen7_9_0_vpc_pipe_bv_cluster_fe_registers,  },
+	{ A7XX_CLUSTER_FE, A7XX_PIPE_BV, STATE_FORCE_CTXT_1,
+		gen7_9_0_vpc_pipe_bv_cluster_fe_registers,  },
+	{ A7XX_CLUSTER_PC_VS, A7XX_PIPE_BV, STATE_FORCE_CTXT_0,
+		gen7_9_0_vpc_pipe_bv_cluster_pc_vs_registers,  },
+	{ A7XX_CLUSTER_PC_VS, A7XX_PIPE_BV, STATE_FORCE_CTXT_1,
+		gen7_9_0_vpc_pipe_bv_cluster_pc_vs_registers,  },
+	{ A7XX_CLUSTER_VPC_PS, A7XX_PIPE_BV, STATE_FORCE_CTXT_0,
+		gen7_9_0_vpc_pipe_bv_cluster_vpc_ps_registers,  },
+	{ A7XX_CLUSTER_VPC_PS, A7XX_PIPE_BV, STATE_FORCE_CTXT_1,
+		gen7_9_0_vpc_pipe_bv_cluster_vpc_ps_registers,  },
+};
+
+static struct gen7_sptp_cluster_registers gen7_9_0_sptp_clusters[] = {
+	{ A7XX_CLUSTER_NONE, A7XX_SP_NCTX_REG, A7XX_PIPE_BR, 0, A7XX_HLSQ_STATE,
+		gen7_9_0_non_context_sp_pipe_br_hlsq_state_registers, 0xae00},
+	{ A7XX_CLUSTER_NONE, A7XX_SP_NCTX_REG, A7XX_PIPE_BR, 0, A7XX_SP_TOP,
+		gen7_9_0_non_context_sp_pipe_br_sp_top_registers, 0xae00},
+	{ A7XX_CLUSTER_NONE, A7XX_SP_NCTX_REG, A7XX_PIPE_BR, 0, A7XX_USPTP,
+		gen7_9_0_non_context_sp_pipe_br_usptp_registers, 0xae00},
+	{ A7XX_CLUSTER_NONE, A7XX_SP_NCTX_REG, A7XX_PIPE_BR, 0, A7XX_HLSQ_DP_STR,
+		gen7_9_0_non_context_sp_pipe_br_hlsq_dp_str_registers, 0xae00},
+	{ A7XX_CLUSTER_NONE, A7XX_SP_NCTX_REG, A7XX_PIPE_LPAC, 0, A7XX_HLSQ_STATE,
+		gen7_9_0_non_context_sp_pipe_lpac_hlsq_state_registers, 0xaf80},
+	{ A7XX_CLUSTER_NONE, A7XX_SP_NCTX_REG, A7XX_PIPE_LPAC, 0, A7XX_SP_TOP,
+		gen7_9_0_non_context_sp_pipe_lpac_sp_top_registers, 0xaf80},
+	{ A7XX_CLUSTER_NONE, A7XX_SP_NCTX_REG, A7XX_PIPE_LPAC, 0, A7XX_USPTP,
+		gen7_9_0_non_context_sp_pipe_lpac_usptp_registers, 0xaf80},
+	{ A7XX_CLUSTER_NONE, A7XX_TP0_NCTX_REG, A7XX_PIPE_NONE, 0, A7XX_USPTP,
+		gen7_9_0_non_context_tpl1_pipe_none_usptp_registers, 0xb600},
+	{ A7XX_CLUSTER_NONE, A7XX_TP0_NCTX_REG, A7XX_PIPE_BR, 0, A7XX_USPTP,
+		gen7_9_0_non_context_tpl1_pipe_br_usptp_registers, 0xb600},
+	{ A7XX_CLUSTER_NONE, A7XX_TP0_NCTX_REG, A7XX_PIPE_LPAC, 0, A7XX_USPTP,
+		gen7_9_0_non_context_tpl1_pipe_lpac_usptp_registers, 0xb780},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX0_3D_CVS_REG, A7XX_PIPE_BR, 0, A7XX_HLSQ_STATE,
+		gen7_9_0_sp_pipe_br_cluster_sp_vs_hlsq_state_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX0_3D_CVS_REG, A7XX_PIPE_BR, 0, A7XX_SP_TOP,
+		gen7_9_0_sp_pipe_br_cluster_sp_vs_sp_top_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX0_3D_CVS_REG, A7XX_PIPE_BR, 0, A7XX_USPTP,
+		gen7_9_0_sp_pipe_br_cluster_sp_vs_usptp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX0_3D_CVS_REG, A7XX_PIPE_BV, 0, A7XX_HLSQ_STATE,
+		gen7_9_0_sp_pipe_bv_cluster_sp_vs_hlsq_state_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX0_3D_CVS_REG, A7XX_PIPE_BV, 0, A7XX_SP_TOP,
+		gen7_9_0_sp_pipe_bv_cluster_sp_vs_sp_top_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX0_3D_CVS_REG, A7XX_PIPE_BV, 0, A7XX_USPTP,
+		gen7_9_0_sp_pipe_bv_cluster_sp_vs_usptp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX1_3D_CVS_REG, A7XX_PIPE_BR, 1, A7XX_HLSQ_STATE,
+		gen7_9_0_sp_pipe_br_cluster_sp_vs_hlsq_state_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX1_3D_CVS_REG, A7XX_PIPE_BR, 1, A7XX_SP_TOP,
+		gen7_9_0_sp_pipe_br_cluster_sp_vs_sp_top_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX1_3D_CVS_REG, A7XX_PIPE_BR, 1, A7XX_USPTP,
+		gen7_9_0_sp_pipe_br_cluster_sp_vs_usptp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX1_3D_CVS_REG, A7XX_PIPE_BV, 1, A7XX_HLSQ_STATE,
+		gen7_9_0_sp_pipe_bv_cluster_sp_vs_hlsq_state_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX1_3D_CVS_REG, A7XX_PIPE_BV, 1, A7XX_SP_TOP,
+		gen7_9_0_sp_pipe_bv_cluster_sp_vs_sp_top_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX1_3D_CVS_REG, A7XX_PIPE_BV, 1, A7XX_USPTP,
+		gen7_9_0_sp_pipe_bv_cluster_sp_vs_usptp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX0_3D_CPS_REG, A7XX_PIPE_BR, 0, A7XX_HLSQ_STATE,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_state_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX0_3D_CPS_REG, A7XX_PIPE_BR, 0, A7XX_HLSQ_DP,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_dp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX0_3D_CPS_REG, A7XX_PIPE_BR, 0, A7XX_SP_TOP,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_sp_top_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX0_3D_CPS_REG, A7XX_PIPE_BR, 0, A7XX_USPTP,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_usptp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX0_3D_CPS_REG, A7XX_PIPE_BR, 0, A7XX_HLSQ_DP_STR,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_dp_str_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX0_3D_CPS_REG, A7XX_PIPE_LPAC, 0, A7XX_HLSQ_STATE,
+		gen7_9_0_sp_pipe_lpac_cluster_sp_ps_hlsq_state_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX0_3D_CPS_REG, A7XX_PIPE_LPAC, 0, A7XX_HLSQ_DP,
+		gen7_9_0_sp_pipe_lpac_cluster_sp_ps_hlsq_dp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX0_3D_CPS_REG, A7XX_PIPE_LPAC, 0, A7XX_SP_TOP,
+		gen7_9_0_sp_pipe_lpac_cluster_sp_ps_sp_top_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX0_3D_CPS_REG, A7XX_PIPE_LPAC, 0, A7XX_USPTP,
+		gen7_9_0_sp_pipe_lpac_cluster_sp_ps_usptp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX1_3D_CPS_REG, A7XX_PIPE_BR, 1, A7XX_HLSQ_STATE,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_state_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX1_3D_CPS_REG, A7XX_PIPE_BR, 1, A7XX_HLSQ_DP,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_dp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX1_3D_CPS_REG, A7XX_PIPE_BR, 1, A7XX_SP_TOP,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_sp_top_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX1_3D_CPS_REG, A7XX_PIPE_BR, 1, A7XX_USPTP,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_usptp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX1_3D_CPS_REG, A7XX_PIPE_BR, 1, A7XX_HLSQ_DP_STR,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_dp_str_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX2_3D_CPS_REG, A7XX_PIPE_BR, 2, A7XX_HLSQ_DP,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_dp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX2_3D_CPS_REG, A7XX_PIPE_BR, 2, A7XX_SP_TOP,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_sp_top_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX2_3D_CPS_REG, A7XX_PIPE_BR, 2, A7XX_USPTP,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_usptp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX2_3D_CPS_REG, A7XX_PIPE_BR, 2, A7XX_HLSQ_DP_STR,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_dp_str_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX3_3D_CPS_REG, A7XX_PIPE_BR, 3, A7XX_HLSQ_DP,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_dp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX3_3D_CPS_REG, A7XX_PIPE_BR, 3, A7XX_SP_TOP,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_sp_top_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX3_3D_CPS_REG, A7XX_PIPE_BR, 3, A7XX_USPTP,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_usptp_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX3_3D_CPS_REG, A7XX_PIPE_BR, 3, A7XX_HLSQ_DP_STR,
+		gen7_9_0_sp_pipe_br_cluster_sp_ps_hlsq_dp_str_registers, 0xa800},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX0_3D_CVS_REG, A7XX_PIPE_BR, 0, A7XX_USPTP,
+		gen7_9_0_tpl1_pipe_br_cluster_sp_vs_usptp_registers, 0xb000},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX0_3D_CVS_REG, A7XX_PIPE_BV, 0, A7XX_USPTP,
+		gen7_9_0_tpl1_pipe_bv_cluster_sp_vs_usptp_registers, 0xb000},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX1_3D_CVS_REG, A7XX_PIPE_BR, 1, A7XX_USPTP,
+		gen7_9_0_tpl1_pipe_br_cluster_sp_vs_usptp_registers, 0xb000},
+	{ A7XX_CLUSTER_SP_VS, A7XX_SP_CTX1_3D_CVS_REG, A7XX_PIPE_BV, 1, A7XX_USPTP,
+		gen7_9_0_tpl1_pipe_bv_cluster_sp_vs_usptp_registers, 0xb000},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX0_3D_CPS_REG, A7XX_PIPE_BR, 0, A7XX_USPTP,
+		gen7_9_0_tpl1_pipe_br_cluster_sp_ps_usptp_registers, 0xb000},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX0_3D_CPS_REG, A7XX_PIPE_LPAC, 0, A7XX_USPTP,
+		gen7_9_0_tpl1_pipe_lpac_cluster_sp_ps_usptp_registers, 0xb000},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX1_3D_CPS_REG, A7XX_PIPE_BR, 1, A7XX_USPTP,
+		gen7_9_0_tpl1_pipe_br_cluster_sp_ps_usptp_registers, 0xb000},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX2_3D_CPS_REG, A7XX_PIPE_BR, 2, A7XX_USPTP,
+		gen7_9_0_tpl1_pipe_br_cluster_sp_ps_usptp_registers, 0xb000},
+	{ A7XX_CLUSTER_SP_PS, A7XX_SP_CTX3_3D_CPS_REG, A7XX_PIPE_BR, 3, A7XX_USPTP,
+		gen7_9_0_tpl1_pipe_br_cluster_sp_ps_usptp_registers, 0xb000},
+};
+
+static struct a6xx_indexed_registers gen7_9_0_cp_indexed_reg_list[] = {
+	{ "CP_SQE_STAT", REG_A6XX_CP_SQE_STAT_ADDR,
+		REG_A6XX_CP_SQE_STAT_DATA, 0x00040},
+	{ "CP_DRAW_STATE", REG_A6XX_CP_DRAW_STATE_ADDR,
+		REG_A6XX_CP_DRAW_STATE_DATA, 0x00200},
+	{ "CP_ROQ", REG_A6XX_CP_ROQ_DBG_ADDR,
+		REG_A6XX_CP_ROQ_DBG_DATA, 0x00800},
+	{ "CP_UCODE_DBG_DATA", REG_A6XX_CP_SQE_UCODE_DBG_ADDR,
+		REG_A6XX_CP_SQE_UCODE_DBG_DATA, 0x08000},
+	{ "CP_BV_SQE_STAT_ADDR", REG_A7XX_CP_BV_DRAW_STATE_ADDR,
+		REG_A7XX_CP_BV_DRAW_STATE_DATA, 0x00200},
+	{ "CP_BV_ROQ_DBG_ADDR", REG_A7XX_CP_BV_ROQ_DBG_ADDR,
+		REG_A7XX_CP_BV_ROQ_DBG_DATA, 0x00800},
+	{ "CP_BV_SQE_UCODE_DBG_ADDR", REG_A7XX_CP_BV_SQE_UCODE_DBG_ADDR,
+		REG_A7XX_CP_BV_SQE_UCODE_DBG_DATA, 0x08000},
+	{ "CP_BV_SQE_STAT_ADDR", REG_A7XX_CP_BV_SQE_STAT_ADDR,
+		REG_A7XX_CP_BV_SQE_STAT_DATA, 0x00040},
+	{ "CP_RESOURCE_TBL", REG_A7XX_CP_RESOURCE_TBL_DBG_ADDR,
+		REG_A7XX_CP_RESOURCE_TBL_DBG_DATA, 0x04100},
+	{ "CP_LPAC_DRAW_STATE_ADDR", REG_A7XX_CP_LPAC_DRAW_STATE_ADDR,
+		REG_A7XX_CP_LPAC_DRAW_STATE_DATA, 0x00200},
+	{ "CP_LPAC_ROQ", REG_A7XX_CP_LPAC_ROQ_DBG_ADDR,
+		REG_A7XX_CP_LPAC_ROQ_DBG_DATA, 0x00200},
+	{ "CP_SQE_AC_UCODE_DBG_ADDR", REG_A7XX_CP_SQE_AC_UCODE_DBG_ADDR,
+		REG_A7XX_CP_SQE_AC_UCODE_DBG_DATA, 0x08000},
+	{ "CP_SQE_AC_STAT_ADDR", REG_A7XX_CP_SQE_AC_STAT_ADDR,
+		REG_A7XX_CP_SQE_AC_STAT_DATA, 0x00040},
+	{ "CP_LPAC_FIFO_DBG_ADDR", REG_A7XX_CP_LPAC_FIFO_DBG_ADDR,
+		REG_A7XX_CP_LPAC_FIFO_DBG_DATA, 0x00040},
+	{ "CP_AQE_ROQ_0", REG_A7XX_CP_AQE_ROQ_DBG_ADDR_0,
+		REG_A7XX_CP_AQE_ROQ_DBG_DATA_0, 0x00100},
+	{ "CP_AQE_ROQ_1", REG_A7XX_CP_AQE_ROQ_DBG_ADDR_1,
+		REG_A7XX_CP_AQE_ROQ_DBG_DATA_1, 0x00100},
+	{ "CP_AQE_UCODE_DBG_0", REG_A7XX_CP_AQE_UCODE_DBG_ADDR_0,
+		REG_A7XX_CP_AQE_UCODE_DBG_DATA_0, 0x08000},
+	{ "CP_AQE_UCODE_DBG_1", REG_A7XX_CP_AQE_UCODE_DBG_ADDR_1,
+		REG_A7XX_CP_AQE_UCODE_DBG_DATA_1, 0x08000},
+	{ "CP_AQE_STAT_0", REG_A7XX_CP_AQE_STAT_ADDR_0,
+		REG_A7XX_CP_AQE_STAT_DATA_0, 0x00040},
+	{ "CP_AQE_STAT_1", REG_A7XX_CP_AQE_STAT_ADDR_1,
+		REG_A7XX_CP_AQE_STAT_DATA_1, 0x00040},
+};
+
+static struct gen7_reg_list gen7_9_0_reg_list[] = {
+	{ gen7_9_0_gpu_registers, NULL},
+	{ gen7_9_0_cx_misc_registers, NULL},
+	{ gen7_9_0_cx_dbgc_registers, NULL},
+	{ gen7_9_0_dbgc_registers, NULL},
+	{ NULL, NULL},
+};
+
+static const u32 gen7_9_0_cpr_registers[] = {
+	0x26800, 0x26805, 0x26808, 0x2680d, 0x26814, 0x26815, 0x2681c, 0x2681c,
+	0x26820, 0x26839, 0x26840, 0x26841, 0x26848, 0x26849, 0x26850, 0x26851,
+	0x26880, 0x268a1, 0x26980, 0x269b0, 0x269c0, 0x269c8, 0x269e0, 0x269ee,
+	0x269fb, 0x269ff, 0x26a02, 0x26a07, 0x26a09, 0x26a0b, 0x26a10, 0x26b0f,
+	0x27440, 0x27441, 0x27444, 0x27444, 0x27480, 0x274a2, 0x274ac, 0x274c4,
+	UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_cpr_registers), 8));
+
+static const u32 gen7_9_0_dpm_registers[] = {
+	0x1aa00, 0x1aa06, 0x1aa09, 0x1aa0a, 0x1aa0c, 0x1aa0d, 0x1aa0f, 0x1aa12,
+	0x1aa14, 0x1aa47, 0x1aa50, 0x1aa51,
+	UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_dpm_registers), 8));
+
+static const u32 gen7_9_0_dpm_leakage_registers[] = {
+	0x21c00, 0x21c00, 0x21c08, 0x21c09, 0x21c0e, 0x21c0f, 0x21c4f, 0x21c50,
+	0x21c52, 0x21c52, 0x21c54, 0x21c56, 0x21c58, 0x21c5a, 0x21c5c, 0x21c60,
+	UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_dpm_leakage_registers), 8));
+
+static const u32 gen7_9_0_gfx_gpu_acd_registers[] = {
+	0x18c00, 0x18c16, 0x18c20, 0x18c2d, 0x18c30, 0x18c31, 0x18c35, 0x18c35,
+	0x18c37, 0x18c37, 0x18c3a, 0x18c3a, 0x18c42, 0x18c42, 0x18c56, 0x18c58,
+	0x18c5b, 0x18c5d, 0x18c5f, 0x18c62,
+	UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_gfx_gpu_acd_registers), 8));
+
+static const u32 gen7_9_0_gpucc_registers[] = {
+	0x24000, 0x2400f, 0x24400, 0x2440f, 0x24800, 0x24805, 0x24c00, 0x24cff,
+	0x25400, 0x25404, 0x25800, 0x25804, 0x25c00, 0x25c04, 0x26000, 0x26004,
+	0x26400, 0x26405, 0x26414, 0x2641d, 0x2642a, 0x26430, 0x26432, 0x26434,
+	0x26441, 0x2644b, 0x2644d, 0x26463, 0x26466, 0x26468, 0x26478, 0x2647a,
+	0x26489, 0x2648a, 0x2649c, 0x2649e, 0x264a0, 0x264a6, 0x264c5, 0x264c7,
+	0x264d6, 0x264d8, 0x264e8, 0x264e9, 0x264f9, 0x264fc, 0x2650b, 0x2650b,
+	0x2651c, 0x2651e, 0x26540, 0x2654e, 0x26554, 0x26573, 0x26576, 0x2657a,
+	UINT_MAX, UINT_MAX,
+
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_gpucc_registers), 8));
+
+static const u32 gen7_9_0_isense_registers[] = {
+	0x22c3a, 0x22c3c, 0x22c41, 0x22c41, 0x22c46, 0x22c47, 0x22c4c, 0x22c4c,
+	0x22c51, 0x22c51, 0x22c56, 0x22c56, 0x22c5b, 0x22c5b, 0x22c60, 0x22c60,
+	0x22c65, 0x22c65, 0x22c6a, 0x22c70, 0x22c75, 0x22c75, 0x22c7a, 0x22c7a,
+	0x22c7f, 0x22c7f, 0x22c84, 0x22c85, 0x22c8a, 0x22c8a, 0x22c8f, 0x22c8f,
+	0x23000, 0x23009, 0x2300e, 0x2300e, 0x23013, 0x23013, 0x23018, 0x23018,
+	0x2301d, 0x2301d, 0x23022, 0x23022, 0x23027, 0x23032, 0x23037, 0x23037,
+	0x2303c, 0x2303c, 0x23041, 0x23041, 0x23046, 0x23046, 0x2304b, 0x2304b,
+	0x23050, 0x23050, 0x23055, 0x23055, 0x2305a, 0x2305a, 0x2305f, 0x2305f,
+	0x23064, 0x23064, 0x23069, 0x2306a, 0x2306f, 0x2306f, 0x23074, 0x23075,
+	0x2307a, 0x2307e, 0x23083, 0x23083, 0x23088, 0x23088, 0x2308d, 0x2308d,
+	0x23092, 0x23092, 0x230e2, 0x230e2,
+	UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_isense_registers), 8));
+
+static const u32 gen7_9_0_rscc_registers[] = {
+	0x14000, 0x14036, 0x14040, 0x14047, 0x14080, 0x14084, 0x14089, 0x1408c,
+	0x14091, 0x14094, 0x14099, 0x1409c, 0x140a1, 0x140a4, 0x140a9, 0x140ac,
+	0x14100, 0x14104, 0x14114, 0x14119, 0x14124, 0x14132, 0x14154, 0x1416b,
+	0x14340, 0x14342, 0x14344, 0x1437c, 0x143f0, 0x143f8, 0x143fa, 0x143fe,
+	0x14400, 0x14404, 0x14406, 0x1440a, 0x1440c, 0x14410, 0x14412, 0x14416,
+	0x14418, 0x1441c, 0x1441e, 0x14422, 0x14424, 0x14424, 0x14498, 0x144a0,
+	0x144a2, 0x144a6, 0x144a8, 0x144ac, 0x144ae, 0x144b2, 0x144b4, 0x144b8,
+	0x144ba, 0x144be, 0x144c0, 0x144c4, 0x144c6, 0x144ca, 0x144cc, 0x144cc,
+	0x14540, 0x14548, 0x1454a, 0x1454e, 0x14550, 0x14554, 0x14556, 0x1455a,
+	0x1455c, 0x14560, 0x14562, 0x14566, 0x14568, 0x1456c, 0x1456e, 0x14572,
+	0x14574, 0x14574, 0x145e8, 0x145f0, 0x145f2, 0x145f6, 0x145f8, 0x145fc,
+	0x145fe, 0x14602, 0x14604, 0x14608, 0x1460a, 0x1460e, 0x14610, 0x14614,
+	0x14616, 0x1461a, 0x1461c, 0x1461c, 0x14690, 0x14698, 0x1469a, 0x1469e,
+	0x146a0, 0x146a4, 0x146a6, 0x146aa, 0x146ac, 0x146b0, 0x146b2, 0x146b6,
+	0x146b8, 0x146bc, 0x146be, 0x146c2, 0x146c4, 0x146c4, 0x14738, 0x14740,
+	0x14742, 0x14746, 0x14748, 0x1474c, 0x1474e, 0x14752, 0x14754, 0x14758,
+	0x1475a, 0x1475e, 0x14760, 0x14764, 0x14766, 0x1476a, 0x1476c, 0x1476c,
+	0x147e0, 0x147e8, 0x147ea, 0x147ee, 0x147f0, 0x147f4, 0x147f6, 0x147fa,
+	0x147fc, 0x14800, 0x14802, 0x14806, 0x14808, 0x1480c, 0x1480e, 0x14812,
+	0x14814, 0x14814, 0x14888, 0x14890, 0x14892, 0x14896, 0x14898, 0x1489c,
+	0x1489e, 0x148a2, 0x148a4, 0x148a8, 0x148aa, 0x148ae, 0x148b0, 0x148b4,
+	0x148b6, 0x148ba, 0x148bc, 0x148bc, 0x14930, 0x14938, 0x1493a, 0x1493e,
+	0x14940, 0x14944, 0x14946, 0x1494a, 0x1494c, 0x14950, 0x14952, 0x14956,
+	0x14958, 0x1495c, 0x1495e, 0x14962, 0x14964, 0x14964,
+	UINT_MAX, UINT_MAX,
+};
+static_assert(IS_ALIGNED(sizeof(gen7_9_0_rscc_registers), 8));
+
+static const u32 *gen7_9_0_external_core_regs[] = {
+	gen7_9_0_gpucc_registers,
+	gen7_9_0_gxclkctl_registers,
+	gen7_9_0_cpr_registers,
+	gen7_9_0_dpm_registers,
+	gen7_9_0_dpm_leakage_registers,
+	gen7_9_0_gfx_gpu_acd_registers,
+};
+#endif /*_ADRENO_GEN7_9_0_SNAPSHOT_H */
diff --git a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
deleted file mode 100644
index 7067376e25e1..000000000000
--- a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
+++ /dev/null
@@ -1,2803 +0,0 @@
-#ifndef ADRENO_PM4_XML
-#define ADRENO_PM4_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng gen_header.py tool in this git repository:
-http://gitlab.freedesktop.org/mesa/mesa/
-git clone https://gitlab.freedesktop.org/mesa/mesa.git
-
-The rules-ng-ng source files this header was generated from are:
-
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_pm4.xml    (  85856 bytes, from Fri Feb 23 13:07:00 2024)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/adreno/adreno_common.xml (  15434 bytes, from Fri Jun  2 14:59:26 2023)
-*/
-
-#ifdef __KERNEL__
-#include <linux/bug.h>
-#define assert(x) BUG_ON(!(x))
-#else
-#include <assert.h>
-#endif
-
-#ifdef __cplusplus
-#define __struct_cast(X)
-#else
-#define __struct_cast(X) (struct X)
-#endif
-
-enum vgt_event_type {
-	VS_DEALLOC = 0,
-	PS_DEALLOC = 1,
-	VS_DONE_TS = 2,
-	PS_DONE_TS = 3,
-	CACHE_FLUSH_TS = 4,
-	CONTEXT_DONE = 5,
-	CACHE_FLUSH = 6,
-	VIZQUERY_START = 7,
-	HLSQ_FLUSH = 7,
-	VIZQUERY_END = 8,
-	SC_WAIT_WC = 9,
-	WRITE_PRIMITIVE_COUNTS = 9,
-	START_PRIMITIVE_CTRS = 11,
-	STOP_PRIMITIVE_CTRS = 12,
-	RST_PIX_CNT = 13,
-	RST_VTX_CNT = 14,
-	TILE_FLUSH = 15,
-	STAT_EVENT = 16,
-	CACHE_FLUSH_AND_INV_TS_EVENT = 20,
-	ZPASS_DONE = 21,
-	CACHE_FLUSH_AND_INV_EVENT = 22,
-	RB_DONE_TS = 22,
-	PERFCOUNTER_START = 23,
-	PERFCOUNTER_STOP = 24,
-	VS_FETCH_DONE = 27,
-	FACENESS_FLUSH = 28,
-	WT_DONE_TS = 8,
-	START_FRAGMENT_CTRS = 13,
-	STOP_FRAGMENT_CTRS = 14,
-	START_COMPUTE_CTRS = 15,
-	STOP_COMPUTE_CTRS = 16,
-	FLUSH_SO_0 = 17,
-	FLUSH_SO_1 = 18,
-	FLUSH_SO_2 = 19,
-	FLUSH_SO_3 = 20,
-	PC_CCU_INVALIDATE_DEPTH = 24,
-	PC_CCU_INVALIDATE_COLOR = 25,
-	PC_CCU_RESOLVE_TS = 26,
-	PC_CCU_FLUSH_DEPTH_TS = 28,
-	PC_CCU_FLUSH_COLOR_TS = 29,
-	BLIT = 30,
-	LRZ_CLEAR = 37,
-	LRZ_FLUSH = 38,
-	BLIT_OP_FILL_2D = 39,
-	BLIT_OP_COPY_2D = 40,
-	UNK_40 = 40,
-	BLIT_OP_SCALE_2D = 42,
-	CONTEXT_DONE_2D = 43,
-	UNK_2C = 44,
-	UNK_2D = 45,
-	CACHE_INVALIDATE = 49,
-	LABEL = 63,
-	DUMMY_EVENT = 1,
-	CCU_INVALIDATE_DEPTH = 24,
-	CCU_INVALIDATE_COLOR = 25,
-	CCU_RESOLVE_CLEAN = 26,
-	CCU_FLUSH_DEPTH = 28,
-	CCU_FLUSH_COLOR = 29,
-	CCU_RESOLVE = 30,
-	CCU_END_RESOLVE_GROUP = 31,
-	CCU_CLEAN_DEPTH = 32,
-	CCU_CLEAN_COLOR = 33,
-	CACHE_RESET = 48,
-	CACHE_CLEAN = 49,
-	CACHE_FLUSH7 = 50,
-	CACHE_INVALIDATE7 = 51,
-};
-
-enum pc_di_primtype {
-	DI_PT_NONE = 0,
-	DI_PT_POINTLIST_PSIZE = 1,
-	DI_PT_LINELIST = 2,
-	DI_PT_LINESTRIP = 3,
-	DI_PT_TRILIST = 4,
-	DI_PT_TRIFAN = 5,
-	DI_PT_TRISTRIP = 6,
-	DI_PT_LINELOOP = 7,
-	DI_PT_RECTLIST = 8,
-	DI_PT_POINTLIST = 9,
-	DI_PT_LINE_ADJ = 10,
-	DI_PT_LINESTRIP_ADJ = 11,
-	DI_PT_TRI_ADJ = 12,
-	DI_PT_TRISTRIP_ADJ = 13,
-	DI_PT_PATCHES0 = 31,
-	DI_PT_PATCHES1 = 32,
-	DI_PT_PATCHES2 = 33,
-	DI_PT_PATCHES3 = 34,
-	DI_PT_PATCHES4 = 35,
-	DI_PT_PATCHES5 = 36,
-	DI_PT_PATCHES6 = 37,
-	DI_PT_PATCHES7 = 38,
-	DI_PT_PATCHES8 = 39,
-	DI_PT_PATCHES9 = 40,
-	DI_PT_PATCHES10 = 41,
-	DI_PT_PATCHES11 = 42,
-	DI_PT_PATCHES12 = 43,
-	DI_PT_PATCHES13 = 44,
-	DI_PT_PATCHES14 = 45,
-	DI_PT_PATCHES15 = 46,
-	DI_PT_PATCHES16 = 47,
-	DI_PT_PATCHES17 = 48,
-	DI_PT_PATCHES18 = 49,
-	DI_PT_PATCHES19 = 50,
-	DI_PT_PATCHES20 = 51,
-	DI_PT_PATCHES21 = 52,
-	DI_PT_PATCHES22 = 53,
-	DI_PT_PATCHES23 = 54,
-	DI_PT_PATCHES24 = 55,
-	DI_PT_PATCHES25 = 56,
-	DI_PT_PATCHES26 = 57,
-	DI_PT_PATCHES27 = 58,
-	DI_PT_PATCHES28 = 59,
-	DI_PT_PATCHES29 = 60,
-	DI_PT_PATCHES30 = 61,
-	DI_PT_PATCHES31 = 62,
-};
-
-enum pc_di_src_sel {
-	DI_SRC_SEL_DMA = 0,
-	DI_SRC_SEL_IMMEDIATE = 1,
-	DI_SRC_SEL_AUTO_INDEX = 2,
-	DI_SRC_SEL_AUTO_XFB = 3,
-};
-
-enum pc_di_face_cull_sel {
-	DI_FACE_CULL_NONE = 0,
-	DI_FACE_CULL_FETCH = 1,
-	DI_FACE_BACKFACE_CULL = 2,
-	DI_FACE_FRONTFACE_CULL = 3,
-};
-
-enum pc_di_index_size {
-	INDEX_SIZE_IGN = 0,
-	INDEX_SIZE_16_BIT = 0,
-	INDEX_SIZE_32_BIT = 1,
-	INDEX_SIZE_8_BIT = 2,
-	INDEX_SIZE_INVALID = 0,
-};
-
-enum pc_di_vis_cull_mode {
-	IGNORE_VISIBILITY = 0,
-	USE_VISIBILITY = 1,
-};
-
-enum adreno_pm4_packet_type {
-	CP_TYPE0_PKT = 0x00000000,
-	CP_TYPE1_PKT = 0x40000000,
-	CP_TYPE2_PKT = 0x80000000,
-	CP_TYPE3_PKT = 0xc0000000,
-	CP_TYPE4_PKT = 0x40000000,
-	CP_TYPE7_PKT = 0x70000000,
-};
-
-enum adreno_pm4_type3_packets {
-	CP_ME_INIT = 72,
-	CP_NOP = 16,
-	CP_PREEMPT_ENABLE = 28,
-	CP_PREEMPT_TOKEN = 30,
-	CP_INDIRECT_BUFFER = 63,
-	CP_INDIRECT_BUFFER_CHAIN = 87,
-	CP_INDIRECT_BUFFER_PFD = 55,
-	CP_WAIT_FOR_IDLE = 38,
-	CP_WAIT_REG_MEM = 60,
-	CP_WAIT_REG_EQ = 82,
-	CP_WAIT_REG_GTE = 83,
-	CP_WAIT_UNTIL_READ = 92,
-	CP_WAIT_IB_PFD_COMPLETE = 93,
-	CP_REG_RMW = 33,
-	CP_SET_BIN_DATA = 47,
-	CP_SET_BIN_DATA5 = 47,
-	CP_REG_TO_MEM = 62,
-	CP_MEM_WRITE = 61,
-	CP_MEM_WRITE_CNTR = 79,
-	CP_COND_EXEC = 68,
-	CP_COND_WRITE = 69,
-	CP_COND_WRITE5 = 69,
-	CP_EVENT_WRITE = 70,
-	CP_EVENT_WRITE7 = 70,
-	CP_EVENT_WRITE_SHD = 88,
-	CP_EVENT_WRITE_CFL = 89,
-	CP_EVENT_WRITE_ZPD = 91,
-	CP_RUN_OPENCL = 49,
-	CP_DRAW_INDX = 34,
-	CP_DRAW_INDX_2 = 54,
-	CP_DRAW_INDX_BIN = 52,
-	CP_DRAW_INDX_2_BIN = 53,
-	CP_VIZ_QUERY = 35,
-	CP_SET_STATE = 37,
-	CP_SET_CONSTANT = 45,
-	CP_IM_LOAD = 39,
-	CP_IM_LOAD_IMMEDIATE = 43,
-	CP_LOAD_CONSTANT_CONTEXT = 46,
-	CP_INVALIDATE_STATE = 59,
-	CP_SET_SHADER_BASES = 74,
-	CP_SET_BIN_MASK = 80,
-	CP_SET_BIN_SELECT = 81,
-	CP_CONTEXT_UPDATE = 94,
-	CP_INTERRUPT = 64,
-	CP_IM_STORE = 44,
-	CP_SET_DRAW_INIT_FLAGS = 75,
-	CP_SET_PROTECTED_MODE = 95,
-	CP_BOOTSTRAP_UCODE = 111,
-	CP_LOAD_STATE = 48,
-	CP_LOAD_STATE4 = 48,
-	CP_COND_INDIRECT_BUFFER_PFE = 58,
-	CP_COND_INDIRECT_BUFFER_PFD = 50,
-	CP_INDIRECT_BUFFER_PFE = 63,
-	CP_SET_BIN = 76,
-	CP_TEST_TWO_MEMS = 113,
-	CP_REG_WR_NO_CTXT = 120,
-	CP_RECORD_PFP_TIMESTAMP = 17,
-	CP_SET_SECURE_MODE = 102,
-	CP_WAIT_FOR_ME = 19,
-	CP_SET_DRAW_STATE = 67,
-	CP_DRAW_INDX_OFFSET = 56,
-	CP_DRAW_INDIRECT = 40,
-	CP_DRAW_INDX_INDIRECT = 41,
-	CP_DRAW_INDIRECT_MULTI = 42,
-	CP_DRAW_AUTO = 36,
-	CP_DRAW_PRED_ENABLE_GLOBAL = 25,
-	CP_DRAW_PRED_ENABLE_LOCAL = 26,
-	CP_DRAW_PRED_SET = 78,
-	CP_WIDE_REG_WRITE = 116,
-	CP_SCRATCH_TO_REG = 77,
-	CP_REG_TO_SCRATCH = 74,
-	CP_WAIT_MEM_WRITES = 18,
-	CP_COND_REG_EXEC = 71,
-	CP_MEM_TO_REG = 66,
-	CP_EXEC_CS_INDIRECT = 65,
-	CP_EXEC_CS = 51,
-	CP_PERFCOUNTER_ACTION = 80,
-	CP_SMMU_TABLE_UPDATE = 83,
-	CP_SET_MARKER = 101,
-	CP_SET_PSEUDO_REG = 86,
-	CP_CONTEXT_REG_BUNCH = 92,
-	CP_YIELD_ENABLE = 28,
-	CP_SKIP_IB2_ENABLE_GLOBAL = 29,
-	CP_SKIP_IB2_ENABLE_LOCAL = 35,
-	CP_SET_SUBDRAW_SIZE = 53,
-	CP_WHERE_AM_I = 98,
-	CP_SET_VISIBILITY_OVERRIDE = 100,
-	CP_PREEMPT_ENABLE_GLOBAL = 105,
-	CP_PREEMPT_ENABLE_LOCAL = 106,
-	CP_CONTEXT_SWITCH_YIELD = 107,
-	CP_SET_RENDER_MODE = 108,
-	CP_COMPUTE_CHECKPOINT = 110,
-	CP_MEM_TO_MEM = 115,
-	CP_BLIT = 44,
-	CP_REG_TEST = 57,
-	CP_SET_MODE = 99,
-	CP_LOAD_STATE6_GEOM = 50,
-	CP_LOAD_STATE6_FRAG = 52,
-	CP_LOAD_STATE6 = 54,
-	IN_IB_PREFETCH_END = 23,
-	IN_SUBBLK_PREFETCH = 31,
-	IN_INSTR_PREFETCH = 32,
-	IN_INSTR_MATCH = 71,
-	IN_CONST_PREFETCH = 73,
-	IN_INCR_UPDT_STATE = 85,
-	IN_INCR_UPDT_CONST = 86,
-	IN_INCR_UPDT_INSTR = 87,
-	PKT4 = 4,
-	IN_IB_END = 10,
-	IN_GMU_INTERRUPT = 11,
-	IN_PREEMPT = 15,
-	CP_SCRATCH_WRITE = 76,
-	CP_REG_TO_MEM_OFFSET_MEM = 116,
-	CP_REG_TO_MEM_OFFSET_REG = 114,
-	CP_WAIT_MEM_GTE = 20,
-	CP_WAIT_TWO_REGS = 112,
-	CP_MEMCPY = 117,
-	CP_SET_BIN_DATA5_OFFSET = 46,
-	CP_SET_UNK_BIN_DATA = 45,
-	CP_CONTEXT_SWITCH = 84,
-	CP_SET_CTXSWITCH_IB = 85,
-	CP_REG_WRITE = 109,
-	CP_START_BIN = 80,
-	CP_END_BIN = 81,
-	CP_PREEMPT_DISABLE = 108,
-	CP_WAIT_TIMESTAMP = 20,
-	CP_GLOBAL_TIMESTAMP = 21,
-	CP_LOCAL_TIMESTAMP = 22,
-	CP_THREAD_CONTROL = 23,
-	CP_RESOURCE_LIST = 24,
-	CP_BV_BR_COUNT_OPS = 27,
-	CP_MODIFY_TIMESTAMP = 28,
-	CP_CONTEXT_REG_BUNCH2 = 93,
-	CP_MEM_TO_SCRATCH_MEM = 73,
-	CP_FIXED_STRIDE_DRAW_TABLE = 127,
-	CP_RESET_CONTEXT_STATE = 31,
-};
-
-enum adreno_state_block {
-	SB_VERT_TEX = 0,
-	SB_VERT_MIPADDR = 1,
-	SB_FRAG_TEX = 2,
-	SB_FRAG_MIPADDR = 3,
-	SB_VERT_SHADER = 4,
-	SB_GEOM_SHADER = 5,
-	SB_FRAG_SHADER = 6,
-	SB_COMPUTE_SHADER = 7,
-};
-
-enum adreno_state_type {
-	ST_SHADER = 0,
-	ST_CONSTANTS = 1,
-};
-
-enum adreno_state_src {
-	SS_DIRECT = 0,
-	SS_INVALID_ALL_IC = 2,
-	SS_INVALID_PART_IC = 3,
-	SS_INDIRECT = 4,
-	SS_INDIRECT_TCM = 5,
-	SS_INDIRECT_STM = 6,
-};
-
-enum a4xx_state_block {
-	SB4_VS_TEX = 0,
-	SB4_HS_TEX = 1,
-	SB4_DS_TEX = 2,
-	SB4_GS_TEX = 3,
-	SB4_FS_TEX = 4,
-	SB4_CS_TEX = 5,
-	SB4_VS_SHADER = 8,
-	SB4_HS_SHADER = 9,
-	SB4_DS_SHADER = 10,
-	SB4_GS_SHADER = 11,
-	SB4_FS_SHADER = 12,
-	SB4_CS_SHADER = 13,
-	SB4_SSBO = 14,
-	SB4_CS_SSBO = 15,
-};
-
-enum a4xx_state_type {
-	ST4_SHADER = 0,
-	ST4_CONSTANTS = 1,
-	ST4_UBO = 2,
-};
-
-enum a4xx_state_src {
-	SS4_DIRECT = 0,
-	SS4_INDIRECT = 2,
-};
-
-enum a6xx_state_block {
-	SB6_VS_TEX = 0,
-	SB6_HS_TEX = 1,
-	SB6_DS_TEX = 2,
-	SB6_GS_TEX = 3,
-	SB6_FS_TEX = 4,
-	SB6_CS_TEX = 5,
-	SB6_VS_SHADER = 8,
-	SB6_HS_SHADER = 9,
-	SB6_DS_SHADER = 10,
-	SB6_GS_SHADER = 11,
-	SB6_FS_SHADER = 12,
-	SB6_CS_SHADER = 13,
-	SB6_IBO = 14,
-	SB6_CS_IBO = 15,
-};
-
-enum a6xx_state_type {
-	ST6_SHADER = 0,
-	ST6_CONSTANTS = 1,
-	ST6_UBO = 2,
-	ST6_IBO = 3,
-};
-
-enum a6xx_state_src {
-	SS6_DIRECT = 0,
-	SS6_BINDLESS = 1,
-	SS6_INDIRECT = 2,
-	SS6_UBO = 3,
-};
-
-enum a4xx_index_size {
-	INDEX4_SIZE_8_BIT = 0,
-	INDEX4_SIZE_16_BIT = 1,
-	INDEX4_SIZE_32_BIT = 2,
-};
-
-enum a6xx_patch_type {
-	TESS_QUADS = 0,
-	TESS_TRIANGLES = 1,
-	TESS_ISOLINES = 2,
-};
-
-enum a6xx_draw_indirect_opcode {
-	INDIRECT_OP_NORMAL = 2,
-	INDIRECT_OP_INDEXED = 4,
-	INDIRECT_OP_INDIRECT_COUNT = 6,
-	INDIRECT_OP_INDIRECT_COUNT_INDEXED = 7,
-};
-
-enum cp_draw_pred_src {
-	PRED_SRC_MEM = 5,
-};
-
-enum cp_draw_pred_test {
-	NE_0_PASS = 0,
-	EQ_0_PASS = 1,
-};
-
-enum cp_cond_function {
-	WRITE_ALWAYS = 0,
-	WRITE_LT = 1,
-	WRITE_LE = 2,
-	WRITE_EQ = 3,
-	WRITE_NE = 4,
-	WRITE_GE = 5,
-	WRITE_GT = 6,
-};
-
-enum poll_memory_type {
-	POLL_REGISTER = 0,
-	POLL_MEMORY = 1,
-	POLL_SCRATCH = 2,
-	POLL_ON_CHIP = 3,
-};
-
-enum render_mode_cmd {
-	BYPASS = 1,
-	BINNING = 2,
-	GMEM = 3,
-	BLIT2D = 5,
-	BLIT2DSCALE = 7,
-	END2D = 8,
-};
-
-enum event_write_src {
-	EV_WRITE_USER_32B = 0,
-	EV_WRITE_USER_64B = 1,
-	EV_WRITE_TIMESTAMP_SUM = 2,
-	EV_WRITE_ALWAYSON = 3,
-	EV_WRITE_REGS_CONTENT = 4,
-};
-
-enum event_write_dst {
-	EV_DST_RAM = 0,
-	EV_DST_ONCHIP = 1,
-};
-
-enum cp_blit_cmd {
-	BLIT_OP_FILL = 0,
-	BLIT_OP_COPY = 1,
-	BLIT_OP_SCALE = 3,
-};
-
-enum a6xx_marker {
-	RM6_BYPASS = 1,
-	RM6_BINNING = 2,
-	RM6_GMEM = 4,
-	RM6_ENDVIS = 5,
-	RM6_RESOLVE = 6,
-	RM6_YIELD = 7,
-	RM6_COMPUTE = 8,
-	RM6_BLIT2DSCALE = 12,
-	RM6_IB1LIST_START = 13,
-	RM6_IB1LIST_END = 14,
-	RM6_IFPC_ENABLE = 256,
-	RM6_IFPC_DISABLE = 257,
-};
-
-enum pseudo_reg {
-	SMMU_INFO = 0,
-	NON_SECURE_SAVE_ADDR = 1,
-	SECURE_SAVE_ADDR = 2,
-	NON_PRIV_SAVE_ADDR = 3,
-	COUNTER = 4,
-	DRAW_STRM_ADDRESS = 8,
-	DRAW_STRM_SIZE_ADDRESS = 9,
-	PRIM_STRM_ADDRESS = 10,
-	UNK_STRM_ADDRESS = 11,
-	UNK_STRM_SIZE_ADDRESS = 12,
-	BINDLESS_BASE_0_ADDR = 16,
-	BINDLESS_BASE_1_ADDR = 17,
-	BINDLESS_BASE_2_ADDR = 18,
-	BINDLESS_BASE_3_ADDR = 19,
-	BINDLESS_BASE_4_ADDR = 20,
-	BINDLESS_BASE_5_ADDR = 21,
-	BINDLESS_BASE_6_ADDR = 22,
-};
-
-enum source_type {
-	SOURCE_REG = 0,
-	SOURCE_SCRATCH_MEM = 1,
-};
-
-enum compare_mode {
-	PRED_TEST = 1,
-	REG_COMPARE = 2,
-	RENDER_MODE = 3,
-	REG_COMPARE_IMM = 4,
-	THREAD_MODE = 5,
-};
-
-enum ctxswitch_ib {
-	RESTORE_IB = 0,
-	YIELD_RESTORE_IB = 1,
-	SAVE_IB = 2,
-	RB_SAVE_IB = 3,
-};
-
-enum reg_tracker {
-	TRACK_CNTL_REG = 1,
-	TRACK_RENDER_CNTL = 2,
-	UNK_EVENT_WRITE = 4,
-	TRACK_LRZ = 8,
-};
-
-enum ts_wait_value_src {
-	TS_WAIT_GE_32B = 0,
-	TS_WAIT_GE_64B = 1,
-	TS_WAIT_GE_TIMESTAMP_SUM = 2,
-};
-
-enum ts_wait_type {
-	TS_WAIT_RAM = 0,
-	TS_WAIT_ONCHIP = 1,
-};
-
-enum pipe_count_op {
-	PIPE_CLEAR_BV_BR = 1,
-	PIPE_SET_BR_OFFSET = 2,
-	PIPE_BR_WAIT_FOR_BV = 3,
-	PIPE_BV_WAIT_FOR_BR = 4,
-};
-
-enum timestamp_op {
-	MODIFY_TIMESTAMP_CLEAR = 0,
-	MODIFY_TIMESTAMP_ADD_GLOBAL = 1,
-	MODIFY_TIMESTAMP_ADD_LOCAL = 2,
-};
-
-enum cp_thread {
-	CP_SET_THREAD_BR = 1,
-	CP_SET_THREAD_BV = 2,
-	CP_SET_THREAD_BOTH = 3,
-};
-
-#define REG_CP_LOAD_STATE_0					0x00000000
-#define CP_LOAD_STATE_0_DST_OFF__MASK				0x0000ffff
-#define CP_LOAD_STATE_0_DST_OFF__SHIFT				0
-static inline uint32_t CP_LOAD_STATE_0_DST_OFF(uint32_t val)
-{
-	return ((val) << CP_LOAD_STATE_0_DST_OFF__SHIFT) & CP_LOAD_STATE_0_DST_OFF__MASK;
-}
-#define CP_LOAD_STATE_0_STATE_SRC__MASK				0x00070000
-#define CP_LOAD_STATE_0_STATE_SRC__SHIFT			16
-static inline uint32_t CP_LOAD_STATE_0_STATE_SRC(enum adreno_state_src val)
-{
-	return ((val) << CP_LOAD_STATE_0_STATE_SRC__SHIFT) & CP_LOAD_STATE_0_STATE_SRC__MASK;
-}
-#define CP_LOAD_STATE_0_STATE_BLOCK__MASK			0x00380000
-#define CP_LOAD_STATE_0_STATE_BLOCK__SHIFT			19
-static inline uint32_t CP_LOAD_STATE_0_STATE_BLOCK(enum adreno_state_block val)
-{
-	return ((val) << CP_LOAD_STATE_0_STATE_BLOCK__SHIFT) & CP_LOAD_STATE_0_STATE_BLOCK__MASK;
-}
-#define CP_LOAD_STATE_0_NUM_UNIT__MASK				0xffc00000
-#define CP_LOAD_STATE_0_NUM_UNIT__SHIFT				22
-static inline uint32_t CP_LOAD_STATE_0_NUM_UNIT(uint32_t val)
-{
-	return ((val) << CP_LOAD_STATE_0_NUM_UNIT__SHIFT) & CP_LOAD_STATE_0_NUM_UNIT__MASK;
-}
-
-#define REG_CP_LOAD_STATE_1					0x00000001
-#define CP_LOAD_STATE_1_STATE_TYPE__MASK			0x00000003
-#define CP_LOAD_STATE_1_STATE_TYPE__SHIFT			0
-static inline uint32_t CP_LOAD_STATE_1_STATE_TYPE(enum adreno_state_type val)
-{
-	return ((val) << CP_LOAD_STATE_1_STATE_TYPE__SHIFT) & CP_LOAD_STATE_1_STATE_TYPE__MASK;
-}
-#define CP_LOAD_STATE_1_EXT_SRC_ADDR__MASK			0xfffffffc
-#define CP_LOAD_STATE_1_EXT_SRC_ADDR__SHIFT			2
-static inline uint32_t CP_LOAD_STATE_1_EXT_SRC_ADDR(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << CP_LOAD_STATE_1_EXT_SRC_ADDR__SHIFT) & CP_LOAD_STATE_1_EXT_SRC_ADDR__MASK;
-}
-
-#define REG_CP_LOAD_STATE4_0					0x00000000
-#define CP_LOAD_STATE4_0_DST_OFF__MASK				0x00003fff
-#define CP_LOAD_STATE4_0_DST_OFF__SHIFT				0
-static inline uint32_t CP_LOAD_STATE4_0_DST_OFF(uint32_t val)
-{
-	return ((val) << CP_LOAD_STATE4_0_DST_OFF__SHIFT) & CP_LOAD_STATE4_0_DST_OFF__MASK;
-}
-#define CP_LOAD_STATE4_0_STATE_SRC__MASK			0x00030000
-#define CP_LOAD_STATE4_0_STATE_SRC__SHIFT			16
-static inline uint32_t CP_LOAD_STATE4_0_STATE_SRC(enum a4xx_state_src val)
-{
-	return ((val) << CP_LOAD_STATE4_0_STATE_SRC__SHIFT) & CP_LOAD_STATE4_0_STATE_SRC__MASK;
-}
-#define CP_LOAD_STATE4_0_STATE_BLOCK__MASK			0x003c0000
-#define CP_LOAD_STATE4_0_STATE_BLOCK__SHIFT			18
-static inline uint32_t CP_LOAD_STATE4_0_STATE_BLOCK(enum a4xx_state_block val)
-{
-	return ((val) << CP_LOAD_STATE4_0_STATE_BLOCK__SHIFT) & CP_LOAD_STATE4_0_STATE_BLOCK__MASK;
-}
-#define CP_LOAD_STATE4_0_NUM_UNIT__MASK				0xffc00000
-#define CP_LOAD_STATE4_0_NUM_UNIT__SHIFT			22
-static inline uint32_t CP_LOAD_STATE4_0_NUM_UNIT(uint32_t val)
-{
-	return ((val) << CP_LOAD_STATE4_0_NUM_UNIT__SHIFT) & CP_LOAD_STATE4_0_NUM_UNIT__MASK;
-}
-
-#define REG_CP_LOAD_STATE4_1					0x00000001
-#define CP_LOAD_STATE4_1_STATE_TYPE__MASK			0x00000003
-#define CP_LOAD_STATE4_1_STATE_TYPE__SHIFT			0
-static inline uint32_t CP_LOAD_STATE4_1_STATE_TYPE(enum a4xx_state_type val)
-{
-	return ((val) << CP_LOAD_STATE4_1_STATE_TYPE__SHIFT) & CP_LOAD_STATE4_1_STATE_TYPE__MASK;
-}
-#define CP_LOAD_STATE4_1_EXT_SRC_ADDR__MASK			0xfffffffc
-#define CP_LOAD_STATE4_1_EXT_SRC_ADDR__SHIFT			2
-static inline uint32_t CP_LOAD_STATE4_1_EXT_SRC_ADDR(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << CP_LOAD_STATE4_1_EXT_SRC_ADDR__SHIFT) & CP_LOAD_STATE4_1_EXT_SRC_ADDR__MASK;
-}
-
-#define REG_CP_LOAD_STATE4_2					0x00000002
-#define CP_LOAD_STATE4_2_EXT_SRC_ADDR_HI__MASK			0xffffffff
-#define CP_LOAD_STATE4_2_EXT_SRC_ADDR_HI__SHIFT			0
-static inline uint32_t CP_LOAD_STATE4_2_EXT_SRC_ADDR_HI(uint32_t val)
-{
-	return ((val) << CP_LOAD_STATE4_2_EXT_SRC_ADDR_HI__SHIFT) & CP_LOAD_STATE4_2_EXT_SRC_ADDR_HI__MASK;
-}
-
-#define REG_CP_LOAD_STATE6_0					0x00000000
-#define CP_LOAD_STATE6_0_DST_OFF__MASK				0x00003fff
-#define CP_LOAD_STATE6_0_DST_OFF__SHIFT				0
-static inline uint32_t CP_LOAD_STATE6_0_DST_OFF(uint32_t val)
-{
-	return ((val) << CP_LOAD_STATE6_0_DST_OFF__SHIFT) & CP_LOAD_STATE6_0_DST_OFF__MASK;
-}
-#define CP_LOAD_STATE6_0_STATE_TYPE__MASK			0x0000c000
-#define CP_LOAD_STATE6_0_STATE_TYPE__SHIFT			14
-static inline uint32_t CP_LOAD_STATE6_0_STATE_TYPE(enum a6xx_state_type val)
-{
-	return ((val) << CP_LOAD_STATE6_0_STATE_TYPE__SHIFT) & CP_LOAD_STATE6_0_STATE_TYPE__MASK;
-}
-#define CP_LOAD_STATE6_0_STATE_SRC__MASK			0x00030000
-#define CP_LOAD_STATE6_0_STATE_SRC__SHIFT			16
-static inline uint32_t CP_LOAD_STATE6_0_STATE_SRC(enum a6xx_state_src val)
-{
-	return ((val) << CP_LOAD_STATE6_0_STATE_SRC__SHIFT) & CP_LOAD_STATE6_0_STATE_SRC__MASK;
-}
-#define CP_LOAD_STATE6_0_STATE_BLOCK__MASK			0x003c0000
-#define CP_LOAD_STATE6_0_STATE_BLOCK__SHIFT			18
-static inline uint32_t CP_LOAD_STATE6_0_STATE_BLOCK(enum a6xx_state_block val)
-{
-	return ((val) << CP_LOAD_STATE6_0_STATE_BLOCK__SHIFT) & CP_LOAD_STATE6_0_STATE_BLOCK__MASK;
-}
-#define CP_LOAD_STATE6_0_NUM_UNIT__MASK				0xffc00000
-#define CP_LOAD_STATE6_0_NUM_UNIT__SHIFT			22
-static inline uint32_t CP_LOAD_STATE6_0_NUM_UNIT(uint32_t val)
-{
-	return ((val) << CP_LOAD_STATE6_0_NUM_UNIT__SHIFT) & CP_LOAD_STATE6_0_NUM_UNIT__MASK;
-}
-
-#define REG_CP_LOAD_STATE6_1					0x00000001
-#define CP_LOAD_STATE6_1_EXT_SRC_ADDR__MASK			0xfffffffc
-#define CP_LOAD_STATE6_1_EXT_SRC_ADDR__SHIFT			2
-static inline uint32_t CP_LOAD_STATE6_1_EXT_SRC_ADDR(uint32_t val)
-{
-	assert(!(val & 0x3));
-	return (((val >> 2)) << CP_LOAD_STATE6_1_EXT_SRC_ADDR__SHIFT) & CP_LOAD_STATE6_1_EXT_SRC_ADDR__MASK;
-}
-
-#define REG_CP_LOAD_STATE6_2					0x00000002
-#define CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI__MASK			0xffffffff
-#define CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI__SHIFT			0
-static inline uint32_t CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(uint32_t val)
-{
-	return ((val) << CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI__SHIFT) & CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI__MASK;
-}
-
-#define REG_CP_LOAD_STATE6_EXT_SRC_ADDR				0x00000001
-
-#define REG_CP_DRAW_INDX_0					0x00000000
-#define CP_DRAW_INDX_0_VIZ_QUERY__MASK				0xffffffff
-#define CP_DRAW_INDX_0_VIZ_QUERY__SHIFT				0
-static inline uint32_t CP_DRAW_INDX_0_VIZ_QUERY(uint32_t val)
-{
-	return ((val) << CP_DRAW_INDX_0_VIZ_QUERY__SHIFT) & CP_DRAW_INDX_0_VIZ_QUERY__MASK;
-}
-
-#define REG_CP_DRAW_INDX_1					0x00000001
-#define CP_DRAW_INDX_1_PRIM_TYPE__MASK				0x0000003f
-#define CP_DRAW_INDX_1_PRIM_TYPE__SHIFT				0
-static inline uint32_t CP_DRAW_INDX_1_PRIM_TYPE(enum pc_di_primtype val)
-{
-	return ((val) << CP_DRAW_INDX_1_PRIM_TYPE__SHIFT) & CP_DRAW_INDX_1_PRIM_TYPE__MASK;
-}
-#define CP_DRAW_INDX_1_SOURCE_SELECT__MASK			0x000000c0
-#define CP_DRAW_INDX_1_SOURCE_SELECT__SHIFT			6
-static inline uint32_t CP_DRAW_INDX_1_SOURCE_SELECT(enum pc_di_src_sel val)
-{
-	return ((val) << CP_DRAW_INDX_1_SOURCE_SELECT__SHIFT) & CP_DRAW_INDX_1_SOURCE_SELECT__MASK;
-}
-#define CP_DRAW_INDX_1_VIS_CULL__MASK				0x00000600
-#define CP_DRAW_INDX_1_VIS_CULL__SHIFT				9
-static inline uint32_t CP_DRAW_INDX_1_VIS_CULL(enum pc_di_vis_cull_mode val)
-{
-	return ((val) << CP_DRAW_INDX_1_VIS_CULL__SHIFT) & CP_DRAW_INDX_1_VIS_CULL__MASK;
-}
-#define CP_DRAW_INDX_1_INDEX_SIZE__MASK				0x00000800
-#define CP_DRAW_INDX_1_INDEX_SIZE__SHIFT			11
-static inline uint32_t CP_DRAW_INDX_1_INDEX_SIZE(enum pc_di_index_size val)
-{
-	return ((val) << CP_DRAW_INDX_1_INDEX_SIZE__SHIFT) & CP_DRAW_INDX_1_INDEX_SIZE__MASK;
-}
-#define CP_DRAW_INDX_1_NOT_EOP					0x00001000
-#define CP_DRAW_INDX_1_SMALL_INDEX				0x00002000
-#define CP_DRAW_INDX_1_PRE_DRAW_INITIATOR_ENABLE		0x00004000
-#define CP_DRAW_INDX_1_NUM_INSTANCES__MASK			0xff000000
-#define CP_DRAW_INDX_1_NUM_INSTANCES__SHIFT			24
-static inline uint32_t CP_DRAW_INDX_1_NUM_INSTANCES(uint32_t val)
-{
-	return ((val) << CP_DRAW_INDX_1_NUM_INSTANCES__SHIFT) & CP_DRAW_INDX_1_NUM_INSTANCES__MASK;
-}
-
-#define REG_CP_DRAW_INDX_2					0x00000002
-#define CP_DRAW_INDX_2_NUM_INDICES__MASK			0xffffffff
-#define CP_DRAW_INDX_2_NUM_INDICES__SHIFT			0
-static inline uint32_t CP_DRAW_INDX_2_NUM_INDICES(uint32_t val)
-{
-	return ((val) << CP_DRAW_INDX_2_NUM_INDICES__SHIFT) & CP_DRAW_INDX_2_NUM_INDICES__MASK;
-}
-
-#define REG_CP_DRAW_INDX_3					0x00000003
-#define CP_DRAW_INDX_3_INDX_BASE__MASK				0xffffffff
-#define CP_DRAW_INDX_3_INDX_BASE__SHIFT				0
-static inline uint32_t CP_DRAW_INDX_3_INDX_BASE(uint32_t val)
-{
-	return ((val) << CP_DRAW_INDX_3_INDX_BASE__SHIFT) & CP_DRAW_INDX_3_INDX_BASE__MASK;
-}
-
-#define REG_CP_DRAW_INDX_4					0x00000004
-#define CP_DRAW_INDX_4_INDX_SIZE__MASK				0xffffffff
-#define CP_DRAW_INDX_4_INDX_SIZE__SHIFT				0
-static inline uint32_t CP_DRAW_INDX_4_INDX_SIZE(uint32_t val)
-{
-	return ((val) << CP_DRAW_INDX_4_INDX_SIZE__SHIFT) & CP_DRAW_INDX_4_INDX_SIZE__MASK;
-}
-
-#define REG_CP_DRAW_INDX_2_0					0x00000000
-#define CP_DRAW_INDX_2_0_VIZ_QUERY__MASK			0xffffffff
-#define CP_DRAW_INDX_2_0_VIZ_QUERY__SHIFT			0
-static inline uint32_t CP_DRAW_INDX_2_0_VIZ_QUERY(uint32_t val)
-{
-	return ((val) << CP_DRAW_INDX_2_0_VIZ_QUERY__SHIFT) & CP_DRAW_INDX_2_0_VIZ_QUERY__MASK;
-}
-
-#define REG_CP_DRAW_INDX_2_1					0x00000001
-#define CP_DRAW_INDX_2_1_PRIM_TYPE__MASK			0x0000003f
-#define CP_DRAW_INDX_2_1_PRIM_TYPE__SHIFT			0
-static inline uint32_t CP_DRAW_INDX_2_1_PRIM_TYPE(enum pc_di_primtype val)
-{
-	return ((val) << CP_DRAW_INDX_2_1_PRIM_TYPE__SHIFT) & CP_DRAW_INDX_2_1_PRIM_TYPE__MASK;
-}
-#define CP_DRAW_INDX_2_1_SOURCE_SELECT__MASK			0x000000c0
-#define CP_DRAW_INDX_2_1_SOURCE_SELECT__SHIFT			6
-static inline uint32_t CP_DRAW_INDX_2_1_SOURCE_SELECT(enum pc_di_src_sel val)
-{
-	return ((val) << CP_DRAW_INDX_2_1_SOURCE_SELECT__SHIFT) & CP_DRAW_INDX_2_1_SOURCE_SELECT__MASK;
-}
-#define CP_DRAW_INDX_2_1_VIS_CULL__MASK				0x00000600
-#define CP_DRAW_INDX_2_1_VIS_CULL__SHIFT			9
-static inline uint32_t CP_DRAW_INDX_2_1_VIS_CULL(enum pc_di_vis_cull_mode val)
-{
-	return ((val) << CP_DRAW_INDX_2_1_VIS_CULL__SHIFT) & CP_DRAW_INDX_2_1_VIS_CULL__MASK;
-}
-#define CP_DRAW_INDX_2_1_INDEX_SIZE__MASK			0x00000800
-#define CP_DRAW_INDX_2_1_INDEX_SIZE__SHIFT			11
-static inline uint32_t CP_DRAW_INDX_2_1_INDEX_SIZE(enum pc_di_index_size val)
-{
-	return ((val) << CP_DRAW_INDX_2_1_INDEX_SIZE__SHIFT) & CP_DRAW_INDX_2_1_INDEX_SIZE__MASK;
-}
-#define CP_DRAW_INDX_2_1_NOT_EOP				0x00001000
-#define CP_DRAW_INDX_2_1_SMALL_INDEX				0x00002000
-#define CP_DRAW_INDX_2_1_PRE_DRAW_INITIATOR_ENABLE		0x00004000
-#define CP_DRAW_INDX_2_1_NUM_INSTANCES__MASK			0xff000000
-#define CP_DRAW_INDX_2_1_NUM_INSTANCES__SHIFT			24
-static inline uint32_t CP_DRAW_INDX_2_1_NUM_INSTANCES(uint32_t val)
-{
-	return ((val) << CP_DRAW_INDX_2_1_NUM_INSTANCES__SHIFT) & CP_DRAW_INDX_2_1_NUM_INSTANCES__MASK;
-}
-
-#define REG_CP_DRAW_INDX_2_2					0x00000002
-#define CP_DRAW_INDX_2_2_NUM_INDICES__MASK			0xffffffff
-#define CP_DRAW_INDX_2_2_NUM_INDICES__SHIFT			0
-static inline uint32_t CP_DRAW_INDX_2_2_NUM_INDICES(uint32_t val)
-{
-	return ((val) << CP_DRAW_INDX_2_2_NUM_INDICES__SHIFT) & CP_DRAW_INDX_2_2_NUM_INDICES__MASK;
-}
-
-#define REG_CP_DRAW_INDX_OFFSET_0				0x00000000
-#define CP_DRAW_INDX_OFFSET_0_PRIM_TYPE__MASK			0x0000003f
-#define CP_DRAW_INDX_OFFSET_0_PRIM_TYPE__SHIFT			0
-static inline uint32_t CP_DRAW_INDX_OFFSET_0_PRIM_TYPE(enum pc_di_primtype val)
-{
-	return ((val) << CP_DRAW_INDX_OFFSET_0_PRIM_TYPE__SHIFT) & CP_DRAW_INDX_OFFSET_0_PRIM_TYPE__MASK;
-}
-#define CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT__MASK		0x000000c0
-#define CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT__SHIFT		6
-static inline uint32_t CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(enum pc_di_src_sel val)
-{
-	return ((val) << CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT__SHIFT) & CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT__MASK;
-}
-#define CP_DRAW_INDX_OFFSET_0_VIS_CULL__MASK			0x00000300
-#define CP_DRAW_INDX_OFFSET_0_VIS_CULL__SHIFT			8
-static inline uint32_t CP_DRAW_INDX_OFFSET_0_VIS_CULL(enum pc_di_vis_cull_mode val)
-{
-	return ((val) << CP_DRAW_INDX_OFFSET_0_VIS_CULL__SHIFT) & CP_DRAW_INDX_OFFSET_0_VIS_CULL__MASK;
-}
-#define CP_DRAW_INDX_OFFSET_0_INDEX_SIZE__MASK			0x00000c00
-#define CP_DRAW_INDX_OFFSET_0_INDEX_SIZE__SHIFT			10
-static inline uint32_t CP_DRAW_INDX_OFFSET_0_INDEX_SIZE(enum a4xx_index_size val)
-{
-	return ((val) << CP_DRAW_INDX_OFFSET_0_INDEX_SIZE__SHIFT) & CP_DRAW_INDX_OFFSET_0_INDEX_SIZE__MASK;
-}
-#define CP_DRAW_INDX_OFFSET_0_PATCH_TYPE__MASK			0x00003000
-#define CP_DRAW_INDX_OFFSET_0_PATCH_TYPE__SHIFT			12
-static inline uint32_t CP_DRAW_INDX_OFFSET_0_PATCH_TYPE(enum a6xx_patch_type val)
-{
-	return ((val) << CP_DRAW_INDX_OFFSET_0_PATCH_TYPE__SHIFT) & CP_DRAW_INDX_OFFSET_0_PATCH_TYPE__MASK;
-}
-#define CP_DRAW_INDX_OFFSET_0_GS_ENABLE				0x00010000
-#define CP_DRAW_INDX_OFFSET_0_TESS_ENABLE			0x00020000
-
-#define REG_CP_DRAW_INDX_OFFSET_1				0x00000001
-#define CP_DRAW_INDX_OFFSET_1_NUM_INSTANCES__MASK		0xffffffff
-#define CP_DRAW_INDX_OFFSET_1_NUM_INSTANCES__SHIFT		0
-static inline uint32_t CP_DRAW_INDX_OFFSET_1_NUM_INSTANCES(uint32_t val)
-{
-	return ((val) << CP_DRAW_INDX_OFFSET_1_NUM_INSTANCES__SHIFT) & CP_DRAW_INDX_OFFSET_1_NUM_INSTANCES__MASK;
-}
-
-#define REG_CP_DRAW_INDX_OFFSET_2				0x00000002
-#define CP_DRAW_INDX_OFFSET_2_NUM_INDICES__MASK			0xffffffff
-#define CP_DRAW_INDX_OFFSET_2_NUM_INDICES__SHIFT		0
-static inline uint32_t CP_DRAW_INDX_OFFSET_2_NUM_INDICES(uint32_t val)
-{
-	return ((val) << CP_DRAW_INDX_OFFSET_2_NUM_INDICES__SHIFT) & CP_DRAW_INDX_OFFSET_2_NUM_INDICES__MASK;
-}
-
-#define REG_CP_DRAW_INDX_OFFSET_3				0x00000003
-#define CP_DRAW_INDX_OFFSET_3_FIRST_INDX__MASK			0xffffffff
-#define CP_DRAW_INDX_OFFSET_3_FIRST_INDX__SHIFT			0
-static inline uint32_t CP_DRAW_INDX_OFFSET_3_FIRST_INDX(uint32_t val)
-{
-	return ((val) << CP_DRAW_INDX_OFFSET_3_FIRST_INDX__SHIFT) & CP_DRAW_INDX_OFFSET_3_FIRST_INDX__MASK;
-}
-
-#define REG_A5XX_CP_DRAW_INDX_OFFSET_4				0x00000004
-#define A5XX_CP_DRAW_INDX_OFFSET_4_INDX_BASE_LO__MASK		0xffffffff
-#define A5XX_CP_DRAW_INDX_OFFSET_4_INDX_BASE_LO__SHIFT		0
-static inline uint32_t A5XX_CP_DRAW_INDX_OFFSET_4_INDX_BASE_LO(uint32_t val)
-{
-	return ((val) << A5XX_CP_DRAW_INDX_OFFSET_4_INDX_BASE_LO__SHIFT) & A5XX_CP_DRAW_INDX_OFFSET_4_INDX_BASE_LO__MASK;
-}
-
-#define REG_A5XX_CP_DRAW_INDX_OFFSET_5				0x00000005
-#define A5XX_CP_DRAW_INDX_OFFSET_5_INDX_BASE_HI__MASK		0xffffffff
-#define A5XX_CP_DRAW_INDX_OFFSET_5_INDX_BASE_HI__SHIFT		0
-static inline uint32_t A5XX_CP_DRAW_INDX_OFFSET_5_INDX_BASE_HI(uint32_t val)
-{
-	return ((val) << A5XX_CP_DRAW_INDX_OFFSET_5_INDX_BASE_HI__SHIFT) & A5XX_CP_DRAW_INDX_OFFSET_5_INDX_BASE_HI__MASK;
-}
-
-#define REG_A5XX_CP_DRAW_INDX_OFFSET_INDX_BASE			0x00000004
-
-#define REG_A5XX_CP_DRAW_INDX_OFFSET_6				0x00000006
-#define A5XX_CP_DRAW_INDX_OFFSET_6_MAX_INDICES__MASK		0xffffffff
-#define A5XX_CP_DRAW_INDX_OFFSET_6_MAX_INDICES__SHIFT		0
-static inline uint32_t A5XX_CP_DRAW_INDX_OFFSET_6_MAX_INDICES(uint32_t val)
-{
-	return ((val) << A5XX_CP_DRAW_INDX_OFFSET_6_MAX_INDICES__SHIFT) & A5XX_CP_DRAW_INDX_OFFSET_6_MAX_INDICES__MASK;
-}
-
-#define REG_CP_DRAW_INDX_OFFSET_4				0x00000004
-#define CP_DRAW_INDX_OFFSET_4_INDX_BASE__MASK			0xffffffff
-#define CP_DRAW_INDX_OFFSET_4_INDX_BASE__SHIFT			0
-static inline uint32_t CP_DRAW_INDX_OFFSET_4_INDX_BASE(uint64_t val)
-{
-	return ((val) << CP_DRAW_INDX_OFFSET_4_INDX_BASE__SHIFT) & CP_DRAW_INDX_OFFSET_4_INDX_BASE__MASK;
-}
-
-#define REG_CP_DRAW_INDX_OFFSET_5				0x00000005
-#define CP_DRAW_INDX_OFFSET_5_INDX_SIZE__MASK			0xffffffff
-#define CP_DRAW_INDX_OFFSET_5_INDX_SIZE__SHIFT			0
-static inline uint32_t CP_DRAW_INDX_OFFSET_5_INDX_SIZE(uint32_t val)
-{
-	return ((val) << CP_DRAW_INDX_OFFSET_5_INDX_SIZE__SHIFT) & CP_DRAW_INDX_OFFSET_5_INDX_SIZE__MASK;
-}
-
-#define REG_A4XX_CP_DRAW_INDIRECT_0				0x00000000
-#define A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE__MASK			0x0000003f
-#define A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE__SHIFT		0
-static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE(enum pc_di_primtype val)
-{
-	return ((val) << A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE__MASK;
-}
-#define A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT__MASK		0x000000c0
-#define A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT__SHIFT		6
-static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT(enum pc_di_src_sel val)
-{
-	return ((val) << A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT__MASK;
-}
-#define A4XX_CP_DRAW_INDIRECT_0_VIS_CULL__MASK			0x00000300
-#define A4XX_CP_DRAW_INDIRECT_0_VIS_CULL__SHIFT			8
-static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_VIS_CULL(enum pc_di_vis_cull_mode val)
-{
-	return ((val) << A4XX_CP_DRAW_INDIRECT_0_VIS_CULL__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_VIS_CULL__MASK;
-}
-#define A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE__MASK		0x00000c00
-#define A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE__SHIFT		10
-static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE(enum a4xx_index_size val)
-{
-	return ((val) << A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE__MASK;
-}
-#define A4XX_CP_DRAW_INDIRECT_0_PATCH_TYPE__MASK		0x00003000
-#define A4XX_CP_DRAW_INDIRECT_0_PATCH_TYPE__SHIFT		12
-static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_PATCH_TYPE(enum a6xx_patch_type val)
-{
-	return ((val) << A4XX_CP_DRAW_INDIRECT_0_PATCH_TYPE__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_PATCH_TYPE__MASK;
-}
-#define A4XX_CP_DRAW_INDIRECT_0_GS_ENABLE			0x00010000
-#define A4XX_CP_DRAW_INDIRECT_0_TESS_ENABLE			0x00020000
-
-#define REG_A4XX_CP_DRAW_INDIRECT_1				0x00000001
-#define A4XX_CP_DRAW_INDIRECT_1_INDIRECT__MASK			0xffffffff
-#define A4XX_CP_DRAW_INDIRECT_1_INDIRECT__SHIFT			0
-static inline uint32_t A4XX_CP_DRAW_INDIRECT_1_INDIRECT(uint32_t val)
-{
-	return ((val) << A4XX_CP_DRAW_INDIRECT_1_INDIRECT__SHIFT) & A4XX_CP_DRAW_INDIRECT_1_INDIRECT__MASK;
-}
-
-#define REG_A5XX_CP_DRAW_INDIRECT_1				0x00000001
-#define A5XX_CP_DRAW_INDIRECT_1_INDIRECT_LO__MASK		0xffffffff
-#define A5XX_CP_DRAW_INDIRECT_1_INDIRECT_LO__SHIFT		0
-static inline uint32_t A5XX_CP_DRAW_INDIRECT_1_INDIRECT_LO(uint32_t val)
-{
-	return ((val) << A5XX_CP_DRAW_INDIRECT_1_INDIRECT_LO__SHIFT) & A5XX_CP_DRAW_INDIRECT_1_INDIRECT_LO__MASK;
-}
-
-#define REG_A5XX_CP_DRAW_INDIRECT_2				0x00000002
-#define A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI__MASK		0xffffffff
-#define A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI__SHIFT		0
-static inline uint32_t A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI(uint32_t val)
-{
-	return ((val) << A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI__SHIFT) & A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI__MASK;
-}
-
-#define REG_A5XX_CP_DRAW_INDIRECT_INDIRECT			0x00000001
-
-#define REG_A4XX_CP_DRAW_INDX_INDIRECT_0			0x00000000
-#define A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE__MASK		0x0000003f
-#define A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE__SHIFT		0
-static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE(enum pc_di_primtype val)
-{
-	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE__MASK;
-}
-#define A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT__MASK	0x000000c0
-#define A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT__SHIFT	6
-static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT(enum pc_di_src_sel val)
-{
-	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT__MASK;
-}
-#define A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL__MASK		0x00000300
-#define A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL__SHIFT		8
-static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL(enum pc_di_vis_cull_mode val)
-{
-	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL__MASK;
-}
-#define A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE__MASK		0x00000c00
-#define A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE__SHIFT		10
-static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE(enum a4xx_index_size val)
-{
-	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE__MASK;
-}
-#define A4XX_CP_DRAW_INDX_INDIRECT_0_PATCH_TYPE__MASK		0x00003000
-#define A4XX_CP_DRAW_INDX_INDIRECT_0_PATCH_TYPE__SHIFT		12
-static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_PATCH_TYPE(enum a6xx_patch_type val)
-{
-	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_PATCH_TYPE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_PATCH_TYPE__MASK;
-}
-#define A4XX_CP_DRAW_INDX_INDIRECT_0_GS_ENABLE			0x00010000
-#define A4XX_CP_DRAW_INDX_INDIRECT_0_TESS_ENABLE		0x00020000
-
-#define REG_A4XX_CP_DRAW_INDX_INDIRECT_1			0x00000001
-#define A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE__MASK		0xffffffff
-#define A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE__SHIFT		0
-static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE(uint32_t val)
-{
-	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE__MASK;
-}
-
-#define REG_A4XX_CP_DRAW_INDX_INDIRECT_2			0x00000002
-#define A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE__MASK		0xffffffff
-#define A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE__SHIFT		0
-static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE(uint32_t val)
-{
-	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE__MASK;
-}
-
-#define REG_A4XX_CP_DRAW_INDX_INDIRECT_3			0x00000003
-#define A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT__MASK		0xffffffff
-#define A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT__SHIFT		0
-static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT(uint32_t val)
-{
-	return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT__MASK;
-}
-
-#define REG_A5XX_CP_DRAW_INDX_INDIRECT_1			0x00000001
-#define A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO__MASK		0xffffffff
-#define A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO__SHIFT	0
-static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO(uint32_t val)
-{
-	return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO__MASK;
-}
-
-#define REG_A5XX_CP_DRAW_INDX_INDIRECT_2			0x00000002
-#define A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI__MASK		0xffffffff
-#define A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI__SHIFT	0
-static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI(uint32_t val)
-{
-	return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI__MASK;
-}
-
-#define REG_A5XX_CP_DRAW_INDX_INDIRECT_INDX_BASE		0x00000001
-
-#define REG_A5XX_CP_DRAW_INDX_INDIRECT_3			0x00000003
-#define A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES__MASK		0xffffffff
-#define A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES__SHIFT		0
-static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES(uint32_t val)
-{
-	return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES__MASK;
-}
-
-#define REG_A5XX_CP_DRAW_INDX_INDIRECT_4			0x00000004
-#define A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO__MASK		0xffffffff
-#define A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO__SHIFT		0
-static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO(uint32_t val)
-{
-	return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO__MASK;
-}
-
-#define REG_A5XX_CP_DRAW_INDX_INDIRECT_5			0x00000005
-#define A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI__MASK		0xffffffff
-#define A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI__SHIFT		0
-static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI(uint32_t val)
-{
-	return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI__MASK;
-}
-
-#define REG_A5XX_CP_DRAW_INDX_INDIRECT_INDIRECT			0x00000004
-
-#define REG_A6XX_CP_DRAW_INDIRECT_MULTI_0			0x00000000
-#define A6XX_CP_DRAW_INDIRECT_MULTI_0_PRIM_TYPE__MASK		0x0000003f
-#define A6XX_CP_DRAW_INDIRECT_MULTI_0_PRIM_TYPE__SHIFT		0
-static inline uint32_t A6XX_CP_DRAW_INDIRECT_MULTI_0_PRIM_TYPE(enum pc_di_primtype val)
-{
-	return ((val) << A6XX_CP_DRAW_INDIRECT_MULTI_0_PRIM_TYPE__SHIFT) & A6XX_CP_DRAW_INDIRECT_MULTI_0_PRIM_TYPE__MASK;
-}
-#define A6XX_CP_DRAW_INDIRECT_MULTI_0_SOURCE_SELECT__MASK	0x000000c0
-#define A6XX_CP_DRAW_INDIRECT_MULTI_0_SOURCE_SELECT__SHIFT	6
-static inline uint32_t A6XX_CP_DRAW_INDIRECT_MULTI_0_SOURCE_SELECT(enum pc_di_src_sel val)
-{
-	return ((val) << A6XX_CP_DRAW_INDIRECT_MULTI_0_SOURCE_SELECT__SHIFT) & A6XX_CP_DRAW_INDIRECT_MULTI_0_SOURCE_SELECT__MASK;
-}
-#define A6XX_CP_DRAW_INDIRECT_MULTI_0_VIS_CULL__MASK		0x00000300
-#define A6XX_CP_DRAW_INDIRECT_MULTI_0_VIS_CULL__SHIFT		8
-static inline uint32_t A6XX_CP_DRAW_INDIRECT_MULTI_0_VIS_CULL(enum pc_di_vis_cull_mode val)
-{
-	return ((val) << A6XX_CP_DRAW_INDIRECT_MULTI_0_VIS_CULL__SHIFT) & A6XX_CP_DRAW_INDIRECT_MULTI_0_VIS_CULL__MASK;
-}
-#define A6XX_CP_DRAW_INDIRECT_MULTI_0_INDEX_SIZE__MASK		0x00000c00
-#define A6XX_CP_DRAW_INDIRECT_MULTI_0_INDEX_SIZE__SHIFT		10
-static inline uint32_t A6XX_CP_DRAW_INDIRECT_MULTI_0_INDEX_SIZE(enum a4xx_index_size val)
-{
-	return ((val) << A6XX_CP_DRAW_INDIRECT_MULTI_0_INDEX_SIZE__SHIFT) & A6XX_CP_DRAW_INDIRECT_MULTI_0_INDEX_SIZE__MASK;
-}
-#define A6XX_CP_DRAW_INDIRECT_MULTI_0_PATCH_TYPE__MASK		0x00003000
-#define A6XX_CP_DRAW_INDIRECT_MULTI_0_PATCH_TYPE__SHIFT		12
-static inline uint32_t A6XX_CP_DRAW_INDIRECT_MULTI_0_PATCH_TYPE(enum a6xx_patch_type val)
-{
-	return ((val) << A6XX_CP_DRAW_INDIRECT_MULTI_0_PATCH_TYPE__SHIFT) & A6XX_CP_DRAW_INDIRECT_MULTI_0_PATCH_TYPE__MASK;
-}
-#define A6XX_CP_DRAW_INDIRECT_MULTI_0_GS_ENABLE			0x00010000
-#define A6XX_CP_DRAW_INDIRECT_MULTI_0_TESS_ENABLE		0x00020000
-
-#define REG_A6XX_CP_DRAW_INDIRECT_MULTI_1			0x00000001
-#define A6XX_CP_DRAW_INDIRECT_MULTI_1_OPCODE__MASK		0x0000000f
-#define A6XX_CP_DRAW_INDIRECT_MULTI_1_OPCODE__SHIFT		0
-static inline uint32_t A6XX_CP_DRAW_INDIRECT_MULTI_1_OPCODE(enum a6xx_draw_indirect_opcode val)
-{
-	return ((val) << A6XX_CP_DRAW_INDIRECT_MULTI_1_OPCODE__SHIFT) & A6XX_CP_DRAW_INDIRECT_MULTI_1_OPCODE__MASK;
-}
-#define A6XX_CP_DRAW_INDIRECT_MULTI_1_DST_OFF__MASK		0x003fff00
-#define A6XX_CP_DRAW_INDIRECT_MULTI_1_DST_OFF__SHIFT		8
-static inline uint32_t A6XX_CP_DRAW_INDIRECT_MULTI_1_DST_OFF(uint32_t val)
-{
-	return ((val) << A6XX_CP_DRAW_INDIRECT_MULTI_1_DST_OFF__SHIFT) & A6XX_CP_DRAW_INDIRECT_MULTI_1_DST_OFF__MASK;
-}
-
-#define REG_A6XX_CP_DRAW_INDIRECT_MULTI_DRAW_COUNT		0x00000002
-
-#define REG_INDIRECT_OP_NORMAL_CP_DRAW_INDIRECT_MULTI_INDIRECT	0x00000003
-
-#define REG_INDIRECT_OP_NORMAL_CP_DRAW_INDIRECT_MULTI_STRIDE	0x00000005
-
-#define REG_INDIRECT_OP_INDEXED_CP_DRAW_INDIRECT_MULTI_INDEX	0x00000003
-
-#define REG_INDIRECT_OP_INDEXED_CP_DRAW_INDIRECT_MULTI_MAX_INDICES	0x00000005
-
-#define REG_INDIRECT_OP_INDEXED_CP_DRAW_INDIRECT_MULTI_INDIRECT	0x00000006
-
-#define REG_INDIRECT_OP_INDEXED_CP_DRAW_INDIRECT_MULTI_STRIDE	0x00000008
-
-#define REG_INDIRECT_OP_INDIRECT_COUNT_CP_DRAW_INDIRECT_MULTI_INDIRECT	0x00000003
-
-#define REG_INDIRECT_OP_INDIRECT_COUNT_CP_DRAW_INDIRECT_MULTI_INDIRECT_COUNT	0x00000005
-
-#define REG_INDIRECT_OP_INDIRECT_COUNT_CP_DRAW_INDIRECT_MULTI_STRIDE	0x00000007
-
-#define REG_INDIRECT_OP_INDIRECT_COUNT_INDEXED_CP_DRAW_INDIRECT_MULTI_INDEX	0x00000003
-
-#define REG_INDIRECT_OP_INDIRECT_COUNT_INDEXED_CP_DRAW_INDIRECT_MULTI_MAX_INDICES	0x00000005
-
-#define REG_INDIRECT_OP_INDIRECT_COUNT_INDEXED_CP_DRAW_INDIRECT_MULTI_INDIRECT	0x00000006
-
-#define REG_INDIRECT_OP_INDIRECT_COUNT_INDEXED_CP_DRAW_INDIRECT_MULTI_INDIRECT_COUNT	0x00000008
-
-#define REG_INDIRECT_OP_INDIRECT_COUNT_INDEXED_CP_DRAW_INDIRECT_MULTI_STRIDE	0x0000000a
-
-#define REG_CP_DRAW_AUTO_0					0x00000000
-#define CP_DRAW_AUTO_0_PRIM_TYPE__MASK				0x0000003f
-#define CP_DRAW_AUTO_0_PRIM_TYPE__SHIFT				0
-static inline uint32_t CP_DRAW_AUTO_0_PRIM_TYPE(enum pc_di_primtype val)
-{
-	return ((val) << CP_DRAW_AUTO_0_PRIM_TYPE__SHIFT) & CP_DRAW_AUTO_0_PRIM_TYPE__MASK;
-}
-#define CP_DRAW_AUTO_0_SOURCE_SELECT__MASK			0x000000c0
-#define CP_DRAW_AUTO_0_SOURCE_SELECT__SHIFT			6
-static inline uint32_t CP_DRAW_AUTO_0_SOURCE_SELECT(enum pc_di_src_sel val)
-{
-	return ((val) << CP_DRAW_AUTO_0_SOURCE_SELECT__SHIFT) & CP_DRAW_AUTO_0_SOURCE_SELECT__MASK;
-}
-#define CP_DRAW_AUTO_0_VIS_CULL__MASK				0x00000300
-#define CP_DRAW_AUTO_0_VIS_CULL__SHIFT				8
-static inline uint32_t CP_DRAW_AUTO_0_VIS_CULL(enum pc_di_vis_cull_mode val)
-{
-	return ((val) << CP_DRAW_AUTO_0_VIS_CULL__SHIFT) & CP_DRAW_AUTO_0_VIS_CULL__MASK;
-}
-#define CP_DRAW_AUTO_0_INDEX_SIZE__MASK				0x00000c00
-#define CP_DRAW_AUTO_0_INDEX_SIZE__SHIFT			10
-static inline uint32_t CP_DRAW_AUTO_0_INDEX_SIZE(enum a4xx_index_size val)
-{
-	return ((val) << CP_DRAW_AUTO_0_INDEX_SIZE__SHIFT) & CP_DRAW_AUTO_0_INDEX_SIZE__MASK;
-}
-#define CP_DRAW_AUTO_0_PATCH_TYPE__MASK				0x00003000
-#define CP_DRAW_AUTO_0_PATCH_TYPE__SHIFT			12
-static inline uint32_t CP_DRAW_AUTO_0_PATCH_TYPE(enum a6xx_patch_type val)
-{
-	return ((val) << CP_DRAW_AUTO_0_PATCH_TYPE__SHIFT) & CP_DRAW_AUTO_0_PATCH_TYPE__MASK;
-}
-#define CP_DRAW_AUTO_0_GS_ENABLE				0x00010000
-#define CP_DRAW_AUTO_0_TESS_ENABLE				0x00020000
-
-#define REG_CP_DRAW_AUTO_1					0x00000001
-#define CP_DRAW_AUTO_1_NUM_INSTANCES__MASK			0xffffffff
-#define CP_DRAW_AUTO_1_NUM_INSTANCES__SHIFT			0
-static inline uint32_t CP_DRAW_AUTO_1_NUM_INSTANCES(uint32_t val)
-{
-	return ((val) << CP_DRAW_AUTO_1_NUM_INSTANCES__SHIFT) & CP_DRAW_AUTO_1_NUM_INSTANCES__MASK;
-}
-
-#define REG_CP_DRAW_AUTO_NUM_VERTICES_BASE			0x00000002
-
-#define REG_CP_DRAW_AUTO_4					0x00000004
-#define CP_DRAW_AUTO_4_NUM_VERTICES_OFFSET__MASK		0xffffffff
-#define CP_DRAW_AUTO_4_NUM_VERTICES_OFFSET__SHIFT		0
-static inline uint32_t CP_DRAW_AUTO_4_NUM_VERTICES_OFFSET(uint32_t val)
-{
-	return ((val) << CP_DRAW_AUTO_4_NUM_VERTICES_OFFSET__SHIFT) & CP_DRAW_AUTO_4_NUM_VERTICES_OFFSET__MASK;
-}
-
-#define REG_CP_DRAW_AUTO_5					0x00000005
-#define CP_DRAW_AUTO_5_STRIDE__MASK				0xffffffff
-#define CP_DRAW_AUTO_5_STRIDE__SHIFT				0
-static inline uint32_t CP_DRAW_AUTO_5_STRIDE(uint32_t val)
-{
-	return ((val) << CP_DRAW_AUTO_5_STRIDE__SHIFT) & CP_DRAW_AUTO_5_STRIDE__MASK;
-}
-
-#define REG_CP_DRAW_PRED_ENABLE_GLOBAL_0			0x00000000
-#define CP_DRAW_PRED_ENABLE_GLOBAL_0_ENABLE			0x00000001
-
-#define REG_CP_DRAW_PRED_ENABLE_LOCAL_0				0x00000000
-#define CP_DRAW_PRED_ENABLE_LOCAL_0_ENABLE			0x00000001
-
-#define REG_CP_DRAW_PRED_SET_0					0x00000000
-#define CP_DRAW_PRED_SET_0_SRC__MASK				0x000000f0
-#define CP_DRAW_PRED_SET_0_SRC__SHIFT				4
-static inline uint32_t CP_DRAW_PRED_SET_0_SRC(enum cp_draw_pred_src val)
-{
-	return ((val) << CP_DRAW_PRED_SET_0_SRC__SHIFT) & CP_DRAW_PRED_SET_0_SRC__MASK;
-}
-#define CP_DRAW_PRED_SET_0_TEST__MASK				0x00000100
-#define CP_DRAW_PRED_SET_0_TEST__SHIFT				8
-static inline uint32_t CP_DRAW_PRED_SET_0_TEST(enum cp_draw_pred_test val)
-{
-	return ((val) << CP_DRAW_PRED_SET_0_TEST__SHIFT) & CP_DRAW_PRED_SET_0_TEST__MASK;
-}
-
-#define REG_CP_DRAW_PRED_SET_MEM_ADDR				0x00000001
-
-#define REG_CP_SET_DRAW_STATE_(i0) (0x00000000 + 0x3*(i0))
-
-static inline uint32_t REG_CP_SET_DRAW_STATE__0(uint32_t i0) { return 0x00000000 + 0x3*i0; }
-#define CP_SET_DRAW_STATE__0_COUNT__MASK			0x0000ffff
-#define CP_SET_DRAW_STATE__0_COUNT__SHIFT			0
-static inline uint32_t CP_SET_DRAW_STATE__0_COUNT(uint32_t val)
-{
-	return ((val) << CP_SET_DRAW_STATE__0_COUNT__SHIFT) & CP_SET_DRAW_STATE__0_COUNT__MASK;
-}
-#define CP_SET_DRAW_STATE__0_DIRTY				0x00010000
-#define CP_SET_DRAW_STATE__0_DISABLE				0x00020000
-#define CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS			0x00040000
-#define CP_SET_DRAW_STATE__0_LOAD_IMMED				0x00080000
-#define CP_SET_DRAW_STATE__0_BINNING				0x00100000
-#define CP_SET_DRAW_STATE__0_GMEM				0x00200000
-#define CP_SET_DRAW_STATE__0_SYSMEM				0x00400000
-#define CP_SET_DRAW_STATE__0_GROUP_ID__MASK			0x1f000000
-#define CP_SET_DRAW_STATE__0_GROUP_ID__SHIFT			24
-static inline uint32_t CP_SET_DRAW_STATE__0_GROUP_ID(uint32_t val)
-{
-	return ((val) << CP_SET_DRAW_STATE__0_GROUP_ID__SHIFT) & CP_SET_DRAW_STATE__0_GROUP_ID__MASK;
-}
-
-static inline uint32_t REG_CP_SET_DRAW_STATE__1(uint32_t i0) { return 0x00000001 + 0x3*i0; }
-#define CP_SET_DRAW_STATE__1_ADDR_LO__MASK			0xffffffff
-#define CP_SET_DRAW_STATE__1_ADDR_LO__SHIFT			0
-static inline uint32_t CP_SET_DRAW_STATE__1_ADDR_LO(uint32_t val)
-{
-	return ((val) << CP_SET_DRAW_STATE__1_ADDR_LO__SHIFT) & CP_SET_DRAW_STATE__1_ADDR_LO__MASK;
-}
-
-static inline uint32_t REG_CP_SET_DRAW_STATE__2(uint32_t i0) { return 0x00000002 + 0x3*i0; }
-#define CP_SET_DRAW_STATE__2_ADDR_HI__MASK			0xffffffff
-#define CP_SET_DRAW_STATE__2_ADDR_HI__SHIFT			0
-static inline uint32_t CP_SET_DRAW_STATE__2_ADDR_HI(uint32_t val)
-{
-	return ((val) << CP_SET_DRAW_STATE__2_ADDR_HI__SHIFT) & CP_SET_DRAW_STATE__2_ADDR_HI__MASK;
-}
-
-#define REG_CP_SET_BIN_0					0x00000000
-
-#define REG_CP_SET_BIN_1					0x00000001
-#define CP_SET_BIN_1_X1__MASK					0x0000ffff
-#define CP_SET_BIN_1_X1__SHIFT					0
-static inline uint32_t CP_SET_BIN_1_X1(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_1_X1__SHIFT) & CP_SET_BIN_1_X1__MASK;
-}
-#define CP_SET_BIN_1_Y1__MASK					0xffff0000
-#define CP_SET_BIN_1_Y1__SHIFT					16
-static inline uint32_t CP_SET_BIN_1_Y1(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_1_Y1__SHIFT) & CP_SET_BIN_1_Y1__MASK;
-}
-
-#define REG_CP_SET_BIN_2					0x00000002
-#define CP_SET_BIN_2_X2__MASK					0x0000ffff
-#define CP_SET_BIN_2_X2__SHIFT					0
-static inline uint32_t CP_SET_BIN_2_X2(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_2_X2__SHIFT) & CP_SET_BIN_2_X2__MASK;
-}
-#define CP_SET_BIN_2_Y2__MASK					0xffff0000
-#define CP_SET_BIN_2_Y2__SHIFT					16
-static inline uint32_t CP_SET_BIN_2_Y2(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_2_Y2__SHIFT) & CP_SET_BIN_2_Y2__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA_0					0x00000000
-#define CP_SET_BIN_DATA_0_BIN_DATA_ADDR__MASK			0xffffffff
-#define CP_SET_BIN_DATA_0_BIN_DATA_ADDR__SHIFT			0
-static inline uint32_t CP_SET_BIN_DATA_0_BIN_DATA_ADDR(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA_0_BIN_DATA_ADDR__SHIFT) & CP_SET_BIN_DATA_0_BIN_DATA_ADDR__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA_1					0x00000001
-#define CP_SET_BIN_DATA_1_BIN_SIZE_ADDRESS__MASK		0xffffffff
-#define CP_SET_BIN_DATA_1_BIN_SIZE_ADDRESS__SHIFT		0
-static inline uint32_t CP_SET_BIN_DATA_1_BIN_SIZE_ADDRESS(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA_1_BIN_SIZE_ADDRESS__SHIFT) & CP_SET_BIN_DATA_1_BIN_SIZE_ADDRESS__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA5_0					0x00000000
-#define CP_SET_BIN_DATA5_0_VSC_SIZE__MASK			0x003f0000
-#define CP_SET_BIN_DATA5_0_VSC_SIZE__SHIFT			16
-static inline uint32_t CP_SET_BIN_DATA5_0_VSC_SIZE(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_0_VSC_SIZE__SHIFT) & CP_SET_BIN_DATA5_0_VSC_SIZE__MASK;
-}
-#define CP_SET_BIN_DATA5_0_VSC_N__MASK				0x07c00000
-#define CP_SET_BIN_DATA5_0_VSC_N__SHIFT				22
-static inline uint32_t CP_SET_BIN_DATA5_0_VSC_N(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_0_VSC_N__SHIFT) & CP_SET_BIN_DATA5_0_VSC_N__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA5_1					0x00000001
-#define CP_SET_BIN_DATA5_1_BIN_DATA_ADDR_LO__MASK		0xffffffff
-#define CP_SET_BIN_DATA5_1_BIN_DATA_ADDR_LO__SHIFT		0
-static inline uint32_t CP_SET_BIN_DATA5_1_BIN_DATA_ADDR_LO(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_1_BIN_DATA_ADDR_LO__SHIFT) & CP_SET_BIN_DATA5_1_BIN_DATA_ADDR_LO__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA5_2					0x00000002
-#define CP_SET_BIN_DATA5_2_BIN_DATA_ADDR_HI__MASK		0xffffffff
-#define CP_SET_BIN_DATA5_2_BIN_DATA_ADDR_HI__SHIFT		0
-static inline uint32_t CP_SET_BIN_DATA5_2_BIN_DATA_ADDR_HI(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_2_BIN_DATA_ADDR_HI__SHIFT) & CP_SET_BIN_DATA5_2_BIN_DATA_ADDR_HI__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA5_3					0x00000003
-#define CP_SET_BIN_DATA5_3_BIN_SIZE_ADDRESS_LO__MASK		0xffffffff
-#define CP_SET_BIN_DATA5_3_BIN_SIZE_ADDRESS_LO__SHIFT		0
-static inline uint32_t CP_SET_BIN_DATA5_3_BIN_SIZE_ADDRESS_LO(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_3_BIN_SIZE_ADDRESS_LO__SHIFT) & CP_SET_BIN_DATA5_3_BIN_SIZE_ADDRESS_LO__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA5_4					0x00000004
-#define CP_SET_BIN_DATA5_4_BIN_SIZE_ADDRESS_HI__MASK		0xffffffff
-#define CP_SET_BIN_DATA5_4_BIN_SIZE_ADDRESS_HI__SHIFT		0
-static inline uint32_t CP_SET_BIN_DATA5_4_BIN_SIZE_ADDRESS_HI(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_4_BIN_SIZE_ADDRESS_HI__SHIFT) & CP_SET_BIN_DATA5_4_BIN_SIZE_ADDRESS_HI__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA5_5					0x00000005
-#define CP_SET_BIN_DATA5_5_BIN_PRIM_STRM_LO__MASK		0xffffffff
-#define CP_SET_BIN_DATA5_5_BIN_PRIM_STRM_LO__SHIFT		0
-static inline uint32_t CP_SET_BIN_DATA5_5_BIN_PRIM_STRM_LO(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_5_BIN_PRIM_STRM_LO__SHIFT) & CP_SET_BIN_DATA5_5_BIN_PRIM_STRM_LO__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA5_6					0x00000006
-#define CP_SET_BIN_DATA5_6_BIN_PRIM_STRM_HI__MASK		0xffffffff
-#define CP_SET_BIN_DATA5_6_BIN_PRIM_STRM_HI__SHIFT		0
-static inline uint32_t CP_SET_BIN_DATA5_6_BIN_PRIM_STRM_HI(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_6_BIN_PRIM_STRM_HI__SHIFT) & CP_SET_BIN_DATA5_6_BIN_PRIM_STRM_HI__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA5_7					0x00000007
-
-#define REG_CP_SET_BIN_DATA5_9					0x00000009
-
-#define REG_CP_SET_BIN_DATA5_OFFSET_0				0x00000000
-#define CP_SET_BIN_DATA5_OFFSET_0_VSC_SIZE__MASK		0x003f0000
-#define CP_SET_BIN_DATA5_OFFSET_0_VSC_SIZE__SHIFT		16
-static inline uint32_t CP_SET_BIN_DATA5_OFFSET_0_VSC_SIZE(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_OFFSET_0_VSC_SIZE__SHIFT) & CP_SET_BIN_DATA5_OFFSET_0_VSC_SIZE__MASK;
-}
-#define CP_SET_BIN_DATA5_OFFSET_0_VSC_N__MASK			0x07c00000
-#define CP_SET_BIN_DATA5_OFFSET_0_VSC_N__SHIFT			22
-static inline uint32_t CP_SET_BIN_DATA5_OFFSET_0_VSC_N(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_OFFSET_0_VSC_N__SHIFT) & CP_SET_BIN_DATA5_OFFSET_0_VSC_N__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA5_OFFSET_1				0x00000001
-#define CP_SET_BIN_DATA5_OFFSET_1_BIN_DATA_OFFSET__MASK		0xffffffff
-#define CP_SET_BIN_DATA5_OFFSET_1_BIN_DATA_OFFSET__SHIFT	0
-static inline uint32_t CP_SET_BIN_DATA5_OFFSET_1_BIN_DATA_OFFSET(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_OFFSET_1_BIN_DATA_OFFSET__SHIFT) & CP_SET_BIN_DATA5_OFFSET_1_BIN_DATA_OFFSET__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA5_OFFSET_2				0x00000002
-#define CP_SET_BIN_DATA5_OFFSET_2_BIN_SIZE_OFFSET__MASK		0xffffffff
-#define CP_SET_BIN_DATA5_OFFSET_2_BIN_SIZE_OFFSET__SHIFT	0
-static inline uint32_t CP_SET_BIN_DATA5_OFFSET_2_BIN_SIZE_OFFSET(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_OFFSET_2_BIN_SIZE_OFFSET__SHIFT) & CP_SET_BIN_DATA5_OFFSET_2_BIN_SIZE_OFFSET__MASK;
-}
-
-#define REG_CP_SET_BIN_DATA5_OFFSET_3				0x00000003
-#define CP_SET_BIN_DATA5_OFFSET_3_BIN_DATA2_OFFSET__MASK	0xffffffff
-#define CP_SET_BIN_DATA5_OFFSET_3_BIN_DATA2_OFFSET__SHIFT	0
-static inline uint32_t CP_SET_BIN_DATA5_OFFSET_3_BIN_DATA2_OFFSET(uint32_t val)
-{
-	return ((val) << CP_SET_BIN_DATA5_OFFSET_3_BIN_DATA2_OFFSET__SHIFT) & CP_SET_BIN_DATA5_OFFSET_3_BIN_DATA2_OFFSET__MASK;
-}
-
-#define REG_CP_REG_RMW_0					0x00000000
-#define CP_REG_RMW_0_DST_REG__MASK				0x0003ffff
-#define CP_REG_RMW_0_DST_REG__SHIFT				0
-static inline uint32_t CP_REG_RMW_0_DST_REG(uint32_t val)
-{
-	return ((val) << CP_REG_RMW_0_DST_REG__SHIFT) & CP_REG_RMW_0_DST_REG__MASK;
-}
-#define CP_REG_RMW_0_ROTATE__MASK				0x1f000000
-#define CP_REG_RMW_0_ROTATE__SHIFT				24
-static inline uint32_t CP_REG_RMW_0_ROTATE(uint32_t val)
-{
-	return ((val) << CP_REG_RMW_0_ROTATE__SHIFT) & CP_REG_RMW_0_ROTATE__MASK;
-}
-#define CP_REG_RMW_0_SRC1_ADD					0x20000000
-#define CP_REG_RMW_0_SRC1_IS_REG				0x40000000
-#define CP_REG_RMW_0_SRC0_IS_REG				0x80000000
-
-#define REG_CP_REG_RMW_1					0x00000001
-#define CP_REG_RMW_1_SRC0__MASK					0xffffffff
-#define CP_REG_RMW_1_SRC0__SHIFT				0
-static inline uint32_t CP_REG_RMW_1_SRC0(uint32_t val)
-{
-	return ((val) << CP_REG_RMW_1_SRC0__SHIFT) & CP_REG_RMW_1_SRC0__MASK;
-}
-
-#define REG_CP_REG_RMW_2					0x00000002
-#define CP_REG_RMW_2_SRC1__MASK					0xffffffff
-#define CP_REG_RMW_2_SRC1__SHIFT				0
-static inline uint32_t CP_REG_RMW_2_SRC1(uint32_t val)
-{
-	return ((val) << CP_REG_RMW_2_SRC1__SHIFT) & CP_REG_RMW_2_SRC1__MASK;
-}
-
-#define REG_CP_REG_TO_MEM_0					0x00000000
-#define CP_REG_TO_MEM_0_REG__MASK				0x0003ffff
-#define CP_REG_TO_MEM_0_REG__SHIFT				0
-static inline uint32_t CP_REG_TO_MEM_0_REG(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_0_REG__SHIFT) & CP_REG_TO_MEM_0_REG__MASK;
-}
-#define CP_REG_TO_MEM_0_CNT__MASK				0x3ffc0000
-#define CP_REG_TO_MEM_0_CNT__SHIFT				18
-static inline uint32_t CP_REG_TO_MEM_0_CNT(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_0_CNT__SHIFT) & CP_REG_TO_MEM_0_CNT__MASK;
-}
-#define CP_REG_TO_MEM_0_64B					0x40000000
-#define CP_REG_TO_MEM_0_ACCUMULATE				0x80000000
-
-#define REG_CP_REG_TO_MEM_1					0x00000001
-#define CP_REG_TO_MEM_1_DEST__MASK				0xffffffff
-#define CP_REG_TO_MEM_1_DEST__SHIFT				0
-static inline uint32_t CP_REG_TO_MEM_1_DEST(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_1_DEST__SHIFT) & CP_REG_TO_MEM_1_DEST__MASK;
-}
-
-#define REG_CP_REG_TO_MEM_2					0x00000002
-#define CP_REG_TO_MEM_2_DEST_HI__MASK				0xffffffff
-#define CP_REG_TO_MEM_2_DEST_HI__SHIFT				0
-static inline uint32_t CP_REG_TO_MEM_2_DEST_HI(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_2_DEST_HI__SHIFT) & CP_REG_TO_MEM_2_DEST_HI__MASK;
-}
-
-#define REG_CP_REG_TO_MEM_OFFSET_REG_0				0x00000000
-#define CP_REG_TO_MEM_OFFSET_REG_0_REG__MASK			0x0003ffff
-#define CP_REG_TO_MEM_OFFSET_REG_0_REG__SHIFT			0
-static inline uint32_t CP_REG_TO_MEM_OFFSET_REG_0_REG(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_OFFSET_REG_0_REG__SHIFT) & CP_REG_TO_MEM_OFFSET_REG_0_REG__MASK;
-}
-#define CP_REG_TO_MEM_OFFSET_REG_0_CNT__MASK			0x3ffc0000
-#define CP_REG_TO_MEM_OFFSET_REG_0_CNT__SHIFT			18
-static inline uint32_t CP_REG_TO_MEM_OFFSET_REG_0_CNT(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_OFFSET_REG_0_CNT__SHIFT) & CP_REG_TO_MEM_OFFSET_REG_0_CNT__MASK;
-}
-#define CP_REG_TO_MEM_OFFSET_REG_0_64B				0x40000000
-#define CP_REG_TO_MEM_OFFSET_REG_0_ACCUMULATE			0x80000000
-
-#define REG_CP_REG_TO_MEM_OFFSET_REG_1				0x00000001
-#define CP_REG_TO_MEM_OFFSET_REG_1_DEST__MASK			0xffffffff
-#define CP_REG_TO_MEM_OFFSET_REG_1_DEST__SHIFT			0
-static inline uint32_t CP_REG_TO_MEM_OFFSET_REG_1_DEST(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_OFFSET_REG_1_DEST__SHIFT) & CP_REG_TO_MEM_OFFSET_REG_1_DEST__MASK;
-}
-
-#define REG_CP_REG_TO_MEM_OFFSET_REG_2				0x00000002
-#define CP_REG_TO_MEM_OFFSET_REG_2_DEST_HI__MASK		0xffffffff
-#define CP_REG_TO_MEM_OFFSET_REG_2_DEST_HI__SHIFT		0
-static inline uint32_t CP_REG_TO_MEM_OFFSET_REG_2_DEST_HI(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_OFFSET_REG_2_DEST_HI__SHIFT) & CP_REG_TO_MEM_OFFSET_REG_2_DEST_HI__MASK;
-}
-
-#define REG_CP_REG_TO_MEM_OFFSET_REG_3				0x00000003
-#define CP_REG_TO_MEM_OFFSET_REG_3_OFFSET0__MASK		0x0003ffff
-#define CP_REG_TO_MEM_OFFSET_REG_3_OFFSET0__SHIFT		0
-static inline uint32_t CP_REG_TO_MEM_OFFSET_REG_3_OFFSET0(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_OFFSET_REG_3_OFFSET0__SHIFT) & CP_REG_TO_MEM_OFFSET_REG_3_OFFSET0__MASK;
-}
-#define CP_REG_TO_MEM_OFFSET_REG_3_OFFSET0_SCRATCH		0x00080000
-
-#define REG_CP_REG_TO_MEM_OFFSET_MEM_0				0x00000000
-#define CP_REG_TO_MEM_OFFSET_MEM_0_REG__MASK			0x0003ffff
-#define CP_REG_TO_MEM_OFFSET_MEM_0_REG__SHIFT			0
-static inline uint32_t CP_REG_TO_MEM_OFFSET_MEM_0_REG(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_OFFSET_MEM_0_REG__SHIFT) & CP_REG_TO_MEM_OFFSET_MEM_0_REG__MASK;
-}
-#define CP_REG_TO_MEM_OFFSET_MEM_0_CNT__MASK			0x3ffc0000
-#define CP_REG_TO_MEM_OFFSET_MEM_0_CNT__SHIFT			18
-static inline uint32_t CP_REG_TO_MEM_OFFSET_MEM_0_CNT(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_OFFSET_MEM_0_CNT__SHIFT) & CP_REG_TO_MEM_OFFSET_MEM_0_CNT__MASK;
-}
-#define CP_REG_TO_MEM_OFFSET_MEM_0_64B				0x40000000
-#define CP_REG_TO_MEM_OFFSET_MEM_0_ACCUMULATE			0x80000000
-
-#define REG_CP_REG_TO_MEM_OFFSET_MEM_1				0x00000001
-#define CP_REG_TO_MEM_OFFSET_MEM_1_DEST__MASK			0xffffffff
-#define CP_REG_TO_MEM_OFFSET_MEM_1_DEST__SHIFT			0
-static inline uint32_t CP_REG_TO_MEM_OFFSET_MEM_1_DEST(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_OFFSET_MEM_1_DEST__SHIFT) & CP_REG_TO_MEM_OFFSET_MEM_1_DEST__MASK;
-}
-
-#define REG_CP_REG_TO_MEM_OFFSET_MEM_2				0x00000002
-#define CP_REG_TO_MEM_OFFSET_MEM_2_DEST_HI__MASK		0xffffffff
-#define CP_REG_TO_MEM_OFFSET_MEM_2_DEST_HI__SHIFT		0
-static inline uint32_t CP_REG_TO_MEM_OFFSET_MEM_2_DEST_HI(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_OFFSET_MEM_2_DEST_HI__SHIFT) & CP_REG_TO_MEM_OFFSET_MEM_2_DEST_HI__MASK;
-}
-
-#define REG_CP_REG_TO_MEM_OFFSET_MEM_3				0x00000003
-#define CP_REG_TO_MEM_OFFSET_MEM_3_OFFSET_LO__MASK		0xffffffff
-#define CP_REG_TO_MEM_OFFSET_MEM_3_OFFSET_LO__SHIFT		0
-static inline uint32_t CP_REG_TO_MEM_OFFSET_MEM_3_OFFSET_LO(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_OFFSET_MEM_3_OFFSET_LO__SHIFT) & CP_REG_TO_MEM_OFFSET_MEM_3_OFFSET_LO__MASK;
-}
-
-#define REG_CP_REG_TO_MEM_OFFSET_MEM_4				0x00000004
-#define CP_REG_TO_MEM_OFFSET_MEM_4_OFFSET_HI__MASK		0xffffffff
-#define CP_REG_TO_MEM_OFFSET_MEM_4_OFFSET_HI__SHIFT		0
-static inline uint32_t CP_REG_TO_MEM_OFFSET_MEM_4_OFFSET_HI(uint32_t val)
-{
-	return ((val) << CP_REG_TO_MEM_OFFSET_MEM_4_OFFSET_HI__SHIFT) & CP_REG_TO_MEM_OFFSET_MEM_4_OFFSET_HI__MASK;
-}
-
-#define REG_CP_MEM_TO_REG_0					0x00000000
-#define CP_MEM_TO_REG_0_REG__MASK				0x0003ffff
-#define CP_MEM_TO_REG_0_REG__SHIFT				0
-static inline uint32_t CP_MEM_TO_REG_0_REG(uint32_t val)
-{
-	return ((val) << CP_MEM_TO_REG_0_REG__SHIFT) & CP_MEM_TO_REG_0_REG__MASK;
-}
-#define CP_MEM_TO_REG_0_CNT__MASK				0x3ff80000
-#define CP_MEM_TO_REG_0_CNT__SHIFT				19
-static inline uint32_t CP_MEM_TO_REG_0_CNT(uint32_t val)
-{
-	return ((val) << CP_MEM_TO_REG_0_CNT__SHIFT) & CP_MEM_TO_REG_0_CNT__MASK;
-}
-#define CP_MEM_TO_REG_0_SHIFT_BY_2				0x40000000
-#define CP_MEM_TO_REG_0_UNK31					0x80000000
-
-#define REG_CP_MEM_TO_REG_1					0x00000001
-#define CP_MEM_TO_REG_1_SRC__MASK				0xffffffff
-#define CP_MEM_TO_REG_1_SRC__SHIFT				0
-static inline uint32_t CP_MEM_TO_REG_1_SRC(uint32_t val)
-{
-	return ((val) << CP_MEM_TO_REG_1_SRC__SHIFT) & CP_MEM_TO_REG_1_SRC__MASK;
-}
-
-#define REG_CP_MEM_TO_REG_2					0x00000002
-#define CP_MEM_TO_REG_2_SRC_HI__MASK				0xffffffff
-#define CP_MEM_TO_REG_2_SRC_HI__SHIFT				0
-static inline uint32_t CP_MEM_TO_REG_2_SRC_HI(uint32_t val)
-{
-	return ((val) << CP_MEM_TO_REG_2_SRC_HI__SHIFT) & CP_MEM_TO_REG_2_SRC_HI__MASK;
-}
-
-#define REG_CP_MEM_TO_MEM_0					0x00000000
-#define CP_MEM_TO_MEM_0_NEG_A					0x00000001
-#define CP_MEM_TO_MEM_0_NEG_B					0x00000002
-#define CP_MEM_TO_MEM_0_NEG_C					0x00000004
-#define CP_MEM_TO_MEM_0_DOUBLE					0x20000000
-#define CP_MEM_TO_MEM_0_WAIT_FOR_MEM_WRITES			0x40000000
-#define CP_MEM_TO_MEM_0_UNK31					0x80000000
-
-#define REG_CP_MEMCPY_0						0x00000000
-#define CP_MEMCPY_0_DWORDS__MASK				0xffffffff
-#define CP_MEMCPY_0_DWORDS__SHIFT				0
-static inline uint32_t CP_MEMCPY_0_DWORDS(uint32_t val)
-{
-	return ((val) << CP_MEMCPY_0_DWORDS__SHIFT) & CP_MEMCPY_0_DWORDS__MASK;
-}
-
-#define REG_CP_MEMCPY_1						0x00000001
-#define CP_MEMCPY_1_SRC_LO__MASK				0xffffffff
-#define CP_MEMCPY_1_SRC_LO__SHIFT				0
-static inline uint32_t CP_MEMCPY_1_SRC_LO(uint32_t val)
-{
-	return ((val) << CP_MEMCPY_1_SRC_LO__SHIFT) & CP_MEMCPY_1_SRC_LO__MASK;
-}
-
-#define REG_CP_MEMCPY_2						0x00000002
-#define CP_MEMCPY_2_SRC_HI__MASK				0xffffffff
-#define CP_MEMCPY_2_SRC_HI__SHIFT				0
-static inline uint32_t CP_MEMCPY_2_SRC_HI(uint32_t val)
-{
-	return ((val) << CP_MEMCPY_2_SRC_HI__SHIFT) & CP_MEMCPY_2_SRC_HI__MASK;
-}
-
-#define REG_CP_MEMCPY_3						0x00000003
-#define CP_MEMCPY_3_DST_LO__MASK				0xffffffff
-#define CP_MEMCPY_3_DST_LO__SHIFT				0
-static inline uint32_t CP_MEMCPY_3_DST_LO(uint32_t val)
-{
-	return ((val) << CP_MEMCPY_3_DST_LO__SHIFT) & CP_MEMCPY_3_DST_LO__MASK;
-}
-
-#define REG_CP_MEMCPY_4						0x00000004
-#define CP_MEMCPY_4_DST_HI__MASK				0xffffffff
-#define CP_MEMCPY_4_DST_HI__SHIFT				0
-static inline uint32_t CP_MEMCPY_4_DST_HI(uint32_t val)
-{
-	return ((val) << CP_MEMCPY_4_DST_HI__SHIFT) & CP_MEMCPY_4_DST_HI__MASK;
-}
-
-#define REG_CP_REG_TO_SCRATCH_0					0x00000000
-#define CP_REG_TO_SCRATCH_0_REG__MASK				0x0003ffff
-#define CP_REG_TO_SCRATCH_0_REG__SHIFT				0
-static inline uint32_t CP_REG_TO_SCRATCH_0_REG(uint32_t val)
-{
-	return ((val) << CP_REG_TO_SCRATCH_0_REG__SHIFT) & CP_REG_TO_SCRATCH_0_REG__MASK;
-}
-#define CP_REG_TO_SCRATCH_0_SCRATCH__MASK			0x00700000
-#define CP_REG_TO_SCRATCH_0_SCRATCH__SHIFT			20
-static inline uint32_t CP_REG_TO_SCRATCH_0_SCRATCH(uint32_t val)
-{
-	return ((val) << CP_REG_TO_SCRATCH_0_SCRATCH__SHIFT) & CP_REG_TO_SCRATCH_0_SCRATCH__MASK;
-}
-#define CP_REG_TO_SCRATCH_0_CNT__MASK				0x07000000
-#define CP_REG_TO_SCRATCH_0_CNT__SHIFT				24
-static inline uint32_t CP_REG_TO_SCRATCH_0_CNT(uint32_t val)
-{
-	return ((val) << CP_REG_TO_SCRATCH_0_CNT__SHIFT) & CP_REG_TO_SCRATCH_0_CNT__MASK;
-}
-
-#define REG_CP_SCRATCH_TO_REG_0					0x00000000
-#define CP_SCRATCH_TO_REG_0_REG__MASK				0x0003ffff
-#define CP_SCRATCH_TO_REG_0_REG__SHIFT				0
-static inline uint32_t CP_SCRATCH_TO_REG_0_REG(uint32_t val)
-{
-	return ((val) << CP_SCRATCH_TO_REG_0_REG__SHIFT) & CP_SCRATCH_TO_REG_0_REG__MASK;
-}
-#define CP_SCRATCH_TO_REG_0_UNK18				0x00040000
-#define CP_SCRATCH_TO_REG_0_SCRATCH__MASK			0x00700000
-#define CP_SCRATCH_TO_REG_0_SCRATCH__SHIFT			20
-static inline uint32_t CP_SCRATCH_TO_REG_0_SCRATCH(uint32_t val)
-{
-	return ((val) << CP_SCRATCH_TO_REG_0_SCRATCH__SHIFT) & CP_SCRATCH_TO_REG_0_SCRATCH__MASK;
-}
-#define CP_SCRATCH_TO_REG_0_CNT__MASK				0x07000000
-#define CP_SCRATCH_TO_REG_0_CNT__SHIFT				24
-static inline uint32_t CP_SCRATCH_TO_REG_0_CNT(uint32_t val)
-{
-	return ((val) << CP_SCRATCH_TO_REG_0_CNT__SHIFT) & CP_SCRATCH_TO_REG_0_CNT__MASK;
-}
-
-#define REG_CP_SCRATCH_WRITE_0					0x00000000
-#define CP_SCRATCH_WRITE_0_SCRATCH__MASK			0x00700000
-#define CP_SCRATCH_WRITE_0_SCRATCH__SHIFT			20
-static inline uint32_t CP_SCRATCH_WRITE_0_SCRATCH(uint32_t val)
-{
-	return ((val) << CP_SCRATCH_WRITE_0_SCRATCH__SHIFT) & CP_SCRATCH_WRITE_0_SCRATCH__MASK;
-}
-
-#define REG_CP_MEM_WRITE_0					0x00000000
-#define CP_MEM_WRITE_0_ADDR_LO__MASK				0xffffffff
-#define CP_MEM_WRITE_0_ADDR_LO__SHIFT				0
-static inline uint32_t CP_MEM_WRITE_0_ADDR_LO(uint32_t val)
-{
-	return ((val) << CP_MEM_WRITE_0_ADDR_LO__SHIFT) & CP_MEM_WRITE_0_ADDR_LO__MASK;
-}
-
-#define REG_CP_MEM_WRITE_1					0x00000001
-#define CP_MEM_WRITE_1_ADDR_HI__MASK				0xffffffff
-#define CP_MEM_WRITE_1_ADDR_HI__SHIFT				0
-static inline uint32_t CP_MEM_WRITE_1_ADDR_HI(uint32_t val)
-{
-	return ((val) << CP_MEM_WRITE_1_ADDR_HI__SHIFT) & CP_MEM_WRITE_1_ADDR_HI__MASK;
-}
-
-#define REG_CP_COND_WRITE_0					0x00000000
-#define CP_COND_WRITE_0_FUNCTION__MASK				0x00000007
-#define CP_COND_WRITE_0_FUNCTION__SHIFT				0
-static inline uint32_t CP_COND_WRITE_0_FUNCTION(enum cp_cond_function val)
-{
-	return ((val) << CP_COND_WRITE_0_FUNCTION__SHIFT) & CP_COND_WRITE_0_FUNCTION__MASK;
-}
-#define CP_COND_WRITE_0_POLL_MEMORY				0x00000010
-#define CP_COND_WRITE_0_WRITE_MEMORY				0x00000100
-
-#define REG_CP_COND_WRITE_1					0x00000001
-#define CP_COND_WRITE_1_POLL_ADDR__MASK				0xffffffff
-#define CP_COND_WRITE_1_POLL_ADDR__SHIFT			0
-static inline uint32_t CP_COND_WRITE_1_POLL_ADDR(uint32_t val)
-{
-	return ((val) << CP_COND_WRITE_1_POLL_ADDR__SHIFT) & CP_COND_WRITE_1_POLL_ADDR__MASK;
-}
-
-#define REG_CP_COND_WRITE_2					0x00000002
-#define CP_COND_WRITE_2_REF__MASK				0xffffffff
-#define CP_COND_WRITE_2_REF__SHIFT				0
-static inline uint32_t CP_COND_WRITE_2_REF(uint32_t val)
-{
-	return ((val) << CP_COND_WRITE_2_REF__SHIFT) & CP_COND_WRITE_2_REF__MASK;
-}
-
-#define REG_CP_COND_WRITE_3					0x00000003
-#define CP_COND_WRITE_3_MASK__MASK				0xffffffff
-#define CP_COND_WRITE_3_MASK__SHIFT				0
-static inline uint32_t CP_COND_WRITE_3_MASK(uint32_t val)
-{
-	return ((val) << CP_COND_WRITE_3_MASK__SHIFT) & CP_COND_WRITE_3_MASK__MASK;
-}
-
-#define REG_CP_COND_WRITE_4					0x00000004
-#define CP_COND_WRITE_4_WRITE_ADDR__MASK			0xffffffff
-#define CP_COND_WRITE_4_WRITE_ADDR__SHIFT			0
-static inline uint32_t CP_COND_WRITE_4_WRITE_ADDR(uint32_t val)
-{
-	return ((val) << CP_COND_WRITE_4_WRITE_ADDR__SHIFT) & CP_COND_WRITE_4_WRITE_ADDR__MASK;
-}
-
-#define REG_CP_COND_WRITE_5					0x00000005
-#define CP_COND_WRITE_5_WRITE_DATA__MASK			0xffffffff
-#define CP_COND_WRITE_5_WRITE_DATA__SHIFT			0
-static inline uint32_t CP_COND_WRITE_5_WRITE_DATA(uint32_t val)
-{
-	return ((val) << CP_COND_WRITE_5_WRITE_DATA__SHIFT) & CP_COND_WRITE_5_WRITE_DATA__MASK;
-}
-
-#define REG_CP_COND_WRITE5_0					0x00000000
-#define CP_COND_WRITE5_0_FUNCTION__MASK				0x00000007
-#define CP_COND_WRITE5_0_FUNCTION__SHIFT			0
-static inline uint32_t CP_COND_WRITE5_0_FUNCTION(enum cp_cond_function val)
-{
-	return ((val) << CP_COND_WRITE5_0_FUNCTION__SHIFT) & CP_COND_WRITE5_0_FUNCTION__MASK;
-}
-#define CP_COND_WRITE5_0_SIGNED_COMPARE				0x00000008
-#define CP_COND_WRITE5_0_POLL__MASK				0x00000030
-#define CP_COND_WRITE5_0_POLL__SHIFT				4
-static inline uint32_t CP_COND_WRITE5_0_POLL(enum poll_memory_type val)
-{
-	return ((val) << CP_COND_WRITE5_0_POLL__SHIFT) & CP_COND_WRITE5_0_POLL__MASK;
-}
-#define CP_COND_WRITE5_0_WRITE_MEMORY				0x00000100
-
-#define REG_CP_COND_WRITE5_1					0x00000001
-#define CP_COND_WRITE5_1_POLL_ADDR_LO__MASK			0xffffffff
-#define CP_COND_WRITE5_1_POLL_ADDR_LO__SHIFT			0
-static inline uint32_t CP_COND_WRITE5_1_POLL_ADDR_LO(uint32_t val)
-{
-	return ((val) << CP_COND_WRITE5_1_POLL_ADDR_LO__SHIFT) & CP_COND_WRITE5_1_POLL_ADDR_LO__MASK;
-}
-
-#define REG_CP_COND_WRITE5_2					0x00000002
-#define CP_COND_WRITE5_2_POLL_ADDR_HI__MASK			0xffffffff
-#define CP_COND_WRITE5_2_POLL_ADDR_HI__SHIFT			0
-static inline uint32_t CP_COND_WRITE5_2_POLL_ADDR_HI(uint32_t val)
-{
-	return ((val) << CP_COND_WRITE5_2_POLL_ADDR_HI__SHIFT) & CP_COND_WRITE5_2_POLL_ADDR_HI__MASK;
-}
-
-#define REG_CP_COND_WRITE5_3					0x00000003
-#define CP_COND_WRITE5_3_REF__MASK				0xffffffff
-#define CP_COND_WRITE5_3_REF__SHIFT				0
-static inline uint32_t CP_COND_WRITE5_3_REF(uint32_t val)
-{
-	return ((val) << CP_COND_WRITE5_3_REF__SHIFT) & CP_COND_WRITE5_3_REF__MASK;
-}
-
-#define REG_CP_COND_WRITE5_4					0x00000004
-#define CP_COND_WRITE5_4_MASK__MASK				0xffffffff
-#define CP_COND_WRITE5_4_MASK__SHIFT				0
-static inline uint32_t CP_COND_WRITE5_4_MASK(uint32_t val)
-{
-	return ((val) << CP_COND_WRITE5_4_MASK__SHIFT) & CP_COND_WRITE5_4_MASK__MASK;
-}
-
-#define REG_CP_COND_WRITE5_5					0x00000005
-#define CP_COND_WRITE5_5_WRITE_ADDR_LO__MASK			0xffffffff
-#define CP_COND_WRITE5_5_WRITE_ADDR_LO__SHIFT			0
-static inline uint32_t CP_COND_WRITE5_5_WRITE_ADDR_LO(uint32_t val)
-{
-	return ((val) << CP_COND_WRITE5_5_WRITE_ADDR_LO__SHIFT) & CP_COND_WRITE5_5_WRITE_ADDR_LO__MASK;
-}
-
-#define REG_CP_COND_WRITE5_6					0x00000006
-#define CP_COND_WRITE5_6_WRITE_ADDR_HI__MASK			0xffffffff
-#define CP_COND_WRITE5_6_WRITE_ADDR_HI__SHIFT			0
-static inline uint32_t CP_COND_WRITE5_6_WRITE_ADDR_HI(uint32_t val)
-{
-	return ((val) << CP_COND_WRITE5_6_WRITE_ADDR_HI__SHIFT) & CP_COND_WRITE5_6_WRITE_ADDR_HI__MASK;
-}
-
-#define REG_CP_COND_WRITE5_7					0x00000007
-#define CP_COND_WRITE5_7_WRITE_DATA__MASK			0xffffffff
-#define CP_COND_WRITE5_7_WRITE_DATA__SHIFT			0
-static inline uint32_t CP_COND_WRITE5_7_WRITE_DATA(uint32_t val)
-{
-	return ((val) << CP_COND_WRITE5_7_WRITE_DATA__SHIFT) & CP_COND_WRITE5_7_WRITE_DATA__MASK;
-}
-
-#define REG_CP_WAIT_MEM_GTE_0					0x00000000
-#define CP_WAIT_MEM_GTE_0_RESERVED__MASK			0xffffffff
-#define CP_WAIT_MEM_GTE_0_RESERVED__SHIFT			0
-static inline uint32_t CP_WAIT_MEM_GTE_0_RESERVED(uint32_t val)
-{
-	return ((val) << CP_WAIT_MEM_GTE_0_RESERVED__SHIFT) & CP_WAIT_MEM_GTE_0_RESERVED__MASK;
-}
-
-#define REG_CP_WAIT_MEM_GTE_1					0x00000001
-#define CP_WAIT_MEM_GTE_1_POLL_ADDR_LO__MASK			0xffffffff
-#define CP_WAIT_MEM_GTE_1_POLL_ADDR_LO__SHIFT			0
-static inline uint32_t CP_WAIT_MEM_GTE_1_POLL_ADDR_LO(uint32_t val)
-{
-	return ((val) << CP_WAIT_MEM_GTE_1_POLL_ADDR_LO__SHIFT) & CP_WAIT_MEM_GTE_1_POLL_ADDR_LO__MASK;
-}
-
-#define REG_CP_WAIT_MEM_GTE_2					0x00000002
-#define CP_WAIT_MEM_GTE_2_POLL_ADDR_HI__MASK			0xffffffff
-#define CP_WAIT_MEM_GTE_2_POLL_ADDR_HI__SHIFT			0
-static inline uint32_t CP_WAIT_MEM_GTE_2_POLL_ADDR_HI(uint32_t val)
-{
-	return ((val) << CP_WAIT_MEM_GTE_2_POLL_ADDR_HI__SHIFT) & CP_WAIT_MEM_GTE_2_POLL_ADDR_HI__MASK;
-}
-
-#define REG_CP_WAIT_MEM_GTE_3					0x00000003
-#define CP_WAIT_MEM_GTE_3_REF__MASK				0xffffffff
-#define CP_WAIT_MEM_GTE_3_REF__SHIFT				0
-static inline uint32_t CP_WAIT_MEM_GTE_3_REF(uint32_t val)
-{
-	return ((val) << CP_WAIT_MEM_GTE_3_REF__SHIFT) & CP_WAIT_MEM_GTE_3_REF__MASK;
-}
-
-#define REG_CP_WAIT_REG_MEM_0					0x00000000
-#define CP_WAIT_REG_MEM_0_FUNCTION__MASK			0x00000007
-#define CP_WAIT_REG_MEM_0_FUNCTION__SHIFT			0
-static inline uint32_t CP_WAIT_REG_MEM_0_FUNCTION(enum cp_cond_function val)
-{
-	return ((val) << CP_WAIT_REG_MEM_0_FUNCTION__SHIFT) & CP_WAIT_REG_MEM_0_FUNCTION__MASK;
-}
-#define CP_WAIT_REG_MEM_0_SIGNED_COMPARE			0x00000008
-#define CP_WAIT_REG_MEM_0_POLL__MASK				0x00000030
-#define CP_WAIT_REG_MEM_0_POLL__SHIFT				4
-static inline uint32_t CP_WAIT_REG_MEM_0_POLL(enum poll_memory_type val)
-{
-	return ((val) << CP_WAIT_REG_MEM_0_POLL__SHIFT) & CP_WAIT_REG_MEM_0_POLL__MASK;
-}
-#define CP_WAIT_REG_MEM_0_WRITE_MEMORY				0x00000100
-
-#define REG_CP_WAIT_REG_MEM_1					0x00000001
-#define CP_WAIT_REG_MEM_1_POLL_ADDR_LO__MASK			0xffffffff
-#define CP_WAIT_REG_MEM_1_POLL_ADDR_LO__SHIFT			0
-static inline uint32_t CP_WAIT_REG_MEM_1_POLL_ADDR_LO(uint32_t val)
-{
-	return ((val) << CP_WAIT_REG_MEM_1_POLL_ADDR_LO__SHIFT) & CP_WAIT_REG_MEM_1_POLL_ADDR_LO__MASK;
-}
-
-#define REG_CP_WAIT_REG_MEM_2					0x00000002
-#define CP_WAIT_REG_MEM_2_POLL_ADDR_HI__MASK			0xffffffff
-#define CP_WAIT_REG_MEM_2_POLL_ADDR_HI__SHIFT			0
-static inline uint32_t CP_WAIT_REG_MEM_2_POLL_ADDR_HI(uint32_t val)
-{
-	return ((val) << CP_WAIT_REG_MEM_2_POLL_ADDR_HI__SHIFT) & CP_WAIT_REG_MEM_2_POLL_ADDR_HI__MASK;
-}
-
-#define REG_CP_WAIT_REG_MEM_3					0x00000003
-#define CP_WAIT_REG_MEM_3_REF__MASK				0xffffffff
-#define CP_WAIT_REG_MEM_3_REF__SHIFT				0
-static inline uint32_t CP_WAIT_REG_MEM_3_REF(uint32_t val)
-{
-	return ((val) << CP_WAIT_REG_MEM_3_REF__SHIFT) & CP_WAIT_REG_MEM_3_REF__MASK;
-}
-
-#define REG_CP_WAIT_REG_MEM_4					0x00000004
-#define CP_WAIT_REG_MEM_4_MASK__MASK				0xffffffff
-#define CP_WAIT_REG_MEM_4_MASK__SHIFT				0
-static inline uint32_t CP_WAIT_REG_MEM_4_MASK(uint32_t val)
-{
-	return ((val) << CP_WAIT_REG_MEM_4_MASK__SHIFT) & CP_WAIT_REG_MEM_4_MASK__MASK;
-}
-
-#define REG_CP_WAIT_REG_MEM_5					0x00000005
-#define CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES__MASK		0xffffffff
-#define CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES__SHIFT		0
-static inline uint32_t CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(uint32_t val)
-{
-	return ((val) << CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES__SHIFT) & CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES__MASK;
-}
-
-#define REG_CP_WAIT_TWO_REGS_0					0x00000000
-#define CP_WAIT_TWO_REGS_0_REG0__MASK				0x0003ffff
-#define CP_WAIT_TWO_REGS_0_REG0__SHIFT				0
-static inline uint32_t CP_WAIT_TWO_REGS_0_REG0(uint32_t val)
-{
-	return ((val) << CP_WAIT_TWO_REGS_0_REG0__SHIFT) & CP_WAIT_TWO_REGS_0_REG0__MASK;
-}
-
-#define REG_CP_WAIT_TWO_REGS_1					0x00000001
-#define CP_WAIT_TWO_REGS_1_REG1__MASK				0x0003ffff
-#define CP_WAIT_TWO_REGS_1_REG1__SHIFT				0
-static inline uint32_t CP_WAIT_TWO_REGS_1_REG1(uint32_t val)
-{
-	return ((val) << CP_WAIT_TWO_REGS_1_REG1__SHIFT) & CP_WAIT_TWO_REGS_1_REG1__MASK;
-}
-
-#define REG_CP_WAIT_TWO_REGS_2					0x00000002
-#define CP_WAIT_TWO_REGS_2_REF__MASK				0xffffffff
-#define CP_WAIT_TWO_REGS_2_REF__SHIFT				0
-static inline uint32_t CP_WAIT_TWO_REGS_2_REF(uint32_t val)
-{
-	return ((val) << CP_WAIT_TWO_REGS_2_REF__SHIFT) & CP_WAIT_TWO_REGS_2_REF__MASK;
-}
-
-#define REG_CP_DISPATCH_COMPUTE_0				0x00000000
-
-#define REG_CP_DISPATCH_COMPUTE_1				0x00000001
-#define CP_DISPATCH_COMPUTE_1_X__MASK				0xffffffff
-#define CP_DISPATCH_COMPUTE_1_X__SHIFT				0
-static inline uint32_t CP_DISPATCH_COMPUTE_1_X(uint32_t val)
-{
-	return ((val) << CP_DISPATCH_COMPUTE_1_X__SHIFT) & CP_DISPATCH_COMPUTE_1_X__MASK;
-}
-
-#define REG_CP_DISPATCH_COMPUTE_2				0x00000002
-#define CP_DISPATCH_COMPUTE_2_Y__MASK				0xffffffff
-#define CP_DISPATCH_COMPUTE_2_Y__SHIFT				0
-static inline uint32_t CP_DISPATCH_COMPUTE_2_Y(uint32_t val)
-{
-	return ((val) << CP_DISPATCH_COMPUTE_2_Y__SHIFT) & CP_DISPATCH_COMPUTE_2_Y__MASK;
-}
-
-#define REG_CP_DISPATCH_COMPUTE_3				0x00000003
-#define CP_DISPATCH_COMPUTE_3_Z__MASK				0xffffffff
-#define CP_DISPATCH_COMPUTE_3_Z__SHIFT				0
-static inline uint32_t CP_DISPATCH_COMPUTE_3_Z(uint32_t val)
-{
-	return ((val) << CP_DISPATCH_COMPUTE_3_Z__SHIFT) & CP_DISPATCH_COMPUTE_3_Z__MASK;
-}
-
-#define REG_CP_SET_RENDER_MODE_0				0x00000000
-#define CP_SET_RENDER_MODE_0_MODE__MASK				0x000001ff
-#define CP_SET_RENDER_MODE_0_MODE__SHIFT			0
-static inline uint32_t CP_SET_RENDER_MODE_0_MODE(enum render_mode_cmd val)
-{
-	return ((val) << CP_SET_RENDER_MODE_0_MODE__SHIFT) & CP_SET_RENDER_MODE_0_MODE__MASK;
-}
-
-#define REG_CP_SET_RENDER_MODE_1				0x00000001
-#define CP_SET_RENDER_MODE_1_ADDR_0_LO__MASK			0xffffffff
-#define CP_SET_RENDER_MODE_1_ADDR_0_LO__SHIFT			0
-static inline uint32_t CP_SET_RENDER_MODE_1_ADDR_0_LO(uint32_t val)
-{
-	return ((val) << CP_SET_RENDER_MODE_1_ADDR_0_LO__SHIFT) & CP_SET_RENDER_MODE_1_ADDR_0_LO__MASK;
-}
-
-#define REG_CP_SET_RENDER_MODE_2				0x00000002
-#define CP_SET_RENDER_MODE_2_ADDR_0_HI__MASK			0xffffffff
-#define CP_SET_RENDER_MODE_2_ADDR_0_HI__SHIFT			0
-static inline uint32_t CP_SET_RENDER_MODE_2_ADDR_0_HI(uint32_t val)
-{
-	return ((val) << CP_SET_RENDER_MODE_2_ADDR_0_HI__SHIFT) & CP_SET_RENDER_MODE_2_ADDR_0_HI__MASK;
-}
-
-#define REG_CP_SET_RENDER_MODE_3				0x00000003
-#define CP_SET_RENDER_MODE_3_VSC_ENABLE				0x00000008
-#define CP_SET_RENDER_MODE_3_GMEM_ENABLE			0x00000010
-
-#define REG_CP_SET_RENDER_MODE_4				0x00000004
-
-#define REG_CP_SET_RENDER_MODE_5				0x00000005
-#define CP_SET_RENDER_MODE_5_ADDR_1_LEN__MASK			0xffffffff
-#define CP_SET_RENDER_MODE_5_ADDR_1_LEN__SHIFT			0
-static inline uint32_t CP_SET_RENDER_MODE_5_ADDR_1_LEN(uint32_t val)
-{
-	return ((val) << CP_SET_RENDER_MODE_5_ADDR_1_LEN__SHIFT) & CP_SET_RENDER_MODE_5_ADDR_1_LEN__MASK;
-}
-
-#define REG_CP_SET_RENDER_MODE_6				0x00000006
-#define CP_SET_RENDER_MODE_6_ADDR_1_LO__MASK			0xffffffff
-#define CP_SET_RENDER_MODE_6_ADDR_1_LO__SHIFT			0
-static inline uint32_t CP_SET_RENDER_MODE_6_ADDR_1_LO(uint32_t val)
-{
-	return ((val) << CP_SET_RENDER_MODE_6_ADDR_1_LO__SHIFT) & CP_SET_RENDER_MODE_6_ADDR_1_LO__MASK;
-}
-
-#define REG_CP_SET_RENDER_MODE_7				0x00000007
-#define CP_SET_RENDER_MODE_7_ADDR_1_HI__MASK			0xffffffff
-#define CP_SET_RENDER_MODE_7_ADDR_1_HI__SHIFT			0
-static inline uint32_t CP_SET_RENDER_MODE_7_ADDR_1_HI(uint32_t val)
-{
-	return ((val) << CP_SET_RENDER_MODE_7_ADDR_1_HI__SHIFT) & CP_SET_RENDER_MODE_7_ADDR_1_HI__MASK;
-}
-
-#define REG_CP_COMPUTE_CHECKPOINT_0				0x00000000
-#define CP_COMPUTE_CHECKPOINT_0_ADDR_0_LO__MASK			0xffffffff
-#define CP_COMPUTE_CHECKPOINT_0_ADDR_0_LO__SHIFT		0
-static inline uint32_t CP_COMPUTE_CHECKPOINT_0_ADDR_0_LO(uint32_t val)
-{
-	return ((val) << CP_COMPUTE_CHECKPOINT_0_ADDR_0_LO__SHIFT) & CP_COMPUTE_CHECKPOINT_0_ADDR_0_LO__MASK;
-}
-
-#define REG_CP_COMPUTE_CHECKPOINT_1				0x00000001
-#define CP_COMPUTE_CHECKPOINT_1_ADDR_0_HI__MASK			0xffffffff
-#define CP_COMPUTE_CHECKPOINT_1_ADDR_0_HI__SHIFT		0
-static inline uint32_t CP_COMPUTE_CHECKPOINT_1_ADDR_0_HI(uint32_t val)
-{
-	return ((val) << CP_COMPUTE_CHECKPOINT_1_ADDR_0_HI__SHIFT) & CP_COMPUTE_CHECKPOINT_1_ADDR_0_HI__MASK;
-}
-
-#define REG_CP_COMPUTE_CHECKPOINT_2				0x00000002
-
-#define REG_CP_COMPUTE_CHECKPOINT_3				0x00000003
-
-#define REG_CP_COMPUTE_CHECKPOINT_4				0x00000004
-#define CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN__MASK		0xffffffff
-#define CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN__SHIFT		0
-static inline uint32_t CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN(uint32_t val)
-{
-	return ((val) << CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN__SHIFT) & CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN__MASK;
-}
-
-#define REG_CP_COMPUTE_CHECKPOINT_5				0x00000005
-#define CP_COMPUTE_CHECKPOINT_5_ADDR_1_LO__MASK			0xffffffff
-#define CP_COMPUTE_CHECKPOINT_5_ADDR_1_LO__SHIFT		0
-static inline uint32_t CP_COMPUTE_CHECKPOINT_5_ADDR_1_LO(uint32_t val)
-{
-	return ((val) << CP_COMPUTE_CHECKPOINT_5_ADDR_1_LO__SHIFT) & CP_COMPUTE_CHECKPOINT_5_ADDR_1_LO__MASK;
-}
-
-#define REG_CP_COMPUTE_CHECKPOINT_6				0x00000006
-#define CP_COMPUTE_CHECKPOINT_6_ADDR_1_HI__MASK			0xffffffff
-#define CP_COMPUTE_CHECKPOINT_6_ADDR_1_HI__SHIFT		0
-static inline uint32_t CP_COMPUTE_CHECKPOINT_6_ADDR_1_HI(uint32_t val)
-{
-	return ((val) << CP_COMPUTE_CHECKPOINT_6_ADDR_1_HI__SHIFT) & CP_COMPUTE_CHECKPOINT_6_ADDR_1_HI__MASK;
-}
-
-#define REG_CP_COMPUTE_CHECKPOINT_7				0x00000007
-
-#define REG_CP_PERFCOUNTER_ACTION_0				0x00000000
-
-#define REG_CP_PERFCOUNTER_ACTION_1				0x00000001
-#define CP_PERFCOUNTER_ACTION_1_ADDR_0_LO__MASK			0xffffffff
-#define CP_PERFCOUNTER_ACTION_1_ADDR_0_LO__SHIFT		0
-static inline uint32_t CP_PERFCOUNTER_ACTION_1_ADDR_0_LO(uint32_t val)
-{
-	return ((val) << CP_PERFCOUNTER_ACTION_1_ADDR_0_LO__SHIFT) & CP_PERFCOUNTER_ACTION_1_ADDR_0_LO__MASK;
-}
-
-#define REG_CP_PERFCOUNTER_ACTION_2				0x00000002
-#define CP_PERFCOUNTER_ACTION_2_ADDR_0_HI__MASK			0xffffffff
-#define CP_PERFCOUNTER_ACTION_2_ADDR_0_HI__SHIFT		0
-static inline uint32_t CP_PERFCOUNTER_ACTION_2_ADDR_0_HI(uint32_t val)
-{
-	return ((val) << CP_PERFCOUNTER_ACTION_2_ADDR_0_HI__SHIFT) & CP_PERFCOUNTER_ACTION_2_ADDR_0_HI__MASK;
-}
-
-#define REG_CP_EVENT_WRITE_0					0x00000000
-#define CP_EVENT_WRITE_0_EVENT__MASK				0x000000ff
-#define CP_EVENT_WRITE_0_EVENT__SHIFT				0
-static inline uint32_t CP_EVENT_WRITE_0_EVENT(enum vgt_event_type val)
-{
-	return ((val) << CP_EVENT_WRITE_0_EVENT__SHIFT) & CP_EVENT_WRITE_0_EVENT__MASK;
-}
-#define CP_EVENT_WRITE_0_TIMESTAMP				0x40000000
-#define CP_EVENT_WRITE_0_IRQ					0x80000000
-
-#define REG_CP_EVENT_WRITE_1					0x00000001
-#define CP_EVENT_WRITE_1_ADDR_0_LO__MASK			0xffffffff
-#define CP_EVENT_WRITE_1_ADDR_0_LO__SHIFT			0
-static inline uint32_t CP_EVENT_WRITE_1_ADDR_0_LO(uint32_t val)
-{
-	return ((val) << CP_EVENT_WRITE_1_ADDR_0_LO__SHIFT) & CP_EVENT_WRITE_1_ADDR_0_LO__MASK;
-}
-
-#define REG_CP_EVENT_WRITE_2					0x00000002
-#define CP_EVENT_WRITE_2_ADDR_0_HI__MASK			0xffffffff
-#define CP_EVENT_WRITE_2_ADDR_0_HI__SHIFT			0
-static inline uint32_t CP_EVENT_WRITE_2_ADDR_0_HI(uint32_t val)
-{
-	return ((val) << CP_EVENT_WRITE_2_ADDR_0_HI__SHIFT) & CP_EVENT_WRITE_2_ADDR_0_HI__MASK;
-}
-
-#define REG_CP_EVENT_WRITE_3					0x00000003
-
-#define REG_CP_EVENT_WRITE7_0					0x00000000
-#define CP_EVENT_WRITE7_0_EVENT__MASK				0x000000ff
-#define CP_EVENT_WRITE7_0_EVENT__SHIFT				0
-static inline uint32_t CP_EVENT_WRITE7_0_EVENT(enum vgt_event_type val)
-{
-	return ((val) << CP_EVENT_WRITE7_0_EVENT__SHIFT) & CP_EVENT_WRITE7_0_EVENT__MASK;
-}
-#define CP_EVENT_WRITE7_0_WRITE_SAMPLE_COUNT			0x00001000
-#define CP_EVENT_WRITE7_0_SAMPLE_COUNT_END_OFFSET		0x00002000
-#define CP_EVENT_WRITE7_0_WRITE_SAMPLE_COUNT_DIFF		0x00004000
-#define CP_EVENT_WRITE7_0_INC_BV_COUNT				0x00010000
-#define CP_EVENT_WRITE7_0_INC_BR_COUNT				0x00020000
-#define CP_EVENT_WRITE7_0_CLEAR_RENDER_RESOURCE			0x00040000
-#define CP_EVENT_WRITE7_0_CLEAR_LRZ_RESOURCE			0x00080000
-#define CP_EVENT_WRITE7_0_WRITE_SRC__MASK			0x00700000
-#define CP_EVENT_WRITE7_0_WRITE_SRC__SHIFT			20
-static inline uint32_t CP_EVENT_WRITE7_0_WRITE_SRC(enum event_write_src val)
-{
-	return ((val) << CP_EVENT_WRITE7_0_WRITE_SRC__SHIFT) & CP_EVENT_WRITE7_0_WRITE_SRC__MASK;
-}
-#define CP_EVENT_WRITE7_0_WRITE_DST__MASK			0x01000000
-#define CP_EVENT_WRITE7_0_WRITE_DST__SHIFT			24
-static inline uint32_t CP_EVENT_WRITE7_0_WRITE_DST(enum event_write_dst val)
-{
-	return ((val) << CP_EVENT_WRITE7_0_WRITE_DST__SHIFT) & CP_EVENT_WRITE7_0_WRITE_DST__MASK;
-}
-#define CP_EVENT_WRITE7_0_WRITE_ENABLED				0x08000000
-
-#define REG_EV_DST_RAM_CP_EVENT_WRITE7_1			0x00000001
-#define EV_DST_RAM_CP_EVENT_WRITE7_1_ADDR_0_LO__MASK		0xffffffff
-#define EV_DST_RAM_CP_EVENT_WRITE7_1_ADDR_0_LO__SHIFT		0
-static inline uint32_t EV_DST_RAM_CP_EVENT_WRITE7_1_ADDR_0_LO(uint32_t val)
-{
-	return ((val) << EV_DST_RAM_CP_EVENT_WRITE7_1_ADDR_0_LO__SHIFT) & EV_DST_RAM_CP_EVENT_WRITE7_1_ADDR_0_LO__MASK;
-}
-
-#define REG_EV_DST_RAM_CP_EVENT_WRITE7_2			0x00000002
-#define EV_DST_RAM_CP_EVENT_WRITE7_2_ADDR_0_HI__MASK		0xffffffff
-#define EV_DST_RAM_CP_EVENT_WRITE7_2_ADDR_0_HI__SHIFT		0
-static inline uint32_t EV_DST_RAM_CP_EVENT_WRITE7_2_ADDR_0_HI(uint32_t val)
-{
-	return ((val) << EV_DST_RAM_CP_EVENT_WRITE7_2_ADDR_0_HI__SHIFT) & EV_DST_RAM_CP_EVENT_WRITE7_2_ADDR_0_HI__MASK;
-}
-
-#define REG_EV_DST_RAM_CP_EVENT_WRITE7_3			0x00000003
-#define EV_DST_RAM_CP_EVENT_WRITE7_3_PAYLOAD_0__MASK		0xffffffff
-#define EV_DST_RAM_CP_EVENT_WRITE7_3_PAYLOAD_0__SHIFT		0
-static inline uint32_t EV_DST_RAM_CP_EVENT_WRITE7_3_PAYLOAD_0(uint32_t val)
-{
-	return ((val) << EV_DST_RAM_CP_EVENT_WRITE7_3_PAYLOAD_0__SHIFT) & EV_DST_RAM_CP_EVENT_WRITE7_3_PAYLOAD_0__MASK;
-}
-
-#define REG_EV_DST_RAM_CP_EVENT_WRITE7_4			0x00000004
-#define EV_DST_RAM_CP_EVENT_WRITE7_4_PAYLOAD_1__MASK		0xffffffff
-#define EV_DST_RAM_CP_EVENT_WRITE7_4_PAYLOAD_1__SHIFT		0
-static inline uint32_t EV_DST_RAM_CP_EVENT_WRITE7_4_PAYLOAD_1(uint32_t val)
-{
-	return ((val) << EV_DST_RAM_CP_EVENT_WRITE7_4_PAYLOAD_1__SHIFT) & EV_DST_RAM_CP_EVENT_WRITE7_4_PAYLOAD_1__MASK;
-}
-
-#define REG_EV_DST_ONCHIP_CP_EVENT_WRITE7_1			0x00000001
-#define EV_DST_ONCHIP_CP_EVENT_WRITE7_1_ONCHIP_ADDR_0__MASK	0xffffffff
-#define EV_DST_ONCHIP_CP_EVENT_WRITE7_1_ONCHIP_ADDR_0__SHIFT	0
-static inline uint32_t EV_DST_ONCHIP_CP_EVENT_WRITE7_1_ONCHIP_ADDR_0(uint32_t val)
-{
-	return ((val) << EV_DST_ONCHIP_CP_EVENT_WRITE7_1_ONCHIP_ADDR_0__SHIFT) & EV_DST_ONCHIP_CP_EVENT_WRITE7_1_ONCHIP_ADDR_0__MASK;
-}
-
-#define REG_EV_DST_ONCHIP_CP_EVENT_WRITE7_3			0x00000003
-#define EV_DST_ONCHIP_CP_EVENT_WRITE7_3_PAYLOAD_0__MASK		0xffffffff
-#define EV_DST_ONCHIP_CP_EVENT_WRITE7_3_PAYLOAD_0__SHIFT	0
-static inline uint32_t EV_DST_ONCHIP_CP_EVENT_WRITE7_3_PAYLOAD_0(uint32_t val)
-{
-	return ((val) << EV_DST_ONCHIP_CP_EVENT_WRITE7_3_PAYLOAD_0__SHIFT) & EV_DST_ONCHIP_CP_EVENT_WRITE7_3_PAYLOAD_0__MASK;
-}
-
-#define REG_EV_DST_ONCHIP_CP_EVENT_WRITE7_4			0x00000004
-#define EV_DST_ONCHIP_CP_EVENT_WRITE7_4_PAYLOAD_1__MASK		0xffffffff
-#define EV_DST_ONCHIP_CP_EVENT_WRITE7_4_PAYLOAD_1__SHIFT	0
-static inline uint32_t EV_DST_ONCHIP_CP_EVENT_WRITE7_4_PAYLOAD_1(uint32_t val)
-{
-	return ((val) << EV_DST_ONCHIP_CP_EVENT_WRITE7_4_PAYLOAD_1__SHIFT) & EV_DST_ONCHIP_CP_EVENT_WRITE7_4_PAYLOAD_1__MASK;
-}
-
-#define REG_CP_BLIT_0						0x00000000
-#define CP_BLIT_0_OP__MASK					0x0000000f
-#define CP_BLIT_0_OP__SHIFT					0
-static inline uint32_t CP_BLIT_0_OP(enum cp_blit_cmd val)
-{
-	return ((val) << CP_BLIT_0_OP__SHIFT) & CP_BLIT_0_OP__MASK;
-}
-
-#define REG_CP_BLIT_1						0x00000001
-#define CP_BLIT_1_SRC_X1__MASK					0x00003fff
-#define CP_BLIT_1_SRC_X1__SHIFT					0
-static inline uint32_t CP_BLIT_1_SRC_X1(uint32_t val)
-{
-	return ((val) << CP_BLIT_1_SRC_X1__SHIFT) & CP_BLIT_1_SRC_X1__MASK;
-}
-#define CP_BLIT_1_SRC_Y1__MASK					0x3fff0000
-#define CP_BLIT_1_SRC_Y1__SHIFT					16
-static inline uint32_t CP_BLIT_1_SRC_Y1(uint32_t val)
-{
-	return ((val) << CP_BLIT_1_SRC_Y1__SHIFT) & CP_BLIT_1_SRC_Y1__MASK;
-}
-
-#define REG_CP_BLIT_2						0x00000002
-#define CP_BLIT_2_SRC_X2__MASK					0x00003fff
-#define CP_BLIT_2_SRC_X2__SHIFT					0
-static inline uint32_t CP_BLIT_2_SRC_X2(uint32_t val)
-{
-	return ((val) << CP_BLIT_2_SRC_X2__SHIFT) & CP_BLIT_2_SRC_X2__MASK;
-}
-#define CP_BLIT_2_SRC_Y2__MASK					0x3fff0000
-#define CP_BLIT_2_SRC_Y2__SHIFT					16
-static inline uint32_t CP_BLIT_2_SRC_Y2(uint32_t val)
-{
-	return ((val) << CP_BLIT_2_SRC_Y2__SHIFT) & CP_BLIT_2_SRC_Y2__MASK;
-}
-
-#define REG_CP_BLIT_3						0x00000003
-#define CP_BLIT_3_DST_X1__MASK					0x00003fff
-#define CP_BLIT_3_DST_X1__SHIFT					0
-static inline uint32_t CP_BLIT_3_DST_X1(uint32_t val)
-{
-	return ((val) << CP_BLIT_3_DST_X1__SHIFT) & CP_BLIT_3_DST_X1__MASK;
-}
-#define CP_BLIT_3_DST_Y1__MASK					0x3fff0000
-#define CP_BLIT_3_DST_Y1__SHIFT					16
-static inline uint32_t CP_BLIT_3_DST_Y1(uint32_t val)
-{
-	return ((val) << CP_BLIT_3_DST_Y1__SHIFT) & CP_BLIT_3_DST_Y1__MASK;
-}
-
-#define REG_CP_BLIT_4						0x00000004
-#define CP_BLIT_4_DST_X2__MASK					0x00003fff
-#define CP_BLIT_4_DST_X2__SHIFT					0
-static inline uint32_t CP_BLIT_4_DST_X2(uint32_t val)
-{
-	return ((val) << CP_BLIT_4_DST_X2__SHIFT) & CP_BLIT_4_DST_X2__MASK;
-}
-#define CP_BLIT_4_DST_Y2__MASK					0x3fff0000
-#define CP_BLIT_4_DST_Y2__SHIFT					16
-static inline uint32_t CP_BLIT_4_DST_Y2(uint32_t val)
-{
-	return ((val) << CP_BLIT_4_DST_Y2__SHIFT) & CP_BLIT_4_DST_Y2__MASK;
-}
-
-#define REG_CP_EXEC_CS_0					0x00000000
-
-#define REG_CP_EXEC_CS_1					0x00000001
-#define CP_EXEC_CS_1_NGROUPS_X__MASK				0xffffffff
-#define CP_EXEC_CS_1_NGROUPS_X__SHIFT				0
-static inline uint32_t CP_EXEC_CS_1_NGROUPS_X(uint32_t val)
-{
-	return ((val) << CP_EXEC_CS_1_NGROUPS_X__SHIFT) & CP_EXEC_CS_1_NGROUPS_X__MASK;
-}
-
-#define REG_CP_EXEC_CS_2					0x00000002
-#define CP_EXEC_CS_2_NGROUPS_Y__MASK				0xffffffff
-#define CP_EXEC_CS_2_NGROUPS_Y__SHIFT				0
-static inline uint32_t CP_EXEC_CS_2_NGROUPS_Y(uint32_t val)
-{
-	return ((val) << CP_EXEC_CS_2_NGROUPS_Y__SHIFT) & CP_EXEC_CS_2_NGROUPS_Y__MASK;
-}
-
-#define REG_CP_EXEC_CS_3					0x00000003
-#define CP_EXEC_CS_3_NGROUPS_Z__MASK				0xffffffff
-#define CP_EXEC_CS_3_NGROUPS_Z__SHIFT				0
-static inline uint32_t CP_EXEC_CS_3_NGROUPS_Z(uint32_t val)
-{
-	return ((val) << CP_EXEC_CS_3_NGROUPS_Z__SHIFT) & CP_EXEC_CS_3_NGROUPS_Z__MASK;
-}
-
-#define REG_A4XX_CP_EXEC_CS_INDIRECT_0				0x00000000
-
-#define REG_A4XX_CP_EXEC_CS_INDIRECT_1				0x00000001
-#define A4XX_CP_EXEC_CS_INDIRECT_1_ADDR__MASK			0xffffffff
-#define A4XX_CP_EXEC_CS_INDIRECT_1_ADDR__SHIFT			0
-static inline uint32_t A4XX_CP_EXEC_CS_INDIRECT_1_ADDR(uint32_t val)
-{
-	return ((val) << A4XX_CP_EXEC_CS_INDIRECT_1_ADDR__SHIFT) & A4XX_CP_EXEC_CS_INDIRECT_1_ADDR__MASK;
-}
-
-#define REG_A4XX_CP_EXEC_CS_INDIRECT_2				0x00000002
-#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX__MASK		0x00000ffc
-#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX__SHIFT		2
-static inline uint32_t A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX(uint32_t val)
-{
-	return ((val) << A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX__SHIFT) & A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX__MASK;
-}
-#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY__MASK		0x003ff000
-#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY__SHIFT		12
-static inline uint32_t A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY(uint32_t val)
-{
-	return ((val) << A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY__SHIFT) & A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY__MASK;
-}
-#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ__MASK		0xffc00000
-#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ__SHIFT		22
-static inline uint32_t A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ(uint32_t val)
-{
-	return ((val) << A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ__SHIFT) & A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ__MASK;
-}
-
-#define REG_A5XX_CP_EXEC_CS_INDIRECT_1				0x00000001
-#define A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO__MASK		0xffffffff
-#define A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO__SHIFT		0
-static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO(uint32_t val)
-{
-	return ((val) << A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO__MASK;
-}
-
-#define REG_A5XX_CP_EXEC_CS_INDIRECT_2				0x00000002
-#define A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI__MASK		0xffffffff
-#define A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI__SHIFT		0
-static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI(uint32_t val)
-{
-	return ((val) << A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI__MASK;
-}
-
-#define REG_A5XX_CP_EXEC_CS_INDIRECT_3				0x00000003
-#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX__MASK		0x00000ffc
-#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX__SHIFT		2
-static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX(uint32_t val)
-{
-	return ((val) << A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX__MASK;
-}
-#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY__MASK		0x003ff000
-#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY__SHIFT		12
-static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY(uint32_t val)
-{
-	return ((val) << A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY__MASK;
-}
-#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ__MASK		0xffc00000
-#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ__SHIFT		22
-static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ(uint32_t val)
-{
-	return ((val) << A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ__MASK;
-}
-
-#define REG_A6XX_CP_SET_MARKER_0				0x00000000
-#define A6XX_CP_SET_MARKER_0_MODE__MASK				0x000001ff
-#define A6XX_CP_SET_MARKER_0_MODE__SHIFT			0
-static inline uint32_t A6XX_CP_SET_MARKER_0_MODE(enum a6xx_marker val)
-{
-	return ((val) << A6XX_CP_SET_MARKER_0_MODE__SHIFT) & A6XX_CP_SET_MARKER_0_MODE__MASK;
-}
-#define A6XX_CP_SET_MARKER_0_MARKER__MASK			0x0000000f
-#define A6XX_CP_SET_MARKER_0_MARKER__SHIFT			0
-static inline uint32_t A6XX_CP_SET_MARKER_0_MARKER(enum a6xx_marker val)
-{
-	return ((val) << A6XX_CP_SET_MARKER_0_MARKER__SHIFT) & A6XX_CP_SET_MARKER_0_MARKER__MASK;
-}
-
-#define REG_A6XX_CP_SET_PSEUDO_REG_(i0) (0x00000000 + 0x3*(i0))
-
-static inline uint32_t REG_A6XX_CP_SET_PSEUDO_REG__0(uint32_t i0) { return 0x00000000 + 0x3*i0; }
-#define A6XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG__MASK		0x000007ff
-#define A6XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG__SHIFT		0
-static inline uint32_t A6XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG(enum pseudo_reg val)
-{
-	return ((val) << A6XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG__SHIFT) & A6XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG__MASK;
-}
-
-static inline uint32_t REG_A6XX_CP_SET_PSEUDO_REG__1(uint32_t i0) { return 0x00000001 + 0x3*i0; }
-#define A6XX_CP_SET_PSEUDO_REG__1_LO__MASK			0xffffffff
-#define A6XX_CP_SET_PSEUDO_REG__1_LO__SHIFT			0
-static inline uint32_t A6XX_CP_SET_PSEUDO_REG__1_LO(uint32_t val)
-{
-	return ((val) << A6XX_CP_SET_PSEUDO_REG__1_LO__SHIFT) & A6XX_CP_SET_PSEUDO_REG__1_LO__MASK;
-}
-
-static inline uint32_t REG_A6XX_CP_SET_PSEUDO_REG__2(uint32_t i0) { return 0x00000002 + 0x3*i0; }
-#define A6XX_CP_SET_PSEUDO_REG__2_HI__MASK			0xffffffff
-#define A6XX_CP_SET_PSEUDO_REG__2_HI__SHIFT			0
-static inline uint32_t A6XX_CP_SET_PSEUDO_REG__2_HI(uint32_t val)
-{
-	return ((val) << A6XX_CP_SET_PSEUDO_REG__2_HI__SHIFT) & A6XX_CP_SET_PSEUDO_REG__2_HI__MASK;
-}
-
-#define REG_A6XX_CP_REG_TEST_0					0x00000000
-#define A6XX_CP_REG_TEST_0_REG__MASK				0x0003ffff
-#define A6XX_CP_REG_TEST_0_REG__SHIFT				0
-static inline uint32_t A6XX_CP_REG_TEST_0_REG(uint32_t val)
-{
-	return ((val) << A6XX_CP_REG_TEST_0_REG__SHIFT) & A6XX_CP_REG_TEST_0_REG__MASK;
-}
-#define A6XX_CP_REG_TEST_0_SCRATCH_MEM_OFFSET__MASK		0x0003ffff
-#define A6XX_CP_REG_TEST_0_SCRATCH_MEM_OFFSET__SHIFT		0
-static inline uint32_t A6XX_CP_REG_TEST_0_SCRATCH_MEM_OFFSET(uint32_t val)
-{
-	return ((val) << A6XX_CP_REG_TEST_0_SCRATCH_MEM_OFFSET__SHIFT) & A6XX_CP_REG_TEST_0_SCRATCH_MEM_OFFSET__MASK;
-}
-#define A6XX_CP_REG_TEST_0_SOURCE__MASK				0x00040000
-#define A6XX_CP_REG_TEST_0_SOURCE__SHIFT			18
-static inline uint32_t A6XX_CP_REG_TEST_0_SOURCE(enum source_type val)
-{
-	return ((val) << A6XX_CP_REG_TEST_0_SOURCE__SHIFT) & A6XX_CP_REG_TEST_0_SOURCE__MASK;
-}
-#define A6XX_CP_REG_TEST_0_BIT__MASK				0x01f00000
-#define A6XX_CP_REG_TEST_0_BIT__SHIFT				20
-static inline uint32_t A6XX_CP_REG_TEST_0_BIT(uint32_t val)
-{
-	return ((val) << A6XX_CP_REG_TEST_0_BIT__SHIFT) & A6XX_CP_REG_TEST_0_BIT__MASK;
-}
-#define A6XX_CP_REG_TEST_0_SKIP_WAIT_FOR_ME			0x02000000
-#define A6XX_CP_REG_TEST_0_PRED_BIT__MASK			0x7c000000
-#define A6XX_CP_REG_TEST_0_PRED_BIT__SHIFT			26
-static inline uint32_t A6XX_CP_REG_TEST_0_PRED_BIT(uint32_t val)
-{
-	return ((val) << A6XX_CP_REG_TEST_0_PRED_BIT__SHIFT) & A6XX_CP_REG_TEST_0_PRED_BIT__MASK;
-}
-#define A6XX_CP_REG_TEST_0_PRED_UPDATE				0x80000000
-
-#define REG_A6XX_CP_REG_TEST_PRED_MASK				0x00000001
-
-#define REG_A6XX_CP_REG_TEST_PRED_VAL				0x00000002
-
-#define REG_CP_COND_REG_EXEC_0					0x00000000
-#define CP_COND_REG_EXEC_0_REG0__MASK				0x0003ffff
-#define CP_COND_REG_EXEC_0_REG0__SHIFT				0
-static inline uint32_t CP_COND_REG_EXEC_0_REG0(uint32_t val)
-{
-	return ((val) << CP_COND_REG_EXEC_0_REG0__SHIFT) & CP_COND_REG_EXEC_0_REG0__MASK;
-}
-#define CP_COND_REG_EXEC_0_PRED_BIT__MASK			0x007c0000
-#define CP_COND_REG_EXEC_0_PRED_BIT__SHIFT			18
-static inline uint32_t CP_COND_REG_EXEC_0_PRED_BIT(uint32_t val)
-{
-	return ((val) << CP_COND_REG_EXEC_0_PRED_BIT__SHIFT) & CP_COND_REG_EXEC_0_PRED_BIT__MASK;
-}
-#define CP_COND_REG_EXEC_0_SKIP_WAIT_FOR_ME			0x00800000
-#define CP_COND_REG_EXEC_0_ONCHIP_MEM				0x01000000
-#define CP_COND_REG_EXEC_0_BINNING				0x02000000
-#define CP_COND_REG_EXEC_0_GMEM					0x04000000
-#define CP_COND_REG_EXEC_0_SYSMEM				0x08000000
-#define CP_COND_REG_EXEC_0_BV					0x02000000
-#define CP_COND_REG_EXEC_0_BR					0x04000000
-#define CP_COND_REG_EXEC_0_LPAC					0x08000000
-#define CP_COND_REG_EXEC_0_MODE__MASK				0xf0000000
-#define CP_COND_REG_EXEC_0_MODE__SHIFT				28
-static inline uint32_t CP_COND_REG_EXEC_0_MODE(enum compare_mode val)
-{
-	return ((val) << CP_COND_REG_EXEC_0_MODE__SHIFT) & CP_COND_REG_EXEC_0_MODE__MASK;
-}
-
-#define REG_PRED_TEST_CP_COND_REG_EXEC_1			0x00000001
-#define PRED_TEST_CP_COND_REG_EXEC_1_DWORDS__MASK		0x00ffffff
-#define PRED_TEST_CP_COND_REG_EXEC_1_DWORDS__SHIFT		0
-static inline uint32_t PRED_TEST_CP_COND_REG_EXEC_1_DWORDS(uint32_t val)
-{
-	return ((val) << PRED_TEST_CP_COND_REG_EXEC_1_DWORDS__SHIFT) & PRED_TEST_CP_COND_REG_EXEC_1_DWORDS__MASK;
-}
-
-#define REG_REG_COMPARE_CP_COND_REG_EXEC_1			0x00000001
-#define REG_COMPARE_CP_COND_REG_EXEC_1_REG1__MASK		0x0003ffff
-#define REG_COMPARE_CP_COND_REG_EXEC_1_REG1__SHIFT		0
-static inline uint32_t REG_COMPARE_CP_COND_REG_EXEC_1_REG1(uint32_t val)
-{
-	return ((val) << REG_COMPARE_CP_COND_REG_EXEC_1_REG1__SHIFT) & REG_COMPARE_CP_COND_REG_EXEC_1_REG1__MASK;
-}
-#define REG_COMPARE_CP_COND_REG_EXEC_1_ONCHIP_MEM		0x01000000
-
-#define REG_RENDER_MODE_CP_COND_REG_EXEC_1			0x00000001
-#define RENDER_MODE_CP_COND_REG_EXEC_1_DWORDS__MASK		0x00ffffff
-#define RENDER_MODE_CP_COND_REG_EXEC_1_DWORDS__SHIFT		0
-static inline uint32_t RENDER_MODE_CP_COND_REG_EXEC_1_DWORDS(uint32_t val)
-{
-	return ((val) << RENDER_MODE_CP_COND_REG_EXEC_1_DWORDS__SHIFT) & RENDER_MODE_CP_COND_REG_EXEC_1_DWORDS__MASK;
-}
-
-#define REG_REG_COMPARE_IMM_CP_COND_REG_EXEC_1			0x00000001
-#define REG_COMPARE_IMM_CP_COND_REG_EXEC_1_IMM__MASK		0xffffffff
-#define REG_COMPARE_IMM_CP_COND_REG_EXEC_1_IMM__SHIFT		0
-static inline uint32_t REG_COMPARE_IMM_CP_COND_REG_EXEC_1_IMM(uint32_t val)
-{
-	return ((val) << REG_COMPARE_IMM_CP_COND_REG_EXEC_1_IMM__SHIFT) & REG_COMPARE_IMM_CP_COND_REG_EXEC_1_IMM__MASK;
-}
-
-#define REG_THREAD_MODE_CP_COND_REG_EXEC_1			0x00000001
-#define THREAD_MODE_CP_COND_REG_EXEC_1_DWORDS__MASK		0x00ffffff
-#define THREAD_MODE_CP_COND_REG_EXEC_1_DWORDS__SHIFT		0
-static inline uint32_t THREAD_MODE_CP_COND_REG_EXEC_1_DWORDS(uint32_t val)
-{
-	return ((val) << THREAD_MODE_CP_COND_REG_EXEC_1_DWORDS__SHIFT) & THREAD_MODE_CP_COND_REG_EXEC_1_DWORDS__MASK;
-}
-
-#define REG_CP_COND_REG_EXEC_2					0x00000002
-#define CP_COND_REG_EXEC_2_DWORDS__MASK				0x00ffffff
-#define CP_COND_REG_EXEC_2_DWORDS__SHIFT			0
-static inline uint32_t CP_COND_REG_EXEC_2_DWORDS(uint32_t val)
-{
-	return ((val) << CP_COND_REG_EXEC_2_DWORDS__SHIFT) & CP_COND_REG_EXEC_2_DWORDS__MASK;
-}
-
-#define REG_CP_COND_EXEC_0					0x00000000
-#define CP_COND_EXEC_0_ADDR0_LO__MASK				0xffffffff
-#define CP_COND_EXEC_0_ADDR0_LO__SHIFT				0
-static inline uint32_t CP_COND_EXEC_0_ADDR0_LO(uint32_t val)
-{
-	return ((val) << CP_COND_EXEC_0_ADDR0_LO__SHIFT) & CP_COND_EXEC_0_ADDR0_LO__MASK;
-}
-
-#define REG_CP_COND_EXEC_1					0x00000001
-#define CP_COND_EXEC_1_ADDR0_HI__MASK				0xffffffff
-#define CP_COND_EXEC_1_ADDR0_HI__SHIFT				0
-static inline uint32_t CP_COND_EXEC_1_ADDR0_HI(uint32_t val)
-{
-	return ((val) << CP_COND_EXEC_1_ADDR0_HI__SHIFT) & CP_COND_EXEC_1_ADDR0_HI__MASK;
-}
-
-#define REG_CP_COND_EXEC_2					0x00000002
-#define CP_COND_EXEC_2_ADDR1_LO__MASK				0xffffffff
-#define CP_COND_EXEC_2_ADDR1_LO__SHIFT				0
-static inline uint32_t CP_COND_EXEC_2_ADDR1_LO(uint32_t val)
-{
-	return ((val) << CP_COND_EXEC_2_ADDR1_LO__SHIFT) & CP_COND_EXEC_2_ADDR1_LO__MASK;
-}
-
-#define REG_CP_COND_EXEC_3					0x00000003
-#define CP_COND_EXEC_3_ADDR1_HI__MASK				0xffffffff
-#define CP_COND_EXEC_3_ADDR1_HI__SHIFT				0
-static inline uint32_t CP_COND_EXEC_3_ADDR1_HI(uint32_t val)
-{
-	return ((val) << CP_COND_EXEC_3_ADDR1_HI__SHIFT) & CP_COND_EXEC_3_ADDR1_HI__MASK;
-}
-
-#define REG_CP_COND_EXEC_4					0x00000004
-#define CP_COND_EXEC_4_REF__MASK				0xffffffff
-#define CP_COND_EXEC_4_REF__SHIFT				0
-static inline uint32_t CP_COND_EXEC_4_REF(uint32_t val)
-{
-	return ((val) << CP_COND_EXEC_4_REF__SHIFT) & CP_COND_EXEC_4_REF__MASK;
-}
-
-#define REG_CP_COND_EXEC_5					0x00000005
-#define CP_COND_EXEC_5_DWORDS__MASK				0xffffffff
-#define CP_COND_EXEC_5_DWORDS__SHIFT				0
-static inline uint32_t CP_COND_EXEC_5_DWORDS(uint32_t val)
-{
-	return ((val) << CP_COND_EXEC_5_DWORDS__SHIFT) & CP_COND_EXEC_5_DWORDS__MASK;
-}
-
-#define REG_CP_SET_CTXSWITCH_IB_0				0x00000000
-#define CP_SET_CTXSWITCH_IB_0_ADDR_LO__MASK			0xffffffff
-#define CP_SET_CTXSWITCH_IB_0_ADDR_LO__SHIFT			0
-static inline uint32_t CP_SET_CTXSWITCH_IB_0_ADDR_LO(uint32_t val)
-{
-	return ((val) << CP_SET_CTXSWITCH_IB_0_ADDR_LO__SHIFT) & CP_SET_CTXSWITCH_IB_0_ADDR_LO__MASK;
-}
-
-#define REG_CP_SET_CTXSWITCH_IB_1				0x00000001
-#define CP_SET_CTXSWITCH_IB_1_ADDR_HI__MASK			0xffffffff
-#define CP_SET_CTXSWITCH_IB_1_ADDR_HI__SHIFT			0
-static inline uint32_t CP_SET_CTXSWITCH_IB_1_ADDR_HI(uint32_t val)
-{
-	return ((val) << CP_SET_CTXSWITCH_IB_1_ADDR_HI__SHIFT) & CP_SET_CTXSWITCH_IB_1_ADDR_HI__MASK;
-}
-
-#define REG_CP_SET_CTXSWITCH_IB_2				0x00000002
-#define CP_SET_CTXSWITCH_IB_2_DWORDS__MASK			0x000fffff
-#define CP_SET_CTXSWITCH_IB_2_DWORDS__SHIFT			0
-static inline uint32_t CP_SET_CTXSWITCH_IB_2_DWORDS(uint32_t val)
-{
-	return ((val) << CP_SET_CTXSWITCH_IB_2_DWORDS__SHIFT) & CP_SET_CTXSWITCH_IB_2_DWORDS__MASK;
-}
-#define CP_SET_CTXSWITCH_IB_2_TYPE__MASK			0x00300000
-#define CP_SET_CTXSWITCH_IB_2_TYPE__SHIFT			20
-static inline uint32_t CP_SET_CTXSWITCH_IB_2_TYPE(enum ctxswitch_ib val)
-{
-	return ((val) << CP_SET_CTXSWITCH_IB_2_TYPE__SHIFT) & CP_SET_CTXSWITCH_IB_2_TYPE__MASK;
-}
-
-#define REG_CP_REG_WRITE_0					0x00000000
-#define CP_REG_WRITE_0_TRACKER__MASK				0x0000000f
-#define CP_REG_WRITE_0_TRACKER__SHIFT				0
-static inline uint32_t CP_REG_WRITE_0_TRACKER(enum reg_tracker val)
-{
-	return ((val) << CP_REG_WRITE_0_TRACKER__SHIFT) & CP_REG_WRITE_0_TRACKER__MASK;
-}
-
-#define REG_CP_REG_WRITE_1					0x00000001
-
-#define REG_CP_REG_WRITE_2					0x00000002
-
-#define REG_CP_SMMU_TABLE_UPDATE_0				0x00000000
-#define CP_SMMU_TABLE_UPDATE_0_TTBR0_LO__MASK			0xffffffff
-#define CP_SMMU_TABLE_UPDATE_0_TTBR0_LO__SHIFT			0
-static inline uint32_t CP_SMMU_TABLE_UPDATE_0_TTBR0_LO(uint32_t val)
-{
-	return ((val) << CP_SMMU_TABLE_UPDATE_0_TTBR0_LO__SHIFT) & CP_SMMU_TABLE_UPDATE_0_TTBR0_LO__MASK;
-}
-
-#define REG_CP_SMMU_TABLE_UPDATE_1				0x00000001
-#define CP_SMMU_TABLE_UPDATE_1_TTBR0_HI__MASK			0x0000ffff
-#define CP_SMMU_TABLE_UPDATE_1_TTBR0_HI__SHIFT			0
-static inline uint32_t CP_SMMU_TABLE_UPDATE_1_TTBR0_HI(uint32_t val)
-{
-	return ((val) << CP_SMMU_TABLE_UPDATE_1_TTBR0_HI__SHIFT) & CP_SMMU_TABLE_UPDATE_1_TTBR0_HI__MASK;
-}
-#define CP_SMMU_TABLE_UPDATE_1_ASID__MASK			0xffff0000
-#define CP_SMMU_TABLE_UPDATE_1_ASID__SHIFT			16
-static inline uint32_t CP_SMMU_TABLE_UPDATE_1_ASID(uint32_t val)
-{
-	return ((val) << CP_SMMU_TABLE_UPDATE_1_ASID__SHIFT) & CP_SMMU_TABLE_UPDATE_1_ASID__MASK;
-}
-
-#define REG_CP_SMMU_TABLE_UPDATE_2				0x00000002
-#define CP_SMMU_TABLE_UPDATE_2_CONTEXTIDR__MASK			0xffffffff
-#define CP_SMMU_TABLE_UPDATE_2_CONTEXTIDR__SHIFT		0
-static inline uint32_t CP_SMMU_TABLE_UPDATE_2_CONTEXTIDR(uint32_t val)
-{
-	return ((val) << CP_SMMU_TABLE_UPDATE_2_CONTEXTIDR__SHIFT) & CP_SMMU_TABLE_UPDATE_2_CONTEXTIDR__MASK;
-}
-
-#define REG_CP_SMMU_TABLE_UPDATE_3				0x00000003
-#define CP_SMMU_TABLE_UPDATE_3_CONTEXTBANK__MASK		0xffffffff
-#define CP_SMMU_TABLE_UPDATE_3_CONTEXTBANK__SHIFT		0
-static inline uint32_t CP_SMMU_TABLE_UPDATE_3_CONTEXTBANK(uint32_t val)
-{
-	return ((val) << CP_SMMU_TABLE_UPDATE_3_CONTEXTBANK__SHIFT) & CP_SMMU_TABLE_UPDATE_3_CONTEXTBANK__MASK;
-}
-
-#define REG_CP_START_BIN_BIN_COUNT				0x00000000
-
-#define REG_CP_START_BIN_PREFIX_ADDR				0x00000001
-
-#define REG_CP_START_BIN_PREFIX_DWORDS				0x00000003
-
-#define REG_CP_START_BIN_BODY_DWORDS				0x00000004
-
-#define REG_CP_WAIT_TIMESTAMP_0					0x00000000
-#define CP_WAIT_TIMESTAMP_0_WAIT_VALUE_SRC__MASK		0x00000003
-#define CP_WAIT_TIMESTAMP_0_WAIT_VALUE_SRC__SHIFT		0
-static inline uint32_t CP_WAIT_TIMESTAMP_0_WAIT_VALUE_SRC(enum ts_wait_value_src val)
-{
-	return ((val) << CP_WAIT_TIMESTAMP_0_WAIT_VALUE_SRC__SHIFT) & CP_WAIT_TIMESTAMP_0_WAIT_VALUE_SRC__MASK;
-}
-#define CP_WAIT_TIMESTAMP_0_WAIT_DST__MASK			0x00000010
-#define CP_WAIT_TIMESTAMP_0_WAIT_DST__SHIFT			4
-static inline uint32_t CP_WAIT_TIMESTAMP_0_WAIT_DST(enum ts_wait_type val)
-{
-	return ((val) << CP_WAIT_TIMESTAMP_0_WAIT_DST__SHIFT) & CP_WAIT_TIMESTAMP_0_WAIT_DST__MASK;
-}
-
-#define REG_TS_WAIT_RAM_CP_WAIT_TIMESTAMP_ADDR			0x00000001
-
-#define REG_TS_WAIT_ONCHIP_CP_WAIT_TIMESTAMP_ONCHIP_ADDR_0	0x00000001
-
-#define REG_CP_WAIT_TIMESTAMP_SRC_0				0x00000003
-
-#define REG_CP_WAIT_TIMESTAMP_SRC_1				0x00000004
-
-#define REG_CP_BV_BR_COUNT_OPS_0				0x00000000
-#define CP_BV_BR_COUNT_OPS_0_OP__MASK				0x0000000f
-#define CP_BV_BR_COUNT_OPS_0_OP__SHIFT				0
-static inline uint32_t CP_BV_BR_COUNT_OPS_0_OP(enum pipe_count_op val)
-{
-	return ((val) << CP_BV_BR_COUNT_OPS_0_OP__SHIFT) & CP_BV_BR_COUNT_OPS_0_OP__MASK;
-}
-
-#define REG_CP_BV_BR_COUNT_OPS_1				0x00000001
-#define CP_BV_BR_COUNT_OPS_1_BR_OFFSET__MASK			0x0000ffff
-#define CP_BV_BR_COUNT_OPS_1_BR_OFFSET__SHIFT			0
-static inline uint32_t CP_BV_BR_COUNT_OPS_1_BR_OFFSET(uint32_t val)
-{
-	return ((val) << CP_BV_BR_COUNT_OPS_1_BR_OFFSET__SHIFT) & CP_BV_BR_COUNT_OPS_1_BR_OFFSET__MASK;
-}
-
-#define REG_CP_MODIFY_TIMESTAMP_0				0x00000000
-#define CP_MODIFY_TIMESTAMP_0_ADD__MASK				0x000000ff
-#define CP_MODIFY_TIMESTAMP_0_ADD__SHIFT			0
-static inline uint32_t CP_MODIFY_TIMESTAMP_0_ADD(uint32_t val)
-{
-	return ((val) << CP_MODIFY_TIMESTAMP_0_ADD__SHIFT) & CP_MODIFY_TIMESTAMP_0_ADD__MASK;
-}
-#define CP_MODIFY_TIMESTAMP_0_OP__MASK				0xf0000000
-#define CP_MODIFY_TIMESTAMP_0_OP__SHIFT				28
-static inline uint32_t CP_MODIFY_TIMESTAMP_0_OP(enum timestamp_op val)
-{
-	return ((val) << CP_MODIFY_TIMESTAMP_0_OP__SHIFT) & CP_MODIFY_TIMESTAMP_0_OP__MASK;
-}
-
-#define REG_CP_MEM_TO_SCRATCH_MEM_0				0x00000000
-#define CP_MEM_TO_SCRATCH_MEM_0_CNT__MASK			0x0000003f
-#define CP_MEM_TO_SCRATCH_MEM_0_CNT__SHIFT			0
-static inline uint32_t CP_MEM_TO_SCRATCH_MEM_0_CNT(uint32_t val)
-{
-	return ((val) << CP_MEM_TO_SCRATCH_MEM_0_CNT__SHIFT) & CP_MEM_TO_SCRATCH_MEM_0_CNT__MASK;
-}
-
-#define REG_CP_MEM_TO_SCRATCH_MEM_1				0x00000001
-#define CP_MEM_TO_SCRATCH_MEM_1_OFFSET__MASK			0x0000003f
-#define CP_MEM_TO_SCRATCH_MEM_1_OFFSET__SHIFT			0
-static inline uint32_t CP_MEM_TO_SCRATCH_MEM_1_OFFSET(uint32_t val)
-{
-	return ((val) << CP_MEM_TO_SCRATCH_MEM_1_OFFSET__SHIFT) & CP_MEM_TO_SCRATCH_MEM_1_OFFSET__MASK;
-}
-
-#define REG_CP_MEM_TO_SCRATCH_MEM_2				0x00000002
-#define CP_MEM_TO_SCRATCH_MEM_2_SRC__MASK			0xffffffff
-#define CP_MEM_TO_SCRATCH_MEM_2_SRC__SHIFT			0
-static inline uint32_t CP_MEM_TO_SCRATCH_MEM_2_SRC(uint32_t val)
-{
-	return ((val) << CP_MEM_TO_SCRATCH_MEM_2_SRC__SHIFT) & CP_MEM_TO_SCRATCH_MEM_2_SRC__MASK;
-}
-
-#define REG_CP_MEM_TO_SCRATCH_MEM_3				0x00000003
-#define CP_MEM_TO_SCRATCH_MEM_3_SRC_HI__MASK			0xffffffff
-#define CP_MEM_TO_SCRATCH_MEM_3_SRC_HI__SHIFT			0
-static inline uint32_t CP_MEM_TO_SCRATCH_MEM_3_SRC_HI(uint32_t val)
-{
-	return ((val) << CP_MEM_TO_SCRATCH_MEM_3_SRC_HI__SHIFT) & CP_MEM_TO_SCRATCH_MEM_3_SRC_HI__MASK;
-}
-
-#define REG_CP_THREAD_CONTROL_0					0x00000000
-#define CP_THREAD_CONTROL_0_THREAD__MASK			0x00000003
-#define CP_THREAD_CONTROL_0_THREAD__SHIFT			0
-static inline uint32_t CP_THREAD_CONTROL_0_THREAD(enum cp_thread val)
-{
-	return ((val) << CP_THREAD_CONTROL_0_THREAD__SHIFT) & CP_THREAD_CONTROL_0_THREAD__MASK;
-}
-#define CP_THREAD_CONTROL_0_CONCURRENT_BIN_DISABLE		0x08000000
-#define CP_THREAD_CONTROL_0_SYNC_THREADS			0x80000000
-
-#define REG_CP_FIXED_STRIDE_DRAW_TABLE_IB_BASE			0x00000000
-
-#define REG_CP_FIXED_STRIDE_DRAW_TABLE_2			0x00000002
-#define CP_FIXED_STRIDE_DRAW_TABLE_2_IB_SIZE__MASK		0x00000fff
-#define CP_FIXED_STRIDE_DRAW_TABLE_2_IB_SIZE__SHIFT		0
-static inline uint32_t CP_FIXED_STRIDE_DRAW_TABLE_2_IB_SIZE(uint32_t val)
-{
-	return ((val) << CP_FIXED_STRIDE_DRAW_TABLE_2_IB_SIZE__SHIFT) & CP_FIXED_STRIDE_DRAW_TABLE_2_IB_SIZE__MASK;
-}
-#define CP_FIXED_STRIDE_DRAW_TABLE_2_STRIDE__MASK		0xfff00000
-#define CP_FIXED_STRIDE_DRAW_TABLE_2_STRIDE__SHIFT		20
-static inline uint32_t CP_FIXED_STRIDE_DRAW_TABLE_2_STRIDE(uint32_t val)
-{
-	return ((val) << CP_FIXED_STRIDE_DRAW_TABLE_2_STRIDE__SHIFT) & CP_FIXED_STRIDE_DRAW_TABLE_2_STRIDE__MASK;
-}
-
-#define REG_CP_FIXED_STRIDE_DRAW_TABLE_3			0x00000003
-#define CP_FIXED_STRIDE_DRAW_TABLE_3_COUNT__MASK		0xffffffff
-#define CP_FIXED_STRIDE_DRAW_TABLE_3_COUNT__SHIFT		0
-static inline uint32_t CP_FIXED_STRIDE_DRAW_TABLE_3_COUNT(uint32_t val)
-{
-	return ((val) << CP_FIXED_STRIDE_DRAW_TABLE_3_COUNT__SHIFT) & CP_FIXED_STRIDE_DRAW_TABLE_3_COUNT__MASK;
-}
-
-#define REG_CP_RESET_CONTEXT_STATE_0				0x00000000
-#define CP_RESET_CONTEXT_STATE_0_CLEAR_ON_CHIP_TS		0x00000001
-#define CP_RESET_CONTEXT_STATE_0_CLEAR_RESOURCE_TABLE		0x00000002
-#define CP_RESET_CONTEXT_STATE_0_CLEAR_GLOBAL_LOCAL_TS		0x00000004
-
-#ifdef __cplusplus
-#endif
-
-#endif /* ADRENO_PM4_XML */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
index 88c2e51ab166..9f2164782844 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
@@ -320,7 +320,7 @@ static bool dpu_crtc_get_scanout_position(struct drm_crtc *crtc,
 }
 
 static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer,
-		struct dpu_plane_state *pstate, struct dpu_format *format)
+		struct dpu_plane_state *pstate, const struct msm_format *format)
 {
 	struct dpu_hw_mixer *lm = mixer->hw_lm;
 	uint32_t blend_op;
@@ -363,7 +363,7 @@ static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer,
 				fg_alpha, bg_alpha, blend_op);
 
 	DRM_DEBUG_ATOMIC("format:%p4cc, alpha_en:%u blend_op:0x%x\n",
-		  &format->base.pixel_format, format->alpha_enable, blend_op);
+		  &format->pixel_format, format->alpha_enable, blend_op);
 }
 
 static void _dpu_crtc_program_lm_output_roi(struct drm_crtc *crtc)
@@ -395,7 +395,7 @@ static void _dpu_crtc_blend_setup_pipe(struct drm_crtc *crtc,
 				       struct dpu_crtc_mixer *mixer,
 				       u32 num_mixers,
 				       enum dpu_stage stage,
-				       struct dpu_format *format,
+				       const struct msm_format *format,
 				       uint64_t modifier,
 				       struct dpu_sw_pipe *pipe,
 				       unsigned int stage_idx,
@@ -412,7 +412,7 @@ static void _dpu_crtc_blend_setup_pipe(struct drm_crtc *crtc,
 
 	trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane),
 				   state, to_dpu_plane_state(state), stage_idx,
-				   format->base.pixel_format,
+				   format->pixel_format,
 				   modifier);
 
 	DRM_DEBUG_ATOMIC("crtc %d stage:%d - plane %d sspp %d fb %d multirect_idx %d\n",
@@ -440,7 +440,7 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 	struct drm_plane_state *state;
 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
 	struct dpu_plane_state *pstate = NULL;
-	struct dpu_format *format;
+	const struct msm_format *format;
 	struct dpu_hw_ctl *ctl = mixer->lm_ctl;
 
 	uint32_t lm_idx;
@@ -459,7 +459,7 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 		pstate = to_dpu_plane_state(state);
 		fb = state->fb;
 
-		format = to_dpu_format(msm_framebuffer_format(pstate->base.fb));
+		format = msm_framebuffer_format(pstate->base.fb);
 
 		if (pstate->stage == DPU_STAGE_BASE && format->alpha_enable)
 			bg_alpha_enable = true;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
index 9a14d2232e4a..119f3ea50a7c 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
@@ -675,7 +675,7 @@ static int dpu_encoder_virt_atomic_check(
 	if (disp_info->intf_type == INTF_WB && conn_state->writeback_job) {
 		fb = conn_state->writeback_job->fb;
 
-		if (fb && DPU_FORMAT_IS_YUV(to_dpu_format(msm_framebuffer_format(fb))))
+		if (fb && MSM_FORMAT_IS_YUV(msm_framebuffer_format(fb)))
 			topology.needs_cdm = true;
 	} else if (disp_info->intf_type == INTF_DP) {
 		if (msm_dp_is_yuv_420_enabled(priv->dp[disp_info->h_tile_instance[0]], adj_mode))
@@ -2184,7 +2184,7 @@ void dpu_encoder_helper_phys_cleanup(struct dpu_encoder_phys *phys_enc)
 }
 
 void dpu_encoder_helper_phys_setup_cdm(struct dpu_encoder_phys *phys_enc,
-				       const struct dpu_format *dpu_fmt,
+				       const struct msm_format *dpu_fmt,
 				       u32 output_type)
 {
 	struct dpu_hw_cdm *hw_cdm;
@@ -2202,9 +2202,9 @@ void dpu_encoder_helper_phys_setup_cdm(struct dpu_encoder_phys *phys_enc,
 	if (!hw_cdm)
 		return;
 
-	if (!DPU_FORMAT_IS_YUV(dpu_fmt)) {
-		DPU_DEBUG("[enc:%d] cdm_disable fmt:%x\n", DRMID(phys_enc->parent),
-			  dpu_fmt->base.pixel_format);
+	if (!MSM_FORMAT_IS_YUV(dpu_fmt)) {
+		DPU_DEBUG("[enc:%d] cdm_disable fmt:%p4cc\n", DRMID(phys_enc->parent),
+			  &dpu_fmt->pixel_format);
 		if (hw_cdm->ops.bind_pingpong_blk)
 			hw_cdm->ops.bind_pingpong_blk(hw_cdm, PINGPONG_NONE);
 
@@ -2217,25 +2217,25 @@ void dpu_encoder_helper_phys_setup_cdm(struct dpu_encoder_phys *phys_enc,
 	cdm_cfg->output_height = phys_enc->cached_mode.vdisplay;
 	cdm_cfg->output_fmt = dpu_fmt;
 	cdm_cfg->output_type = output_type;
-	cdm_cfg->output_bit_depth = DPU_FORMAT_IS_DX(dpu_fmt) ?
+	cdm_cfg->output_bit_depth = MSM_FORMAT_IS_DX(dpu_fmt) ?
 			CDM_CDWN_OUTPUT_10BIT : CDM_CDWN_OUTPUT_8BIT;
 	cdm_cfg->csc_cfg = &dpu_csc10_rgb2yuv_601l;
 
 	/* enable 10 bit logic */
 	switch (cdm_cfg->output_fmt->chroma_sample) {
-	case DPU_CHROMA_RGB:
+	case CHROMA_FULL:
 		cdm_cfg->h_cdwn_type = CDM_CDWN_DISABLE;
 		cdm_cfg->v_cdwn_type = CDM_CDWN_DISABLE;
 		break;
-	case DPU_CHROMA_H2V1:
+	case CHROMA_H2V1:
 		cdm_cfg->h_cdwn_type = CDM_CDWN_COSITE;
 		cdm_cfg->v_cdwn_type = CDM_CDWN_DISABLE;
 		break;
-	case DPU_CHROMA_420:
+	case CHROMA_420:
 		cdm_cfg->h_cdwn_type = CDM_CDWN_COSITE;
 		cdm_cfg->v_cdwn_type = CDM_CDWN_OFFSITE;
 		break;
-	case DPU_CHROMA_H1V2:
+	case CHROMA_H1V2:
 	default:
 		DPU_ERROR("[enc:%d] unsupported chroma sampling type\n",
 			  DRMID(phys_enc->parent));
@@ -2244,9 +2244,9 @@ void dpu_encoder_helper_phys_setup_cdm(struct dpu_encoder_phys *phys_enc,
 		break;
 	}
 
-	DPU_DEBUG("[enc:%d] cdm_enable:%d,%d,%X,%d,%d,%d,%d]\n",
+	DPU_DEBUG("[enc:%d] cdm_enable:%d,%d,%p4cc,%d,%d,%d,%d]\n",
 		  DRMID(phys_enc->parent), cdm_cfg->output_width,
-		  cdm_cfg->output_height, cdm_cfg->output_fmt->base.pixel_format,
+		  cdm_cfg->output_height, &cdm_cfg->output_fmt->pixel_format,
 		  cdm_cfg->output_type, cdm_cfg->output_bit_depth,
 		  cdm_cfg->h_cdwn_type, cdm_cfg->v_cdwn_type);
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h
index 98d1b64a43e8..002e89cc1705 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h
@@ -393,7 +393,7 @@ void dpu_encoder_helper_phys_cleanup(struct dpu_encoder_phys *phys_enc);
  * @output_type: HDMI/WB
  */
 void dpu_encoder_helper_phys_setup_cdm(struct dpu_encoder_phys *phys_enc,
-				       const struct dpu_format *dpu_fmt,
+				       const struct msm_format *dpu_fmt,
 				       u32 output_type);
 
 /**
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c
index fc1d5736d7fc..489be1c0c704 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c
@@ -448,9 +448,6 @@ static void dpu_encoder_phys_cmd_enable_helper(
 
 	_dpu_encoder_phys_cmd_pingpong_config(phys_enc);
 
-	if (!dpu_encoder_phys_cmd_is_master(phys_enc))
-		return;
-
 	ctl = phys_enc->hw_ctl;
 	ctl->ops.update_pending_flush_intf(ctl, phys_enc->hw_intf->idx);
 }
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c
index d9e7dbf0499c..ef69c2f408c3 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c
@@ -235,7 +235,7 @@ static void dpu_encoder_phys_vid_setup_timing_engine(
 {
 	struct drm_display_mode mode;
 	struct dpu_hw_intf_timing_params timing_params = { 0 };
-	const struct dpu_format *fmt = NULL;
+	const struct msm_format *fmt = NULL;
 	u32 fmt_fourcc;
 	unsigned long lock_flags;
 	struct dpu_hw_intf_cfg intf_cfg = { 0 };
@@ -274,7 +274,7 @@ static void dpu_encoder_phys_vid_setup_timing_engine(
 
 	drm_mode_to_intf_timing_params(phys_enc, &mode, &timing_params);
 
-	fmt = dpu_get_dpu_format(fmt_fourcc);
+	fmt = mdp_get_format(&phys_enc->dpu_kms->base, fmt_fourcc, 0);
 	DPU_DEBUG_VIDENC(phys_enc, "fmt_fourcc 0x%X\n", fmt_fourcc);
 
 	if (phys_enc->hw_cdm)
@@ -409,12 +409,12 @@ end:
 static void dpu_encoder_phys_vid_enable(struct dpu_encoder_phys *phys_enc)
 {
 	struct dpu_hw_ctl *ctl;
-	const struct dpu_format *fmt;
+	const struct msm_format *fmt;
 	u32 fmt_fourcc;
 
 	ctl = phys_enc->hw_ctl;
 	fmt_fourcc = dpu_encoder_get_drm_fmt(phys_enc);
-	fmt = dpu_get_dpu_format(fmt_fourcc);
+	fmt = mdp_get_format(&phys_enc->dpu_kms->base, fmt_fourcc, 0);
 
 	DPU_DEBUG_VIDENC(phys_enc, "\n");
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_wb.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_wb.c
index 1924a2b28e53..d3ea91c1d7d2 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_wb.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_wb.c
@@ -322,11 +322,11 @@ static void dpu_encoder_phys_wb_setup(
 	struct dpu_encoder_phys_wb *wb_enc = to_dpu_encoder_phys_wb(phys_enc);
 	struct drm_writeback_job *wb_job;
 	const struct msm_format *format;
-	const struct dpu_format *dpu_fmt;
+	const struct msm_format *dpu_fmt;
 
 	wb_job = wb_enc->wb_job;
 	format = msm_framebuffer_format(wb_enc->wb_job->fb);
-	dpu_fmt = dpu_get_dpu_format_ext(format->pixel_format, wb_job->fb->modifier);
+	dpu_fmt = mdp_get_format(&phys_enc->dpu_kms->base, format->pixel_format, wb_job->fb->modifier);
 
 	DPU_DEBUG("[mode_set:%d, \"%s\",%d,%d]\n",
 			hw_wb->idx - WB_0, mode.name,
@@ -576,11 +576,11 @@ static void dpu_encoder_phys_wb_prepare_wb_job(struct dpu_encoder_phys *phys_enc
 
 	format = msm_framebuffer_format(job->fb);
 
-	wb_cfg->dest.format = dpu_get_dpu_format_ext(
-			format->pixel_format, job->fb->modifier);
+	wb_cfg->dest.format = mdp_get_format(&phys_enc->dpu_kms->base,
+					     format->pixel_format, job->fb->modifier);
 	if (!wb_cfg->dest.format) {
 		/* this error should be detected during atomic_check */
-		DPU_ERROR("failed to get format %x\n", format->pixel_format);
+		DPU_ERROR("failed to get format %p4cc\n", &format->pixel_format);
 		return;
 	}
 
@@ -594,7 +594,7 @@ static void dpu_encoder_phys_wb_prepare_wb_job(struct dpu_encoder_phys *phys_enc
 	wb_cfg->dest.height = job->fb->height;
 	wb_cfg->dest.num_planes = wb_cfg->dest.format->num_planes;
 
-	if ((wb_cfg->dest.format->fetch_planes == DPU_PLANE_PLANAR) &&
+	if ((wb_cfg->dest.format->fetch_type == MDP_PLANE_PLANAR) &&
 			(wb_cfg->dest.format->element[0] == C1_B_Cb))
 		swap(wb_cfg->dest.plane_addr[1], wb_cfg->dest.plane_addr[2]);
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
index e366ab134249..6b1e9a617da3 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
@@ -11,179 +11,12 @@
 #include "dpu_kms.h"
 #include "dpu_formats.h"
 
-#define DPU_UBWC_META_MACRO_W_H		16
-#define DPU_UBWC_META_BLOCK_SIZE	256
 #define DPU_UBWC_PLANE_SIZE_ALIGNMENT	4096
 
-#define DPU_TILE_HEIGHT_DEFAULT	1
-#define DPU_TILE_HEIGHT_TILED	4
-#define DPU_TILE_HEIGHT_UBWC	4
-#define DPU_TILE_HEIGHT_NV12	8
-
 #define DPU_MAX_IMG_WIDTH		0x3FFF
 #define DPU_MAX_IMG_HEIGHT		0x3FFF
 
 /*
- * DPU supported format packing, bpp, and other format
- * information.
- * DPU currently only supports interleaved RGB formats
- * UBWC support for a pixel format is indicated by the flag,
- * there is additional meta data plane for such formats
- */
-
-#define INTERLEAVED_RGB_FMT(fmt, a, r, g, b, e0, e1, e2, e3, uc, alpha,   \
-bp, flg, fm, np)                                                          \
-{                                                                         \
-	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
-	.fetch_planes = DPU_PLANE_INTERLEAVED,                            \
-	.alpha_enable = alpha,                                            \
-	.element = { (e0), (e1), (e2), (e3) },                            \
-	.bits = { g, b, r, a },                                           \
-	.chroma_sample = DPU_CHROMA_RGB,                                  \
-	.unpack_align_msb = 0,                                            \
-	.unpack_tight = 1,                                                \
-	.unpack_count = uc,                                               \
-	.bpp = bp,                                                        \
-	.fetch_mode = fm,                                                 \
-	.flag = {(flg)},                                                  \
-	.num_planes = np,                                                 \
-	.tile_height = DPU_TILE_HEIGHT_DEFAULT                            \
-}
-
-#define INTERLEAVED_RGB_FMT_TILED(fmt, a, r, g, b, e0, e1, e2, e3, uc,    \
-alpha, bp, flg, fm, np, th)                                               \
-{                                                                         \
-	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
-	.fetch_planes = DPU_PLANE_INTERLEAVED,                            \
-	.alpha_enable = alpha,                                            \
-	.element = { (e0), (e1), (e2), (e3) },                            \
-	.bits = { g, b, r, a },                                           \
-	.chroma_sample = DPU_CHROMA_RGB,                                  \
-	.unpack_align_msb = 0,                                            \
-	.unpack_tight = 1,                                                \
-	.unpack_count = uc,                                               \
-	.bpp = bp,                                                        \
-	.fetch_mode = fm,                                                 \
-	.flag = {(flg)},                                                  \
-	.num_planes = np,                                                 \
-	.tile_height = th                                                 \
-}
-
-
-#define INTERLEAVED_YUV_FMT(fmt, a, r, g, b, e0, e1, e2, e3,              \
-alpha, chroma, count, bp, flg, fm, np)                                    \
-{                                                                         \
-	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
-	.fetch_planes = DPU_PLANE_INTERLEAVED,                            \
-	.alpha_enable = alpha,                                            \
-	.element = { (e0), (e1), (e2), (e3)},                             \
-	.bits = { g, b, r, a },                                           \
-	.chroma_sample = chroma,                                          \
-	.unpack_align_msb = 0,                                            \
-	.unpack_tight = 1,                                                \
-	.unpack_count = count,                                            \
-	.bpp = bp,                                                        \
-	.fetch_mode = fm,                                                 \
-	.flag = {(flg)},                                                  \
-	.num_planes = np,                                                 \
-	.tile_height = DPU_TILE_HEIGHT_DEFAULT                            \
-}
-
-#define PSEUDO_YUV_FMT(fmt, a, r, g, b, e0, e1, chroma, flg, fm, np)      \
-{                                                                         \
-	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
-	.fetch_planes = DPU_PLANE_PSEUDO_PLANAR,                          \
-	.alpha_enable = false,                                            \
-	.element = { (e0), (e1), 0, 0 },                                  \
-	.bits = { g, b, r, a },                                           \
-	.chroma_sample = chroma,                                          \
-	.unpack_align_msb = 0,                                            \
-	.unpack_tight = 1,                                                \
-	.unpack_count = 2,                                                \
-	.bpp = 2,                                                         \
-	.fetch_mode = fm,                                                 \
-	.flag = {(flg)},                                                  \
-	.num_planes = np,                                                 \
-	.tile_height = DPU_TILE_HEIGHT_DEFAULT                            \
-}
-
-#define PSEUDO_YUV_FMT_TILED(fmt, a, r, g, b, e0, e1, chroma,             \
-flg, fm, np, th)                                                          \
-{                                                                         \
-	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
-	.fetch_planes = DPU_PLANE_PSEUDO_PLANAR,                          \
-	.alpha_enable = false,                                            \
-	.element = { (e0), (e1), 0, 0 },                                  \
-	.bits = { g, b, r, a },                                           \
-	.chroma_sample = chroma,                                          \
-	.unpack_align_msb = 0,                                            \
-	.unpack_tight = 1,                                                \
-	.unpack_count = 2,                                                \
-	.bpp = 2,                                                         \
-	.fetch_mode = fm,                                                 \
-	.flag = {(flg)},                                                  \
-	.num_planes = np,                                                 \
-	.tile_height = th                                                 \
-}
-
-#define PSEUDO_YUV_FMT_LOOSE(fmt, a, r, g, b, e0, e1, chroma, flg, fm, np)\
-{                                                                         \
-	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
-	.fetch_planes = DPU_PLANE_PSEUDO_PLANAR,                          \
-	.alpha_enable = false,                                            \
-	.element = { (e0), (e1), 0, 0 },                                  \
-	.bits = { g, b, r, a },                                           \
-	.chroma_sample = chroma,                                          \
-	.unpack_align_msb = 1,                                            \
-	.unpack_tight = 0,                                                \
-	.unpack_count = 2,                                                \
-	.bpp = 2,                                                         \
-	.fetch_mode = fm,                                                 \
-	.flag = {(flg)},                                                  \
-	.num_planes = np,                                                 \
-	.tile_height = DPU_TILE_HEIGHT_DEFAULT                            \
-}
-
-#define PSEUDO_YUV_FMT_LOOSE_TILED(fmt, a, r, g, b, e0, e1, chroma,       \
-flg, fm, np, th)                                                          \
-{                                                                         \
-	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
-	.fetch_planes = DPU_PLANE_PSEUDO_PLANAR,                          \
-	.alpha_enable = false,                                            \
-	.element = { (e0), (e1), 0, 0 },                                  \
-	.bits = { g, b, r, a },                                           \
-	.chroma_sample = chroma,                                          \
-	.unpack_align_msb = 1,                                            \
-	.unpack_tight = 0,                                                \
-	.unpack_count = 2,                                                \
-	.bpp = 2,                                                         \
-	.fetch_mode = fm,                                                 \
-	.flag = {(flg)},                                                  \
-	.num_planes = np,                                                 \
-	.tile_height = th                                                 \
-}
-
-
-#define PLANAR_YUV_FMT(fmt, a, r, g, b, e0, e1, e2, alpha, chroma, bp,    \
-flg, fm, np)                                                      \
-{                                                                         \
-	.base.pixel_format = DRM_FORMAT_ ## fmt,                          \
-	.fetch_planes = DPU_PLANE_PLANAR,                                 \
-	.alpha_enable = alpha,                                            \
-	.element = { (e0), (e1), (e2), 0 },                               \
-	.bits = { g, b, r, a },                                           \
-	.chroma_sample = chroma,                                          \
-	.unpack_align_msb = 0,                                            \
-	.unpack_tight = 1,                                                \
-	.unpack_count = 1,                                                \
-	.bpp = bp,                                                        \
-	.fetch_mode = fm,                                                 \
-	.flag = {(flg)},                                                  \
-	.num_planes = np,                                                 \
-	.tile_height = DPU_TILE_HEIGHT_DEFAULT                            \
-}
-
-/*
  * struct dpu_media_color_map - maps drm format to media format
  * @format: DRM base pixel format
  * @color: Media API color related to DRM format
@@ -193,380 +26,11 @@ struct dpu_media_color_map {
 	uint32_t color;
 };
 
-static const struct dpu_format dpu_format_map[] = {
-	INTERLEAVED_RGB_FMT(ARGB8888,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
-		true, 4, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(ABGR8888,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		true, 4, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(XBGR8888,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		false, 4, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(RGBA8888,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
-		true, 4, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(BGRA8888,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
-		true, 4, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(BGRX8888,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
-		false, 4, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(XRGB8888,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
-		false, 4, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(RGBX8888,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
-		false, 4, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(RGB888,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C1_B_Cb, C0_G_Y, C2_R_Cr, 0, 3,
-		false, 3, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(BGR888,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3,
-		false, 3, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(RGB565,
-		0, COLOR_5BIT, COLOR_6BIT, COLOR_5BIT,
-		C1_B_Cb, C0_G_Y, C2_R_Cr, 0, 3,
-		false, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(BGR565,
-		0, COLOR_5BIT, COLOR_6BIT, COLOR_5BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3,
-		false, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(ARGB1555,
-		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
-		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
-		true, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(ABGR1555,
-		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		true, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(RGBA5551,
-		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
-		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
-		true, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(BGRA5551,
-		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
-		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
-		true, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(XRGB1555,
-		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
-		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
-		false, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(XBGR1555,
-		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		false, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(RGBX5551,
-		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
-		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
-		false, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(BGRX5551,
-		COLOR_ALPHA_1BIT, COLOR_5BIT, COLOR_5BIT, COLOR_5BIT,
-		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
-		false, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(ARGB4444,
-		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
-		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
-		true, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(ABGR4444,
-		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		true, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(RGBA4444,
-		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
-		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
-		true, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(BGRA4444,
-		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
-		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
-		true, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(XRGB4444,
-		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
-		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
-		false, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(XBGR4444,
-		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		false, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(RGBX4444,
-		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
-		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
-		false, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(BGRX4444,
-		COLOR_ALPHA_4BIT, COLOR_4BIT, COLOR_4BIT, COLOR_4BIT,
-		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
-		false, 2, 0,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(BGRA1010102,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
-		true, 4, DPU_FORMAT_FLAG_DX,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(RGBA1010102,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
-		true, 4, DPU_FORMAT_FLAG_DX,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(ABGR2101010,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		true, 4, DPU_FORMAT_FLAG_DX,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(ARGB2101010,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
-		true, 4, DPU_FORMAT_FLAG_DX,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(XRGB2101010,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
-		false, 4, DPU_FORMAT_FLAG_DX,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(BGRX1010102,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
-		false, 4, DPU_FORMAT_FLAG_DX,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(XBGR2101010,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		false, 4, DPU_FORMAT_FLAG_DX,
-		DPU_FETCH_LINEAR, 1),
-
-	INTERLEAVED_RGB_FMT(RGBX1010102,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
-		false, 4, DPU_FORMAT_FLAG_DX,
-		DPU_FETCH_LINEAR, 1),
-
-	PSEUDO_YUV_FMT(NV12,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C1_B_Cb, C2_R_Cr,
-		DPU_CHROMA_420, DPU_FORMAT_FLAG_YUV,
-		DPU_FETCH_LINEAR, 2),
-
-	PSEUDO_YUV_FMT(NV21,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C1_B_Cb,
-		DPU_CHROMA_420, DPU_FORMAT_FLAG_YUV,
-		DPU_FETCH_LINEAR, 2),
-
-	PSEUDO_YUV_FMT(NV16,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C1_B_Cb, C2_R_Cr,
-		DPU_CHROMA_H2V1, DPU_FORMAT_FLAG_YUV,
-		DPU_FETCH_LINEAR, 2),
-
-	PSEUDO_YUV_FMT(NV61,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C1_B_Cb,
-		DPU_CHROMA_H2V1, DPU_FORMAT_FLAG_YUV,
-		DPU_FETCH_LINEAR, 2),
-
-	PSEUDO_YUV_FMT_LOOSE(P010,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C1_B_Cb, C2_R_Cr,
-		DPU_CHROMA_420, DPU_FORMAT_FLAG_DX | DPU_FORMAT_FLAG_YUV,
-		DPU_FETCH_LINEAR, 2),
-
-	INTERLEAVED_YUV_FMT(VYUY,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C0_G_Y,
-		false, DPU_CHROMA_H2V1, 4, 2, DPU_FORMAT_FLAG_YUV,
-		DPU_FETCH_LINEAR, 2),
-
-	INTERLEAVED_YUV_FMT(UYVY,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C1_B_Cb, C0_G_Y, C2_R_Cr, C0_G_Y,
-		false, DPU_CHROMA_H2V1, 4, 2, DPU_FORMAT_FLAG_YUV,
-		DPU_FETCH_LINEAR, 2),
-
-	INTERLEAVED_YUV_FMT(YUYV,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C0_G_Y, C1_B_Cb, C0_G_Y, C2_R_Cr,
-		false, DPU_CHROMA_H2V1, 4, 2, DPU_FORMAT_FLAG_YUV,
-		DPU_FETCH_LINEAR, 2),
-
-	INTERLEAVED_YUV_FMT(YVYU,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C0_G_Y, C2_R_Cr, C0_G_Y, C1_B_Cb,
-		false, DPU_CHROMA_H2V1, 4, 2, DPU_FORMAT_FLAG_YUV,
-		DPU_FETCH_LINEAR, 2),
-
-	PLANAR_YUV_FMT(YUV420,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C1_B_Cb, C0_G_Y,
-		false, DPU_CHROMA_420, 1, DPU_FORMAT_FLAG_YUV,
-		DPU_FETCH_LINEAR, 3),
-
-	PLANAR_YUV_FMT(YVU420,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C1_B_Cb, C2_R_Cr, C0_G_Y,
-		false, DPU_CHROMA_420, 1, DPU_FORMAT_FLAG_YUV,
-		DPU_FETCH_LINEAR, 3),
-};
-
-/*
- * UBWC formats table:
- * This table holds the UBWC formats supported.
- * If a compression ratio needs to be used for this or any other format,
- * the data will be passed by user-space.
- */
-static const struct dpu_format dpu_format_map_ubwc[] = {
-	INTERLEAVED_RGB_FMT_TILED(BGR565,
-		0, COLOR_5BIT, COLOR_6BIT, COLOR_5BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3,
-		false, 2, DPU_FORMAT_FLAG_COMPRESSED,
-		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
-
-	INTERLEAVED_RGB_FMT_TILED(ABGR8888,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		true, 4, DPU_FORMAT_FLAG_COMPRESSED,
-		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
-
-	/* ARGB8888 and ABGR8888 purposely have the same color
-	 * ordering.  The hardware only supports ABGR8888 UBWC
-	 * natively.
-	 */
-	INTERLEAVED_RGB_FMT_TILED(ARGB8888,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		true, 4, DPU_FORMAT_FLAG_COMPRESSED,
-		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
-
-	INTERLEAVED_RGB_FMT_TILED(XBGR8888,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		false, 4, DPU_FORMAT_FLAG_COMPRESSED,
-		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
-
-	INTERLEAVED_RGB_FMT_TILED(XRGB8888,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		false, 4, DPU_FORMAT_FLAG_COMPRESSED,
-		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
-
-	INTERLEAVED_RGB_FMT_TILED(ABGR2101010,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		true, 4, DPU_FORMAT_FLAG_DX | DPU_FORMAT_FLAG_COMPRESSED,
-		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
-
-	INTERLEAVED_RGB_FMT_TILED(XBGR2101010,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		true, 4, DPU_FORMAT_FLAG_DX | DPU_FORMAT_FLAG_COMPRESSED,
-		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
-
-	INTERLEAVED_RGB_FMT_TILED(XRGB2101010,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		true, 4, DPU_FORMAT_FLAG_DX | DPU_FORMAT_FLAG_COMPRESSED,
-		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
-
-	/* XRGB2101010 and ARGB2101010 purposely have the same color
-	* ordering.  The hardware only supports ARGB2101010 UBWC
-	* natively.
-	*/
-	INTERLEAVED_RGB_FMT_TILED(ARGB2101010,
-		COLOR_8BIT, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
-		true, 4, DPU_FORMAT_FLAG_DX | DPU_FORMAT_FLAG_COMPRESSED,
-		DPU_FETCH_UBWC, 2, DPU_TILE_HEIGHT_UBWC),
-
-	PSEUDO_YUV_FMT_TILED(NV12,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C1_B_Cb, C2_R_Cr,
-		DPU_CHROMA_420, DPU_FORMAT_FLAG_YUV |
-				DPU_FORMAT_FLAG_COMPRESSED,
-		DPU_FETCH_UBWC, 4, DPU_TILE_HEIGHT_NV12),
-
-	PSEUDO_YUV_FMT_TILED(P010,
-		0, COLOR_8BIT, COLOR_8BIT, COLOR_8BIT,
-		C1_B_Cb, C2_R_Cr,
-		DPU_CHROMA_420, DPU_FORMAT_FLAG_DX |
-				DPU_FORMAT_FLAG_YUV |
-				DPU_FORMAT_FLAG_COMPRESSED,
-		DPU_FETCH_UBWC, 4, DPU_TILE_HEIGHT_UBWC),
-};
-
 /* _dpu_get_v_h_subsample_rate - Get subsample rates for all formats we support
  *   Note: Not using the drm_format_*_subsampling since we have formats
  */
 static void _dpu_get_v_h_subsample_rate(
-	enum dpu_chroma_samp_type chroma_sample,
+	enum mdp_chroma_samp_type chroma_sample,
 	uint32_t *v_sample,
 	uint32_t *h_sample)
 {
@@ -574,15 +38,15 @@ static void _dpu_get_v_h_subsample_rate(
 		return;
 
 	switch (chroma_sample) {
-	case DPU_CHROMA_H2V1:
+	case CHROMA_H2V1:
 		*v_sample = 1;
 		*h_sample = 2;
 		break;
-	case DPU_CHROMA_H1V2:
+	case CHROMA_H1V2:
 		*v_sample = 2;
 		*h_sample = 1;
 		break;
-	case DPU_CHROMA_420:
+	case CHROMA_420:
 		*v_sample = 2;
 		*h_sample = 2;
 		break;
@@ -593,7 +57,7 @@ static void _dpu_get_v_h_subsample_rate(
 	}
 }
 
-static int _dpu_format_get_media_color_ubwc(const struct dpu_format *fmt)
+static int _dpu_format_get_media_color_ubwc(const struct msm_format *fmt)
 {
 	static const struct dpu_media_color_map dpu_media_ubwc_map[] = {
 		{DRM_FORMAT_ABGR8888, COLOR_FMT_RGBA8888_UBWC},
@@ -609,10 +73,10 @@ static int _dpu_format_get_media_color_ubwc(const struct dpu_format *fmt)
 	int color_fmt = -1;
 	int i;
 
-	if (fmt->base.pixel_format == DRM_FORMAT_NV12 ||
-	    fmt->base.pixel_format == DRM_FORMAT_P010) {
-		if (DPU_FORMAT_IS_DX(fmt)) {
-			if (fmt->unpack_tight)
+	if (fmt->pixel_format == DRM_FORMAT_NV12 ||
+	    fmt->pixel_format == DRM_FORMAT_P010) {
+		if (MSM_FORMAT_IS_DX(fmt)) {
+			if (fmt->flags & MSM_FORMAT_FLAG_UNPACK_TIGHT)
 				color_fmt = COLOR_FMT_NV12_BPP10_UBWC;
 			else
 				color_fmt = COLOR_FMT_P010_UBWC;
@@ -622,7 +86,7 @@ static int _dpu_format_get_media_color_ubwc(const struct dpu_format *fmt)
 	}
 
 	for (i = 0; i < ARRAY_SIZE(dpu_media_ubwc_map); ++i)
-		if (fmt->base.pixel_format == dpu_media_ubwc_map[i].format) {
+		if (fmt->pixel_format == dpu_media_ubwc_map[i].format) {
 			color_fmt = dpu_media_ubwc_map[i].color;
 			break;
 		}
@@ -630,14 +94,14 @@ static int _dpu_format_get_media_color_ubwc(const struct dpu_format *fmt)
 }
 
 static int _dpu_format_get_plane_sizes_ubwc(
-		const struct dpu_format *fmt,
+		const struct msm_format *fmt,
 		const uint32_t width,
 		const uint32_t height,
 		struct dpu_hw_fmt_layout *layout)
 {
 	int i;
 	int color;
-	bool meta = DPU_FORMAT_IS_UBWC(fmt);
+	bool meta = MSM_FORMAT_IS_UBWC(fmt);
 
 	memset(layout, 0, sizeof(struct dpu_hw_fmt_layout));
 	layout->format = fmt;
@@ -647,12 +111,12 @@ static int _dpu_format_get_plane_sizes_ubwc(
 
 	color = _dpu_format_get_media_color_ubwc(fmt);
 	if (color < 0) {
-		DRM_ERROR("UBWC format not supported for fmt: %4.4s\n",
-			(char *)&fmt->base.pixel_format);
+		DRM_ERROR("UBWC format not supported for fmt: %p4cc\n",
+			  &fmt->pixel_format);
 		return -EINVAL;
 	}
 
-	if (DPU_FORMAT_IS_YUV(layout->format)) {
+	if (MSM_FORMAT_IS_YUV(layout->format)) {
 		uint32_t y_sclines, uv_sclines;
 		uint32_t y_meta_scanlines = 0;
 		uint32_t uv_meta_scanlines = 0;
@@ -709,7 +173,7 @@ done:
 }
 
 static int _dpu_format_get_plane_sizes_linear(
-		const struct dpu_format *fmt,
+		const struct msm_format *fmt,
 		const uint32_t width,
 		const uint32_t height,
 		struct dpu_hw_fmt_layout *layout,
@@ -724,7 +188,7 @@ static int _dpu_format_get_plane_sizes_linear(
 	layout->num_planes = fmt->num_planes;
 
 	/* Due to memset above, only need to set planes of interest */
-	if (fmt->fetch_planes == DPU_PLANE_INTERLEAVED) {
+	if (fmt->fetch_type == MDP_PLANE_INTERLEAVED) {
 		layout->num_planes = 1;
 		layout->plane_size[0] = width * height * layout->format->bpp;
 		layout->plane_pitch[0] = width * layout->format->bpp;
@@ -742,8 +206,8 @@ static int _dpu_format_get_plane_sizes_linear(
 			return -EINVAL;
 		}
 
-		if ((fmt->base.pixel_format == DRM_FORMAT_NV12) &&
-			(DPU_FORMAT_IS_DX(fmt)))
+		if ((fmt->pixel_format == DRM_FORMAT_NV12) &&
+			(MSM_FORMAT_IS_DX(fmt)))
 			bpp = 2;
 		layout->plane_pitch[0] = width * bpp;
 		layout->plane_pitch[1] = layout->plane_pitch[0] / h_subsample;
@@ -751,7 +215,7 @@ static int _dpu_format_get_plane_sizes_linear(
 		layout->plane_size[1] = layout->plane_pitch[1] *
 				(height / v_subsample);
 
-		if (fmt->fetch_planes == DPU_PLANE_PSEUDO_PLANAR) {
+		if (fmt->fetch_type == MDP_PLANE_PSEUDO_PLANAR) {
 			layout->num_planes = 2;
 			layout->plane_size[1] *= 2;
 			layout->plane_pitch[1] *= 2;
@@ -781,7 +245,7 @@ static int _dpu_format_get_plane_sizes_linear(
 }
 
 static int dpu_format_get_plane_sizes(
-		const struct dpu_format *fmt,
+		const struct msm_format *fmt,
 		const uint32_t w,
 		const uint32_t h,
 		struct dpu_hw_fmt_layout *layout,
@@ -797,7 +261,7 @@ static int dpu_format_get_plane_sizes(
 		return -ERANGE;
 	}
 
-	if (DPU_FORMAT_IS_UBWC(fmt) || DPU_FORMAT_IS_TILE(fmt))
+	if (MSM_FORMAT_IS_UBWC(fmt) || MSM_FORMAT_IS_TILE(fmt))
 		return _dpu_format_get_plane_sizes_ubwc(fmt, w, h, layout);
 
 	return _dpu_format_get_plane_sizes_linear(fmt, w, h, layout, pitches);
@@ -823,10 +287,10 @@ static int _dpu_format_populate_addrs_ubwc(
 		return -EFAULT;
 	}
 
-	meta = DPU_FORMAT_IS_UBWC(layout->format);
+	meta = MSM_FORMAT_IS_UBWC(layout->format);
 
 	/* Per-format logic for verifying active planes */
-	if (DPU_FORMAT_IS_YUV(layout->format)) {
+	if (MSM_FORMAT_IS_YUV(layout->format)) {
 		/************************************************/
 		/*      UBWC            **                      */
 		/*      buffer          **      DPU PLANE       */
@@ -942,7 +406,7 @@ int dpu_format_populate_layout(
 		return -ERANGE;
 	}
 
-	layout->format = to_dpu_format(msm_framebuffer_format(fb));
+	layout->format = msm_framebuffer_format(fb);
 
 	/* Populate the plane sizes etc via get_format */
 	ret = dpu_format_get_plane_sizes(layout->format, fb->width, fb->height,
@@ -951,8 +415,8 @@ int dpu_format_populate_layout(
 		return ret;
 
 	/* Populate the addresses given the fb */
-	if (DPU_FORMAT_IS_UBWC(layout->format) ||
-			DPU_FORMAT_IS_TILE(layout->format))
+	if (MSM_FORMAT_IS_UBWC(layout->format) ||
+			MSM_FORMAT_IS_TILE(layout->format))
 		ret = _dpu_format_populate_addrs_ubwc(aspace, fb, layout);
 	else
 		ret = _dpu_format_populate_addrs_linear(aspace, fb, layout);
@@ -962,23 +426,21 @@ int dpu_format_populate_layout(
 
 int dpu_format_check_modified_format(
 		const struct msm_kms *kms,
-		const struct msm_format *msm_fmt,
+		const struct msm_format *fmt,
 		const struct drm_mode_fb_cmd2 *cmd,
 		struct drm_gem_object **bos)
 {
 	const struct drm_format_info *info;
-	const struct dpu_format *fmt;
 	struct dpu_hw_fmt_layout layout;
 	uint32_t bos_total_size = 0;
 	int ret, i;
 
-	if (!msm_fmt || !cmd || !bos) {
+	if (!fmt || !cmd || !bos) {
 		DRM_ERROR("invalid arguments\n");
 		return -EINVAL;
 	}
 
-	fmt = to_dpu_format(msm_fmt);
-	info = drm_format_info(fmt->base.pixel_format);
+	info = drm_format_info(fmt->pixel_format);
 	if (!info)
 		return -EINVAL;
 
@@ -1004,65 +466,3 @@ int dpu_format_check_modified_format(
 
 	return 0;
 }
-
-const struct dpu_format *dpu_get_dpu_format_ext(
-		const uint32_t format,
-		const uint64_t modifier)
-{
-	uint32_t i = 0;
-	const struct dpu_format *fmt = NULL;
-	const struct dpu_format *map = NULL;
-	ssize_t map_size = 0;
-
-	/*
-	 * Currently only support exactly zero or one modifier.
-	 * All planes use the same modifier.
-	 */
-	DRM_DEBUG_ATOMIC("plane format modifier 0x%llX\n", modifier);
-
-	switch (modifier) {
-	case 0:
-		map = dpu_format_map;
-		map_size = ARRAY_SIZE(dpu_format_map);
-		break;
-	case DRM_FORMAT_MOD_QCOM_COMPRESSED:
-		map = dpu_format_map_ubwc;
-		map_size = ARRAY_SIZE(dpu_format_map_ubwc);
-		DRM_DEBUG_ATOMIC("found fmt: %4.4s  DRM_FORMAT_MOD_QCOM_COMPRESSED\n",
-				(char *)&format);
-		break;
-	default:
-		DPU_ERROR("unsupported format modifier %llX\n", modifier);
-		return NULL;
-	}
-
-	for (i = 0; i < map_size; i++) {
-		if (format == map[i].base.pixel_format) {
-			fmt = &map[i];
-			break;
-		}
-	}
-
-	if (fmt == NULL)
-		DPU_ERROR("unsupported fmt: %4.4s modifier 0x%llX\n",
-			(char *)&format, modifier);
-	else
-		DRM_DEBUG_ATOMIC("fmt %4.4s mod 0x%llX ubwc %d yuv %d\n",
-				(char *)&format, modifier,
-				DPU_FORMAT_IS_UBWC(fmt),
-				DPU_FORMAT_IS_YUV(fmt));
-
-	return fmt;
-}
-
-const struct msm_format *dpu_get_msm_format(
-		struct msm_kms *kms,
-		const uint32_t format,
-		const uint64_t modifiers)
-{
-	const struct dpu_format *fmt = dpu_get_dpu_format_ext(format,
-			modifiers);
-	if (fmt)
-		return &fmt->base;
-	return NULL;
-}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.h
index 84b8b3289f18..210d0ed5f0af 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.h
@@ -10,17 +10,6 @@
 #include "dpu_hw_mdss.h"
 
 /**
- * dpu_get_dpu_format_ext() - Returns dpu format structure pointer.
- * @format:          DRM FourCC Code
- * @modifiers:       format modifier array from client, one per plane
- */
-const struct dpu_format *dpu_get_dpu_format_ext(
-		const uint32_t format,
-		const uint64_t modifier);
-
-#define dpu_get_dpu_format(f) dpu_get_dpu_format_ext(f, 0)
-
-/**
  * dpu_find_format - validate if the pixel format is supported
  * @format:		dpu format
  * @supported_formats:	supported formats by dpu HW
@@ -43,22 +32,10 @@ static inline bool dpu_find_format(u32 format, const u32 *supported_formats,
 }
 
 /**
- * dpu_get_msm_format - get an dpu_format by its msm_format base
- *                     callback function registers with the msm_kms layer
- * @kms:             kms driver
- * @format:          DRM FourCC Code
- * @modifiers:       data layout modifier
- */
-const struct msm_format *dpu_get_msm_format(
-		struct msm_kms *kms,
-		const uint32_t format,
-		const uint64_t modifiers);
-
-/**
  * dpu_format_check_modified_format - validate format and buffers for
  *                   dpu non-standard, i.e. modified format
  * @kms:             kms driver
- * @msm_fmt:         pointer to the msm_fmt base pointer of an dpu_format
+ * @msm_fmt:         pointer to the msm_fmt base pointer of an msm_format
  * @cmd:             fb_cmd2 structure user request
  * @bos:             gem buffer object list
  *
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.c
index 9016b3ade6bc..55d2768a6d4d 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.c
@@ -170,7 +170,7 @@ static int dpu_hw_cdm_setup_cdwn(struct dpu_hw_cdm *ctx, struct dpu_hw_cdm_cfg *
 static int dpu_hw_cdm_enable(struct dpu_hw_cdm *ctx, struct dpu_hw_cdm_cfg *cdm)
 {
 	struct dpu_hw_blk_reg_map *c = &ctx->hw;
-	const struct dpu_format *fmt;
+	const struct msm_format *fmt;
 	u32 opmode = 0;
 	u32 csc = 0;
 
@@ -179,14 +179,14 @@ static int dpu_hw_cdm_enable(struct dpu_hw_cdm *ctx, struct dpu_hw_cdm_cfg *cdm)
 
 	fmt = cdm->output_fmt;
 
-	if (!DPU_FORMAT_IS_YUV(fmt))
+	if (!MSM_FORMAT_IS_YUV(fmt))
 		return -EINVAL;
 
 	dpu_hw_csc_setup(&ctx->hw, CDM_CSC_10_MATRIX_COEFF_0, cdm->csc_cfg, true);
 	dpu_hw_cdm_setup_cdwn(ctx, cdm);
 
 	if (cdm->output_type == CDM_CDWN_OUTPUT_HDMI) {
-		if (fmt->chroma_sample == DPU_CHROMA_H1V2)
+		if (fmt->chroma_sample == CHROMA_H1V2)
 			return -EINVAL; /*unsupported format */
 		opmode = CDM_HDMI_PACK_OP_MODE_EN;
 		opmode |= (fmt->chroma_sample << 1);
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.h
index 348424df87c6..ec71c9886d75 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_cdm.h
@@ -19,7 +19,7 @@ struct dpu_hw_cdm;
  *  @output_bit_depth:     output bit-depth of CDM block
  *  @h_cdwn_type:          downsample type used for horizontal pixels
  *  @v_cdwn_type:          downsample type used for vertical pixels
- *  @output_fmt:           handle to dpu_format of CDM block
+ *  @output_fmt:           handle to msm_format of CDM block
  *  @csc_cfg:              handle to CSC matrix programmed for CDM block
  *  @output_type:          interface to which CDM is paired (HDMI/WB)
  *  @pp_id:                ping-pong block to which CDM is bound to
@@ -30,7 +30,7 @@ struct dpu_hw_cdm_cfg {
 	u32 output_bit_depth;
 	u32 h_cdwn_type;
 	u32 v_cdwn_type;
-	const struct dpu_format *output_fmt;
+	const struct msm_format *output_fmt;
 	const struct dpu_csc_cfg *csc_cfg;
 	u32 output_type;
 	int pp_id;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
index a06f69d0b257..2e50049f2f85 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
@@ -545,6 +545,7 @@ static void dpu_hw_ctl_intf_cfg_v1(struct dpu_hw_ctl *ctx,
 {
 	struct dpu_hw_blk_reg_map *c = &ctx->hw;
 	u32 intf_active = 0;
+	u32 dsc_active = 0;
 	u32 wb_active = 0;
 	u32 mode_sel = 0;
 
@@ -560,6 +561,7 @@ static void dpu_hw_ctl_intf_cfg_v1(struct dpu_hw_ctl *ctx,
 
 	intf_active = DPU_REG_READ(c, CTL_INTF_ACTIVE);
 	wb_active = DPU_REG_READ(c, CTL_WB_ACTIVE);
+	dsc_active = DPU_REG_READ(c, CTL_DSC_ACTIVE);
 
 	if (cfg->intf)
 		intf_active |= BIT(cfg->intf - INTF_0);
@@ -567,17 +569,18 @@ static void dpu_hw_ctl_intf_cfg_v1(struct dpu_hw_ctl *ctx,
 	if (cfg->wb)
 		wb_active |= BIT(cfg->wb - WB_0);
 
+	if (cfg->dsc)
+		dsc_active |= cfg->dsc;
+
 	DPU_REG_WRITE(c, CTL_TOP, mode_sel);
 	DPU_REG_WRITE(c, CTL_INTF_ACTIVE, intf_active);
 	DPU_REG_WRITE(c, CTL_WB_ACTIVE, wb_active);
+	DPU_REG_WRITE(c, CTL_DSC_ACTIVE, dsc_active);
 
 	if (cfg->merge_3d)
 		DPU_REG_WRITE(c, CTL_MERGE_3D_ACTIVE,
 			      BIT(cfg->merge_3d - MERGE_3D_0));
 
-	if (cfg->dsc)
-		DPU_REG_WRITE(c, CTL_DSC_ACTIVE, cfg->dsc);
-
 	if (cfg->cdm)
 		DPU_REG_WRITE(c, CTL_CDM_ACTIVE, cfg->cdm);
 }
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c
index 6a0a74832fb6..b85881aab047 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_interrupts.c
@@ -223,9 +223,11 @@ static void dpu_core_irq_callback_handler(struct dpu_kms *dpu_kms, unsigned int
 
 	VERB("IRQ=[%d, %d]\n", DPU_IRQ_REG(irq_idx), DPU_IRQ_BIT(irq_idx));
 
-	if (!irq_entry->cb)
+	if (!irq_entry->cb) {
 		DRM_ERROR("no registered cb, IRQ=[%d, %d]\n",
 			  DPU_IRQ_REG(irq_idx), DPU_IRQ_BIT(irq_idx));
+		return;
+	}
 
 	atomic_inc(&irq_entry->count);
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c
index 965692ef7892..225c1c7768ff 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c
@@ -96,11 +96,11 @@
 #define INTF_CFG2_DCE_DATA_COMPRESS     BIT(12)
 
 
-static void dpu_hw_intf_setup_timing_engine(struct dpu_hw_intf *ctx,
+static void dpu_hw_intf_setup_timing_engine(struct dpu_hw_intf *intf,
 		const struct dpu_hw_intf_timing_params *p,
-		const struct dpu_format *fmt)
+		const struct msm_format *fmt)
 {
-	struct dpu_hw_blk_reg_map *c = &ctx->hw;
+	struct dpu_hw_blk_reg_map *c = &intf->hw;
 	u32 hsync_period, vsync_period;
 	u32 display_v_start, display_v_end;
 	u32 hsync_start_x, hsync_end_x;
@@ -118,7 +118,7 @@ static void dpu_hw_intf_setup_timing_engine(struct dpu_hw_intf *ctx,
 	/* read interface_cfg */
 	intf_cfg = DPU_REG_READ(c, INTF_CONFIG);
 
-	if (ctx->cap->type == INTF_DP)
+	if (intf->cap->type == INTF_DP)
 		dp_intf = true;
 
 	hsync_period = p->hsync_pulse_width + p->h_back_porch + p->width +
@@ -194,16 +194,16 @@ static void dpu_hw_intf_setup_timing_engine(struct dpu_hw_intf *ctx,
 		(p->vsync_polarity << 1) | /* VSYNC Polarity */
 		(p->hsync_polarity << 0);  /* HSYNC Polarity */
 
-	if (!DPU_FORMAT_IS_YUV(fmt))
-		panel_format = (fmt->bits[C0_G_Y] |
-				(fmt->bits[C1_B_Cb] << 2) |
-				(fmt->bits[C2_R_Cr] << 4) |
+	if (!MSM_FORMAT_IS_YUV(fmt))
+		panel_format = (fmt->bpc_g_y |
+				(fmt->bpc_b_cb << 2) |
+				(fmt->bpc_r_cr << 4) |
 				(0x21 << 8));
 	else
 		/* Interface treats all the pixel data in RGB888 format */
-		panel_format = (COLOR_8BIT |
-				(COLOR_8BIT << 2) |
-				(COLOR_8BIT << 4) |
+		panel_format = (BPC8 |
+				(BPC8 << 2) |
+				(BPC8 << 4) |
 				(0x21 << 8));
 
 	DPU_REG_WRITE(c, INTF_HSYNC_CTL, hsync_ctl);
@@ -223,7 +223,7 @@ static void dpu_hw_intf_setup_timing_engine(struct dpu_hw_intf *ctx,
 	DPU_REG_WRITE(c, INTF_FRAME_LINE_COUNT_EN, 0x3);
 	DPU_REG_WRITE(c, INTF_CONFIG, intf_cfg);
 	DPU_REG_WRITE(c, INTF_PANEL_FORMAT, panel_format);
-	if (ctx->cap->features & BIT(DPU_DATA_HCTL_EN)) {
+	if (intf->cap->features & BIT(DPU_DATA_HCTL_EN)) {
 		/*
 		 * DATA_HCTL_EN controls data timing which can be different from
 		 * video timing. It is recommended to enable it for all cases, except
@@ -518,10 +518,10 @@ static void dpu_hw_intf_disable_autorefresh(struct dpu_hw_intf *intf,
 
 }
 
-static void dpu_hw_intf_program_intf_cmd_cfg(struct dpu_hw_intf *ctx,
+static void dpu_hw_intf_program_intf_cmd_cfg(struct dpu_hw_intf *intf,
 					     struct dpu_hw_intf_cmd_mode_cfg *cmd_mode_cfg)
 {
-	u32 intf_cfg2 = DPU_REG_READ(&ctx->hw, INTF_CONFIG2);
+	u32 intf_cfg2 = DPU_REG_READ(&intf->hw, INTF_CONFIG2);
 
 	if (cmd_mode_cfg->data_compress)
 		intf_cfg2 |= INTF_CFG2_DCE_DATA_COMPRESS;
@@ -529,7 +529,7 @@ static void dpu_hw_intf_program_intf_cmd_cfg(struct dpu_hw_intf *ctx,
 	if (cmd_mode_cfg->wide_bus_en)
 		intf_cfg2 |= INTF_CFG2_DATABUS_WIDEN;
 
-	DPU_REG_WRITE(&ctx->hw, INTF_CONFIG2, intf_cfg2);
+	DPU_REG_WRITE(&intf->hw, INTF_CONFIG2, intf_cfg2);
 }
 
 struct dpu_hw_intf *dpu_hw_intf_init(struct drm_device *dev,
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h
index 6f4c87244f94..f9015c67a574 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h
@@ -81,7 +81,7 @@ struct dpu_hw_intf_cmd_mode_cfg {
 struct dpu_hw_intf_ops {
 	void (*setup_timing_gen)(struct dpu_hw_intf *intf,
 			const struct dpu_hw_intf_timing_params *p,
-			const struct dpu_format *fmt);
+			const struct msm_format *fmt);
 
 	void (*setup_prg_fetch)(struct dpu_hw_intf *intf,
 			const struct dpu_hw_intf_prog_fetch *fetch);
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h
index 5df545904057..66759623fc42 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h
@@ -10,6 +10,8 @@
 
 #include "msm_drv.h"
 
+#include "disp/mdp_format.h"
+
 #define DPU_DBG_NAME			"dpu"
 
 #define DPU_NONE                        0
@@ -35,28 +37,6 @@
 #define DPU_MAX_DE_CURVES		3
 #endif
 
-enum dpu_format_flags {
-	DPU_FORMAT_FLAG_YUV_BIT,
-	DPU_FORMAT_FLAG_DX_BIT,
-	DPU_FORMAT_FLAG_COMPRESSED_BIT,
-	DPU_FORMAT_FLAG_BIT_MAX,
-};
-
-#define DPU_FORMAT_FLAG_YUV		BIT(DPU_FORMAT_FLAG_YUV_BIT)
-#define DPU_FORMAT_FLAG_DX		BIT(DPU_FORMAT_FLAG_DX_BIT)
-#define DPU_FORMAT_FLAG_COMPRESSED	BIT(DPU_FORMAT_FLAG_COMPRESSED_BIT)
-#define DPU_FORMAT_IS_YUV(X)		\
-	(test_bit(DPU_FORMAT_FLAG_YUV_BIT, (X)->flag))
-#define DPU_FORMAT_IS_DX(X)		\
-	(test_bit(DPU_FORMAT_FLAG_DX_BIT, (X)->flag))
-#define DPU_FORMAT_IS_LINEAR(X)		((X)->fetch_mode == DPU_FETCH_LINEAR)
-#define DPU_FORMAT_IS_TILE(X) \
-	(((X)->fetch_mode == DPU_FETCH_UBWC) && \
-			!test_bit(DPU_FORMAT_FLAG_COMPRESSED_BIT, (X)->flag))
-#define DPU_FORMAT_IS_UBWC(X) \
-	(((X)->fetch_mode == DPU_FETCH_UBWC) && \
-			test_bit(DPU_FORMAT_FLAG_COMPRESSED_BIT, (X)->flag))
-
 #define DPU_BLEND_FG_ALPHA_FG_CONST	(0 << 0)
 #define DPU_BLEND_FG_ALPHA_BG_CONST	(1 << 0)
 #define DPU_BLEND_FG_ALPHA_FG_PIXEL	(2 << 0)
@@ -291,67 +271,6 @@ enum dpu_vbif {
 };
 
 /**
- * DPU HW,Component order color map
- */
-enum {
-	C0_G_Y = 0,
-	C1_B_Cb = 1,
-	C2_R_Cr = 2,
-	C3_ALPHA = 3
-};
-
-/**
- * enum dpu_plane_type - defines how the color component pixel packing
- * @DPU_PLANE_INTERLEAVED   : Color components in single plane
- * @DPU_PLANE_PLANAR        : Color component in separate planes
- * @DPU_PLANE_PSEUDO_PLANAR : Chroma components interleaved in separate plane
- */
-enum dpu_plane_type {
-	DPU_PLANE_INTERLEAVED,
-	DPU_PLANE_PLANAR,
-	DPU_PLANE_PSEUDO_PLANAR,
-};
-
-/**
- * enum dpu_chroma_samp_type - chroma sub-samplng type
- * @DPU_CHROMA_RGB   : No chroma subsampling
- * @DPU_CHROMA_H2V1  : Chroma pixels are horizontally subsampled
- * @DPU_CHROMA_H1V2  : Chroma pixels are vertically subsampled
- * @DPU_CHROMA_420   : 420 subsampling
- */
-enum dpu_chroma_samp_type {
-	DPU_CHROMA_RGB,
-	DPU_CHROMA_H2V1,
-	DPU_CHROMA_H1V2,
-	DPU_CHROMA_420
-};
-
-/**
- * dpu_fetch_type - Defines How DPU HW fetches data
- * @DPU_FETCH_LINEAR   : fetch is line by line
- * @DPU_FETCH_TILE     : fetches data in Z order from a tile
- * @DPU_FETCH_UBWC     : fetch and decompress data
- */
-enum dpu_fetch_type {
-	DPU_FETCH_LINEAR,
-	DPU_FETCH_TILE,
-	DPU_FETCH_UBWC
-};
-
-/**
- * Value of enum chosen to fit the number of bits
- * expected by the HW programming.
- */
-enum {
-	COLOR_ALPHA_1BIT = 0,
-	COLOR_ALPHA_4BIT = 1,
-	COLOR_4BIT = 0,
-	COLOR_5BIT = 1, /* No 5-bit Alpha */
-	COLOR_6BIT = 2, /* 6-Bit Alpha also = 2 */
-	COLOR_8BIT = 3, /* 8-Bit Alpha also = 3 */
-};
-
-/**
  * enum dpu_3d_blend_mode
  * Desribes how the 3d data is blended
  * @BLEND_3D_NONE      : 3d blending not enabled
@@ -370,43 +289,6 @@ enum dpu_3d_blend_mode {
 	BLEND_3D_MAX
 };
 
-/** struct dpu_format - defines the format configuration which
- * allows DPU HW to correctly fetch and decode the format
- * @base: base msm_format structure containing fourcc code
- * @fetch_planes: how the color components are packed in pixel format
- * @element: element color ordering
- * @bits: element bit widths
- * @chroma_sample: chroma sub-samplng type
- * @unpack_align_msb: unpack aligned, 0 to LSB, 1 to MSB
- * @unpack_tight: 0 for loose, 1 for tight
- * @unpack_count: 0 = 1 component, 1 = 2 component
- * @bpp: bytes per pixel
- * @alpha_enable: whether the format has an alpha channel
- * @num_planes: number of planes (including meta data planes)
- * @fetch_mode: linear, tiled, or ubwc hw fetch behavior
- * @flag: usage bit flags
- * @tile_width: format tile width
- * @tile_height: format tile height
- */
-struct dpu_format {
-	struct msm_format base;
-	enum dpu_plane_type fetch_planes;
-	u8 element[DPU_MAX_PLANES];
-	u8 bits[DPU_MAX_PLANES];
-	enum dpu_chroma_samp_type chroma_sample;
-	u8 unpack_align_msb;
-	u8 unpack_tight;
-	u8 unpack_count;
-	u8 bpp;
-	u8 alpha_enable;
-	u8 num_planes;
-	enum dpu_fetch_type fetch_mode;
-	DECLARE_BITMAP(flag, DPU_FORMAT_FLAG_BIT_MAX);
-	u16 tile_width;
-	u16 tile_height;
-};
-#define to_dpu_format(x) container_of(x, struct dpu_format, base)
-
 /**
  * struct dpu_hw_fmt_layout - format information of the source pixel data
  * @format: pixel format parameters
@@ -419,7 +301,7 @@ struct dpu_format {
  * @plane_pitch: pitch of each plane
  */
 struct dpu_hw_fmt_layout {
-	const struct dpu_format *format;
+	const struct msm_format *format;
 	uint32_t num_planes;
 	uint32_t width;
 	uint32_t height;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
index 0bf8a83e8df3..2c720f1fc1b2 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
@@ -2,6 +2,8 @@
 /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
  */
 
+#include <linux/debugfs.h>
+
 #include "dpu_hwio.h"
 #include "dpu_hw_catalog.h"
 #include "dpu_hw_lm.h"
@@ -206,7 +208,7 @@ static void _sspp_setup_csc10_opmode(struct dpu_hw_sspp *ctx,
  * Setup source pixel format, flip,
  */
 static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe,
-		const struct dpu_format *fmt, u32 flags)
+		const struct msm_format *fmt, u32 flags)
 {
 	struct dpu_hw_sspp *ctx = pipe->sspp;
 	struct dpu_hw_blk_reg_map *c;
@@ -241,20 +243,20 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe,
 
 	chroma_samp = fmt->chroma_sample;
 	if (flags & DPU_SSPP_SOURCE_ROTATED_90) {
-		if (chroma_samp == DPU_CHROMA_H2V1)
-			chroma_samp = DPU_CHROMA_H1V2;
-		else if (chroma_samp == DPU_CHROMA_H1V2)
-			chroma_samp = DPU_CHROMA_H2V1;
+		if (chroma_samp == CHROMA_H2V1)
+			chroma_samp = CHROMA_H1V2;
+		else if (chroma_samp == CHROMA_H1V2)
+			chroma_samp = CHROMA_H2V1;
 	}
 
-	src_format = (chroma_samp << 23) | (fmt->fetch_planes << 19) |
-		(fmt->bits[C3_ALPHA] << 6) | (fmt->bits[C2_R_Cr] << 4) |
-		(fmt->bits[C1_B_Cb] << 2) | (fmt->bits[C0_G_Y] << 0);
+	src_format = (chroma_samp << 23) | (fmt->fetch_type << 19) |
+		(fmt->bpc_a << 6) | (fmt->bpc_r_cr << 4) |
+		(fmt->bpc_b_cb << 2) | (fmt->bpc_g_y << 0);
 
 	if (flags & DPU_SSPP_ROT_90)
 		src_format |= BIT(11); /* ROT90 */
 
-	if (fmt->alpha_enable && fmt->fetch_planes == DPU_PLANE_INTERLEAVED)
+	if (fmt->alpha_enable && fmt->fetch_type == MDP_PLANE_INTERLEAVED)
 		src_format |= BIT(8); /* SRCC3_EN */
 
 	if (flags & DPU_SSPP_SOLID_FILL)
@@ -263,12 +265,12 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe,
 	unpack = (fmt->element[3] << 24) | (fmt->element[2] << 16) |
 		(fmt->element[1] << 8) | (fmt->element[0] << 0);
 	src_format |= ((fmt->unpack_count - 1) << 12) |
-		(fmt->unpack_tight << 17) |
-		(fmt->unpack_align_msb << 18) |
+		((fmt->flags & MSM_FORMAT_FLAG_UNPACK_TIGHT ? 1 : 0) << 17) |
+		((fmt->flags & MSM_FORMAT_FLAG_UNPACK_ALIGN_MSB ? 1 : 0) << 18) |
 		((fmt->bpp - 1) << 9);
 
-	if (fmt->fetch_mode != DPU_FETCH_LINEAR) {
-		if (DPU_FORMAT_IS_UBWC(fmt))
+	if (fmt->fetch_mode != MDP_FETCH_LINEAR) {
+		if (MSM_FORMAT_IS_UBWC(fmt))
 			opmode |= MDSS_MDP_OP_BWC_EN;
 		src_format |= (fmt->fetch_mode & 3) << 30; /*FRAME_FORMAT */
 		DPU_REG_WRITE(c, SSPP_FETCH_CONFIG,
@@ -295,7 +297,7 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe,
 			break;
 		case UBWC_4_0:
 			DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL,
-					DPU_FORMAT_IS_YUV(fmt) ? 0 : BIT(30));
+					MSM_FORMAT_IS_YUV(fmt) ? 0 : BIT(30));
 			break;
 		}
 	}
@@ -303,20 +305,20 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe,
 	opmode |= MDSS_MDP_OP_PE_OVERRIDE;
 
 	/* if this is YUV pixel format, enable CSC */
-	if (DPU_FORMAT_IS_YUV(fmt))
+	if (MSM_FORMAT_IS_YUV(fmt))
 		src_format |= BIT(15);
 
-	if (DPU_FORMAT_IS_DX(fmt))
+	if (MSM_FORMAT_IS_DX(fmt))
 		src_format |= BIT(14);
 
 	/* update scaler opmode, if appropriate */
 	if (test_bit(DPU_SSPP_CSC, &ctx->cap->features))
 		_sspp_setup_opmode(ctx, VIG_OP_CSC_EN | VIG_OP_CSC_SRC_DATAFMT,
-			DPU_FORMAT_IS_YUV(fmt));
+			MSM_FORMAT_IS_YUV(fmt));
 	else if (test_bit(DPU_SSPP_CSC_10BIT, &ctx->cap->features))
 		_sspp_setup_csc10_opmode(ctx,
 			VIG_CSC_10_EN | VIG_CSC_10_SRC_DATAFMT,
-			DPU_FORMAT_IS_YUV(fmt));
+			MSM_FORMAT_IS_YUV(fmt));
 
 	DPU_REG_WRITE(c, format_off, src_format);
 	DPU_REG_WRITE(c, unpack_pat_off, unpack);
@@ -385,7 +387,7 @@ static void dpu_hw_sspp_setup_pe_config(struct dpu_hw_sspp *ctx,
 
 static void _dpu_hw_sspp_setup_scaler3(struct dpu_hw_sspp *ctx,
 		struct dpu_hw_scaler3_cfg *scaler3_cfg,
-		const struct dpu_format *format)
+		const struct msm_format *format)
 {
 	if (!ctx || !scaler3_cfg)
 		return;
@@ -556,7 +558,7 @@ static void dpu_hw_sspp_setup_qos_ctrl(struct dpu_hw_sspp *ctx,
 }
 
 static void dpu_hw_sspp_setup_cdp(struct dpu_sw_pipe *pipe,
-				  const struct dpu_format *fmt,
+				  const struct msm_format *fmt,
 				  bool enable)
 {
 	struct dpu_hw_sspp *ctx = pipe->sspp;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h
index b7dc52312c39..4a910b808687 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h
@@ -183,7 +183,7 @@ struct dpu_hw_sspp_ops {
 	 * @flags: Extra flags for format config
 	 */
 	void (*setup_format)(struct dpu_sw_pipe *pipe,
-			     const struct dpu_format *fmt, u32 flags);
+			     const struct msm_format *fmt, u32 flags);
 
 	/**
 	 * setup_rects - setup pipe ROI rectangles
@@ -279,7 +279,7 @@ struct dpu_hw_sspp_ops {
 	 */
 	void (*setup_scaler)(struct dpu_hw_sspp *ctx,
 		struct dpu_hw_scaler3_cfg *scaler3_cfg,
-		const struct dpu_format *format);
+		const struct msm_format *format);
 
 	/**
 	 * setup_cdp - setup client driven prefetch
@@ -288,7 +288,7 @@ struct dpu_hw_sspp_ops {
 	 * @enable: whether the CDP should be enabled for this pipe
 	 */
 	void (*setup_cdp)(struct dpu_sw_pipe *pipe,
-			  const struct dpu_format *fmt,
+			  const struct msm_format *fmt,
 			  bool enable);
 };
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c
index dd475827314e..486be346d40d 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.c
@@ -282,7 +282,7 @@ static void _dpu_hw_setup_scaler3_de(struct dpu_hw_blk_reg_map *c,
 void dpu_hw_setup_scaler3(struct dpu_hw_blk_reg_map *c,
 		struct dpu_hw_scaler3_cfg *scaler3_cfg,
 		u32 scaler_offset, u32 scaler_version,
-		const struct dpu_format *format)
+		const struct msm_format *format)
 {
 	u32 op_mode = 0;
 	u32 phase_init, preload, src_y_rgb, src_uv, dst;
@@ -293,7 +293,7 @@ void dpu_hw_setup_scaler3(struct dpu_hw_blk_reg_map *c,
 	op_mode |= BIT(0);
 	op_mode |= (scaler3_cfg->y_rgb_filter_cfg & 0x3) << 16;
 
-	if (format && DPU_FORMAT_IS_YUV(format)) {
+	if (format && MSM_FORMAT_IS_YUV(format)) {
 		op_mode |= BIT(12);
 		op_mode |= (scaler3_cfg->uv_filter_cfg & 0x3) << 24;
 	}
@@ -367,7 +367,7 @@ void dpu_hw_setup_scaler3(struct dpu_hw_blk_reg_map *c,
 	DPU_REG_WRITE(c, QSEED3_DST_SIZE + scaler_offset, dst);
 
 end:
-	if (format && !DPU_FORMAT_IS_DX(format))
+	if (format && !MSM_FORMAT_IS_DX(format))
 		op_mode |= BIT(14);
 
 	if (format && format->alpha_enable) {
@@ -522,16 +522,16 @@ int dpu_hw_collect_misr(struct dpu_hw_blk_reg_map *c,
 #define CDP_PRELOAD_AHEAD_64	BIT(3)
 
 void dpu_setup_cdp(struct dpu_hw_blk_reg_map *c, u32 offset,
-		   const struct dpu_format *fmt, bool enable)
+		   const struct msm_format *fmt, bool enable)
 {
 	u32 cdp_cntl = CDP_PRELOAD_AHEAD_64;
 
 	if (enable)
 		cdp_cntl |= CDP_ENABLE;
-	if (DPU_FORMAT_IS_UBWC(fmt))
+	if (MSM_FORMAT_IS_UBWC(fmt))
 		cdp_cntl |= CDP_UBWC_META_ENABLE;
-	if (DPU_FORMAT_IS_UBWC(fmt) ||
-	    DPU_FORMAT_IS_TILE(fmt))
+	if (MSM_FORMAT_IS_UBWC(fmt) ||
+	    MSM_FORMAT_IS_TILE(fmt))
 		cdp_cntl |= CDP_TILE_AMORTIZE_ENABLE;
 
 	DPU_REG_WRITE(c, offset, cdp_cntl);
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h
index 64ded69fa903..67b08e99335d 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h
@@ -344,14 +344,14 @@ void *dpu_hw_util_get_dir(void);
 void dpu_hw_setup_scaler3(struct dpu_hw_blk_reg_map *c,
 		struct dpu_hw_scaler3_cfg *scaler3_cfg,
 		u32 scaler_offset, u32 scaler_version,
-		const struct dpu_format *format);
+		const struct msm_format *format);
 
 void dpu_hw_csc_setup(struct dpu_hw_blk_reg_map  *c,
 		u32 csc_reg_off,
 		const struct dpu_csc_cfg *data, bool csc10);
 
 void dpu_setup_cdp(struct dpu_hw_blk_reg_map *c, u32 offset,
-		   const struct dpu_format *fmt, bool enable);
+		   const struct msm_format *fmt, bool enable);
 
 u64 _dpu_hw_get_qos_lut(const struct dpu_qos_lut_tbl *tbl,
 		u32 total_fl);
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.c
index e75995f7fcea..93ff01c889b5 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.c
@@ -67,7 +67,7 @@ static void dpu_hw_wb_setup_format(struct dpu_hw_wb *ctx,
 		struct dpu_hw_wb_cfg *data)
 {
 	struct dpu_hw_blk_reg_map *c = &ctx->hw;
-	const struct dpu_format *fmt = data->dest.format;
+	const struct msm_format *fmt = data->dest.format;
 	u32 dst_format, pattern, ystride0, ystride1, outsize, chroma_samp;
 	u32 write_config = 0;
 	u32 opmode = 0;
@@ -76,20 +76,20 @@ static void dpu_hw_wb_setup_format(struct dpu_hw_wb *ctx,
 	chroma_samp = fmt->chroma_sample;
 
 	dst_format = (chroma_samp << 23) |
-		(fmt->fetch_planes << 19) |
-		(fmt->bits[C3_ALPHA] << 6) |
-		(fmt->bits[C2_R_Cr] << 4) |
-		(fmt->bits[C1_B_Cb] << 2) |
-		(fmt->bits[C0_G_Y] << 0);
+		(fmt->fetch_type << 19) |
+		(fmt->bpc_a << 6) |
+		(fmt->bpc_r_cr << 4) |
+		(fmt->bpc_b_cb << 2) |
+		(fmt->bpc_g_y << 0);
 
-	if (fmt->bits[C3_ALPHA] || fmt->alpha_enable) {
+	if (fmt->bpc_a || fmt->alpha_enable) {
 		dst_format |= BIT(8); /* DSTC3_EN */
 		if (!fmt->alpha_enable ||
 			!(ctx->caps->features & BIT(DPU_WB_PIPE_ALPHA)))
 			dst_format |= BIT(14); /* DST_ALPHA_X */
 	}
 
-	if (DPU_FORMAT_IS_YUV(fmt))
+	if (MSM_FORMAT_IS_YUV(fmt))
 		dst_format |= BIT(15);
 
 	pattern = (fmt->element[3] << 24) |
@@ -97,8 +97,8 @@ static void dpu_hw_wb_setup_format(struct dpu_hw_wb *ctx,
 		(fmt->element[1] << 8)  |
 		(fmt->element[0] << 0);
 
-	dst_format |= (fmt->unpack_align_msb << 18) |
-		(fmt->unpack_tight << 17) |
+	dst_format |= ((fmt->flags & MSM_FORMAT_FLAG_UNPACK_ALIGN_MSB ? 1 : 0) << 18) |
+		((fmt->flags & MSM_FORMAT_FLAG_UNPACK_TIGHT ? 1 : 0) << 17) |
 		((fmt->unpack_count - 1) << 12) |
 		((fmt->bpp - 1) << 9);
 
@@ -149,7 +149,7 @@ static void dpu_hw_wb_setup_qos_lut(struct dpu_hw_wb *ctx,
 }
 
 static void dpu_hw_wb_setup_cdp(struct dpu_hw_wb *ctx,
-				const struct dpu_format *fmt,
+				const struct msm_format *fmt,
 				bool enable)
 {
 	if (!ctx)
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.h
index e671796ea379..37497473e16c 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_wb.h
@@ -46,7 +46,7 @@ struct dpu_hw_wb_ops {
 			struct dpu_hw_qos_cfg *cfg);
 
 	void (*setup_cdp)(struct dpu_hw_wb *ctx,
-			  const struct dpu_format *fmt,
+			  const struct msm_format *fmt,
 			  bool enable);
 
 	bool (*setup_clk_force_ctrl)(struct dpu_hw_wb *ctx,
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
index a1f5d7c4ab91..1955848b1b78 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
@@ -348,9 +348,18 @@ static void dpu_kms_global_destroy_state(struct drm_private_obj *obj,
 	kfree(dpu_state);
 }
 
+static void dpu_kms_global_print_state(struct drm_printer *p,
+				       const struct drm_private_state *state)
+{
+	const struct dpu_global_state *global_state = to_dpu_global_state(state);
+
+	dpu_rm_print_state(p, global_state);
+}
+
 static const struct drm_private_state_funcs dpu_kms_global_state_funcs = {
 	.atomic_duplicate_state = dpu_kms_global_duplicate_state,
 	.atomic_destroy_state = dpu_kms_global_destroy_state,
+	.atomic_print_state = dpu_kms_global_print_state,
 };
 
 static int dpu_kms_global_obj_init(struct dpu_kms *dpu_kms)
@@ -364,6 +373,9 @@ static int dpu_kms_global_obj_init(struct dpu_kms *dpu_kms)
 	drm_atomic_private_obj_init(dpu_kms->dev, &dpu_kms->global_state,
 				    &state->base,
 				    &dpu_kms_global_state_funcs);
+
+	state->rm = &dpu_kms->rm;
+
 	return 0;
 }
 
@@ -970,7 +982,6 @@ static const struct msm_kms_funcs kms_funcs = {
 	.enable_vblank   = dpu_kms_enable_vblank,
 	.disable_vblank  = dpu_kms_disable_vblank,
 	.check_modified_format = dpu_format_check_modified_format,
-	.get_format      = dpu_get_msm_format,
 	.destroy         = dpu_kms_destroy,
 	.snapshot        = dpu_kms_mdp_snapshot,
 #ifdef CONFIG_DEBUG_FS
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
index b5db3fc76ca6..e2adc937ea63 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
@@ -130,6 +130,8 @@ struct vsync_info {
 struct dpu_global_state {
 	struct drm_private_state base;
 
+	struct dpu_rm *rm;
+
 	uint32_t pingpong_to_enc_id[PINGPONG_MAX - PINGPONG_0];
 	uint32_t mixer_to_enc_id[LM_MAX - LM_0];
 	uint32_t ctl_to_enc_id[CTL_MAX - CTL_0];
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
index ff975ad51145..1c3a2657450c 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
@@ -113,7 +113,7 @@ static struct dpu_kms *_dpu_plane_get_kms(struct drm_plane *plane)
  * Prefill BW Equation: line src bytes * line_time
  */
 static u64 _dpu_plane_calc_bw(const struct dpu_mdss_cfg *catalog,
-	const struct dpu_format *fmt,
+	const struct msm_format *fmt,
 	const struct drm_display_mode *mode,
 	struct dpu_sw_pipe_cfg *pipe_cfg)
 {
@@ -195,7 +195,7 @@ static u64 _dpu_plane_calc_clk(const struct drm_display_mode *mode,
 static int _dpu_plane_calc_fill_level(struct drm_plane *plane,
 		struct dpu_sw_pipe *pipe,
 		enum dpu_qos_lut_usage lut_usage,
-		const struct dpu_format *fmt, u32 src_width)
+		const struct msm_format *fmt, u32 src_width)
 {
 	struct dpu_plane *pdpu;
 	u32 fixed_buff_size;
@@ -214,8 +214,8 @@ static int _dpu_plane_calc_fill_level(struct drm_plane *plane,
 
 	/* FIXME: in multirect case account for the src_width of all the planes */
 
-	if (fmt->fetch_planes == DPU_PLANE_PSEUDO_PLANAR) {
-		if (fmt->chroma_sample == DPU_CHROMA_420) {
+	if (fmt->fetch_type == MDP_PLANE_PSEUDO_PLANAR) {
+		if (fmt->chroma_sample == CHROMA_420) {
 			/* NV12 */
 			total_fl = (fixed_buff_size / 2) /
 				((src_width + 32) * fmt->bpp);
@@ -234,9 +234,9 @@ static int _dpu_plane_calc_fill_level(struct drm_plane *plane,
 		}
 	}
 
-	DPU_DEBUG_PLANE(pdpu, "pnum:%d fmt: %4.4s w:%u fl:%u\n",
+	DPU_DEBUG_PLANE(pdpu, "pnum:%d fmt: %p4cc w:%u fl:%u\n",
 			pipe->sspp->idx - SSPP_VIG0,
-			(char *)&fmt->base.pixel_format,
+			&fmt->pixel_format,
 			src_width, total_fl);
 
 	return total_fl;
@@ -251,7 +251,7 @@ static int _dpu_plane_calc_fill_level(struct drm_plane *plane,
  */
 static void _dpu_plane_set_qos_lut(struct drm_plane *plane,
 		struct dpu_sw_pipe *pipe,
-		const struct dpu_format *fmt, struct dpu_sw_pipe_cfg *pipe_cfg)
+		const struct msm_format *fmt, struct dpu_sw_pipe_cfg *pipe_cfg)
 {
 	struct dpu_plane *pdpu = to_dpu_plane(plane);
 	struct dpu_hw_qos_cfg cfg;
@@ -260,7 +260,7 @@ static void _dpu_plane_set_qos_lut(struct drm_plane *plane,
 	if (!pdpu->is_rt_pipe) {
 		lut_usage = DPU_QOS_LUT_USAGE_NRT;
 	} else {
-		if (fmt && DPU_FORMAT_IS_LINEAR(fmt))
+		if (fmt && MSM_FORMAT_IS_LINEAR(fmt))
 			lut_usage = DPU_QOS_LUT_USAGE_LINEAR;
 		else
 			lut_usage = DPU_QOS_LUT_USAGE_MACROTILE;
@@ -284,26 +284,26 @@ static void _dpu_plane_set_qos_lut(struct drm_plane *plane,
 		pdpu->is_rt_pipe);
 
 	trace_dpu_perf_set_qos_luts(pipe->sspp->idx - SSPP_VIG0,
-			(fmt) ? fmt->base.pixel_format : 0,
+			(fmt) ? fmt->pixel_format : 0,
 			pdpu->is_rt_pipe, total_fl, cfg.creq_lut, lut_usage);
 
-	DPU_DEBUG_PLANE(pdpu, "pnum:%d fmt: %4.4s rt:%d fl:%u lut:0x%llx\n",
+	DPU_DEBUG_PLANE(pdpu, "pnum:%d fmt: %p4cc rt:%d fl:%u lut:0x%llx\n",
 			pdpu->pipe - SSPP_VIG0,
-			fmt ? (char *)&fmt->base.pixel_format : NULL,
+			fmt ? &fmt->pixel_format : NULL,
 			pdpu->is_rt_pipe, total_fl, cfg.creq_lut);
 
 	trace_dpu_perf_set_danger_luts(pdpu->pipe - SSPP_VIG0,
-			(fmt) ? fmt->base.pixel_format : 0,
+			(fmt) ? fmt->pixel_format : 0,
 			(fmt) ? fmt->fetch_mode : 0,
 			cfg.danger_lut,
 			cfg.safe_lut);
 
-	DPU_DEBUG_PLANE(pdpu, "pnum:%d fmt: %4.4s mode:%d luts[0x%x, 0x%x]\n",
-		pdpu->pipe - SSPP_VIG0,
-		fmt ? (char *)&fmt->base.pixel_format : NULL,
-		fmt ? fmt->fetch_mode : -1,
-		cfg.danger_lut,
-		cfg.safe_lut);
+	DPU_DEBUG_PLANE(pdpu, "pnum:%d fmt: %p4cc mode:%d luts[0x%x, 0x%x]\n",
+			pdpu->pipe - SSPP_VIG0,
+			fmt ? &fmt->pixel_format : NULL,
+			fmt ? fmt->fetch_mode : -1,
+			cfg.danger_lut,
+			cfg.safe_lut);
 
 	pipe->sspp->ops.setup_qos_lut(pipe->sspp, &cfg);
 }
@@ -425,7 +425,7 @@ static void _dpu_plane_set_qos_remap(struct drm_plane *plane,
 static void _dpu_plane_setup_scaler3(struct dpu_hw_sspp *pipe_hw,
 		uint32_t src_w, uint32_t src_h, uint32_t dst_w, uint32_t dst_h,
 		struct dpu_hw_scaler3_cfg *scale_cfg,
-		const struct dpu_format *fmt,
+		const struct msm_format *fmt,
 		uint32_t chroma_subsmpl_h, uint32_t chroma_subsmpl_v,
 		unsigned int rotation)
 {
@@ -477,7 +477,7 @@ static void _dpu_plane_setup_scaler3(struct dpu_hw_sspp *pipe_hw,
 			scale_cfg->preload_y[i] = DPU_QSEED3_DEFAULT_PRELOAD_V;
 		}
 	}
-	if (!(DPU_FORMAT_IS_YUV(fmt)) && (src_h == dst_h)
+	if (!(MSM_FORMAT_IS_YUV(fmt)) && (src_h == dst_h)
 		&& (src_w == dst_w))
 		return;
 
@@ -510,11 +510,11 @@ static void _dpu_plane_setup_pixel_ext(struct dpu_hw_scaler3_cfg *scale_cfg,
 }
 
 static const struct dpu_csc_cfg *_dpu_plane_get_csc(struct dpu_sw_pipe *pipe,
-						    const struct dpu_format *fmt)
+						    const struct msm_format *fmt)
 {
 	const struct dpu_csc_cfg *csc_ptr;
 
-	if (!DPU_FORMAT_IS_YUV(fmt))
+	if (!MSM_FORMAT_IS_YUV(fmt))
 		return NULL;
 
 	if (BIT(DPU_SSPP_CSC_10BIT) & pipe->sspp->cap->features)
@@ -526,12 +526,12 @@ static const struct dpu_csc_cfg *_dpu_plane_get_csc(struct dpu_sw_pipe *pipe,
 }
 
 static void _dpu_plane_setup_scaler(struct dpu_sw_pipe *pipe,
-		const struct dpu_format *fmt, bool color_fill,
+		const struct msm_format *fmt, bool color_fill,
 		struct dpu_sw_pipe_cfg *pipe_cfg,
 		unsigned int rotation)
 {
 	struct dpu_hw_sspp *pipe_hw = pipe->sspp;
-	const struct drm_format_info *info = drm_format_info(fmt->base.pixel_format);
+	const struct drm_format_info *info = drm_format_info(fmt->pixel_format);
 	struct dpu_hw_scaler3_cfg scaler3_cfg;
 	struct dpu_hw_pixel_ext pixel_ext;
 	u32 src_width = drm_rect_width(&pipe_cfg->src_rect);
@@ -577,7 +577,7 @@ static void _dpu_plane_color_fill_pipe(struct dpu_plane_state *pstate,
 				       struct dpu_sw_pipe *pipe,
 				       struct drm_rect *dst_rect,
 				       u32 fill_color,
-				       const struct dpu_format *fmt)
+				       const struct msm_format *fmt)
 {
 	struct dpu_sw_pipe_cfg pipe_cfg;
 
@@ -615,8 +615,9 @@ static void _dpu_plane_color_fill_pipe(struct dpu_plane_state *pstate,
 static void _dpu_plane_color_fill(struct dpu_plane *pdpu,
 		uint32_t color, uint32_t alpha)
 {
-	const struct dpu_format *fmt;
+	const struct msm_format *fmt;
 	const struct drm_plane *plane = &pdpu->base;
+	struct msm_drm_private *priv = plane->dev->dev_private;
 	struct dpu_plane_state *pstate = to_dpu_plane_state(plane->state);
 	u32 fill_color = (color & 0xFFFFFF) | ((alpha & 0xFF) << 24);
 
@@ -626,7 +627,7 @@ static void _dpu_plane_color_fill(struct dpu_plane *pdpu,
 	 * select fill format to match user property expectation,
 	 * h/w only supports RGB variants
 	 */
-	fmt = dpu_get_dpu_format(DRM_FORMAT_ABGR8888);
+	fmt = mdp_get_format(priv->kms, DRM_FORMAT_ABGR8888, 0);
 	/* should not happen ever */
 	if (!fmt)
 		return;
@@ -704,7 +705,7 @@ static void dpu_plane_cleanup_fb(struct drm_plane *plane,
 
 static int dpu_plane_check_inline_rotation(struct dpu_plane *pdpu,
 						const struct dpu_sspp_sub_blks *sblk,
-						struct drm_rect src, const struct dpu_format *fmt)
+						struct drm_rect src, const struct msm_format *fmt)
 {
 	size_t num_formats;
 	const u32 *supported_formats;
@@ -723,8 +724,8 @@ static int dpu_plane_check_inline_rotation(struct dpu_plane *pdpu,
 	supported_formats = sblk->rotation_cfg->rot_format_list;
 	num_formats = sblk->rotation_cfg->rot_num_formats;
 
-	if (!DPU_FORMAT_IS_UBWC(fmt) ||
-		!dpu_find_format(fmt->base.pixel_format, supported_formats, num_formats))
+	if (!MSM_FORMAT_IS_UBWC(fmt) ||
+		!dpu_find_format(fmt->pixel_format, supported_formats, num_formats))
 		return -EINVAL;
 
 	return 0;
@@ -733,15 +734,15 @@ static int dpu_plane_check_inline_rotation(struct dpu_plane *pdpu,
 static int dpu_plane_atomic_check_pipe(struct dpu_plane *pdpu,
 		struct dpu_sw_pipe *pipe,
 		struct dpu_sw_pipe_cfg *pipe_cfg,
-		const struct dpu_format *fmt,
+		const struct msm_format *fmt,
 		const struct drm_display_mode *mode)
 {
 	uint32_t min_src_size;
 	struct dpu_kms *kms = _dpu_plane_get_kms(&pdpu->base);
 
-	min_src_size = DPU_FORMAT_IS_YUV(fmt) ? 2 : 1;
+	min_src_size = MSM_FORMAT_IS_YUV(fmt) ? 2 : 1;
 
-	if (DPU_FORMAT_IS_YUV(fmt) &&
+	if (MSM_FORMAT_IS_YUV(fmt) &&
 	    (!pipe->sspp->cap->sblk->scaler_blk.len ||
 	     !pipe->sspp->cap->sblk->csc_blk.len)) {
 		DPU_DEBUG_PLANE(pdpu,
@@ -758,7 +759,7 @@ static int dpu_plane_atomic_check_pipe(struct dpu_plane *pdpu,
 	}
 
 	/* valid yuv image */
-	if (DPU_FORMAT_IS_YUV(fmt) &&
+	if (MSM_FORMAT_IS_YUV(fmt) &&
 	    (pipe_cfg->src_rect.x1 & 0x1 ||
 	     pipe_cfg->src_rect.y1 & 0x1 ||
 	     drm_rect_width(&pipe_cfg->src_rect) & 0x1 ||
@@ -798,7 +799,7 @@ static int dpu_plane_atomic_check(struct drm_plane *plane,
 	struct dpu_sw_pipe *pipe = &pstate->pipe;
 	struct dpu_sw_pipe *r_pipe = &pstate->r_pipe;
 	const struct drm_crtc_state *crtc_state = NULL;
-	const struct dpu_format *fmt;
+	const struct msm_format *fmt;
 	struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg;
 	struct dpu_sw_pipe_cfg *r_pipe_cfg = &pstate->r_pipe_cfg;
 	struct drm_rect fb_rect = { 0 };
@@ -858,7 +859,7 @@ static int dpu_plane_atomic_check(struct drm_plane *plane,
 		return -E2BIG;
 	}
 
-	fmt = to_dpu_format(msm_framebuffer_format(new_plane_state->fb));
+	fmt = msm_framebuffer_format(new_plane_state->fb);
 
 	max_linewidth = pdpu->catalog->caps->max_linewidth;
 
@@ -870,7 +871,7 @@ static int dpu_plane_atomic_check(struct drm_plane *plane,
 		 * full width is more than max_linewidth, thus each rect is
 		 * wider than allowed.
 		 */
-		if (DPU_FORMAT_IS_UBWC(fmt) &&
+		if (MSM_FORMAT_IS_UBWC(fmt) &&
 		    drm_rect_width(&pipe_cfg->src_rect) > max_linewidth) {
 			DPU_DEBUG_PLANE(pdpu, "invalid src " DRM_RECT_FMT " line:%u, tiled format\n",
 					DRM_RECT_ARG(&pipe_cfg->src_rect), max_linewidth);
@@ -887,7 +888,7 @@ static int dpu_plane_atomic_check(struct drm_plane *plane,
 		    drm_rect_height(&pipe_cfg->src_rect) != drm_rect_height(&pipe_cfg->dst_rect) ||
 		    (!test_bit(DPU_SSPP_SMART_DMA_V1, &pipe->sspp->cap->features) &&
 		     !test_bit(DPU_SSPP_SMART_DMA_V2, &pipe->sspp->cap->features)) ||
-		    DPU_FORMAT_IS_YUV(fmt)) {
+		    MSM_FORMAT_IS_YUV(fmt)) {
 			DPU_DEBUG_PLANE(pdpu, "invalid src " DRM_RECT_FMT " line:%u, can't use split source\n",
 					DRM_RECT_ARG(&pipe_cfg->src_rect), max_linewidth);
 			return -E2BIG;
@@ -945,8 +946,8 @@ static int dpu_plane_atomic_check(struct drm_plane *plane,
 
 static void dpu_plane_flush_csc(struct dpu_plane *pdpu, struct dpu_sw_pipe *pipe)
 {
-	const struct dpu_format *format =
-		to_dpu_format(msm_framebuffer_format(pdpu->base.state->fb));
+	const struct msm_format *format =
+		msm_framebuffer_format(pdpu->base.state->fb);
 	const struct dpu_csc_cfg *csc_ptr;
 
 	if (!pipe->sspp || !pipe->sspp->ops.setup_csc)
@@ -1017,7 +1018,7 @@ void dpu_plane_set_error(struct drm_plane *plane, bool error)
 static void dpu_plane_sspp_update_pipe(struct drm_plane *plane,
 				       struct dpu_sw_pipe *pipe,
 				       struct dpu_sw_pipe_cfg *pipe_cfg,
-				       const struct dpu_format *fmt,
+				       const struct msm_format *fmt,
 				       int frame_rate,
 				       struct dpu_hw_fmt_layout *layout)
 {
@@ -1095,8 +1096,8 @@ static void dpu_plane_sspp_atomic_update(struct drm_plane *plane)
 	struct drm_crtc *crtc = state->crtc;
 	struct drm_framebuffer *fb = state->fb;
 	bool is_rt_pipe;
-	const struct dpu_format *fmt =
-		to_dpu_format(msm_framebuffer_format(fb));
+	const struct msm_format *fmt =
+		msm_framebuffer_format(fb);
 	struct dpu_sw_pipe_cfg *pipe_cfg = &pstate->pipe_cfg;
 	struct dpu_sw_pipe_cfg *r_pipe_cfg = &pstate->r_pipe_cfg;
 	struct dpu_kms *kms = _dpu_plane_get_kms(&pdpu->base);
@@ -1118,9 +1119,9 @@ static void dpu_plane_sspp_atomic_update(struct drm_plane *plane)
 	pdpu->is_rt_pipe = is_rt_pipe;
 
 	DPU_DEBUG_PLANE(pdpu, "FB[%u] " DRM_RECT_FP_FMT "->crtc%u " DRM_RECT_FMT
-			", %4.4s ubwc %d\n", fb->base.id, DRM_RECT_FP_ARG(&state->src),
+			", %p4cc ubwc %d\n", fb->base.id, DRM_RECT_FP_ARG(&state->src),
 			crtc->base.id, DRM_RECT_ARG(&state->dst),
-			(char *)&fmt->base.pixel_format, DPU_FORMAT_IS_UBWC(fmt));
+			&fmt->pixel_format, MSM_FORMAT_IS_UBWC(fmt));
 
 	dpu_plane_sspp_update_pipe(plane, pipe, pipe_cfg, fmt,
 				   drm_mode_vrefresh(&crtc->mode),
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
index cb5ce3c62a22..44938ba7a2b7 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
@@ -758,3 +758,59 @@ int dpu_rm_get_assigned_resources(struct dpu_rm *rm,
 
 	return num_blks;
 }
+
+static void dpu_rm_print_state_helper(struct drm_printer *p,
+					    struct dpu_hw_blk *blk,
+					    uint32_t mapping)
+{
+	if (!blk)
+		drm_puts(p, "- ");
+	else if (!mapping)
+		drm_puts(p, "# ");
+	else
+		drm_printf(p, "%d ", mapping);
+}
+
+
+void dpu_rm_print_state(struct drm_printer *p,
+			const struct dpu_global_state *global_state)
+{
+	const struct dpu_rm *rm = global_state->rm;
+	int i;
+
+	drm_puts(p, "resource mapping:\n");
+	drm_puts(p, "\tpingpong=");
+	for (i = 0; i < ARRAY_SIZE(global_state->pingpong_to_enc_id); i++)
+		dpu_rm_print_state_helper(p, rm->pingpong_blks[i],
+					  global_state->pingpong_to_enc_id[i]);
+	drm_puts(p, "\n");
+
+	drm_puts(p, "\tmixer=");
+	for (i = 0; i < ARRAY_SIZE(global_state->mixer_to_enc_id); i++)
+		dpu_rm_print_state_helper(p, rm->mixer_blks[i],
+					  global_state->mixer_to_enc_id[i]);
+	drm_puts(p, "\n");
+
+	drm_puts(p, "\tctl=");
+	for (i = 0; i < ARRAY_SIZE(global_state->ctl_to_enc_id); i++)
+		dpu_rm_print_state_helper(p, rm->ctl_blks[i],
+					  global_state->ctl_to_enc_id[i]);
+	drm_puts(p, "\n");
+
+	drm_puts(p, "\tdspp=");
+	for (i = 0; i < ARRAY_SIZE(global_state->dspp_to_enc_id); i++)
+		dpu_rm_print_state_helper(p, rm->dspp_blks[i],
+					  global_state->dspp_to_enc_id[i]);
+	drm_puts(p, "\n");
+
+	drm_puts(p, "\tdsc=");
+	for (i = 0; i < ARRAY_SIZE(global_state->dsc_to_enc_id); i++)
+		dpu_rm_print_state_helper(p, rm->dsc_blks[i],
+					  global_state->dsc_to_enc_id[i]);
+	drm_puts(p, "\n");
+
+	drm_puts(p, "\tcdm=");
+	dpu_rm_print_state_helper(p, rm->cdm_blk,
+				  global_state->cdm_to_enc_id);
+	drm_puts(p, "\n");
+}
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h
index e3f83ebc656b..e63db8ace6b9 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h
@@ -90,6 +90,14 @@ int dpu_rm_get_assigned_resources(struct dpu_rm *rm,
 	enum dpu_hw_blk_type type, struct dpu_hw_blk **blks, int blks_size);
 
 /**
+ * dpu_rm_print_state - output the RM private state
+ * @p: DRM printer
+ * @global_state: global state
+ */
+void dpu_rm_print_state(struct drm_printer *p,
+			const struct dpu_global_state *global_state);
+
+/**
  * dpu_rm_get_intf - Return a struct dpu_hw_intf instance given it's index.
  * @rm: DPU Resource Manager handle
  * @intf_idx: INTF's index
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h
index bd92fb2979aa..0fdd41162e4b 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_trace.h
@@ -113,7 +113,7 @@ TRACE_EVENT(tracing_mark_write,
 	),
 	TP_fast_assign(
 			__entry->pid = pid;
-			__assign_str(trace_name, name);
+			__assign_str(trace_name);
 			__entry->trace_begin = trace_begin;
 	),
 	TP_printk("%s|%d|%s", __entry->trace_begin ? "B" : "E",
@@ -130,7 +130,7 @@ TRACE_EVENT(dpu_trace_counter,
 	),
 	TP_fast_assign(
 			__entry->pid = current->tgid;
-			__assign_str(counter_name, name);
+			__assign_str(counter_name);
 			__entry->value = value;
 	),
 	TP_printk("%d|%s|%d", __entry->pid,
@@ -379,7 +379,7 @@ TRACE_EVENT(dpu_enc_rc,
 		__entry->sw_event = sw_event;
 		__entry->idle_pc_supported = idle_pc_supported;
 		__entry->rc_state = rc_state;
-		__assign_str(stage_str, stage);
+		__assign_str(stage_str);
 	),
 	TP_printk("%s: id:%u, sw_event:%d, idle_pc_supported:%s, rc_state:%d",
 		  __get_str(stage_str), __entry->drm_id, __entry->sw_event,
@@ -401,7 +401,7 @@ TRACE_EVENT(dpu_enc_frame_done_cb_not_busy,
 	TP_fast_assign(
 		__entry->drm_id = drm_id;
 		__entry->event = event;
-		__assign_str(intf_mode_str, intf_mode);
+		__assign_str(intf_mode_str);
 		__entry->intf_idx = intf_idx;
 		__entry->wb_idx = wb_idx;
 	),
@@ -446,7 +446,7 @@ TRACE_EVENT(dpu_enc_trigger_flush,
 	),
 	TP_fast_assign(
 		__entry->drm_id = drm_id;
-		__assign_str(intf_mode_str, intf_mode);
+		__assign_str(intf_mode_str);
 		__entry->intf_idx = intf_idx;
 		__entry->wb_idx = wb_idx;
 		__entry->pending_kickoff_cnt = pending_kickoff_cnt;
@@ -946,7 +946,7 @@ TRACE_EVENT(dpu_core_perf_update_clk,
 		__field(	u64,			clk_rate	)
 	),
 	TP_fast_assign(
-		__assign_str(dev_name, dev->unique);
+		__assign_str(dev_name);
 		__entry->stop_req = stop_req;
 		__entry->clk_rate = clk_rate;
 	),
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h b/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h
deleted file mode 100644
index cc8fde450884..000000000000
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h
+++ /dev/null
@@ -1,1181 +0,0 @@
-#ifndef MDP4_XML
-#define MDP4_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-enum mdp4_pipe {
-	VG1 = 0,
-	VG2 = 1,
-	RGB1 = 2,
-	RGB2 = 3,
-	RGB3 = 4,
-	VG3 = 5,
-	VG4 = 6,
-};
-
-enum mdp4_mixer {
-	MIXER0 = 0,
-	MIXER1 = 1,
-	MIXER2 = 2,
-};
-
-enum mdp4_intf {
-	INTF_LCDC_DTV = 0,
-	INTF_DSI_VIDEO = 1,
-	INTF_DSI_CMD = 2,
-	INTF_EBI2_TV = 3,
-};
-
-enum mdp4_cursor_format {
-	CURSOR_ARGB = 1,
-	CURSOR_XRGB = 2,
-};
-
-enum mdp4_frame_format {
-	FRAME_LINEAR = 0,
-	FRAME_TILE_ARGB_4X4 = 1,
-	FRAME_TILE_YCBCR_420 = 2,
-};
-
-enum mdp4_scale_unit {
-	SCALE_FIR = 0,
-	SCALE_MN_PHASE = 1,
-	SCALE_PIXEL_RPT = 2,
-};
-
-enum mdp4_dma {
-	DMA_P = 0,
-	DMA_S = 1,
-	DMA_E = 2,
-};
-
-#define MDP4_IRQ_OVERLAY0_DONE					0x00000001
-#define MDP4_IRQ_OVERLAY1_DONE					0x00000002
-#define MDP4_IRQ_DMA_S_DONE					0x00000004
-#define MDP4_IRQ_DMA_E_DONE					0x00000008
-#define MDP4_IRQ_DMA_P_DONE					0x00000010
-#define MDP4_IRQ_VG1_HISTOGRAM					0x00000020
-#define MDP4_IRQ_VG2_HISTOGRAM					0x00000040
-#define MDP4_IRQ_PRIMARY_VSYNC					0x00000080
-#define MDP4_IRQ_PRIMARY_INTF_UDERRUN				0x00000100
-#define MDP4_IRQ_EXTERNAL_VSYNC					0x00000200
-#define MDP4_IRQ_EXTERNAL_INTF_UDERRUN				0x00000400
-#define MDP4_IRQ_PRIMARY_RDPTR					0x00000800
-#define MDP4_IRQ_DMA_P_HISTOGRAM				0x00020000
-#define MDP4_IRQ_DMA_S_HISTOGRAM				0x04000000
-#define MDP4_IRQ_OVERLAY2_DONE					0x40000000
-#define REG_MDP4_VERSION					0x00000000
-#define MDP4_VERSION_MINOR__MASK				0x00ff0000
-#define MDP4_VERSION_MINOR__SHIFT				16
-static inline uint32_t MDP4_VERSION_MINOR(uint32_t val)
-{
-	return ((val) << MDP4_VERSION_MINOR__SHIFT) & MDP4_VERSION_MINOR__MASK;
-}
-#define MDP4_VERSION_MAJOR__MASK				0xff000000
-#define MDP4_VERSION_MAJOR__SHIFT				24
-static inline uint32_t MDP4_VERSION_MAJOR(uint32_t val)
-{
-	return ((val) << MDP4_VERSION_MAJOR__SHIFT) & MDP4_VERSION_MAJOR__MASK;
-}
-
-#define REG_MDP4_OVLP0_KICK					0x00000004
-
-#define REG_MDP4_OVLP1_KICK					0x00000008
-
-#define REG_MDP4_OVLP2_KICK					0x000000d0
-
-#define REG_MDP4_DMA_P_KICK					0x0000000c
-
-#define REG_MDP4_DMA_S_KICK					0x00000010
-
-#define REG_MDP4_DMA_E_KICK					0x00000014
-
-#define REG_MDP4_DISP_STATUS					0x00000018
-
-#define REG_MDP4_DISP_INTF_SEL					0x00000038
-#define MDP4_DISP_INTF_SEL_PRIM__MASK				0x00000003
-#define MDP4_DISP_INTF_SEL_PRIM__SHIFT				0
-static inline uint32_t MDP4_DISP_INTF_SEL_PRIM(enum mdp4_intf val)
-{
-	return ((val) << MDP4_DISP_INTF_SEL_PRIM__SHIFT) & MDP4_DISP_INTF_SEL_PRIM__MASK;
-}
-#define MDP4_DISP_INTF_SEL_SEC__MASK				0x0000000c
-#define MDP4_DISP_INTF_SEL_SEC__SHIFT				2
-static inline uint32_t MDP4_DISP_INTF_SEL_SEC(enum mdp4_intf val)
-{
-	return ((val) << MDP4_DISP_INTF_SEL_SEC__SHIFT) & MDP4_DISP_INTF_SEL_SEC__MASK;
-}
-#define MDP4_DISP_INTF_SEL_EXT__MASK				0x00000030
-#define MDP4_DISP_INTF_SEL_EXT__SHIFT				4
-static inline uint32_t MDP4_DISP_INTF_SEL_EXT(enum mdp4_intf val)
-{
-	return ((val) << MDP4_DISP_INTF_SEL_EXT__SHIFT) & MDP4_DISP_INTF_SEL_EXT__MASK;
-}
-#define MDP4_DISP_INTF_SEL_DSI_VIDEO				0x00000040
-#define MDP4_DISP_INTF_SEL_DSI_CMD				0x00000080
-
-#define REG_MDP4_RESET_STATUS					0x0000003c
-
-#define REG_MDP4_READ_CNFG					0x0000004c
-
-#define REG_MDP4_INTR_ENABLE					0x00000050
-
-#define REG_MDP4_INTR_STATUS					0x00000054
-
-#define REG_MDP4_INTR_CLEAR					0x00000058
-
-#define REG_MDP4_EBI2_LCD0					0x00000060
-
-#define REG_MDP4_EBI2_LCD1					0x00000064
-
-#define REG_MDP4_PORTMAP_MODE					0x00000070
-
-#define REG_MDP4_CS_CONTROLLER0					0x000000c0
-
-#define REG_MDP4_CS_CONTROLLER1					0x000000c4
-
-#define REG_MDP4_LAYERMIXER2_IN_CFG				0x000100f0
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE0__MASK			0x00000007
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE0__SHIFT			0
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE0(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE0__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE0__MASK;
-}
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE0_MIXER1			0x00000008
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE1__MASK			0x00000070
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE1__SHIFT			4
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE1(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE1__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE1__MASK;
-}
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE1_MIXER1			0x00000080
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE2__MASK			0x00000700
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE2__SHIFT			8
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE2(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE2__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE2__MASK;
-}
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE2_MIXER1			0x00000800
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE3__MASK			0x00007000
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE3__SHIFT			12
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE3(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE3__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE3__MASK;
-}
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE3_MIXER1			0x00008000
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE4__MASK			0x00070000
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE4__SHIFT			16
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE4(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE4__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE4__MASK;
-}
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE4_MIXER1			0x00080000
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE5__MASK			0x00700000
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE5__SHIFT			20
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE5(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE5__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE5__MASK;
-}
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE5_MIXER1			0x00800000
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE6__MASK			0x07000000
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE6__SHIFT			24
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE6(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE6__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE6__MASK;
-}
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE6_MIXER1			0x08000000
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE7__MASK			0x70000000
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE7__SHIFT			28
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE7(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE7__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE7__MASK;
-}
-#define MDP4_LAYERMIXER2_IN_CFG_PIPE7_MIXER1			0x80000000
-
-#define REG_MDP4_LAYERMIXER_IN_CFG_UPDATE_METHOD		0x000100fc
-
-#define REG_MDP4_LAYERMIXER_IN_CFG				0x00010100
-#define MDP4_LAYERMIXER_IN_CFG_PIPE0__MASK			0x00000007
-#define MDP4_LAYERMIXER_IN_CFG_PIPE0__SHIFT			0
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE0(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE0__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE0__MASK;
-}
-#define MDP4_LAYERMIXER_IN_CFG_PIPE0_MIXER1			0x00000008
-#define MDP4_LAYERMIXER_IN_CFG_PIPE1__MASK			0x00000070
-#define MDP4_LAYERMIXER_IN_CFG_PIPE1__SHIFT			4
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE1(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE1__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE1__MASK;
-}
-#define MDP4_LAYERMIXER_IN_CFG_PIPE1_MIXER1			0x00000080
-#define MDP4_LAYERMIXER_IN_CFG_PIPE2__MASK			0x00000700
-#define MDP4_LAYERMIXER_IN_CFG_PIPE2__SHIFT			8
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE2(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE2__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE2__MASK;
-}
-#define MDP4_LAYERMIXER_IN_CFG_PIPE2_MIXER1			0x00000800
-#define MDP4_LAYERMIXER_IN_CFG_PIPE3__MASK			0x00007000
-#define MDP4_LAYERMIXER_IN_CFG_PIPE3__SHIFT			12
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE3(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE3__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE3__MASK;
-}
-#define MDP4_LAYERMIXER_IN_CFG_PIPE3_MIXER1			0x00008000
-#define MDP4_LAYERMIXER_IN_CFG_PIPE4__MASK			0x00070000
-#define MDP4_LAYERMIXER_IN_CFG_PIPE4__SHIFT			16
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE4(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE4__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE4__MASK;
-}
-#define MDP4_LAYERMIXER_IN_CFG_PIPE4_MIXER1			0x00080000
-#define MDP4_LAYERMIXER_IN_CFG_PIPE5__MASK			0x00700000
-#define MDP4_LAYERMIXER_IN_CFG_PIPE5__SHIFT			20
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE5(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE5__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE5__MASK;
-}
-#define MDP4_LAYERMIXER_IN_CFG_PIPE5_MIXER1			0x00800000
-#define MDP4_LAYERMIXER_IN_CFG_PIPE6__MASK			0x07000000
-#define MDP4_LAYERMIXER_IN_CFG_PIPE6__SHIFT			24
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE6(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE6__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE6__MASK;
-}
-#define MDP4_LAYERMIXER_IN_CFG_PIPE6_MIXER1			0x08000000
-#define MDP4_LAYERMIXER_IN_CFG_PIPE7__MASK			0x70000000
-#define MDP4_LAYERMIXER_IN_CFG_PIPE7__SHIFT			28
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE7(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE7__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE7__MASK;
-}
-#define MDP4_LAYERMIXER_IN_CFG_PIPE7_MIXER1			0x80000000
-
-#define REG_MDP4_VG2_SRC_FORMAT					0x00030050
-
-#define REG_MDP4_VG2_CONST_COLOR				0x00031008
-
-#define REG_MDP4_OVERLAY_FLUSH					0x00018000
-#define MDP4_OVERLAY_FLUSH_OVLP0				0x00000001
-#define MDP4_OVERLAY_FLUSH_OVLP1				0x00000002
-#define MDP4_OVERLAY_FLUSH_VG1					0x00000004
-#define MDP4_OVERLAY_FLUSH_VG2					0x00000008
-#define MDP4_OVERLAY_FLUSH_RGB1					0x00000010
-#define MDP4_OVERLAY_FLUSH_RGB2					0x00000020
-
-static inline uint32_t __offset_OVLP(uint32_t idx)
-{
-	switch (idx) {
-		case 0: return 0x00010000;
-		case 1: return 0x00018000;
-		case 2: return 0x00088000;
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP4_OVLP(uint32_t i0) { return 0x00000000 + __offset_OVLP(i0); }
-
-static inline uint32_t REG_MDP4_OVLP_CFG(uint32_t i0) { return 0x00000004 + __offset_OVLP(i0); }
-
-static inline uint32_t REG_MDP4_OVLP_SIZE(uint32_t i0) { return 0x00000008 + __offset_OVLP(i0); }
-#define MDP4_OVLP_SIZE_HEIGHT__MASK				0xffff0000
-#define MDP4_OVLP_SIZE_HEIGHT__SHIFT				16
-static inline uint32_t MDP4_OVLP_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << MDP4_OVLP_SIZE_HEIGHT__SHIFT) & MDP4_OVLP_SIZE_HEIGHT__MASK;
-}
-#define MDP4_OVLP_SIZE_WIDTH__MASK				0x0000ffff
-#define MDP4_OVLP_SIZE_WIDTH__SHIFT				0
-static inline uint32_t MDP4_OVLP_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << MDP4_OVLP_SIZE_WIDTH__SHIFT) & MDP4_OVLP_SIZE_WIDTH__MASK;
-}
-
-static inline uint32_t REG_MDP4_OVLP_BASE(uint32_t i0) { return 0x0000000c + __offset_OVLP(i0); }
-
-static inline uint32_t REG_MDP4_OVLP_STRIDE(uint32_t i0) { return 0x00000010 + __offset_OVLP(i0); }
-
-static inline uint32_t REG_MDP4_OVLP_OPMODE(uint32_t i0) { return 0x00000014 + __offset_OVLP(i0); }
-
-static inline uint32_t __offset_STAGE(uint32_t idx)
-{
-	switch (idx) {
-		case 0: return 0x00000104;
-		case 1: return 0x00000124;
-		case 2: return 0x00000144;
-		case 3: return 0x00000160;
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP4_OVLP_STAGE(uint32_t i0, uint32_t i1) { return 0x00000000 + __offset_OVLP(i0) + __offset_STAGE(i1); }
-
-static inline uint32_t REG_MDP4_OVLP_STAGE_OP(uint32_t i0, uint32_t i1) { return 0x00000000 + __offset_OVLP(i0) + __offset_STAGE(i1); }
-#define MDP4_OVLP_STAGE_OP_FG_ALPHA__MASK			0x00000003
-#define MDP4_OVLP_STAGE_OP_FG_ALPHA__SHIFT			0
-static inline uint32_t MDP4_OVLP_STAGE_OP_FG_ALPHA(enum mdp_alpha_type val)
-{
-	return ((val) << MDP4_OVLP_STAGE_OP_FG_ALPHA__SHIFT) & MDP4_OVLP_STAGE_OP_FG_ALPHA__MASK;
-}
-#define MDP4_OVLP_STAGE_OP_FG_INV_ALPHA				0x00000004
-#define MDP4_OVLP_STAGE_OP_FG_MOD_ALPHA				0x00000008
-#define MDP4_OVLP_STAGE_OP_BG_ALPHA__MASK			0x00000030
-#define MDP4_OVLP_STAGE_OP_BG_ALPHA__SHIFT			4
-static inline uint32_t MDP4_OVLP_STAGE_OP_BG_ALPHA(enum mdp_alpha_type val)
-{
-	return ((val) << MDP4_OVLP_STAGE_OP_BG_ALPHA__SHIFT) & MDP4_OVLP_STAGE_OP_BG_ALPHA__MASK;
-}
-#define MDP4_OVLP_STAGE_OP_BG_INV_ALPHA				0x00000040
-#define MDP4_OVLP_STAGE_OP_BG_MOD_ALPHA				0x00000080
-#define MDP4_OVLP_STAGE_OP_FG_TRANSP				0x00000100
-#define MDP4_OVLP_STAGE_OP_BG_TRANSP				0x00000200
-
-static inline uint32_t REG_MDP4_OVLP_STAGE_FG_ALPHA(uint32_t i0, uint32_t i1) { return 0x00000004 + __offset_OVLP(i0) + __offset_STAGE(i1); }
-
-static inline uint32_t REG_MDP4_OVLP_STAGE_BG_ALPHA(uint32_t i0, uint32_t i1) { return 0x00000008 + __offset_OVLP(i0) + __offset_STAGE(i1); }
-
-static inline uint32_t REG_MDP4_OVLP_STAGE_TRANSP_LOW0(uint32_t i0, uint32_t i1) { return 0x0000000c + __offset_OVLP(i0) + __offset_STAGE(i1); }
-
-static inline uint32_t REG_MDP4_OVLP_STAGE_TRANSP_LOW1(uint32_t i0, uint32_t i1) { return 0x00000010 + __offset_OVLP(i0) + __offset_STAGE(i1); }
-
-static inline uint32_t REG_MDP4_OVLP_STAGE_TRANSP_HIGH0(uint32_t i0, uint32_t i1) { return 0x00000014 + __offset_OVLP(i0) + __offset_STAGE(i1); }
-
-static inline uint32_t REG_MDP4_OVLP_STAGE_TRANSP_HIGH1(uint32_t i0, uint32_t i1) { return 0x00000018 + __offset_OVLP(i0) + __offset_STAGE(i1); }
-
-static inline uint32_t __offset_STAGE_CO3(uint32_t idx)
-{
-	switch (idx) {
-		case 0: return 0x00001004;
-		case 1: return 0x00001404;
-		case 2: return 0x00001804;
-		case 3: return 0x00001b84;
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP4_OVLP_STAGE_CO3(uint32_t i0, uint32_t i1) { return 0x00000000 + __offset_OVLP(i0) + __offset_STAGE_CO3(i1); }
-
-static inline uint32_t REG_MDP4_OVLP_STAGE_CO3_SEL(uint32_t i0, uint32_t i1) { return 0x00000000 + __offset_OVLP(i0) + __offset_STAGE_CO3(i1); }
-#define MDP4_OVLP_STAGE_CO3_SEL_FG_ALPHA			0x00000001
-
-static inline uint32_t REG_MDP4_OVLP_TRANSP_LOW0(uint32_t i0) { return 0x00000180 + __offset_OVLP(i0); }
-
-static inline uint32_t REG_MDP4_OVLP_TRANSP_LOW1(uint32_t i0) { return 0x00000184 + __offset_OVLP(i0); }
-
-static inline uint32_t REG_MDP4_OVLP_TRANSP_HIGH0(uint32_t i0) { return 0x00000188 + __offset_OVLP(i0); }
-
-static inline uint32_t REG_MDP4_OVLP_TRANSP_HIGH1(uint32_t i0) { return 0x0000018c + __offset_OVLP(i0); }
-
-static inline uint32_t REG_MDP4_OVLP_CSC_CONFIG(uint32_t i0) { return 0x00000200 + __offset_OVLP(i0); }
-
-static inline uint32_t REG_MDP4_OVLP_CSC(uint32_t i0) { return 0x00002000 + __offset_OVLP(i0); }
-
-
-static inline uint32_t REG_MDP4_OVLP_CSC_MV(uint32_t i0, uint32_t i1) { return 0x00002400 + __offset_OVLP(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_OVLP_CSC_MV_VAL(uint32_t i0, uint32_t i1) { return 0x00002400 + __offset_OVLP(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_OVLP_CSC_PRE_BV(uint32_t i0, uint32_t i1) { return 0x00002500 + __offset_OVLP(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_OVLP_CSC_PRE_BV_VAL(uint32_t i0, uint32_t i1) { return 0x00002500 + __offset_OVLP(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_OVLP_CSC_POST_BV(uint32_t i0, uint32_t i1) { return 0x00002580 + __offset_OVLP(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_OVLP_CSC_POST_BV_VAL(uint32_t i0, uint32_t i1) { return 0x00002580 + __offset_OVLP(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_OVLP_CSC_PRE_LV(uint32_t i0, uint32_t i1) { return 0x00002600 + __offset_OVLP(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_OVLP_CSC_PRE_LV_VAL(uint32_t i0, uint32_t i1) { return 0x00002600 + __offset_OVLP(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_OVLP_CSC_POST_LV(uint32_t i0, uint32_t i1) { return 0x00002680 + __offset_OVLP(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_OVLP_CSC_POST_LV_VAL(uint32_t i0, uint32_t i1) { return 0x00002680 + __offset_OVLP(i0) + 0x4*i1; }
-
-#define REG_MDP4_DMA_P_OP_MODE					0x00090070
-
-static inline uint32_t REG_MDP4_LUTN(uint32_t i0) { return 0x00094800 + 0x400*i0; }
-
-static inline uint32_t REG_MDP4_LUTN_LUT(uint32_t i0, uint32_t i1) { return 0x00094800 + 0x400*i0 + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_LUTN_LUT_VAL(uint32_t i0, uint32_t i1) { return 0x00094800 + 0x400*i0 + 0x4*i1; }
-
-#define REG_MDP4_DMA_S_OP_MODE					0x000a0028
-
-static inline uint32_t REG_MDP4_DMA_E_QUANT(uint32_t i0) { return 0x000b0070 + 0x4*i0; }
-
-static inline uint32_t __offset_DMA(enum mdp4_dma idx)
-{
-	switch (idx) {
-		case DMA_P: return 0x00090000;
-		case DMA_S: return 0x000a0000;
-		case DMA_E: return 0x000b0000;
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP4_DMA(enum mdp4_dma i0) { return 0x00000000 + __offset_DMA(i0); }
-
-static inline uint32_t REG_MDP4_DMA_CONFIG(enum mdp4_dma i0) { return 0x00000000 + __offset_DMA(i0); }
-#define MDP4_DMA_CONFIG_G_BPC__MASK				0x00000003
-#define MDP4_DMA_CONFIG_G_BPC__SHIFT				0
-static inline uint32_t MDP4_DMA_CONFIG_G_BPC(enum mdp_bpc val)
-{
-	return ((val) << MDP4_DMA_CONFIG_G_BPC__SHIFT) & MDP4_DMA_CONFIG_G_BPC__MASK;
-}
-#define MDP4_DMA_CONFIG_B_BPC__MASK				0x0000000c
-#define MDP4_DMA_CONFIG_B_BPC__SHIFT				2
-static inline uint32_t MDP4_DMA_CONFIG_B_BPC(enum mdp_bpc val)
-{
-	return ((val) << MDP4_DMA_CONFIG_B_BPC__SHIFT) & MDP4_DMA_CONFIG_B_BPC__MASK;
-}
-#define MDP4_DMA_CONFIG_R_BPC__MASK				0x00000030
-#define MDP4_DMA_CONFIG_R_BPC__SHIFT				4
-static inline uint32_t MDP4_DMA_CONFIG_R_BPC(enum mdp_bpc val)
-{
-	return ((val) << MDP4_DMA_CONFIG_R_BPC__SHIFT) & MDP4_DMA_CONFIG_R_BPC__MASK;
-}
-#define MDP4_DMA_CONFIG_PACK_ALIGN_MSB				0x00000080
-#define MDP4_DMA_CONFIG_PACK__MASK				0x0000ff00
-#define MDP4_DMA_CONFIG_PACK__SHIFT				8
-static inline uint32_t MDP4_DMA_CONFIG_PACK(uint32_t val)
-{
-	return ((val) << MDP4_DMA_CONFIG_PACK__SHIFT) & MDP4_DMA_CONFIG_PACK__MASK;
-}
-#define MDP4_DMA_CONFIG_DEFLKR_EN				0x01000000
-#define MDP4_DMA_CONFIG_DITHER_EN				0x01000000
-
-static inline uint32_t REG_MDP4_DMA_SRC_SIZE(enum mdp4_dma i0) { return 0x00000004 + __offset_DMA(i0); }
-#define MDP4_DMA_SRC_SIZE_HEIGHT__MASK				0xffff0000
-#define MDP4_DMA_SRC_SIZE_HEIGHT__SHIFT				16
-static inline uint32_t MDP4_DMA_SRC_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << MDP4_DMA_SRC_SIZE_HEIGHT__SHIFT) & MDP4_DMA_SRC_SIZE_HEIGHT__MASK;
-}
-#define MDP4_DMA_SRC_SIZE_WIDTH__MASK				0x0000ffff
-#define MDP4_DMA_SRC_SIZE_WIDTH__SHIFT				0
-static inline uint32_t MDP4_DMA_SRC_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << MDP4_DMA_SRC_SIZE_WIDTH__SHIFT) & MDP4_DMA_SRC_SIZE_WIDTH__MASK;
-}
-
-static inline uint32_t REG_MDP4_DMA_SRC_BASE(enum mdp4_dma i0) { return 0x00000008 + __offset_DMA(i0); }
-
-static inline uint32_t REG_MDP4_DMA_SRC_STRIDE(enum mdp4_dma i0) { return 0x0000000c + __offset_DMA(i0); }
-
-static inline uint32_t REG_MDP4_DMA_DST_SIZE(enum mdp4_dma i0) { return 0x00000010 + __offset_DMA(i0); }
-#define MDP4_DMA_DST_SIZE_HEIGHT__MASK				0xffff0000
-#define MDP4_DMA_DST_SIZE_HEIGHT__SHIFT				16
-static inline uint32_t MDP4_DMA_DST_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << MDP4_DMA_DST_SIZE_HEIGHT__SHIFT) & MDP4_DMA_DST_SIZE_HEIGHT__MASK;
-}
-#define MDP4_DMA_DST_SIZE_WIDTH__MASK				0x0000ffff
-#define MDP4_DMA_DST_SIZE_WIDTH__SHIFT				0
-static inline uint32_t MDP4_DMA_DST_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << MDP4_DMA_DST_SIZE_WIDTH__SHIFT) & MDP4_DMA_DST_SIZE_WIDTH__MASK;
-}
-
-static inline uint32_t REG_MDP4_DMA_CURSOR_SIZE(enum mdp4_dma i0) { return 0x00000044 + __offset_DMA(i0); }
-#define MDP4_DMA_CURSOR_SIZE_WIDTH__MASK			0x0000007f
-#define MDP4_DMA_CURSOR_SIZE_WIDTH__SHIFT			0
-static inline uint32_t MDP4_DMA_CURSOR_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << MDP4_DMA_CURSOR_SIZE_WIDTH__SHIFT) & MDP4_DMA_CURSOR_SIZE_WIDTH__MASK;
-}
-#define MDP4_DMA_CURSOR_SIZE_HEIGHT__MASK			0x007f0000
-#define MDP4_DMA_CURSOR_SIZE_HEIGHT__SHIFT			16
-static inline uint32_t MDP4_DMA_CURSOR_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << MDP4_DMA_CURSOR_SIZE_HEIGHT__SHIFT) & MDP4_DMA_CURSOR_SIZE_HEIGHT__MASK;
-}
-
-static inline uint32_t REG_MDP4_DMA_CURSOR_BASE(enum mdp4_dma i0) { return 0x00000048 + __offset_DMA(i0); }
-
-static inline uint32_t REG_MDP4_DMA_CURSOR_POS(enum mdp4_dma i0) { return 0x0000004c + __offset_DMA(i0); }
-#define MDP4_DMA_CURSOR_POS_X__MASK				0x0000ffff
-#define MDP4_DMA_CURSOR_POS_X__SHIFT				0
-static inline uint32_t MDP4_DMA_CURSOR_POS_X(uint32_t val)
-{
-	return ((val) << MDP4_DMA_CURSOR_POS_X__SHIFT) & MDP4_DMA_CURSOR_POS_X__MASK;
-}
-#define MDP4_DMA_CURSOR_POS_Y__MASK				0xffff0000
-#define MDP4_DMA_CURSOR_POS_Y__SHIFT				16
-static inline uint32_t MDP4_DMA_CURSOR_POS_Y(uint32_t val)
-{
-	return ((val) << MDP4_DMA_CURSOR_POS_Y__SHIFT) & MDP4_DMA_CURSOR_POS_Y__MASK;
-}
-
-static inline uint32_t REG_MDP4_DMA_CURSOR_BLEND_CONFIG(enum mdp4_dma i0) { return 0x00000060 + __offset_DMA(i0); }
-#define MDP4_DMA_CURSOR_BLEND_CONFIG_CURSOR_EN			0x00000001
-#define MDP4_DMA_CURSOR_BLEND_CONFIG_FORMAT__MASK		0x00000006
-#define MDP4_DMA_CURSOR_BLEND_CONFIG_FORMAT__SHIFT		1
-static inline uint32_t MDP4_DMA_CURSOR_BLEND_CONFIG_FORMAT(enum mdp4_cursor_format val)
-{
-	return ((val) << MDP4_DMA_CURSOR_BLEND_CONFIG_FORMAT__SHIFT) & MDP4_DMA_CURSOR_BLEND_CONFIG_FORMAT__MASK;
-}
-#define MDP4_DMA_CURSOR_BLEND_CONFIG_TRANSP_EN			0x00000008
-
-static inline uint32_t REG_MDP4_DMA_CURSOR_BLEND_PARAM(enum mdp4_dma i0) { return 0x00000064 + __offset_DMA(i0); }
-
-static inline uint32_t REG_MDP4_DMA_BLEND_TRANS_LOW(enum mdp4_dma i0) { return 0x00000068 + __offset_DMA(i0); }
-
-static inline uint32_t REG_MDP4_DMA_BLEND_TRANS_HIGH(enum mdp4_dma i0) { return 0x0000006c + __offset_DMA(i0); }
-
-static inline uint32_t REG_MDP4_DMA_FETCH_CONFIG(enum mdp4_dma i0) { return 0x00001004 + __offset_DMA(i0); }
-
-static inline uint32_t REG_MDP4_DMA_CSC(enum mdp4_dma i0) { return 0x00003000 + __offset_DMA(i0); }
-
-
-static inline uint32_t REG_MDP4_DMA_CSC_MV(enum mdp4_dma i0, uint32_t i1) { return 0x00003400 + __offset_DMA(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_DMA_CSC_MV_VAL(enum mdp4_dma i0, uint32_t i1) { return 0x00003400 + __offset_DMA(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_DMA_CSC_PRE_BV(enum mdp4_dma i0, uint32_t i1) { return 0x00003500 + __offset_DMA(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_DMA_CSC_PRE_BV_VAL(enum mdp4_dma i0, uint32_t i1) { return 0x00003500 + __offset_DMA(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_DMA_CSC_POST_BV(enum mdp4_dma i0, uint32_t i1) { return 0x00003580 + __offset_DMA(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_DMA_CSC_POST_BV_VAL(enum mdp4_dma i0, uint32_t i1) { return 0x00003580 + __offset_DMA(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_DMA_CSC_PRE_LV(enum mdp4_dma i0, uint32_t i1) { return 0x00003600 + __offset_DMA(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_DMA_CSC_PRE_LV_VAL(enum mdp4_dma i0, uint32_t i1) { return 0x00003600 + __offset_DMA(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_DMA_CSC_POST_LV(enum mdp4_dma i0, uint32_t i1) { return 0x00003680 + __offset_DMA(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_DMA_CSC_POST_LV_VAL(enum mdp4_dma i0, uint32_t i1) { return 0x00003680 + __offset_DMA(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_PIPE(enum mdp4_pipe i0) { return 0x00020000 + 0x10000*i0; }
-
-static inline uint32_t REG_MDP4_PIPE_SRC_SIZE(enum mdp4_pipe i0) { return 0x00020000 + 0x10000*i0; }
-#define MDP4_PIPE_SRC_SIZE_HEIGHT__MASK				0xffff0000
-#define MDP4_PIPE_SRC_SIZE_HEIGHT__SHIFT			16
-static inline uint32_t MDP4_PIPE_SRC_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_SIZE_HEIGHT__SHIFT) & MDP4_PIPE_SRC_SIZE_HEIGHT__MASK;
-}
-#define MDP4_PIPE_SRC_SIZE_WIDTH__MASK				0x0000ffff
-#define MDP4_PIPE_SRC_SIZE_WIDTH__SHIFT				0
-static inline uint32_t MDP4_PIPE_SRC_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_SIZE_WIDTH__SHIFT) & MDP4_PIPE_SRC_SIZE_WIDTH__MASK;
-}
-
-static inline uint32_t REG_MDP4_PIPE_SRC_XY(enum mdp4_pipe i0) { return 0x00020004 + 0x10000*i0; }
-#define MDP4_PIPE_SRC_XY_Y__MASK				0xffff0000
-#define MDP4_PIPE_SRC_XY_Y__SHIFT				16
-static inline uint32_t MDP4_PIPE_SRC_XY_Y(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_XY_Y__SHIFT) & MDP4_PIPE_SRC_XY_Y__MASK;
-}
-#define MDP4_PIPE_SRC_XY_X__MASK				0x0000ffff
-#define MDP4_PIPE_SRC_XY_X__SHIFT				0
-static inline uint32_t MDP4_PIPE_SRC_XY_X(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_XY_X__SHIFT) & MDP4_PIPE_SRC_XY_X__MASK;
-}
-
-static inline uint32_t REG_MDP4_PIPE_DST_SIZE(enum mdp4_pipe i0) { return 0x00020008 + 0x10000*i0; }
-#define MDP4_PIPE_DST_SIZE_HEIGHT__MASK				0xffff0000
-#define MDP4_PIPE_DST_SIZE_HEIGHT__SHIFT			16
-static inline uint32_t MDP4_PIPE_DST_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_DST_SIZE_HEIGHT__SHIFT) & MDP4_PIPE_DST_SIZE_HEIGHT__MASK;
-}
-#define MDP4_PIPE_DST_SIZE_WIDTH__MASK				0x0000ffff
-#define MDP4_PIPE_DST_SIZE_WIDTH__SHIFT				0
-static inline uint32_t MDP4_PIPE_DST_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_DST_SIZE_WIDTH__SHIFT) & MDP4_PIPE_DST_SIZE_WIDTH__MASK;
-}
-
-static inline uint32_t REG_MDP4_PIPE_DST_XY(enum mdp4_pipe i0) { return 0x0002000c + 0x10000*i0; }
-#define MDP4_PIPE_DST_XY_Y__MASK				0xffff0000
-#define MDP4_PIPE_DST_XY_Y__SHIFT				16
-static inline uint32_t MDP4_PIPE_DST_XY_Y(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_DST_XY_Y__SHIFT) & MDP4_PIPE_DST_XY_Y__MASK;
-}
-#define MDP4_PIPE_DST_XY_X__MASK				0x0000ffff
-#define MDP4_PIPE_DST_XY_X__SHIFT				0
-static inline uint32_t MDP4_PIPE_DST_XY_X(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_DST_XY_X__SHIFT) & MDP4_PIPE_DST_XY_X__MASK;
-}
-
-static inline uint32_t REG_MDP4_PIPE_SRCP0_BASE(enum mdp4_pipe i0) { return 0x00020010 + 0x10000*i0; }
-
-static inline uint32_t REG_MDP4_PIPE_SRCP1_BASE(enum mdp4_pipe i0) { return 0x00020014 + 0x10000*i0; }
-
-static inline uint32_t REG_MDP4_PIPE_SRCP2_BASE(enum mdp4_pipe i0) { return 0x00020018 + 0x10000*i0; }
-
-static inline uint32_t REG_MDP4_PIPE_SRCP3_BASE(enum mdp4_pipe i0) { return 0x0002001c + 0x10000*i0; }
-
-static inline uint32_t REG_MDP4_PIPE_SRC_STRIDE_A(enum mdp4_pipe i0) { return 0x00020040 + 0x10000*i0; }
-#define MDP4_PIPE_SRC_STRIDE_A_P0__MASK				0x0000ffff
-#define MDP4_PIPE_SRC_STRIDE_A_P0__SHIFT			0
-static inline uint32_t MDP4_PIPE_SRC_STRIDE_A_P0(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_STRIDE_A_P0__SHIFT) & MDP4_PIPE_SRC_STRIDE_A_P0__MASK;
-}
-#define MDP4_PIPE_SRC_STRIDE_A_P1__MASK				0xffff0000
-#define MDP4_PIPE_SRC_STRIDE_A_P1__SHIFT			16
-static inline uint32_t MDP4_PIPE_SRC_STRIDE_A_P1(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_STRIDE_A_P1__SHIFT) & MDP4_PIPE_SRC_STRIDE_A_P1__MASK;
-}
-
-static inline uint32_t REG_MDP4_PIPE_SRC_STRIDE_B(enum mdp4_pipe i0) { return 0x00020044 + 0x10000*i0; }
-#define MDP4_PIPE_SRC_STRIDE_B_P2__MASK				0x0000ffff
-#define MDP4_PIPE_SRC_STRIDE_B_P2__SHIFT			0
-static inline uint32_t MDP4_PIPE_SRC_STRIDE_B_P2(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_STRIDE_B_P2__SHIFT) & MDP4_PIPE_SRC_STRIDE_B_P2__MASK;
-}
-#define MDP4_PIPE_SRC_STRIDE_B_P3__MASK				0xffff0000
-#define MDP4_PIPE_SRC_STRIDE_B_P3__SHIFT			16
-static inline uint32_t MDP4_PIPE_SRC_STRIDE_B_P3(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_STRIDE_B_P3__SHIFT) & MDP4_PIPE_SRC_STRIDE_B_P3__MASK;
-}
-
-static inline uint32_t REG_MDP4_PIPE_SSTILE_FRAME_SIZE(enum mdp4_pipe i0) { return 0x00020048 + 0x10000*i0; }
-#define MDP4_PIPE_SSTILE_FRAME_SIZE_HEIGHT__MASK		0xffff0000
-#define MDP4_PIPE_SSTILE_FRAME_SIZE_HEIGHT__SHIFT		16
-static inline uint32_t MDP4_PIPE_SSTILE_FRAME_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SSTILE_FRAME_SIZE_HEIGHT__SHIFT) & MDP4_PIPE_SSTILE_FRAME_SIZE_HEIGHT__MASK;
-}
-#define MDP4_PIPE_SSTILE_FRAME_SIZE_WIDTH__MASK			0x0000ffff
-#define MDP4_PIPE_SSTILE_FRAME_SIZE_WIDTH__SHIFT		0
-static inline uint32_t MDP4_PIPE_SSTILE_FRAME_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SSTILE_FRAME_SIZE_WIDTH__SHIFT) & MDP4_PIPE_SSTILE_FRAME_SIZE_WIDTH__MASK;
-}
-
-static inline uint32_t REG_MDP4_PIPE_SRC_FORMAT(enum mdp4_pipe i0) { return 0x00020050 + 0x10000*i0; }
-#define MDP4_PIPE_SRC_FORMAT_G_BPC__MASK			0x00000003
-#define MDP4_PIPE_SRC_FORMAT_G_BPC__SHIFT			0
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_G_BPC(enum mdp_bpc val)
-{
-	return ((val) << MDP4_PIPE_SRC_FORMAT_G_BPC__SHIFT) & MDP4_PIPE_SRC_FORMAT_G_BPC__MASK;
-}
-#define MDP4_PIPE_SRC_FORMAT_B_BPC__MASK			0x0000000c
-#define MDP4_PIPE_SRC_FORMAT_B_BPC__SHIFT			2
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_B_BPC(enum mdp_bpc val)
-{
-	return ((val) << MDP4_PIPE_SRC_FORMAT_B_BPC__SHIFT) & MDP4_PIPE_SRC_FORMAT_B_BPC__MASK;
-}
-#define MDP4_PIPE_SRC_FORMAT_R_BPC__MASK			0x00000030
-#define MDP4_PIPE_SRC_FORMAT_R_BPC__SHIFT			4
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_R_BPC(enum mdp_bpc val)
-{
-	return ((val) << MDP4_PIPE_SRC_FORMAT_R_BPC__SHIFT) & MDP4_PIPE_SRC_FORMAT_R_BPC__MASK;
-}
-#define MDP4_PIPE_SRC_FORMAT_A_BPC__MASK			0x000000c0
-#define MDP4_PIPE_SRC_FORMAT_A_BPC__SHIFT			6
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_A_BPC(enum mdp_bpc_alpha val)
-{
-	return ((val) << MDP4_PIPE_SRC_FORMAT_A_BPC__SHIFT) & MDP4_PIPE_SRC_FORMAT_A_BPC__MASK;
-}
-#define MDP4_PIPE_SRC_FORMAT_ALPHA_ENABLE			0x00000100
-#define MDP4_PIPE_SRC_FORMAT_CPP__MASK				0x00000600
-#define MDP4_PIPE_SRC_FORMAT_CPP__SHIFT				9
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_CPP(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_FORMAT_CPP__SHIFT) & MDP4_PIPE_SRC_FORMAT_CPP__MASK;
-}
-#define MDP4_PIPE_SRC_FORMAT_ROTATED_90				0x00001000
-#define MDP4_PIPE_SRC_FORMAT_UNPACK_COUNT__MASK			0x00006000
-#define MDP4_PIPE_SRC_FORMAT_UNPACK_COUNT__SHIFT		13
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_UNPACK_COUNT(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_FORMAT_UNPACK_COUNT__SHIFT) & MDP4_PIPE_SRC_FORMAT_UNPACK_COUNT__MASK;
-}
-#define MDP4_PIPE_SRC_FORMAT_UNPACK_TIGHT			0x00020000
-#define MDP4_PIPE_SRC_FORMAT_UNPACK_ALIGN_MSB			0x00040000
-#define MDP4_PIPE_SRC_FORMAT_FETCH_PLANES__MASK			0x00180000
-#define MDP4_PIPE_SRC_FORMAT_FETCH_PLANES__SHIFT		19
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_FETCH_PLANES(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_FORMAT_FETCH_PLANES__SHIFT) & MDP4_PIPE_SRC_FORMAT_FETCH_PLANES__MASK;
-}
-#define MDP4_PIPE_SRC_FORMAT_SOLID_FILL				0x00400000
-#define MDP4_PIPE_SRC_FORMAT_CHROMA_SAMP__MASK			0x0c000000
-#define MDP4_PIPE_SRC_FORMAT_CHROMA_SAMP__SHIFT			26
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_CHROMA_SAMP(enum mdp_chroma_samp_type val)
-{
-	return ((val) << MDP4_PIPE_SRC_FORMAT_CHROMA_SAMP__SHIFT) & MDP4_PIPE_SRC_FORMAT_CHROMA_SAMP__MASK;
-}
-#define MDP4_PIPE_SRC_FORMAT_FRAME_FORMAT__MASK			0x60000000
-#define MDP4_PIPE_SRC_FORMAT_FRAME_FORMAT__SHIFT		29
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_FRAME_FORMAT(enum mdp4_frame_format val)
-{
-	return ((val) << MDP4_PIPE_SRC_FORMAT_FRAME_FORMAT__SHIFT) & MDP4_PIPE_SRC_FORMAT_FRAME_FORMAT__MASK;
-}
-
-static inline uint32_t REG_MDP4_PIPE_SRC_UNPACK(enum mdp4_pipe i0) { return 0x00020054 + 0x10000*i0; }
-#define MDP4_PIPE_SRC_UNPACK_ELEM0__MASK			0x000000ff
-#define MDP4_PIPE_SRC_UNPACK_ELEM0__SHIFT			0
-static inline uint32_t MDP4_PIPE_SRC_UNPACK_ELEM0(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_UNPACK_ELEM0__SHIFT) & MDP4_PIPE_SRC_UNPACK_ELEM0__MASK;
-}
-#define MDP4_PIPE_SRC_UNPACK_ELEM1__MASK			0x0000ff00
-#define MDP4_PIPE_SRC_UNPACK_ELEM1__SHIFT			8
-static inline uint32_t MDP4_PIPE_SRC_UNPACK_ELEM1(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_UNPACK_ELEM1__SHIFT) & MDP4_PIPE_SRC_UNPACK_ELEM1__MASK;
-}
-#define MDP4_PIPE_SRC_UNPACK_ELEM2__MASK			0x00ff0000
-#define MDP4_PIPE_SRC_UNPACK_ELEM2__SHIFT			16
-static inline uint32_t MDP4_PIPE_SRC_UNPACK_ELEM2(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_UNPACK_ELEM2__SHIFT) & MDP4_PIPE_SRC_UNPACK_ELEM2__MASK;
-}
-#define MDP4_PIPE_SRC_UNPACK_ELEM3__MASK			0xff000000
-#define MDP4_PIPE_SRC_UNPACK_ELEM3__SHIFT			24
-static inline uint32_t MDP4_PIPE_SRC_UNPACK_ELEM3(uint32_t val)
-{
-	return ((val) << MDP4_PIPE_SRC_UNPACK_ELEM3__SHIFT) & MDP4_PIPE_SRC_UNPACK_ELEM3__MASK;
-}
-
-static inline uint32_t REG_MDP4_PIPE_OP_MODE(enum mdp4_pipe i0) { return 0x00020058 + 0x10000*i0; }
-#define MDP4_PIPE_OP_MODE_SCALEX_EN				0x00000001
-#define MDP4_PIPE_OP_MODE_SCALEY_EN				0x00000002
-#define MDP4_PIPE_OP_MODE_SCALEX_UNIT_SEL__MASK			0x0000000c
-#define MDP4_PIPE_OP_MODE_SCALEX_UNIT_SEL__SHIFT		2
-static inline uint32_t MDP4_PIPE_OP_MODE_SCALEX_UNIT_SEL(enum mdp4_scale_unit val)
-{
-	return ((val) << MDP4_PIPE_OP_MODE_SCALEX_UNIT_SEL__SHIFT) & MDP4_PIPE_OP_MODE_SCALEX_UNIT_SEL__MASK;
-}
-#define MDP4_PIPE_OP_MODE_SCALEY_UNIT_SEL__MASK			0x00000030
-#define MDP4_PIPE_OP_MODE_SCALEY_UNIT_SEL__SHIFT		4
-static inline uint32_t MDP4_PIPE_OP_MODE_SCALEY_UNIT_SEL(enum mdp4_scale_unit val)
-{
-	return ((val) << MDP4_PIPE_OP_MODE_SCALEY_UNIT_SEL__SHIFT) & MDP4_PIPE_OP_MODE_SCALEY_UNIT_SEL__MASK;
-}
-#define MDP4_PIPE_OP_MODE_SRC_YCBCR				0x00000200
-#define MDP4_PIPE_OP_MODE_DST_YCBCR				0x00000400
-#define MDP4_PIPE_OP_MODE_CSC_EN				0x00000800
-#define MDP4_PIPE_OP_MODE_FLIP_LR				0x00002000
-#define MDP4_PIPE_OP_MODE_FLIP_UD				0x00004000
-#define MDP4_PIPE_OP_MODE_DITHER_EN				0x00008000
-#define MDP4_PIPE_OP_MODE_IGC_LUT_EN				0x00010000
-#define MDP4_PIPE_OP_MODE_DEINT_EN				0x00040000
-#define MDP4_PIPE_OP_MODE_DEINT_ODD_REF				0x00080000
-
-static inline uint32_t REG_MDP4_PIPE_PHASEX_STEP(enum mdp4_pipe i0) { return 0x0002005c + 0x10000*i0; }
-
-static inline uint32_t REG_MDP4_PIPE_PHASEY_STEP(enum mdp4_pipe i0) { return 0x00020060 + 0x10000*i0; }
-
-static inline uint32_t REG_MDP4_PIPE_FETCH_CONFIG(enum mdp4_pipe i0) { return 0x00021004 + 0x10000*i0; }
-
-static inline uint32_t REG_MDP4_PIPE_SOLID_COLOR(enum mdp4_pipe i0) { return 0x00021008 + 0x10000*i0; }
-
-static inline uint32_t REG_MDP4_PIPE_CSC(enum mdp4_pipe i0) { return 0x00024000 + 0x10000*i0; }
-
-
-static inline uint32_t REG_MDP4_PIPE_CSC_MV(enum mdp4_pipe i0, uint32_t i1) { return 0x00024400 + 0x10000*i0 + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_PIPE_CSC_MV_VAL(enum mdp4_pipe i0, uint32_t i1) { return 0x00024400 + 0x10000*i0 + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_PIPE_CSC_PRE_BV(enum mdp4_pipe i0, uint32_t i1) { return 0x00024500 + 0x10000*i0 + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_PIPE_CSC_PRE_BV_VAL(enum mdp4_pipe i0, uint32_t i1) { return 0x00024500 + 0x10000*i0 + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_PIPE_CSC_POST_BV(enum mdp4_pipe i0, uint32_t i1) { return 0x00024580 + 0x10000*i0 + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_PIPE_CSC_POST_BV_VAL(enum mdp4_pipe i0, uint32_t i1) { return 0x00024580 + 0x10000*i0 + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_PIPE_CSC_PRE_LV(enum mdp4_pipe i0, uint32_t i1) { return 0x00024600 + 0x10000*i0 + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_PIPE_CSC_PRE_LV_VAL(enum mdp4_pipe i0, uint32_t i1) { return 0x00024600 + 0x10000*i0 + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_PIPE_CSC_POST_LV(enum mdp4_pipe i0, uint32_t i1) { return 0x00024680 + 0x10000*i0 + 0x4*i1; }
-
-static inline uint32_t REG_MDP4_PIPE_CSC_POST_LV_VAL(enum mdp4_pipe i0, uint32_t i1) { return 0x00024680 + 0x10000*i0 + 0x4*i1; }
-
-#define REG_MDP4_LCDC						0x000c0000
-
-#define REG_MDP4_LCDC_ENABLE					0x000c0000
-
-#define REG_MDP4_LCDC_HSYNC_CTRL				0x000c0004
-#define MDP4_LCDC_HSYNC_CTRL_PULSEW__MASK			0x0000ffff
-#define MDP4_LCDC_HSYNC_CTRL_PULSEW__SHIFT			0
-static inline uint32_t MDP4_LCDC_HSYNC_CTRL_PULSEW(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_HSYNC_CTRL_PULSEW__SHIFT) & MDP4_LCDC_HSYNC_CTRL_PULSEW__MASK;
-}
-#define MDP4_LCDC_HSYNC_CTRL_PERIOD__MASK			0xffff0000
-#define MDP4_LCDC_HSYNC_CTRL_PERIOD__SHIFT			16
-static inline uint32_t MDP4_LCDC_HSYNC_CTRL_PERIOD(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_HSYNC_CTRL_PERIOD__SHIFT) & MDP4_LCDC_HSYNC_CTRL_PERIOD__MASK;
-}
-
-#define REG_MDP4_LCDC_VSYNC_PERIOD				0x000c0008
-
-#define REG_MDP4_LCDC_VSYNC_LEN					0x000c000c
-
-#define REG_MDP4_LCDC_DISPLAY_HCTRL				0x000c0010
-#define MDP4_LCDC_DISPLAY_HCTRL_START__MASK			0x0000ffff
-#define MDP4_LCDC_DISPLAY_HCTRL_START__SHIFT			0
-static inline uint32_t MDP4_LCDC_DISPLAY_HCTRL_START(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_DISPLAY_HCTRL_START__SHIFT) & MDP4_LCDC_DISPLAY_HCTRL_START__MASK;
-}
-#define MDP4_LCDC_DISPLAY_HCTRL_END__MASK			0xffff0000
-#define MDP4_LCDC_DISPLAY_HCTRL_END__SHIFT			16
-static inline uint32_t MDP4_LCDC_DISPLAY_HCTRL_END(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_DISPLAY_HCTRL_END__SHIFT) & MDP4_LCDC_DISPLAY_HCTRL_END__MASK;
-}
-
-#define REG_MDP4_LCDC_DISPLAY_VSTART				0x000c0014
-
-#define REG_MDP4_LCDC_DISPLAY_VEND				0x000c0018
-
-#define REG_MDP4_LCDC_ACTIVE_HCTL				0x000c001c
-#define MDP4_LCDC_ACTIVE_HCTL_START__MASK			0x00007fff
-#define MDP4_LCDC_ACTIVE_HCTL_START__SHIFT			0
-static inline uint32_t MDP4_LCDC_ACTIVE_HCTL_START(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_ACTIVE_HCTL_START__SHIFT) & MDP4_LCDC_ACTIVE_HCTL_START__MASK;
-}
-#define MDP4_LCDC_ACTIVE_HCTL_END__MASK				0x7fff0000
-#define MDP4_LCDC_ACTIVE_HCTL_END__SHIFT			16
-static inline uint32_t MDP4_LCDC_ACTIVE_HCTL_END(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_ACTIVE_HCTL_END__SHIFT) & MDP4_LCDC_ACTIVE_HCTL_END__MASK;
-}
-#define MDP4_LCDC_ACTIVE_HCTL_ACTIVE_START_X			0x80000000
-
-#define REG_MDP4_LCDC_ACTIVE_VSTART				0x000c0020
-
-#define REG_MDP4_LCDC_ACTIVE_VEND				0x000c0024
-
-#define REG_MDP4_LCDC_BORDER_CLR				0x000c0028
-
-#define REG_MDP4_LCDC_UNDERFLOW_CLR				0x000c002c
-#define MDP4_LCDC_UNDERFLOW_CLR_COLOR__MASK			0x00ffffff
-#define MDP4_LCDC_UNDERFLOW_CLR_COLOR__SHIFT			0
-static inline uint32_t MDP4_LCDC_UNDERFLOW_CLR_COLOR(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_UNDERFLOW_CLR_COLOR__SHIFT) & MDP4_LCDC_UNDERFLOW_CLR_COLOR__MASK;
-}
-#define MDP4_LCDC_UNDERFLOW_CLR_ENABLE_RECOVERY			0x80000000
-
-#define REG_MDP4_LCDC_HSYNC_SKEW				0x000c0030
-
-#define REG_MDP4_LCDC_TEST_CNTL					0x000c0034
-
-#define REG_MDP4_LCDC_CTRL_POLARITY				0x000c0038
-#define MDP4_LCDC_CTRL_POLARITY_HSYNC_LOW			0x00000001
-#define MDP4_LCDC_CTRL_POLARITY_VSYNC_LOW			0x00000002
-#define MDP4_LCDC_CTRL_POLARITY_DATA_EN_LOW			0x00000004
-
-#define REG_MDP4_LCDC_LVDS_INTF_CTL				0x000c2000
-#define MDP4_LCDC_LVDS_INTF_CTL_MODE_SEL			0x00000004
-#define MDP4_LCDC_LVDS_INTF_CTL_RGB_OUT				0x00000008
-#define MDP4_LCDC_LVDS_INTF_CTL_CH_SWAP				0x00000010
-#define MDP4_LCDC_LVDS_INTF_CTL_CH1_RES_BIT			0x00000020
-#define MDP4_LCDC_LVDS_INTF_CTL_CH2_RES_BIT			0x00000040
-#define MDP4_LCDC_LVDS_INTF_CTL_ENABLE				0x00000080
-#define MDP4_LCDC_LVDS_INTF_CTL_CH1_DATA_LANE0_EN		0x00000100
-#define MDP4_LCDC_LVDS_INTF_CTL_CH1_DATA_LANE1_EN		0x00000200
-#define MDP4_LCDC_LVDS_INTF_CTL_CH1_DATA_LANE2_EN		0x00000400
-#define MDP4_LCDC_LVDS_INTF_CTL_CH1_DATA_LANE3_EN		0x00000800
-#define MDP4_LCDC_LVDS_INTF_CTL_CH2_DATA_LANE0_EN		0x00001000
-#define MDP4_LCDC_LVDS_INTF_CTL_CH2_DATA_LANE1_EN		0x00002000
-#define MDP4_LCDC_LVDS_INTF_CTL_CH2_DATA_LANE2_EN		0x00004000
-#define MDP4_LCDC_LVDS_INTF_CTL_CH2_DATA_LANE3_EN		0x00008000
-#define MDP4_LCDC_LVDS_INTF_CTL_CH1_CLK_LANE_EN			0x00010000
-#define MDP4_LCDC_LVDS_INTF_CTL_CH2_CLK_LANE_EN			0x00020000
-
-static inline uint32_t REG_MDP4_LCDC_LVDS_MUX_CTL(uint32_t i0) { return 0x000c2014 + 0x8*i0; }
-
-static inline uint32_t REG_MDP4_LCDC_LVDS_MUX_CTL_3_TO_0(uint32_t i0) { return 0x000c2014 + 0x8*i0; }
-#define MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT0__MASK		0x000000ff
-#define MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT0__SHIFT		0
-static inline uint32_t MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT0(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT0__SHIFT) & MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT0__MASK;
-}
-#define MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT1__MASK		0x0000ff00
-#define MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT1__SHIFT		8
-static inline uint32_t MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT1(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT1__SHIFT) & MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT1__MASK;
-}
-#define MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT2__MASK		0x00ff0000
-#define MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT2__SHIFT		16
-static inline uint32_t MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT2(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT2__SHIFT) & MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT2__MASK;
-}
-#define MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT3__MASK		0xff000000
-#define MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT3__SHIFT		24
-static inline uint32_t MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT3(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT3__SHIFT) & MDP4_LCDC_LVDS_MUX_CTL_3_TO_0_BIT3__MASK;
-}
-
-static inline uint32_t REG_MDP4_LCDC_LVDS_MUX_CTL_6_TO_4(uint32_t i0) { return 0x000c2018 + 0x8*i0; }
-#define MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT4__MASK		0x000000ff
-#define MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT4__SHIFT		0
-static inline uint32_t MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT4(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT4__SHIFT) & MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT4__MASK;
-}
-#define MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT5__MASK		0x0000ff00
-#define MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT5__SHIFT		8
-static inline uint32_t MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT5(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT5__SHIFT) & MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT5__MASK;
-}
-#define MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT6__MASK		0x00ff0000
-#define MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT6__SHIFT		16
-static inline uint32_t MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT6(uint32_t val)
-{
-	return ((val) << MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT6__SHIFT) & MDP4_LCDC_LVDS_MUX_CTL_6_TO_4_BIT6__MASK;
-}
-
-#define REG_MDP4_LCDC_LVDS_PHY_RESET				0x000c2034
-
-#define REG_MDP4_LVDS_PHY_PLL_CTRL_0				0x000c3000
-
-#define REG_MDP4_LVDS_PHY_PLL_CTRL_1				0x000c3004
-
-#define REG_MDP4_LVDS_PHY_PLL_CTRL_2				0x000c3008
-
-#define REG_MDP4_LVDS_PHY_PLL_CTRL_3				0x000c300c
-
-#define REG_MDP4_LVDS_PHY_PLL_CTRL_5				0x000c3014
-
-#define REG_MDP4_LVDS_PHY_PLL_CTRL_6				0x000c3018
-
-#define REG_MDP4_LVDS_PHY_PLL_CTRL_7				0x000c301c
-
-#define REG_MDP4_LVDS_PHY_PLL_CTRL_8				0x000c3020
-
-#define REG_MDP4_LVDS_PHY_PLL_CTRL_9				0x000c3024
-
-#define REG_MDP4_LVDS_PHY_PLL_LOCKED				0x000c3080
-
-#define REG_MDP4_LVDS_PHY_CFG2					0x000c3108
-
-#define REG_MDP4_LVDS_PHY_CFG0					0x000c3100
-#define MDP4_LVDS_PHY_CFG0_SERIALIZATION_ENBLE			0x00000010
-#define MDP4_LVDS_PHY_CFG0_CHANNEL0				0x00000040
-#define MDP4_LVDS_PHY_CFG0_CHANNEL1				0x00000080
-
-#define REG_MDP4_DTV						0x000d0000
-
-#define REG_MDP4_DTV_ENABLE					0x000d0000
-
-#define REG_MDP4_DTV_HSYNC_CTRL					0x000d0004
-#define MDP4_DTV_HSYNC_CTRL_PULSEW__MASK			0x0000ffff
-#define MDP4_DTV_HSYNC_CTRL_PULSEW__SHIFT			0
-static inline uint32_t MDP4_DTV_HSYNC_CTRL_PULSEW(uint32_t val)
-{
-	return ((val) << MDP4_DTV_HSYNC_CTRL_PULSEW__SHIFT) & MDP4_DTV_HSYNC_CTRL_PULSEW__MASK;
-}
-#define MDP4_DTV_HSYNC_CTRL_PERIOD__MASK			0xffff0000
-#define MDP4_DTV_HSYNC_CTRL_PERIOD__SHIFT			16
-static inline uint32_t MDP4_DTV_HSYNC_CTRL_PERIOD(uint32_t val)
-{
-	return ((val) << MDP4_DTV_HSYNC_CTRL_PERIOD__SHIFT) & MDP4_DTV_HSYNC_CTRL_PERIOD__MASK;
-}
-
-#define REG_MDP4_DTV_VSYNC_PERIOD				0x000d0008
-
-#define REG_MDP4_DTV_VSYNC_LEN					0x000d000c
-
-#define REG_MDP4_DTV_DISPLAY_HCTRL				0x000d0018
-#define MDP4_DTV_DISPLAY_HCTRL_START__MASK			0x0000ffff
-#define MDP4_DTV_DISPLAY_HCTRL_START__SHIFT			0
-static inline uint32_t MDP4_DTV_DISPLAY_HCTRL_START(uint32_t val)
-{
-	return ((val) << MDP4_DTV_DISPLAY_HCTRL_START__SHIFT) & MDP4_DTV_DISPLAY_HCTRL_START__MASK;
-}
-#define MDP4_DTV_DISPLAY_HCTRL_END__MASK			0xffff0000
-#define MDP4_DTV_DISPLAY_HCTRL_END__SHIFT			16
-static inline uint32_t MDP4_DTV_DISPLAY_HCTRL_END(uint32_t val)
-{
-	return ((val) << MDP4_DTV_DISPLAY_HCTRL_END__SHIFT) & MDP4_DTV_DISPLAY_HCTRL_END__MASK;
-}
-
-#define REG_MDP4_DTV_DISPLAY_VSTART				0x000d001c
-
-#define REG_MDP4_DTV_DISPLAY_VEND				0x000d0020
-
-#define REG_MDP4_DTV_ACTIVE_HCTL				0x000d002c
-#define MDP4_DTV_ACTIVE_HCTL_START__MASK			0x00007fff
-#define MDP4_DTV_ACTIVE_HCTL_START__SHIFT			0
-static inline uint32_t MDP4_DTV_ACTIVE_HCTL_START(uint32_t val)
-{
-	return ((val) << MDP4_DTV_ACTIVE_HCTL_START__SHIFT) & MDP4_DTV_ACTIVE_HCTL_START__MASK;
-}
-#define MDP4_DTV_ACTIVE_HCTL_END__MASK				0x7fff0000
-#define MDP4_DTV_ACTIVE_HCTL_END__SHIFT				16
-static inline uint32_t MDP4_DTV_ACTIVE_HCTL_END(uint32_t val)
-{
-	return ((val) << MDP4_DTV_ACTIVE_HCTL_END__SHIFT) & MDP4_DTV_ACTIVE_HCTL_END__MASK;
-}
-#define MDP4_DTV_ACTIVE_HCTL_ACTIVE_START_X			0x80000000
-
-#define REG_MDP4_DTV_ACTIVE_VSTART				0x000d0030
-
-#define REG_MDP4_DTV_ACTIVE_VEND				0x000d0038
-
-#define REG_MDP4_DTV_BORDER_CLR					0x000d0040
-
-#define REG_MDP4_DTV_UNDERFLOW_CLR				0x000d0044
-#define MDP4_DTV_UNDERFLOW_CLR_COLOR__MASK			0x00ffffff
-#define MDP4_DTV_UNDERFLOW_CLR_COLOR__SHIFT			0
-static inline uint32_t MDP4_DTV_UNDERFLOW_CLR_COLOR(uint32_t val)
-{
-	return ((val) << MDP4_DTV_UNDERFLOW_CLR_COLOR__SHIFT) & MDP4_DTV_UNDERFLOW_CLR_COLOR__MASK;
-}
-#define MDP4_DTV_UNDERFLOW_CLR_ENABLE_RECOVERY			0x80000000
-
-#define REG_MDP4_DTV_HSYNC_SKEW					0x000d0048
-
-#define REG_MDP4_DTV_TEST_CNTL					0x000d004c
-
-#define REG_MDP4_DTV_CTRL_POLARITY				0x000d0050
-#define MDP4_DTV_CTRL_POLARITY_HSYNC_LOW			0x00000001
-#define MDP4_DTV_CTRL_POLARITY_VSYNC_LOW			0x00000002
-#define MDP4_DTV_CTRL_POLARITY_DATA_EN_LOW			0x00000004
-
-#define REG_MDP4_DSI						0x000e0000
-
-#define REG_MDP4_DSI_ENABLE					0x000e0000
-
-#define REG_MDP4_DSI_HSYNC_CTRL					0x000e0004
-#define MDP4_DSI_HSYNC_CTRL_PULSEW__MASK			0x0000ffff
-#define MDP4_DSI_HSYNC_CTRL_PULSEW__SHIFT			0
-static inline uint32_t MDP4_DSI_HSYNC_CTRL_PULSEW(uint32_t val)
-{
-	return ((val) << MDP4_DSI_HSYNC_CTRL_PULSEW__SHIFT) & MDP4_DSI_HSYNC_CTRL_PULSEW__MASK;
-}
-#define MDP4_DSI_HSYNC_CTRL_PERIOD__MASK			0xffff0000
-#define MDP4_DSI_HSYNC_CTRL_PERIOD__SHIFT			16
-static inline uint32_t MDP4_DSI_HSYNC_CTRL_PERIOD(uint32_t val)
-{
-	return ((val) << MDP4_DSI_HSYNC_CTRL_PERIOD__SHIFT) & MDP4_DSI_HSYNC_CTRL_PERIOD__MASK;
-}
-
-#define REG_MDP4_DSI_VSYNC_PERIOD				0x000e0008
-
-#define REG_MDP4_DSI_VSYNC_LEN					0x000e000c
-
-#define REG_MDP4_DSI_DISPLAY_HCTRL				0x000e0010
-#define MDP4_DSI_DISPLAY_HCTRL_START__MASK			0x0000ffff
-#define MDP4_DSI_DISPLAY_HCTRL_START__SHIFT			0
-static inline uint32_t MDP4_DSI_DISPLAY_HCTRL_START(uint32_t val)
-{
-	return ((val) << MDP4_DSI_DISPLAY_HCTRL_START__SHIFT) & MDP4_DSI_DISPLAY_HCTRL_START__MASK;
-}
-#define MDP4_DSI_DISPLAY_HCTRL_END__MASK			0xffff0000
-#define MDP4_DSI_DISPLAY_HCTRL_END__SHIFT			16
-static inline uint32_t MDP4_DSI_DISPLAY_HCTRL_END(uint32_t val)
-{
-	return ((val) << MDP4_DSI_DISPLAY_HCTRL_END__SHIFT) & MDP4_DSI_DISPLAY_HCTRL_END__MASK;
-}
-
-#define REG_MDP4_DSI_DISPLAY_VSTART				0x000e0014
-
-#define REG_MDP4_DSI_DISPLAY_VEND				0x000e0018
-
-#define REG_MDP4_DSI_ACTIVE_HCTL				0x000e001c
-#define MDP4_DSI_ACTIVE_HCTL_START__MASK			0x00007fff
-#define MDP4_DSI_ACTIVE_HCTL_START__SHIFT			0
-static inline uint32_t MDP4_DSI_ACTIVE_HCTL_START(uint32_t val)
-{
-	return ((val) << MDP4_DSI_ACTIVE_HCTL_START__SHIFT) & MDP4_DSI_ACTIVE_HCTL_START__MASK;
-}
-#define MDP4_DSI_ACTIVE_HCTL_END__MASK				0x7fff0000
-#define MDP4_DSI_ACTIVE_HCTL_END__SHIFT				16
-static inline uint32_t MDP4_DSI_ACTIVE_HCTL_END(uint32_t val)
-{
-	return ((val) << MDP4_DSI_ACTIVE_HCTL_END__SHIFT) & MDP4_DSI_ACTIVE_HCTL_END__MASK;
-}
-#define MDP4_DSI_ACTIVE_HCTL_ACTIVE_START_X			0x80000000
-
-#define REG_MDP4_DSI_ACTIVE_VSTART				0x000e0020
-
-#define REG_MDP4_DSI_ACTIVE_VEND				0x000e0024
-
-#define REG_MDP4_DSI_BORDER_CLR					0x000e0028
-
-#define REG_MDP4_DSI_UNDERFLOW_CLR				0x000e002c
-#define MDP4_DSI_UNDERFLOW_CLR_COLOR__MASK			0x00ffffff
-#define MDP4_DSI_UNDERFLOW_CLR_COLOR__SHIFT			0
-static inline uint32_t MDP4_DSI_UNDERFLOW_CLR_COLOR(uint32_t val)
-{
-	return ((val) << MDP4_DSI_UNDERFLOW_CLR_COLOR__SHIFT) & MDP4_DSI_UNDERFLOW_CLR_COLOR__MASK;
-}
-#define MDP4_DSI_UNDERFLOW_CLR_ENABLE_RECOVERY			0x80000000
-
-#define REG_MDP4_DSI_HSYNC_SKEW					0x000e0030
-
-#define REG_MDP4_DSI_TEST_CNTL					0x000e0034
-
-#define REG_MDP4_DSI_CTRL_POLARITY				0x000e0038
-#define MDP4_DSI_CTRL_POLARITY_HSYNC_LOW			0x00000001
-#define MDP4_DSI_CTRL_POLARITY_VSYNC_LOW			0x00000002
-#define MDP4_DSI_CTRL_POLARITY_DATA_EN_LOW			0x00000004
-
-
-#endif /* MDP4_XML */
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c
index 75f93e346282..b8610aa806ea 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c
@@ -182,8 +182,8 @@ static void blend_setup(struct drm_crtc *crtc)
 		enum mdp4_pipe pipe_id = mdp4_plane_pipe(plane);
 		int idx = idxs[pipe_id];
 		if (idx > 0) {
-			const struct mdp_format *format =
-					to_mdp_format(msm_framebuffer_format(plane->state->fb));
+			const struct msm_format *format =
+					msm_framebuffer_format(plane->state->fb);
 			alpha[idx-1] = format->alpha_enable;
 		}
 	}
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
index 4ba1cb74ad76..6e4e74f9d63d 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
@@ -151,7 +151,6 @@ static const struct mdp_kms_funcs kms_funcs = {
 		.flush_commit    = mdp4_flush_commit,
 		.wait_flush      = mdp4_wait_flush,
 		.complete_commit = mdp4_complete_commit,
-		.get_format      = mdp_get_format,
 		.round_pixclk    = mdp4_round_pixclk,
 		.destroy         = mdp4_destroy,
 	},
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.h b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.h
index 01179e764a29..94b1ba92785f 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.h
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.h
@@ -44,12 +44,12 @@ struct mdp4_kms {
 
 static inline void mdp4_write(struct mdp4_kms *mdp4_kms, u32 reg, u32 data)
 {
-	msm_writel(data, mdp4_kms->mmio + reg);
+	writel(data, mdp4_kms->mmio + reg);
 }
 
 static inline u32 mdp4_read(struct mdp4_kms *mdp4_kms, u32 reg)
 {
-	return msm_readl(mdp4_kms->mmio + reg);
+	return readl(mdp4_kms->mmio + reg);
 }
 
 static inline uint32_t pipe2flush(enum mdp4_pipe pipe)
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c
index b689b618da78..3fefb2088008 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c
@@ -20,12 +20,6 @@ struct mdp4_plane {
 	const char *name;
 
 	enum mdp4_pipe pipe;
-
-	uint32_t caps;
-	uint32_t nformats;
-	uint32_t formats[32];
-
-	bool enabled;
 };
 #define to_mdp4_plane(x) container_of(x, struct mdp4_plane, base)
 
@@ -59,15 +53,6 @@ static struct mdp4_kms *get_kms(struct drm_plane *plane)
 	return to_mdp4_kms(to_mdp_kms(priv->kms));
 }
 
-static void mdp4_plane_destroy(struct drm_plane *plane)
-{
-	struct mdp4_plane *mdp4_plane = to_mdp4_plane(plane);
-
-	drm_plane_cleanup(plane);
-
-	kfree(mdp4_plane);
-}
-
 /* helper to install properties which are common to planes and crtcs */
 static void mdp4_plane_install_properties(struct drm_plane *plane,
 		struct drm_mode_object *obj)
@@ -85,7 +70,6 @@ static int mdp4_plane_set_property(struct drm_plane *plane,
 static const struct drm_plane_funcs mdp4_plane_funcs = {
 		.update_plane = drm_atomic_helper_update_plane,
 		.disable_plane = drm_atomic_helper_disable_plane,
-		.destroy = mdp4_plane_destroy,
 		.set_property = mdp4_plane_set_property,
 		.reset = drm_atomic_helper_plane_reset,
 		.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
@@ -218,7 +202,7 @@ static int mdp4_plane_mode_set(struct drm_plane *plane,
 	struct mdp4_plane *mdp4_plane = to_mdp4_plane(plane);
 	struct mdp4_kms *mdp4_kms = get_kms(plane);
 	enum mdp4_pipe pipe = mdp4_plane->pipe;
-	const struct mdp_format *format;
+	const struct msm_format *format;
 	uint32_t op_mode = 0;
 	uint32_t phasex_step = MDP4_VG_PHASE_STEP_DEFAULT;
 	uint32_t phasey_step = MDP4_VG_PHASE_STEP_DEFAULT;
@@ -241,7 +225,7 @@ static int mdp4_plane_mode_set(struct drm_plane *plane,
 			fb->base.id, src_x, src_y, src_w, src_h,
 			crtc->base.id, crtc_x, crtc_y, crtc_w, crtc_h);
 
-	format = to_mdp_format(msm_framebuffer_format(fb));
+	format = msm_framebuffer_format(fb);
 
 	if (src_w > (crtc_w * DOWN_SCALE_MAX)) {
 		DRM_DEV_ERROR(dev->dev, "Width down scaling exceeds limits!\n");
@@ -267,7 +251,7 @@ static int mdp4_plane_mode_set(struct drm_plane *plane,
 		uint32_t sel_unit = SCALE_FIR;
 		op_mode |= MDP4_PIPE_OP_MODE_SCALEX_EN;
 
-		if (MDP_FORMAT_IS_YUV(format)) {
+		if (MSM_FORMAT_IS_YUV(format)) {
 			if (crtc_w > src_w)
 				sel_unit = SCALE_PIXEL_RPT;
 			else if (crtc_w <= (src_w / 4))
@@ -283,7 +267,7 @@ static int mdp4_plane_mode_set(struct drm_plane *plane,
 		uint32_t sel_unit = SCALE_FIR;
 		op_mode |= MDP4_PIPE_OP_MODE_SCALEY_EN;
 
-		if (MDP_FORMAT_IS_YUV(format)) {
+		if (MSM_FORMAT_IS_YUV(format)) {
 
 			if (crtc_h > src_h)
 				sel_unit = SCALE_PIXEL_RPT;
@@ -316,24 +300,25 @@ static int mdp4_plane_mode_set(struct drm_plane *plane,
 
 	mdp4_write(mdp4_kms, REG_MDP4_PIPE_SRC_FORMAT(pipe),
 			MDP4_PIPE_SRC_FORMAT_A_BPC(format->bpc_a) |
-			MDP4_PIPE_SRC_FORMAT_R_BPC(format->bpc_r) |
-			MDP4_PIPE_SRC_FORMAT_G_BPC(format->bpc_g) |
-			MDP4_PIPE_SRC_FORMAT_B_BPC(format->bpc_b) |
+			MDP4_PIPE_SRC_FORMAT_R_BPC(format->bpc_r_cr) |
+			MDP4_PIPE_SRC_FORMAT_G_BPC(format->bpc_g_y) |
+			MDP4_PIPE_SRC_FORMAT_B_BPC(format->bpc_b_cb) |
 			COND(format->alpha_enable, MDP4_PIPE_SRC_FORMAT_ALPHA_ENABLE) |
-			MDP4_PIPE_SRC_FORMAT_CPP(format->cpp - 1) |
+			MDP4_PIPE_SRC_FORMAT_CPP(format->bpp - 1) |
 			MDP4_PIPE_SRC_FORMAT_UNPACK_COUNT(format->unpack_count - 1) |
 			MDP4_PIPE_SRC_FORMAT_FETCH_PLANES(format->fetch_type) |
 			MDP4_PIPE_SRC_FORMAT_CHROMA_SAMP(format->chroma_sample) |
 			MDP4_PIPE_SRC_FORMAT_FRAME_FORMAT(frame_type) |
-			COND(format->unpack_tight, MDP4_PIPE_SRC_FORMAT_UNPACK_TIGHT));
+			COND(format->flags & MSM_FORMAT_FLAG_UNPACK_TIGHT,
+			     MDP4_PIPE_SRC_FORMAT_UNPACK_TIGHT));
 
 	mdp4_write(mdp4_kms, REG_MDP4_PIPE_SRC_UNPACK(pipe),
-			MDP4_PIPE_SRC_UNPACK_ELEM0(format->unpack[0]) |
-			MDP4_PIPE_SRC_UNPACK_ELEM1(format->unpack[1]) |
-			MDP4_PIPE_SRC_UNPACK_ELEM2(format->unpack[2]) |
-			MDP4_PIPE_SRC_UNPACK_ELEM3(format->unpack[3]));
+			MDP4_PIPE_SRC_UNPACK_ELEM0(format->element[0]) |
+			MDP4_PIPE_SRC_UNPACK_ELEM1(format->element[1]) |
+			MDP4_PIPE_SRC_UNPACK_ELEM2(format->element[2]) |
+			MDP4_PIPE_SRC_UNPACK_ELEM3(format->element[3]));
 
-	if (MDP_FORMAT_IS_YUV(format)) {
+	if (MSM_FORMAT_IS_YUV(format)) {
 		struct csc_cfg *csc = mdp_get_default_csc_cfg(CSC_YUV2RGB);
 
 		op_mode |= MDP4_PIPE_OP_MODE_SRC_YCBCR;
@@ -371,37 +356,81 @@ static const uint64_t supported_format_modifiers[] = {
 	DRM_FORMAT_MOD_INVALID
 };
 
+static const uint32_t mdp4_rgb_formats[] = {
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_BGR565,
+};
+
+static const uint32_t mdp4_rgb_yuv_formats[] = {
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_BGR565,
+
+	DRM_FORMAT_NV12,
+	DRM_FORMAT_NV21,
+	DRM_FORMAT_NV16,
+	DRM_FORMAT_NV61,
+	DRM_FORMAT_VYUY,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_YUV420,
+	DRM_FORMAT_YVU420,
+};
+
 /* initialize plane */
 struct drm_plane *mdp4_plane_init(struct drm_device *dev,
 		enum mdp4_pipe pipe_id, bool private_plane)
 {
 	struct drm_plane *plane = NULL;
 	struct mdp4_plane *mdp4_plane;
-	int ret;
 	enum drm_plane_type type;
+	uint32_t pipe_caps;
+	const uint32_t *formats;
+	size_t nformats;
 
-	mdp4_plane = kzalloc(sizeof(*mdp4_plane), GFP_KERNEL);
-	if (!mdp4_plane) {
-		ret = -ENOMEM;
-		goto fail;
+	type = private_plane ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
+
+	pipe_caps = mdp4_pipe_caps(pipe_id);
+	if (pipe_supports_yuv(pipe_caps)) {
+		formats = mdp4_rgb_yuv_formats;
+		nformats = ARRAY_SIZE(mdp4_rgb_yuv_formats);
+	} else {
+		formats = mdp4_rgb_formats;
+		nformats = ARRAY_SIZE(mdp4_rgb_formats);
 	}
 
+	mdp4_plane = drmm_universal_plane_alloc(dev, struct mdp4_plane, base,
+						0xff, &mdp4_plane_funcs,
+						formats, nformats,
+						supported_format_modifiers,
+						type, NULL);
+	if (IS_ERR(mdp4_plane))
+		return ERR_CAST(mdp4_plane);
+
 	plane = &mdp4_plane->base;
 
 	mdp4_plane->pipe = pipe_id;
 	mdp4_plane->name = pipe_names[pipe_id];
-	mdp4_plane->caps = mdp4_pipe_caps(pipe_id);
-
-	mdp4_plane->nformats = mdp_get_formats(mdp4_plane->formats,
-			ARRAY_SIZE(mdp4_plane->formats),
-			!pipe_supports_yuv(mdp4_plane->caps));
-
-	type = private_plane ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
-	ret = drm_universal_plane_init(dev, plane, 0xff, &mdp4_plane_funcs,
-				 mdp4_plane->formats, mdp4_plane->nformats,
-				 supported_format_modifiers, type, NULL);
-	if (ret)
-		goto fail;
 
 	drm_plane_helper_add(plane, &mdp4_plane_helper_funcs);
 
@@ -410,10 +439,4 @@ struct drm_plane *mdp4_plane_init(struct drm_device *dev,
 	drm_plane_enable_fb_damage_clips(plane);
 
 	return plane;
-
-fail:
-	if (plane)
-		mdp4_plane_destroy(plane);
-
-	return ERR_PTR(ret);
 }
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h
deleted file mode 100644
index 270e11c904bd..000000000000
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h
+++ /dev/null
@@ -1,1979 +0,0 @@
-#ifndef MDP5_XML
-#define MDP5_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-enum mdp5_intf_type {
-	INTF_DISABLED = 0,
-	INTF_DSI = 1,
-	INTF_HDMI = 3,
-	INTF_LCDC = 5,
-	INTF_eDP = 9,
-	INTF_VIRTUAL = 100,
-	INTF_WB = 101,
-};
-
-enum mdp5_intfnum {
-	NO_INTF = 0,
-	INTF0 = 1,
-	INTF1 = 2,
-	INTF2 = 3,
-	INTF3 = 4,
-};
-
-enum mdp5_pipe {
-	SSPP_NONE = 0,
-	SSPP_VIG0 = 1,
-	SSPP_VIG1 = 2,
-	SSPP_VIG2 = 3,
-	SSPP_RGB0 = 4,
-	SSPP_RGB1 = 5,
-	SSPP_RGB2 = 6,
-	SSPP_DMA0 = 7,
-	SSPP_DMA1 = 8,
-	SSPP_VIG3 = 9,
-	SSPP_RGB3 = 10,
-	SSPP_CURSOR0 = 11,
-	SSPP_CURSOR1 = 12,
-};
-
-enum mdp5_format {
-	DUMMY = 0,
-};
-
-enum mdp5_ctl_mode {
-	MODE_NONE = 0,
-	MODE_WB_0_BLOCK = 1,
-	MODE_WB_1_BLOCK = 2,
-	MODE_WB_0_LINE = 3,
-	MODE_WB_1_LINE = 4,
-	MODE_WB_2_LINE = 5,
-};
-
-enum mdp5_pack_3d {
-	PACK_3D_FRAME_INT = 0,
-	PACK_3D_H_ROW_INT = 1,
-	PACK_3D_V_ROW_INT = 2,
-	PACK_3D_COL_INT = 3,
-};
-
-enum mdp5_scale_filter {
-	SCALE_FILTER_NEAREST = 0,
-	SCALE_FILTER_BIL = 1,
-	SCALE_FILTER_PCMN = 2,
-	SCALE_FILTER_CA = 3,
-};
-
-enum mdp5_pipe_bwc {
-	BWC_LOSSLESS = 0,
-	BWC_Q_HIGH = 1,
-	BWC_Q_MED = 2,
-};
-
-enum mdp5_cursor_format {
-	CURSOR_FMT_ARGB8888 = 0,
-	CURSOR_FMT_ARGB1555 = 2,
-	CURSOR_FMT_ARGB4444 = 4,
-};
-
-enum mdp5_cursor_alpha {
-	CURSOR_ALPHA_CONST = 0,
-	CURSOR_ALPHA_PER_PIXEL = 2,
-};
-
-enum mdp5_igc_type {
-	IGC_VIG = 0,
-	IGC_RGB = 1,
-	IGC_DMA = 2,
-	IGC_DSPP = 3,
-};
-
-enum mdp5_data_format {
-	DATA_FORMAT_RGB = 0,
-	DATA_FORMAT_YUV = 1,
-};
-
-enum mdp5_block_size {
-	BLOCK_SIZE_64 = 0,
-	BLOCK_SIZE_128 = 1,
-};
-
-enum mdp5_rotate_mode {
-	ROTATE_0 = 0,
-	ROTATE_90 = 1,
-};
-
-enum mdp5_chroma_downsample_method {
-	DS_MTHD_NO_PIXEL_DROP = 0,
-	DS_MTHD_PIXEL_DROP = 1,
-};
-
-#define MDP5_IRQ_WB_0_DONE					0x00000001
-#define MDP5_IRQ_WB_1_DONE					0x00000002
-#define MDP5_IRQ_WB_2_DONE					0x00000010
-#define MDP5_IRQ_PING_PONG_0_DONE				0x00000100
-#define MDP5_IRQ_PING_PONG_1_DONE				0x00000200
-#define MDP5_IRQ_PING_PONG_2_DONE				0x00000400
-#define MDP5_IRQ_PING_PONG_3_DONE				0x00000800
-#define MDP5_IRQ_PING_PONG_0_RD_PTR				0x00001000
-#define MDP5_IRQ_PING_PONG_1_RD_PTR				0x00002000
-#define MDP5_IRQ_PING_PONG_2_RD_PTR				0x00004000
-#define MDP5_IRQ_PING_PONG_3_RD_PTR				0x00008000
-#define MDP5_IRQ_PING_PONG_0_WR_PTR				0x00010000
-#define MDP5_IRQ_PING_PONG_1_WR_PTR				0x00020000
-#define MDP5_IRQ_PING_PONG_2_WR_PTR				0x00040000
-#define MDP5_IRQ_PING_PONG_3_WR_PTR				0x00080000
-#define MDP5_IRQ_PING_PONG_0_AUTO_REF				0x00100000
-#define MDP5_IRQ_PING_PONG_1_AUTO_REF				0x00200000
-#define MDP5_IRQ_PING_PONG_2_AUTO_REF				0x00400000
-#define MDP5_IRQ_PING_PONG_3_AUTO_REF				0x00800000
-#define MDP5_IRQ_INTF0_UNDER_RUN				0x01000000
-#define MDP5_IRQ_INTF0_VSYNC					0x02000000
-#define MDP5_IRQ_INTF1_UNDER_RUN				0x04000000
-#define MDP5_IRQ_INTF1_VSYNC					0x08000000
-#define MDP5_IRQ_INTF2_UNDER_RUN				0x10000000
-#define MDP5_IRQ_INTF2_VSYNC					0x20000000
-#define MDP5_IRQ_INTF3_UNDER_RUN				0x40000000
-#define MDP5_IRQ_INTF3_VSYNC					0x80000000
-#define REG_MDSS_HW_VERSION					0x00000000
-#define MDSS_HW_VERSION_STEP__MASK				0x0000ffff
-#define MDSS_HW_VERSION_STEP__SHIFT				0
-static inline uint32_t MDSS_HW_VERSION_STEP(uint32_t val)
-{
-	return ((val) << MDSS_HW_VERSION_STEP__SHIFT) & MDSS_HW_VERSION_STEP__MASK;
-}
-#define MDSS_HW_VERSION_MINOR__MASK				0x0fff0000
-#define MDSS_HW_VERSION_MINOR__SHIFT				16
-static inline uint32_t MDSS_HW_VERSION_MINOR(uint32_t val)
-{
-	return ((val) << MDSS_HW_VERSION_MINOR__SHIFT) & MDSS_HW_VERSION_MINOR__MASK;
-}
-#define MDSS_HW_VERSION_MAJOR__MASK				0xf0000000
-#define MDSS_HW_VERSION_MAJOR__SHIFT				28
-static inline uint32_t MDSS_HW_VERSION_MAJOR(uint32_t val)
-{
-	return ((val) << MDSS_HW_VERSION_MAJOR__SHIFT) & MDSS_HW_VERSION_MAJOR__MASK;
-}
-
-#define REG_MDSS_HW_INTR_STATUS					0x00000010
-#define MDSS_HW_INTR_STATUS_INTR_MDP				0x00000001
-#define MDSS_HW_INTR_STATUS_INTR_DSI0				0x00000010
-#define MDSS_HW_INTR_STATUS_INTR_DSI1				0x00000020
-#define MDSS_HW_INTR_STATUS_INTR_HDMI				0x00000100
-#define MDSS_HW_INTR_STATUS_INTR_EDP				0x00001000
-
-#define REG_MDP5_HW_VERSION					0x00000000
-#define MDP5_HW_VERSION_STEP__MASK				0x0000ffff
-#define MDP5_HW_VERSION_STEP__SHIFT				0
-static inline uint32_t MDP5_HW_VERSION_STEP(uint32_t val)
-{
-	return ((val) << MDP5_HW_VERSION_STEP__SHIFT) & MDP5_HW_VERSION_STEP__MASK;
-}
-#define MDP5_HW_VERSION_MINOR__MASK				0x0fff0000
-#define MDP5_HW_VERSION_MINOR__SHIFT				16
-static inline uint32_t MDP5_HW_VERSION_MINOR(uint32_t val)
-{
-	return ((val) << MDP5_HW_VERSION_MINOR__SHIFT) & MDP5_HW_VERSION_MINOR__MASK;
-}
-#define MDP5_HW_VERSION_MAJOR__MASK				0xf0000000
-#define MDP5_HW_VERSION_MAJOR__SHIFT				28
-static inline uint32_t MDP5_HW_VERSION_MAJOR(uint32_t val)
-{
-	return ((val) << MDP5_HW_VERSION_MAJOR__SHIFT) & MDP5_HW_VERSION_MAJOR__MASK;
-}
-
-#define REG_MDP5_DISP_INTF_SEL					0x00000004
-#define MDP5_DISP_INTF_SEL_INTF0__MASK				0x000000ff
-#define MDP5_DISP_INTF_SEL_INTF0__SHIFT				0
-static inline uint32_t MDP5_DISP_INTF_SEL_INTF0(enum mdp5_intf_type val)
-{
-	return ((val) << MDP5_DISP_INTF_SEL_INTF0__SHIFT) & MDP5_DISP_INTF_SEL_INTF0__MASK;
-}
-#define MDP5_DISP_INTF_SEL_INTF1__MASK				0x0000ff00
-#define MDP5_DISP_INTF_SEL_INTF1__SHIFT				8
-static inline uint32_t MDP5_DISP_INTF_SEL_INTF1(enum mdp5_intf_type val)
-{
-	return ((val) << MDP5_DISP_INTF_SEL_INTF1__SHIFT) & MDP5_DISP_INTF_SEL_INTF1__MASK;
-}
-#define MDP5_DISP_INTF_SEL_INTF2__MASK				0x00ff0000
-#define MDP5_DISP_INTF_SEL_INTF2__SHIFT				16
-static inline uint32_t MDP5_DISP_INTF_SEL_INTF2(enum mdp5_intf_type val)
-{
-	return ((val) << MDP5_DISP_INTF_SEL_INTF2__SHIFT) & MDP5_DISP_INTF_SEL_INTF2__MASK;
-}
-#define MDP5_DISP_INTF_SEL_INTF3__MASK				0xff000000
-#define MDP5_DISP_INTF_SEL_INTF3__SHIFT				24
-static inline uint32_t MDP5_DISP_INTF_SEL_INTF3(enum mdp5_intf_type val)
-{
-	return ((val) << MDP5_DISP_INTF_SEL_INTF3__SHIFT) & MDP5_DISP_INTF_SEL_INTF3__MASK;
-}
-
-#define REG_MDP5_INTR_EN					0x00000010
-
-#define REG_MDP5_INTR_STATUS					0x00000014
-
-#define REG_MDP5_INTR_CLEAR					0x00000018
-
-#define REG_MDP5_HIST_INTR_EN					0x0000001c
-
-#define REG_MDP5_HIST_INTR_STATUS				0x00000020
-
-#define REG_MDP5_HIST_INTR_CLEAR				0x00000024
-
-#define REG_MDP5_SPARE_0					0x00000028
-#define MDP5_SPARE_0_SPLIT_DPL_SINGLE_FLUSH_EN			0x00000001
-
-static inline uint32_t REG_MDP5_SMP_ALLOC_W(uint32_t i0) { return 0x00000080 + 0x4*i0; }
-
-static inline uint32_t REG_MDP5_SMP_ALLOC_W_REG(uint32_t i0) { return 0x00000080 + 0x4*i0; }
-#define MDP5_SMP_ALLOC_W_REG_CLIENT0__MASK			0x000000ff
-#define MDP5_SMP_ALLOC_W_REG_CLIENT0__SHIFT			0
-static inline uint32_t MDP5_SMP_ALLOC_W_REG_CLIENT0(uint32_t val)
-{
-	return ((val) << MDP5_SMP_ALLOC_W_REG_CLIENT0__SHIFT) & MDP5_SMP_ALLOC_W_REG_CLIENT0__MASK;
-}
-#define MDP5_SMP_ALLOC_W_REG_CLIENT1__MASK			0x0000ff00
-#define MDP5_SMP_ALLOC_W_REG_CLIENT1__SHIFT			8
-static inline uint32_t MDP5_SMP_ALLOC_W_REG_CLIENT1(uint32_t val)
-{
-	return ((val) << MDP5_SMP_ALLOC_W_REG_CLIENT1__SHIFT) & MDP5_SMP_ALLOC_W_REG_CLIENT1__MASK;
-}
-#define MDP5_SMP_ALLOC_W_REG_CLIENT2__MASK			0x00ff0000
-#define MDP5_SMP_ALLOC_W_REG_CLIENT2__SHIFT			16
-static inline uint32_t MDP5_SMP_ALLOC_W_REG_CLIENT2(uint32_t val)
-{
-	return ((val) << MDP5_SMP_ALLOC_W_REG_CLIENT2__SHIFT) & MDP5_SMP_ALLOC_W_REG_CLIENT2__MASK;
-}
-
-static inline uint32_t REG_MDP5_SMP_ALLOC_R(uint32_t i0) { return 0x00000130 + 0x4*i0; }
-
-static inline uint32_t REG_MDP5_SMP_ALLOC_R_REG(uint32_t i0) { return 0x00000130 + 0x4*i0; }
-#define MDP5_SMP_ALLOC_R_REG_CLIENT0__MASK			0x000000ff
-#define MDP5_SMP_ALLOC_R_REG_CLIENT0__SHIFT			0
-static inline uint32_t MDP5_SMP_ALLOC_R_REG_CLIENT0(uint32_t val)
-{
-	return ((val) << MDP5_SMP_ALLOC_R_REG_CLIENT0__SHIFT) & MDP5_SMP_ALLOC_R_REG_CLIENT0__MASK;
-}
-#define MDP5_SMP_ALLOC_R_REG_CLIENT1__MASK			0x0000ff00
-#define MDP5_SMP_ALLOC_R_REG_CLIENT1__SHIFT			8
-static inline uint32_t MDP5_SMP_ALLOC_R_REG_CLIENT1(uint32_t val)
-{
-	return ((val) << MDP5_SMP_ALLOC_R_REG_CLIENT1__SHIFT) & MDP5_SMP_ALLOC_R_REG_CLIENT1__MASK;
-}
-#define MDP5_SMP_ALLOC_R_REG_CLIENT2__MASK			0x00ff0000
-#define MDP5_SMP_ALLOC_R_REG_CLIENT2__SHIFT			16
-static inline uint32_t MDP5_SMP_ALLOC_R_REG_CLIENT2(uint32_t val)
-{
-	return ((val) << MDP5_SMP_ALLOC_R_REG_CLIENT2__SHIFT) & MDP5_SMP_ALLOC_R_REG_CLIENT2__MASK;
-}
-
-static inline uint32_t __offset_IGC(enum mdp5_igc_type idx)
-{
-	switch (idx) {
-		case IGC_VIG: return 0x00000200;
-		case IGC_RGB: return 0x00000210;
-		case IGC_DMA: return 0x00000220;
-		case IGC_DSPP: return 0x00000300;
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_IGC(enum mdp5_igc_type i0) { return 0x00000000 + __offset_IGC(i0); }
-
-static inline uint32_t REG_MDP5_IGC_LUT(enum mdp5_igc_type i0, uint32_t i1) { return 0x00000000 + __offset_IGC(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP5_IGC_LUT_REG(enum mdp5_igc_type i0, uint32_t i1) { return 0x00000000 + __offset_IGC(i0) + 0x4*i1; }
-#define MDP5_IGC_LUT_REG_VAL__MASK				0x00000fff
-#define MDP5_IGC_LUT_REG_VAL__SHIFT				0
-static inline uint32_t MDP5_IGC_LUT_REG_VAL(uint32_t val)
-{
-	return ((val) << MDP5_IGC_LUT_REG_VAL__SHIFT) & MDP5_IGC_LUT_REG_VAL__MASK;
-}
-#define MDP5_IGC_LUT_REG_INDEX_UPDATE				0x02000000
-#define MDP5_IGC_LUT_REG_DISABLE_PIPE_0				0x10000000
-#define MDP5_IGC_LUT_REG_DISABLE_PIPE_1				0x20000000
-#define MDP5_IGC_LUT_REG_DISABLE_PIPE_2				0x40000000
-
-#define REG_MDP5_SPLIT_DPL_EN					0x000002f4
-
-#define REG_MDP5_SPLIT_DPL_UPPER				0x000002f8
-#define MDP5_SPLIT_DPL_UPPER_SMART_PANEL			0x00000002
-#define MDP5_SPLIT_DPL_UPPER_SMART_PANEL_FREE_RUN		0x00000004
-#define MDP5_SPLIT_DPL_UPPER_INTF1_SW_TRG_MUX			0x00000010
-#define MDP5_SPLIT_DPL_UPPER_INTF2_SW_TRG_MUX			0x00000100
-
-#define REG_MDP5_SPLIT_DPL_LOWER				0x000003f0
-#define MDP5_SPLIT_DPL_LOWER_SMART_PANEL			0x00000002
-#define MDP5_SPLIT_DPL_LOWER_SMART_PANEL_FREE_RUN		0x00000004
-#define MDP5_SPLIT_DPL_LOWER_INTF1_TG_SYNC			0x00000010
-#define MDP5_SPLIT_DPL_LOWER_INTF2_TG_SYNC			0x00000100
-
-static inline uint32_t __offset_CTL(uint32_t idx)
-{
-	switch (idx) {
-		case 0: return (mdp5_cfg->ctl.base[0]);
-		case 1: return (mdp5_cfg->ctl.base[1]);
-		case 2: return (mdp5_cfg->ctl.base[2]);
-		case 3: return (mdp5_cfg->ctl.base[3]);
-		case 4: return (mdp5_cfg->ctl.base[4]);
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_CTL(uint32_t i0) { return 0x00000000 + __offset_CTL(i0); }
-
-static inline uint32_t __offset_LAYER(uint32_t idx)
-{
-	switch (idx) {
-		case 0: return 0x00000000;
-		case 1: return 0x00000004;
-		case 2: return 0x00000008;
-		case 3: return 0x0000000c;
-		case 4: return 0x00000010;
-		case 5: return 0x00000024;
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_CTL_LAYER(uint32_t i0, uint32_t i1) { return 0x00000000 + __offset_CTL(i0) + __offset_LAYER(i1); }
-
-static inline uint32_t REG_MDP5_CTL_LAYER_REG(uint32_t i0, uint32_t i1) { return 0x00000000 + __offset_CTL(i0) + __offset_LAYER(i1); }
-#define MDP5_CTL_LAYER_REG_VIG0__MASK				0x00000007
-#define MDP5_CTL_LAYER_REG_VIG0__SHIFT				0
-static inline uint32_t MDP5_CTL_LAYER_REG_VIG0(uint32_t val)
-{
-	return ((val) << MDP5_CTL_LAYER_REG_VIG0__SHIFT) & MDP5_CTL_LAYER_REG_VIG0__MASK;
-}
-#define MDP5_CTL_LAYER_REG_VIG1__MASK				0x00000038
-#define MDP5_CTL_LAYER_REG_VIG1__SHIFT				3
-static inline uint32_t MDP5_CTL_LAYER_REG_VIG1(uint32_t val)
-{
-	return ((val) << MDP5_CTL_LAYER_REG_VIG1__SHIFT) & MDP5_CTL_LAYER_REG_VIG1__MASK;
-}
-#define MDP5_CTL_LAYER_REG_VIG2__MASK				0x000001c0
-#define MDP5_CTL_LAYER_REG_VIG2__SHIFT				6
-static inline uint32_t MDP5_CTL_LAYER_REG_VIG2(uint32_t val)
-{
-	return ((val) << MDP5_CTL_LAYER_REG_VIG2__SHIFT) & MDP5_CTL_LAYER_REG_VIG2__MASK;
-}
-#define MDP5_CTL_LAYER_REG_RGB0__MASK				0x00000e00
-#define MDP5_CTL_LAYER_REG_RGB0__SHIFT				9
-static inline uint32_t MDP5_CTL_LAYER_REG_RGB0(uint32_t val)
-{
-	return ((val) << MDP5_CTL_LAYER_REG_RGB0__SHIFT) & MDP5_CTL_LAYER_REG_RGB0__MASK;
-}
-#define MDP5_CTL_LAYER_REG_RGB1__MASK				0x00007000
-#define MDP5_CTL_LAYER_REG_RGB1__SHIFT				12
-static inline uint32_t MDP5_CTL_LAYER_REG_RGB1(uint32_t val)
-{
-	return ((val) << MDP5_CTL_LAYER_REG_RGB1__SHIFT) & MDP5_CTL_LAYER_REG_RGB1__MASK;
-}
-#define MDP5_CTL_LAYER_REG_RGB2__MASK				0x00038000
-#define MDP5_CTL_LAYER_REG_RGB2__SHIFT				15
-static inline uint32_t MDP5_CTL_LAYER_REG_RGB2(uint32_t val)
-{
-	return ((val) << MDP5_CTL_LAYER_REG_RGB2__SHIFT) & MDP5_CTL_LAYER_REG_RGB2__MASK;
-}
-#define MDP5_CTL_LAYER_REG_DMA0__MASK				0x001c0000
-#define MDP5_CTL_LAYER_REG_DMA0__SHIFT				18
-static inline uint32_t MDP5_CTL_LAYER_REG_DMA0(uint32_t val)
-{
-	return ((val) << MDP5_CTL_LAYER_REG_DMA0__SHIFT) & MDP5_CTL_LAYER_REG_DMA0__MASK;
-}
-#define MDP5_CTL_LAYER_REG_DMA1__MASK				0x00e00000
-#define MDP5_CTL_LAYER_REG_DMA1__SHIFT				21
-static inline uint32_t MDP5_CTL_LAYER_REG_DMA1(uint32_t val)
-{
-	return ((val) << MDP5_CTL_LAYER_REG_DMA1__SHIFT) & MDP5_CTL_LAYER_REG_DMA1__MASK;
-}
-#define MDP5_CTL_LAYER_REG_BORDER_COLOR				0x01000000
-#define MDP5_CTL_LAYER_REG_CURSOR_OUT				0x02000000
-#define MDP5_CTL_LAYER_REG_VIG3__MASK				0x1c000000
-#define MDP5_CTL_LAYER_REG_VIG3__SHIFT				26
-static inline uint32_t MDP5_CTL_LAYER_REG_VIG3(uint32_t val)
-{
-	return ((val) << MDP5_CTL_LAYER_REG_VIG3__SHIFT) & MDP5_CTL_LAYER_REG_VIG3__MASK;
-}
-#define MDP5_CTL_LAYER_REG_RGB3__MASK				0xe0000000
-#define MDP5_CTL_LAYER_REG_RGB3__SHIFT				29
-static inline uint32_t MDP5_CTL_LAYER_REG_RGB3(uint32_t val)
-{
-	return ((val) << MDP5_CTL_LAYER_REG_RGB3__SHIFT) & MDP5_CTL_LAYER_REG_RGB3__MASK;
-}
-
-static inline uint32_t REG_MDP5_CTL_OP(uint32_t i0) { return 0x00000014 + __offset_CTL(i0); }
-#define MDP5_CTL_OP_MODE__MASK					0x0000000f
-#define MDP5_CTL_OP_MODE__SHIFT					0
-static inline uint32_t MDP5_CTL_OP_MODE(enum mdp5_ctl_mode val)
-{
-	return ((val) << MDP5_CTL_OP_MODE__SHIFT) & MDP5_CTL_OP_MODE__MASK;
-}
-#define MDP5_CTL_OP_INTF_NUM__MASK				0x00000070
-#define MDP5_CTL_OP_INTF_NUM__SHIFT				4
-static inline uint32_t MDP5_CTL_OP_INTF_NUM(enum mdp5_intfnum val)
-{
-	return ((val) << MDP5_CTL_OP_INTF_NUM__SHIFT) & MDP5_CTL_OP_INTF_NUM__MASK;
-}
-#define MDP5_CTL_OP_CMD_MODE					0x00020000
-#define MDP5_CTL_OP_PACK_3D_ENABLE				0x00080000
-#define MDP5_CTL_OP_PACK_3D__MASK				0x00300000
-#define MDP5_CTL_OP_PACK_3D__SHIFT				20
-static inline uint32_t MDP5_CTL_OP_PACK_3D(enum mdp5_pack_3d val)
-{
-	return ((val) << MDP5_CTL_OP_PACK_3D__SHIFT) & MDP5_CTL_OP_PACK_3D__MASK;
-}
-
-static inline uint32_t REG_MDP5_CTL_FLUSH(uint32_t i0) { return 0x00000018 + __offset_CTL(i0); }
-#define MDP5_CTL_FLUSH_VIG0					0x00000001
-#define MDP5_CTL_FLUSH_VIG1					0x00000002
-#define MDP5_CTL_FLUSH_VIG2					0x00000004
-#define MDP5_CTL_FLUSH_RGB0					0x00000008
-#define MDP5_CTL_FLUSH_RGB1					0x00000010
-#define MDP5_CTL_FLUSH_RGB2					0x00000020
-#define MDP5_CTL_FLUSH_LM0					0x00000040
-#define MDP5_CTL_FLUSH_LM1					0x00000080
-#define MDP5_CTL_FLUSH_LM2					0x00000100
-#define MDP5_CTL_FLUSH_LM3					0x00000200
-#define MDP5_CTL_FLUSH_LM4					0x00000400
-#define MDP5_CTL_FLUSH_DMA0					0x00000800
-#define MDP5_CTL_FLUSH_DMA1					0x00001000
-#define MDP5_CTL_FLUSH_DSPP0					0x00002000
-#define MDP5_CTL_FLUSH_DSPP1					0x00004000
-#define MDP5_CTL_FLUSH_DSPP2					0x00008000
-#define MDP5_CTL_FLUSH_WB					0x00010000
-#define MDP5_CTL_FLUSH_CTL					0x00020000
-#define MDP5_CTL_FLUSH_VIG3					0x00040000
-#define MDP5_CTL_FLUSH_RGB3					0x00080000
-#define MDP5_CTL_FLUSH_LM5					0x00100000
-#define MDP5_CTL_FLUSH_DSPP3					0x00200000
-#define MDP5_CTL_FLUSH_CURSOR_0					0x00400000
-#define MDP5_CTL_FLUSH_CURSOR_1					0x00800000
-#define MDP5_CTL_FLUSH_CHROMADOWN_0				0x04000000
-#define MDP5_CTL_FLUSH_TIMING_3					0x10000000
-#define MDP5_CTL_FLUSH_TIMING_2					0x20000000
-#define MDP5_CTL_FLUSH_TIMING_1					0x40000000
-#define MDP5_CTL_FLUSH_TIMING_0					0x80000000
-
-static inline uint32_t REG_MDP5_CTL_START(uint32_t i0) { return 0x0000001c + __offset_CTL(i0); }
-
-static inline uint32_t REG_MDP5_CTL_PACK_3D(uint32_t i0) { return 0x00000020 + __offset_CTL(i0); }
-
-static inline uint32_t __offset_LAYER_EXT(uint32_t idx)
-{
-	switch (idx) {
-		case 0: return 0x00000040;
-		case 1: return 0x00000044;
-		case 2: return 0x00000048;
-		case 3: return 0x0000004c;
-		case 4: return 0x00000050;
-		case 5: return 0x00000054;
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_CTL_LAYER_EXT(uint32_t i0, uint32_t i1) { return 0x00000000 + __offset_CTL(i0) + __offset_LAYER_EXT(i1); }
-
-static inline uint32_t REG_MDP5_CTL_LAYER_EXT_REG(uint32_t i0, uint32_t i1) { return 0x00000000 + __offset_CTL(i0) + __offset_LAYER_EXT(i1); }
-#define MDP5_CTL_LAYER_EXT_REG_VIG0_BIT3			0x00000001
-#define MDP5_CTL_LAYER_EXT_REG_VIG1_BIT3			0x00000004
-#define MDP5_CTL_LAYER_EXT_REG_VIG2_BIT3			0x00000010
-#define MDP5_CTL_LAYER_EXT_REG_VIG3_BIT3			0x00000040
-#define MDP5_CTL_LAYER_EXT_REG_RGB0_BIT3			0x00000100
-#define MDP5_CTL_LAYER_EXT_REG_RGB1_BIT3			0x00000400
-#define MDP5_CTL_LAYER_EXT_REG_RGB2_BIT3			0x00001000
-#define MDP5_CTL_LAYER_EXT_REG_RGB3_BIT3			0x00004000
-#define MDP5_CTL_LAYER_EXT_REG_DMA0_BIT3			0x00010000
-#define MDP5_CTL_LAYER_EXT_REG_DMA1_BIT3			0x00040000
-#define MDP5_CTL_LAYER_EXT_REG_CURSOR0__MASK			0x00f00000
-#define MDP5_CTL_LAYER_EXT_REG_CURSOR0__SHIFT			20
-static inline uint32_t MDP5_CTL_LAYER_EXT_REG_CURSOR0(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP5_CTL_LAYER_EXT_REG_CURSOR0__SHIFT) & MDP5_CTL_LAYER_EXT_REG_CURSOR0__MASK;
-}
-#define MDP5_CTL_LAYER_EXT_REG_CURSOR1__MASK			0x3c000000
-#define MDP5_CTL_LAYER_EXT_REG_CURSOR1__SHIFT			26
-static inline uint32_t MDP5_CTL_LAYER_EXT_REG_CURSOR1(enum mdp_mixer_stage_id val)
-{
-	return ((val) << MDP5_CTL_LAYER_EXT_REG_CURSOR1__SHIFT) & MDP5_CTL_LAYER_EXT_REG_CURSOR1__MASK;
-}
-
-static inline uint32_t __offset_PIPE(enum mdp5_pipe idx)
-{
-	switch (idx) {
-		case SSPP_NONE: return (INVALID_IDX(idx));
-		case SSPP_VIG0: return (mdp5_cfg->pipe_vig.base[0]);
-		case SSPP_VIG1: return (mdp5_cfg->pipe_vig.base[1]);
-		case SSPP_VIG2: return (mdp5_cfg->pipe_vig.base[2]);
-		case SSPP_RGB0: return (mdp5_cfg->pipe_rgb.base[0]);
-		case SSPP_RGB1: return (mdp5_cfg->pipe_rgb.base[1]);
-		case SSPP_RGB2: return (mdp5_cfg->pipe_rgb.base[2]);
-		case SSPP_DMA0: return (mdp5_cfg->pipe_dma.base[0]);
-		case SSPP_DMA1: return (mdp5_cfg->pipe_dma.base[1]);
-		case SSPP_VIG3: return (mdp5_cfg->pipe_vig.base[3]);
-		case SSPP_RGB3: return (mdp5_cfg->pipe_rgb.base[3]);
-		case SSPP_CURSOR0: return (mdp5_cfg->pipe_cursor.base[0]);
-		case SSPP_CURSOR1: return (mdp5_cfg->pipe_cursor.base[1]);
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_PIPE(enum mdp5_pipe i0) { return 0x00000000 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_OP_MODE(enum mdp5_pipe i0) { return 0x00000200 + __offset_PIPE(i0); }
-#define MDP5_PIPE_OP_MODE_CSC_DST_DATA_FORMAT__MASK		0x00080000
-#define MDP5_PIPE_OP_MODE_CSC_DST_DATA_FORMAT__SHIFT		19
-static inline uint32_t MDP5_PIPE_OP_MODE_CSC_DST_DATA_FORMAT(enum mdp5_data_format val)
-{
-	return ((val) << MDP5_PIPE_OP_MODE_CSC_DST_DATA_FORMAT__SHIFT) & MDP5_PIPE_OP_MODE_CSC_DST_DATA_FORMAT__MASK;
-}
-#define MDP5_PIPE_OP_MODE_CSC_SRC_DATA_FORMAT__MASK		0x00040000
-#define MDP5_PIPE_OP_MODE_CSC_SRC_DATA_FORMAT__SHIFT		18
-static inline uint32_t MDP5_PIPE_OP_MODE_CSC_SRC_DATA_FORMAT(enum mdp5_data_format val)
-{
-	return ((val) << MDP5_PIPE_OP_MODE_CSC_SRC_DATA_FORMAT__SHIFT) & MDP5_PIPE_OP_MODE_CSC_SRC_DATA_FORMAT__MASK;
-}
-#define MDP5_PIPE_OP_MODE_CSC_1_EN				0x00020000
-
-static inline uint32_t REG_MDP5_PIPE_HIST_CTL_BASE(enum mdp5_pipe i0) { return 0x000002c4 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_HIST_LUT_BASE(enum mdp5_pipe i0) { return 0x000002f0 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_HIST_LUT_SWAP(enum mdp5_pipe i0) { return 0x00000300 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_MATRIX_COEFF_0(enum mdp5_pipe i0) { return 0x00000320 + __offset_PIPE(i0); }
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_0_COEFF_11__MASK		0x00001fff
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_0_COEFF_11__SHIFT		0
-static inline uint32_t MDP5_PIPE_CSC_1_MATRIX_COEFF_0_COEFF_11(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_MATRIX_COEFF_0_COEFF_11__SHIFT) & MDP5_PIPE_CSC_1_MATRIX_COEFF_0_COEFF_11__MASK;
-}
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_0_COEFF_12__MASK		0x1fff0000
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_0_COEFF_12__SHIFT		16
-static inline uint32_t MDP5_PIPE_CSC_1_MATRIX_COEFF_0_COEFF_12(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_MATRIX_COEFF_0_COEFF_12__SHIFT) & MDP5_PIPE_CSC_1_MATRIX_COEFF_0_COEFF_12__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_MATRIX_COEFF_1(enum mdp5_pipe i0) { return 0x00000324 + __offset_PIPE(i0); }
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_1_COEFF_13__MASK		0x00001fff
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_1_COEFF_13__SHIFT		0
-static inline uint32_t MDP5_PIPE_CSC_1_MATRIX_COEFF_1_COEFF_13(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_MATRIX_COEFF_1_COEFF_13__SHIFT) & MDP5_PIPE_CSC_1_MATRIX_COEFF_1_COEFF_13__MASK;
-}
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_1_COEFF_21__MASK		0x1fff0000
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_1_COEFF_21__SHIFT		16
-static inline uint32_t MDP5_PIPE_CSC_1_MATRIX_COEFF_1_COEFF_21(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_MATRIX_COEFF_1_COEFF_21__SHIFT) & MDP5_PIPE_CSC_1_MATRIX_COEFF_1_COEFF_21__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_MATRIX_COEFF_2(enum mdp5_pipe i0) { return 0x00000328 + __offset_PIPE(i0); }
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_2_COEFF_22__MASK		0x00001fff
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_2_COEFF_22__SHIFT		0
-static inline uint32_t MDP5_PIPE_CSC_1_MATRIX_COEFF_2_COEFF_22(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_MATRIX_COEFF_2_COEFF_22__SHIFT) & MDP5_PIPE_CSC_1_MATRIX_COEFF_2_COEFF_22__MASK;
-}
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_2_COEFF_23__MASK		0x1fff0000
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_2_COEFF_23__SHIFT		16
-static inline uint32_t MDP5_PIPE_CSC_1_MATRIX_COEFF_2_COEFF_23(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_MATRIX_COEFF_2_COEFF_23__SHIFT) & MDP5_PIPE_CSC_1_MATRIX_COEFF_2_COEFF_23__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_MATRIX_COEFF_3(enum mdp5_pipe i0) { return 0x0000032c + __offset_PIPE(i0); }
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_3_COEFF_31__MASK		0x00001fff
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_3_COEFF_31__SHIFT		0
-static inline uint32_t MDP5_PIPE_CSC_1_MATRIX_COEFF_3_COEFF_31(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_MATRIX_COEFF_3_COEFF_31__SHIFT) & MDP5_PIPE_CSC_1_MATRIX_COEFF_3_COEFF_31__MASK;
-}
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_3_COEFF_32__MASK		0x1fff0000
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_3_COEFF_32__SHIFT		16
-static inline uint32_t MDP5_PIPE_CSC_1_MATRIX_COEFF_3_COEFF_32(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_MATRIX_COEFF_3_COEFF_32__SHIFT) & MDP5_PIPE_CSC_1_MATRIX_COEFF_3_COEFF_32__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_MATRIX_COEFF_4(enum mdp5_pipe i0) { return 0x00000330 + __offset_PIPE(i0); }
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_4_COEFF_33__MASK		0x00001fff
-#define MDP5_PIPE_CSC_1_MATRIX_COEFF_4_COEFF_33__SHIFT		0
-static inline uint32_t MDP5_PIPE_CSC_1_MATRIX_COEFF_4_COEFF_33(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_MATRIX_COEFF_4_COEFF_33__SHIFT) & MDP5_PIPE_CSC_1_MATRIX_COEFF_4_COEFF_33__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_PRE_CLAMP(enum mdp5_pipe i0, uint32_t i1) { return 0x00000334 + __offset_PIPE(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_PRE_CLAMP_REG(enum mdp5_pipe i0, uint32_t i1) { return 0x00000334 + __offset_PIPE(i0) + 0x4*i1; }
-#define MDP5_PIPE_CSC_1_PRE_CLAMP_REG_HIGH__MASK		0x000000ff
-#define MDP5_PIPE_CSC_1_PRE_CLAMP_REG_HIGH__SHIFT		0
-static inline uint32_t MDP5_PIPE_CSC_1_PRE_CLAMP_REG_HIGH(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_PRE_CLAMP_REG_HIGH__SHIFT) & MDP5_PIPE_CSC_1_PRE_CLAMP_REG_HIGH__MASK;
-}
-#define MDP5_PIPE_CSC_1_PRE_CLAMP_REG_LOW__MASK			0x0000ff00
-#define MDP5_PIPE_CSC_1_PRE_CLAMP_REG_LOW__SHIFT		8
-static inline uint32_t MDP5_PIPE_CSC_1_PRE_CLAMP_REG_LOW(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_PRE_CLAMP_REG_LOW__SHIFT) & MDP5_PIPE_CSC_1_PRE_CLAMP_REG_LOW__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_POST_CLAMP(enum mdp5_pipe i0, uint32_t i1) { return 0x00000340 + __offset_PIPE(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_POST_CLAMP_REG(enum mdp5_pipe i0, uint32_t i1) { return 0x00000340 + __offset_PIPE(i0) + 0x4*i1; }
-#define MDP5_PIPE_CSC_1_POST_CLAMP_REG_HIGH__MASK		0x000000ff
-#define MDP5_PIPE_CSC_1_POST_CLAMP_REG_HIGH__SHIFT		0
-static inline uint32_t MDP5_PIPE_CSC_1_POST_CLAMP_REG_HIGH(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_POST_CLAMP_REG_HIGH__SHIFT) & MDP5_PIPE_CSC_1_POST_CLAMP_REG_HIGH__MASK;
-}
-#define MDP5_PIPE_CSC_1_POST_CLAMP_REG_LOW__MASK		0x0000ff00
-#define MDP5_PIPE_CSC_1_POST_CLAMP_REG_LOW__SHIFT		8
-static inline uint32_t MDP5_PIPE_CSC_1_POST_CLAMP_REG_LOW(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_POST_CLAMP_REG_LOW__SHIFT) & MDP5_PIPE_CSC_1_POST_CLAMP_REG_LOW__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_PRE_BIAS(enum mdp5_pipe i0, uint32_t i1) { return 0x0000034c + __offset_PIPE(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_PRE_BIAS_REG(enum mdp5_pipe i0, uint32_t i1) { return 0x0000034c + __offset_PIPE(i0) + 0x4*i1; }
-#define MDP5_PIPE_CSC_1_PRE_BIAS_REG_VALUE__MASK		0x000001ff
-#define MDP5_PIPE_CSC_1_PRE_BIAS_REG_VALUE__SHIFT		0
-static inline uint32_t MDP5_PIPE_CSC_1_PRE_BIAS_REG_VALUE(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_PRE_BIAS_REG_VALUE__SHIFT) & MDP5_PIPE_CSC_1_PRE_BIAS_REG_VALUE__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_POST_BIAS(enum mdp5_pipe i0, uint32_t i1) { return 0x00000358 + __offset_PIPE(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP5_PIPE_CSC_1_POST_BIAS_REG(enum mdp5_pipe i0, uint32_t i1) { return 0x00000358 + __offset_PIPE(i0) + 0x4*i1; }
-#define MDP5_PIPE_CSC_1_POST_BIAS_REG_VALUE__MASK		0x000001ff
-#define MDP5_PIPE_CSC_1_POST_BIAS_REG_VALUE__SHIFT		0
-static inline uint32_t MDP5_PIPE_CSC_1_POST_BIAS_REG_VALUE(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_CSC_1_POST_BIAS_REG_VALUE__SHIFT) & MDP5_PIPE_CSC_1_POST_BIAS_REG_VALUE__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_SRC_SIZE(enum mdp5_pipe i0) { return 0x00000000 + __offset_PIPE(i0); }
-#define MDP5_PIPE_SRC_SIZE_HEIGHT__MASK				0xffff0000
-#define MDP5_PIPE_SRC_SIZE_HEIGHT__SHIFT			16
-static inline uint32_t MDP5_PIPE_SRC_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_SIZE_HEIGHT__SHIFT) & MDP5_PIPE_SRC_SIZE_HEIGHT__MASK;
-}
-#define MDP5_PIPE_SRC_SIZE_WIDTH__MASK				0x0000ffff
-#define MDP5_PIPE_SRC_SIZE_WIDTH__SHIFT				0
-static inline uint32_t MDP5_PIPE_SRC_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_SIZE_WIDTH__SHIFT) & MDP5_PIPE_SRC_SIZE_WIDTH__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_SRC_IMG_SIZE(enum mdp5_pipe i0) { return 0x00000004 + __offset_PIPE(i0); }
-#define MDP5_PIPE_SRC_IMG_SIZE_HEIGHT__MASK			0xffff0000
-#define MDP5_PIPE_SRC_IMG_SIZE_HEIGHT__SHIFT			16
-static inline uint32_t MDP5_PIPE_SRC_IMG_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_IMG_SIZE_HEIGHT__SHIFT) & MDP5_PIPE_SRC_IMG_SIZE_HEIGHT__MASK;
-}
-#define MDP5_PIPE_SRC_IMG_SIZE_WIDTH__MASK			0x0000ffff
-#define MDP5_PIPE_SRC_IMG_SIZE_WIDTH__SHIFT			0
-static inline uint32_t MDP5_PIPE_SRC_IMG_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_IMG_SIZE_WIDTH__SHIFT) & MDP5_PIPE_SRC_IMG_SIZE_WIDTH__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_SRC_XY(enum mdp5_pipe i0) { return 0x00000008 + __offset_PIPE(i0); }
-#define MDP5_PIPE_SRC_XY_Y__MASK				0xffff0000
-#define MDP5_PIPE_SRC_XY_Y__SHIFT				16
-static inline uint32_t MDP5_PIPE_SRC_XY_Y(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_XY_Y__SHIFT) & MDP5_PIPE_SRC_XY_Y__MASK;
-}
-#define MDP5_PIPE_SRC_XY_X__MASK				0x0000ffff
-#define MDP5_PIPE_SRC_XY_X__SHIFT				0
-static inline uint32_t MDP5_PIPE_SRC_XY_X(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_XY_X__SHIFT) & MDP5_PIPE_SRC_XY_X__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_OUT_SIZE(enum mdp5_pipe i0) { return 0x0000000c + __offset_PIPE(i0); }
-#define MDP5_PIPE_OUT_SIZE_HEIGHT__MASK				0xffff0000
-#define MDP5_PIPE_OUT_SIZE_HEIGHT__SHIFT			16
-static inline uint32_t MDP5_PIPE_OUT_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_OUT_SIZE_HEIGHT__SHIFT) & MDP5_PIPE_OUT_SIZE_HEIGHT__MASK;
-}
-#define MDP5_PIPE_OUT_SIZE_WIDTH__MASK				0x0000ffff
-#define MDP5_PIPE_OUT_SIZE_WIDTH__SHIFT				0
-static inline uint32_t MDP5_PIPE_OUT_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_OUT_SIZE_WIDTH__SHIFT) & MDP5_PIPE_OUT_SIZE_WIDTH__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_OUT_XY(enum mdp5_pipe i0) { return 0x00000010 + __offset_PIPE(i0); }
-#define MDP5_PIPE_OUT_XY_Y__MASK				0xffff0000
-#define MDP5_PIPE_OUT_XY_Y__SHIFT				16
-static inline uint32_t MDP5_PIPE_OUT_XY_Y(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_OUT_XY_Y__SHIFT) & MDP5_PIPE_OUT_XY_Y__MASK;
-}
-#define MDP5_PIPE_OUT_XY_X__MASK				0x0000ffff
-#define MDP5_PIPE_OUT_XY_X__SHIFT				0
-static inline uint32_t MDP5_PIPE_OUT_XY_X(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_OUT_XY_X__SHIFT) & MDP5_PIPE_OUT_XY_X__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_SRC0_ADDR(enum mdp5_pipe i0) { return 0x00000014 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_SRC1_ADDR(enum mdp5_pipe i0) { return 0x00000018 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_SRC2_ADDR(enum mdp5_pipe i0) { return 0x0000001c + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_SRC3_ADDR(enum mdp5_pipe i0) { return 0x00000020 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_SRC_STRIDE_A(enum mdp5_pipe i0) { return 0x00000024 + __offset_PIPE(i0); }
-#define MDP5_PIPE_SRC_STRIDE_A_P0__MASK				0x0000ffff
-#define MDP5_PIPE_SRC_STRIDE_A_P0__SHIFT			0
-static inline uint32_t MDP5_PIPE_SRC_STRIDE_A_P0(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_STRIDE_A_P0__SHIFT) & MDP5_PIPE_SRC_STRIDE_A_P0__MASK;
-}
-#define MDP5_PIPE_SRC_STRIDE_A_P1__MASK				0xffff0000
-#define MDP5_PIPE_SRC_STRIDE_A_P1__SHIFT			16
-static inline uint32_t MDP5_PIPE_SRC_STRIDE_A_P1(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_STRIDE_A_P1__SHIFT) & MDP5_PIPE_SRC_STRIDE_A_P1__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_SRC_STRIDE_B(enum mdp5_pipe i0) { return 0x00000028 + __offset_PIPE(i0); }
-#define MDP5_PIPE_SRC_STRIDE_B_P2__MASK				0x0000ffff
-#define MDP5_PIPE_SRC_STRIDE_B_P2__SHIFT			0
-static inline uint32_t MDP5_PIPE_SRC_STRIDE_B_P2(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_STRIDE_B_P2__SHIFT) & MDP5_PIPE_SRC_STRIDE_B_P2__MASK;
-}
-#define MDP5_PIPE_SRC_STRIDE_B_P3__MASK				0xffff0000
-#define MDP5_PIPE_SRC_STRIDE_B_P3__SHIFT			16
-static inline uint32_t MDP5_PIPE_SRC_STRIDE_B_P3(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_STRIDE_B_P3__SHIFT) & MDP5_PIPE_SRC_STRIDE_B_P3__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_STILE_FRAME_SIZE(enum mdp5_pipe i0) { return 0x0000002c + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_SRC_FORMAT(enum mdp5_pipe i0) { return 0x00000030 + __offset_PIPE(i0); }
-#define MDP5_PIPE_SRC_FORMAT_G_BPC__MASK			0x00000003
-#define MDP5_PIPE_SRC_FORMAT_G_BPC__SHIFT			0
-static inline uint32_t MDP5_PIPE_SRC_FORMAT_G_BPC(enum mdp_bpc val)
-{
-	return ((val) << MDP5_PIPE_SRC_FORMAT_G_BPC__SHIFT) & MDP5_PIPE_SRC_FORMAT_G_BPC__MASK;
-}
-#define MDP5_PIPE_SRC_FORMAT_B_BPC__MASK			0x0000000c
-#define MDP5_PIPE_SRC_FORMAT_B_BPC__SHIFT			2
-static inline uint32_t MDP5_PIPE_SRC_FORMAT_B_BPC(enum mdp_bpc val)
-{
-	return ((val) << MDP5_PIPE_SRC_FORMAT_B_BPC__SHIFT) & MDP5_PIPE_SRC_FORMAT_B_BPC__MASK;
-}
-#define MDP5_PIPE_SRC_FORMAT_R_BPC__MASK			0x00000030
-#define MDP5_PIPE_SRC_FORMAT_R_BPC__SHIFT			4
-static inline uint32_t MDP5_PIPE_SRC_FORMAT_R_BPC(enum mdp_bpc val)
-{
-	return ((val) << MDP5_PIPE_SRC_FORMAT_R_BPC__SHIFT) & MDP5_PIPE_SRC_FORMAT_R_BPC__MASK;
-}
-#define MDP5_PIPE_SRC_FORMAT_A_BPC__MASK			0x000000c0
-#define MDP5_PIPE_SRC_FORMAT_A_BPC__SHIFT			6
-static inline uint32_t MDP5_PIPE_SRC_FORMAT_A_BPC(enum mdp_bpc_alpha val)
-{
-	return ((val) << MDP5_PIPE_SRC_FORMAT_A_BPC__SHIFT) & MDP5_PIPE_SRC_FORMAT_A_BPC__MASK;
-}
-#define MDP5_PIPE_SRC_FORMAT_ALPHA_ENABLE			0x00000100
-#define MDP5_PIPE_SRC_FORMAT_CPP__MASK				0x00000600
-#define MDP5_PIPE_SRC_FORMAT_CPP__SHIFT				9
-static inline uint32_t MDP5_PIPE_SRC_FORMAT_CPP(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_FORMAT_CPP__SHIFT) & MDP5_PIPE_SRC_FORMAT_CPP__MASK;
-}
-#define MDP5_PIPE_SRC_FORMAT_ROT90				0x00000800
-#define MDP5_PIPE_SRC_FORMAT_UNPACK_COUNT__MASK			0x00003000
-#define MDP5_PIPE_SRC_FORMAT_UNPACK_COUNT__SHIFT		12
-static inline uint32_t MDP5_PIPE_SRC_FORMAT_UNPACK_COUNT(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_FORMAT_UNPACK_COUNT__SHIFT) & MDP5_PIPE_SRC_FORMAT_UNPACK_COUNT__MASK;
-}
-#define MDP5_PIPE_SRC_FORMAT_UNPACK_TIGHT			0x00020000
-#define MDP5_PIPE_SRC_FORMAT_UNPACK_ALIGN_MSB			0x00040000
-#define MDP5_PIPE_SRC_FORMAT_FETCH_TYPE__MASK			0x00180000
-#define MDP5_PIPE_SRC_FORMAT_FETCH_TYPE__SHIFT			19
-static inline uint32_t MDP5_PIPE_SRC_FORMAT_FETCH_TYPE(enum mdp_fetch_type val)
-{
-	return ((val) << MDP5_PIPE_SRC_FORMAT_FETCH_TYPE__SHIFT) & MDP5_PIPE_SRC_FORMAT_FETCH_TYPE__MASK;
-}
-#define MDP5_PIPE_SRC_FORMAT_CHROMA_SAMP__MASK			0x01800000
-#define MDP5_PIPE_SRC_FORMAT_CHROMA_SAMP__SHIFT			23
-static inline uint32_t MDP5_PIPE_SRC_FORMAT_CHROMA_SAMP(enum mdp_chroma_samp_type val)
-{
-	return ((val) << MDP5_PIPE_SRC_FORMAT_CHROMA_SAMP__SHIFT) & MDP5_PIPE_SRC_FORMAT_CHROMA_SAMP__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_SRC_UNPACK(enum mdp5_pipe i0) { return 0x00000034 + __offset_PIPE(i0); }
-#define MDP5_PIPE_SRC_UNPACK_ELEM0__MASK			0x000000ff
-#define MDP5_PIPE_SRC_UNPACK_ELEM0__SHIFT			0
-static inline uint32_t MDP5_PIPE_SRC_UNPACK_ELEM0(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_UNPACK_ELEM0__SHIFT) & MDP5_PIPE_SRC_UNPACK_ELEM0__MASK;
-}
-#define MDP5_PIPE_SRC_UNPACK_ELEM1__MASK			0x0000ff00
-#define MDP5_PIPE_SRC_UNPACK_ELEM1__SHIFT			8
-static inline uint32_t MDP5_PIPE_SRC_UNPACK_ELEM1(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_UNPACK_ELEM1__SHIFT) & MDP5_PIPE_SRC_UNPACK_ELEM1__MASK;
-}
-#define MDP5_PIPE_SRC_UNPACK_ELEM2__MASK			0x00ff0000
-#define MDP5_PIPE_SRC_UNPACK_ELEM2__SHIFT			16
-static inline uint32_t MDP5_PIPE_SRC_UNPACK_ELEM2(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_UNPACK_ELEM2__SHIFT) & MDP5_PIPE_SRC_UNPACK_ELEM2__MASK;
-}
-#define MDP5_PIPE_SRC_UNPACK_ELEM3__MASK			0xff000000
-#define MDP5_PIPE_SRC_UNPACK_ELEM3__SHIFT			24
-static inline uint32_t MDP5_PIPE_SRC_UNPACK_ELEM3(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SRC_UNPACK_ELEM3__SHIFT) & MDP5_PIPE_SRC_UNPACK_ELEM3__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_SRC_OP_MODE(enum mdp5_pipe i0) { return 0x00000038 + __offset_PIPE(i0); }
-#define MDP5_PIPE_SRC_OP_MODE_BWC_EN				0x00000001
-#define MDP5_PIPE_SRC_OP_MODE_BWC__MASK				0x00000006
-#define MDP5_PIPE_SRC_OP_MODE_BWC__SHIFT			1
-static inline uint32_t MDP5_PIPE_SRC_OP_MODE_BWC(enum mdp5_pipe_bwc val)
-{
-	return ((val) << MDP5_PIPE_SRC_OP_MODE_BWC__SHIFT) & MDP5_PIPE_SRC_OP_MODE_BWC__MASK;
-}
-#define MDP5_PIPE_SRC_OP_MODE_FLIP_LR				0x00002000
-#define MDP5_PIPE_SRC_OP_MODE_FLIP_UD				0x00004000
-#define MDP5_PIPE_SRC_OP_MODE_IGC_EN				0x00010000
-#define MDP5_PIPE_SRC_OP_MODE_IGC_ROM_0				0x00020000
-#define MDP5_PIPE_SRC_OP_MODE_IGC_ROM_1				0x00040000
-#define MDP5_PIPE_SRC_OP_MODE_DEINTERLACE			0x00400000
-#define MDP5_PIPE_SRC_OP_MODE_DEINTERLACE_ODD			0x00800000
-#define MDP5_PIPE_SRC_OP_MODE_SW_PIX_EXT_OVERRIDE		0x80000000
-
-static inline uint32_t REG_MDP5_PIPE_SRC_CONSTANT_COLOR(enum mdp5_pipe i0) { return 0x0000003c + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_FETCH_CONFIG(enum mdp5_pipe i0) { return 0x00000048 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_VC1_RANGE(enum mdp5_pipe i0) { return 0x0000004c + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_REQPRIO_FIFO_WM_0(enum mdp5_pipe i0) { return 0x00000050 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_REQPRIO_FIFO_WM_1(enum mdp5_pipe i0) { return 0x00000054 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_REQPRIO_FIFO_WM_2(enum mdp5_pipe i0) { return 0x00000058 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_SRC_ADDR_SW_STATUS(enum mdp5_pipe i0) { return 0x00000070 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_CURRENT_SRC0_ADDR(enum mdp5_pipe i0) { return 0x000000a4 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_CURRENT_SRC1_ADDR(enum mdp5_pipe i0) { return 0x000000a8 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_CURRENT_SRC2_ADDR(enum mdp5_pipe i0) { return 0x000000ac + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_CURRENT_SRC3_ADDR(enum mdp5_pipe i0) { return 0x000000b0 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_DECIMATION(enum mdp5_pipe i0) { return 0x000000b4 + __offset_PIPE(i0); }
-#define MDP5_PIPE_DECIMATION_VERT__MASK				0x000000ff
-#define MDP5_PIPE_DECIMATION_VERT__SHIFT			0
-static inline uint32_t MDP5_PIPE_DECIMATION_VERT(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_DECIMATION_VERT__SHIFT) & MDP5_PIPE_DECIMATION_VERT__MASK;
-}
-#define MDP5_PIPE_DECIMATION_HORZ__MASK				0x0000ff00
-#define MDP5_PIPE_DECIMATION_HORZ__SHIFT			8
-static inline uint32_t MDP5_PIPE_DECIMATION_HORZ(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_DECIMATION_HORZ__SHIFT) & MDP5_PIPE_DECIMATION_HORZ__MASK;
-}
-
-static inline uint32_t __offset_SW_PIX_EXT(enum mdp_component_type idx)
-{
-	switch (idx) {
-		case COMP_0: return 0x00000100;
-		case COMP_1_2: return 0x00000110;
-		case COMP_3: return 0x00000120;
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_PIPE_SW_PIX_EXT(enum mdp5_pipe i0, enum mdp_component_type i1) { return 0x00000000 + __offset_PIPE(i0) + __offset_SW_PIX_EXT(i1); }
-
-static inline uint32_t REG_MDP5_PIPE_SW_PIX_EXT_LR(enum mdp5_pipe i0, enum mdp_component_type i1) { return 0x00000000 + __offset_PIPE(i0) + __offset_SW_PIX_EXT(i1); }
-#define MDP5_PIPE_SW_PIX_EXT_LR_LEFT_RPT__MASK			0x000000ff
-#define MDP5_PIPE_SW_PIX_EXT_LR_LEFT_RPT__SHIFT			0
-static inline uint32_t MDP5_PIPE_SW_PIX_EXT_LR_LEFT_RPT(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SW_PIX_EXT_LR_LEFT_RPT__SHIFT) & MDP5_PIPE_SW_PIX_EXT_LR_LEFT_RPT__MASK;
-}
-#define MDP5_PIPE_SW_PIX_EXT_LR_LEFT_OVF__MASK			0x0000ff00
-#define MDP5_PIPE_SW_PIX_EXT_LR_LEFT_OVF__SHIFT			8
-static inline uint32_t MDP5_PIPE_SW_PIX_EXT_LR_LEFT_OVF(int32_t val)
-{
-	return ((val) << MDP5_PIPE_SW_PIX_EXT_LR_LEFT_OVF__SHIFT) & MDP5_PIPE_SW_PIX_EXT_LR_LEFT_OVF__MASK;
-}
-#define MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_RPT__MASK			0x00ff0000
-#define MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_RPT__SHIFT		16
-static inline uint32_t MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_RPT(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_RPT__SHIFT) & MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_RPT__MASK;
-}
-#define MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_OVF__MASK			0xff000000
-#define MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_OVF__SHIFT		24
-static inline uint32_t MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_OVF(int32_t val)
-{
-	return ((val) << MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_OVF__SHIFT) & MDP5_PIPE_SW_PIX_EXT_LR_RIGHT_OVF__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_SW_PIX_EXT_TB(enum mdp5_pipe i0, enum mdp_component_type i1) { return 0x00000004 + __offset_PIPE(i0) + __offset_SW_PIX_EXT(i1); }
-#define MDP5_PIPE_SW_PIX_EXT_TB_TOP_RPT__MASK			0x000000ff
-#define MDP5_PIPE_SW_PIX_EXT_TB_TOP_RPT__SHIFT			0
-static inline uint32_t MDP5_PIPE_SW_PIX_EXT_TB_TOP_RPT(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SW_PIX_EXT_TB_TOP_RPT__SHIFT) & MDP5_PIPE_SW_PIX_EXT_TB_TOP_RPT__MASK;
-}
-#define MDP5_PIPE_SW_PIX_EXT_TB_TOP_OVF__MASK			0x0000ff00
-#define MDP5_PIPE_SW_PIX_EXT_TB_TOP_OVF__SHIFT			8
-static inline uint32_t MDP5_PIPE_SW_PIX_EXT_TB_TOP_OVF(int32_t val)
-{
-	return ((val) << MDP5_PIPE_SW_PIX_EXT_TB_TOP_OVF__SHIFT) & MDP5_PIPE_SW_PIX_EXT_TB_TOP_OVF__MASK;
-}
-#define MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_RPT__MASK		0x00ff0000
-#define MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_RPT__SHIFT		16
-static inline uint32_t MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_RPT(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_RPT__SHIFT) & MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_RPT__MASK;
-}
-#define MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_OVF__MASK		0xff000000
-#define MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_OVF__SHIFT		24
-static inline uint32_t MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_OVF(int32_t val)
-{
-	return ((val) << MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_OVF__SHIFT) & MDP5_PIPE_SW_PIX_EXT_TB_BOTTOM_OVF__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS(enum mdp5_pipe i0, enum mdp_component_type i1) { return 0x00000008 + __offset_PIPE(i0) + __offset_SW_PIX_EXT(i1); }
-#define MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_LEFT_RIGHT__MASK	0x0000ffff
-#define MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_LEFT_RIGHT__SHIFT	0
-static inline uint32_t MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_LEFT_RIGHT(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_LEFT_RIGHT__SHIFT) & MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_LEFT_RIGHT__MASK;
-}
-#define MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_TOP_BOTTOM__MASK	0xffff0000
-#define MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_TOP_BOTTOM__SHIFT	16
-static inline uint32_t MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_TOP_BOTTOM(uint32_t val)
-{
-	return ((val) << MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_TOP_BOTTOM__SHIFT) & MDP5_PIPE_SW_PIX_EXT_REQ_PIXELS_TOP_BOTTOM__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_SCALE_CONFIG(enum mdp5_pipe i0) { return 0x00000204 + __offset_PIPE(i0); }
-#define MDP5_PIPE_SCALE_CONFIG_SCALEX_EN			0x00000001
-#define MDP5_PIPE_SCALE_CONFIG_SCALEY_EN			0x00000002
-#define MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_0__MASK	0x00000300
-#define MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_0__SHIFT	8
-static inline uint32_t MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_0(enum mdp5_scale_filter val)
-{
-	return ((val) << MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_0__SHIFT) & MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_0__MASK;
-}
-#define MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_0__MASK	0x00000c00
-#define MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_0__SHIFT	10
-static inline uint32_t MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_0(enum mdp5_scale_filter val)
-{
-	return ((val) << MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_0__SHIFT) & MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_0__MASK;
-}
-#define MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_1_2__MASK	0x00003000
-#define MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_1_2__SHIFT	12
-static inline uint32_t MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_1_2(enum mdp5_scale_filter val)
-{
-	return ((val) << MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_1_2__SHIFT) & MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_1_2__MASK;
-}
-#define MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_1_2__MASK	0x0000c000
-#define MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_1_2__SHIFT	14
-static inline uint32_t MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_1_2(enum mdp5_scale_filter val)
-{
-	return ((val) << MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_1_2__SHIFT) & MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_1_2__MASK;
-}
-#define MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_3__MASK	0x00030000
-#define MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_3__SHIFT	16
-static inline uint32_t MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_3(enum mdp5_scale_filter val)
-{
-	return ((val) << MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_3__SHIFT) & MDP5_PIPE_SCALE_CONFIG_SCALEX_FILTER_COMP_3__MASK;
-}
-#define MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_3__MASK	0x000c0000
-#define MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_3__SHIFT	18
-static inline uint32_t MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_3(enum mdp5_scale_filter val)
-{
-	return ((val) << MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_3__SHIFT) & MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_3__MASK;
-}
-
-static inline uint32_t REG_MDP5_PIPE_SCALE_PHASE_STEP_X(enum mdp5_pipe i0) { return 0x00000210 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_SCALE_PHASE_STEP_Y(enum mdp5_pipe i0) { return 0x00000214 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_SCALE_CR_PHASE_STEP_X(enum mdp5_pipe i0) { return 0x00000218 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_SCALE_CR_PHASE_STEP_Y(enum mdp5_pipe i0) { return 0x0000021c + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_SCALE_INIT_PHASE_X(enum mdp5_pipe i0) { return 0x00000220 + __offset_PIPE(i0); }
-
-static inline uint32_t REG_MDP5_PIPE_SCALE_INIT_PHASE_Y(enum mdp5_pipe i0) { return 0x00000224 + __offset_PIPE(i0); }
-
-static inline uint32_t __offset_LM(uint32_t idx)
-{
-	switch (idx) {
-		case 0: return (mdp5_cfg->lm.base[0]);
-		case 1: return (mdp5_cfg->lm.base[1]);
-		case 2: return (mdp5_cfg->lm.base[2]);
-		case 3: return (mdp5_cfg->lm.base[3]);
-		case 4: return (mdp5_cfg->lm.base[4]);
-		case 5: return (mdp5_cfg->lm.base[5]);
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_LM(uint32_t i0) { return 0x00000000 + __offset_LM(i0); }
-
-static inline uint32_t REG_MDP5_LM_BLEND_COLOR_OUT(uint32_t i0) { return 0x00000000 + __offset_LM(i0); }
-#define MDP5_LM_BLEND_COLOR_OUT_STAGE0_FG_ALPHA			0x00000002
-#define MDP5_LM_BLEND_COLOR_OUT_STAGE1_FG_ALPHA			0x00000004
-#define MDP5_LM_BLEND_COLOR_OUT_STAGE2_FG_ALPHA			0x00000008
-#define MDP5_LM_BLEND_COLOR_OUT_STAGE3_FG_ALPHA			0x00000010
-#define MDP5_LM_BLEND_COLOR_OUT_STAGE4_FG_ALPHA			0x00000020
-#define MDP5_LM_BLEND_COLOR_OUT_STAGE5_FG_ALPHA			0x00000040
-#define MDP5_LM_BLEND_COLOR_OUT_STAGE6_FG_ALPHA			0x00000080
-#define MDP5_LM_BLEND_COLOR_OUT_SPLIT_LEFT_RIGHT		0x80000000
-
-static inline uint32_t REG_MDP5_LM_OUT_SIZE(uint32_t i0) { return 0x00000004 + __offset_LM(i0); }
-#define MDP5_LM_OUT_SIZE_HEIGHT__MASK				0xffff0000
-#define MDP5_LM_OUT_SIZE_HEIGHT__SHIFT				16
-static inline uint32_t MDP5_LM_OUT_SIZE_HEIGHT(uint32_t val)
-{
-	return ((val) << MDP5_LM_OUT_SIZE_HEIGHT__SHIFT) & MDP5_LM_OUT_SIZE_HEIGHT__MASK;
-}
-#define MDP5_LM_OUT_SIZE_WIDTH__MASK				0x0000ffff
-#define MDP5_LM_OUT_SIZE_WIDTH__SHIFT				0
-static inline uint32_t MDP5_LM_OUT_SIZE_WIDTH(uint32_t val)
-{
-	return ((val) << MDP5_LM_OUT_SIZE_WIDTH__SHIFT) & MDP5_LM_OUT_SIZE_WIDTH__MASK;
-}
-
-static inline uint32_t REG_MDP5_LM_BORDER_COLOR_0(uint32_t i0) { return 0x00000008 + __offset_LM(i0); }
-
-static inline uint32_t REG_MDP5_LM_BORDER_COLOR_1(uint32_t i0) { return 0x00000010 + __offset_LM(i0); }
-
-static inline uint32_t __offset_BLEND(uint32_t idx)
-{
-	switch (idx) {
-		case 0: return 0x00000020;
-		case 1: return 0x00000050;
-		case 2: return 0x00000080;
-		case 3: return 0x000000b0;
-		case 4: return 0x00000230;
-		case 5: return 0x00000260;
-		case 6: return 0x00000290;
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_LM_BLEND(uint32_t i0, uint32_t i1) { return 0x00000000 + __offset_LM(i0) + __offset_BLEND(i1); }
-
-static inline uint32_t REG_MDP5_LM_BLEND_OP_MODE(uint32_t i0, uint32_t i1) { return 0x00000000 + __offset_LM(i0) + __offset_BLEND(i1); }
-#define MDP5_LM_BLEND_OP_MODE_FG_ALPHA__MASK			0x00000003
-#define MDP5_LM_BLEND_OP_MODE_FG_ALPHA__SHIFT			0
-static inline uint32_t MDP5_LM_BLEND_OP_MODE_FG_ALPHA(enum mdp_alpha_type val)
-{
-	return ((val) << MDP5_LM_BLEND_OP_MODE_FG_ALPHA__SHIFT) & MDP5_LM_BLEND_OP_MODE_FG_ALPHA__MASK;
-}
-#define MDP5_LM_BLEND_OP_MODE_FG_INV_ALPHA			0x00000004
-#define MDP5_LM_BLEND_OP_MODE_FG_MOD_ALPHA			0x00000008
-#define MDP5_LM_BLEND_OP_MODE_FG_INV_MOD_ALPHA			0x00000010
-#define MDP5_LM_BLEND_OP_MODE_FG_TRANSP_EN			0x00000020
-#define MDP5_LM_BLEND_OP_MODE_BG_ALPHA__MASK			0x00000300
-#define MDP5_LM_BLEND_OP_MODE_BG_ALPHA__SHIFT			8
-static inline uint32_t MDP5_LM_BLEND_OP_MODE_BG_ALPHA(enum mdp_alpha_type val)
-{
-	return ((val) << MDP5_LM_BLEND_OP_MODE_BG_ALPHA__SHIFT) & MDP5_LM_BLEND_OP_MODE_BG_ALPHA__MASK;
-}
-#define MDP5_LM_BLEND_OP_MODE_BG_INV_ALPHA			0x00000400
-#define MDP5_LM_BLEND_OP_MODE_BG_MOD_ALPHA			0x00000800
-#define MDP5_LM_BLEND_OP_MODE_BG_INV_MOD_ALPHA			0x00001000
-#define MDP5_LM_BLEND_OP_MODE_BG_TRANSP_EN			0x00002000
-
-static inline uint32_t REG_MDP5_LM_BLEND_FG_ALPHA(uint32_t i0, uint32_t i1) { return 0x00000004 + __offset_LM(i0) + __offset_BLEND(i1); }
-
-static inline uint32_t REG_MDP5_LM_BLEND_BG_ALPHA(uint32_t i0, uint32_t i1) { return 0x00000008 + __offset_LM(i0) + __offset_BLEND(i1); }
-
-static inline uint32_t REG_MDP5_LM_BLEND_FG_TRANSP_LOW0(uint32_t i0, uint32_t i1) { return 0x0000000c + __offset_LM(i0) + __offset_BLEND(i1); }
-
-static inline uint32_t REG_MDP5_LM_BLEND_FG_TRANSP_LOW1(uint32_t i0, uint32_t i1) { return 0x00000010 + __offset_LM(i0) + __offset_BLEND(i1); }
-
-static inline uint32_t REG_MDP5_LM_BLEND_FG_TRANSP_HIGH0(uint32_t i0, uint32_t i1) { return 0x00000014 + __offset_LM(i0) + __offset_BLEND(i1); }
-
-static inline uint32_t REG_MDP5_LM_BLEND_FG_TRANSP_HIGH1(uint32_t i0, uint32_t i1) { return 0x00000018 + __offset_LM(i0) + __offset_BLEND(i1); }
-
-static inline uint32_t REG_MDP5_LM_BLEND_BG_TRANSP_LOW0(uint32_t i0, uint32_t i1) { return 0x0000001c + __offset_LM(i0) + __offset_BLEND(i1); }
-
-static inline uint32_t REG_MDP5_LM_BLEND_BG_TRANSP_LOW1(uint32_t i0, uint32_t i1) { return 0x00000020 + __offset_LM(i0) + __offset_BLEND(i1); }
-
-static inline uint32_t REG_MDP5_LM_BLEND_BG_TRANSP_HIGH0(uint32_t i0, uint32_t i1) { return 0x00000024 + __offset_LM(i0) + __offset_BLEND(i1); }
-
-static inline uint32_t REG_MDP5_LM_BLEND_BG_TRANSP_HIGH1(uint32_t i0, uint32_t i1) { return 0x00000028 + __offset_LM(i0) + __offset_BLEND(i1); }
-
-static inline uint32_t REG_MDP5_LM_CURSOR_IMG_SIZE(uint32_t i0) { return 0x000000e0 + __offset_LM(i0); }
-#define MDP5_LM_CURSOR_IMG_SIZE_SRC_W__MASK			0x0000ffff
-#define MDP5_LM_CURSOR_IMG_SIZE_SRC_W__SHIFT			0
-static inline uint32_t MDP5_LM_CURSOR_IMG_SIZE_SRC_W(uint32_t val)
-{
-	return ((val) << MDP5_LM_CURSOR_IMG_SIZE_SRC_W__SHIFT) & MDP5_LM_CURSOR_IMG_SIZE_SRC_W__MASK;
-}
-#define MDP5_LM_CURSOR_IMG_SIZE_SRC_H__MASK			0xffff0000
-#define MDP5_LM_CURSOR_IMG_SIZE_SRC_H__SHIFT			16
-static inline uint32_t MDP5_LM_CURSOR_IMG_SIZE_SRC_H(uint32_t val)
-{
-	return ((val) << MDP5_LM_CURSOR_IMG_SIZE_SRC_H__SHIFT) & MDP5_LM_CURSOR_IMG_SIZE_SRC_H__MASK;
-}
-
-static inline uint32_t REG_MDP5_LM_CURSOR_SIZE(uint32_t i0) { return 0x000000e4 + __offset_LM(i0); }
-#define MDP5_LM_CURSOR_SIZE_ROI_W__MASK				0x0000ffff
-#define MDP5_LM_CURSOR_SIZE_ROI_W__SHIFT			0
-static inline uint32_t MDP5_LM_CURSOR_SIZE_ROI_W(uint32_t val)
-{
-	return ((val) << MDP5_LM_CURSOR_SIZE_ROI_W__SHIFT) & MDP5_LM_CURSOR_SIZE_ROI_W__MASK;
-}
-#define MDP5_LM_CURSOR_SIZE_ROI_H__MASK				0xffff0000
-#define MDP5_LM_CURSOR_SIZE_ROI_H__SHIFT			16
-static inline uint32_t MDP5_LM_CURSOR_SIZE_ROI_H(uint32_t val)
-{
-	return ((val) << MDP5_LM_CURSOR_SIZE_ROI_H__SHIFT) & MDP5_LM_CURSOR_SIZE_ROI_H__MASK;
-}
-
-static inline uint32_t REG_MDP5_LM_CURSOR_XY(uint32_t i0) { return 0x000000e8 + __offset_LM(i0); }
-#define MDP5_LM_CURSOR_XY_SRC_X__MASK				0x0000ffff
-#define MDP5_LM_CURSOR_XY_SRC_X__SHIFT				0
-static inline uint32_t MDP5_LM_CURSOR_XY_SRC_X(uint32_t val)
-{
-	return ((val) << MDP5_LM_CURSOR_XY_SRC_X__SHIFT) & MDP5_LM_CURSOR_XY_SRC_X__MASK;
-}
-#define MDP5_LM_CURSOR_XY_SRC_Y__MASK				0xffff0000
-#define MDP5_LM_CURSOR_XY_SRC_Y__SHIFT				16
-static inline uint32_t MDP5_LM_CURSOR_XY_SRC_Y(uint32_t val)
-{
-	return ((val) << MDP5_LM_CURSOR_XY_SRC_Y__SHIFT) & MDP5_LM_CURSOR_XY_SRC_Y__MASK;
-}
-
-static inline uint32_t REG_MDP5_LM_CURSOR_STRIDE(uint32_t i0) { return 0x000000dc + __offset_LM(i0); }
-#define MDP5_LM_CURSOR_STRIDE_STRIDE__MASK			0x0000ffff
-#define MDP5_LM_CURSOR_STRIDE_STRIDE__SHIFT			0
-static inline uint32_t MDP5_LM_CURSOR_STRIDE_STRIDE(uint32_t val)
-{
-	return ((val) << MDP5_LM_CURSOR_STRIDE_STRIDE__SHIFT) & MDP5_LM_CURSOR_STRIDE_STRIDE__MASK;
-}
-
-static inline uint32_t REG_MDP5_LM_CURSOR_FORMAT(uint32_t i0) { return 0x000000ec + __offset_LM(i0); }
-#define MDP5_LM_CURSOR_FORMAT_FORMAT__MASK			0x00000007
-#define MDP5_LM_CURSOR_FORMAT_FORMAT__SHIFT			0
-static inline uint32_t MDP5_LM_CURSOR_FORMAT_FORMAT(enum mdp5_cursor_format val)
-{
-	return ((val) << MDP5_LM_CURSOR_FORMAT_FORMAT__SHIFT) & MDP5_LM_CURSOR_FORMAT_FORMAT__MASK;
-}
-
-static inline uint32_t REG_MDP5_LM_CURSOR_BASE_ADDR(uint32_t i0) { return 0x000000f0 + __offset_LM(i0); }
-
-static inline uint32_t REG_MDP5_LM_CURSOR_START_XY(uint32_t i0) { return 0x000000f4 + __offset_LM(i0); }
-#define MDP5_LM_CURSOR_START_XY_X_START__MASK			0x0000ffff
-#define MDP5_LM_CURSOR_START_XY_X_START__SHIFT			0
-static inline uint32_t MDP5_LM_CURSOR_START_XY_X_START(uint32_t val)
-{
-	return ((val) << MDP5_LM_CURSOR_START_XY_X_START__SHIFT) & MDP5_LM_CURSOR_START_XY_X_START__MASK;
-}
-#define MDP5_LM_CURSOR_START_XY_Y_START__MASK			0xffff0000
-#define MDP5_LM_CURSOR_START_XY_Y_START__SHIFT			16
-static inline uint32_t MDP5_LM_CURSOR_START_XY_Y_START(uint32_t val)
-{
-	return ((val) << MDP5_LM_CURSOR_START_XY_Y_START__SHIFT) & MDP5_LM_CURSOR_START_XY_Y_START__MASK;
-}
-
-static inline uint32_t REG_MDP5_LM_CURSOR_BLEND_CONFIG(uint32_t i0) { return 0x000000f8 + __offset_LM(i0); }
-#define MDP5_LM_CURSOR_BLEND_CONFIG_BLEND_EN			0x00000001
-#define MDP5_LM_CURSOR_BLEND_CONFIG_BLEND_ALPHA_SEL__MASK	0x00000006
-#define MDP5_LM_CURSOR_BLEND_CONFIG_BLEND_ALPHA_SEL__SHIFT	1
-static inline uint32_t MDP5_LM_CURSOR_BLEND_CONFIG_BLEND_ALPHA_SEL(enum mdp5_cursor_alpha val)
-{
-	return ((val) << MDP5_LM_CURSOR_BLEND_CONFIG_BLEND_ALPHA_SEL__SHIFT) & MDP5_LM_CURSOR_BLEND_CONFIG_BLEND_ALPHA_SEL__MASK;
-}
-#define MDP5_LM_CURSOR_BLEND_CONFIG_BLEND_TRANSP_EN		0x00000008
-
-static inline uint32_t REG_MDP5_LM_CURSOR_BLEND_PARAM(uint32_t i0) { return 0x000000fc + __offset_LM(i0); }
-
-static inline uint32_t REG_MDP5_LM_CURSOR_BLEND_TRANSP_LOW0(uint32_t i0) { return 0x00000100 + __offset_LM(i0); }
-
-static inline uint32_t REG_MDP5_LM_CURSOR_BLEND_TRANSP_LOW1(uint32_t i0) { return 0x00000104 + __offset_LM(i0); }
-
-static inline uint32_t REG_MDP5_LM_CURSOR_BLEND_TRANSP_HIGH0(uint32_t i0) { return 0x00000108 + __offset_LM(i0); }
-
-static inline uint32_t REG_MDP5_LM_CURSOR_BLEND_TRANSP_HIGH1(uint32_t i0) { return 0x0000010c + __offset_LM(i0); }
-
-static inline uint32_t REG_MDP5_LM_GC_LUT_BASE(uint32_t i0) { return 0x00000110 + __offset_LM(i0); }
-
-static inline uint32_t __offset_DSPP(uint32_t idx)
-{
-	switch (idx) {
-		case 0: return (mdp5_cfg->dspp.base[0]);
-		case 1: return (mdp5_cfg->dspp.base[1]);
-		case 2: return (mdp5_cfg->dspp.base[2]);
-		case 3: return (mdp5_cfg->dspp.base[3]);
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_DSPP(uint32_t i0) { return 0x00000000 + __offset_DSPP(i0); }
-
-static inline uint32_t REG_MDP5_DSPP_OP_MODE(uint32_t i0) { return 0x00000000 + __offset_DSPP(i0); }
-#define MDP5_DSPP_OP_MODE_IGC_LUT_EN				0x00000001
-#define MDP5_DSPP_OP_MODE_IGC_TBL_IDX__MASK			0x0000000e
-#define MDP5_DSPP_OP_MODE_IGC_TBL_IDX__SHIFT			1
-static inline uint32_t MDP5_DSPP_OP_MODE_IGC_TBL_IDX(uint32_t val)
-{
-	return ((val) << MDP5_DSPP_OP_MODE_IGC_TBL_IDX__SHIFT) & MDP5_DSPP_OP_MODE_IGC_TBL_IDX__MASK;
-}
-#define MDP5_DSPP_OP_MODE_PCC_EN				0x00000010
-#define MDP5_DSPP_OP_MODE_DITHER_EN				0x00000100
-#define MDP5_DSPP_OP_MODE_HIST_EN				0x00010000
-#define MDP5_DSPP_OP_MODE_AUTO_CLEAR				0x00020000
-#define MDP5_DSPP_OP_MODE_HIST_LUT_EN				0x00080000
-#define MDP5_DSPP_OP_MODE_PA_EN					0x00100000
-#define MDP5_DSPP_OP_MODE_GAMUT_EN				0x00800000
-#define MDP5_DSPP_OP_MODE_GAMUT_ORDER				0x01000000
-
-static inline uint32_t REG_MDP5_DSPP_PCC_BASE(uint32_t i0) { return 0x00000030 + __offset_DSPP(i0); }
-
-static inline uint32_t REG_MDP5_DSPP_DITHER_DEPTH(uint32_t i0) { return 0x00000150 + __offset_DSPP(i0); }
-
-static inline uint32_t REG_MDP5_DSPP_HIST_CTL_BASE(uint32_t i0) { return 0x00000210 + __offset_DSPP(i0); }
-
-static inline uint32_t REG_MDP5_DSPP_HIST_LUT_BASE(uint32_t i0) { return 0x00000230 + __offset_DSPP(i0); }
-
-static inline uint32_t REG_MDP5_DSPP_HIST_LUT_SWAP(uint32_t i0) { return 0x00000234 + __offset_DSPP(i0); }
-
-static inline uint32_t REG_MDP5_DSPP_PA_BASE(uint32_t i0) { return 0x00000238 + __offset_DSPP(i0); }
-
-static inline uint32_t REG_MDP5_DSPP_GAMUT_BASE(uint32_t i0) { return 0x000002dc + __offset_DSPP(i0); }
-
-static inline uint32_t REG_MDP5_DSPP_GC_BASE(uint32_t i0) { return 0x000002b0 + __offset_DSPP(i0); }
-
-static inline uint32_t __offset_PP(uint32_t idx)
-{
-	switch (idx) {
-		case 0: return (mdp5_cfg->pp.base[0]);
-		case 1: return (mdp5_cfg->pp.base[1]);
-		case 2: return (mdp5_cfg->pp.base[2]);
-		case 3: return (mdp5_cfg->pp.base[3]);
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_PP(uint32_t i0) { return 0x00000000 + __offset_PP(i0); }
-
-static inline uint32_t REG_MDP5_PP_TEAR_CHECK_EN(uint32_t i0) { return 0x00000000 + __offset_PP(i0); }
-
-static inline uint32_t REG_MDP5_PP_SYNC_CONFIG_VSYNC(uint32_t i0) { return 0x00000004 + __offset_PP(i0); }
-#define MDP5_PP_SYNC_CONFIG_VSYNC_COUNT__MASK			0x0007ffff
-#define MDP5_PP_SYNC_CONFIG_VSYNC_COUNT__SHIFT			0
-static inline uint32_t MDP5_PP_SYNC_CONFIG_VSYNC_COUNT(uint32_t val)
-{
-	return ((val) << MDP5_PP_SYNC_CONFIG_VSYNC_COUNT__SHIFT) & MDP5_PP_SYNC_CONFIG_VSYNC_COUNT__MASK;
-}
-#define MDP5_PP_SYNC_CONFIG_VSYNC_COUNTER_EN			0x00080000
-#define MDP5_PP_SYNC_CONFIG_VSYNC_IN_EN				0x00100000
-
-static inline uint32_t REG_MDP5_PP_SYNC_CONFIG_HEIGHT(uint32_t i0) { return 0x00000008 + __offset_PP(i0); }
-
-static inline uint32_t REG_MDP5_PP_SYNC_WRCOUNT(uint32_t i0) { return 0x0000000c + __offset_PP(i0); }
-#define MDP5_PP_SYNC_WRCOUNT_LINE_COUNT__MASK			0x0000ffff
-#define MDP5_PP_SYNC_WRCOUNT_LINE_COUNT__SHIFT			0
-static inline uint32_t MDP5_PP_SYNC_WRCOUNT_LINE_COUNT(uint32_t val)
-{
-	return ((val) << MDP5_PP_SYNC_WRCOUNT_LINE_COUNT__SHIFT) & MDP5_PP_SYNC_WRCOUNT_LINE_COUNT__MASK;
-}
-#define MDP5_PP_SYNC_WRCOUNT_FRAME_COUNT__MASK			0xffff0000
-#define MDP5_PP_SYNC_WRCOUNT_FRAME_COUNT__SHIFT			16
-static inline uint32_t MDP5_PP_SYNC_WRCOUNT_FRAME_COUNT(uint32_t val)
-{
-	return ((val) << MDP5_PP_SYNC_WRCOUNT_FRAME_COUNT__SHIFT) & MDP5_PP_SYNC_WRCOUNT_FRAME_COUNT__MASK;
-}
-
-static inline uint32_t REG_MDP5_PP_VSYNC_INIT_VAL(uint32_t i0) { return 0x00000010 + __offset_PP(i0); }
-
-static inline uint32_t REG_MDP5_PP_INT_COUNT_VAL(uint32_t i0) { return 0x00000014 + __offset_PP(i0); }
-#define MDP5_PP_INT_COUNT_VAL_LINE_COUNT__MASK			0x0000ffff
-#define MDP5_PP_INT_COUNT_VAL_LINE_COUNT__SHIFT			0
-static inline uint32_t MDP5_PP_INT_COUNT_VAL_LINE_COUNT(uint32_t val)
-{
-	return ((val) << MDP5_PP_INT_COUNT_VAL_LINE_COUNT__SHIFT) & MDP5_PP_INT_COUNT_VAL_LINE_COUNT__MASK;
-}
-#define MDP5_PP_INT_COUNT_VAL_FRAME_COUNT__MASK			0xffff0000
-#define MDP5_PP_INT_COUNT_VAL_FRAME_COUNT__SHIFT		16
-static inline uint32_t MDP5_PP_INT_COUNT_VAL_FRAME_COUNT(uint32_t val)
-{
-	return ((val) << MDP5_PP_INT_COUNT_VAL_FRAME_COUNT__SHIFT) & MDP5_PP_INT_COUNT_VAL_FRAME_COUNT__MASK;
-}
-
-static inline uint32_t REG_MDP5_PP_SYNC_THRESH(uint32_t i0) { return 0x00000018 + __offset_PP(i0); }
-#define MDP5_PP_SYNC_THRESH_START__MASK				0x0000ffff
-#define MDP5_PP_SYNC_THRESH_START__SHIFT			0
-static inline uint32_t MDP5_PP_SYNC_THRESH_START(uint32_t val)
-{
-	return ((val) << MDP5_PP_SYNC_THRESH_START__SHIFT) & MDP5_PP_SYNC_THRESH_START__MASK;
-}
-#define MDP5_PP_SYNC_THRESH_CONTINUE__MASK			0xffff0000
-#define MDP5_PP_SYNC_THRESH_CONTINUE__SHIFT			16
-static inline uint32_t MDP5_PP_SYNC_THRESH_CONTINUE(uint32_t val)
-{
-	return ((val) << MDP5_PP_SYNC_THRESH_CONTINUE__SHIFT) & MDP5_PP_SYNC_THRESH_CONTINUE__MASK;
-}
-
-static inline uint32_t REG_MDP5_PP_START_POS(uint32_t i0) { return 0x0000001c + __offset_PP(i0); }
-
-static inline uint32_t REG_MDP5_PP_RD_PTR_IRQ(uint32_t i0) { return 0x00000020 + __offset_PP(i0); }
-
-static inline uint32_t REG_MDP5_PP_WR_PTR_IRQ(uint32_t i0) { return 0x00000024 + __offset_PP(i0); }
-
-static inline uint32_t REG_MDP5_PP_OUT_LINE_COUNT(uint32_t i0) { return 0x00000028 + __offset_PP(i0); }
-
-static inline uint32_t REG_MDP5_PP_PP_LINE_COUNT(uint32_t i0) { return 0x0000002c + __offset_PP(i0); }
-
-static inline uint32_t REG_MDP5_PP_AUTOREFRESH_CONFIG(uint32_t i0) { return 0x00000030 + __offset_PP(i0); }
-
-static inline uint32_t REG_MDP5_PP_FBC_MODE(uint32_t i0) { return 0x00000034 + __offset_PP(i0); }
-
-static inline uint32_t REG_MDP5_PP_FBC_BUDGET_CTL(uint32_t i0) { return 0x00000038 + __offset_PP(i0); }
-
-static inline uint32_t REG_MDP5_PP_FBC_LOSSY_MODE(uint32_t i0) { return 0x0000003c + __offset_PP(i0); }
-
-static inline uint32_t __offset_WB(uint32_t idx)
-{
-	switch (idx) {
-#if 0  /* TEMPORARY until patch that adds wb.base[] is merged */
-		case 0: return (mdp5_cfg->wb.base[0]);
-		case 1: return (mdp5_cfg->wb.base[1]);
-		case 2: return (mdp5_cfg->wb.base[2]);
-		case 3: return (mdp5_cfg->wb.base[3]);
-		case 4: return (mdp5_cfg->wb.base[4]);
-#endif
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_WB(uint32_t i0) { return 0x00000000 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_DST_FORMAT(uint32_t i0) { return 0x00000000 + __offset_WB(i0); }
-#define MDP5_WB_DST_FORMAT_DSTC0_OUT__MASK			0x00000003
-#define MDP5_WB_DST_FORMAT_DSTC0_OUT__SHIFT			0
-static inline uint32_t MDP5_WB_DST_FORMAT_DSTC0_OUT(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_FORMAT_DSTC0_OUT__SHIFT) & MDP5_WB_DST_FORMAT_DSTC0_OUT__MASK;
-}
-#define MDP5_WB_DST_FORMAT_DSTC1_OUT__MASK			0x0000000c
-#define MDP5_WB_DST_FORMAT_DSTC1_OUT__SHIFT			2
-static inline uint32_t MDP5_WB_DST_FORMAT_DSTC1_OUT(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_FORMAT_DSTC1_OUT__SHIFT) & MDP5_WB_DST_FORMAT_DSTC1_OUT__MASK;
-}
-#define MDP5_WB_DST_FORMAT_DSTC2_OUT__MASK			0x00000030
-#define MDP5_WB_DST_FORMAT_DSTC2_OUT__SHIFT			4
-static inline uint32_t MDP5_WB_DST_FORMAT_DSTC2_OUT(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_FORMAT_DSTC2_OUT__SHIFT) & MDP5_WB_DST_FORMAT_DSTC2_OUT__MASK;
-}
-#define MDP5_WB_DST_FORMAT_DSTC3_OUT__MASK			0x000000c0
-#define MDP5_WB_DST_FORMAT_DSTC3_OUT__SHIFT			6
-static inline uint32_t MDP5_WB_DST_FORMAT_DSTC3_OUT(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_FORMAT_DSTC3_OUT__SHIFT) & MDP5_WB_DST_FORMAT_DSTC3_OUT__MASK;
-}
-#define MDP5_WB_DST_FORMAT_DSTC3_EN				0x00000100
-#define MDP5_WB_DST_FORMAT_DST_BPP__MASK			0x00000600
-#define MDP5_WB_DST_FORMAT_DST_BPP__SHIFT			9
-static inline uint32_t MDP5_WB_DST_FORMAT_DST_BPP(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_FORMAT_DST_BPP__SHIFT) & MDP5_WB_DST_FORMAT_DST_BPP__MASK;
-}
-#define MDP5_WB_DST_FORMAT_PACK_COUNT__MASK			0x00003000
-#define MDP5_WB_DST_FORMAT_PACK_COUNT__SHIFT			12
-static inline uint32_t MDP5_WB_DST_FORMAT_PACK_COUNT(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_FORMAT_PACK_COUNT__SHIFT) & MDP5_WB_DST_FORMAT_PACK_COUNT__MASK;
-}
-#define MDP5_WB_DST_FORMAT_DST_ALPHA_X				0x00004000
-#define MDP5_WB_DST_FORMAT_PACK_TIGHT				0x00020000
-#define MDP5_WB_DST_FORMAT_PACK_ALIGN_MSB			0x00040000
-#define MDP5_WB_DST_FORMAT_WRITE_PLANES__MASK			0x00180000
-#define MDP5_WB_DST_FORMAT_WRITE_PLANES__SHIFT			19
-static inline uint32_t MDP5_WB_DST_FORMAT_WRITE_PLANES(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_FORMAT_WRITE_PLANES__SHIFT) & MDP5_WB_DST_FORMAT_WRITE_PLANES__MASK;
-}
-#define MDP5_WB_DST_FORMAT_DST_DITHER_EN			0x00400000
-#define MDP5_WB_DST_FORMAT_DST_CHROMA_SAMP__MASK		0x03800000
-#define MDP5_WB_DST_FORMAT_DST_CHROMA_SAMP__SHIFT		23
-static inline uint32_t MDP5_WB_DST_FORMAT_DST_CHROMA_SAMP(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_FORMAT_DST_CHROMA_SAMP__SHIFT) & MDP5_WB_DST_FORMAT_DST_CHROMA_SAMP__MASK;
-}
-#define MDP5_WB_DST_FORMAT_DST_CHROMA_SITE__MASK		0x3c000000
-#define MDP5_WB_DST_FORMAT_DST_CHROMA_SITE__SHIFT		26
-static inline uint32_t MDP5_WB_DST_FORMAT_DST_CHROMA_SITE(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_FORMAT_DST_CHROMA_SITE__SHIFT) & MDP5_WB_DST_FORMAT_DST_CHROMA_SITE__MASK;
-}
-#define MDP5_WB_DST_FORMAT_FRAME_FORMAT__MASK			0xc0000000
-#define MDP5_WB_DST_FORMAT_FRAME_FORMAT__SHIFT			30
-static inline uint32_t MDP5_WB_DST_FORMAT_FRAME_FORMAT(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_FORMAT_FRAME_FORMAT__SHIFT) & MDP5_WB_DST_FORMAT_FRAME_FORMAT__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_DST_OP_MODE(uint32_t i0) { return 0x00000004 + __offset_WB(i0); }
-#define MDP5_WB_DST_OP_MODE_BWC_ENC_EN				0x00000001
-#define MDP5_WB_DST_OP_MODE_BWC_ENC_OP__MASK			0x00000006
-#define MDP5_WB_DST_OP_MODE_BWC_ENC_OP__SHIFT			1
-static inline uint32_t MDP5_WB_DST_OP_MODE_BWC_ENC_OP(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_OP_MODE_BWC_ENC_OP__SHIFT) & MDP5_WB_DST_OP_MODE_BWC_ENC_OP__MASK;
-}
-#define MDP5_WB_DST_OP_MODE_BLOCK_SIZE__MASK			0x00000010
-#define MDP5_WB_DST_OP_MODE_BLOCK_SIZE__SHIFT			4
-static inline uint32_t MDP5_WB_DST_OP_MODE_BLOCK_SIZE(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_OP_MODE_BLOCK_SIZE__SHIFT) & MDP5_WB_DST_OP_MODE_BLOCK_SIZE__MASK;
-}
-#define MDP5_WB_DST_OP_MODE_ROT_MODE__MASK			0x00000020
-#define MDP5_WB_DST_OP_MODE_ROT_MODE__SHIFT			5
-static inline uint32_t MDP5_WB_DST_OP_MODE_ROT_MODE(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_OP_MODE_ROT_MODE__SHIFT) & MDP5_WB_DST_OP_MODE_ROT_MODE__MASK;
-}
-#define MDP5_WB_DST_OP_MODE_ROT_EN				0x00000040
-#define MDP5_WB_DST_OP_MODE_CSC_EN				0x00000100
-#define MDP5_WB_DST_OP_MODE_CSC_SRC_DATA_FORMAT__MASK		0x00000200
-#define MDP5_WB_DST_OP_MODE_CSC_SRC_DATA_FORMAT__SHIFT		9
-static inline uint32_t MDP5_WB_DST_OP_MODE_CSC_SRC_DATA_FORMAT(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_OP_MODE_CSC_SRC_DATA_FORMAT__SHIFT) & MDP5_WB_DST_OP_MODE_CSC_SRC_DATA_FORMAT__MASK;
-}
-#define MDP5_WB_DST_OP_MODE_CSC_DST_DATA_FORMAT__MASK		0x00000400
-#define MDP5_WB_DST_OP_MODE_CSC_DST_DATA_FORMAT__SHIFT		10
-static inline uint32_t MDP5_WB_DST_OP_MODE_CSC_DST_DATA_FORMAT(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_OP_MODE_CSC_DST_DATA_FORMAT__SHIFT) & MDP5_WB_DST_OP_MODE_CSC_DST_DATA_FORMAT__MASK;
-}
-#define MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_EN		0x00000800
-#define MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_FORMAT__MASK	0x00001000
-#define MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_FORMAT__SHIFT	12
-static inline uint32_t MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_FORMAT(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_FORMAT__SHIFT) & MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_FORMAT__MASK;
-}
-#define MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_H_MTHD__MASK	0x00002000
-#define MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_H_MTHD__SHIFT	13
-static inline uint32_t MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_H_MTHD(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_H_MTHD__SHIFT) & MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_H_MTHD__MASK;
-}
-#define MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_V_MTHD__MASK	0x00004000
-#define MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_V_MTHD__SHIFT	14
-static inline uint32_t MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_V_MTHD(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_V_MTHD__SHIFT) & MDP5_WB_DST_OP_MODE_CHROMA_DWN_SAMPLE_V_MTHD__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_DST_PACK_PATTERN(uint32_t i0) { return 0x00000008 + __offset_WB(i0); }
-#define MDP5_WB_DST_PACK_PATTERN_ELEMENT0__MASK			0x00000003
-#define MDP5_WB_DST_PACK_PATTERN_ELEMENT0__SHIFT		0
-static inline uint32_t MDP5_WB_DST_PACK_PATTERN_ELEMENT0(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_PACK_PATTERN_ELEMENT0__SHIFT) & MDP5_WB_DST_PACK_PATTERN_ELEMENT0__MASK;
-}
-#define MDP5_WB_DST_PACK_PATTERN_ELEMENT1__MASK			0x00000300
-#define MDP5_WB_DST_PACK_PATTERN_ELEMENT1__SHIFT		8
-static inline uint32_t MDP5_WB_DST_PACK_PATTERN_ELEMENT1(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_PACK_PATTERN_ELEMENT1__SHIFT) & MDP5_WB_DST_PACK_PATTERN_ELEMENT1__MASK;
-}
-#define MDP5_WB_DST_PACK_PATTERN_ELEMENT2__MASK			0x00030000
-#define MDP5_WB_DST_PACK_PATTERN_ELEMENT2__SHIFT		16
-static inline uint32_t MDP5_WB_DST_PACK_PATTERN_ELEMENT2(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_PACK_PATTERN_ELEMENT2__SHIFT) & MDP5_WB_DST_PACK_PATTERN_ELEMENT2__MASK;
-}
-#define MDP5_WB_DST_PACK_PATTERN_ELEMENT3__MASK			0x03000000
-#define MDP5_WB_DST_PACK_PATTERN_ELEMENT3__SHIFT		24
-static inline uint32_t MDP5_WB_DST_PACK_PATTERN_ELEMENT3(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_PACK_PATTERN_ELEMENT3__SHIFT) & MDP5_WB_DST_PACK_PATTERN_ELEMENT3__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_DST0_ADDR(uint32_t i0) { return 0x0000000c + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_DST1_ADDR(uint32_t i0) { return 0x00000010 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_DST2_ADDR(uint32_t i0) { return 0x00000014 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_DST3_ADDR(uint32_t i0) { return 0x00000018 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_DST_YSTRIDE0(uint32_t i0) { return 0x0000001c + __offset_WB(i0); }
-#define MDP5_WB_DST_YSTRIDE0_DST0_YSTRIDE__MASK			0x0000ffff
-#define MDP5_WB_DST_YSTRIDE0_DST0_YSTRIDE__SHIFT		0
-static inline uint32_t MDP5_WB_DST_YSTRIDE0_DST0_YSTRIDE(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_YSTRIDE0_DST0_YSTRIDE__SHIFT) & MDP5_WB_DST_YSTRIDE0_DST0_YSTRIDE__MASK;
-}
-#define MDP5_WB_DST_YSTRIDE0_DST1_YSTRIDE__MASK			0xffff0000
-#define MDP5_WB_DST_YSTRIDE0_DST1_YSTRIDE__SHIFT		16
-static inline uint32_t MDP5_WB_DST_YSTRIDE0_DST1_YSTRIDE(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_YSTRIDE0_DST1_YSTRIDE__SHIFT) & MDP5_WB_DST_YSTRIDE0_DST1_YSTRIDE__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_DST_YSTRIDE1(uint32_t i0) { return 0x00000020 + __offset_WB(i0); }
-#define MDP5_WB_DST_YSTRIDE1_DST2_YSTRIDE__MASK			0x0000ffff
-#define MDP5_WB_DST_YSTRIDE1_DST2_YSTRIDE__SHIFT		0
-static inline uint32_t MDP5_WB_DST_YSTRIDE1_DST2_YSTRIDE(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_YSTRIDE1_DST2_YSTRIDE__SHIFT) & MDP5_WB_DST_YSTRIDE1_DST2_YSTRIDE__MASK;
-}
-#define MDP5_WB_DST_YSTRIDE1_DST3_YSTRIDE__MASK			0xffff0000
-#define MDP5_WB_DST_YSTRIDE1_DST3_YSTRIDE__SHIFT		16
-static inline uint32_t MDP5_WB_DST_YSTRIDE1_DST3_YSTRIDE(uint32_t val)
-{
-	return ((val) << MDP5_WB_DST_YSTRIDE1_DST3_YSTRIDE__SHIFT) & MDP5_WB_DST_YSTRIDE1_DST3_YSTRIDE__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_DST_DITHER_BITDEPTH(uint32_t i0) { return 0x00000024 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_DITHER_MATRIX_ROW0(uint32_t i0) { return 0x00000030 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_DITHER_MATRIX_ROW1(uint32_t i0) { return 0x00000034 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_DITHER_MATRIX_ROW2(uint32_t i0) { return 0x00000038 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_DITHER_MATRIX_ROW3(uint32_t i0) { return 0x0000003c + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_DST_WRITE_CONFIG(uint32_t i0) { return 0x00000048 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_ROTATION_DNSCALER(uint32_t i0) { return 0x00000050 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_N16_INIT_PHASE_X_0_3(uint32_t i0) { return 0x00000060 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_N16_INIT_PHASE_X_1_2(uint32_t i0) { return 0x00000064 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_N16_INIT_PHASE_Y_0_3(uint32_t i0) { return 0x00000068 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_N16_INIT_PHASE_Y_1_2(uint32_t i0) { return 0x0000006c + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_OUT_SIZE(uint32_t i0) { return 0x00000074 + __offset_WB(i0); }
-#define MDP5_WB_OUT_SIZE_DST_W__MASK				0x0000ffff
-#define MDP5_WB_OUT_SIZE_DST_W__SHIFT				0
-static inline uint32_t MDP5_WB_OUT_SIZE_DST_W(uint32_t val)
-{
-	return ((val) << MDP5_WB_OUT_SIZE_DST_W__SHIFT) & MDP5_WB_OUT_SIZE_DST_W__MASK;
-}
-#define MDP5_WB_OUT_SIZE_DST_H__MASK				0xffff0000
-#define MDP5_WB_OUT_SIZE_DST_H__SHIFT				16
-static inline uint32_t MDP5_WB_OUT_SIZE_DST_H(uint32_t val)
-{
-	return ((val) << MDP5_WB_OUT_SIZE_DST_H__SHIFT) & MDP5_WB_OUT_SIZE_DST_H__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_ALPHA_X_VALUE(uint32_t i0) { return 0x00000078 + __offset_WB(i0); }
-
-static inline uint32_t REG_MDP5_WB_CSC_MATRIX_COEFF_0(uint32_t i0) { return 0x00000260 + __offset_WB(i0); }
-#define MDP5_WB_CSC_MATRIX_COEFF_0_COEFF_11__MASK		0x00001fff
-#define MDP5_WB_CSC_MATRIX_COEFF_0_COEFF_11__SHIFT		0
-static inline uint32_t MDP5_WB_CSC_MATRIX_COEFF_0_COEFF_11(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_MATRIX_COEFF_0_COEFF_11__SHIFT) & MDP5_WB_CSC_MATRIX_COEFF_0_COEFF_11__MASK;
-}
-#define MDP5_WB_CSC_MATRIX_COEFF_0_COEFF_12__MASK		0x1fff0000
-#define MDP5_WB_CSC_MATRIX_COEFF_0_COEFF_12__SHIFT		16
-static inline uint32_t MDP5_WB_CSC_MATRIX_COEFF_0_COEFF_12(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_MATRIX_COEFF_0_COEFF_12__SHIFT) & MDP5_WB_CSC_MATRIX_COEFF_0_COEFF_12__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_CSC_MATRIX_COEFF_1(uint32_t i0) { return 0x00000264 + __offset_WB(i0); }
-#define MDP5_WB_CSC_MATRIX_COEFF_1_COEFF_13__MASK		0x00001fff
-#define MDP5_WB_CSC_MATRIX_COEFF_1_COEFF_13__SHIFT		0
-static inline uint32_t MDP5_WB_CSC_MATRIX_COEFF_1_COEFF_13(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_MATRIX_COEFF_1_COEFF_13__SHIFT) & MDP5_WB_CSC_MATRIX_COEFF_1_COEFF_13__MASK;
-}
-#define MDP5_WB_CSC_MATRIX_COEFF_1_COEFF_21__MASK		0x1fff0000
-#define MDP5_WB_CSC_MATRIX_COEFF_1_COEFF_21__SHIFT		16
-static inline uint32_t MDP5_WB_CSC_MATRIX_COEFF_1_COEFF_21(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_MATRIX_COEFF_1_COEFF_21__SHIFT) & MDP5_WB_CSC_MATRIX_COEFF_1_COEFF_21__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_CSC_MATRIX_COEFF_2(uint32_t i0) { return 0x00000268 + __offset_WB(i0); }
-#define MDP5_WB_CSC_MATRIX_COEFF_2_COEFF_22__MASK		0x00001fff
-#define MDP5_WB_CSC_MATRIX_COEFF_2_COEFF_22__SHIFT		0
-static inline uint32_t MDP5_WB_CSC_MATRIX_COEFF_2_COEFF_22(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_MATRIX_COEFF_2_COEFF_22__SHIFT) & MDP5_WB_CSC_MATRIX_COEFF_2_COEFF_22__MASK;
-}
-#define MDP5_WB_CSC_MATRIX_COEFF_2_COEFF_23__MASK		0x1fff0000
-#define MDP5_WB_CSC_MATRIX_COEFF_2_COEFF_23__SHIFT		16
-static inline uint32_t MDP5_WB_CSC_MATRIX_COEFF_2_COEFF_23(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_MATRIX_COEFF_2_COEFF_23__SHIFT) & MDP5_WB_CSC_MATRIX_COEFF_2_COEFF_23__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_CSC_MATRIX_COEFF_3(uint32_t i0) { return 0x0000026c + __offset_WB(i0); }
-#define MDP5_WB_CSC_MATRIX_COEFF_3_COEFF_31__MASK		0x00001fff
-#define MDP5_WB_CSC_MATRIX_COEFF_3_COEFF_31__SHIFT		0
-static inline uint32_t MDP5_WB_CSC_MATRIX_COEFF_3_COEFF_31(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_MATRIX_COEFF_3_COEFF_31__SHIFT) & MDP5_WB_CSC_MATRIX_COEFF_3_COEFF_31__MASK;
-}
-#define MDP5_WB_CSC_MATRIX_COEFF_3_COEFF_32__MASK		0x1fff0000
-#define MDP5_WB_CSC_MATRIX_COEFF_3_COEFF_32__SHIFT		16
-static inline uint32_t MDP5_WB_CSC_MATRIX_COEFF_3_COEFF_32(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_MATRIX_COEFF_3_COEFF_32__SHIFT) & MDP5_WB_CSC_MATRIX_COEFF_3_COEFF_32__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_CSC_MATRIX_COEFF_4(uint32_t i0) { return 0x00000270 + __offset_WB(i0); }
-#define MDP5_WB_CSC_MATRIX_COEFF_4_COEFF_33__MASK		0x00001fff
-#define MDP5_WB_CSC_MATRIX_COEFF_4_COEFF_33__SHIFT		0
-static inline uint32_t MDP5_WB_CSC_MATRIX_COEFF_4_COEFF_33(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_MATRIX_COEFF_4_COEFF_33__SHIFT) & MDP5_WB_CSC_MATRIX_COEFF_4_COEFF_33__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_CSC_COMP_PRECLAMP(uint32_t i0, uint32_t i1) { return 0x00000274 + __offset_WB(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP5_WB_CSC_COMP_PRECLAMP_REG(uint32_t i0, uint32_t i1) { return 0x00000274 + __offset_WB(i0) + 0x4*i1; }
-#define MDP5_WB_CSC_COMP_PRECLAMP_REG_HIGH__MASK		0x000000ff
-#define MDP5_WB_CSC_COMP_PRECLAMP_REG_HIGH__SHIFT		0
-static inline uint32_t MDP5_WB_CSC_COMP_PRECLAMP_REG_HIGH(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_COMP_PRECLAMP_REG_HIGH__SHIFT) & MDP5_WB_CSC_COMP_PRECLAMP_REG_HIGH__MASK;
-}
-#define MDP5_WB_CSC_COMP_PRECLAMP_REG_LOW__MASK			0x0000ff00
-#define MDP5_WB_CSC_COMP_PRECLAMP_REG_LOW__SHIFT		8
-static inline uint32_t MDP5_WB_CSC_COMP_PRECLAMP_REG_LOW(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_COMP_PRECLAMP_REG_LOW__SHIFT) & MDP5_WB_CSC_COMP_PRECLAMP_REG_LOW__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_CSC_COMP_POSTCLAMP(uint32_t i0, uint32_t i1) { return 0x00000280 + __offset_WB(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP5_WB_CSC_COMP_POSTCLAMP_REG(uint32_t i0, uint32_t i1) { return 0x00000280 + __offset_WB(i0) + 0x4*i1; }
-#define MDP5_WB_CSC_COMP_POSTCLAMP_REG_HIGH__MASK		0x000000ff
-#define MDP5_WB_CSC_COMP_POSTCLAMP_REG_HIGH__SHIFT		0
-static inline uint32_t MDP5_WB_CSC_COMP_POSTCLAMP_REG_HIGH(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_COMP_POSTCLAMP_REG_HIGH__SHIFT) & MDP5_WB_CSC_COMP_POSTCLAMP_REG_HIGH__MASK;
-}
-#define MDP5_WB_CSC_COMP_POSTCLAMP_REG_LOW__MASK		0x0000ff00
-#define MDP5_WB_CSC_COMP_POSTCLAMP_REG_LOW__SHIFT		8
-static inline uint32_t MDP5_WB_CSC_COMP_POSTCLAMP_REG_LOW(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_COMP_POSTCLAMP_REG_LOW__SHIFT) & MDP5_WB_CSC_COMP_POSTCLAMP_REG_LOW__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_CSC_COMP_PREBIAS(uint32_t i0, uint32_t i1) { return 0x0000028c + __offset_WB(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP5_WB_CSC_COMP_PREBIAS_REG(uint32_t i0, uint32_t i1) { return 0x0000028c + __offset_WB(i0) + 0x4*i1; }
-#define MDP5_WB_CSC_COMP_PREBIAS_REG_VALUE__MASK		0x000001ff
-#define MDP5_WB_CSC_COMP_PREBIAS_REG_VALUE__SHIFT		0
-static inline uint32_t MDP5_WB_CSC_COMP_PREBIAS_REG_VALUE(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_COMP_PREBIAS_REG_VALUE__SHIFT) & MDP5_WB_CSC_COMP_PREBIAS_REG_VALUE__MASK;
-}
-
-static inline uint32_t REG_MDP5_WB_CSC_COMP_POSTBIAS(uint32_t i0, uint32_t i1) { return 0x00000298 + __offset_WB(i0) + 0x4*i1; }
-
-static inline uint32_t REG_MDP5_WB_CSC_COMP_POSTBIAS_REG(uint32_t i0, uint32_t i1) { return 0x00000298 + __offset_WB(i0) + 0x4*i1; }
-#define MDP5_WB_CSC_COMP_POSTBIAS_REG_VALUE__MASK		0x000001ff
-#define MDP5_WB_CSC_COMP_POSTBIAS_REG_VALUE__SHIFT		0
-static inline uint32_t MDP5_WB_CSC_COMP_POSTBIAS_REG_VALUE(uint32_t val)
-{
-	return ((val) << MDP5_WB_CSC_COMP_POSTBIAS_REG_VALUE__SHIFT) & MDP5_WB_CSC_COMP_POSTBIAS_REG_VALUE__MASK;
-}
-
-static inline uint32_t __offset_INTF(uint32_t idx)
-{
-	switch (idx) {
-		case 0: return (mdp5_cfg->intf.base[0]);
-		case 1: return (mdp5_cfg->intf.base[1]);
-		case 2: return (mdp5_cfg->intf.base[2]);
-		case 3: return (mdp5_cfg->intf.base[3]);
-		case 4: return (mdp5_cfg->intf.base[4]);
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_INTF(uint32_t i0) { return 0x00000000 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_TIMING_ENGINE_EN(uint32_t i0) { return 0x00000000 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_CONFIG(uint32_t i0) { return 0x00000004 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_HSYNC_CTL(uint32_t i0) { return 0x00000008 + __offset_INTF(i0); }
-#define MDP5_INTF_HSYNC_CTL_PULSEW__MASK			0x0000ffff
-#define MDP5_INTF_HSYNC_CTL_PULSEW__SHIFT			0
-static inline uint32_t MDP5_INTF_HSYNC_CTL_PULSEW(uint32_t val)
-{
-	return ((val) << MDP5_INTF_HSYNC_CTL_PULSEW__SHIFT) & MDP5_INTF_HSYNC_CTL_PULSEW__MASK;
-}
-#define MDP5_INTF_HSYNC_CTL_PERIOD__MASK			0xffff0000
-#define MDP5_INTF_HSYNC_CTL_PERIOD__SHIFT			16
-static inline uint32_t MDP5_INTF_HSYNC_CTL_PERIOD(uint32_t val)
-{
-	return ((val) << MDP5_INTF_HSYNC_CTL_PERIOD__SHIFT) & MDP5_INTF_HSYNC_CTL_PERIOD__MASK;
-}
-
-static inline uint32_t REG_MDP5_INTF_VSYNC_PERIOD_F0(uint32_t i0) { return 0x0000000c + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_VSYNC_PERIOD_F1(uint32_t i0) { return 0x00000010 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_VSYNC_LEN_F0(uint32_t i0) { return 0x00000014 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_VSYNC_LEN_F1(uint32_t i0) { return 0x00000018 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_DISPLAY_VSTART_F0(uint32_t i0) { return 0x0000001c + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_DISPLAY_VSTART_F1(uint32_t i0) { return 0x00000020 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_DISPLAY_VEND_F0(uint32_t i0) { return 0x00000024 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_DISPLAY_VEND_F1(uint32_t i0) { return 0x00000028 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_ACTIVE_VSTART_F0(uint32_t i0) { return 0x0000002c + __offset_INTF(i0); }
-#define MDP5_INTF_ACTIVE_VSTART_F0_VAL__MASK			0x7fffffff
-#define MDP5_INTF_ACTIVE_VSTART_F0_VAL__SHIFT			0
-static inline uint32_t MDP5_INTF_ACTIVE_VSTART_F0_VAL(uint32_t val)
-{
-	return ((val) << MDP5_INTF_ACTIVE_VSTART_F0_VAL__SHIFT) & MDP5_INTF_ACTIVE_VSTART_F0_VAL__MASK;
-}
-#define MDP5_INTF_ACTIVE_VSTART_F0_ACTIVE_V_ENABLE		0x80000000
-
-static inline uint32_t REG_MDP5_INTF_ACTIVE_VSTART_F1(uint32_t i0) { return 0x00000030 + __offset_INTF(i0); }
-#define MDP5_INTF_ACTIVE_VSTART_F1_VAL__MASK			0x7fffffff
-#define MDP5_INTF_ACTIVE_VSTART_F1_VAL__SHIFT			0
-static inline uint32_t MDP5_INTF_ACTIVE_VSTART_F1_VAL(uint32_t val)
-{
-	return ((val) << MDP5_INTF_ACTIVE_VSTART_F1_VAL__SHIFT) & MDP5_INTF_ACTIVE_VSTART_F1_VAL__MASK;
-}
-
-static inline uint32_t REG_MDP5_INTF_ACTIVE_VEND_F0(uint32_t i0) { return 0x00000034 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_ACTIVE_VEND_F1(uint32_t i0) { return 0x00000038 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_DISPLAY_HCTL(uint32_t i0) { return 0x0000003c + __offset_INTF(i0); }
-#define MDP5_INTF_DISPLAY_HCTL_START__MASK			0x0000ffff
-#define MDP5_INTF_DISPLAY_HCTL_START__SHIFT			0
-static inline uint32_t MDP5_INTF_DISPLAY_HCTL_START(uint32_t val)
-{
-	return ((val) << MDP5_INTF_DISPLAY_HCTL_START__SHIFT) & MDP5_INTF_DISPLAY_HCTL_START__MASK;
-}
-#define MDP5_INTF_DISPLAY_HCTL_END__MASK			0xffff0000
-#define MDP5_INTF_DISPLAY_HCTL_END__SHIFT			16
-static inline uint32_t MDP5_INTF_DISPLAY_HCTL_END(uint32_t val)
-{
-	return ((val) << MDP5_INTF_DISPLAY_HCTL_END__SHIFT) & MDP5_INTF_DISPLAY_HCTL_END__MASK;
-}
-
-static inline uint32_t REG_MDP5_INTF_ACTIVE_HCTL(uint32_t i0) { return 0x00000040 + __offset_INTF(i0); }
-#define MDP5_INTF_ACTIVE_HCTL_START__MASK			0x00007fff
-#define MDP5_INTF_ACTIVE_HCTL_START__SHIFT			0
-static inline uint32_t MDP5_INTF_ACTIVE_HCTL_START(uint32_t val)
-{
-	return ((val) << MDP5_INTF_ACTIVE_HCTL_START__SHIFT) & MDP5_INTF_ACTIVE_HCTL_START__MASK;
-}
-#define MDP5_INTF_ACTIVE_HCTL_END__MASK				0x7fff0000
-#define MDP5_INTF_ACTIVE_HCTL_END__SHIFT			16
-static inline uint32_t MDP5_INTF_ACTIVE_HCTL_END(uint32_t val)
-{
-	return ((val) << MDP5_INTF_ACTIVE_HCTL_END__SHIFT) & MDP5_INTF_ACTIVE_HCTL_END__MASK;
-}
-#define MDP5_INTF_ACTIVE_HCTL_ACTIVE_H_ENABLE			0x80000000
-
-static inline uint32_t REG_MDP5_INTF_BORDER_COLOR(uint32_t i0) { return 0x00000044 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_UNDERFLOW_COLOR(uint32_t i0) { return 0x00000048 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_HSYNC_SKEW(uint32_t i0) { return 0x0000004c + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_POLARITY_CTL(uint32_t i0) { return 0x00000050 + __offset_INTF(i0); }
-#define MDP5_INTF_POLARITY_CTL_HSYNC_LOW			0x00000001
-#define MDP5_INTF_POLARITY_CTL_VSYNC_LOW			0x00000002
-#define MDP5_INTF_POLARITY_CTL_DATA_EN_LOW			0x00000004
-
-static inline uint32_t REG_MDP5_INTF_TEST_CTL(uint32_t i0) { return 0x00000054 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_TP_COLOR0(uint32_t i0) { return 0x00000058 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_TP_COLOR1(uint32_t i0) { return 0x0000005c + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_DSI_CMD_MODE_TRIGGER_EN(uint32_t i0) { return 0x00000084 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_PANEL_FORMAT(uint32_t i0) { return 0x00000090 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_FRAME_LINE_COUNT_EN(uint32_t i0) { return 0x000000a8 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_FRAME_COUNT(uint32_t i0) { return 0x000000ac + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_LINE_COUNT(uint32_t i0) { return 0x000000b0 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_DEFLICKER_CONFIG(uint32_t i0) { return 0x000000f0 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_DEFLICKER_STRNG_COEFF(uint32_t i0) { return 0x000000f4 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_DEFLICKER_WEAK_COEFF(uint32_t i0) { return 0x000000f8 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_TPG_ENABLE(uint32_t i0) { return 0x00000100 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_TPG_MAIN_CONTROL(uint32_t i0) { return 0x00000104 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_TPG_VIDEO_CONFIG(uint32_t i0) { return 0x00000108 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_TPG_COMPONENT_LIMITS(uint32_t i0) { return 0x0000010c + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_TPG_RECTANGLE(uint32_t i0) { return 0x00000110 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_TPG_INITIAL_VALUE(uint32_t i0) { return 0x00000114 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_TPG_BLK_WHITE_PATTERN_FRAME(uint32_t i0) { return 0x00000118 + __offset_INTF(i0); }
-
-static inline uint32_t REG_MDP5_INTF_TPG_RGB_MAPPING(uint32_t i0) { return 0x0000011c + __offset_INTF(i0); }
-
-static inline uint32_t __offset_AD(uint32_t idx)
-{
-	switch (idx) {
-		case 0: return (mdp5_cfg->ad.base[0]);
-		case 1: return (mdp5_cfg->ad.base[1]);
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MDP5_AD(uint32_t i0) { return 0x00000000 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_BYPASS(uint32_t i0) { return 0x00000000 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_CTRL_0(uint32_t i0) { return 0x00000004 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_CTRL_1(uint32_t i0) { return 0x00000008 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_FRAME_SIZE(uint32_t i0) { return 0x0000000c + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_CON_CTRL_0(uint32_t i0) { return 0x00000010 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_CON_CTRL_1(uint32_t i0) { return 0x00000014 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_STR_MAN(uint32_t i0) { return 0x00000018 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_VAR(uint32_t i0) { return 0x0000001c + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_DITH(uint32_t i0) { return 0x00000020 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_DITH_CTRL(uint32_t i0) { return 0x00000024 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_AMP_LIM(uint32_t i0) { return 0x00000028 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_SLOPE(uint32_t i0) { return 0x0000002c + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_BW_LVL(uint32_t i0) { return 0x00000030 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_LOGO_POS(uint32_t i0) { return 0x00000034 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_LUT_FI(uint32_t i0) { return 0x00000038 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_LUT_CC(uint32_t i0) { return 0x0000007c + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_STR_LIM(uint32_t i0) { return 0x000000c8 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_CALIB_AB(uint32_t i0) { return 0x000000cc + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_CALIB_CD(uint32_t i0) { return 0x000000d0 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_MODE_SEL(uint32_t i0) { return 0x000000d4 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_TFILT_CTRL(uint32_t i0) { return 0x000000d8 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_BL_MINMAX(uint32_t i0) { return 0x000000dc + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_BL(uint32_t i0) { return 0x000000e0 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_BL_MAX(uint32_t i0) { return 0x000000e8 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_AL(uint32_t i0) { return 0x000000ec + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_AL_MIN(uint32_t i0) { return 0x000000f0 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_AL_FILT(uint32_t i0) { return 0x000000f4 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_CFG_BUF(uint32_t i0) { return 0x000000f8 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_LUT_AL(uint32_t i0) { return 0x00000100 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_TARG_STR(uint32_t i0) { return 0x00000144 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_START_CALC(uint32_t i0) { return 0x00000148 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_STR_OUT(uint32_t i0) { return 0x0000014c + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_BL_OUT(uint32_t i0) { return 0x00000154 + __offset_AD(i0); }
-
-static inline uint32_t REG_MDP5_AD_CALC_DONE(uint32_t i0) { return 0x00000158 + __offset_AD(i0); }
-
-
-#endif /* MDP5_XML */
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.h
index 26c5d8b4ab46..4b988e69fbfc 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.h
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.h
@@ -69,6 +69,16 @@ struct mdp5_mdp_block {
 	uint32_t caps;			/* MDP capabilities: MDP_CAP_xxx bits */
 };
 
+struct mdp5_wb_instance {
+	int id;
+	int lm;
+};
+
+struct mdp5_wb_block {
+	MDP5_SUB_BLOCK_DEFINITION;
+	struct mdp5_wb_instance instances[MAX_BASES];
+};
+
 #define MDP5_INTF_NUM_MAX	5
 
 struct mdp5_intf_block {
@@ -98,6 +108,7 @@ struct mdp5_cfg_hw {
 	struct mdp5_sub_block pp;
 	struct mdp5_sub_block dsc;
 	struct mdp5_sub_block cdm;
+	struct mdp5_wb_block wb;
 	struct mdp5_intf_block intf;
 	struct mdp5_perf_block perf;
 
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
index 4a3db2ea1689..0f653e62b4a0 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
@@ -216,7 +216,7 @@ static void blend_setup(struct drm_crtc *crtc)
 	struct mdp5_kms *mdp5_kms = get_kms(crtc);
 	struct drm_plane *plane;
 	struct mdp5_plane_state *pstate, *pstates[STAGE_MAX + 1] = {NULL};
-	const struct mdp_format *format;
+	const struct msm_format *format;
 	struct mdp5_hw_mixer *mixer = pipeline->mixer;
 	uint32_t lm = mixer->lm;
 	struct mdp5_hw_mixer *r_mixer = pipeline->r_mixer;
@@ -274,7 +274,7 @@ static void blend_setup(struct drm_crtc *crtc)
 		ctl_blend_flags |= MDP5_CTL_BLEND_OP_FLAG_BORDER_OUT;
 		DBG("Border Color is enabled");
 	} else if (plane_cnt) {
-		format = to_mdp_format(msm_framebuffer_format(pstates[STAGE_BASE]->base.fb));
+		format = msm_framebuffer_format(pstates[STAGE_BASE]->base.fb);
 
 		if (format->alpha_enable)
 			bg_alpha_enabled = true;
@@ -285,8 +285,7 @@ static void blend_setup(struct drm_crtc *crtc)
 		if (!pstates[i])
 			continue;
 
-		format = to_mdp_format(
-			msm_framebuffer_format(pstates[i]->base.fb));
+		format = msm_framebuffer_format(pstates[i]->base.fb);
 		plane = pstates[i]->base.plane;
 		blend_op = MDP5_LM_BLEND_OP_MODE_FG_ALPHA(FG_CONST) |
 			MDP5_LM_BLEND_OP_MODE_BG_ALPHA(BG_CONST);
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
index a874fd95cc20..374704cce656 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
@@ -224,7 +224,6 @@ static const struct mdp_kms_funcs kms_funcs = {
 		.prepare_commit  = mdp5_prepare_commit,
 		.wait_flush      = mdp5_wait_flush,
 		.complete_commit = mdp5_complete_commit,
-		.get_format      = mdp_get_format,
 		.destroy         = mdp5_kms_destroy,
 	},
 	.set_irqmask         = mdp5_set_irqmask,
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h
index fac9f05aa639..36b6842dfc9c 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h
@@ -171,13 +171,13 @@ struct mdp5_encoder {
 static inline void mdp5_write(struct mdp5_kms *mdp5_kms, u32 reg, u32 data)
 {
 	WARN_ON(mdp5_kms->enable_count <= 0);
-	msm_writel(data, mdp5_kms->mmio + reg);
+	writel(data, mdp5_kms->mmio + reg);
 }
 
 static inline u32 mdp5_read(struct mdp5_kms *mdp5_kms, u32 reg)
 {
 	WARN_ON(mdp5_kms->enable_count <= 0);
-	return msm_readl(mdp5_kms->mmio + reg);
+	return readl(mdp5_kms->mmio + reg);
 }
 
 static inline const char *stage2name(enum mdp_mixer_stage_id stage)
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
index 0d5ff03cb091..62de248ed1b0 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
@@ -17,9 +17,6 @@
 
 struct mdp5_plane {
 	struct drm_plane base;
-
-	uint32_t nformats;
-	uint32_t formats[32];
 };
 #define to_mdp5_plane(x) container_of(x, struct mdp5_plane, base)
 
@@ -38,15 +35,6 @@ static bool plane_enabled(struct drm_plane_state *state)
 	return state->visible;
 }
 
-static void mdp5_plane_destroy(struct drm_plane *plane)
-{
-	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
-
-	drm_plane_cleanup(plane);
-
-	kfree(mdp5_plane);
-}
-
 /* helper to install properties which are common to planes and crtcs */
 static void mdp5_plane_install_properties(struct drm_plane *plane,
 		struct drm_mode_object *obj)
@@ -138,7 +126,6 @@ static void mdp5_plane_destroy_state(struct drm_plane *plane,
 static const struct drm_plane_funcs mdp5_plane_funcs = {
 		.update_plane = drm_atomic_helper_update_plane,
 		.disable_plane = drm_atomic_helper_disable_plane,
-		.destroy = mdp5_plane_destroy,
 		.reset = mdp5_plane_reset,
 		.atomic_duplicate_state = mdp5_plane_duplicate_state,
 		.atomic_destroy_state = mdp5_plane_destroy_state,
@@ -231,12 +218,12 @@ static int mdp5_plane_atomic_check_with_state(struct drm_crtc_state *crtc_state,
 
 	if (plane_enabled(state)) {
 		unsigned int rotation;
-		const struct mdp_format *format;
+		const struct msm_format *format;
 		struct mdp5_kms *mdp5_kms = get_kms(plane);
 		uint32_t blkcfg = 0;
 
-		format = to_mdp_format(msm_framebuffer_format(state->fb));
-		if (MDP_FORMAT_IS_YUV(format))
+		format = msm_framebuffer_format(state->fb);
+		if (MSM_FORMAT_IS_YUV(format))
 			caps |= MDP_PIPE_CAP_SCALE | MDP_PIPE_CAP_CSC;
 
 		if (((state->src_w >> 16) != state->crtc_w) ||
@@ -271,8 +258,8 @@ static int mdp5_plane_atomic_check_with_state(struct drm_crtc_state *crtc_state,
 			new_hwpipe = true;
 
 		if (mdp5_kms->smp) {
-			const struct mdp_format *format =
-				to_mdp_format(msm_framebuffer_format(state->fb));
+			const struct msm_format *format =
+				msm_framebuffer_format(state->fb);
 
 			blkcfg = mdp5_smp_calculate(mdp5_kms->smp, format,
 					state->src_w >> 16, false);
@@ -633,14 +620,14 @@ static int calc_scaley_steps(struct drm_plane *plane,
 	return 0;
 }
 
-static uint32_t get_scale_config(const struct mdp_format *format,
+static uint32_t get_scale_config(const struct msm_format *format,
 		uint32_t src, uint32_t dst, bool horz)
 {
-	const struct drm_format_info *info = drm_format_info(format->base.pixel_format);
-	bool scaling = format->is_yuv ? true : (src != dst);
+	const struct drm_format_info *info = drm_format_info(format->pixel_format);
+	bool yuv = MSM_FORMAT_IS_YUV(format);
+	bool scaling = yuv ? true : (src != dst);
 	uint32_t sub;
 	uint32_t ya_filter, uv_filter;
-	bool yuv = format->is_yuv;
 
 	if (!scaling)
 		return 0;
@@ -664,12 +651,12 @@ static uint32_t get_scale_config(const struct mdp_format *format,
 			COND(yuv, MDP5_PIPE_SCALE_CONFIG_SCALEY_FILTER_COMP_1_2(uv_filter));
 }
 
-static void calc_pixel_ext(const struct mdp_format *format,
+static void calc_pixel_ext(const struct msm_format *format,
 		uint32_t src, uint32_t dst, uint32_t phase_step[2],
 		int pix_ext_edge1[COMP_MAX], int pix_ext_edge2[COMP_MAX],
 		bool horz)
 {
-	bool scaling = format->is_yuv ? true : (src != dst);
+	bool scaling = MSM_FORMAT_IS_YUV(format) ? true : (src != dst);
 	int i;
 
 	/*
@@ -687,11 +674,11 @@ static void calc_pixel_ext(const struct mdp_format *format,
 }
 
 static void mdp5_write_pixel_ext(struct mdp5_kms *mdp5_kms, enum mdp5_pipe pipe,
-	const struct mdp_format *format,
+	const struct msm_format *format,
 	uint32_t src_w, int pe_left[COMP_MAX], int pe_right[COMP_MAX],
 	uint32_t src_h, int pe_top[COMP_MAX], int pe_bottom[COMP_MAX])
 {
-	const struct drm_format_info *info = drm_format_info(format->base.pixel_format);
+	const struct drm_format_info *info = drm_format_info(format->pixel_format);
 	uint32_t lr, tb, req;
 	int i;
 
@@ -699,7 +686,7 @@ static void mdp5_write_pixel_ext(struct mdp5_kms *mdp5_kms, enum mdp5_pipe pipe,
 		uint32_t roi_w = src_w;
 		uint32_t roi_h = src_h;
 
-		if (format->is_yuv && i == COMP_1_2) {
+		if (MSM_FORMAT_IS_YUV(format) && i == COMP_1_2) {
 			roi_w /= info->hsub;
 			roi_h /= info->vsub;
 		}
@@ -773,8 +760,8 @@ static void mdp5_hwpipe_mode_set(struct mdp5_kms *mdp5_kms,
 {
 	enum mdp5_pipe pipe = hwpipe->pipe;
 	bool has_pe = hwpipe->caps & MDP_PIPE_CAP_SW_PIX_EXT;
-	const struct mdp_format *format =
-			to_mdp_format(msm_framebuffer_format(fb));
+	const struct msm_format *format =
+			msm_framebuffer_format(fb);
 
 	mdp5_write(mdp5_kms, REG_MDP5_PIPE_SRC_IMG_SIZE(pipe),
 			MDP5_PIPE_SRC_IMG_SIZE_WIDTH(src_img_w) |
@@ -798,21 +785,22 @@ static void mdp5_hwpipe_mode_set(struct mdp5_kms *mdp5_kms,
 
 	mdp5_write(mdp5_kms, REG_MDP5_PIPE_SRC_FORMAT(pipe),
 			MDP5_PIPE_SRC_FORMAT_A_BPC(format->bpc_a) |
-			MDP5_PIPE_SRC_FORMAT_R_BPC(format->bpc_r) |
-			MDP5_PIPE_SRC_FORMAT_G_BPC(format->bpc_g) |
-			MDP5_PIPE_SRC_FORMAT_B_BPC(format->bpc_b) |
+			MDP5_PIPE_SRC_FORMAT_R_BPC(format->bpc_r_cr) |
+			MDP5_PIPE_SRC_FORMAT_G_BPC(format->bpc_g_y) |
+			MDP5_PIPE_SRC_FORMAT_B_BPC(format->bpc_b_cb) |
 			COND(format->alpha_enable, MDP5_PIPE_SRC_FORMAT_ALPHA_ENABLE) |
-			MDP5_PIPE_SRC_FORMAT_CPP(format->cpp - 1) |
+			MDP5_PIPE_SRC_FORMAT_CPP(format->bpp - 1) |
 			MDP5_PIPE_SRC_FORMAT_UNPACK_COUNT(format->unpack_count - 1) |
-			COND(format->unpack_tight, MDP5_PIPE_SRC_FORMAT_UNPACK_TIGHT) |
+			COND(format->flags & MSM_FORMAT_FLAG_UNPACK_TIGHT,
+			     MDP5_PIPE_SRC_FORMAT_UNPACK_TIGHT) |
 			MDP5_PIPE_SRC_FORMAT_FETCH_TYPE(format->fetch_type) |
 			MDP5_PIPE_SRC_FORMAT_CHROMA_SAMP(format->chroma_sample));
 
 	mdp5_write(mdp5_kms, REG_MDP5_PIPE_SRC_UNPACK(pipe),
-			MDP5_PIPE_SRC_UNPACK_ELEM0(format->unpack[0]) |
-			MDP5_PIPE_SRC_UNPACK_ELEM1(format->unpack[1]) |
-			MDP5_PIPE_SRC_UNPACK_ELEM2(format->unpack[2]) |
-			MDP5_PIPE_SRC_UNPACK_ELEM3(format->unpack[3]));
+			MDP5_PIPE_SRC_UNPACK_ELEM0(format->element[0]) |
+			MDP5_PIPE_SRC_UNPACK_ELEM1(format->element[1]) |
+			MDP5_PIPE_SRC_UNPACK_ELEM2(format->element[2]) |
+			MDP5_PIPE_SRC_UNPACK_ELEM3(format->element[3]));
 
 	mdp5_write(mdp5_kms, REG_MDP5_PIPE_SRC_OP_MODE(pipe),
 			(hflip ? MDP5_PIPE_SRC_OP_MODE_FLIP_LR : 0) |
@@ -845,7 +833,7 @@ static void mdp5_hwpipe_mode_set(struct mdp5_kms *mdp5_kms,
 	}
 
 	if (hwpipe->caps & MDP_PIPE_CAP_CSC) {
-		if (MDP_FORMAT_IS_YUV(format))
+		if (MSM_FORMAT_IS_YUV(format))
 			csc_enable(mdp5_kms, pipe,
 					mdp_get_default_csc_cfg(CSC_YUV2RGB));
 		else
@@ -864,7 +852,7 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 	struct mdp5_kms *mdp5_kms = get_kms(plane);
 	enum mdp5_pipe pipe = hwpipe->pipe;
 	struct mdp5_hw_pipe *right_hwpipe;
-	const struct mdp_format *format;
+	const struct msm_format *format;
 	uint32_t nplanes, config = 0;
 	struct phase_step step = { { 0 } };
 	struct pixel_ext pe = { { 0 } };
@@ -885,8 +873,8 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 	if (WARN_ON(nplanes > pipe2nclients(pipe)))
 		return -EINVAL;
 
-	format = to_mdp_format(msm_framebuffer_format(fb));
-	pix_format = format->base.pixel_format;
+	format = msm_framebuffer_format(fb);
+	pix_format = format->pixel_format;
 
 	src_x = src->x1;
 	src_y = src->y1;
@@ -1007,31 +995,48 @@ uint32_t mdp5_plane_get_flush(struct drm_plane *plane)
 	return mask;
 }
 
+static const uint32_t mdp5_plane_formats[] = {
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_BGR565,
+
+	DRM_FORMAT_NV12,
+	DRM_FORMAT_NV21,
+	DRM_FORMAT_NV16,
+	DRM_FORMAT_NV61,
+	DRM_FORMAT_VYUY,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_YUV420,
+	DRM_FORMAT_YVU420,
+};
+
 /* initialize plane */
 struct drm_plane *mdp5_plane_init(struct drm_device *dev,
 				  enum drm_plane_type type)
 {
 	struct drm_plane *plane = NULL;
 	struct mdp5_plane *mdp5_plane;
-	int ret;
 
-	mdp5_plane = kzalloc(sizeof(*mdp5_plane), GFP_KERNEL);
-	if (!mdp5_plane) {
-		ret = -ENOMEM;
-		goto fail;
-	}
+	mdp5_plane = drmm_universal_plane_alloc(dev, struct mdp5_plane, base,
+						0xff, &mdp5_plane_funcs,
+						mdp5_plane_formats, ARRAY_SIZE(mdp5_plane_formats),
+						NULL, type, NULL);
+	if (IS_ERR(mdp5_plane))
+		return ERR_CAST(mdp5_plane);
 
 	plane = &mdp5_plane->base;
 
-	mdp5_plane->nformats = mdp_get_formats(mdp5_plane->formats,
-		ARRAY_SIZE(mdp5_plane->formats), false);
-
-	ret = drm_universal_plane_init(dev, plane, 0xff, &mdp5_plane_funcs,
-			mdp5_plane->formats, mdp5_plane->nformats,
-			NULL, type, NULL);
-	if (ret)
-		goto fail;
-
 	drm_plane_helper_add(plane, &mdp5_plane_helper_funcs);
 
 	mdp5_plane_install_properties(plane, &plane->base);
@@ -1039,10 +1044,4 @@ struct drm_plane *mdp5_plane_init(struct drm_device *dev,
 	drm_plane_enable_fb_damage_clips(plane);
 
 	return plane;
-
-fail:
-	if (plane)
-		mdp5_plane_destroy(plane);
-
-	return ERR_PTR(ret);
 }
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c
index b4bebb425d22..3a7f7edda96b 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.c
@@ -114,10 +114,10 @@ static void set_fifo_thresholds(struct mdp5_smp *smp,
  * presumably happens during the dma from scanout buffer).
  */
 uint32_t mdp5_smp_calculate(struct mdp5_smp *smp,
-		const struct mdp_format *format,
+		const struct msm_format *format,
 		u32 width, bool hdecim)
 {
-	const struct drm_format_info *info = drm_format_info(format->base.pixel_format);
+	const struct drm_format_info *info = drm_format_info(format->pixel_format);
 	struct mdp5_kms *mdp5_kms = get_kms(smp);
 	int rev = mdp5_cfg_get_hw_rev(mdp5_kms->cfg);
 	int i, hsub, nplanes, nlines;
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.h
index 21732ed485be..1be9832382d7 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.h
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_smp.h
@@ -74,7 +74,7 @@ void mdp5_smp_dump(struct mdp5_smp *smp, struct drm_printer *p,
 		   struct mdp5_global_state *global_state);
 
 uint32_t mdp5_smp_calculate(struct mdp5_smp *smp,
-		const struct mdp_format *format,
+		const struct msm_format *format,
 		u32 width, bool hdecim);
 
 int mdp5_smp_assign(struct mdp5_smp *smp, struct mdp5_smp_state *state,
diff --git a/drivers/gpu/drm/msm/disp/mdp_common.xml.h b/drivers/gpu/drm/msm/disp/mdp_common.xml.h
deleted file mode 100644
index 4dd8d7db2862..000000000000
--- a/drivers/gpu/drm/msm/disp/mdp_common.xml.h
+++ /dev/null
@@ -1,111 +0,0 @@
-#ifndef MDP_COMMON_XML
-#define MDP_COMMON_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-enum mdp_chroma_samp_type {
-	CHROMA_FULL = 0,
-	CHROMA_H2V1 = 1,
-	CHROMA_H1V2 = 2,
-	CHROMA_420 = 3,
-};
-
-enum mdp_fetch_type {
-	MDP_PLANE_INTERLEAVED = 0,
-	MDP_PLANE_PLANAR = 1,
-	MDP_PLANE_PSEUDO_PLANAR = 2,
-};
-
-enum mdp_mixer_stage_id {
-	STAGE_UNUSED = 0,
-	STAGE_BASE = 1,
-	STAGE0 = 2,
-	STAGE1 = 3,
-	STAGE2 = 4,
-	STAGE3 = 5,
-	STAGE4 = 6,
-	STAGE5 = 7,
-	STAGE6 = 8,
-	STAGE_MAX = 8,
-};
-
-enum mdp_alpha_type {
-	FG_CONST = 0,
-	BG_CONST = 1,
-	FG_PIXEL = 2,
-	BG_PIXEL = 3,
-};
-
-enum mdp_component_type {
-	COMP_0 = 0,
-	COMP_1_2 = 1,
-	COMP_3 = 2,
-	COMP_MAX = 3,
-};
-
-enum mdp_bpc {
-	BPC1 = 0,
-	BPC5 = 1,
-	BPC6 = 2,
-	BPC8 = 3,
-};
-
-enum mdp_bpc_alpha {
-	BPC1A = 0,
-	BPC4A = 1,
-	BPC6A = 2,
-	BPC8A = 3,
-};
-
-
-#endif /* MDP_COMMON_XML */
diff --git a/drivers/gpu/drm/msm/disp/mdp_format.c b/drivers/gpu/drm/msm/disp/mdp_format.c
index 025595336f26..426782d50cb4 100644
--- a/drivers/gpu/drm/msm/disp/mdp_format.c
+++ b/drivers/gpu/drm/msm/disp/mdp_format.c
@@ -62,115 +62,573 @@ static struct csc_cfg csc_convert[CSC_MAX] = {
 	},
 };
 
-#define FMT(name, a, r, g, b, e0, e1, e2, e3, alpha, tight, c, cnt, fp, cs, yuv) { \
-		.base = { .pixel_format = DRM_FORMAT_ ## name }, \
-		.bpc_a = BPC ## a ## A,                          \
-		.bpc_r = BPC ## r,                               \
-		.bpc_g = BPC ## g,                               \
-		.bpc_b = BPC ## b,                               \
-		.unpack = { e0, e1, e2, e3 },                    \
-		.alpha_enable = alpha,                           \
-		.unpack_tight = tight,                           \
-		.cpp = c,                                        \
-		.unpack_count = cnt,                             \
-		.fetch_type = fp,                                \
-		.chroma_sample = cs,                             \
-		.is_yuv = yuv,                                   \
+#define MDP_TILE_HEIGHT_DEFAULT	1
+#define MDP_TILE_HEIGHT_UBWC	4
+#define MDP_TILE_HEIGHT_NV12	8
+
+#define INTERLEAVED_RGB_FMT(fmt, a, r, g, b, e0, e1, e2, e3, uc, alpha,   \
+bp, flg, fm, np)                                                          \
+{                                                                         \
+	.pixel_format = DRM_FORMAT_ ## fmt,                               \
+	.fetch_type = MDP_PLANE_INTERLEAVED,                              \
+	.alpha_enable = alpha,                                            \
+	.element = { (e0), (e1), (e2), (e3) },                            \
+	.bpc_g_y = g,                                                     \
+	.bpc_b_cb = b,                                                    \
+	.bpc_r_cr = r,                                                    \
+	.bpc_a = a,                                                       \
+	.chroma_sample = CHROMA_FULL,                                     \
+	.unpack_count = uc,                                               \
+	.bpp = bp,                                                        \
+	.fetch_mode = fm,                                                 \
+	.flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | flg,                      \
+	.num_planes = np,                                                 \
+	.tile_height = MDP_TILE_HEIGHT_DEFAULT                            \
 }
 
-#define BPC0A 0
+#define INTERLEAVED_RGB_FMT_TILED(fmt, a, r, g, b, e0, e1, e2, e3, uc,    \
+alpha, bp, flg, fm, np, th)                                               \
+{                                                                         \
+	.pixel_format = DRM_FORMAT_ ## fmt,                               \
+	.fetch_type = MDP_PLANE_INTERLEAVED,                              \
+	.alpha_enable = alpha,                                            \
+	.element = { (e0), (e1), (e2), (e3) },                            \
+	.bpc_g_y = g,                                                     \
+	.bpc_b_cb = b,                                                    \
+	.bpc_r_cr = r,                                                    \
+	.bpc_a = a,                                                       \
+	.chroma_sample = CHROMA_FULL,                                     \
+	.unpack_count = uc,                                               \
+	.bpp = bp,                                                        \
+	.fetch_mode = fm,                                                 \
+	.flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | flg,                      \
+	.num_planes = np,                                                 \
+	.tile_height = th                                                 \
+}
 
-/*
- * Note: Keep RGB formats 1st, followed by YUV formats to avoid breaking
- * mdp_get_rgb_formats()'s implementation.
- */
-static const struct mdp_format formats[] = {
-	/*  name      a  r  g  b   e0 e1 e2 e3  alpha   tight  cpp cnt ... */
-	FMT(ARGB8888, 8, 8, 8, 8,  1, 0, 2, 3,  true,   true,  4,  4,
-			MDP_PLANE_INTERLEAVED, CHROMA_FULL, false),
-	FMT(ABGR8888, 8, 8, 8, 8,  2, 0, 1, 3,  true,   true,  4,  4,
-			MDP_PLANE_INTERLEAVED, CHROMA_FULL, false),
-	FMT(RGBA8888, 8, 8, 8, 8,  3, 1, 0, 2,  true,   true,  4,  4,
-			MDP_PLANE_INTERLEAVED, CHROMA_FULL, false),
-	FMT(BGRA8888, 8, 8, 8, 8,  3, 2, 0, 1,  true,   true,  4,  4,
-			MDP_PLANE_INTERLEAVED, CHROMA_FULL, false),
-	FMT(XRGB8888, 8, 8, 8, 8,  1, 0, 2, 3,  false,  true,  4,  4,
-			MDP_PLANE_INTERLEAVED, CHROMA_FULL, false),
-	FMT(XBGR8888, 8, 8, 8, 8,  2, 0, 1, 3,  false,   true,  4,  4,
-			MDP_PLANE_INTERLEAVED, CHROMA_FULL, false),
-	FMT(RGBX8888, 8, 8, 8, 8,  3, 1, 0, 2,  false,   true,  4,  4,
-			MDP_PLANE_INTERLEAVED, CHROMA_FULL, false),
-	FMT(BGRX8888, 8, 8, 8, 8,  3, 2, 0, 1,  false,   true,  4,  4,
-			MDP_PLANE_INTERLEAVED, CHROMA_FULL, false),
-	FMT(RGB888,   0, 8, 8, 8,  1, 0, 2, 0,  false,  true,  3,  3,
-			MDP_PLANE_INTERLEAVED, CHROMA_FULL, false),
-	FMT(BGR888,   0, 8, 8, 8,  2, 0, 1, 0,  false,  true,  3,  3,
-			MDP_PLANE_INTERLEAVED, CHROMA_FULL, false),
-	FMT(RGB565,   0, 5, 6, 5,  1, 0, 2, 0,  false,  true,  2,  3,
-			MDP_PLANE_INTERLEAVED, CHROMA_FULL, false),
-	FMT(BGR565,   0, 5, 6, 5,  2, 0, 1, 0,  false,  true,  2,  3,
-			MDP_PLANE_INTERLEAVED, CHROMA_FULL, false),
+#define INTERLEAVED_YUV_FMT(fmt, a, r, g, b, e0, e1, e2, e3,              \
+alpha, chroma, count, bp, flg, fm, np)                                    \
+{                                                                         \
+	.pixel_format = DRM_FORMAT_ ## fmt,                               \
+	.fetch_type = MDP_PLANE_INTERLEAVED,                              \
+	.alpha_enable = alpha,                                            \
+	.element = { (e0), (e1), (e2), (e3)},                             \
+	.bpc_g_y = g,                                                     \
+	.bpc_b_cb = b,                                                    \
+	.bpc_r_cr = r,                                                    \
+	.bpc_a = a,                                                       \
+	.chroma_sample = chroma,                                          \
+	.unpack_count = count,                                            \
+	.bpp = bp,                                                        \
+	.fetch_mode = fm,                                                 \
+	.flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | flg,                      \
+	.num_planes = np,                                                 \
+	.tile_height = MDP_TILE_HEIGHT_DEFAULT                            \
+}
+
+#define PSEUDO_YUV_FMT(fmt, a, r, g, b, e0, e1, chroma, flg, fm, np)      \
+{                                                                         \
+	.pixel_format = DRM_FORMAT_ ## fmt,                               \
+	.fetch_type = MDP_PLANE_PSEUDO_PLANAR,                            \
+	.alpha_enable = 0,                                                \
+	.element = { (e0), (e1), 0, 0 },                                  \
+	.bpc_g_y = g,                                                     \
+	.bpc_b_cb = b,                                                    \
+	.bpc_r_cr = r,                                                    \
+	.bpc_a = a,                                                       \
+	.chroma_sample = chroma,                                          \
+	.unpack_count = 2,                                                \
+	.bpp = 2,                                                         \
+	.fetch_mode = fm,                                                 \
+	.flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | flg,                      \
+	.num_planes = np,                                                 \
+	.tile_height = MDP_TILE_HEIGHT_DEFAULT                            \
+}
+
+#define PSEUDO_YUV_FMT_TILED(fmt, a, r, g, b, e0, e1, chroma,             \
+flg, fm, np, th)                                                          \
+{                                                                         \
+	.pixel_format = DRM_FORMAT_ ## fmt,                               \
+	.fetch_type = MDP_PLANE_PSEUDO_PLANAR,                            \
+	.alpha_enable = 0,                                                \
+	.element = { (e0), (e1), 0, 0 },                                  \
+	.bpc_g_y = g,                                                     \
+	.bpc_b_cb = b,                                                    \
+	.bpc_r_cr = r,                                                    \
+	.bpc_a = a,                                                       \
+	.chroma_sample = chroma,                                          \
+	.unpack_count = 2,                                                \
+	.bpp = 2,                                                         \
+	.fetch_mode = fm,                                                 \
+	.flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | flg,                      \
+	.num_planes = np,                                                 \
+	.tile_height = th                                                 \
+}
+
+#define PSEUDO_YUV_FMT_LOOSE(fmt, a, r, g, b, e0, e1, chroma, flg, fm, np)\
+{                                                                         \
+	.pixel_format = DRM_FORMAT_ ## fmt,                               \
+	.fetch_type = MDP_PLANE_PSEUDO_PLANAR,                            \
+	.alpha_enable = 0,                                                \
+	.element = { (e0), (e1), 0, 0 },                                  \
+	.bpc_g_y = g,                                                     \
+	.bpc_b_cb = b,                                                    \
+	.bpc_r_cr = r,                                                    \
+	.bpc_a = a,                                                       \
+	.chroma_sample = chroma,                                          \
+	.unpack_count = 2,                                                \
+	.bpp = 2,                                                         \
+	.fetch_mode = fm,                                                 \
+	.flags = MSM_FORMAT_FLAG_UNPACK_ALIGN_MSB | flg,                  \
+	.num_planes = np,                                                 \
+	.tile_height = MDP_TILE_HEIGHT_DEFAULT                            \
+}
+
+#define PSEUDO_YUV_FMT_LOOSE_TILED(fmt, a, r, g, b, e0, e1, chroma,       \
+flg, fm, np, th)                                                          \
+{                                                                         \
+	.pixel_format = DRM_FORMAT_ ## fmt,                               \
+	.fetch_type = MDP_PLANE_PSEUDO_PLANAR,                            \
+	.alpha_enable = 0,                                                \
+	.element = { (e0), (e1), 0, 0 },                                  \
+	.bpc_g_y = g,                                                     \
+	.bpc_b_cb = b,                                                    \
+	.bpc_r_cr = r,                                                    \
+	.bpc_a = a,                                                       \
+	.chroma_sample = chroma,                                          \
+	.unpack_count = 2,                                                \
+	.bpp = 2,                                                         \
+	.fetch_mode = fm,                                                 \
+	.flags = MSM_FORMAT_FLAG_UNPACK_ALIGN_MSB | flg,                  \
+	.num_planes = np,                                                 \
+	.tile_height = th                                                 \
+}
+
+#define PLANAR_YUV_FMT(fmt, a, r, g, b, e0, e1, e2, alpha, chroma, bp,    \
+flg, fm, np)                                                      \
+{                                                                         \
+	.pixel_format = DRM_FORMAT_ ## fmt,                               \
+	.fetch_type = MDP_PLANE_PLANAR,                                   \
+	.alpha_enable = alpha,                                            \
+	.element = { (e0), (e1), (e2), 0 },                               \
+	.bpc_g_y = g,                                                     \
+	.bpc_b_cb = b,                                                    \
+	.bpc_r_cr = r,                                                    \
+	.bpc_a = a,                                                       \
+	.chroma_sample = chroma,                                          \
+	.unpack_count = 1,                                                \
+	.bpp = bp,                                                        \
+	.fetch_mode = fm,                                                 \
+	.flags = MSM_FORMAT_FLAG_UNPACK_TIGHT | flg,                      \
+	.num_planes = np,                                                 \
+	.tile_height = MDP_TILE_HEIGHT_DEFAULT                            \
+}
+
+static const struct msm_format mdp_formats[] = {
+	INTERLEAVED_RGB_FMT(ARGB8888,
+		BPC8A, BPC8, BPC8, BPC8,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		true, 4, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ABGR8888,
+		BPC8A, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XBGR8888,
+		BPC8A, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		false, 4, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBA8888,
+		BPC8A, BPC8, BPC8, BPC8,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		true, 4, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRA8888,
+		BPC8A, BPC8, BPC8, BPC8,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		true, 4, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRX8888,
+		BPC8A, BPC8, BPC8, BPC8,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		false, 4, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XRGB8888,
+		BPC8A, BPC8, BPC8, BPC8,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		false, 4, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBX8888,
+		BPC8A, BPC8, BPC8, BPC8,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		false, 4, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGB888,
+		0, BPC8, BPC8, BPC8,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, 0, 3,
+		false, 3, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGR888,
+		0, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3,
+		false, 3, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGB565,
+		0, BPC5, BPC6, BPC5,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, 0, 3,
+		false, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGR565,
+		0, BPC5, BPC6, BPC5,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3,
+		false, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ARGB1555,
+		BPC1A, BPC5, BPC5, BPC5,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		true, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ABGR1555,
+		BPC1A, BPC5, BPC5, BPC5,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBA5551,
+		BPC1A, BPC5, BPC5, BPC5,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		true, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRA5551,
+		BPC1A, BPC5, BPC5, BPC5,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		true, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XRGB1555,
+		BPC1A, BPC5, BPC5, BPC5,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		false, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XBGR1555,
+		BPC1A, BPC5, BPC5, BPC5,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		false, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBX5551,
+		BPC1A, BPC5, BPC5, BPC5,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		false, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRX5551,
+		BPC1A, BPC5, BPC5, BPC5,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		false, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ARGB4444,
+		BPC4A, BPC4, BPC4, BPC4,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		true, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ABGR4444,
+		BPC4A, BPC4, BPC4, BPC4,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBA4444,
+		BPC4A, BPC4, BPC4, BPC4,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		true, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRA4444,
+		BPC4A, BPC4, BPC4, BPC4,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		true, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XRGB4444,
+		BPC4A, BPC4, BPC4, BPC4,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		false, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XBGR4444,
+		BPC4A, BPC4, BPC4, BPC4,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		false, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBX4444,
+		BPC4A, BPC4, BPC4, BPC4,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		false, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRX4444,
+		BPC4A, BPC4, BPC4, BPC4,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		false, 2, 0,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRA1010102,
+		BPC8A, BPC8, BPC8, BPC8,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		true, 4, MSM_FORMAT_FLAG_DX,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBA1010102,
+		BPC8A, BPC8, BPC8, BPC8,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		true, 4, MSM_FORMAT_FLAG_DX,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ABGR2101010,
+		BPC8A, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, MSM_FORMAT_FLAG_DX,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(ARGB2101010,
+		BPC8A, BPC8, BPC8, BPC8,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		true, 4, MSM_FORMAT_FLAG_DX,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XRGB2101010,
+		BPC8A, BPC8, BPC8, BPC8,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C3_ALPHA, 4,
+		false, 4, MSM_FORMAT_FLAG_DX,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(BGRX1010102,
+		BPC8A, BPC8, BPC8, BPC8,
+		C3_ALPHA, C2_R_Cr, C0_G_Y, C1_B_Cb, 4,
+		false, 4, MSM_FORMAT_FLAG_DX,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(XBGR2101010,
+		BPC8A, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		false, 4, MSM_FORMAT_FLAG_DX,
+		MDP_FETCH_LINEAR, 1),
+
+	INTERLEAVED_RGB_FMT(RGBX1010102,
+		BPC8A, BPC8, BPC8, BPC8,
+		C3_ALPHA, C1_B_Cb, C0_G_Y, C2_R_Cr, 4,
+		false, 4, MSM_FORMAT_FLAG_DX,
+		MDP_FETCH_LINEAR, 1),
 
 	/* --- RGB formats above / YUV formats below this line --- */
 
 	/* 2 plane YUV */
-	FMT(NV12,     0, 8, 8, 8,  1, 2, 0, 0,  false,  true,  2, 2,
-			MDP_PLANE_PSEUDO_PLANAR, CHROMA_420, true),
-	FMT(NV21,     0, 8, 8, 8,  2, 1, 0, 0,  false,  true,  2, 2,
-			MDP_PLANE_PSEUDO_PLANAR, CHROMA_420, true),
-	FMT(NV16,     0, 8, 8, 8,  1, 2, 0, 0,  false,  true,  2, 2,
-			MDP_PLANE_PSEUDO_PLANAR, CHROMA_H2V1, true),
-	FMT(NV61,     0, 8, 8, 8,  2, 1, 0, 0,  false,  true,  2, 2,
-			MDP_PLANE_PSEUDO_PLANAR, CHROMA_H2V1, true),
+	PSEUDO_YUV_FMT(NV12,
+		0, BPC8, BPC8, BPC8,
+		C1_B_Cb, C2_R_Cr,
+		CHROMA_420, MSM_FORMAT_FLAG_YUV,
+		MDP_FETCH_LINEAR, 2),
+
+	PSEUDO_YUV_FMT(NV21,
+		0, BPC8, BPC8, BPC8,
+		C2_R_Cr, C1_B_Cb,
+		CHROMA_420, MSM_FORMAT_FLAG_YUV,
+		MDP_FETCH_LINEAR, 2),
+
+	PSEUDO_YUV_FMT(NV16,
+		0, BPC8, BPC8, BPC8,
+		C1_B_Cb, C2_R_Cr,
+		CHROMA_H2V1, MSM_FORMAT_FLAG_YUV,
+		MDP_FETCH_LINEAR, 2),
+
+	PSEUDO_YUV_FMT(NV61,
+		0, BPC8, BPC8, BPC8,
+		C2_R_Cr, C1_B_Cb,
+		CHROMA_H2V1, MSM_FORMAT_FLAG_YUV,
+		MDP_FETCH_LINEAR, 2),
+
+	PSEUDO_YUV_FMT_LOOSE(P010,
+		0, BPC8, BPC8, BPC8,
+		C1_B_Cb, C2_R_Cr,
+		CHROMA_420, MSM_FORMAT_FLAG_DX | MSM_FORMAT_FLAG_YUV,
+		MDP_FETCH_LINEAR, 2),
+
 	/* 1 plane YUV */
-	FMT(VYUY,     0, 8, 8, 8,  2, 0, 1, 0,  false,  true,  2, 4,
-			MDP_PLANE_INTERLEAVED, CHROMA_H2V1, true),
-	FMT(UYVY,     0, 8, 8, 8,  1, 0, 2, 0,  false,  true,  2, 4,
-			MDP_PLANE_INTERLEAVED, CHROMA_H2V1, true),
-	FMT(YUYV,     0, 8, 8, 8,  0, 1, 0, 2,  false,  true,  2, 4,
-			MDP_PLANE_INTERLEAVED, CHROMA_H2V1, true),
-	FMT(YVYU,     0, 8, 8, 8,  0, 2, 0, 1,  false,  true,  2, 4,
-			MDP_PLANE_INTERLEAVED, CHROMA_H2V1, true),
+	INTERLEAVED_YUV_FMT(VYUY,
+		0, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C0_G_Y,
+		false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV,
+		MDP_FETCH_LINEAR, 2),
+
+	INTERLEAVED_YUV_FMT(UYVY,
+		0, BPC8, BPC8, BPC8,
+		C1_B_Cb, C0_G_Y, C2_R_Cr, C0_G_Y,
+		false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV,
+		MDP_FETCH_LINEAR, 2),
+
+	INTERLEAVED_YUV_FMT(YUYV,
+		0, BPC8, BPC8, BPC8,
+		C0_G_Y, C1_B_Cb, C0_G_Y, C2_R_Cr,
+		false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV,
+		MDP_FETCH_LINEAR, 2),
+
+	INTERLEAVED_YUV_FMT(YVYU,
+		0, BPC8, BPC8, BPC8,
+		C0_G_Y, C2_R_Cr, C0_G_Y, C1_B_Cb,
+		false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV,
+		MDP_FETCH_LINEAR, 2),
+
 	/* 3 plane YUV */
-	FMT(YUV420,   0, 8, 8, 8,  2, 1, 0, 0,  false,  true,  1, 1,
-			MDP_PLANE_PLANAR, CHROMA_420, true),
-	FMT(YVU420,   0, 8, 8, 8,  1, 2, 0, 0,  false,  true,  1, 1,
-			MDP_PLANE_PLANAR, CHROMA_420, true),
+	PLANAR_YUV_FMT(YUV420,
+		0, BPC8, BPC8, BPC8,
+		C2_R_Cr, C1_B_Cb, C0_G_Y,
+		false, CHROMA_420, 1, MSM_FORMAT_FLAG_YUV,
+		MDP_FETCH_LINEAR, 3),
+
+	PLANAR_YUV_FMT(YVU420,
+		0, BPC8, BPC8, BPC8,
+		C1_B_Cb, C2_R_Cr, C0_G_Y,
+		false, CHROMA_420, 1, MSM_FORMAT_FLAG_YUV,
+		MDP_FETCH_LINEAR, 3),
 };
 
 /*
- * Note:
- * @rgb_only must be set to true, when requesting
- * supported formats for RGB pipes.
+ * UBWC formats table:
+ * This table holds the UBWC formats supported.
+ * If a compression ratio needs to be used for this or any other format,
+ * the data will be passed by user-space.
  */
-uint32_t mdp_get_formats(uint32_t *pixel_formats, uint32_t max_formats,
-		bool rgb_only)
-{
-	uint32_t i;
-	for (i = 0; i < ARRAY_SIZE(formats); i++) {
-		const struct mdp_format *f = &formats[i];
+static const struct msm_format mdp_formats_ubwc[] = {
+	INTERLEAVED_RGB_FMT_TILED(BGR565,
+		0, BPC5, BPC6, BPC5,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, 0, 3,
+		false, 2, MSM_FORMAT_FLAG_COMPRESSED,
+		MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC),
 
-		if (i == max_formats)
-			break;
+	INTERLEAVED_RGB_FMT_TILED(ABGR8888,
+		BPC8A, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, MSM_FORMAT_FLAG_COMPRESSED,
+		MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC),
 
-		if (rgb_only && MDP_FORMAT_IS_YUV(f))
-			break;
+	/* ARGB8888 and ABGR8888 purposely have the same color
+	 * ordering.  The hardware only supports ABGR8888 UBWC
+	 * natively.
+	 */
+	INTERLEAVED_RGB_FMT_TILED(ARGB8888,
+		BPC8A, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, MSM_FORMAT_FLAG_COMPRESSED,
+		MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC),
 
-		pixel_formats[i] = f->base.pixel_format;
-	}
+	INTERLEAVED_RGB_FMT_TILED(XBGR8888,
+		BPC8A, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		false, 4, MSM_FORMAT_FLAG_COMPRESSED,
+		MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC),
 
-	return i;
-}
+	INTERLEAVED_RGB_FMT_TILED(XRGB8888,
+		BPC8A, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		false, 4, MSM_FORMAT_FLAG_COMPRESSED,
+		MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC),
+
+	INTERLEAVED_RGB_FMT_TILED(ABGR2101010,
+		BPC8A, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, MSM_FORMAT_FLAG_DX | MSM_FORMAT_FLAG_COMPRESSED,
+		MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC),
+
+	INTERLEAVED_RGB_FMT_TILED(XBGR2101010,
+		BPC8A, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, MSM_FORMAT_FLAG_DX | MSM_FORMAT_FLAG_COMPRESSED,
+		MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC),
+
+	INTERLEAVED_RGB_FMT_TILED(XRGB2101010,
+		BPC8A, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, MSM_FORMAT_FLAG_DX | MSM_FORMAT_FLAG_COMPRESSED,
+		MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC),
+
+	/* XRGB2101010 and ARGB2101010 purposely have the same color
+	* ordering.  The hardware only supports ARGB2101010 UBWC
+	* natively.
+	*/
+	INTERLEAVED_RGB_FMT_TILED(ARGB2101010,
+		BPC8A, BPC8, BPC8, BPC8,
+		C2_R_Cr, C0_G_Y, C1_B_Cb, C3_ALPHA, 4,
+		true, 4, MSM_FORMAT_FLAG_DX | MSM_FORMAT_FLAG_COMPRESSED,
+		MDP_FETCH_UBWC, 2, MDP_TILE_HEIGHT_UBWC),
+
+	PSEUDO_YUV_FMT_TILED(NV12,
+		0, BPC8, BPC8, BPC8,
+		C1_B_Cb, C2_R_Cr,
+		CHROMA_420, MSM_FORMAT_FLAG_YUV |
+				MSM_FORMAT_FLAG_COMPRESSED,
+		MDP_FETCH_UBWC, 4, MDP_TILE_HEIGHT_NV12),
+
+	PSEUDO_YUV_FMT_TILED(P010,
+		0, BPC8, BPC8, BPC8,
+		C1_B_Cb, C2_R_Cr,
+		CHROMA_420, MSM_FORMAT_FLAG_DX |
+				MSM_FORMAT_FLAG_YUV |
+				MSM_FORMAT_FLAG_COMPRESSED,
+		MDP_FETCH_UBWC, 4, MDP_TILE_HEIGHT_UBWC),
+};
 
 const struct msm_format *mdp_get_format(struct msm_kms *kms, uint32_t format,
 		uint64_t modifier)
 {
+	const struct msm_format *map = NULL;
+	ssize_t map_size;
 	int i;
-	for (i = 0; i < ARRAY_SIZE(formats); i++) {
-		const struct mdp_format *f = &formats[i];
-		if (f->base.pixel_format == format)
-			return &f->base;
+
+	switch (modifier) {
+	case 0:
+		map = mdp_formats;
+		map_size = ARRAY_SIZE(mdp_formats);
+		break;
+	case DRM_FORMAT_MOD_QCOM_COMPRESSED:
+		map = mdp_formats_ubwc;
+		map_size = ARRAY_SIZE(mdp_formats_ubwc);
+		break;
+	default:
+		drm_err(kms->dev, "unsupported format modifier %llX\n", modifier);
+		return NULL;
 	}
+
+	for (i = 0; i < map_size; i++) {
+		const struct msm_format *f = &map[i];
+
+		if (f->pixel_format == format)
+			return f;
+	}
+
+	drm_err(kms->dev, "unsupported fmt: %p4cc modifier 0x%llX\n",
+		&format, modifier);
+
 	return NULL;
 }
 
diff --git a/drivers/gpu/drm/msm/disp/mdp_format.h b/drivers/gpu/drm/msm/disp/mdp_format.h
new file mode 100644
index 000000000000..a00d646ff4d4
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/mdp_format.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ */
+
+#ifndef __MSM_FORMAT_H__
+#define __MSM_FORMAT_H__
+
+#include "mdp_common.xml.h"
+
+enum msm_format_flags {
+	MSM_FORMAT_FLAG_YUV_BIT,
+	MSM_FORMAT_FLAG_DX_BIT,
+	MSM_FORMAT_FLAG_COMPRESSED_BIT,
+	MSM_FORMAT_FLAG_UNPACK_TIGHT_BIT,
+	MSM_FORMAT_FLAG_UNPACK_ALIGN_MSB_BIT,
+};
+
+#define MSM_FORMAT_FLAG_YUV		BIT(MSM_FORMAT_FLAG_YUV_BIT)
+#define MSM_FORMAT_FLAG_DX		BIT(MSM_FORMAT_FLAG_DX_BIT)
+#define MSM_FORMAT_FLAG_COMPRESSED	BIT(MSM_FORMAT_FLAG_COMPRESSED_BIT)
+#define MSM_FORMAT_FLAG_UNPACK_TIGHT	BIT(MSM_FORMAT_FLAG_UNPACK_TIGHT_BIT)
+#define MSM_FORMAT_FLAG_UNPACK_ALIGN_MSB BIT(MSM_FORMAT_FLAG_UNPACK_ALIGN_MSB_BIT)
+
+/**
+ * DPU HW,Component order color map
+ */
+enum {
+	C0_G_Y = 0,
+	C1_B_Cb = 1,
+	C2_R_Cr = 2,
+	C3_ALPHA = 3
+};
+
+/**
+ * struct msm_format: defines the format configuration
+ * @pixel_format: format fourcc
+ * @element: element color ordering
+ * @fetch_type: how the color components are packed in pixel format
+ * @chroma_sample: chroma sub-samplng type
+ * @alpha_enable: whether the format has an alpha channel
+ * @unpack_count: number of the components to unpack
+ * @bpp: bytes per pixel
+ * @flags: usage bit flags
+ * @num_planes: number of planes (including meta data planes)
+ * @fetch_mode: linear, tiled, or ubwc hw fetch behavior
+ * @tile_height: format tile height
+ */
+struct msm_format {
+	uint32_t pixel_format;
+	enum mdp_bpc bpc_g_y, bpc_b_cb, bpc_r_cr;
+	enum mdp_bpc_alpha bpc_a;
+	u8 element[4];
+	enum mdp_fetch_type fetch_type;
+	enum mdp_chroma_samp_type chroma_sample;
+	bool alpha_enable;
+	u8 unpack_count;
+	u8 bpp;
+	unsigned long flags;
+	u8 num_planes;
+	enum mdp_fetch_mode fetch_mode;
+	u16 tile_height;
+};
+
+#define MSM_FORMAT_IS_YUV(X)		((X)->flags & MSM_FORMAT_FLAG_YUV)
+#define MSM_FORMAT_IS_DX(X)		((X)->flags & MSM_FORMAT_FLAG_DX)
+#define MSM_FORMAT_IS_LINEAR(X)		((X)->fetch_mode == MDP_FETCH_LINEAR)
+#define MSM_FORMAT_IS_TILE(X) \
+	(((X)->fetch_mode == MDP_FETCH_UBWC) && \
+	 !((X)->flags & MSM_FORMAT_FLAG_COMPRESSED))
+#define MSM_FORMAT_IS_UBWC(X) \
+	(((X)->fetch_mode == MDP_FETCH_UBWC) && \
+	 ((X)->flags & MSM_FORMAT_FLAG_COMPRESSED))
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/mdp_kms.h b/drivers/gpu/drm/msm/disp/mdp_kms.h
index b0286d5d5130..068fbeac6edb 100644
--- a/drivers/gpu/drm/msm/disp/mdp_kms.h
+++ b/drivers/gpu/drm/msm/disp/mdp_kms.h
@@ -11,6 +11,7 @@
 #include <linux/platform_device.h>
 #include <linux/regulator/consumer.h>
 
+#include "mdp_format.h"
 #include "msm_drv.h"
 #include "msm_kms.h"
 #include "mdp_common.xml.h"
@@ -77,23 +78,6 @@ void mdp_irq_update(struct mdp_kms *mdp_kms);
  * pixel format helpers:
  */
 
-struct mdp_format {
-	struct msm_format base;
-	enum mdp_bpc bpc_r, bpc_g, bpc_b;
-	enum mdp_bpc_alpha bpc_a;
-	uint8_t unpack[4];
-	bool alpha_enable, unpack_tight;
-	uint8_t cpp, unpack_count;
-	enum mdp_fetch_type fetch_type;
-	enum mdp_chroma_samp_type chroma_sample;
-	bool is_yuv;
-};
-#define to_mdp_format(x) container_of(x, struct mdp_format, base)
-#define MDP_FORMAT_IS_YUV(mdp_format) ((mdp_format)->is_yuv)
-
-uint32_t mdp_get_formats(uint32_t *formats, uint32_t max_formats, bool rgb_only);
-const struct msm_format *mdp_get_format(struct msm_kms *kms, uint32_t format, uint64_t modifier);
-
 /* MDP capabilities */
 #define MDP_CAP_SMP		BIT(0)	/* Shared Memory Pool                 */
 #define MDP_CAP_DSC		BIT(1)	/* VESA Display Stream Compression    */
diff --git a/drivers/gpu/drm/msm/dp/dp_audio.c b/drivers/gpu/drm/msm/dp/dp_audio.c
index 7634e4b74208..a599fc5d63c5 100644
--- a/drivers/gpu/drm/msm/dp/dp_audio.c
+++ b/drivers/gpu/drm/msm/dp/dp_audio.c
@@ -22,9 +22,7 @@ struct dp_audio_private {
 	struct platform_device *pdev;
 	struct drm_device *drm_dev;
 	struct dp_catalog *catalog;
-	struct dp_panel *panel;
 
-	bool engine_on;
 	u32 channels;
 
 	struct dp_audio dp_audio;
@@ -34,11 +32,7 @@ static u32 dp_audio_get_header(struct dp_catalog *catalog,
 		enum dp_catalog_audio_sdp_type sdp,
 		enum dp_catalog_audio_header_type header)
 {
-	catalog->sdp_type = sdp;
-	catalog->sdp_header = header;
-	dp_catalog_audio_get_header(catalog);
-
-	return catalog->audio_data;
+	return dp_catalog_audio_get_header(catalog, sdp, header);
 }
 
 static void dp_audio_set_header(struct dp_catalog *catalog,
@@ -46,10 +40,7 @@ static void dp_audio_set_header(struct dp_catalog *catalog,
 		enum dp_catalog_audio_sdp_type sdp,
 		enum dp_catalog_audio_header_type header)
 {
-	catalog->sdp_type = sdp;
-	catalog->sdp_header = header;
-	catalog->audio_data = data;
-	dp_catalog_audio_set_header(catalog);
+	dp_catalog_audio_set_header(catalog, sdp, header, data);
 }
 
 static void dp_audio_stream_sdp(struct dp_audio_private *audio)
@@ -319,8 +310,7 @@ static void dp_audio_setup_acr(struct dp_audio_private *audio)
 		break;
 	}
 
-	catalog->audio_data = select;
-	dp_catalog_audio_config_acr(catalog);
+	dp_catalog_audio_config_acr(catalog, select);
 }
 
 static void dp_audio_safe_to_exit_level(struct dp_audio_private *audio)
@@ -346,18 +336,14 @@ static void dp_audio_safe_to_exit_level(struct dp_audio_private *audio)
 		break;
 	}
 
-	catalog->audio_data = safe_to_exit_level;
-	dp_catalog_audio_sfe_level(catalog);
+	dp_catalog_audio_sfe_level(catalog, safe_to_exit_level);
 }
 
 static void dp_audio_enable(struct dp_audio_private *audio, bool enable)
 {
 	struct dp_catalog *catalog = audio->catalog;
 
-	catalog->audio_data = enable;
-	dp_catalog_audio_enable(catalog);
-
-	audio->engine_on = enable;
+	dp_catalog_audio_enable(catalog, enable);
 }
 
 static struct dp_audio_private *dp_audio_get_data(struct platform_device *pdev)
@@ -571,7 +557,6 @@ struct dp_audio *dp_audio_get(struct platform_device *pdev,
 	}
 
 	audio->pdev = pdev;
-	audio->panel = panel;
 	audio->catalog = catalog;
 
 	dp_audio = &audio->dp_audio;
diff --git a/drivers/gpu/drm/msm/dp/dp_aux.c b/drivers/gpu/drm/msm/dp/dp_aux.c
index adbd5a367395..da46a433bf74 100644
--- a/drivers/gpu/drm/msm/dp/dp_aux.c
+++ b/drivers/gpu/drm/msm/dp/dp_aux.c
@@ -38,6 +38,7 @@ struct dp_aux_private {
 	bool no_send_stop;
 	bool initted;
 	bool is_edp;
+	bool enable_xfers;
 	u32 offset;
 	u32 segment;
 
@@ -87,8 +88,7 @@ static ssize_t dp_aux_write(struct dp_aux_private *aux,
 		/* index = 0, write */
 		if (i == 0)
 			reg |= DP_AUX_DATA_INDEX_WRITE;
-		aux->catalog->aux_data = reg;
-		dp_catalog_aux_write_data(aux->catalog);
+		dp_catalog_aux_write_data(aux->catalog, reg);
 	}
 
 	dp_catalog_aux_clear_trans(aux->catalog, false);
@@ -106,8 +106,7 @@ static ssize_t dp_aux_write(struct dp_aux_private *aux,
 	}
 
 	reg |= DP_AUX_TRANS_CTRL_GO;
-	aux->catalog->aux_data = reg;
-	dp_catalog_aux_write_trans(aux->catalog);
+	dp_catalog_aux_write_trans(aux->catalog, reg);
 
 	return len;
 }
@@ -145,8 +144,7 @@ static ssize_t dp_aux_cmd_fifo_rx(struct dp_aux_private *aux,
 	data = DP_AUX_DATA_INDEX_WRITE; /* INDEX_WRITE */
 	data |= DP_AUX_DATA_READ;  /* read */
 
-	aux->catalog->aux_data = data;
-	dp_catalog_aux_write_data(aux->catalog);
+	dp_catalog_aux_write_data(aux->catalog, data);
 
 	dp = msg->buffer;
 
@@ -305,19 +303,14 @@ static ssize_t dp_aux_transfer(struct drm_dp_aux *dp_aux,
 	}
 
 	/*
-	 * For eDP it's important to give a reasonably long wait here for HPD
-	 * to be asserted. This is because the panel driver may have _just_
-	 * turned on the panel and then tried to do an AUX transfer. The panel
-	 * driver has no way of knowing when the panel is ready, so it's up
-	 * to us to wait. For DP we never get into this situation so let's
-	 * avoid ever doing the extra long wait for DP.
+	 * If we're using DP and an external display isn't connected then the
+	 * transfer won't succeed. Return right away. If we don't do this we
+	 * can end up with long timeouts if someone tries to access the DP AUX
+	 * character device when no DP device is connected.
 	 */
-	if (aux->is_edp) {
-		ret = dp_catalog_aux_wait_for_hpd_connect_state(aux->catalog);
-		if (ret) {
-			DRM_DEBUG_DP("Panel not ready for aux transactions\n");
-			goto exit;
-		}
+	if (!aux->is_edp && !aux->enable_xfers) {
+		ret = -ENXIO;
+		goto exit;
 	}
 
 	dp_aux_update_offset_and_segment(aux, msg);
@@ -436,6 +429,14 @@ irqreturn_t dp_aux_isr(struct drm_dp_aux *dp_aux)
 	return IRQ_HANDLED;
 }
 
+void dp_aux_enable_xfers(struct drm_dp_aux *dp_aux, bool enabled)
+{
+	struct dp_aux_private *aux;
+
+	aux = container_of(dp_aux, struct dp_aux_private, dp_aux);
+	aux->enable_xfers = enabled;
+}
+
 void dp_aux_reconfig(struct drm_dp_aux *dp_aux)
 {
 	struct dp_aux_private *aux;
@@ -513,7 +514,7 @@ static int dp_wait_hpd_asserted(struct drm_dp_aux *dp_aux,
 	aux = container_of(dp_aux, struct dp_aux_private, dp_aux);
 
 	pm_runtime_get_sync(aux->dev);
-	ret = dp_catalog_aux_wait_for_hpd_connect_state(aux->catalog);
+	ret = dp_catalog_aux_wait_for_hpd_connect_state(aux->catalog, wait_us);
 	pm_runtime_put_sync(aux->dev);
 
 	return ret;
diff --git a/drivers/gpu/drm/msm/dp/dp_aux.h b/drivers/gpu/drm/msm/dp/dp_aux.h
index f47d591c1f54..4f65e892a807 100644
--- a/drivers/gpu/drm/msm/dp/dp_aux.h
+++ b/drivers/gpu/drm/msm/dp/dp_aux.h
@@ -12,6 +12,7 @@
 int dp_aux_register(struct drm_dp_aux *dp_aux);
 void dp_aux_unregister(struct drm_dp_aux *dp_aux);
 irqreturn_t dp_aux_isr(struct drm_dp_aux *dp_aux);
+void dp_aux_enable_xfers(struct drm_dp_aux *dp_aux, bool enabled);
 void dp_aux_init(struct drm_dp_aux *dp_aux);
 void dp_aux_deinit(struct drm_dp_aux *dp_aux);
 void dp_aux_reconfig(struct drm_dp_aux *dp_aux);
diff --git a/drivers/gpu/drm/msm/dp/dp_catalog.c b/drivers/gpu/drm/msm/dp/dp_catalog.c
index 3e7c84cdef47..6e55cbf69674 100644
--- a/drivers/gpu/drm/msm/dp/dp_catalog.c
+++ b/drivers/gpu/drm/msm/dp/dp_catalog.c
@@ -81,7 +81,6 @@ struct dp_catalog_private {
 	struct dss_io_data io;
 	u32 (*audio_map)[DP_AUDIO_SDP_HEADER_MAX];
 	struct dp_catalog dp_catalog;
-	u8 aux_lut_cfg_index[PHY_AUX_CFG_MAX];
 };
 
 void dp_catalog_snapshot(struct dp_catalog *dp_catalog, struct msm_disp_state *disp_state)
@@ -170,21 +169,21 @@ u32 dp_catalog_aux_read_data(struct dp_catalog *dp_catalog)
 	return dp_read_aux(catalog, REG_DP_AUX_DATA);
 }
 
-int dp_catalog_aux_write_data(struct dp_catalog *dp_catalog)
+int dp_catalog_aux_write_data(struct dp_catalog *dp_catalog, u32 data)
 {
 	struct dp_catalog_private *catalog = container_of(dp_catalog,
 				struct dp_catalog_private, dp_catalog);
 
-	dp_write_aux(catalog, REG_DP_AUX_DATA, dp_catalog->aux_data);
+	dp_write_aux(catalog, REG_DP_AUX_DATA, data);
 	return 0;
 }
 
-int dp_catalog_aux_write_trans(struct dp_catalog *dp_catalog)
+int dp_catalog_aux_write_trans(struct dp_catalog *dp_catalog, u32 data)
 {
 	struct dp_catalog_private *catalog = container_of(dp_catalog,
 				struct dp_catalog_private, dp_catalog);
 
-	dp_write_aux(catalog, REG_DP_AUX_TRANS_CTRL, dp_catalog->aux_data);
+	dp_write_aux(catalog, REG_DP_AUX_TRANS_CTRL, data);
 	return 0;
 }
 
@@ -263,17 +262,18 @@ void dp_catalog_aux_enable(struct dp_catalog *dp_catalog, bool enable)
 	dp_write_aux(catalog, REG_DP_AUX_CTRL, aux_ctrl);
 }
 
-int dp_catalog_aux_wait_for_hpd_connect_state(struct dp_catalog *dp_catalog)
+int dp_catalog_aux_wait_for_hpd_connect_state(struct dp_catalog *dp_catalog,
+					      unsigned long wait_us)
 {
 	u32 state;
 	struct dp_catalog_private *catalog = container_of(dp_catalog,
 				struct dp_catalog_private, dp_catalog);
 
-	/* poll for hpd connected status every 2ms and timeout after 500ms */
+	/* poll for hpd connected status every 2ms and timeout after wait_us */
 	return readl_poll_timeout(catalog->io.aux.base +
 				REG_DP_DP_HPD_INT_STATUS,
 				state, state & DP_DP_HPD_STATE_STATUS_CONNECTED,
-				2000, 500000);
+				min(wait_us, 2000), wait_us);
 }
 
 static void dump_regs(void __iomem *base, int len)
@@ -469,7 +469,7 @@ void dp_catalog_setup_peripheral_flush(struct dp_catalog *dp_catalog)
 
 void dp_catalog_ctrl_config_msa(struct dp_catalog *dp_catalog,
 					u32 rate, u32 stream_rate_khz,
-					bool fixed_nvid, bool is_ycbcr_420)
+					bool is_ycbcr_420)
 {
 	u32 pixel_m, pixel_n;
 	u32 mvid, nvid, pixel_div = 0, dispcc_input_rate;
@@ -881,19 +881,17 @@ u32 dp_catalog_ctrl_read_phy_pattern(struct dp_catalog *dp_catalog)
 }
 
 /* panel related catalog functions */
-int dp_catalog_panel_timing_cfg(struct dp_catalog *dp_catalog)
+int dp_catalog_panel_timing_cfg(struct dp_catalog *dp_catalog, u32 total,
+				u32 sync_start, u32 width_blanking, u32 dp_active)
 {
 	struct dp_catalog_private *catalog = container_of(dp_catalog,
 				struct dp_catalog_private, dp_catalog);
 	u32 reg;
 
-	dp_write_link(catalog, REG_DP_TOTAL_HOR_VER,
-				dp_catalog->total);
-	dp_write_link(catalog, REG_DP_START_HOR_VER_FROM_SYNC,
-				dp_catalog->sync_start);
-	dp_write_link(catalog, REG_DP_HSYNC_VSYNC_WIDTH_POLARITY,
-				dp_catalog->width_blanking);
-	dp_write_link(catalog, REG_DP_ACTIVE_HOR_VER, dp_catalog->dp_active);
+	dp_write_link(catalog, REG_DP_TOTAL_HOR_VER, total);
+	dp_write_link(catalog, REG_DP_START_HOR_VER_FROM_SYNC, sync_start);
+	dp_write_link(catalog, REG_DP_HSYNC_VSYNC_WIDTH_POLARITY, width_blanking);
+	dp_write_link(catalog, REG_DP_ACTIVE_HOR_VER, dp_active);
 
 	reg = dp_read_p0(catalog, MMSS_DP_INTF_CONFIG);
 
@@ -1162,34 +1160,28 @@ struct dp_catalog *dp_catalog_get(struct device *dev)
 	return &catalog->dp_catalog;
 }
 
-void dp_catalog_audio_get_header(struct dp_catalog *dp_catalog)
+u32 dp_catalog_audio_get_header(struct dp_catalog *dp_catalog,
+				enum dp_catalog_audio_sdp_type sdp,
+				enum dp_catalog_audio_header_type header)
 {
 	struct dp_catalog_private *catalog;
 	u32 (*sdp_map)[DP_AUDIO_SDP_HEADER_MAX];
-	enum dp_catalog_audio_sdp_type sdp;
-	enum dp_catalog_audio_header_type header;
-
-	if (!dp_catalog)
-		return;
 
 	catalog = container_of(dp_catalog,
 		struct dp_catalog_private, dp_catalog);
 
 	sdp_map = catalog->audio_map;
-	sdp     = dp_catalog->sdp_type;
-	header  = dp_catalog->sdp_header;
 
-	dp_catalog->audio_data = dp_read_link(catalog,
-			sdp_map[sdp][header]);
+	return dp_read_link(catalog, sdp_map[sdp][header]);
 }
 
-void dp_catalog_audio_set_header(struct dp_catalog *dp_catalog)
+void dp_catalog_audio_set_header(struct dp_catalog *dp_catalog,
+				 enum dp_catalog_audio_sdp_type sdp,
+				 enum dp_catalog_audio_header_type header,
+				 u32 data)
 {
 	struct dp_catalog_private *catalog;
 	u32 (*sdp_map)[DP_AUDIO_SDP_HEADER_MAX];
-	enum dp_catalog_audio_sdp_type sdp;
-	enum dp_catalog_audio_header_type header;
-	u32 data;
 
 	if (!dp_catalog)
 		return;
@@ -1198,17 +1190,14 @@ void dp_catalog_audio_set_header(struct dp_catalog *dp_catalog)
 		struct dp_catalog_private, dp_catalog);
 
 	sdp_map = catalog->audio_map;
-	sdp     = dp_catalog->sdp_type;
-	header  = dp_catalog->sdp_header;
-	data    = dp_catalog->audio_data;
 
 	dp_write_link(catalog, sdp_map[sdp][header], data);
 }
 
-void dp_catalog_audio_config_acr(struct dp_catalog *dp_catalog)
+void dp_catalog_audio_config_acr(struct dp_catalog *dp_catalog, u32 select)
 {
 	struct dp_catalog_private *catalog;
-	u32 acr_ctrl, select;
+	u32 acr_ctrl;
 
 	if (!dp_catalog)
 		return;
@@ -1216,7 +1205,6 @@ void dp_catalog_audio_config_acr(struct dp_catalog *dp_catalog)
 	catalog = container_of(dp_catalog,
 		struct dp_catalog_private, dp_catalog);
 
-	select = dp_catalog->audio_data;
 	acr_ctrl = select << 4 | BIT(31) | BIT(8) | BIT(14);
 
 	drm_dbg_dp(catalog->drm_dev, "select: %#x, acr_ctrl: %#x\n",
@@ -1225,10 +1213,9 @@ void dp_catalog_audio_config_acr(struct dp_catalog *dp_catalog)
 	dp_write_link(catalog, MMSS_DP_AUDIO_ACR_CTRL, acr_ctrl);
 }
 
-void dp_catalog_audio_enable(struct dp_catalog *dp_catalog)
+void dp_catalog_audio_enable(struct dp_catalog *dp_catalog, bool enable)
 {
 	struct dp_catalog_private *catalog;
-	bool enable;
 	u32 audio_ctrl;
 
 	if (!dp_catalog)
@@ -1237,7 +1224,6 @@ void dp_catalog_audio_enable(struct dp_catalog *dp_catalog)
 	catalog = container_of(dp_catalog,
 		struct dp_catalog_private, dp_catalog);
 
-	enable = !!dp_catalog->audio_data;
 	audio_ctrl = dp_read_link(catalog, MMSS_DP_AUDIO_CFG);
 
 	if (enable)
@@ -1332,10 +1318,10 @@ void dp_catalog_audio_init(struct dp_catalog *dp_catalog)
 	catalog->audio_map = sdp_map;
 }
 
-void dp_catalog_audio_sfe_level(struct dp_catalog *dp_catalog)
+void dp_catalog_audio_sfe_level(struct dp_catalog *dp_catalog, u32 safe_to_exit_level)
 {
 	struct dp_catalog_private *catalog;
-	u32 mainlink_levels, safe_to_exit_level;
+	u32 mainlink_levels;
 
 	if (!dp_catalog)
 		return;
@@ -1343,7 +1329,6 @@ void dp_catalog_audio_sfe_level(struct dp_catalog *dp_catalog)
 	catalog = container_of(dp_catalog,
 		struct dp_catalog_private, dp_catalog);
 
-	safe_to_exit_level = dp_catalog->audio_data;
 	mainlink_levels = dp_read_link(catalog, REG_DP_MAINLINK_LEVELS);
 	mainlink_levels &= 0xFE0;
 	mainlink_levels |= safe_to_exit_level;
diff --git a/drivers/gpu/drm/msm/dp/dp_catalog.h b/drivers/gpu/drm/msm/dp/dp_catalog.h
index 75ec290127c7..4679d50b8c73 100644
--- a/drivers/gpu/drm/msm/dp/dp_catalog.h
+++ b/drivers/gpu/drm/msm/dp/dp_catalog.h
@@ -28,26 +28,9 @@
 #define DP_INTR_FRAME_END		BIT(6)
 #define DP_INTR_CRC_UPDATED		BIT(9)
 
-#define DP_AUX_CFG_MAX_VALUE_CNT 3
-
 #define DP_HW_VERSION_1_0	0x10000000
 #define DP_HW_VERSION_1_2	0x10020000
 
-/* PHY AUX config registers */
-enum dp_phy_aux_config_type {
-	PHY_AUX_CFG0,
-	PHY_AUX_CFG1,
-	PHY_AUX_CFG2,
-	PHY_AUX_CFG3,
-	PHY_AUX_CFG4,
-	PHY_AUX_CFG5,
-	PHY_AUX_CFG6,
-	PHY_AUX_CFG7,
-	PHY_AUX_CFG8,
-	PHY_AUX_CFG9,
-	PHY_AUX_CFG_MAX,
-};
-
 enum dp_catalog_audio_sdp_type {
 	DP_AUDIO_SDP_STREAM,
 	DP_AUDIO_SDP_TIMESTAMP,
@@ -65,14 +48,6 @@ enum dp_catalog_audio_header_type {
 };
 
 struct dp_catalog {
-	u32 aux_data;
-	u32 total;
-	u32 sync_start;
-	u32 width_blanking;
-	u32 dp_active;
-	enum dp_catalog_audio_sdp_type sdp_type;
-	enum dp_catalog_audio_header_type sdp_header;
-	u32 audio_data;
 	bool wide_bus_en;
 };
 
@@ -81,13 +56,14 @@ void dp_catalog_snapshot(struct dp_catalog *dp_catalog, struct msm_disp_state *d
 
 /* AUX APIs */
 u32 dp_catalog_aux_read_data(struct dp_catalog *dp_catalog);
-int dp_catalog_aux_write_data(struct dp_catalog *dp_catalog);
-int dp_catalog_aux_write_trans(struct dp_catalog *dp_catalog);
+int dp_catalog_aux_write_data(struct dp_catalog *dp_catalog, u32 data);
+int dp_catalog_aux_write_trans(struct dp_catalog *dp_catalog, u32 data);
 int dp_catalog_aux_clear_trans(struct dp_catalog *dp_catalog, bool read);
 int dp_catalog_aux_clear_hw_interrupts(struct dp_catalog *dp_catalog);
 void dp_catalog_aux_reset(struct dp_catalog *dp_catalog);
 void dp_catalog_aux_enable(struct dp_catalog *dp_catalog, bool enable);
-int dp_catalog_aux_wait_for_hpd_connect_state(struct dp_catalog *dp_catalog);
+int dp_catalog_aux_wait_for_hpd_connect_state(struct dp_catalog *dp_catalog,
+					      unsigned long wait_us);
 u32 dp_catalog_aux_get_irq(struct dp_catalog *dp_catalog);
 
 /* DP Controller APIs */
@@ -99,7 +75,7 @@ void dp_catalog_ctrl_psr_mainlink_enable(struct dp_catalog *dp_catalog, bool ena
 void dp_catalog_setup_peripheral_flush(struct dp_catalog *dp_catalog);
 void dp_catalog_ctrl_config_misc(struct dp_catalog *dp_catalog, u32 cc, u32 tb);
 void dp_catalog_ctrl_config_msa(struct dp_catalog *dp_catalog, u32 rate,
-				u32 stream_rate_khz, bool fixed_nvid, bool is_ycbcr_420);
+				u32 stream_rate_khz, bool is_ycbcr_420);
 int dp_catalog_ctrl_set_pattern_state_bit(struct dp_catalog *dp_catalog, u32 pattern);
 u32 dp_catalog_hw_revision(const struct dp_catalog *dp_catalog);
 void dp_catalog_ctrl_reset(struct dp_catalog *dp_catalog);
@@ -124,7 +100,8 @@ void dp_catalog_ctrl_send_phy_pattern(struct dp_catalog *dp_catalog,
 u32 dp_catalog_ctrl_read_phy_pattern(struct dp_catalog *dp_catalog);
 
 /* DP Panel APIs */
-int dp_catalog_panel_timing_cfg(struct dp_catalog *dp_catalog);
+int dp_catalog_panel_timing_cfg(struct dp_catalog *dp_catalog, u32 total,
+				u32 sync_start, u32 width_blanking, u32 dp_active);
 void dp_catalog_panel_enable_vsc_sdp(struct dp_catalog *dp_catalog, struct dp_sdp *vsc_sdp);
 void dp_catalog_panel_disable_vsc_sdp(struct dp_catalog *dp_catalog);
 void dp_catalog_dump_regs(struct dp_catalog *dp_catalog);
@@ -135,12 +112,17 @@ void dp_catalog_panel_tpg_disable(struct dp_catalog *dp_catalog);
 struct dp_catalog *dp_catalog_get(struct device *dev);
 
 /* DP Audio APIs */
-void dp_catalog_audio_get_header(struct dp_catalog *catalog);
-void dp_catalog_audio_set_header(struct dp_catalog *catalog);
-void dp_catalog_audio_config_acr(struct dp_catalog *catalog);
-void dp_catalog_audio_enable(struct dp_catalog *catalog);
+u32 dp_catalog_audio_get_header(struct dp_catalog *dp_catalog,
+				enum dp_catalog_audio_sdp_type sdp,
+				enum dp_catalog_audio_header_type header);
+void dp_catalog_audio_set_header(struct dp_catalog *dp_catalog,
+				 enum dp_catalog_audio_sdp_type sdp,
+				 enum dp_catalog_audio_header_type header,
+				 u32 data);
+void dp_catalog_audio_config_acr(struct dp_catalog *catalog, u32 select);
+void dp_catalog_audio_enable(struct dp_catalog *catalog, bool enable);
 void dp_catalog_audio_config_sdp(struct dp_catalog *catalog);
 void dp_catalog_audio_init(struct dp_catalog *catalog);
-void dp_catalog_audio_sfe_level(struct dp_catalog *catalog);
+void dp_catalog_audio_sfe_level(struct dp_catalog *catalog, u32 safe_to_exit_level);
 
 #endif /* _DP_CATALOG_H_ */
diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c
index c4dda1faef67..7bc8a9f0657a 100644
--- a/drivers/gpu/drm/msm/dp/dp_ctrl.c
+++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c
@@ -1052,14 +1052,14 @@ static int dp_ctrl_update_vx_px(struct dp_ctrl_private *ctrl)
 	if (ret)
 		return ret;
 
-	if (voltage_swing_level >= DP_TRAIN_VOLTAGE_SWING_MAX) {
+	if (voltage_swing_level >= DP_TRAIN_LEVEL_MAX) {
 		drm_dbg_dp(ctrl->drm_dev,
 				"max. voltage swing level reached %d\n",
 				voltage_swing_level);
 		max_level_reached |= DP_TRAIN_MAX_SWING_REACHED;
 	}
 
-	if (pre_emphasis_level >= DP_TRAIN_PRE_EMPHASIS_MAX) {
+	if (pre_emphasis_level >= DP_TRAIN_LEVEL_MAX) {
 		drm_dbg_dp(ctrl->drm_dev,
 				"max. pre-emphasis level reached %d\n",
 				pre_emphasis_level);
@@ -1150,7 +1150,7 @@ static int dp_ctrl_link_train_1(struct dp_ctrl_private *ctrl,
 		}
 
 		if (ctrl->link->phy_params.v_level >=
-			DP_TRAIN_VOLTAGE_SWING_MAX) {
+			DP_TRAIN_LEVEL_MAX) {
 			DRM_ERROR_RATELIMITED("max v_level reached\n");
 			return -EAGAIN;
 		}
@@ -1566,21 +1566,6 @@ void dp_ctrl_phy_exit(struct dp_ctrl *dp_ctrl)
 			phy, phy->init_count, phy->power_count);
 }
 
-static bool dp_ctrl_use_fixed_nvid(struct dp_ctrl_private *ctrl)
-{
-	const u8 *dpcd = ctrl->panel->dpcd;
-
-	/*
-	 * For better interop experience, used a fixed NVID=0x8000
-	 * whenever connected to a VGA dongle downstream.
-	 */
-	if (drm_dp_is_branch(dpcd))
-		return (drm_dp_has_quirk(&ctrl->panel->desc,
-					 DP_DPCD_QUIRK_CONSTANT_N));
-
-	return false;
-}
-
 static int dp_ctrl_reinitialize_mainlink(struct dp_ctrl_private *ctrl)
 {
 	struct phy *phy = ctrl->phy;
@@ -2022,7 +2007,7 @@ int dp_ctrl_on_stream(struct dp_ctrl *dp_ctrl, bool force_link_train)
 
 	dp_catalog_ctrl_config_msa(ctrl->catalog,
 		ctrl->link->link_params.rate,
-		pixel_rate_orig, dp_ctrl_use_fixed_nvid(ctrl),
+		pixel_rate_orig,
 		ctrl->panel->dp_mode.out_fmt_is_yuv_420);
 
 	dp_ctrl_setup_tr_unit(ctrl);
diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.h b/drivers/gpu/drm/msm/dp/dp_ctrl.h
index fa014cee7e21..ffcbd9a25748 100644
--- a/drivers/gpu/drm/msm/dp/dp_ctrl.h
+++ b/drivers/gpu/drm/msm/dp/dp_ctrl.h
@@ -12,7 +12,6 @@
 #include "dp_catalog.h"
 
 struct dp_ctrl {
-	atomic_t aborted;
 	bool wide_bus_en;
 };
 
diff --git a/drivers/gpu/drm/msm/dp/dp_debug.c b/drivers/gpu/drm/msm/dp/dp_debug.c
index eca5a02f9003..b8611f6d2296 100644
--- a/drivers/gpu/drm/msm/dp/dp_debug.c
+++ b/drivers/gpu/drm/msm/dp/dp_debug.c
@@ -21,8 +21,6 @@ struct dp_debug_private {
 	struct dp_link *link;
 	struct dp_panel *panel;
 	struct drm_connector *connector;
-
-	struct dp_debug dp_debug;
 };
 
 static int dp_debug_show(struct seq_file *seq, void *p)
@@ -199,10 +197,24 @@ static const struct file_operations test_active_fops = {
 	.write = dp_test_active_write
 };
 
-static void dp_debug_init(struct dp_debug *dp_debug, struct dentry *root, bool is_edp)
+int dp_debug_init(struct device *dev, struct dp_panel *panel,
+		  struct dp_link *link,
+		  struct drm_connector *connector,
+		  struct dentry *root, bool is_edp)
 {
-	struct dp_debug_private *debug = container_of(dp_debug,
-			struct dp_debug_private, dp_debug);
+	struct dp_debug_private *debug;
+
+	if (!dev || !panel || !link) {
+		DRM_ERROR("invalid input\n");
+		return -EINVAL;
+	}
+
+	debug = devm_kzalloc(dev, sizeof(*debug), GFP_KERNEL);
+	if (!debug)
+		return -ENOMEM;
+
+	debug->link = link;
+	debug->panel = panel;
 
 	debugfs_create_file("dp_debug", 0444, root,
 			debug, &dp_debug_fops);
@@ -220,41 +232,6 @@ static void dp_debug_init(struct dp_debug *dp_debug, struct dentry *root, bool i
 				    root,
 				    debug, &dp_test_type_fops);
 	}
-}
 
-struct dp_debug *dp_debug_get(struct device *dev, struct dp_panel *panel,
-		struct dp_link *link,
-		struct drm_connector *connector,
-		struct dentry *root, bool is_edp)
-{
-	struct dp_debug_private *debug;
-	struct dp_debug *dp_debug;
-	int rc;
-
-	if (!dev || !panel || !link) {
-		DRM_ERROR("invalid input\n");
-		rc = -EINVAL;
-		goto error;
-	}
-
-	debug = devm_kzalloc(dev, sizeof(*debug), GFP_KERNEL);
-	if (!debug) {
-		rc = -ENOMEM;
-		goto error;
-	}
-
-	debug->dp_debug.debug_en = false;
-	debug->link = link;
-	debug->panel = panel;
-
-	dp_debug = &debug->dp_debug;
-	dp_debug->vdisplay = 0;
-	dp_debug->hdisplay = 0;
-	dp_debug->vrefresh = 0;
-
-	dp_debug_init(dp_debug, root, is_edp);
-
-	return dp_debug;
- error:
-	return ERR_PTR(rc);
+	return 0;
 }
diff --git a/drivers/gpu/drm/msm/dp/dp_debug.h b/drivers/gpu/drm/msm/dp/dp_debug.h
index 9b3b2e702f65..7e1aa892fc09 100644
--- a/drivers/gpu/drm/msm/dp/dp_debug.h
+++ b/drivers/gpu/drm/msm/dp/dp_debug.h
@@ -9,22 +9,6 @@
 #include "dp_panel.h"
 #include "dp_link.h"
 
-/**
- * struct dp_debug
- * @debug_en: specifies whether debug mode enabled
- * @vdisplay: used to filter out vdisplay value
- * @hdisplay: used to filter out hdisplay value
- * @vrefresh: used to filter out vrefresh value
- * @tpg_state: specifies whether tpg feature is enabled
- */
-struct dp_debug {
-	bool debug_en;
-	int aspect_ratio;
-	int vdisplay;
-	int hdisplay;
-	int vrefresh;
-};
-
 #if defined(CONFIG_DEBUG_FS)
 
 /**
@@ -41,22 +25,22 @@ struct dp_debug {
  * This function sets up the debug module and provides a way
  * for debugfs input to be communicated with existing modules
  */
-struct dp_debug *dp_debug_get(struct device *dev, struct dp_panel *panel,
-		struct dp_link *link,
-		struct drm_connector *connector,
-		struct dentry *root,
-		bool is_edp);
+int dp_debug_init(struct device *dev, struct dp_panel *panel,
+		  struct dp_link *link,
+		  struct drm_connector *connector,
+		  struct dentry *root,
+		  bool is_edp);
 
 #else
 
 static inline
-struct dp_debug *dp_debug_get(struct device *dev, struct dp_panel *panel,
-		struct dp_link *link,
-		struct drm_connector *connector,
-		struct dentry *root,
-		bool is_edp)
+int dp_debug_init(struct device *dev, struct dp_panel *panel,
+		  struct dp_link *link,
+		  struct drm_connector *connector,
+		  struct dentry *root,
+		  bool is_edp)
 {
-	return ERR_PTR(-EINVAL);
+	return -EINVAL;
 }
 
 #endif /* defined(CONFIG_DEBUG_FS) */
diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c
index ffbfde922589..672a7ba52eda 100644
--- a/drivers/gpu/drm/msm/dp/dp_display.c
+++ b/drivers/gpu/drm/msm/dp/dp_display.c
@@ -74,7 +74,6 @@ struct dp_event {
 };
 
 struct dp_display_private {
-	char *name;
 	int irq;
 
 	unsigned int id;
@@ -82,18 +81,15 @@ struct dp_display_private {
 	/* state variables */
 	bool core_initialized;
 	bool phy_initialized;
-	bool hpd_irq_on;
 	bool audio_supported;
 
 	struct drm_device *drm_dev;
-	struct dentry *root;
 
 	struct dp_catalog *catalog;
 	struct drm_dp_aux *aux;
 	struct dp_link    *link;
 	struct dp_panel   *panel;
 	struct dp_ctrl    *ctrl;
-	struct dp_debug   *debug;
 
 	struct dp_display_mode dp_mode;
 	struct msm_dp dp_display;
@@ -119,55 +115,49 @@ struct dp_display_private {
 struct msm_dp_desc {
 	phys_addr_t io_start;
 	unsigned int id;
-	unsigned int connector_type;
 	bool wide_bus_supported;
 };
 
 static const struct msm_dp_desc sc7180_dp_descs[] = {
-	{ .io_start = 0x0ae90000, .id = MSM_DP_CONTROLLER_0, .connector_type = DRM_MODE_CONNECTOR_DisplayPort },
+	{ .io_start = 0x0ae90000, .id = MSM_DP_CONTROLLER_0 },
 	{}
 };
 
 static const struct msm_dp_desc sc7280_dp_descs[] = {
-	{ .io_start = 0x0ae90000, .id = MSM_DP_CONTROLLER_0, .connector_type = DRM_MODE_CONNECTOR_DisplayPort, .wide_bus_supported = true },
-	{ .io_start = 0x0aea0000, .id = MSM_DP_CONTROLLER_1, .connector_type = DRM_MODE_CONNECTOR_eDP, .wide_bus_supported = true },
+	{ .io_start = 0x0ae90000, .id = MSM_DP_CONTROLLER_0, .wide_bus_supported = true },
+	{ .io_start = 0x0aea0000, .id = MSM_DP_CONTROLLER_1, .wide_bus_supported = true },
 	{}
 };
 
 static const struct msm_dp_desc sc8180x_dp_descs[] = {
-	{ .io_start = 0x0ae90000, .id = MSM_DP_CONTROLLER_0, .connector_type = DRM_MODE_CONNECTOR_DisplayPort },
-	{ .io_start = 0x0ae98000, .id = MSM_DP_CONTROLLER_1, .connector_type = DRM_MODE_CONNECTOR_DisplayPort },
-	{ .io_start = 0x0ae9a000, .id = MSM_DP_CONTROLLER_2, .connector_type = DRM_MODE_CONNECTOR_eDP },
+	{ .io_start = 0x0ae90000, .id = MSM_DP_CONTROLLER_0 },
+	{ .io_start = 0x0ae98000, .id = MSM_DP_CONTROLLER_1 },
+	{ .io_start = 0x0ae9a000, .id = MSM_DP_CONTROLLER_2 },
 	{}
 };
 
 static const struct msm_dp_desc sc8280xp_dp_descs[] = {
-	{ .io_start = 0x0ae90000, .id = MSM_DP_CONTROLLER_0, .connector_type = DRM_MODE_CONNECTOR_DisplayPort, .wide_bus_supported = true },
-	{ .io_start = 0x0ae98000, .id = MSM_DP_CONTROLLER_1, .connector_type = DRM_MODE_CONNECTOR_DisplayPort, .wide_bus_supported = true },
-	{ .io_start = 0x0ae9a000, .id = MSM_DP_CONTROLLER_2, .connector_type = DRM_MODE_CONNECTOR_DisplayPort, .wide_bus_supported = true },
-	{ .io_start = 0x0aea0000, .id = MSM_DP_CONTROLLER_3, .connector_type = DRM_MODE_CONNECTOR_DisplayPort, .wide_bus_supported = true },
-	{ .io_start = 0x22090000, .id = MSM_DP_CONTROLLER_0, .connector_type = DRM_MODE_CONNECTOR_DisplayPort, .wide_bus_supported = true },
-	{ .io_start = 0x22098000, .id = MSM_DP_CONTROLLER_1, .connector_type = DRM_MODE_CONNECTOR_DisplayPort, .wide_bus_supported = true },
-	{ .io_start = 0x2209a000, .id = MSM_DP_CONTROLLER_2, .connector_type = DRM_MODE_CONNECTOR_DisplayPort, .wide_bus_supported = true },
-	{ .io_start = 0x220a0000, .id = MSM_DP_CONTROLLER_3, .connector_type = DRM_MODE_CONNECTOR_DisplayPort, .wide_bus_supported = true },
+	{ .io_start = 0x0ae90000, .id = MSM_DP_CONTROLLER_0, .wide_bus_supported = true },
+	{ .io_start = 0x0ae98000, .id = MSM_DP_CONTROLLER_1, .wide_bus_supported = true },
+	{ .io_start = 0x0ae9a000, .id = MSM_DP_CONTROLLER_2, .wide_bus_supported = true },
+	{ .io_start = 0x0aea0000, .id = MSM_DP_CONTROLLER_3, .wide_bus_supported = true },
+	{ .io_start = 0x22090000, .id = MSM_DP_CONTROLLER_0, .wide_bus_supported = true },
+	{ .io_start = 0x22098000, .id = MSM_DP_CONTROLLER_1, .wide_bus_supported = true },
+	{ .io_start = 0x2209a000, .id = MSM_DP_CONTROLLER_2, .wide_bus_supported = true },
+	{ .io_start = 0x220a0000, .id = MSM_DP_CONTROLLER_3, .wide_bus_supported = true },
 	{}
 };
 
-static const struct msm_dp_desc sc8280xp_edp_descs[] = {
-	{ .io_start = 0x0ae9a000, .id = MSM_DP_CONTROLLER_2, .connector_type = DRM_MODE_CONNECTOR_eDP, .wide_bus_supported = true },
-	{ .io_start = 0x0aea0000, .id = MSM_DP_CONTROLLER_3, .connector_type = DRM_MODE_CONNECTOR_eDP, .wide_bus_supported = true },
-	{ .io_start = 0x2209a000, .id = MSM_DP_CONTROLLER_2, .connector_type = DRM_MODE_CONNECTOR_eDP, .wide_bus_supported = true },
-	{ .io_start = 0x220a0000, .id = MSM_DP_CONTROLLER_3, .connector_type = DRM_MODE_CONNECTOR_eDP, .wide_bus_supported = true },
-	{}
-};
-
-static const struct msm_dp_desc sm8350_dp_descs[] = {
-	{ .io_start = 0x0ae90000, .id = MSM_DP_CONTROLLER_0, .connector_type = DRM_MODE_CONNECTOR_DisplayPort },
+static const struct msm_dp_desc sm8650_dp_descs[] = {
+	{ .io_start = 0x0af54000, .id = MSM_DP_CONTROLLER_0 },
 	{}
 };
 
-static const struct msm_dp_desc sm8650_dp_descs[] = {
-	{ .io_start = 0x0af54000, .id = MSM_DP_CONTROLLER_0, .connector_type = DRM_MODE_CONNECTOR_DisplayPort },
+static const struct msm_dp_desc x1e80100_dp_descs[] = {
+	{ .io_start = 0x0ae90000, .id = MSM_DP_CONTROLLER_0, .wide_bus_supported = true },
+	{ .io_start = 0x0ae98000, .id = MSM_DP_CONTROLLER_1, .wide_bus_supported = true },
+	{ .io_start = 0x0ae9a000, .id = MSM_DP_CONTROLLER_2, .wide_bus_supported = true },
+	{ .io_start = 0x0aea0000, .id = MSM_DP_CONTROLLER_3, .wide_bus_supported = true },
 	{}
 };
 
@@ -178,10 +168,11 @@ static const struct of_device_id dp_dt_match[] = {
 	{ .compatible = "qcom,sc8180x-dp", .data = &sc8180x_dp_descs },
 	{ .compatible = "qcom,sc8180x-edp", .data = &sc8180x_dp_descs },
 	{ .compatible = "qcom,sc8280xp-dp", .data = &sc8280xp_dp_descs },
-	{ .compatible = "qcom,sc8280xp-edp", .data = &sc8280xp_edp_descs },
+	{ .compatible = "qcom,sc8280xp-edp", .data = &sc8280xp_dp_descs },
 	{ .compatible = "qcom,sdm845-dp", .data = &sc7180_dp_descs },
-	{ .compatible = "qcom,sm8350-dp", .data = &sm8350_dp_descs },
+	{ .compatible = "qcom,sm8350-dp", .data = &sc7180_dp_descs },
 	{ .compatible = "qcom,sm8650-dp", .data = &sm8650_dp_descs },
+	{ .compatible = "qcom,x1e80100-dp", .data = &x1e80100_dp_descs },
 	{}
 };
 
@@ -555,6 +546,8 @@ static int dp_hpd_plug_handle(struct dp_display_private *dp, u32 data)
 	int ret;
 	struct platform_device *pdev = dp->dp_display.pdev;
 
+	dp_aux_enable_xfers(dp->aux, true);
+
 	mutex_lock(&dp->event_mutex);
 
 	state =  dp->hpd_state;
@@ -620,6 +613,8 @@ static int dp_hpd_unplug_handle(struct dp_display_private *dp, u32 data)
 	u32 state;
 	struct platform_device *pdev = dp->dp_display.pdev;
 
+	dp_aux_enable_xfers(dp->aux, false);
+
 	mutex_lock(&dp->event_mutex);
 
 	state = dp->hpd_state;
@@ -728,6 +723,14 @@ static int dp_init_sub_modules(struct dp_display_private *dp)
 	if (IS_ERR(phy))
 		return PTR_ERR(phy);
 
+	rc = phy_set_mode_ext(phy, PHY_MODE_DP,
+			      dp->dp_display.is_edp ? PHY_SUBMODE_EDP : PHY_SUBMODE_DP);
+	if (rc) {
+		DRM_ERROR("failed to set phy submode, rc = %d\n", rc);
+		dp->catalog = NULL;
+		goto error;
+	}
+
 	dp->catalog = dp_catalog_get(dev);
 	if (IS_ERR(dp->catalog)) {
 		rc = PTR_ERR(dp->catalog);
@@ -803,7 +806,6 @@ static int dp_display_set_mode(struct msm_dp *dp_display,
 
 	drm_mode_copy(&dp->panel->dp_mode.drm_mode, &mode->drm_mode);
 	dp->panel->dp_mode.bpp = mode->bpp;
-	dp->panel->dp_mode.capabilities = mode->capabilities;
 	dp->panel->dp_mode.out_fmt_is_yuv_420 = mode->out_fmt_is_yuv_420;
 	dp_panel_init_panel_info(dp->panel);
 	return 0;
@@ -1243,6 +1245,25 @@ static int dp_auxbus_done_probe(struct drm_dp_aux *aux)
 	return dp_display_probe_tail(aux->dev);
 }
 
+static int dp_display_get_connector_type(struct platform_device *pdev,
+					 const struct msm_dp_desc *desc)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct device_node *aux_bus = of_get_child_by_name(node, "aux-bus");
+	struct device_node *panel = of_get_child_by_name(aux_bus, "panel");
+	int connector_type;
+
+	if (panel)
+		connector_type = DRM_MODE_CONNECTOR_eDP;
+	else
+		connector_type = DRM_MODE_SUBCONNECTOR_DisplayPort;
+
+	of_node_put(panel);
+	of_node_put(aux_bus);
+
+	return connector_type;
+}
+
 static int dp_display_probe(struct platform_device *pdev)
 {
 	int rc = 0;
@@ -1263,9 +1284,8 @@ static int dp_display_probe(struct platform_device *pdev)
 		return -EINVAL;
 
 	dp->dp_display.pdev = pdev;
-	dp->name = "drm_dp";
 	dp->id = desc->id;
-	dp->dp_display.connector_type = desc->connector_type;
+	dp->dp_display.connector_type = dp_display_get_connector_type(pdev, desc);
 	dp->wide_bus_supported = desc->wide_bus_supported;
 	dp->dp_display.is_edp =
 		(dp->dp_display.connector_type == DRM_MODE_CONNECTOR_eDP);
@@ -1433,14 +1453,9 @@ void dp_display_debugfs_init(struct msm_dp *dp_display, struct dentry *root, boo
 	dp = container_of(dp_display, struct dp_display_private, dp_display);
 	dev = &dp->dp_display.pdev->dev;
 
-	dp->debug = dp_debug_get(dev, dp->panel,
-					dp->link, dp->dp_display.connector,
-					root, is_edp);
-	if (IS_ERR(dp->debug)) {
-		rc = PTR_ERR(dp->debug);
+	rc = dp_debug_init(dev, dp->panel, dp->link, dp->dp_display.connector, root, is_edp);
+	if (rc)
 		DRM_ERROR("failed to initialize debug, rc = %d\n", rc);
-		dp->debug = NULL;
-	}
 }
 
 int msm_dp_modeset_init(struct msm_dp *dp_display, struct drm_device *dev,
diff --git a/drivers/gpu/drm/msm/dp/dp_display.h b/drivers/gpu/drm/msm/dp/dp_display.h
index 234dada88687..ec7fa67e0569 100644
--- a/drivers/gpu/drm/msm/dp/dp_display.h
+++ b/drivers/gpu/drm/msm/dp/dp_display.h
@@ -16,7 +16,6 @@ struct msm_dp {
 	struct drm_device *drm_dev;
 	struct platform_device *pdev;
 	struct device *codec_dev;
-	struct drm_bridge *bridge;
 	struct drm_connector *connector;
 	struct drm_bridge *next_bridge;
 	bool link_ready;
@@ -28,8 +27,6 @@ struct msm_dp {
 
 	hdmi_codec_plugged_cb plugged_cb;
 
-	bool wide_bus_en;
-
 	struct dp_audio *dp_audio;
 	bool psr_supported;
 };
diff --git a/drivers/gpu/drm/msm/dp/dp_drm.c b/drivers/gpu/drm/msm/dp/dp_drm.c
index a819a4ff76a9..1b9be5bd97f1 100644
--- a/drivers/gpu/drm/msm/dp/dp_drm.c
+++ b/drivers/gpu/drm/msm/dp/dp_drm.c
@@ -347,8 +347,6 @@ int dp_bridge_init(struct msm_dp *dp_display, struct drm_device *dev,
 		}
 	}
 
-	dp_display->bridge = bridge;
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/msm/dp/dp_link.c b/drivers/gpu/drm/msm/dp/dp_link.c
index 49dfac1fd1ef..d8967615d84d 100644
--- a/drivers/gpu/drm/msm/dp/dp_link.c
+++ b/drivers/gpu/drm/msm/dp/dp_link.c
@@ -36,7 +36,6 @@ struct dp_link_request {
 
 struct dp_link_private {
 	u32 prev_sink_count;
-	struct device *dev;
 	struct drm_device *drm_dev;
 	struct drm_dp_aux *aux;
 	struct dp_link dp_link;
@@ -804,8 +803,6 @@ int dp_link_psm_config(struct dp_link *dp_link,
 	if (ret)
 		DRM_ERROR("Failed to %s low power mode\n", enable ?
 							"enter" : "exit");
-	else
-		dp_link->psm_enabled = enable;
 
 	mutex_unlock(&link->psm_mutex);
 	return ret;
@@ -1109,6 +1106,7 @@ int dp_link_get_colorimetry_config(struct dp_link *dp_link)
 int dp_link_adjust_levels(struct dp_link *dp_link, u8 *link_status)
 {
 	int i;
+	u8 max_p_level;
 	int v_max = 0, p_max = 0;
 	struct dp_link_private *link;
 
@@ -1140,30 +1138,29 @@ int dp_link_adjust_levels(struct dp_link *dp_link, u8 *link_status)
 	 * Adjust the voltage swing and pre-emphasis level combination to within
 	 * the allowable range.
 	 */
-	if (dp_link->phy_params.v_level > DP_TRAIN_VOLTAGE_SWING_MAX) {
+	if (dp_link->phy_params.v_level > DP_TRAIN_LEVEL_MAX) {
 		drm_dbg_dp(link->drm_dev,
 			"Requested vSwingLevel=%d, change to %d\n",
 			dp_link->phy_params.v_level,
-			DP_TRAIN_VOLTAGE_SWING_MAX);
-		dp_link->phy_params.v_level = DP_TRAIN_VOLTAGE_SWING_MAX;
+			DP_TRAIN_LEVEL_MAX);
+		dp_link->phy_params.v_level = DP_TRAIN_LEVEL_MAX;
 	}
 
-	if (dp_link->phy_params.p_level > DP_TRAIN_PRE_EMPHASIS_MAX) {
+	if (dp_link->phy_params.p_level > DP_TRAIN_LEVEL_MAX) {
 		drm_dbg_dp(link->drm_dev,
 			"Requested preEmphasisLevel=%d, change to %d\n",
 			dp_link->phy_params.p_level,
-			DP_TRAIN_PRE_EMPHASIS_MAX);
-		dp_link->phy_params.p_level = DP_TRAIN_PRE_EMPHASIS_MAX;
+			DP_TRAIN_LEVEL_MAX);
+		dp_link->phy_params.p_level = DP_TRAIN_LEVEL_MAX;
 	}
 
-	if ((dp_link->phy_params.p_level > DP_TRAIN_PRE_EMPHASIS_LVL_1)
-		&& (dp_link->phy_params.v_level ==
-			DP_TRAIN_VOLTAGE_SWING_LVL_2)) {
+	max_p_level = DP_TRAIN_LEVEL_MAX - dp_link->phy_params.v_level;
+	if (dp_link->phy_params.p_level > max_p_level) {
 		drm_dbg_dp(link->drm_dev,
 			"Requested preEmphasisLevel=%d, change to %d\n",
 			dp_link->phy_params.p_level,
-			DP_TRAIN_PRE_EMPHASIS_LVL_1);
-		dp_link->phy_params.p_level = DP_TRAIN_PRE_EMPHASIS_LVL_1;
+			max_p_level);
+		dp_link->phy_params.p_level = max_p_level;
 	}
 
 	drm_dbg_dp(link->drm_dev, "adjusted: v_level=%d, p_level=%d\n",
@@ -1226,7 +1223,6 @@ struct dp_link *dp_link_get(struct device *dev, struct drm_dp_aux *aux)
 	if (!link)
 		return ERR_PTR(-ENOMEM);
 
-	link->dev   = dev;
 	link->aux   = aux;
 
 	mutex_init(&link->psm_mutex);
diff --git a/drivers/gpu/drm/msm/dp/dp_link.h b/drivers/gpu/drm/msm/dp/dp_link.h
index 83da170bc56b..5846337bb56f 100644
--- a/drivers/gpu/drm/msm/dp/dp_link.h
+++ b/drivers/gpu/drm/msm/dp/dp_link.h
@@ -19,19 +19,7 @@ struct dp_link_info {
 	unsigned long capabilities;
 };
 
-enum dp_link_voltage_level {
-	DP_TRAIN_VOLTAGE_SWING_LVL_0	= 0,
-	DP_TRAIN_VOLTAGE_SWING_LVL_1	= 1,
-	DP_TRAIN_VOLTAGE_SWING_LVL_2	= 2,
-	DP_TRAIN_VOLTAGE_SWING_MAX	= DP_TRAIN_VOLTAGE_SWING_LVL_2,
-};
-
-enum dp_link_preemaphasis_level {
-	DP_TRAIN_PRE_EMPHASIS_LVL_0	= 0,
-	DP_TRAIN_PRE_EMPHASIS_LVL_1	= 1,
-	DP_TRAIN_PRE_EMPHASIS_LVL_2	= 2,
-	DP_TRAIN_PRE_EMPHASIS_MAX	= DP_TRAIN_PRE_EMPHASIS_LVL_2,
-};
+#define DP_TRAIN_LEVEL_MAX	3
 
 struct dp_link_test_video {
 	u32 test_video_pattern;
@@ -74,7 +62,6 @@ struct dp_link_phy_params {
 struct dp_link {
 	u32 sink_request;
 	u32 test_response;
-	bool psm_enabled;
 
 	u8 sink_count;
 	struct dp_link_test_video test_video;
diff --git a/drivers/gpu/drm/msm/dp/dp_panel.c b/drivers/gpu/drm/msm/dp/dp_panel.c
index 8e7069453952..07db8f37cd06 100644
--- a/drivers/gpu/drm/msm/dp/dp_panel.c
+++ b/drivers/gpu/drm/msm/dp/dp_panel.c
@@ -353,6 +353,10 @@ int dp_panel_timing_cfg(struct dp_panel *dp_panel)
 	struct dp_catalog *catalog;
 	struct dp_panel_private *panel;
 	struct drm_display_mode *drm_mode;
+	u32 width_blanking;
+	u32 sync_start;
+	u32 dp_active;
+	u32 total;
 
 	panel = container_of(dp_panel, struct dp_panel_private, dp_panel);
 	catalog = panel->catalog;
@@ -376,13 +380,13 @@ int dp_panel_timing_cfg(struct dp_panel *dp_panel)
 	data <<= 16;
 	data |= total_hor;
 
-	catalog->total = data;
+	total = data;
 
 	data = (drm_mode->vtotal - drm_mode->vsync_start);
 	data <<= 16;
 	data |= (drm_mode->htotal - drm_mode->hsync_start);
 
-	catalog->sync_start = data;
+	sync_start = data;
 
 	data = drm_mode->vsync_end - drm_mode->vsync_start;
 	data <<= 16;
@@ -390,15 +394,15 @@ int dp_panel_timing_cfg(struct dp_panel *dp_panel)
 	data |= drm_mode->hsync_end - drm_mode->hsync_start;
 	data |= (panel->dp_panel.dp_mode.h_active_low << 15);
 
-	catalog->width_blanking = data;
+	width_blanking = data;
 
 	data = drm_mode->vdisplay;
 	data <<= 16;
 	data |= drm_mode->hdisplay;
 
-	catalog->dp_active = data;
+	dp_active = data;
 
-	dp_catalog_panel_timing_cfg(catalog);
+	dp_catalog_panel_timing_cfg(catalog, total, sync_start, width_blanking, dp_active);
 
 	if (dp_panel->dp_mode.out_fmt_is_yuv_420)
 		dp_panel_setup_vsc_sdp_yuv_420(dp_panel);
diff --git a/drivers/gpu/drm/msm/dp/dp_panel.h b/drivers/gpu/drm/msm/dp/dp_panel.h
index e843f5062d1f..4ea42fa936ae 100644
--- a/drivers/gpu/drm/msm/dp/dp_panel.h
+++ b/drivers/gpu/drm/msm/dp/dp_panel.h
@@ -15,7 +15,6 @@ struct edid;
 
 struct dp_display_mode {
 	struct drm_display_mode drm_mode;
-	u32 capabilities;
 	u32 bpp;
 	u32 h_active_low;
 	u32 v_active_low;
@@ -40,7 +39,6 @@ struct dp_panel {
 	u8 downstream_ports[DP_MAX_DOWNSTREAM_PORTS];
 
 	struct dp_link_info link_info;
-	struct drm_dp_desc desc;
 	struct edid *edid;
 	struct drm_connector *connector;
 	struct dp_display_mode dp_mode;
@@ -48,7 +46,6 @@ struct dp_panel {
 	bool video_test;
 	bool vsc_sdp_supported;
 
-	u32 vic;
 	u32 max_dp_lanes;
 	u32 max_dp_link_rate;
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c
index 37c4c07005fe..efd7c23b662f 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.c
+++ b/drivers/gpu/drm/msm/dsi/dsi.c
@@ -120,6 +120,22 @@ static int dsi_bind(struct device *dev, struct device *master, void *data)
 	struct msm_drm_private *priv = dev_get_drvdata(master);
 	struct msm_dsi *msm_dsi = dev_get_drvdata(dev);
 
+	/*
+	 * Next bridge doesn't exist for the secondary DSI host in a bonded
+	 * pair.
+	 */
+	if (!msm_dsi_is_bonded_dsi(msm_dsi) ||
+	    msm_dsi_is_master_dsi(msm_dsi)) {
+		struct drm_bridge *ext_bridge;
+
+		ext_bridge = devm_drm_of_get_bridge(&msm_dsi->pdev->dev,
+						    msm_dsi->pdev->dev.of_node, 1, 0);
+		if (IS_ERR(ext_bridge))
+			return PTR_ERR(ext_bridge);
+
+		msm_dsi->next_bridge = ext_bridge;
+	}
+
 	priv->dsi[msm_dsi->id] = msm_dsi;
 
 	return 0;
@@ -216,7 +232,6 @@ void __exit msm_dsi_unregister(void)
 int msm_dsi_modeset_init(struct msm_dsi *msm_dsi, struct drm_device *dev,
 			 struct drm_encoder *encoder)
 {
-	struct drm_bridge *bridge;
 	int ret;
 
 	msm_dsi->dev = dev;
@@ -236,14 +251,7 @@ int msm_dsi_modeset_init(struct msm_dsi *msm_dsi, struct drm_device *dev,
 		return 0;
 	}
 
-	bridge = msm_dsi_manager_bridge_init(msm_dsi, encoder);
-	if (IS_ERR(bridge)) {
-		ret = PTR_ERR(bridge);
-		DRM_DEV_ERROR(dev->dev, "failed to create dsi bridge: %d\n", ret);
-		return ret;
-	}
-
-	ret = msm_dsi_manager_ext_bridge_init(msm_dsi->id, bridge);
+	ret = msm_dsi_manager_connector_init(msm_dsi, encoder);
 	if (ret) {
 		DRM_DEV_ERROR(dev->dev,
 			"failed to create dsi connector: %d\n", ret);
diff --git a/drivers/gpu/drm/msm/dsi/dsi.h b/drivers/gpu/drm/msm/dsi/dsi.h
index 2ad9a842c678..afc290408ba4 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.h
+++ b/drivers/gpu/drm/msm/dsi/dsi.h
@@ -38,6 +38,8 @@ struct msm_dsi {
 	struct mipi_dsi_host *host;
 	struct msm_dsi_phy *phy;
 
+	struct drm_bridge *next_bridge;
+
 	struct device *phy_dev;
 	bool phy_enabled;
 
@@ -45,9 +47,8 @@ struct msm_dsi {
 };
 
 /* dsi manager */
-struct drm_bridge *msm_dsi_manager_bridge_init(struct msm_dsi *msm_dsi,
-					       struct drm_encoder *encoder);
-int msm_dsi_manager_ext_bridge_init(u8 id, struct drm_bridge *int_bridge);
+int msm_dsi_manager_connector_init(struct msm_dsi *msm_dsi,
+				   struct drm_encoder *encoder);
 int msm_dsi_manager_cmd_xfer(int id, const struct mipi_dsi_msg *msg);
 bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 dma_base, u32 len);
 int msm_dsi_manager_register(struct msm_dsi *msm_dsi);
diff --git a/drivers/gpu/drm/msm/dsi/dsi.xml.h b/drivers/gpu/drm/msm/dsi/dsi.xml.h
deleted file mode 100644
index 2a7d980e12c3..000000000000
--- a/drivers/gpu/drm/msm/dsi/dsi.xml.h
+++ /dev/null
@@ -1,790 +0,0 @@
-#ifndef DSI_XML
-#define DSI_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-enum dsi_traffic_mode {
-	NON_BURST_SYNCH_PULSE = 0,
-	NON_BURST_SYNCH_EVENT = 1,
-	BURST_MODE = 2,
-};
-
-enum dsi_vid_dst_format {
-	VID_DST_FORMAT_RGB565 = 0,
-	VID_DST_FORMAT_RGB666 = 1,
-	VID_DST_FORMAT_RGB666_LOOSE = 2,
-	VID_DST_FORMAT_RGB888 = 3,
-};
-
-enum dsi_rgb_swap {
-	SWAP_RGB = 0,
-	SWAP_RBG = 1,
-	SWAP_BGR = 2,
-	SWAP_BRG = 3,
-	SWAP_GRB = 4,
-	SWAP_GBR = 5,
-};
-
-enum dsi_cmd_trigger {
-	TRIGGER_NONE = 0,
-	TRIGGER_SEOF = 1,
-	TRIGGER_TE = 2,
-	TRIGGER_SW = 4,
-	TRIGGER_SW_SEOF = 5,
-	TRIGGER_SW_TE = 6,
-};
-
-enum dsi_cmd_dst_format {
-	CMD_DST_FORMAT_RGB111 = 0,
-	CMD_DST_FORMAT_RGB332 = 3,
-	CMD_DST_FORMAT_RGB444 = 4,
-	CMD_DST_FORMAT_RGB565 = 6,
-	CMD_DST_FORMAT_RGB666 = 7,
-	CMD_DST_FORMAT_RGB888 = 8,
-};
-
-enum dsi_lane_swap {
-	LANE_SWAP_0123 = 0,
-	LANE_SWAP_3012 = 1,
-	LANE_SWAP_2301 = 2,
-	LANE_SWAP_1230 = 3,
-	LANE_SWAP_0321 = 4,
-	LANE_SWAP_1032 = 5,
-	LANE_SWAP_2103 = 6,
-	LANE_SWAP_3210 = 7,
-};
-
-enum video_config_bpp {
-	VIDEO_CONFIG_18BPP = 0,
-	VIDEO_CONFIG_24BPP = 1,
-};
-
-enum video_pattern_sel {
-	VID_PRBS = 0,
-	VID_INCREMENTAL = 1,
-	VID_FIXED = 2,
-	VID_MDSS_GENERAL_PATTERN = 3,
-};
-
-enum cmd_mdp_stream0_pattern_sel {
-	CMD_MDP_PRBS = 0,
-	CMD_MDP_INCREMENTAL = 1,
-	CMD_MDP_FIXED = 2,
-	CMD_MDP_MDSS_GENERAL_PATTERN = 3,
-};
-
-enum cmd_dma_pattern_sel {
-	CMD_DMA_PRBS = 0,
-	CMD_DMA_INCREMENTAL = 1,
-	CMD_DMA_FIXED = 2,
-	CMD_DMA_CUSTOM_PATTERN_DMA_FIFO = 3,
-};
-
-#define DSI_IRQ_CMD_DMA_DONE					0x00000001
-#define DSI_IRQ_MASK_CMD_DMA_DONE				0x00000002
-#define DSI_IRQ_CMD_MDP_DONE					0x00000100
-#define DSI_IRQ_MASK_CMD_MDP_DONE				0x00000200
-#define DSI_IRQ_VIDEO_DONE					0x00010000
-#define DSI_IRQ_MASK_VIDEO_DONE					0x00020000
-#define DSI_IRQ_BTA_DONE					0x00100000
-#define DSI_IRQ_MASK_BTA_DONE					0x00200000
-#define DSI_IRQ_ERROR						0x01000000
-#define DSI_IRQ_MASK_ERROR					0x02000000
-#define REG_DSI_6G_HW_VERSION					0x00000000
-#define DSI_6G_HW_VERSION_MAJOR__MASK				0xf0000000
-#define DSI_6G_HW_VERSION_MAJOR__SHIFT				28
-static inline uint32_t DSI_6G_HW_VERSION_MAJOR(uint32_t val)
-{
-	return ((val) << DSI_6G_HW_VERSION_MAJOR__SHIFT) & DSI_6G_HW_VERSION_MAJOR__MASK;
-}
-#define DSI_6G_HW_VERSION_MINOR__MASK				0x0fff0000
-#define DSI_6G_HW_VERSION_MINOR__SHIFT				16
-static inline uint32_t DSI_6G_HW_VERSION_MINOR(uint32_t val)
-{
-	return ((val) << DSI_6G_HW_VERSION_MINOR__SHIFT) & DSI_6G_HW_VERSION_MINOR__MASK;
-}
-#define DSI_6G_HW_VERSION_STEP__MASK				0x0000ffff
-#define DSI_6G_HW_VERSION_STEP__SHIFT				0
-static inline uint32_t DSI_6G_HW_VERSION_STEP(uint32_t val)
-{
-	return ((val) << DSI_6G_HW_VERSION_STEP__SHIFT) & DSI_6G_HW_VERSION_STEP__MASK;
-}
-
-#define REG_DSI_CTRL						0x00000000
-#define DSI_CTRL_ENABLE						0x00000001
-#define DSI_CTRL_VID_MODE_EN					0x00000002
-#define DSI_CTRL_CMD_MODE_EN					0x00000004
-#define DSI_CTRL_LANE0						0x00000010
-#define DSI_CTRL_LANE1						0x00000020
-#define DSI_CTRL_LANE2						0x00000040
-#define DSI_CTRL_LANE3						0x00000080
-#define DSI_CTRL_CLK_EN						0x00000100
-#define DSI_CTRL_ECC_CHECK					0x00100000
-#define DSI_CTRL_CRC_CHECK					0x01000000
-
-#define REG_DSI_STATUS0						0x00000004
-#define DSI_STATUS0_CMD_MODE_ENGINE_BUSY			0x00000001
-#define DSI_STATUS0_CMD_MODE_DMA_BUSY				0x00000002
-#define DSI_STATUS0_CMD_MODE_MDP_BUSY				0x00000004
-#define DSI_STATUS0_VIDEO_MODE_ENGINE_BUSY			0x00000008
-#define DSI_STATUS0_DSI_BUSY					0x00000010
-#define DSI_STATUS0_INTERLEAVE_OP_CONTENTION			0x80000000
-
-#define REG_DSI_FIFO_STATUS					0x00000008
-#define DSI_FIFO_STATUS_VIDEO_MDP_FIFO_OVERFLOW			0x00000001
-#define DSI_FIFO_STATUS_VIDEO_MDP_FIFO_UNDERFLOW		0x00000008
-#define DSI_FIFO_STATUS_CMD_MDP_FIFO_UNDERFLOW			0x00000080
-#define DSI_FIFO_STATUS_CMD_DMA_FIFO_RD_WATERMARK_REACH		0x00000100
-#define DSI_FIFO_STATUS_CMD_DMA_FIFO_WR_WATERMARK_REACH		0x00000200
-#define DSI_FIFO_STATUS_CMD_DMA_FIFO_UNDERFLOW			0x00000400
-#define DSI_FIFO_STATUS_DLN0_LP_FIFO_EMPTY			0x00001000
-#define DSI_FIFO_STATUS_DLN0_LP_FIFO_FULL			0x00002000
-#define DSI_FIFO_STATUS_DLN0_LP_FIFO_OVERFLOW			0x00004000
-#define DSI_FIFO_STATUS_DLN0_HS_FIFO_EMPTY			0x00010000
-#define DSI_FIFO_STATUS_DLN0_HS_FIFO_FULL			0x00020000
-#define DSI_FIFO_STATUS_DLN0_HS_FIFO_OVERFLOW			0x00040000
-#define DSI_FIFO_STATUS_DLN0_HS_FIFO_UNDERFLOW			0x00080000
-#define DSI_FIFO_STATUS_DLN1_HS_FIFO_EMPTY			0x00100000
-#define DSI_FIFO_STATUS_DLN1_HS_FIFO_FULL			0x00200000
-#define DSI_FIFO_STATUS_DLN1_HS_FIFO_OVERFLOW			0x00400000
-#define DSI_FIFO_STATUS_DLN1_HS_FIFO_UNDERFLOW			0x00800000
-#define DSI_FIFO_STATUS_DLN2_HS_FIFO_EMPTY			0x01000000
-#define DSI_FIFO_STATUS_DLN2_HS_FIFO_FULL			0x02000000
-#define DSI_FIFO_STATUS_DLN2_HS_FIFO_OVERFLOW			0x04000000
-#define DSI_FIFO_STATUS_DLN2_HS_FIFO_UNDERFLOW			0x08000000
-#define DSI_FIFO_STATUS_DLN3_HS_FIFO_EMPTY			0x10000000
-#define DSI_FIFO_STATUS_DLN3_HS_FIFO_FULL			0x20000000
-#define DSI_FIFO_STATUS_DLN3_HS_FIFO_OVERFLOW			0x40000000
-#define DSI_FIFO_STATUS_DLN3_HS_FIFO_UNDERFLOW			0x80000000
-
-#define REG_DSI_VID_CFG0					0x0000000c
-#define DSI_VID_CFG0_VIRT_CHANNEL__MASK				0x00000003
-#define DSI_VID_CFG0_VIRT_CHANNEL__SHIFT			0
-static inline uint32_t DSI_VID_CFG0_VIRT_CHANNEL(uint32_t val)
-{
-	return ((val) << DSI_VID_CFG0_VIRT_CHANNEL__SHIFT) & DSI_VID_CFG0_VIRT_CHANNEL__MASK;
-}
-#define DSI_VID_CFG0_DST_FORMAT__MASK				0x00000030
-#define DSI_VID_CFG0_DST_FORMAT__SHIFT				4
-static inline uint32_t DSI_VID_CFG0_DST_FORMAT(enum dsi_vid_dst_format val)
-{
-	return ((val) << DSI_VID_CFG0_DST_FORMAT__SHIFT) & DSI_VID_CFG0_DST_FORMAT__MASK;
-}
-#define DSI_VID_CFG0_TRAFFIC_MODE__MASK				0x00000300
-#define DSI_VID_CFG0_TRAFFIC_MODE__SHIFT			8
-static inline uint32_t DSI_VID_CFG0_TRAFFIC_MODE(enum dsi_traffic_mode val)
-{
-	return ((val) << DSI_VID_CFG0_TRAFFIC_MODE__SHIFT) & DSI_VID_CFG0_TRAFFIC_MODE__MASK;
-}
-#define DSI_VID_CFG0_BLLP_POWER_STOP				0x00001000
-#define DSI_VID_CFG0_EOF_BLLP_POWER_STOP			0x00008000
-#define DSI_VID_CFG0_HSA_POWER_STOP				0x00010000
-#define DSI_VID_CFG0_HBP_POWER_STOP				0x00100000
-#define DSI_VID_CFG0_HFP_POWER_STOP				0x01000000
-#define DSI_VID_CFG0_PULSE_MODE_HSA_HE				0x10000000
-
-#define REG_DSI_VID_CFG1					0x0000001c
-#define DSI_VID_CFG1_R_SEL					0x00000001
-#define DSI_VID_CFG1_G_SEL					0x00000010
-#define DSI_VID_CFG1_B_SEL					0x00000100
-#define DSI_VID_CFG1_RGB_SWAP__MASK				0x00007000
-#define DSI_VID_CFG1_RGB_SWAP__SHIFT				12
-static inline uint32_t DSI_VID_CFG1_RGB_SWAP(enum dsi_rgb_swap val)
-{
-	return ((val) << DSI_VID_CFG1_RGB_SWAP__SHIFT) & DSI_VID_CFG1_RGB_SWAP__MASK;
-}
-
-#define REG_DSI_ACTIVE_H					0x00000020
-#define DSI_ACTIVE_H_START__MASK				0x00000fff
-#define DSI_ACTIVE_H_START__SHIFT				0
-static inline uint32_t DSI_ACTIVE_H_START(uint32_t val)
-{
-	return ((val) << DSI_ACTIVE_H_START__SHIFT) & DSI_ACTIVE_H_START__MASK;
-}
-#define DSI_ACTIVE_H_END__MASK					0x0fff0000
-#define DSI_ACTIVE_H_END__SHIFT					16
-static inline uint32_t DSI_ACTIVE_H_END(uint32_t val)
-{
-	return ((val) << DSI_ACTIVE_H_END__SHIFT) & DSI_ACTIVE_H_END__MASK;
-}
-
-#define REG_DSI_ACTIVE_V					0x00000024
-#define DSI_ACTIVE_V_START__MASK				0x00000fff
-#define DSI_ACTIVE_V_START__SHIFT				0
-static inline uint32_t DSI_ACTIVE_V_START(uint32_t val)
-{
-	return ((val) << DSI_ACTIVE_V_START__SHIFT) & DSI_ACTIVE_V_START__MASK;
-}
-#define DSI_ACTIVE_V_END__MASK					0x0fff0000
-#define DSI_ACTIVE_V_END__SHIFT					16
-static inline uint32_t DSI_ACTIVE_V_END(uint32_t val)
-{
-	return ((val) << DSI_ACTIVE_V_END__SHIFT) & DSI_ACTIVE_V_END__MASK;
-}
-
-#define REG_DSI_TOTAL						0x00000028
-#define DSI_TOTAL_H_TOTAL__MASK					0x00000fff
-#define DSI_TOTAL_H_TOTAL__SHIFT				0
-static inline uint32_t DSI_TOTAL_H_TOTAL(uint32_t val)
-{
-	return ((val) << DSI_TOTAL_H_TOTAL__SHIFT) & DSI_TOTAL_H_TOTAL__MASK;
-}
-#define DSI_TOTAL_V_TOTAL__MASK					0x0fff0000
-#define DSI_TOTAL_V_TOTAL__SHIFT				16
-static inline uint32_t DSI_TOTAL_V_TOTAL(uint32_t val)
-{
-	return ((val) << DSI_TOTAL_V_TOTAL__SHIFT) & DSI_TOTAL_V_TOTAL__MASK;
-}
-
-#define REG_DSI_ACTIVE_HSYNC					0x0000002c
-#define DSI_ACTIVE_HSYNC_START__MASK				0x00000fff
-#define DSI_ACTIVE_HSYNC_START__SHIFT				0
-static inline uint32_t DSI_ACTIVE_HSYNC_START(uint32_t val)
-{
-	return ((val) << DSI_ACTIVE_HSYNC_START__SHIFT) & DSI_ACTIVE_HSYNC_START__MASK;
-}
-#define DSI_ACTIVE_HSYNC_END__MASK				0x0fff0000
-#define DSI_ACTIVE_HSYNC_END__SHIFT				16
-static inline uint32_t DSI_ACTIVE_HSYNC_END(uint32_t val)
-{
-	return ((val) << DSI_ACTIVE_HSYNC_END__SHIFT) & DSI_ACTIVE_HSYNC_END__MASK;
-}
-
-#define REG_DSI_ACTIVE_VSYNC_HPOS				0x00000030
-#define DSI_ACTIVE_VSYNC_HPOS_START__MASK			0x00000fff
-#define DSI_ACTIVE_VSYNC_HPOS_START__SHIFT			0
-static inline uint32_t DSI_ACTIVE_VSYNC_HPOS_START(uint32_t val)
-{
-	return ((val) << DSI_ACTIVE_VSYNC_HPOS_START__SHIFT) & DSI_ACTIVE_VSYNC_HPOS_START__MASK;
-}
-#define DSI_ACTIVE_VSYNC_HPOS_END__MASK				0x0fff0000
-#define DSI_ACTIVE_VSYNC_HPOS_END__SHIFT			16
-static inline uint32_t DSI_ACTIVE_VSYNC_HPOS_END(uint32_t val)
-{
-	return ((val) << DSI_ACTIVE_VSYNC_HPOS_END__SHIFT) & DSI_ACTIVE_VSYNC_HPOS_END__MASK;
-}
-
-#define REG_DSI_ACTIVE_VSYNC_VPOS				0x00000034
-#define DSI_ACTIVE_VSYNC_VPOS_START__MASK			0x00000fff
-#define DSI_ACTIVE_VSYNC_VPOS_START__SHIFT			0
-static inline uint32_t DSI_ACTIVE_VSYNC_VPOS_START(uint32_t val)
-{
-	return ((val) << DSI_ACTIVE_VSYNC_VPOS_START__SHIFT) & DSI_ACTIVE_VSYNC_VPOS_START__MASK;
-}
-#define DSI_ACTIVE_VSYNC_VPOS_END__MASK				0x0fff0000
-#define DSI_ACTIVE_VSYNC_VPOS_END__SHIFT			16
-static inline uint32_t DSI_ACTIVE_VSYNC_VPOS_END(uint32_t val)
-{
-	return ((val) << DSI_ACTIVE_VSYNC_VPOS_END__SHIFT) & DSI_ACTIVE_VSYNC_VPOS_END__MASK;
-}
-
-#define REG_DSI_CMD_DMA_CTRL					0x00000038
-#define DSI_CMD_DMA_CTRL_BROADCAST_EN				0x80000000
-#define DSI_CMD_DMA_CTRL_FROM_FRAME_BUFFER			0x10000000
-#define DSI_CMD_DMA_CTRL_LOW_POWER				0x04000000
-
-#define REG_DSI_CMD_CFG0					0x0000003c
-#define DSI_CMD_CFG0_DST_FORMAT__MASK				0x0000000f
-#define DSI_CMD_CFG0_DST_FORMAT__SHIFT				0
-static inline uint32_t DSI_CMD_CFG0_DST_FORMAT(enum dsi_cmd_dst_format val)
-{
-	return ((val) << DSI_CMD_CFG0_DST_FORMAT__SHIFT) & DSI_CMD_CFG0_DST_FORMAT__MASK;
-}
-#define DSI_CMD_CFG0_R_SEL					0x00000010
-#define DSI_CMD_CFG0_G_SEL					0x00000100
-#define DSI_CMD_CFG0_B_SEL					0x00001000
-#define DSI_CMD_CFG0_INTERLEAVE_MAX__MASK			0x00f00000
-#define DSI_CMD_CFG0_INTERLEAVE_MAX__SHIFT			20
-static inline uint32_t DSI_CMD_CFG0_INTERLEAVE_MAX(uint32_t val)
-{
-	return ((val) << DSI_CMD_CFG0_INTERLEAVE_MAX__SHIFT) & DSI_CMD_CFG0_INTERLEAVE_MAX__MASK;
-}
-#define DSI_CMD_CFG0_RGB_SWAP__MASK				0x00070000
-#define DSI_CMD_CFG0_RGB_SWAP__SHIFT				16
-static inline uint32_t DSI_CMD_CFG0_RGB_SWAP(enum dsi_rgb_swap val)
-{
-	return ((val) << DSI_CMD_CFG0_RGB_SWAP__SHIFT) & DSI_CMD_CFG0_RGB_SWAP__MASK;
-}
-
-#define REG_DSI_CMD_CFG1					0x00000040
-#define DSI_CMD_CFG1_WR_MEM_START__MASK				0x000000ff
-#define DSI_CMD_CFG1_WR_MEM_START__SHIFT			0
-static inline uint32_t DSI_CMD_CFG1_WR_MEM_START(uint32_t val)
-{
-	return ((val) << DSI_CMD_CFG1_WR_MEM_START__SHIFT) & DSI_CMD_CFG1_WR_MEM_START__MASK;
-}
-#define DSI_CMD_CFG1_WR_MEM_CONTINUE__MASK			0x0000ff00
-#define DSI_CMD_CFG1_WR_MEM_CONTINUE__SHIFT			8
-static inline uint32_t DSI_CMD_CFG1_WR_MEM_CONTINUE(uint32_t val)
-{
-	return ((val) << DSI_CMD_CFG1_WR_MEM_CONTINUE__SHIFT) & DSI_CMD_CFG1_WR_MEM_CONTINUE__MASK;
-}
-#define DSI_CMD_CFG1_INSERT_DCS_COMMAND				0x00010000
-
-#define REG_DSI_DMA_BASE					0x00000044
-
-#define REG_DSI_DMA_LEN						0x00000048
-
-#define REG_DSI_CMD_MDP_STREAM0_CTRL				0x00000054
-#define DSI_CMD_MDP_STREAM0_CTRL_DATA_TYPE__MASK		0x0000003f
-#define DSI_CMD_MDP_STREAM0_CTRL_DATA_TYPE__SHIFT		0
-static inline uint32_t DSI_CMD_MDP_STREAM0_CTRL_DATA_TYPE(uint32_t val)
-{
-	return ((val) << DSI_CMD_MDP_STREAM0_CTRL_DATA_TYPE__SHIFT) & DSI_CMD_MDP_STREAM0_CTRL_DATA_TYPE__MASK;
-}
-#define DSI_CMD_MDP_STREAM0_CTRL_VIRTUAL_CHANNEL__MASK		0x00000300
-#define DSI_CMD_MDP_STREAM0_CTRL_VIRTUAL_CHANNEL__SHIFT		8
-static inline uint32_t DSI_CMD_MDP_STREAM0_CTRL_VIRTUAL_CHANNEL(uint32_t val)
-{
-	return ((val) << DSI_CMD_MDP_STREAM0_CTRL_VIRTUAL_CHANNEL__SHIFT) & DSI_CMD_MDP_STREAM0_CTRL_VIRTUAL_CHANNEL__MASK;
-}
-#define DSI_CMD_MDP_STREAM0_CTRL_WORD_COUNT__MASK		0xffff0000
-#define DSI_CMD_MDP_STREAM0_CTRL_WORD_COUNT__SHIFT		16
-static inline uint32_t DSI_CMD_MDP_STREAM0_CTRL_WORD_COUNT(uint32_t val)
-{
-	return ((val) << DSI_CMD_MDP_STREAM0_CTRL_WORD_COUNT__SHIFT) & DSI_CMD_MDP_STREAM0_CTRL_WORD_COUNT__MASK;
-}
-
-#define REG_DSI_CMD_MDP_STREAM0_TOTAL				0x00000058
-#define DSI_CMD_MDP_STREAM0_TOTAL_H_TOTAL__MASK			0x00000fff
-#define DSI_CMD_MDP_STREAM0_TOTAL_H_TOTAL__SHIFT		0
-static inline uint32_t DSI_CMD_MDP_STREAM0_TOTAL_H_TOTAL(uint32_t val)
-{
-	return ((val) << DSI_CMD_MDP_STREAM0_TOTAL_H_TOTAL__SHIFT) & DSI_CMD_MDP_STREAM0_TOTAL_H_TOTAL__MASK;
-}
-#define DSI_CMD_MDP_STREAM0_TOTAL_V_TOTAL__MASK			0x0fff0000
-#define DSI_CMD_MDP_STREAM0_TOTAL_V_TOTAL__SHIFT		16
-static inline uint32_t DSI_CMD_MDP_STREAM0_TOTAL_V_TOTAL(uint32_t val)
-{
-	return ((val) << DSI_CMD_MDP_STREAM0_TOTAL_V_TOTAL__SHIFT) & DSI_CMD_MDP_STREAM0_TOTAL_V_TOTAL__MASK;
-}
-
-#define REG_DSI_CMD_MDP_STREAM1_CTRL				0x0000005c
-#define DSI_CMD_MDP_STREAM1_CTRL_DATA_TYPE__MASK		0x0000003f
-#define DSI_CMD_MDP_STREAM1_CTRL_DATA_TYPE__SHIFT		0
-static inline uint32_t DSI_CMD_MDP_STREAM1_CTRL_DATA_TYPE(uint32_t val)
-{
-	return ((val) << DSI_CMD_MDP_STREAM1_CTRL_DATA_TYPE__SHIFT) & DSI_CMD_MDP_STREAM1_CTRL_DATA_TYPE__MASK;
-}
-#define DSI_CMD_MDP_STREAM1_CTRL_VIRTUAL_CHANNEL__MASK		0x00000300
-#define DSI_CMD_MDP_STREAM1_CTRL_VIRTUAL_CHANNEL__SHIFT		8
-static inline uint32_t DSI_CMD_MDP_STREAM1_CTRL_VIRTUAL_CHANNEL(uint32_t val)
-{
-	return ((val) << DSI_CMD_MDP_STREAM1_CTRL_VIRTUAL_CHANNEL__SHIFT) & DSI_CMD_MDP_STREAM1_CTRL_VIRTUAL_CHANNEL__MASK;
-}
-#define DSI_CMD_MDP_STREAM1_CTRL_WORD_COUNT__MASK		0xffff0000
-#define DSI_CMD_MDP_STREAM1_CTRL_WORD_COUNT__SHIFT		16
-static inline uint32_t DSI_CMD_MDP_STREAM1_CTRL_WORD_COUNT(uint32_t val)
-{
-	return ((val) << DSI_CMD_MDP_STREAM1_CTRL_WORD_COUNT__SHIFT) & DSI_CMD_MDP_STREAM1_CTRL_WORD_COUNT__MASK;
-}
-
-#define REG_DSI_CMD_MDP_STREAM1_TOTAL				0x00000060
-#define DSI_CMD_MDP_STREAM1_TOTAL_H_TOTAL__MASK			0x0000ffff
-#define DSI_CMD_MDP_STREAM1_TOTAL_H_TOTAL__SHIFT		0
-static inline uint32_t DSI_CMD_MDP_STREAM1_TOTAL_H_TOTAL(uint32_t val)
-{
-	return ((val) << DSI_CMD_MDP_STREAM1_TOTAL_H_TOTAL__SHIFT) & DSI_CMD_MDP_STREAM1_TOTAL_H_TOTAL__MASK;
-}
-#define DSI_CMD_MDP_STREAM1_TOTAL_V_TOTAL__MASK			0xffff0000
-#define DSI_CMD_MDP_STREAM1_TOTAL_V_TOTAL__SHIFT		16
-static inline uint32_t DSI_CMD_MDP_STREAM1_TOTAL_V_TOTAL(uint32_t val)
-{
-	return ((val) << DSI_CMD_MDP_STREAM1_TOTAL_V_TOTAL__SHIFT) & DSI_CMD_MDP_STREAM1_TOTAL_V_TOTAL__MASK;
-}
-
-#define REG_DSI_ACK_ERR_STATUS					0x00000064
-
-static inline uint32_t REG_DSI_RDBK(uint32_t i0) { return 0x00000068 + 0x4*i0; }
-
-static inline uint32_t REG_DSI_RDBK_DATA(uint32_t i0) { return 0x00000068 + 0x4*i0; }
-
-#define REG_DSI_TRIG_CTRL					0x00000080
-#define DSI_TRIG_CTRL_DMA_TRIGGER__MASK				0x00000007
-#define DSI_TRIG_CTRL_DMA_TRIGGER__SHIFT			0
-static inline uint32_t DSI_TRIG_CTRL_DMA_TRIGGER(enum dsi_cmd_trigger val)
-{
-	return ((val) << DSI_TRIG_CTRL_DMA_TRIGGER__SHIFT) & DSI_TRIG_CTRL_DMA_TRIGGER__MASK;
-}
-#define DSI_TRIG_CTRL_MDP_TRIGGER__MASK				0x00000070
-#define DSI_TRIG_CTRL_MDP_TRIGGER__SHIFT			4
-static inline uint32_t DSI_TRIG_CTRL_MDP_TRIGGER(enum dsi_cmd_trigger val)
-{
-	return ((val) << DSI_TRIG_CTRL_MDP_TRIGGER__SHIFT) & DSI_TRIG_CTRL_MDP_TRIGGER__MASK;
-}
-#define DSI_TRIG_CTRL_STREAM__MASK				0x00000300
-#define DSI_TRIG_CTRL_STREAM__SHIFT				8
-static inline uint32_t DSI_TRIG_CTRL_STREAM(uint32_t val)
-{
-	return ((val) << DSI_TRIG_CTRL_STREAM__SHIFT) & DSI_TRIG_CTRL_STREAM__MASK;
-}
-#define DSI_TRIG_CTRL_BLOCK_DMA_WITHIN_FRAME			0x00001000
-#define DSI_TRIG_CTRL_TE					0x80000000
-
-#define REG_DSI_TRIG_DMA					0x0000008c
-
-#define REG_DSI_DLN0_PHY_ERR					0x000000b0
-#define DSI_DLN0_PHY_ERR_DLN0_ERR_ESC				0x00000001
-#define DSI_DLN0_PHY_ERR_DLN0_ERR_SYNC_ESC			0x00000010
-#define DSI_DLN0_PHY_ERR_DLN0_ERR_CONTROL			0x00000100
-#define DSI_DLN0_PHY_ERR_DLN0_ERR_CONTENTION_LP0		0x00001000
-#define DSI_DLN0_PHY_ERR_DLN0_ERR_CONTENTION_LP1		0x00010000
-
-#define REG_DSI_LP_TIMER_CTRL					0x000000b4
-#define DSI_LP_TIMER_CTRL_LP_RX_TO__MASK			0x0000ffff
-#define DSI_LP_TIMER_CTRL_LP_RX_TO__SHIFT			0
-static inline uint32_t DSI_LP_TIMER_CTRL_LP_RX_TO(uint32_t val)
-{
-	return ((val) << DSI_LP_TIMER_CTRL_LP_RX_TO__SHIFT) & DSI_LP_TIMER_CTRL_LP_RX_TO__MASK;
-}
-#define DSI_LP_TIMER_CTRL_BTA_TO__MASK				0xffff0000
-#define DSI_LP_TIMER_CTRL_BTA_TO__SHIFT				16
-static inline uint32_t DSI_LP_TIMER_CTRL_BTA_TO(uint32_t val)
-{
-	return ((val) << DSI_LP_TIMER_CTRL_BTA_TO__SHIFT) & DSI_LP_TIMER_CTRL_BTA_TO__MASK;
-}
-
-#define REG_DSI_HS_TIMER_CTRL					0x000000b8
-#define DSI_HS_TIMER_CTRL_HS_TX_TO__MASK			0x0000ffff
-#define DSI_HS_TIMER_CTRL_HS_TX_TO__SHIFT			0
-static inline uint32_t DSI_HS_TIMER_CTRL_HS_TX_TO(uint32_t val)
-{
-	return ((val) << DSI_HS_TIMER_CTRL_HS_TX_TO__SHIFT) & DSI_HS_TIMER_CTRL_HS_TX_TO__MASK;
-}
-#define DSI_HS_TIMER_CTRL_TIMER_RESOLUTION__MASK		0x000f0000
-#define DSI_HS_TIMER_CTRL_TIMER_RESOLUTION__SHIFT		16
-static inline uint32_t DSI_HS_TIMER_CTRL_TIMER_RESOLUTION(uint32_t val)
-{
-	return ((val) << DSI_HS_TIMER_CTRL_TIMER_RESOLUTION__SHIFT) & DSI_HS_TIMER_CTRL_TIMER_RESOLUTION__MASK;
-}
-#define DSI_HS_TIMER_CTRL_HS_TX_TO_STOP_EN			0x10000000
-
-#define REG_DSI_TIMEOUT_STATUS					0x000000bc
-
-#define REG_DSI_CLKOUT_TIMING_CTRL				0x000000c0
-#define DSI_CLKOUT_TIMING_CTRL_T_CLK_PRE__MASK			0x0000003f
-#define DSI_CLKOUT_TIMING_CTRL_T_CLK_PRE__SHIFT			0
-static inline uint32_t DSI_CLKOUT_TIMING_CTRL_T_CLK_PRE(uint32_t val)
-{
-	return ((val) << DSI_CLKOUT_TIMING_CTRL_T_CLK_PRE__SHIFT) & DSI_CLKOUT_TIMING_CTRL_T_CLK_PRE__MASK;
-}
-#define DSI_CLKOUT_TIMING_CTRL_T_CLK_POST__MASK			0x00003f00
-#define DSI_CLKOUT_TIMING_CTRL_T_CLK_POST__SHIFT		8
-static inline uint32_t DSI_CLKOUT_TIMING_CTRL_T_CLK_POST(uint32_t val)
-{
-	return ((val) << DSI_CLKOUT_TIMING_CTRL_T_CLK_POST__SHIFT) & DSI_CLKOUT_TIMING_CTRL_T_CLK_POST__MASK;
-}
-
-#define REG_DSI_EOT_PACKET_CTRL					0x000000c8
-#define DSI_EOT_PACKET_CTRL_TX_EOT_APPEND			0x00000001
-#define DSI_EOT_PACKET_CTRL_RX_EOT_IGNORE			0x00000010
-
-#define REG_DSI_LANE_STATUS					0x000000a4
-#define DSI_LANE_STATUS_DLN0_STOPSTATE				0x00000001
-#define DSI_LANE_STATUS_DLN1_STOPSTATE				0x00000002
-#define DSI_LANE_STATUS_DLN2_STOPSTATE				0x00000004
-#define DSI_LANE_STATUS_DLN3_STOPSTATE				0x00000008
-#define DSI_LANE_STATUS_CLKLN_STOPSTATE				0x00000010
-#define DSI_LANE_STATUS_DLN0_ULPS_ACTIVE_NOT			0x00000100
-#define DSI_LANE_STATUS_DLN1_ULPS_ACTIVE_NOT			0x00000200
-#define DSI_LANE_STATUS_DLN2_ULPS_ACTIVE_NOT			0x00000400
-#define DSI_LANE_STATUS_DLN3_ULPS_ACTIVE_NOT			0x00000800
-#define DSI_LANE_STATUS_CLKLN_ULPS_ACTIVE_NOT			0x00001000
-#define DSI_LANE_STATUS_DLN0_DIRECTION				0x00010000
-
-#define REG_DSI_LANE_CTRL					0x000000a8
-#define DSI_LANE_CTRL_HS_REQ_SEL_PHY				0x01000000
-#define DSI_LANE_CTRL_CLKLN_HS_FORCE_REQUEST			0x10000000
-
-#define REG_DSI_LANE_SWAP_CTRL					0x000000ac
-#define DSI_LANE_SWAP_CTRL_DLN_SWAP_SEL__MASK			0x00000007
-#define DSI_LANE_SWAP_CTRL_DLN_SWAP_SEL__SHIFT			0
-static inline uint32_t DSI_LANE_SWAP_CTRL_DLN_SWAP_SEL(enum dsi_lane_swap val)
-{
-	return ((val) << DSI_LANE_SWAP_CTRL_DLN_SWAP_SEL__SHIFT) & DSI_LANE_SWAP_CTRL_DLN_SWAP_SEL__MASK;
-}
-
-#define REG_DSI_ERR_INT_MASK0					0x00000108
-
-#define REG_DSI_INTR_CTRL					0x0000010c
-
-#define REG_DSI_RESET						0x00000114
-
-#define REG_DSI_CLK_CTRL					0x00000118
-#define DSI_CLK_CTRL_AHBS_HCLK_ON				0x00000001
-#define DSI_CLK_CTRL_AHBM_SCLK_ON				0x00000002
-#define DSI_CLK_CTRL_PCLK_ON					0x00000004
-#define DSI_CLK_CTRL_DSICLK_ON					0x00000008
-#define DSI_CLK_CTRL_BYTECLK_ON					0x00000010
-#define DSI_CLK_CTRL_ESCCLK_ON					0x00000020
-#define DSI_CLK_CTRL_FORCE_ON_DYN_AHBM_HCLK			0x00000200
-
-#define REG_DSI_CLK_STATUS					0x0000011c
-#define DSI_CLK_STATUS_DSI_AON_AHBM_HCLK_ACTIVE			0x00000001
-#define DSI_CLK_STATUS_DSI_DYN_AHBM_HCLK_ACTIVE			0x00000002
-#define DSI_CLK_STATUS_DSI_AON_AHBS_HCLK_ACTIVE			0x00000004
-#define DSI_CLK_STATUS_DSI_DYN_AHBS_HCLK_ACTIVE			0x00000008
-#define DSI_CLK_STATUS_DSI_AON_DSICLK_ACTIVE			0x00000010
-#define DSI_CLK_STATUS_DSI_DYN_DSICLK_ACTIVE			0x00000020
-#define DSI_CLK_STATUS_DSI_AON_BYTECLK_ACTIVE			0x00000040
-#define DSI_CLK_STATUS_DSI_DYN_BYTECLK_ACTIVE			0x00000080
-#define DSI_CLK_STATUS_DSI_AON_ESCCLK_ACTIVE			0x00000100
-#define DSI_CLK_STATUS_DSI_AON_PCLK_ACTIVE			0x00000200
-#define DSI_CLK_STATUS_DSI_DYN_PCLK_ACTIVE			0x00000400
-#define DSI_CLK_STATUS_DSI_DYN_CMD_PCLK_ACTIVE			0x00001000
-#define DSI_CLK_STATUS_DSI_CMD_PCLK_ACTIVE			0x00002000
-#define DSI_CLK_STATUS_DSI_VID_PCLK_ACTIVE			0x00004000
-#define DSI_CLK_STATUS_DSI_CAM_BIST_PCLK_ACT			0x00008000
-#define DSI_CLK_STATUS_PLL_UNLOCKED				0x00010000
-
-#define REG_DSI_PHY_RESET					0x00000128
-#define DSI_PHY_RESET_RESET					0x00000001
-
-#define REG_DSI_TEST_PATTERN_GEN_VIDEO_INIT_VAL			0x00000160
-
-#define REG_DSI_TPG_MAIN_CONTROL				0x00000198
-#define DSI_TPG_MAIN_CONTROL_CHECKERED_RECTANGLE_PATTERN	0x00000100
-
-#define REG_DSI_TPG_VIDEO_CONFIG				0x000001a0
-#define DSI_TPG_VIDEO_CONFIG_BPP__MASK				0x00000003
-#define DSI_TPG_VIDEO_CONFIG_BPP__SHIFT				0
-static inline uint32_t DSI_TPG_VIDEO_CONFIG_BPP(enum video_config_bpp val)
-{
-	return ((val) << DSI_TPG_VIDEO_CONFIG_BPP__SHIFT) & DSI_TPG_VIDEO_CONFIG_BPP__MASK;
-}
-#define DSI_TPG_VIDEO_CONFIG_RGB				0x00000004
-
-#define REG_DSI_TEST_PATTERN_GEN_CTRL				0x00000158
-#define DSI_TEST_PATTERN_GEN_CTRL_CMD_DMA_PATTERN_SEL__MASK	0x00030000
-#define DSI_TEST_PATTERN_GEN_CTRL_CMD_DMA_PATTERN_SEL__SHIFT	16
-static inline uint32_t DSI_TEST_PATTERN_GEN_CTRL_CMD_DMA_PATTERN_SEL(enum cmd_dma_pattern_sel val)
-{
-	return ((val) << DSI_TEST_PATTERN_GEN_CTRL_CMD_DMA_PATTERN_SEL__SHIFT) & DSI_TEST_PATTERN_GEN_CTRL_CMD_DMA_PATTERN_SEL__MASK;
-}
-#define DSI_TEST_PATTERN_GEN_CTRL_CMD_MDP_STREAM0_PATTERN_SEL__MASK	0x00000300
-#define DSI_TEST_PATTERN_GEN_CTRL_CMD_MDP_STREAM0_PATTERN_SEL__SHIFT	8
-static inline uint32_t DSI_TEST_PATTERN_GEN_CTRL_CMD_MDP_STREAM0_PATTERN_SEL(enum cmd_mdp_stream0_pattern_sel val)
-{
-	return ((val) << DSI_TEST_PATTERN_GEN_CTRL_CMD_MDP_STREAM0_PATTERN_SEL__SHIFT) & DSI_TEST_PATTERN_GEN_CTRL_CMD_MDP_STREAM0_PATTERN_SEL__MASK;
-}
-#define DSI_TEST_PATTERN_GEN_CTRL_VIDEO_PATTERN_SEL__MASK	0x00000030
-#define DSI_TEST_PATTERN_GEN_CTRL_VIDEO_PATTERN_SEL__SHIFT	4
-static inline uint32_t DSI_TEST_PATTERN_GEN_CTRL_VIDEO_PATTERN_SEL(enum video_pattern_sel val)
-{
-	return ((val) << DSI_TEST_PATTERN_GEN_CTRL_VIDEO_PATTERN_SEL__SHIFT) & DSI_TEST_PATTERN_GEN_CTRL_VIDEO_PATTERN_SEL__MASK;
-}
-#define DSI_TEST_PATTERN_GEN_CTRL_TPG_DMA_FIFO_MODE		0x00000004
-#define DSI_TEST_PATTERN_GEN_CTRL_CMD_DMA_TPG_EN		0x00000002
-#define DSI_TEST_PATTERN_GEN_CTRL_EN				0x00000001
-
-#define REG_DSI_TEST_PATTERN_GEN_CMD_MDP_INIT_VAL0		0x00000168
-
-#define REG_DSI_TEST_PATTERN_GEN_CMD_STREAM0_TRIGGER		0x00000180
-#define DSI_TEST_PATTERN_GEN_CMD_STREAM0_TRIGGER_SW_TRIGGER	0x00000001
-
-#define REG_DSI_TPG_MAIN_CONTROL2				0x0000019c
-#define DSI_TPG_MAIN_CONTROL2_CMD_MDP0_CHECKERED_RECTANGLE_PATTERN	0x00000080
-#define DSI_TPG_MAIN_CONTROL2_CMD_MDP1_CHECKERED_RECTANGLE_PATTERN	0x00010000
-#define DSI_TPG_MAIN_CONTROL2_CMD_MDP2_CHECKERED_RECTANGLE_PATTERN	0x02000000
-
-#define REG_DSI_T_CLK_PRE_EXTEND				0x0000017c
-#define DSI_T_CLK_PRE_EXTEND_INC_BY_2_BYTECLK			0x00000001
-
-#define REG_DSI_CMD_MODE_MDP_CTRL2				0x000001b4
-#define DSI_CMD_MODE_MDP_CTRL2_DST_FORMAT2__MASK		0x0000000f
-#define DSI_CMD_MODE_MDP_CTRL2_DST_FORMAT2__SHIFT		0
-static inline uint32_t DSI_CMD_MODE_MDP_CTRL2_DST_FORMAT2(enum dsi_cmd_dst_format val)
-{
-	return ((val) << DSI_CMD_MODE_MDP_CTRL2_DST_FORMAT2__SHIFT) & DSI_CMD_MODE_MDP_CTRL2_DST_FORMAT2__MASK;
-}
-#define DSI_CMD_MODE_MDP_CTRL2_R_SEL				0x00000010
-#define DSI_CMD_MODE_MDP_CTRL2_G_SEL				0x00000020
-#define DSI_CMD_MODE_MDP_CTRL2_B_SEL				0x00000040
-#define DSI_CMD_MODE_MDP_CTRL2_BYTE_MSB_LSB_FLIP		0x00000080
-#define DSI_CMD_MODE_MDP_CTRL2_RGB_SWAP__MASK			0x00000700
-#define DSI_CMD_MODE_MDP_CTRL2_RGB_SWAP__SHIFT			8
-static inline uint32_t DSI_CMD_MODE_MDP_CTRL2_RGB_SWAP(enum dsi_rgb_swap val)
-{
-	return ((val) << DSI_CMD_MODE_MDP_CTRL2_RGB_SWAP__SHIFT) & DSI_CMD_MODE_MDP_CTRL2_RGB_SWAP__MASK;
-}
-#define DSI_CMD_MODE_MDP_CTRL2_INPUT_RGB_SWAP__MASK		0x00007000
-#define DSI_CMD_MODE_MDP_CTRL2_INPUT_RGB_SWAP__SHIFT		12
-static inline uint32_t DSI_CMD_MODE_MDP_CTRL2_INPUT_RGB_SWAP(enum dsi_rgb_swap val)
-{
-	return ((val) << DSI_CMD_MODE_MDP_CTRL2_INPUT_RGB_SWAP__SHIFT) & DSI_CMD_MODE_MDP_CTRL2_INPUT_RGB_SWAP__MASK;
-}
-#define DSI_CMD_MODE_MDP_CTRL2_BURST_MODE			0x00010000
-#define DSI_CMD_MODE_MDP_CTRL2_DATABUS_WIDEN			0x00100000
-
-#define REG_DSI_CMD_MODE_MDP_STREAM2_CTRL			0x000001b8
-#define DSI_CMD_MODE_MDP_STREAM2_CTRL_DATA_TYPE__MASK		0x0000003f
-#define DSI_CMD_MODE_MDP_STREAM2_CTRL_DATA_TYPE__SHIFT		0
-static inline uint32_t DSI_CMD_MODE_MDP_STREAM2_CTRL_DATA_TYPE(uint32_t val)
-{
-	return ((val) << DSI_CMD_MODE_MDP_STREAM2_CTRL_DATA_TYPE__SHIFT) & DSI_CMD_MODE_MDP_STREAM2_CTRL_DATA_TYPE__MASK;
-}
-#define DSI_CMD_MODE_MDP_STREAM2_CTRL_VIRTUAL_CHANNEL__MASK	0x00000300
-#define DSI_CMD_MODE_MDP_STREAM2_CTRL_VIRTUAL_CHANNEL__SHIFT	8
-static inline uint32_t DSI_CMD_MODE_MDP_STREAM2_CTRL_VIRTUAL_CHANNEL(uint32_t val)
-{
-	return ((val) << DSI_CMD_MODE_MDP_STREAM2_CTRL_VIRTUAL_CHANNEL__SHIFT) & DSI_CMD_MODE_MDP_STREAM2_CTRL_VIRTUAL_CHANNEL__MASK;
-}
-#define DSI_CMD_MODE_MDP_STREAM2_CTRL_WORD_COUNT__MASK		0xffff0000
-#define DSI_CMD_MODE_MDP_STREAM2_CTRL_WORD_COUNT__SHIFT		16
-static inline uint32_t DSI_CMD_MODE_MDP_STREAM2_CTRL_WORD_COUNT(uint32_t val)
-{
-	return ((val) << DSI_CMD_MODE_MDP_STREAM2_CTRL_WORD_COUNT__SHIFT) & DSI_CMD_MODE_MDP_STREAM2_CTRL_WORD_COUNT__MASK;
-}
-
-#define REG_DSI_RDBK_DATA_CTRL					0x000001d0
-#define DSI_RDBK_DATA_CTRL_COUNT__MASK				0x00ff0000
-#define DSI_RDBK_DATA_CTRL_COUNT__SHIFT				16
-static inline uint32_t DSI_RDBK_DATA_CTRL_COUNT(uint32_t val)
-{
-	return ((val) << DSI_RDBK_DATA_CTRL_COUNT__SHIFT) & DSI_RDBK_DATA_CTRL_COUNT__MASK;
-}
-#define DSI_RDBK_DATA_CTRL_CLR					0x00000001
-
-#define REG_DSI_VERSION						0x000001f0
-#define DSI_VERSION_MAJOR__MASK					0xff000000
-#define DSI_VERSION_MAJOR__SHIFT				24
-static inline uint32_t DSI_VERSION_MAJOR(uint32_t val)
-{
-	return ((val) << DSI_VERSION_MAJOR__SHIFT) & DSI_VERSION_MAJOR__MASK;
-}
-
-#define REG_DSI_CPHY_MODE_CTRL					0x000002d4
-
-#define REG_DSI_VIDEO_COMPRESSION_MODE_CTRL			0x0000029c
-#define DSI_VIDEO_COMPRESSION_MODE_CTRL_WC__MASK		0xffff0000
-#define DSI_VIDEO_COMPRESSION_MODE_CTRL_WC__SHIFT		16
-static inline uint32_t DSI_VIDEO_COMPRESSION_MODE_CTRL_WC(uint32_t val)
-{
-	return ((val) << DSI_VIDEO_COMPRESSION_MODE_CTRL_WC__SHIFT) & DSI_VIDEO_COMPRESSION_MODE_CTRL_WC__MASK;
-}
-#define DSI_VIDEO_COMPRESSION_MODE_CTRL_DATATYPE__MASK		0x00003f00
-#define DSI_VIDEO_COMPRESSION_MODE_CTRL_DATATYPE__SHIFT		8
-static inline uint32_t DSI_VIDEO_COMPRESSION_MODE_CTRL_DATATYPE(uint32_t val)
-{
-	return ((val) << DSI_VIDEO_COMPRESSION_MODE_CTRL_DATATYPE__SHIFT) & DSI_VIDEO_COMPRESSION_MODE_CTRL_DATATYPE__MASK;
-}
-#define DSI_VIDEO_COMPRESSION_MODE_CTRL_PKT_PER_LINE__MASK	0x000000c0
-#define DSI_VIDEO_COMPRESSION_MODE_CTRL_PKT_PER_LINE__SHIFT	6
-static inline uint32_t DSI_VIDEO_COMPRESSION_MODE_CTRL_PKT_PER_LINE(uint32_t val)
-{
-	return ((val) << DSI_VIDEO_COMPRESSION_MODE_CTRL_PKT_PER_LINE__SHIFT) & DSI_VIDEO_COMPRESSION_MODE_CTRL_PKT_PER_LINE__MASK;
-}
-#define DSI_VIDEO_COMPRESSION_MODE_CTRL_EOL_BYTE_NUM__MASK	0x00000030
-#define DSI_VIDEO_COMPRESSION_MODE_CTRL_EOL_BYTE_NUM__SHIFT	4
-static inline uint32_t DSI_VIDEO_COMPRESSION_MODE_CTRL_EOL_BYTE_NUM(uint32_t val)
-{
-	return ((val) << DSI_VIDEO_COMPRESSION_MODE_CTRL_EOL_BYTE_NUM__SHIFT) & DSI_VIDEO_COMPRESSION_MODE_CTRL_EOL_BYTE_NUM__MASK;
-}
-#define DSI_VIDEO_COMPRESSION_MODE_CTRL_EN			0x00000001
-
-#define REG_DSI_COMMAND_COMPRESSION_MODE_CTRL			0x000002a4
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_DATATYPE__MASK	0x3f000000
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_DATATYPE__SHIFT	24
-static inline uint32_t DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_DATATYPE(uint32_t val)
-{
-	return ((val) << DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_DATATYPE__SHIFT) & DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_DATATYPE__MASK;
-}
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_PKT_PER_LINE__MASK	0x00c00000
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_PKT_PER_LINE__SHIFT	22
-static inline uint32_t DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_PKT_PER_LINE(uint32_t val)
-{
-	return ((val) << DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_PKT_PER_LINE__SHIFT) & DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_PKT_PER_LINE__MASK;
-}
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_EOL_BYTE_NUM__MASK	0x00300000
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_EOL_BYTE_NUM__SHIFT	20
-static inline uint32_t DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_EOL_BYTE_NUM(uint32_t val)
-{
-	return ((val) << DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_EOL_BYTE_NUM__SHIFT) & DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_EOL_BYTE_NUM__MASK;
-}
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM1_EN		0x00010000
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_DATATYPE__MASK	0x00003f00
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_DATATYPE__SHIFT	8
-static inline uint32_t DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_DATATYPE(uint32_t val)
-{
-	return ((val) << DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_DATATYPE__SHIFT) & DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_DATATYPE__MASK;
-}
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_PKT_PER_LINE__MASK	0x000000c0
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_PKT_PER_LINE__SHIFT	6
-static inline uint32_t DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_PKT_PER_LINE(uint32_t val)
-{
-	return ((val) << DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_PKT_PER_LINE__SHIFT) & DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_PKT_PER_LINE__MASK;
-}
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_EOL_BYTE_NUM__MASK	0x00000030
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_EOL_BYTE_NUM__SHIFT	4
-static inline uint32_t DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_EOL_BYTE_NUM(uint32_t val)
-{
-	return ((val) << DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_EOL_BYTE_NUM__SHIFT) & DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_EOL_BYTE_NUM__MASK;
-}
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_EN		0x00000001
-
-#define REG_DSI_COMMAND_COMPRESSION_MODE_CTRL2			0x000002a8
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM1_SLICE_WIDTH__MASK	0xffff0000
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM1_SLICE_WIDTH__SHIFT	16
-static inline uint32_t DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM1_SLICE_WIDTH(uint32_t val)
-{
-	return ((val) << DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM1_SLICE_WIDTH__SHIFT) & DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM1_SLICE_WIDTH__MASK;
-}
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM0_SLICE_WIDTH__MASK	0x0000ffff
-#define DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM0_SLICE_WIDTH__SHIFT	0
-static inline uint32_t DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM0_SLICE_WIDTH(uint32_t val)
-{
-	return ((val) << DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM0_SLICE_WIDTH__SHIFT) & DSI_COMMAND_COMPRESSION_MODE_CTRL2_STREAM0_SLICE_WIDTH__MASK;
-}
-
-
-#endif /* DSI_XML */
diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
index 9d86a6aca6f2..a50f4dda5941 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
@@ -55,7 +55,7 @@ static int dsi_get_version(const void __iomem *base, u32 *major, u32 *minor)
 	 * scratch register which we never touch)
 	 */
 
-	ver = msm_readl(base + REG_DSI_VERSION);
+	ver = readl(base + REG_DSI_VERSION);
 	if (ver) {
 		/* older dsi host, there is no register shift */
 		ver = FIELD(ver, DSI_VERSION_MAJOR);
@@ -73,12 +73,12 @@ static int dsi_get_version(const void __iomem *base, u32 *major, u32 *minor)
 		 * registers are shifted down, read DSI_VERSION again with
 		 * the shifted offset
 		 */
-		ver = msm_readl(base + DSI_6G_REG_SHIFT + REG_DSI_VERSION);
+		ver = readl(base + DSI_6G_REG_SHIFT + REG_DSI_VERSION);
 		ver = FIELD(ver, DSI_VERSION_MAJOR);
 		if (ver == MSM_DSI_VER_MAJOR_6G) {
 			/* 6G version */
 			*major = ver;
-			*minor = msm_readl(base + REG_DSI_6G_HW_VERSION);
+			*minor = readl(base + REG_DSI_6G_HW_VERSION);
 			return 0;
 		} else {
 			return -EINVAL;
@@ -186,11 +186,11 @@ struct msm_dsi_host {
 
 static inline u32 dsi_read(struct msm_dsi_host *msm_host, u32 reg)
 {
-	return msm_readl(msm_host->ctrl_base + reg);
+	return readl(msm_host->ctrl_base + reg);
 }
 static inline void dsi_write(struct msm_dsi_host *msm_host, u32 reg, u32 data)
 {
-	msm_writel(data, msm_host->ctrl_base + reg);
+	writel(data, msm_host->ctrl_base + reg);
 }
 
 static const struct msm_dsi_cfg_handler *dsi_get_config(
@@ -356,8 +356,8 @@ int dsi_link_clk_set_rate_6g(struct msm_dsi_host *msm_host)
 {
 	int ret;
 
-	DBG("Set clk rates: pclk=%d, byteclk=%lu",
-		msm_host->mode->clock, msm_host->byte_clk_rate);
+	DBG("Set clk rates: pclk=%lu, byteclk=%lu",
+	    msm_host->pixel_clk_rate, msm_host->byte_clk_rate);
 
 	ret = dev_pm_opp_set_rate(&msm_host->pdev->dev,
 				  msm_host->byte_clk_rate);
@@ -430,9 +430,9 @@ int dsi_link_clk_set_rate_v2(struct msm_dsi_host *msm_host)
 {
 	int ret;
 
-	DBG("Set clk rates: pclk=%d, byteclk=%lu, esc_clk=%lu, dsi_src_clk=%lu",
-		msm_host->mode->clock, msm_host->byte_clk_rate,
-		msm_host->esc_clk_rate, msm_host->src_clk_rate);
+	DBG("Set clk rates: pclk=%lu, byteclk=%lu, esc_clk=%lu, dsi_src_clk=%lu",
+	    msm_host->pixel_clk_rate, msm_host->byte_clk_rate,
+	    msm_host->esc_clk_rate, msm_host->src_clk_rate);
 
 	ret = clk_set_rate(msm_host->byte_clk, msm_host->byte_clk_rate);
 	if (ret) {
diff --git a/drivers/gpu/drm/msm/dsi/dsi_manager.c b/drivers/gpu/drm/msm/dsi/dsi_manager.c
index af2a287cb3bd..5b3f3068fd92 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_manager.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_manager.c
@@ -423,7 +423,18 @@ static enum drm_mode_status dsi_mgr_bridge_mode_valid(struct drm_bridge *bridge,
 	return msm_dsi_host_check_dsc(host, mode);
 }
 
+static int dsi_mgr_bridge_attach(struct drm_bridge *bridge,
+				 enum drm_bridge_attach_flags flags)
+{
+	int id = dsi_mgr_bridge_get_id(bridge);
+	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
+
+	return drm_bridge_attach(bridge->encoder, msm_dsi->next_bridge,
+				 bridge, flags);
+}
+
 static const struct drm_bridge_funcs dsi_mgr_bridge_funcs = {
+	.attach = dsi_mgr_bridge_attach,
 	.pre_enable = dsi_mgr_bridge_pre_enable,
 	.post_disable = dsi_mgr_bridge_post_disable,
 	.mode_set = dsi_mgr_bridge_mode_set,
@@ -431,17 +442,19 @@ static const struct drm_bridge_funcs dsi_mgr_bridge_funcs = {
 };
 
 /* initialize bridge */
-struct drm_bridge *msm_dsi_manager_bridge_init(struct msm_dsi *msm_dsi,
-					       struct drm_encoder *encoder)
+int msm_dsi_manager_connector_init(struct msm_dsi *msm_dsi,
+				   struct drm_encoder *encoder)
 {
+	struct drm_device *dev = msm_dsi->dev;
 	struct drm_bridge *bridge;
 	struct dsi_bridge *dsi_bridge;
+	struct drm_connector *connector;
 	int ret;
 
 	dsi_bridge = devm_kzalloc(msm_dsi->dev->dev,
 				sizeof(*dsi_bridge), GFP_KERNEL);
 	if (!dsi_bridge)
-		return ERR_PTR(-ENOMEM);
+		return -ENOMEM;
 
 	dsi_bridge->id = msm_dsi->id;
 
@@ -450,60 +463,22 @@ struct drm_bridge *msm_dsi_manager_bridge_init(struct msm_dsi *msm_dsi,
 
 	ret = devm_drm_bridge_add(msm_dsi->dev->dev, bridge);
 	if (ret)
-		return ERR_PTR(ret);
+		return ret;
 
-	ret = drm_bridge_attach(encoder, bridge, NULL, 0);
+	ret = drm_bridge_attach(encoder, bridge, NULL, DRM_BRIDGE_ATTACH_NO_CONNECTOR);
 	if (ret)
-		return ERR_PTR(ret);
-
-	return bridge;
-}
-
-int msm_dsi_manager_ext_bridge_init(u8 id, struct drm_bridge *int_bridge)
-{
-	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
-	struct drm_device *dev = msm_dsi->dev;
-	struct drm_encoder *encoder;
-	struct drm_bridge *ext_bridge;
-	int ret;
+		return ret;
 
-	ext_bridge = devm_drm_of_get_bridge(&msm_dsi->pdev->dev,
-					    msm_dsi->pdev->dev.of_node, 1, 0);
-	if (IS_ERR(ext_bridge))
-		return PTR_ERR(ext_bridge);
-
-	encoder = int_bridge->encoder;
-
-	/*
-	 * Try first to create the bridge without it creating its own
-	 * connector.. currently some bridges support this, and others
-	 * do not (and some support both modes)
-	 */
-	ret = drm_bridge_attach(encoder, ext_bridge, int_bridge,
-			DRM_BRIDGE_ATTACH_NO_CONNECTOR);
-	if (ret == -EINVAL) {
-		/*
-		 * link the internal dsi bridge to the external bridge,
-		 * connector is created by the next bridge.
-		 */
-		ret = drm_bridge_attach(encoder, ext_bridge, int_bridge, 0);
-		if (ret < 0)
-			return ret;
-	} else {
-		struct drm_connector *connector;
-
-		/* We are in charge of the connector, create one now. */
-		connector = drm_bridge_connector_init(dev, encoder);
-		if (IS_ERR(connector)) {
-			DRM_ERROR("Unable to create bridge connector\n");
-			return PTR_ERR(connector);
-		}
-
-		ret = drm_connector_attach_encoder(connector, encoder);
-		if (ret < 0)
-			return ret;
+	connector = drm_bridge_connector_init(dev, encoder);
+	if (IS_ERR(connector)) {
+		DRM_ERROR("Unable to create bridge connector\n");
+		return PTR_ERR(connector);
 	}
 
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret < 0)
+		return ret;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi_phy_10nm.xml.h b/drivers/gpu/drm/msm/dsi/dsi_phy_10nm.xml.h
deleted file mode 100644
index a2ae8777e59e..000000000000
--- a/drivers/gpu/drm/msm/dsi/dsi_phy_10nm.xml.h
+++ /dev/null
@@ -1,227 +0,0 @@
-#ifndef DSI_PHY_10NM_XML
-#define DSI_PHY_10NM_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-#define REG_DSI_10nm_PHY_CMN_REVISION_ID0			0x00000000
-
-#define REG_DSI_10nm_PHY_CMN_REVISION_ID1			0x00000004
-
-#define REG_DSI_10nm_PHY_CMN_REVISION_ID2			0x00000008
-
-#define REG_DSI_10nm_PHY_CMN_REVISION_ID3			0x0000000c
-
-#define REG_DSI_10nm_PHY_CMN_CLK_CFG0				0x00000010
-
-#define REG_DSI_10nm_PHY_CMN_CLK_CFG1				0x00000014
-
-#define REG_DSI_10nm_PHY_CMN_GLBL_CTRL				0x00000018
-
-#define REG_DSI_10nm_PHY_CMN_RBUF_CTRL				0x0000001c
-
-#define REG_DSI_10nm_PHY_CMN_VREG_CTRL				0x00000020
-
-#define REG_DSI_10nm_PHY_CMN_CTRL_0				0x00000024
-
-#define REG_DSI_10nm_PHY_CMN_CTRL_1				0x00000028
-
-#define REG_DSI_10nm_PHY_CMN_CTRL_2				0x0000002c
-
-#define REG_DSI_10nm_PHY_CMN_LANE_CFG0				0x00000030
-
-#define REG_DSI_10nm_PHY_CMN_LANE_CFG1				0x00000034
-
-#define REG_DSI_10nm_PHY_CMN_PLL_CNTRL				0x00000038
-
-#define REG_DSI_10nm_PHY_CMN_LANE_CTRL0				0x00000098
-
-#define REG_DSI_10nm_PHY_CMN_LANE_CTRL1				0x0000009c
-
-#define REG_DSI_10nm_PHY_CMN_LANE_CTRL2				0x000000a0
-
-#define REG_DSI_10nm_PHY_CMN_LANE_CTRL3				0x000000a4
-
-#define REG_DSI_10nm_PHY_CMN_LANE_CTRL4				0x000000a8
-
-#define REG_DSI_10nm_PHY_CMN_TIMING_CTRL_0			0x000000ac
-
-#define REG_DSI_10nm_PHY_CMN_TIMING_CTRL_1			0x000000b0
-
-#define REG_DSI_10nm_PHY_CMN_TIMING_CTRL_2			0x000000b4
-
-#define REG_DSI_10nm_PHY_CMN_TIMING_CTRL_3			0x000000b8
-
-#define REG_DSI_10nm_PHY_CMN_TIMING_CTRL_4			0x000000bc
-
-#define REG_DSI_10nm_PHY_CMN_TIMING_CTRL_5			0x000000c0
-
-#define REG_DSI_10nm_PHY_CMN_TIMING_CTRL_6			0x000000c4
-
-#define REG_DSI_10nm_PHY_CMN_TIMING_CTRL_7			0x000000c8
-
-#define REG_DSI_10nm_PHY_CMN_TIMING_CTRL_8			0x000000cc
-
-#define REG_DSI_10nm_PHY_CMN_TIMING_CTRL_9			0x000000d0
-
-#define REG_DSI_10nm_PHY_CMN_TIMING_CTRL_10			0x000000d4
-
-#define REG_DSI_10nm_PHY_CMN_TIMING_CTRL_11			0x000000d8
-
-#define REG_DSI_10nm_PHY_CMN_PHY_STATUS				0x000000ec
-
-#define REG_DSI_10nm_PHY_CMN_LANE_STATUS0			0x000000f4
-
-#define REG_DSI_10nm_PHY_CMN_LANE_STATUS1			0x000000f8
-
-static inline uint32_t REG_DSI_10nm_PHY_LN(uint32_t i0) { return 0x00000000 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_10nm_PHY_LN_CFG0(uint32_t i0) { return 0x00000000 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_10nm_PHY_LN_CFG1(uint32_t i0) { return 0x00000004 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_10nm_PHY_LN_CFG2(uint32_t i0) { return 0x00000008 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_10nm_PHY_LN_CFG3(uint32_t i0) { return 0x0000000c + 0x80*i0; }
-
-static inline uint32_t REG_DSI_10nm_PHY_LN_TEST_DATAPATH(uint32_t i0) { return 0x00000010 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_10nm_PHY_LN_PIN_SWAP(uint32_t i0) { return 0x00000014 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_10nm_PHY_LN_HSTX_STR_CTRL(uint32_t i0) { return 0x00000018 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_10nm_PHY_LN_OFFSET_TOP_CTRL(uint32_t i0) { return 0x0000001c + 0x80*i0; }
-
-static inline uint32_t REG_DSI_10nm_PHY_LN_OFFSET_BOT_CTRL(uint32_t i0) { return 0x00000020 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_10nm_PHY_LN_LPTX_STR_CTRL(uint32_t i0) { return 0x00000024 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_10nm_PHY_LN_LPRX_CTRL(uint32_t i0) { return 0x00000028 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_10nm_PHY_LN_TX_DCTRL(uint32_t i0) { return 0x0000002c + 0x80*i0; }
-
-#define REG_DSI_10nm_PHY_PLL_ANALOG_CONTROLS_ONE		0x00000000
-
-#define REG_DSI_10nm_PHY_PLL_ANALOG_CONTROLS_TWO		0x00000004
-
-#define REG_DSI_10nm_PHY_PLL_ANALOG_CONTROLS_THREE		0x00000010
-
-#define REG_DSI_10nm_PHY_PLL_DSM_DIVIDER			0x0000001c
-
-#define REG_DSI_10nm_PHY_PLL_FEEDBACK_DIVIDER			0x00000020
-
-#define REG_DSI_10nm_PHY_PLL_SYSTEM_MUXES			0x00000024
-
-#define REG_DSI_10nm_PHY_PLL_CMODE				0x0000002c
-
-#define REG_DSI_10nm_PHY_PLL_CALIBRATION_SETTINGS		0x00000030
-
-#define REG_DSI_10nm_PHY_PLL_BAND_SEL_CAL_SETTINGS_THREE	0x00000054
-
-#define REG_DSI_10nm_PHY_PLL_FREQ_DETECT_SETTINGS_ONE		0x00000064
-
-#define REG_DSI_10nm_PHY_PLL_PFILT				0x0000007c
-
-#define REG_DSI_10nm_PHY_PLL_IFILT				0x00000080
-
-#define REG_DSI_10nm_PHY_PLL_OUTDIV				0x00000094
-
-#define REG_DSI_10nm_PHY_PLL_CORE_OVERRIDE			0x000000a4
-
-#define REG_DSI_10nm_PHY_PLL_CORE_INPUT_OVERRIDE		0x000000a8
-
-#define REG_DSI_10nm_PHY_PLL_PLL_DIGITAL_TIMERS_TWO		0x000000b4
-
-#define REG_DSI_10nm_PHY_PLL_DECIMAL_DIV_START_1		0x000000cc
-
-#define REG_DSI_10nm_PHY_PLL_FRAC_DIV_START_LOW_1		0x000000d0
-
-#define REG_DSI_10nm_PHY_PLL_FRAC_DIV_START_MID_1		0x000000d4
-
-#define REG_DSI_10nm_PHY_PLL_FRAC_DIV_START_HIGH_1		0x000000d8
-
-#define REG_DSI_10nm_PHY_PLL_SSC_STEPSIZE_LOW_1			0x0000010c
-
-#define REG_DSI_10nm_PHY_PLL_SSC_STEPSIZE_HIGH_1		0x00000110
-
-#define REG_DSI_10nm_PHY_PLL_SSC_DIV_PER_LOW_1			0x00000114
-
-#define REG_DSI_10nm_PHY_PLL_SSC_DIV_PER_HIGH_1			0x00000118
-
-#define REG_DSI_10nm_PHY_PLL_SSC_DIV_ADJPER_LOW_1		0x0000011c
-
-#define REG_DSI_10nm_PHY_PLL_SSC_DIV_ADJPER_HIGH_1		0x00000120
-
-#define REG_DSI_10nm_PHY_PLL_SSC_CONTROL			0x0000013c
-
-#define REG_DSI_10nm_PHY_PLL_PLL_OUTDIV_RATE			0x00000140
-
-#define REG_DSI_10nm_PHY_PLL_PLL_LOCKDET_RATE_1			0x00000144
-
-#define REG_DSI_10nm_PHY_PLL_PLL_PROP_GAIN_RATE_1		0x0000014c
-
-#define REG_DSI_10nm_PHY_PLL_PLL_BAND_SET_RATE_1		0x00000154
-
-#define REG_DSI_10nm_PHY_PLL_PLL_INT_GAIN_IFILT_BAND_1		0x0000015c
-
-#define REG_DSI_10nm_PHY_PLL_PLL_FL_INT_GAIN_PFILT_BAND_1	0x00000164
-
-#define REG_DSI_10nm_PHY_PLL_PLL_LOCK_OVERRIDE			0x00000180
-
-#define REG_DSI_10nm_PHY_PLL_PLL_LOCK_DELAY			0x00000184
-
-#define REG_DSI_10nm_PHY_PLL_CLOCK_INVERTERS			0x0000018c
-
-#define REG_DSI_10nm_PHY_PLL_COMMON_STATUS_ONE			0x000001a0
-
-
-#endif /* DSI_PHY_10NM_XML */
diff --git a/drivers/gpu/drm/msm/dsi/dsi_phy_14nm.xml.h b/drivers/gpu/drm/msm/dsi/dsi_phy_14nm.xml.h
deleted file mode 100644
index 24e2fdc0cde1..000000000000
--- a/drivers/gpu/drm/msm/dsi/dsi_phy_14nm.xml.h
+++ /dev/null
@@ -1,309 +0,0 @@
-#ifndef DSI_PHY_14NM_XML
-#define DSI_PHY_14NM_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-#define REG_DSI_14nm_PHY_CMN_REVISION_ID0			0x00000000
-
-#define REG_DSI_14nm_PHY_CMN_REVISION_ID1			0x00000004
-
-#define REG_DSI_14nm_PHY_CMN_REVISION_ID2			0x00000008
-
-#define REG_DSI_14nm_PHY_CMN_REVISION_ID3			0x0000000c
-
-#define REG_DSI_14nm_PHY_CMN_CLK_CFG0				0x00000010
-#define DSI_14nm_PHY_CMN_CLK_CFG0_DIV_CTRL_3_0__MASK		0x000000f0
-#define DSI_14nm_PHY_CMN_CLK_CFG0_DIV_CTRL_3_0__SHIFT		4
-static inline uint32_t DSI_14nm_PHY_CMN_CLK_CFG0_DIV_CTRL_3_0(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_CMN_CLK_CFG0_DIV_CTRL_3_0__SHIFT) & DSI_14nm_PHY_CMN_CLK_CFG0_DIV_CTRL_3_0__MASK;
-}
-#define DSI_14nm_PHY_CMN_CLK_CFG0_DIV_CTRL_7_4__MASK		0x000000f0
-#define DSI_14nm_PHY_CMN_CLK_CFG0_DIV_CTRL_7_4__SHIFT		4
-static inline uint32_t DSI_14nm_PHY_CMN_CLK_CFG0_DIV_CTRL_7_4(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_CMN_CLK_CFG0_DIV_CTRL_7_4__SHIFT) & DSI_14nm_PHY_CMN_CLK_CFG0_DIV_CTRL_7_4__MASK;
-}
-
-#define REG_DSI_14nm_PHY_CMN_CLK_CFG1				0x00000014
-#define DSI_14nm_PHY_CMN_CLK_CFG1_DSICLK_SEL			0x00000001
-
-#define REG_DSI_14nm_PHY_CMN_GLBL_TEST_CTRL			0x00000018
-#define DSI_14nm_PHY_CMN_GLBL_TEST_CTRL_BITCLK_HS_SEL		0x00000004
-
-#define REG_DSI_14nm_PHY_CMN_CTRL_0				0x0000001c
-
-#define REG_DSI_14nm_PHY_CMN_CTRL_1				0x00000020
-
-#define REG_DSI_14nm_PHY_CMN_HW_TRIGGER				0x00000024
-
-#define REG_DSI_14nm_PHY_CMN_SW_CFG0				0x00000028
-
-#define REG_DSI_14nm_PHY_CMN_SW_CFG1				0x0000002c
-
-#define REG_DSI_14nm_PHY_CMN_SW_CFG2				0x00000030
-
-#define REG_DSI_14nm_PHY_CMN_HW_CFG0				0x00000034
-
-#define REG_DSI_14nm_PHY_CMN_HW_CFG1				0x00000038
-
-#define REG_DSI_14nm_PHY_CMN_HW_CFG2				0x0000003c
-
-#define REG_DSI_14nm_PHY_CMN_HW_CFG3				0x00000040
-
-#define REG_DSI_14nm_PHY_CMN_HW_CFG4				0x00000044
-
-#define REG_DSI_14nm_PHY_CMN_PLL_CNTRL				0x00000048
-#define DSI_14nm_PHY_CMN_PLL_CNTRL_PLL_START			0x00000001
-
-#define REG_DSI_14nm_PHY_CMN_LDO_CNTRL				0x0000004c
-#define DSI_14nm_PHY_CMN_LDO_CNTRL_VREG_CTRL__MASK		0x0000003f
-#define DSI_14nm_PHY_CMN_LDO_CNTRL_VREG_CTRL__SHIFT		0
-static inline uint32_t DSI_14nm_PHY_CMN_LDO_CNTRL_VREG_CTRL(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_CMN_LDO_CNTRL_VREG_CTRL__SHIFT) & DSI_14nm_PHY_CMN_LDO_CNTRL_VREG_CTRL__MASK;
-}
-
-static inline uint32_t REG_DSI_14nm_PHY_LN(uint32_t i0) { return 0x00000000 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_CFG0(uint32_t i0) { return 0x00000000 + 0x80*i0; }
-#define DSI_14nm_PHY_LN_CFG0_PREPARE_DLY__MASK			0x000000c0
-#define DSI_14nm_PHY_LN_CFG0_PREPARE_DLY__SHIFT			6
-static inline uint32_t DSI_14nm_PHY_LN_CFG0_PREPARE_DLY(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_LN_CFG0_PREPARE_DLY__SHIFT) & DSI_14nm_PHY_LN_CFG0_PREPARE_DLY__MASK;
-}
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_CFG1(uint32_t i0) { return 0x00000004 + 0x80*i0; }
-#define DSI_14nm_PHY_LN_CFG1_HALFBYTECLK_EN			0x00000001
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_CFG2(uint32_t i0) { return 0x00000008 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_CFG3(uint32_t i0) { return 0x0000000c + 0x80*i0; }
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_TEST_DATAPATH(uint32_t i0) { return 0x00000010 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_TEST_STR(uint32_t i0) { return 0x00000014 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_TIMING_CTRL_4(uint32_t i0) { return 0x00000018 + 0x80*i0; }
-#define DSI_14nm_PHY_LN_TIMING_CTRL_4_HS_EXIT__MASK		0x000000ff
-#define DSI_14nm_PHY_LN_TIMING_CTRL_4_HS_EXIT__SHIFT		0
-static inline uint32_t DSI_14nm_PHY_LN_TIMING_CTRL_4_HS_EXIT(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_LN_TIMING_CTRL_4_HS_EXIT__SHIFT) & DSI_14nm_PHY_LN_TIMING_CTRL_4_HS_EXIT__MASK;
-}
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_TIMING_CTRL_5(uint32_t i0) { return 0x0000001c + 0x80*i0; }
-#define DSI_14nm_PHY_LN_TIMING_CTRL_5_HS_ZERO__MASK		0x000000ff
-#define DSI_14nm_PHY_LN_TIMING_CTRL_5_HS_ZERO__SHIFT		0
-static inline uint32_t DSI_14nm_PHY_LN_TIMING_CTRL_5_HS_ZERO(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_LN_TIMING_CTRL_5_HS_ZERO__SHIFT) & DSI_14nm_PHY_LN_TIMING_CTRL_5_HS_ZERO__MASK;
-}
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_TIMING_CTRL_6(uint32_t i0) { return 0x00000020 + 0x80*i0; }
-#define DSI_14nm_PHY_LN_TIMING_CTRL_6_HS_PREPARE__MASK		0x000000ff
-#define DSI_14nm_PHY_LN_TIMING_CTRL_6_HS_PREPARE__SHIFT		0
-static inline uint32_t DSI_14nm_PHY_LN_TIMING_CTRL_6_HS_PREPARE(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_LN_TIMING_CTRL_6_HS_PREPARE__SHIFT) & DSI_14nm_PHY_LN_TIMING_CTRL_6_HS_PREPARE__MASK;
-}
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_TIMING_CTRL_7(uint32_t i0) { return 0x00000024 + 0x80*i0; }
-#define DSI_14nm_PHY_LN_TIMING_CTRL_7_HS_TRAIL__MASK		0x000000ff
-#define DSI_14nm_PHY_LN_TIMING_CTRL_7_HS_TRAIL__SHIFT		0
-static inline uint32_t DSI_14nm_PHY_LN_TIMING_CTRL_7_HS_TRAIL(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_LN_TIMING_CTRL_7_HS_TRAIL__SHIFT) & DSI_14nm_PHY_LN_TIMING_CTRL_7_HS_TRAIL__MASK;
-}
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_TIMING_CTRL_8(uint32_t i0) { return 0x00000028 + 0x80*i0; }
-#define DSI_14nm_PHY_LN_TIMING_CTRL_8_HS_RQST__MASK		0x000000ff
-#define DSI_14nm_PHY_LN_TIMING_CTRL_8_HS_RQST__SHIFT		0
-static inline uint32_t DSI_14nm_PHY_LN_TIMING_CTRL_8_HS_RQST(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_LN_TIMING_CTRL_8_HS_RQST__SHIFT) & DSI_14nm_PHY_LN_TIMING_CTRL_8_HS_RQST__MASK;
-}
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_TIMING_CTRL_9(uint32_t i0) { return 0x0000002c + 0x80*i0; }
-#define DSI_14nm_PHY_LN_TIMING_CTRL_9_TA_GO__MASK		0x00000007
-#define DSI_14nm_PHY_LN_TIMING_CTRL_9_TA_GO__SHIFT		0
-static inline uint32_t DSI_14nm_PHY_LN_TIMING_CTRL_9_TA_GO(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_LN_TIMING_CTRL_9_TA_GO__SHIFT) & DSI_14nm_PHY_LN_TIMING_CTRL_9_TA_GO__MASK;
-}
-#define DSI_14nm_PHY_LN_TIMING_CTRL_9_TA_SURE__MASK		0x00000070
-#define DSI_14nm_PHY_LN_TIMING_CTRL_9_TA_SURE__SHIFT		4
-static inline uint32_t DSI_14nm_PHY_LN_TIMING_CTRL_9_TA_SURE(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_LN_TIMING_CTRL_9_TA_SURE__SHIFT) & DSI_14nm_PHY_LN_TIMING_CTRL_9_TA_SURE__MASK;
-}
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_TIMING_CTRL_10(uint32_t i0) { return 0x00000030 + 0x80*i0; }
-#define DSI_14nm_PHY_LN_TIMING_CTRL_10_TA_GET__MASK		0x00000007
-#define DSI_14nm_PHY_LN_TIMING_CTRL_10_TA_GET__SHIFT		0
-static inline uint32_t DSI_14nm_PHY_LN_TIMING_CTRL_10_TA_GET(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_LN_TIMING_CTRL_10_TA_GET__SHIFT) & DSI_14nm_PHY_LN_TIMING_CTRL_10_TA_GET__MASK;
-}
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_TIMING_CTRL_11(uint32_t i0) { return 0x00000034 + 0x80*i0; }
-#define DSI_14nm_PHY_LN_TIMING_CTRL_11_TRIG3_CMD__MASK		0x000000ff
-#define DSI_14nm_PHY_LN_TIMING_CTRL_11_TRIG3_CMD__SHIFT		0
-static inline uint32_t DSI_14nm_PHY_LN_TIMING_CTRL_11_TRIG3_CMD(uint32_t val)
-{
-	return ((val) << DSI_14nm_PHY_LN_TIMING_CTRL_11_TRIG3_CMD__SHIFT) & DSI_14nm_PHY_LN_TIMING_CTRL_11_TRIG3_CMD__MASK;
-}
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_STRENGTH_CTRL_0(uint32_t i0) { return 0x00000038 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_STRENGTH_CTRL_1(uint32_t i0) { return 0x0000003c + 0x80*i0; }
-
-static inline uint32_t REG_DSI_14nm_PHY_LN_VREG_CNTRL(uint32_t i0) { return 0x00000064 + 0x80*i0; }
-
-#define REG_DSI_14nm_PHY_PLL_IE_TRIM				0x00000000
-
-#define REG_DSI_14nm_PHY_PLL_IP_TRIM				0x00000004
-
-#define REG_DSI_14nm_PHY_PLL_IPTAT_TRIM				0x00000010
-
-#define REG_DSI_14nm_PHY_PLL_CLKBUFLR_EN			0x0000001c
-
-#define REG_DSI_14nm_PHY_PLL_SYSCLK_EN_RESET			0x00000028
-
-#define REG_DSI_14nm_PHY_PLL_RESETSM_CNTRL			0x0000002c
-
-#define REG_DSI_14nm_PHY_PLL_RESETSM_CNTRL2			0x00000030
-
-#define REG_DSI_14nm_PHY_PLL_RESETSM_CNTRL3			0x00000034
-
-#define REG_DSI_14nm_PHY_PLL_RESETSM_CNTRL4			0x00000038
-
-#define REG_DSI_14nm_PHY_PLL_RESETSM_CNTRL5			0x0000003c
-
-#define REG_DSI_14nm_PHY_PLL_KVCO_DIV_REF1			0x00000040
-
-#define REG_DSI_14nm_PHY_PLL_KVCO_DIV_REF2			0x00000044
-
-#define REG_DSI_14nm_PHY_PLL_KVCO_COUNT1			0x00000048
-
-#define REG_DSI_14nm_PHY_PLL_KVCO_COUNT2			0x0000004c
-
-#define REG_DSI_14nm_PHY_PLL_VREF_CFG1				0x0000005c
-
-#define REG_DSI_14nm_PHY_PLL_KVCO_CODE				0x00000058
-
-#define REG_DSI_14nm_PHY_PLL_VCO_DIV_REF1			0x0000006c
-
-#define REG_DSI_14nm_PHY_PLL_VCO_DIV_REF2			0x00000070
-
-#define REG_DSI_14nm_PHY_PLL_VCO_COUNT1				0x00000074
-
-#define REG_DSI_14nm_PHY_PLL_VCO_COUNT2				0x00000078
-
-#define REG_DSI_14nm_PHY_PLL_PLLLOCK_CMP1			0x0000007c
-
-#define REG_DSI_14nm_PHY_PLL_PLLLOCK_CMP2			0x00000080
-
-#define REG_DSI_14nm_PHY_PLL_PLLLOCK_CMP3			0x00000084
-
-#define REG_DSI_14nm_PHY_PLL_PLLLOCK_CMP_EN			0x00000088
-
-#define REG_DSI_14nm_PHY_PLL_PLL_VCO_TUNE			0x0000008c
-
-#define REG_DSI_14nm_PHY_PLL_DEC_START				0x00000090
-
-#define REG_DSI_14nm_PHY_PLL_SSC_EN_CENTER			0x00000094
-
-#define REG_DSI_14nm_PHY_PLL_SSC_ADJ_PER1			0x00000098
-
-#define REG_DSI_14nm_PHY_PLL_SSC_ADJ_PER2			0x0000009c
-
-#define REG_DSI_14nm_PHY_PLL_SSC_PER1				0x000000a0
-
-#define REG_DSI_14nm_PHY_PLL_SSC_PER2				0x000000a4
-
-#define REG_DSI_14nm_PHY_PLL_SSC_STEP_SIZE1			0x000000a8
-
-#define REG_DSI_14nm_PHY_PLL_SSC_STEP_SIZE2			0x000000ac
-
-#define REG_DSI_14nm_PHY_PLL_DIV_FRAC_START1			0x000000b4
-
-#define REG_DSI_14nm_PHY_PLL_DIV_FRAC_START2			0x000000b8
-
-#define REG_DSI_14nm_PHY_PLL_DIV_FRAC_START3			0x000000bc
-
-#define REG_DSI_14nm_PHY_PLL_TXCLK_EN				0x000000c0
-
-#define REG_DSI_14nm_PHY_PLL_PLL_CRCTRL				0x000000c4
-
-#define REG_DSI_14nm_PHY_PLL_RESET_SM_READY_STATUS		0x000000cc
-
-#define REG_DSI_14nm_PHY_PLL_PLL_MISC1				0x000000e8
-
-#define REG_DSI_14nm_PHY_PLL_CP_SET_CUR				0x000000f0
-
-#define REG_DSI_14nm_PHY_PLL_PLL_ICPMSET			0x000000f4
-
-#define REG_DSI_14nm_PHY_PLL_PLL_ICPCSET			0x000000f8
-
-#define REG_DSI_14nm_PHY_PLL_PLL_ICP_SET			0x000000fc
-
-#define REG_DSI_14nm_PHY_PLL_PLL_LPF1				0x00000100
-
-#define REG_DSI_14nm_PHY_PLL_PLL_LPF2_POSTDIV			0x00000104
-
-#define REG_DSI_14nm_PHY_PLL_PLL_BANDGAP			0x00000108
-
-
-#endif /* DSI_PHY_14NM_XML */
diff --git a/drivers/gpu/drm/msm/dsi/dsi_phy_20nm.xml.h b/drivers/gpu/drm/msm/dsi/dsi_phy_20nm.xml.h
deleted file mode 100644
index 6352541f37e9..000000000000
--- a/drivers/gpu/drm/msm/dsi/dsi_phy_20nm.xml.h
+++ /dev/null
@@ -1,237 +0,0 @@
-#ifndef DSI_PHY_20NM_XML
-#define DSI_PHY_20NM_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-static inline uint32_t REG_DSI_20nm_PHY_LN(uint32_t i0) { return 0x00000000 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_20nm_PHY_LN_CFG_0(uint32_t i0) { return 0x00000000 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_20nm_PHY_LN_CFG_1(uint32_t i0) { return 0x00000004 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_20nm_PHY_LN_CFG_2(uint32_t i0) { return 0x00000008 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_20nm_PHY_LN_CFG_3(uint32_t i0) { return 0x0000000c + 0x40*i0; }
-
-static inline uint32_t REG_DSI_20nm_PHY_LN_CFG_4(uint32_t i0) { return 0x00000010 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_20nm_PHY_LN_TEST_DATAPATH(uint32_t i0) { return 0x00000014 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_20nm_PHY_LN_DEBUG_SEL(uint32_t i0) { return 0x00000018 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_20nm_PHY_LN_TEST_STR_0(uint32_t i0) { return 0x0000001c + 0x40*i0; }
-
-static inline uint32_t REG_DSI_20nm_PHY_LN_TEST_STR_1(uint32_t i0) { return 0x00000020 + 0x40*i0; }
-
-#define REG_DSI_20nm_PHY_LNCK_CFG_0				0x00000100
-
-#define REG_DSI_20nm_PHY_LNCK_CFG_1				0x00000104
-
-#define REG_DSI_20nm_PHY_LNCK_CFG_2				0x00000108
-
-#define REG_DSI_20nm_PHY_LNCK_CFG_3				0x0000010c
-
-#define REG_DSI_20nm_PHY_LNCK_CFG_4				0x00000110
-
-#define REG_DSI_20nm_PHY_LNCK_TEST_DATAPATH			0x00000114
-
-#define REG_DSI_20nm_PHY_LNCK_DEBUG_SEL				0x00000118
-
-#define REG_DSI_20nm_PHY_LNCK_TEST_STR0				0x0000011c
-
-#define REG_DSI_20nm_PHY_LNCK_TEST_STR1				0x00000120
-
-#define REG_DSI_20nm_PHY_TIMING_CTRL_0				0x00000140
-#define DSI_20nm_PHY_TIMING_CTRL_0_CLK_ZERO__MASK		0x000000ff
-#define DSI_20nm_PHY_TIMING_CTRL_0_CLK_ZERO__SHIFT		0
-static inline uint32_t DSI_20nm_PHY_TIMING_CTRL_0_CLK_ZERO(uint32_t val)
-{
-	return ((val) << DSI_20nm_PHY_TIMING_CTRL_0_CLK_ZERO__SHIFT) & DSI_20nm_PHY_TIMING_CTRL_0_CLK_ZERO__MASK;
-}
-
-#define REG_DSI_20nm_PHY_TIMING_CTRL_1				0x00000144
-#define DSI_20nm_PHY_TIMING_CTRL_1_CLK_TRAIL__MASK		0x000000ff
-#define DSI_20nm_PHY_TIMING_CTRL_1_CLK_TRAIL__SHIFT		0
-static inline uint32_t DSI_20nm_PHY_TIMING_CTRL_1_CLK_TRAIL(uint32_t val)
-{
-	return ((val) << DSI_20nm_PHY_TIMING_CTRL_1_CLK_TRAIL__SHIFT) & DSI_20nm_PHY_TIMING_CTRL_1_CLK_TRAIL__MASK;
-}
-
-#define REG_DSI_20nm_PHY_TIMING_CTRL_2				0x00000148
-#define DSI_20nm_PHY_TIMING_CTRL_2_CLK_PREPARE__MASK		0x000000ff
-#define DSI_20nm_PHY_TIMING_CTRL_2_CLK_PREPARE__SHIFT		0
-static inline uint32_t DSI_20nm_PHY_TIMING_CTRL_2_CLK_PREPARE(uint32_t val)
-{
-	return ((val) << DSI_20nm_PHY_TIMING_CTRL_2_CLK_PREPARE__SHIFT) & DSI_20nm_PHY_TIMING_CTRL_2_CLK_PREPARE__MASK;
-}
-
-#define REG_DSI_20nm_PHY_TIMING_CTRL_3				0x0000014c
-#define DSI_20nm_PHY_TIMING_CTRL_3_CLK_ZERO_8			0x00000001
-
-#define REG_DSI_20nm_PHY_TIMING_CTRL_4				0x00000150
-#define DSI_20nm_PHY_TIMING_CTRL_4_HS_EXIT__MASK		0x000000ff
-#define DSI_20nm_PHY_TIMING_CTRL_4_HS_EXIT__SHIFT		0
-static inline uint32_t DSI_20nm_PHY_TIMING_CTRL_4_HS_EXIT(uint32_t val)
-{
-	return ((val) << DSI_20nm_PHY_TIMING_CTRL_4_HS_EXIT__SHIFT) & DSI_20nm_PHY_TIMING_CTRL_4_HS_EXIT__MASK;
-}
-
-#define REG_DSI_20nm_PHY_TIMING_CTRL_5				0x00000154
-#define DSI_20nm_PHY_TIMING_CTRL_5_HS_ZERO__MASK		0x000000ff
-#define DSI_20nm_PHY_TIMING_CTRL_5_HS_ZERO__SHIFT		0
-static inline uint32_t DSI_20nm_PHY_TIMING_CTRL_5_HS_ZERO(uint32_t val)
-{
-	return ((val) << DSI_20nm_PHY_TIMING_CTRL_5_HS_ZERO__SHIFT) & DSI_20nm_PHY_TIMING_CTRL_5_HS_ZERO__MASK;
-}
-
-#define REG_DSI_20nm_PHY_TIMING_CTRL_6				0x00000158
-#define DSI_20nm_PHY_TIMING_CTRL_6_HS_PREPARE__MASK		0x000000ff
-#define DSI_20nm_PHY_TIMING_CTRL_6_HS_PREPARE__SHIFT		0
-static inline uint32_t DSI_20nm_PHY_TIMING_CTRL_6_HS_PREPARE(uint32_t val)
-{
-	return ((val) << DSI_20nm_PHY_TIMING_CTRL_6_HS_PREPARE__SHIFT) & DSI_20nm_PHY_TIMING_CTRL_6_HS_PREPARE__MASK;
-}
-
-#define REG_DSI_20nm_PHY_TIMING_CTRL_7				0x0000015c
-#define DSI_20nm_PHY_TIMING_CTRL_7_HS_TRAIL__MASK		0x000000ff
-#define DSI_20nm_PHY_TIMING_CTRL_7_HS_TRAIL__SHIFT		0
-static inline uint32_t DSI_20nm_PHY_TIMING_CTRL_7_HS_TRAIL(uint32_t val)
-{
-	return ((val) << DSI_20nm_PHY_TIMING_CTRL_7_HS_TRAIL__SHIFT) & DSI_20nm_PHY_TIMING_CTRL_7_HS_TRAIL__MASK;
-}
-
-#define REG_DSI_20nm_PHY_TIMING_CTRL_8				0x00000160
-#define DSI_20nm_PHY_TIMING_CTRL_8_HS_RQST__MASK		0x000000ff
-#define DSI_20nm_PHY_TIMING_CTRL_8_HS_RQST__SHIFT		0
-static inline uint32_t DSI_20nm_PHY_TIMING_CTRL_8_HS_RQST(uint32_t val)
-{
-	return ((val) << DSI_20nm_PHY_TIMING_CTRL_8_HS_RQST__SHIFT) & DSI_20nm_PHY_TIMING_CTRL_8_HS_RQST__MASK;
-}
-
-#define REG_DSI_20nm_PHY_TIMING_CTRL_9				0x00000164
-#define DSI_20nm_PHY_TIMING_CTRL_9_TA_GO__MASK			0x00000007
-#define DSI_20nm_PHY_TIMING_CTRL_9_TA_GO__SHIFT			0
-static inline uint32_t DSI_20nm_PHY_TIMING_CTRL_9_TA_GO(uint32_t val)
-{
-	return ((val) << DSI_20nm_PHY_TIMING_CTRL_9_TA_GO__SHIFT) & DSI_20nm_PHY_TIMING_CTRL_9_TA_GO__MASK;
-}
-#define DSI_20nm_PHY_TIMING_CTRL_9_TA_SURE__MASK		0x00000070
-#define DSI_20nm_PHY_TIMING_CTRL_9_TA_SURE__SHIFT		4
-static inline uint32_t DSI_20nm_PHY_TIMING_CTRL_9_TA_SURE(uint32_t val)
-{
-	return ((val) << DSI_20nm_PHY_TIMING_CTRL_9_TA_SURE__SHIFT) & DSI_20nm_PHY_TIMING_CTRL_9_TA_SURE__MASK;
-}
-
-#define REG_DSI_20nm_PHY_TIMING_CTRL_10				0x00000168
-#define DSI_20nm_PHY_TIMING_CTRL_10_TA_GET__MASK		0x00000007
-#define DSI_20nm_PHY_TIMING_CTRL_10_TA_GET__SHIFT		0
-static inline uint32_t DSI_20nm_PHY_TIMING_CTRL_10_TA_GET(uint32_t val)
-{
-	return ((val) << DSI_20nm_PHY_TIMING_CTRL_10_TA_GET__SHIFT) & DSI_20nm_PHY_TIMING_CTRL_10_TA_GET__MASK;
-}
-
-#define REG_DSI_20nm_PHY_TIMING_CTRL_11				0x0000016c
-#define DSI_20nm_PHY_TIMING_CTRL_11_TRIG3_CMD__MASK		0x000000ff
-#define DSI_20nm_PHY_TIMING_CTRL_11_TRIG3_CMD__SHIFT		0
-static inline uint32_t DSI_20nm_PHY_TIMING_CTRL_11_TRIG3_CMD(uint32_t val)
-{
-	return ((val) << DSI_20nm_PHY_TIMING_CTRL_11_TRIG3_CMD__SHIFT) & DSI_20nm_PHY_TIMING_CTRL_11_TRIG3_CMD__MASK;
-}
-
-#define REG_DSI_20nm_PHY_CTRL_0					0x00000170
-
-#define REG_DSI_20nm_PHY_CTRL_1					0x00000174
-
-#define REG_DSI_20nm_PHY_CTRL_2					0x00000178
-
-#define REG_DSI_20nm_PHY_CTRL_3					0x0000017c
-
-#define REG_DSI_20nm_PHY_CTRL_4					0x00000180
-
-#define REG_DSI_20nm_PHY_STRENGTH_0				0x00000184
-
-#define REG_DSI_20nm_PHY_STRENGTH_1				0x00000188
-
-#define REG_DSI_20nm_PHY_BIST_CTRL_0				0x000001b4
-
-#define REG_DSI_20nm_PHY_BIST_CTRL_1				0x000001b8
-
-#define REG_DSI_20nm_PHY_BIST_CTRL_2				0x000001bc
-
-#define REG_DSI_20nm_PHY_BIST_CTRL_3				0x000001c0
-
-#define REG_DSI_20nm_PHY_BIST_CTRL_4				0x000001c4
-
-#define REG_DSI_20nm_PHY_BIST_CTRL_5				0x000001c8
-
-#define REG_DSI_20nm_PHY_GLBL_TEST_CTRL				0x000001d4
-#define DSI_20nm_PHY_GLBL_TEST_CTRL_BITCLK_HS_SEL		0x00000001
-
-#define REG_DSI_20nm_PHY_LDO_CNTRL				0x000001dc
-
-#define REG_DSI_20nm_PHY_REGULATOR_CTRL_0			0x00000000
-
-#define REG_DSI_20nm_PHY_REGULATOR_CTRL_1			0x00000004
-
-#define REG_DSI_20nm_PHY_REGULATOR_CTRL_2			0x00000008
-
-#define REG_DSI_20nm_PHY_REGULATOR_CTRL_3			0x0000000c
-
-#define REG_DSI_20nm_PHY_REGULATOR_CTRL_4			0x00000010
-
-#define REG_DSI_20nm_PHY_REGULATOR_CTRL_5			0x00000014
-
-#define REG_DSI_20nm_PHY_REGULATOR_CAL_PWR_CFG			0x00000018
-
-
-#endif /* DSI_PHY_20NM_XML */
diff --git a/drivers/gpu/drm/msm/dsi/dsi_phy_28nm.xml.h b/drivers/gpu/drm/msm/dsi/dsi_phy_28nm.xml.h
deleted file mode 100644
index 178bd4fd7893..000000000000
--- a/drivers/gpu/drm/msm/dsi/dsi_phy_28nm.xml.h
+++ /dev/null
@@ -1,384 +0,0 @@
-#ifndef DSI_PHY_28NM_XML
-#define DSI_PHY_28NM_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-static inline uint32_t REG_DSI_28nm_PHY_LN(uint32_t i0) { return 0x00000000 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_PHY_LN_CFG_0(uint32_t i0) { return 0x00000000 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_PHY_LN_CFG_1(uint32_t i0) { return 0x00000004 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_PHY_LN_CFG_2(uint32_t i0) { return 0x00000008 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_PHY_LN_CFG_3(uint32_t i0) { return 0x0000000c + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_PHY_LN_CFG_4(uint32_t i0) { return 0x00000010 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_PHY_LN_TEST_DATAPATH(uint32_t i0) { return 0x00000014 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_PHY_LN_DEBUG_SEL(uint32_t i0) { return 0x00000018 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_PHY_LN_TEST_STR_0(uint32_t i0) { return 0x0000001c + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_PHY_LN_TEST_STR_1(uint32_t i0) { return 0x00000020 + 0x40*i0; }
-
-#define REG_DSI_28nm_PHY_LNCK_CFG_0				0x00000100
-
-#define REG_DSI_28nm_PHY_LNCK_CFG_1				0x00000104
-
-#define REG_DSI_28nm_PHY_LNCK_CFG_2				0x00000108
-
-#define REG_DSI_28nm_PHY_LNCK_CFG_3				0x0000010c
-
-#define REG_DSI_28nm_PHY_LNCK_CFG_4				0x00000110
-
-#define REG_DSI_28nm_PHY_LNCK_TEST_DATAPATH			0x00000114
-
-#define REG_DSI_28nm_PHY_LNCK_DEBUG_SEL				0x00000118
-
-#define REG_DSI_28nm_PHY_LNCK_TEST_STR0				0x0000011c
-
-#define REG_DSI_28nm_PHY_LNCK_TEST_STR1				0x00000120
-
-#define REG_DSI_28nm_PHY_TIMING_CTRL_0				0x00000140
-#define DSI_28nm_PHY_TIMING_CTRL_0_CLK_ZERO__MASK		0x000000ff
-#define DSI_28nm_PHY_TIMING_CTRL_0_CLK_ZERO__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_TIMING_CTRL_0_CLK_ZERO(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_TIMING_CTRL_0_CLK_ZERO__SHIFT) & DSI_28nm_PHY_TIMING_CTRL_0_CLK_ZERO__MASK;
-}
-
-#define REG_DSI_28nm_PHY_TIMING_CTRL_1				0x00000144
-#define DSI_28nm_PHY_TIMING_CTRL_1_CLK_TRAIL__MASK		0x000000ff
-#define DSI_28nm_PHY_TIMING_CTRL_1_CLK_TRAIL__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_TIMING_CTRL_1_CLK_TRAIL(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_TIMING_CTRL_1_CLK_TRAIL__SHIFT) & DSI_28nm_PHY_TIMING_CTRL_1_CLK_TRAIL__MASK;
-}
-
-#define REG_DSI_28nm_PHY_TIMING_CTRL_2				0x00000148
-#define DSI_28nm_PHY_TIMING_CTRL_2_CLK_PREPARE__MASK		0x000000ff
-#define DSI_28nm_PHY_TIMING_CTRL_2_CLK_PREPARE__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_TIMING_CTRL_2_CLK_PREPARE(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_TIMING_CTRL_2_CLK_PREPARE__SHIFT) & DSI_28nm_PHY_TIMING_CTRL_2_CLK_PREPARE__MASK;
-}
-
-#define REG_DSI_28nm_PHY_TIMING_CTRL_3				0x0000014c
-#define DSI_28nm_PHY_TIMING_CTRL_3_CLK_ZERO_8			0x00000001
-
-#define REG_DSI_28nm_PHY_TIMING_CTRL_4				0x00000150
-#define DSI_28nm_PHY_TIMING_CTRL_4_HS_EXIT__MASK		0x000000ff
-#define DSI_28nm_PHY_TIMING_CTRL_4_HS_EXIT__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_TIMING_CTRL_4_HS_EXIT(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_TIMING_CTRL_4_HS_EXIT__SHIFT) & DSI_28nm_PHY_TIMING_CTRL_4_HS_EXIT__MASK;
-}
-
-#define REG_DSI_28nm_PHY_TIMING_CTRL_5				0x00000154
-#define DSI_28nm_PHY_TIMING_CTRL_5_HS_ZERO__MASK		0x000000ff
-#define DSI_28nm_PHY_TIMING_CTRL_5_HS_ZERO__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_TIMING_CTRL_5_HS_ZERO(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_TIMING_CTRL_5_HS_ZERO__SHIFT) & DSI_28nm_PHY_TIMING_CTRL_5_HS_ZERO__MASK;
-}
-
-#define REG_DSI_28nm_PHY_TIMING_CTRL_6				0x00000158
-#define DSI_28nm_PHY_TIMING_CTRL_6_HS_PREPARE__MASK		0x000000ff
-#define DSI_28nm_PHY_TIMING_CTRL_6_HS_PREPARE__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_TIMING_CTRL_6_HS_PREPARE(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_TIMING_CTRL_6_HS_PREPARE__SHIFT) & DSI_28nm_PHY_TIMING_CTRL_6_HS_PREPARE__MASK;
-}
-
-#define REG_DSI_28nm_PHY_TIMING_CTRL_7				0x0000015c
-#define DSI_28nm_PHY_TIMING_CTRL_7_HS_TRAIL__MASK		0x000000ff
-#define DSI_28nm_PHY_TIMING_CTRL_7_HS_TRAIL__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_TIMING_CTRL_7_HS_TRAIL(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_TIMING_CTRL_7_HS_TRAIL__SHIFT) & DSI_28nm_PHY_TIMING_CTRL_7_HS_TRAIL__MASK;
-}
-
-#define REG_DSI_28nm_PHY_TIMING_CTRL_8				0x00000160
-#define DSI_28nm_PHY_TIMING_CTRL_8_HS_RQST__MASK		0x000000ff
-#define DSI_28nm_PHY_TIMING_CTRL_8_HS_RQST__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_TIMING_CTRL_8_HS_RQST(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_TIMING_CTRL_8_HS_RQST__SHIFT) & DSI_28nm_PHY_TIMING_CTRL_8_HS_RQST__MASK;
-}
-
-#define REG_DSI_28nm_PHY_TIMING_CTRL_9				0x00000164
-#define DSI_28nm_PHY_TIMING_CTRL_9_TA_GO__MASK			0x00000007
-#define DSI_28nm_PHY_TIMING_CTRL_9_TA_GO__SHIFT			0
-static inline uint32_t DSI_28nm_PHY_TIMING_CTRL_9_TA_GO(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_TIMING_CTRL_9_TA_GO__SHIFT) & DSI_28nm_PHY_TIMING_CTRL_9_TA_GO__MASK;
-}
-#define DSI_28nm_PHY_TIMING_CTRL_9_TA_SURE__MASK		0x00000070
-#define DSI_28nm_PHY_TIMING_CTRL_9_TA_SURE__SHIFT		4
-static inline uint32_t DSI_28nm_PHY_TIMING_CTRL_9_TA_SURE(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_TIMING_CTRL_9_TA_SURE__SHIFT) & DSI_28nm_PHY_TIMING_CTRL_9_TA_SURE__MASK;
-}
-
-#define REG_DSI_28nm_PHY_TIMING_CTRL_10				0x00000168
-#define DSI_28nm_PHY_TIMING_CTRL_10_TA_GET__MASK		0x00000007
-#define DSI_28nm_PHY_TIMING_CTRL_10_TA_GET__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_TIMING_CTRL_10_TA_GET(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_TIMING_CTRL_10_TA_GET__SHIFT) & DSI_28nm_PHY_TIMING_CTRL_10_TA_GET__MASK;
-}
-
-#define REG_DSI_28nm_PHY_TIMING_CTRL_11				0x0000016c
-#define DSI_28nm_PHY_TIMING_CTRL_11_TRIG3_CMD__MASK		0x000000ff
-#define DSI_28nm_PHY_TIMING_CTRL_11_TRIG3_CMD__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_TIMING_CTRL_11_TRIG3_CMD(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_TIMING_CTRL_11_TRIG3_CMD__SHIFT) & DSI_28nm_PHY_TIMING_CTRL_11_TRIG3_CMD__MASK;
-}
-
-#define REG_DSI_28nm_PHY_CTRL_0					0x00000170
-
-#define REG_DSI_28nm_PHY_CTRL_1					0x00000174
-
-#define REG_DSI_28nm_PHY_CTRL_2					0x00000178
-
-#define REG_DSI_28nm_PHY_CTRL_3					0x0000017c
-
-#define REG_DSI_28nm_PHY_CTRL_4					0x00000180
-
-#define REG_DSI_28nm_PHY_STRENGTH_0				0x00000184
-
-#define REG_DSI_28nm_PHY_STRENGTH_1				0x00000188
-
-#define REG_DSI_28nm_PHY_BIST_CTRL_0				0x000001b4
-
-#define REG_DSI_28nm_PHY_BIST_CTRL_1				0x000001b8
-
-#define REG_DSI_28nm_PHY_BIST_CTRL_2				0x000001bc
-
-#define REG_DSI_28nm_PHY_BIST_CTRL_3				0x000001c0
-
-#define REG_DSI_28nm_PHY_BIST_CTRL_4				0x000001c4
-
-#define REG_DSI_28nm_PHY_BIST_CTRL_5				0x000001c8
-
-#define REG_DSI_28nm_PHY_GLBL_TEST_CTRL				0x000001d4
-#define DSI_28nm_PHY_GLBL_TEST_CTRL_BITCLK_HS_SEL		0x00000001
-
-#define REG_DSI_28nm_PHY_LDO_CNTRL				0x000001dc
-
-#define REG_DSI_28nm_PHY_REGULATOR_CTRL_0			0x00000000
-
-#define REG_DSI_28nm_PHY_REGULATOR_CTRL_1			0x00000004
-
-#define REG_DSI_28nm_PHY_REGULATOR_CTRL_2			0x00000008
-
-#define REG_DSI_28nm_PHY_REGULATOR_CTRL_3			0x0000000c
-
-#define REG_DSI_28nm_PHY_REGULATOR_CTRL_4			0x00000010
-
-#define REG_DSI_28nm_PHY_REGULATOR_CTRL_5			0x00000014
-
-#define REG_DSI_28nm_PHY_REGULATOR_CAL_PWR_CFG			0x00000018
-
-#define REG_DSI_28nm_PHY_PLL_REFCLK_CFG				0x00000000
-#define DSI_28nm_PHY_PLL_REFCLK_CFG_DBLR			0x00000001
-
-#define REG_DSI_28nm_PHY_PLL_POSTDIV1_CFG			0x00000004
-
-#define REG_DSI_28nm_PHY_PLL_CHGPUMP_CFG			0x00000008
-
-#define REG_DSI_28nm_PHY_PLL_VCOLPF_CFG				0x0000000c
-
-#define REG_DSI_28nm_PHY_PLL_VREG_CFG				0x00000010
-#define DSI_28nm_PHY_PLL_VREG_CFG_POSTDIV1_BYPASS_B		0x00000002
-
-#define REG_DSI_28nm_PHY_PLL_PWRGEN_CFG				0x00000014
-
-#define REG_DSI_28nm_PHY_PLL_DMUX_CFG				0x00000018
-
-#define REG_DSI_28nm_PHY_PLL_AMUX_CFG				0x0000001c
-
-#define REG_DSI_28nm_PHY_PLL_GLB_CFG				0x00000020
-#define DSI_28nm_PHY_PLL_GLB_CFG_PLL_PWRDN_B			0x00000001
-#define DSI_28nm_PHY_PLL_GLB_CFG_PLL_LDO_PWRDN_B		0x00000002
-#define DSI_28nm_PHY_PLL_GLB_CFG_PLL_PWRGEN_PWRDN_B		0x00000004
-#define DSI_28nm_PHY_PLL_GLB_CFG_PLL_ENABLE			0x00000008
-
-#define REG_DSI_28nm_PHY_PLL_POSTDIV2_CFG			0x00000024
-
-#define REG_DSI_28nm_PHY_PLL_POSTDIV3_CFG			0x00000028
-
-#define REG_DSI_28nm_PHY_PLL_LPFR_CFG				0x0000002c
-
-#define REG_DSI_28nm_PHY_PLL_LPFC1_CFG				0x00000030
-
-#define REG_DSI_28nm_PHY_PLL_LPFC2_CFG				0x00000034
-
-#define REG_DSI_28nm_PHY_PLL_SDM_CFG0				0x00000038
-#define DSI_28nm_PHY_PLL_SDM_CFG0_BYP_DIV__MASK			0x0000003f
-#define DSI_28nm_PHY_PLL_SDM_CFG0_BYP_DIV__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_PLL_SDM_CFG0_BYP_DIV(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_PLL_SDM_CFG0_BYP_DIV__SHIFT) & DSI_28nm_PHY_PLL_SDM_CFG0_BYP_DIV__MASK;
-}
-#define DSI_28nm_PHY_PLL_SDM_CFG0_BYP				0x00000040
-
-#define REG_DSI_28nm_PHY_PLL_SDM_CFG1				0x0000003c
-#define DSI_28nm_PHY_PLL_SDM_CFG1_DC_OFFSET__MASK		0x0000003f
-#define DSI_28nm_PHY_PLL_SDM_CFG1_DC_OFFSET__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_PLL_SDM_CFG1_DC_OFFSET(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_PLL_SDM_CFG1_DC_OFFSET__SHIFT) & DSI_28nm_PHY_PLL_SDM_CFG1_DC_OFFSET__MASK;
-}
-#define DSI_28nm_PHY_PLL_SDM_CFG1_DITHER_EN__MASK		0x00000040
-#define DSI_28nm_PHY_PLL_SDM_CFG1_DITHER_EN__SHIFT		6
-static inline uint32_t DSI_28nm_PHY_PLL_SDM_CFG1_DITHER_EN(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_PLL_SDM_CFG1_DITHER_EN__SHIFT) & DSI_28nm_PHY_PLL_SDM_CFG1_DITHER_EN__MASK;
-}
-
-#define REG_DSI_28nm_PHY_PLL_SDM_CFG2				0x00000040
-#define DSI_28nm_PHY_PLL_SDM_CFG2_FREQ_SEED_7_0__MASK		0x000000ff
-#define DSI_28nm_PHY_PLL_SDM_CFG2_FREQ_SEED_7_0__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_PLL_SDM_CFG2_FREQ_SEED_7_0(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_PLL_SDM_CFG2_FREQ_SEED_7_0__SHIFT) & DSI_28nm_PHY_PLL_SDM_CFG2_FREQ_SEED_7_0__MASK;
-}
-
-#define REG_DSI_28nm_PHY_PLL_SDM_CFG3				0x00000044
-#define DSI_28nm_PHY_PLL_SDM_CFG3_FREQ_SEED_15_8__MASK		0x000000ff
-#define DSI_28nm_PHY_PLL_SDM_CFG3_FREQ_SEED_15_8__SHIFT		0
-static inline uint32_t DSI_28nm_PHY_PLL_SDM_CFG3_FREQ_SEED_15_8(uint32_t val)
-{
-	return ((val) << DSI_28nm_PHY_PLL_SDM_CFG3_FREQ_SEED_15_8__SHIFT) & DSI_28nm_PHY_PLL_SDM_CFG3_FREQ_SEED_15_8__MASK;
-}
-
-#define REG_DSI_28nm_PHY_PLL_SDM_CFG4				0x00000048
-
-#define REG_DSI_28nm_PHY_PLL_SSC_CFG0				0x0000004c
-
-#define REG_DSI_28nm_PHY_PLL_SSC_CFG1				0x00000050
-
-#define REG_DSI_28nm_PHY_PLL_SSC_CFG2				0x00000054
-
-#define REG_DSI_28nm_PHY_PLL_SSC_CFG3				0x00000058
-
-#define REG_DSI_28nm_PHY_PLL_LKDET_CFG0				0x0000005c
-
-#define REG_DSI_28nm_PHY_PLL_LKDET_CFG1				0x00000060
-
-#define REG_DSI_28nm_PHY_PLL_LKDET_CFG2				0x00000064
-
-#define REG_DSI_28nm_PHY_PLL_TEST_CFG				0x00000068
-#define DSI_28nm_PHY_PLL_TEST_CFG_PLL_SW_RESET			0x00000001
-
-#define REG_DSI_28nm_PHY_PLL_CAL_CFG0				0x0000006c
-
-#define REG_DSI_28nm_PHY_PLL_CAL_CFG1				0x00000070
-
-#define REG_DSI_28nm_PHY_PLL_CAL_CFG2				0x00000074
-
-#define REG_DSI_28nm_PHY_PLL_CAL_CFG3				0x00000078
-
-#define REG_DSI_28nm_PHY_PLL_CAL_CFG4				0x0000007c
-
-#define REG_DSI_28nm_PHY_PLL_CAL_CFG5				0x00000080
-
-#define REG_DSI_28nm_PHY_PLL_CAL_CFG6				0x00000084
-
-#define REG_DSI_28nm_PHY_PLL_CAL_CFG7				0x00000088
-
-#define REG_DSI_28nm_PHY_PLL_CAL_CFG8				0x0000008c
-
-#define REG_DSI_28nm_PHY_PLL_CAL_CFG9				0x00000090
-
-#define REG_DSI_28nm_PHY_PLL_CAL_CFG10				0x00000094
-
-#define REG_DSI_28nm_PHY_PLL_CAL_CFG11				0x00000098
-
-#define REG_DSI_28nm_PHY_PLL_EFUSE_CFG				0x0000009c
-
-#define REG_DSI_28nm_PHY_PLL_DEBUG_BUS_SEL			0x000000a0
-
-#define REG_DSI_28nm_PHY_PLL_CTRL_42				0x000000a4
-
-#define REG_DSI_28nm_PHY_PLL_CTRL_43				0x000000a8
-
-#define REG_DSI_28nm_PHY_PLL_CTRL_44				0x000000ac
-
-#define REG_DSI_28nm_PHY_PLL_CTRL_45				0x000000b0
-
-#define REG_DSI_28nm_PHY_PLL_CTRL_46				0x000000b4
-
-#define REG_DSI_28nm_PHY_PLL_CTRL_47				0x000000b8
-
-#define REG_DSI_28nm_PHY_PLL_CTRL_48				0x000000bc
-
-#define REG_DSI_28nm_PHY_PLL_STATUS				0x000000c0
-#define DSI_28nm_PHY_PLL_STATUS_PLL_RDY				0x00000001
-
-#define REG_DSI_28nm_PHY_PLL_DEBUG_BUS0				0x000000c4
-
-#define REG_DSI_28nm_PHY_PLL_DEBUG_BUS1				0x000000c8
-
-#define REG_DSI_28nm_PHY_PLL_DEBUG_BUS2				0x000000cc
-
-#define REG_DSI_28nm_PHY_PLL_DEBUG_BUS3				0x000000d0
-
-#define REG_DSI_28nm_PHY_PLL_CTRL_54				0x000000d4
-
-
-#endif /* DSI_PHY_28NM_XML */
diff --git a/drivers/gpu/drm/msm/dsi/dsi_phy_28nm_8960.xml.h b/drivers/gpu/drm/msm/dsi/dsi_phy_28nm_8960.xml.h
deleted file mode 100644
index 5f900bb53519..000000000000
--- a/drivers/gpu/drm/msm/dsi/dsi_phy_28nm_8960.xml.h
+++ /dev/null
@@ -1,286 +0,0 @@
-#ifndef DSI_PHY_28NM_8960_XML
-#define DSI_PHY_28NM_8960_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-static inline uint32_t REG_DSI_28nm_8960_PHY_LN(uint32_t i0) { return 0x00000000 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_8960_PHY_LN_CFG_0(uint32_t i0) { return 0x00000000 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_8960_PHY_LN_CFG_1(uint32_t i0) { return 0x00000004 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_8960_PHY_LN_CFG_2(uint32_t i0) { return 0x00000008 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_8960_PHY_LN_TEST_DATAPATH(uint32_t i0) { return 0x0000000c + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_8960_PHY_LN_TEST_STR_0(uint32_t i0) { return 0x00000014 + 0x40*i0; }
-
-static inline uint32_t REG_DSI_28nm_8960_PHY_LN_TEST_STR_1(uint32_t i0) { return 0x00000018 + 0x40*i0; }
-
-#define REG_DSI_28nm_8960_PHY_LNCK_CFG_0			0x00000100
-
-#define REG_DSI_28nm_8960_PHY_LNCK_CFG_1			0x00000104
-
-#define REG_DSI_28nm_8960_PHY_LNCK_CFG_2			0x00000108
-
-#define REG_DSI_28nm_8960_PHY_LNCK_TEST_DATAPATH		0x0000010c
-
-#define REG_DSI_28nm_8960_PHY_LNCK_TEST_STR0			0x00000114
-
-#define REG_DSI_28nm_8960_PHY_LNCK_TEST_STR1			0x00000118
-
-#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_0			0x00000140
-#define DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO__MASK		0x000000ff
-#define DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO__SHIFT		0
-static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO(uint32_t val)
-{
-	return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO__MASK;
-}
-
-#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_1			0x00000144
-#define DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL__MASK		0x000000ff
-#define DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL__SHIFT	0
-static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL(uint32_t val)
-{
-	return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL__MASK;
-}
-
-#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_2			0x00000148
-#define DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE__MASK	0x000000ff
-#define DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE__SHIFT	0
-static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE(uint32_t val)
-{
-	return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE__MASK;
-}
-
-#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_3			0x0000014c
-
-#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_4			0x00000150
-#define DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT__MASK		0x000000ff
-#define DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT__SHIFT		0
-static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT(uint32_t val)
-{
-	return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT__MASK;
-}
-
-#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_5			0x00000154
-#define DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO__MASK		0x000000ff
-#define DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO__SHIFT		0
-static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO(uint32_t val)
-{
-	return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO__MASK;
-}
-
-#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_6			0x00000158
-#define DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE__MASK	0x000000ff
-#define DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE__SHIFT	0
-static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE(uint32_t val)
-{
-	return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE__MASK;
-}
-
-#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_7			0x0000015c
-#define DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL__MASK		0x000000ff
-#define DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL__SHIFT		0
-static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL(uint32_t val)
-{
-	return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL__MASK;
-}
-
-#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_8			0x00000160
-#define DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST__MASK		0x000000ff
-#define DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST__SHIFT		0
-static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST(uint32_t val)
-{
-	return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST__MASK;
-}
-
-#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_9			0x00000164
-#define DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO__MASK		0x00000007
-#define DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO__SHIFT		0
-static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO(uint32_t val)
-{
-	return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO__MASK;
-}
-#define DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE__MASK		0x00000070
-#define DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE__SHIFT		4
-static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE(uint32_t val)
-{
-	return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE__MASK;
-}
-
-#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_10			0x00000168
-#define DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET__MASK		0x00000007
-#define DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET__SHIFT		0
-static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET(uint32_t val)
-{
-	return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET__MASK;
-}
-
-#define REG_DSI_28nm_8960_PHY_TIMING_CTRL_11			0x0000016c
-#define DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD__MASK	0x000000ff
-#define DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD__SHIFT	0
-static inline uint32_t DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD(uint32_t val)
-{
-	return ((val) << DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD__SHIFT) & DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD__MASK;
-}
-
-#define REG_DSI_28nm_8960_PHY_CTRL_0				0x00000170
-
-#define REG_DSI_28nm_8960_PHY_CTRL_1				0x00000174
-
-#define REG_DSI_28nm_8960_PHY_CTRL_2				0x00000178
-
-#define REG_DSI_28nm_8960_PHY_CTRL_3				0x0000017c
-
-#define REG_DSI_28nm_8960_PHY_STRENGTH_0			0x00000180
-
-#define REG_DSI_28nm_8960_PHY_STRENGTH_1			0x00000184
-
-#define REG_DSI_28nm_8960_PHY_STRENGTH_2			0x00000188
-
-#define REG_DSI_28nm_8960_PHY_BIST_CTRL_0			0x0000018c
-
-#define REG_DSI_28nm_8960_PHY_BIST_CTRL_1			0x00000190
-
-#define REG_DSI_28nm_8960_PHY_BIST_CTRL_2			0x00000194
-
-#define REG_DSI_28nm_8960_PHY_BIST_CTRL_3			0x00000198
-
-#define REG_DSI_28nm_8960_PHY_BIST_CTRL_4			0x0000019c
-
-#define REG_DSI_28nm_8960_PHY_LDO_CTRL				0x000001b0
-
-#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_0		0x00000000
-
-#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_1		0x00000004
-
-#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_2		0x00000008
-
-#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_3		0x0000000c
-
-#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_4		0x00000010
-
-#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_5		0x00000014
-
-#define REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CAL_PWR_CFG	0x00000018
-
-#define REG_DSI_28nm_8960_PHY_MISC_CAL_HW_TRIGGER		0x00000028
-
-#define REG_DSI_28nm_8960_PHY_MISC_CAL_SW_CFG_0			0x0000002c
-
-#define REG_DSI_28nm_8960_PHY_MISC_CAL_SW_CFG_1			0x00000030
-
-#define REG_DSI_28nm_8960_PHY_MISC_CAL_SW_CFG_2			0x00000034
-
-#define REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_0			0x00000038
-
-#define REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_1			0x0000003c
-
-#define REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_2			0x00000040
-
-#define REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_3			0x00000044
-
-#define REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_4			0x00000048
-
-#define REG_DSI_28nm_8960_PHY_MISC_CAL_STATUS			0x00000050
-#define DSI_28nm_8960_PHY_MISC_CAL_STATUS_CAL_BUSY		0x00000010
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_0			0x00000000
-#define DSI_28nm_8960_PHY_PLL_CTRL_0_ENABLE			0x00000001
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_1			0x00000004
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_2			0x00000008
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_3			0x0000000c
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_4			0x00000010
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_5			0x00000014
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_6			0x00000018
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_7			0x0000001c
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_8			0x00000020
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_9			0x00000024
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_10			0x00000028
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_11			0x0000002c
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_12			0x00000030
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_13			0x00000034
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_14			0x00000038
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_15			0x0000003c
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_16			0x00000040
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_17			0x00000044
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_18			0x00000048
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_19			0x0000004c
-
-#define REG_DSI_28nm_8960_PHY_PLL_CTRL_20			0x00000050
-
-#define REG_DSI_28nm_8960_PHY_PLL_RDY				0x00000080
-#define DSI_28nm_8960_PHY_PLL_RDY_PLL_RDY			0x00000001
-
-
-#endif /* DSI_PHY_28NM_8960_XML */
diff --git a/drivers/gpu/drm/msm/dsi/dsi_phy_7nm.xml.h b/drivers/gpu/drm/msm/dsi/dsi_phy_7nm.xml.h
deleted file mode 100644
index 584cbd0205ef..000000000000
--- a/drivers/gpu/drm/msm/dsi/dsi_phy_7nm.xml.h
+++ /dev/null
@@ -1,483 +0,0 @@
-#ifndef DSI_PHY_7NM_XML
-#define DSI_PHY_7NM_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-#define REG_DSI_7nm_PHY_CMN_REVISION_ID0			0x00000000
-
-#define REG_DSI_7nm_PHY_CMN_REVISION_ID1			0x00000004
-
-#define REG_DSI_7nm_PHY_CMN_REVISION_ID2			0x00000008
-
-#define REG_DSI_7nm_PHY_CMN_REVISION_ID3			0x0000000c
-
-#define REG_DSI_7nm_PHY_CMN_CLK_CFG0				0x00000010
-
-#define REG_DSI_7nm_PHY_CMN_CLK_CFG1				0x00000014
-
-#define REG_DSI_7nm_PHY_CMN_GLBL_CTRL				0x00000018
-
-#define REG_DSI_7nm_PHY_CMN_RBUF_CTRL				0x0000001c
-
-#define REG_DSI_7nm_PHY_CMN_VREG_CTRL_0				0x00000020
-
-#define REG_DSI_7nm_PHY_CMN_CTRL_0				0x00000024
-
-#define REG_DSI_7nm_PHY_CMN_CTRL_1				0x00000028
-
-#define REG_DSI_7nm_PHY_CMN_CTRL_2				0x0000002c
-
-#define REG_DSI_7nm_PHY_CMN_CTRL_3				0x00000030
-
-#define REG_DSI_7nm_PHY_CMN_LANE_CFG0				0x00000034
-
-#define REG_DSI_7nm_PHY_CMN_LANE_CFG1				0x00000038
-
-#define REG_DSI_7nm_PHY_CMN_PLL_CNTRL				0x0000003c
-
-#define REG_DSI_7nm_PHY_CMN_DPHY_SOT				0x00000040
-
-#define REG_DSI_7nm_PHY_CMN_LANE_CTRL0				0x000000a0
-
-#define REG_DSI_7nm_PHY_CMN_LANE_CTRL1				0x000000a4
-
-#define REG_DSI_7nm_PHY_CMN_LANE_CTRL2				0x000000a8
-
-#define REG_DSI_7nm_PHY_CMN_LANE_CTRL3				0x000000ac
-
-#define REG_DSI_7nm_PHY_CMN_LANE_CTRL4				0x000000b0
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_0			0x000000b4
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_1			0x000000b8
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_2			0x000000bc
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_3			0x000000c0
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_4			0x000000c4
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_5			0x000000c8
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_6			0x000000cc
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_7			0x000000d0
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_8			0x000000d4
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_9			0x000000d8
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_10			0x000000dc
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_11			0x000000e0
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_12			0x000000e4
-
-#define REG_DSI_7nm_PHY_CMN_TIMING_CTRL_13			0x000000e8
-
-#define REG_DSI_7nm_PHY_CMN_GLBL_HSTX_STR_CTRL_0		0x000000ec
-
-#define REG_DSI_7nm_PHY_CMN_GLBL_HSTX_STR_CTRL_1		0x000000f0
-
-#define REG_DSI_7nm_PHY_CMN_GLBL_RESCODE_OFFSET_TOP_CTRL	0x000000f4
-
-#define REG_DSI_7nm_PHY_CMN_GLBL_RESCODE_OFFSET_BOT_CTRL	0x000000f8
-
-#define REG_DSI_7nm_PHY_CMN_GLBL_RESCODE_OFFSET_MID_CTRL	0x000000fc
-
-#define REG_DSI_7nm_PHY_CMN_GLBL_LPTX_STR_CTRL			0x00000100
-
-#define REG_DSI_7nm_PHY_CMN_GLBL_PEMPH_CTRL_0			0x00000104
-
-#define REG_DSI_7nm_PHY_CMN_GLBL_PEMPH_CTRL_1			0x00000108
-
-#define REG_DSI_7nm_PHY_CMN_GLBL_STR_SWI_CAL_SEL_CTRL		0x0000010c
-
-#define REG_DSI_7nm_PHY_CMN_VREG_CTRL_1				0x00000110
-
-#define REG_DSI_7nm_PHY_CMN_CTRL_4				0x00000114
-
-#define REG_DSI_7nm_PHY_CMN_GLBL_DIGTOP_SPARE4			0x00000128
-
-#define REG_DSI_7nm_PHY_CMN_PHY_STATUS				0x00000140
-
-#define REG_DSI_7nm_PHY_CMN_LANE_STATUS0			0x00000148
-
-#define REG_DSI_7nm_PHY_CMN_LANE_STATUS1			0x0000014c
-
-#define REG_DSI_7nm_PHY_CMN_GLBL_DIGTOP_SPARE10			0x000001ac
-
-static inline uint32_t REG_DSI_7nm_PHY_LN(uint32_t i0) { return 0x00000000 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_7nm_PHY_LN_CFG0(uint32_t i0) { return 0x00000000 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_7nm_PHY_LN_CFG1(uint32_t i0) { return 0x00000004 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_7nm_PHY_LN_CFG2(uint32_t i0) { return 0x00000008 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_7nm_PHY_LN_TEST_DATAPATH(uint32_t i0) { return 0x0000000c + 0x80*i0; }
-
-static inline uint32_t REG_DSI_7nm_PHY_LN_PIN_SWAP(uint32_t i0) { return 0x00000010 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_7nm_PHY_LN_LPRX_CTRL(uint32_t i0) { return 0x00000014 + 0x80*i0; }
-
-static inline uint32_t REG_DSI_7nm_PHY_LN_TX_DCTRL(uint32_t i0) { return 0x00000018 + 0x80*i0; }
-
-#define REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_ONE			0x00000000
-
-#define REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_TWO			0x00000004
-
-#define REG_DSI_7nm_PHY_PLL_INT_LOOP_SETTINGS			0x00000008
-
-#define REG_DSI_7nm_PHY_PLL_INT_LOOP_SETTINGS_TWO		0x0000000c
-
-#define REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_THREE		0x00000010
-
-#define REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_FOUR		0x00000014
-
-#define REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_FIVE		0x00000018
-
-#define REG_DSI_7nm_PHY_PLL_INT_LOOP_CONTROLS			0x0000001c
-
-#define REG_DSI_7nm_PHY_PLL_DSM_DIVIDER				0x00000020
-
-#define REG_DSI_7nm_PHY_PLL_FEEDBACK_DIVIDER			0x00000024
-
-#define REG_DSI_7nm_PHY_PLL_SYSTEM_MUXES			0x00000028
-
-#define REG_DSI_7nm_PHY_PLL_FREQ_UPDATE_CONTROL_OVERRIDES	0x0000002c
-
-#define REG_DSI_7nm_PHY_PLL_CMODE				0x00000030
-
-#define REG_DSI_7nm_PHY_PLL_PSM_CTRL				0x00000034
-
-#define REG_DSI_7nm_PHY_PLL_RSM_CTRL				0x00000038
-
-#define REG_DSI_7nm_PHY_PLL_VCO_TUNE_MAP			0x0000003c
-
-#define REG_DSI_7nm_PHY_PLL_PLL_CNTRL				0x00000040
-
-#define REG_DSI_7nm_PHY_PLL_CALIBRATION_SETTINGS		0x00000044
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_CAL_TIMER_LOW		0x00000048
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_CAL_TIMER_HIGH		0x0000004c
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_CAL_SETTINGS		0x00000050
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_MIN			0x00000054
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_MAX			0x00000058
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_PFILT			0x0000005c
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_IFILT			0x00000060
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_CAL_SETTINGS_TWO		0x00000064
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_CAL_SETTINGS_THREE		0x00000068
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_CAL_SETTINGS_FOUR		0x0000006c
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_ICODE_HIGH			0x00000070
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_ICODE_LOW			0x00000074
-
-#define REG_DSI_7nm_PHY_PLL_FREQ_DETECT_SETTINGS_ONE		0x00000078
-
-#define REG_DSI_7nm_PHY_PLL_FREQ_DETECT_THRESH			0x0000007c
-
-#define REG_DSI_7nm_PHY_PLL_FREQ_DET_REFCLK_HIGH		0x00000080
-
-#define REG_DSI_7nm_PHY_PLL_FREQ_DET_REFCLK_LOW			0x00000084
-
-#define REG_DSI_7nm_PHY_PLL_FREQ_DET_PLLCLK_HIGH		0x00000088
-
-#define REG_DSI_7nm_PHY_PLL_FREQ_DET_PLLCLK_LOW			0x0000008c
-
-#define REG_DSI_7nm_PHY_PLL_PFILT				0x00000090
-
-#define REG_DSI_7nm_PHY_PLL_IFILT				0x00000094
-
-#define REG_DSI_7nm_PHY_PLL_PLL_GAIN				0x00000098
-
-#define REG_DSI_7nm_PHY_PLL_ICODE_LOW				0x0000009c
-
-#define REG_DSI_7nm_PHY_PLL_ICODE_HIGH				0x000000a0
-
-#define REG_DSI_7nm_PHY_PLL_LOCKDET				0x000000a4
-
-#define REG_DSI_7nm_PHY_PLL_OUTDIV				0x000000a8
-
-#define REG_DSI_7nm_PHY_PLL_FASTLOCK_CONTROL			0x000000ac
-
-#define REG_DSI_7nm_PHY_PLL_PASS_OUT_OVERRIDE_ONE		0x000000b0
-
-#define REG_DSI_7nm_PHY_PLL_PASS_OUT_OVERRIDE_TWO		0x000000b4
-
-#define REG_DSI_7nm_PHY_PLL_CORE_OVERRIDE			0x000000b8
-
-#define REG_DSI_7nm_PHY_PLL_CORE_INPUT_OVERRIDE			0x000000bc
-
-#define REG_DSI_7nm_PHY_PLL_RATE_CHANGE				0x000000c0
-
-#define REG_DSI_7nm_PHY_PLL_PLL_DIGITAL_TIMERS			0x000000c4
-
-#define REG_DSI_7nm_PHY_PLL_PLL_DIGITAL_TIMERS_TWO		0x000000c8
-
-#define REG_DSI_7nm_PHY_PLL_DECIMAL_DIV_START			0x000000cc
-
-#define REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_LOW			0x000000d0
-
-#define REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_MID			0x000000d4
-
-#define REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_HIGH			0x000000d8
-
-#define REG_DSI_7nm_PHY_PLL_DEC_FRAC_MUXES			0x000000dc
-
-#define REG_DSI_7nm_PHY_PLL_DECIMAL_DIV_START_1			0x000000e0
-
-#define REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_LOW_1		0x000000e4
-
-#define REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_MID_1		0x000000e8
-
-#define REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_HIGH_1		0x000000ec
-
-#define REG_DSI_7nm_PHY_PLL_DECIMAL_DIV_START_2			0x000000f0
-
-#define REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_LOW_2		0x000000f4
-
-#define REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_MID_2		0x000000f8
-
-#define REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_HIGH_2		0x000000fc
-
-#define REG_DSI_7nm_PHY_PLL_MASH_CONTROL			0x00000100
-
-#define REG_DSI_7nm_PHY_PLL_SSC_STEPSIZE_LOW			0x00000104
-
-#define REG_DSI_7nm_PHY_PLL_SSC_STEPSIZE_HIGH			0x00000108
-
-#define REG_DSI_7nm_PHY_PLL_SSC_DIV_PER_LOW			0x0000010c
-
-#define REG_DSI_7nm_PHY_PLL_SSC_DIV_PER_HIGH			0x00000110
-
-#define REG_DSI_7nm_PHY_PLL_SSC_ADJPER_LOW			0x00000114
-
-#define REG_DSI_7nm_PHY_PLL_SSC_ADJPER_HIGH			0x00000118
-
-#define REG_DSI_7nm_PHY_PLL_SSC_MUX_CONTROL			0x0000011c
-
-#define REG_DSI_7nm_PHY_PLL_SSC_STEPSIZE_LOW_1			0x00000120
-
-#define REG_DSI_7nm_PHY_PLL_SSC_STEPSIZE_HIGH_1			0x00000124
-
-#define REG_DSI_7nm_PHY_PLL_SSC_DIV_PER_LOW_1			0x00000128
-
-#define REG_DSI_7nm_PHY_PLL_SSC_DIV_PER_HIGH_1			0x0000012c
-
-#define REG_DSI_7nm_PHY_PLL_SSC_ADJPER_LOW_1			0x00000130
-
-#define REG_DSI_7nm_PHY_PLL_SSC_ADJPER_HIGH_1			0x00000134
-
-#define REG_DSI_7nm_PHY_PLL_SSC_STEPSIZE_LOW_2			0x00000138
-
-#define REG_DSI_7nm_PHY_PLL_SSC_STEPSIZE_HIGH_2			0x0000013c
-
-#define REG_DSI_7nm_PHY_PLL_SSC_DIV_PER_LOW_2			0x00000140
-
-#define REG_DSI_7nm_PHY_PLL_SSC_DIV_PER_HIGH_2			0x00000144
-
-#define REG_DSI_7nm_PHY_PLL_SSC_ADJPER_LOW_2			0x00000148
-
-#define REG_DSI_7nm_PHY_PLL_SSC_ADJPER_HIGH_2			0x0000014c
-
-#define REG_DSI_7nm_PHY_PLL_SSC_CONTROL				0x00000150
-
-#define REG_DSI_7nm_PHY_PLL_PLL_OUTDIV_RATE			0x00000154
-
-#define REG_DSI_7nm_PHY_PLL_PLL_LOCKDET_RATE_1			0x00000158
-
-#define REG_DSI_7nm_PHY_PLL_PLL_LOCKDET_RATE_2			0x0000015c
-
-#define REG_DSI_7nm_PHY_PLL_PLL_PROP_GAIN_RATE_1		0x00000160
-
-#define REG_DSI_7nm_PHY_PLL_PLL_PROP_GAIN_RATE_2		0x00000164
-
-#define REG_DSI_7nm_PHY_PLL_PLL_BAND_SEL_RATE_1			0x00000168
-
-#define REG_DSI_7nm_PHY_PLL_PLL_BAND_SEL_RATE_2			0x0000016c
-
-#define REG_DSI_7nm_PHY_PLL_PLL_INT_GAIN_IFILT_BAND_1		0x00000170
-
-#define REG_DSI_7nm_PHY_PLL_PLL_INT_GAIN_IFILT_BAND_2		0x00000174
-
-#define REG_DSI_7nm_PHY_PLL_PLL_FL_INT_GAIN_PFILT_BAND_1	0x00000178
-
-#define REG_DSI_7nm_PHY_PLL_PLL_FL_INT_GAIN_PFILT_BAND_2	0x0000017c
-
-#define REG_DSI_7nm_PHY_PLL_PLL_FASTLOCK_EN_BAND		0x00000180
-
-#define REG_DSI_7nm_PHY_PLL_FREQ_TUNE_ACCUM_INIT_MID		0x00000184
-
-#define REG_DSI_7nm_PHY_PLL_FREQ_TUNE_ACCUM_INIT_HIGH		0x00000188
-
-#define REG_DSI_7nm_PHY_PLL_FREQ_TUNE_ACCUM_INIT_MUX		0x0000018c
-
-#define REG_DSI_7nm_PHY_PLL_PLL_LOCK_OVERRIDE			0x00000190
-
-#define REG_DSI_7nm_PHY_PLL_PLL_LOCK_DELAY			0x00000194
-
-#define REG_DSI_7nm_PHY_PLL_PLL_LOCK_MIN_DELAY			0x00000198
-
-#define REG_DSI_7nm_PHY_PLL_CLOCK_INVERTERS			0x0000019c
-
-#define REG_DSI_7nm_PHY_PLL_SPARE_AND_JPC_OVERRIDES		0x000001a0
-
-#define REG_DSI_7nm_PHY_PLL_BIAS_CONTROL_1			0x000001a4
-
-#define REG_DSI_7nm_PHY_PLL_BIAS_CONTROL_2			0x000001a8
-
-#define REG_DSI_7nm_PHY_PLL_ALOG_OBSV_BUS_CTRL_1		0x000001ac
-
-#define REG_DSI_7nm_PHY_PLL_COMMON_STATUS_ONE			0x000001b0
-
-#define REG_DSI_7nm_PHY_PLL_COMMON_STATUS_TWO			0x000001b4
-
-#define REG_DSI_7nm_PHY_PLL_BAND_SEL_CAL			0x000001b8
-
-#define REG_DSI_7nm_PHY_PLL_ICODE_ACCUM_STATUS_LOW		0x000001bc
-
-#define REG_DSI_7nm_PHY_PLL_ICODE_ACCUM_STATUS_HIGH		0x000001c0
-
-#define REG_DSI_7nm_PHY_PLL_FD_OUT_LOW				0x000001c4
-
-#define REG_DSI_7nm_PHY_PLL_FD_OUT_HIGH				0x000001c8
-
-#define REG_DSI_7nm_PHY_PLL_ALOG_OBSV_BUS_STATUS_1		0x000001cc
-
-#define REG_DSI_7nm_PHY_PLL_PLL_MISC_CONFIG			0x000001d0
-
-#define REG_DSI_7nm_PHY_PLL_FLL_CONFIG				0x000001d4
-
-#define REG_DSI_7nm_PHY_PLL_FLL_FREQ_ACQ_TIME			0x000001d8
-
-#define REG_DSI_7nm_PHY_PLL_FLL_CODE0				0x000001dc
-
-#define REG_DSI_7nm_PHY_PLL_FLL_CODE1				0x000001e0
-
-#define REG_DSI_7nm_PHY_PLL_FLL_GAIN0				0x000001e4
-
-#define REG_DSI_7nm_PHY_PLL_FLL_GAIN1				0x000001e8
-
-#define REG_DSI_7nm_PHY_PLL_SW_RESET				0x000001ec
-
-#define REG_DSI_7nm_PHY_PLL_FAST_PWRUP				0x000001f0
-
-#define REG_DSI_7nm_PHY_PLL_LOCKTIME0				0x000001f4
-
-#define REG_DSI_7nm_PHY_PLL_LOCKTIME1				0x000001f8
-
-#define REG_DSI_7nm_PHY_PLL_DEBUG_BUS_SEL			0x000001fc
-
-#define REG_DSI_7nm_PHY_PLL_DEBUG_BUS0				0x00000200
-
-#define REG_DSI_7nm_PHY_PLL_DEBUG_BUS1				0x00000204
-
-#define REG_DSI_7nm_PHY_PLL_DEBUG_BUS2				0x00000208
-
-#define REG_DSI_7nm_PHY_PLL_DEBUG_BUS3				0x0000020c
-
-#define REG_DSI_7nm_PHY_PLL_ANALOG_FLL_CONTROL_OVERRIDES	0x00000210
-
-#define REG_DSI_7nm_PHY_PLL_VCO_CONFIG				0x00000214
-
-#define REG_DSI_7nm_PHY_PLL_VCO_CAL_CODE1_MODE0_STATUS		0x00000218
-
-#define REG_DSI_7nm_PHY_PLL_VCO_CAL_CODE1_MODE1_STATUS		0x0000021c
-
-#define REG_DSI_7nm_PHY_PLL_RESET_SM_STATUS			0x00000220
-
-#define REG_DSI_7nm_PHY_PLL_TDC_OFFSET				0x00000224
-
-#define REG_DSI_7nm_PHY_PLL_PS3_PWRDOWN_CONTROLS		0x00000228
-
-#define REG_DSI_7nm_PHY_PLL_PS4_PWRDOWN_CONTROLS		0x0000022c
-
-#define REG_DSI_7nm_PHY_PLL_PLL_RST_CONTROLS			0x00000230
-
-#define REG_DSI_7nm_PHY_PLL_GEAR_BAND_SELECT_CONTROLS		0x00000234
-
-#define REG_DSI_7nm_PHY_PLL_PSM_CLK_CONTROLS			0x00000238
-
-#define REG_DSI_7nm_PHY_PLL_SYSTEM_MUXES_2			0x0000023c
-
-#define REG_DSI_7nm_PHY_PLL_VCO_CONFIG_1			0x00000240
-
-#define REG_DSI_7nm_PHY_PLL_VCO_CONFIG_2			0x00000244
-
-#define REG_DSI_7nm_PHY_PLL_CLOCK_INVERTERS_1			0x00000248
-
-#define REG_DSI_7nm_PHY_PLL_CLOCK_INVERTERS_2			0x0000024c
-
-#define REG_DSI_7nm_PHY_PLL_CMODE_1				0x00000250
-
-#define REG_DSI_7nm_PHY_PLL_CMODE_2				0x00000254
-
-#define REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_FIVE_1		0x00000258
-
-#define REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_FIVE_2		0x0000025c
-
-#define REG_DSI_7nm_PHY_PLL_PERF_OPTIMIZE			0x00000260
-
-
-#endif /* DSI_PHY_7NM_XML */
diff --git a/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h b/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
deleted file mode 100644
index 7062f7164216..000000000000
--- a/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
+++ /dev/null
@@ -1,131 +0,0 @@
-#ifndef MMSS_CC_XML
-#define MMSS_CC_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-enum mmss_cc_clk {
-	CLK = 0,
-	PCLK = 1,
-};
-
-#define REG_MMSS_CC_AHB						0x00000008
-
-static inline uint32_t __offset_CLK(enum mmss_cc_clk idx)
-{
-	switch (idx) {
-		case CLK: return 0x0000004c;
-		case PCLK: return 0x00000130;
-		default: return INVALID_IDX(idx);
-	}
-}
-static inline uint32_t REG_MMSS_CC_CLK(enum mmss_cc_clk i0) { return 0x00000000 + __offset_CLK(i0); }
-
-static inline uint32_t REG_MMSS_CC_CLK_CC(enum mmss_cc_clk i0) { return 0x00000000 + __offset_CLK(i0); }
-#define MMSS_CC_CLK_CC_CLK_EN					0x00000001
-#define MMSS_CC_CLK_CC_ROOT_EN					0x00000004
-#define MMSS_CC_CLK_CC_MND_EN					0x00000020
-#define MMSS_CC_CLK_CC_MND_MODE__MASK				0x000000c0
-#define MMSS_CC_CLK_CC_MND_MODE__SHIFT				6
-static inline uint32_t MMSS_CC_CLK_CC_MND_MODE(uint32_t val)
-{
-	return ((val) << MMSS_CC_CLK_CC_MND_MODE__SHIFT) & MMSS_CC_CLK_CC_MND_MODE__MASK;
-}
-#define MMSS_CC_CLK_CC_PMXO_SEL__MASK				0x00000300
-#define MMSS_CC_CLK_CC_PMXO_SEL__SHIFT				8
-static inline uint32_t MMSS_CC_CLK_CC_PMXO_SEL(uint32_t val)
-{
-	return ((val) << MMSS_CC_CLK_CC_PMXO_SEL__SHIFT) & MMSS_CC_CLK_CC_PMXO_SEL__MASK;
-}
-
-static inline uint32_t REG_MMSS_CC_CLK_MD(enum mmss_cc_clk i0) { return 0x00000004 + __offset_CLK(i0); }
-#define MMSS_CC_CLK_MD_D__MASK					0x000000ff
-#define MMSS_CC_CLK_MD_D__SHIFT					0
-static inline uint32_t MMSS_CC_CLK_MD_D(uint32_t val)
-{
-	return ((val) << MMSS_CC_CLK_MD_D__SHIFT) & MMSS_CC_CLK_MD_D__MASK;
-}
-#define MMSS_CC_CLK_MD_M__MASK					0x0000ff00
-#define MMSS_CC_CLK_MD_M__SHIFT					8
-static inline uint32_t MMSS_CC_CLK_MD_M(uint32_t val)
-{
-	return ((val) << MMSS_CC_CLK_MD_M__SHIFT) & MMSS_CC_CLK_MD_M__MASK;
-}
-
-static inline uint32_t REG_MMSS_CC_CLK_NS(enum mmss_cc_clk i0) { return 0x00000008 + __offset_CLK(i0); }
-#define MMSS_CC_CLK_NS_SRC__MASK				0x0000000f
-#define MMSS_CC_CLK_NS_SRC__SHIFT				0
-static inline uint32_t MMSS_CC_CLK_NS_SRC(uint32_t val)
-{
-	return ((val) << MMSS_CC_CLK_NS_SRC__SHIFT) & MMSS_CC_CLK_NS_SRC__MASK;
-}
-#define MMSS_CC_CLK_NS_PRE_DIV_FUNC__MASK			0x00fff000
-#define MMSS_CC_CLK_NS_PRE_DIV_FUNC__SHIFT			12
-static inline uint32_t MMSS_CC_CLK_NS_PRE_DIV_FUNC(uint32_t val)
-{
-	return ((val) << MMSS_CC_CLK_NS_PRE_DIV_FUNC__SHIFT) & MMSS_CC_CLK_NS_PRE_DIV_FUNC__MASK;
-}
-#define MMSS_CC_CLK_NS_VAL__MASK				0xff000000
-#define MMSS_CC_CLK_NS_VAL__SHIFT				24
-static inline uint32_t MMSS_CC_CLK_NS_VAL(uint32_t val)
-{
-	return ((val) << MMSS_CC_CLK_NS_VAL__SHIFT) & MMSS_CC_CLK_NS_VAL__MASK;
-}
-
-#define REG_MMSS_CC_DSI2_PIXEL_CC				0x00000094
-
-#define REG_MMSS_CC_DSI2_PIXEL_NS				0x000000e4
-
-#define REG_MMSS_CC_DSI2_PIXEL_CC2				0x00000264
-
-
-#endif /* MMSS_CC_XML */
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
index e4275d3ad581..5a5dc3faa971 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
@@ -12,10 +12,10 @@
 
 #include "dsi.h"
 
-#define dsi_phy_read(offset) msm_readl((offset))
-#define dsi_phy_write(offset, data) msm_writel((data), (offset))
-#define dsi_phy_write_udelay(offset, data, delay_us) { msm_writel((data), (offset)); udelay(delay_us); }
-#define dsi_phy_write_ndelay(offset, data, delay_ns) { msm_writel((data), (offset)); ndelay(delay_ns); }
+#define dsi_phy_read(offset) readl((offset))
+#define dsi_phy_write(offset, data) writel((data), (offset))
+#define dsi_phy_write_udelay(offset, data, delay_us) { writel((data), (offset)); udelay(delay_us); }
+#define dsi_phy_write_ndelay(offset, data, delay_ns) { writel((data), (offset)); ndelay(delay_ns); }
 
 struct msm_dsi_phy_ops {
 	int (*pll_init)(struct msm_dsi_phy *phy);
diff --git a/drivers/gpu/drm/msm/dsi/sfpb.xml.h b/drivers/gpu/drm/msm/dsi/sfpb.xml.h
deleted file mode 100644
index 344a1a1620cd..000000000000
--- a/drivers/gpu/drm/msm/dsi/sfpb.xml.h
+++ /dev/null
@@ -1,70 +0,0 @@
-#ifndef SFPB_XML
-#define SFPB_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-enum sfpb_ahb_arb_master_port_en {
-	SFPB_MASTER_PORT_ENABLE = 3,
-	SFPB_MASTER_PORT_DISABLE = 0,
-};
-
-#define REG_SFPB_GPREG						0x00000058
-#define SFPB_GPREG_MASTER_PORT_EN__MASK				0x00001800
-#define SFPB_GPREG_MASTER_PORT_EN__SHIFT			11
-static inline uint32_t SFPB_GPREG_MASTER_PORT_EN(enum sfpb_ahb_arb_master_port_en val)
-{
-	return ((val) << SFPB_GPREG_MASTER_PORT_EN__SHIFT) & SFPB_GPREG_MASTER_PORT_EN__MASK;
-}
-
-
-#endif /* SFPB_XML */
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c
index c8ebd75176bb..24abcb7254cc 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.c
@@ -5,8 +5,8 @@
  * Author: Rob Clark <robdclark@gmail.com>
  */
 
+#include <linux/gpio/consumer.h>
 #include <linux/of_irq.h>
-#include <linux/of_gpio.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.h b/drivers/gpu/drm/msm/hdmi/hdmi.h
index ec5786440391..4586baf36415 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.h
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.h
@@ -115,17 +115,17 @@ void msm_hdmi_set_mode(struct hdmi *hdmi, bool power_on);
 
 static inline void hdmi_write(struct hdmi *hdmi, u32 reg, u32 data)
 {
-	msm_writel(data, hdmi->mmio + reg);
+	writel(data, hdmi->mmio + reg);
 }
 
 static inline u32 hdmi_read(struct hdmi *hdmi, u32 reg)
 {
-	return msm_readl(hdmi->mmio + reg);
+	return readl(hdmi->mmio + reg);
 }
 
 static inline u32 hdmi_qfprom_read(struct hdmi *hdmi, u32 reg)
 {
-	return msm_readl(hdmi->qfprom_mmio + reg);
+	return readl(hdmi->qfprom_mmio + reg);
 }
 
 /*
@@ -166,12 +166,12 @@ struct hdmi_phy {
 
 static inline void hdmi_phy_write(struct hdmi_phy *phy, u32 reg, u32 data)
 {
-	msm_writel(data, phy->mmio + reg);
+	writel(data, phy->mmio + reg);
 }
 
 static inline u32 hdmi_phy_read(struct hdmi_phy *phy, u32 reg)
 {
-	return msm_readl(phy->mmio + reg);
+	return readl(phy->mmio + reg);
 }
 
 int msm_hdmi_phy_resource_enable(struct hdmi_phy *phy);
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.xml.h b/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
deleted file mode 100644
index 973b460486a5..000000000000
--- a/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
+++ /dev/null
@@ -1,1399 +0,0 @@
-#ifndef HDMI_XML
-#define HDMI_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-enum hdmi_hdcp_key_state {
-	HDCP_KEYS_STATE_NO_KEYS = 0,
-	HDCP_KEYS_STATE_NOT_CHECKED = 1,
-	HDCP_KEYS_STATE_CHECKING = 2,
-	HDCP_KEYS_STATE_VALID = 3,
-	HDCP_KEYS_STATE_AKSV_NOT_VALID = 4,
-	HDCP_KEYS_STATE_CHKSUM_MISMATCH = 5,
-	HDCP_KEYS_STATE_PROD_AKSV = 6,
-	HDCP_KEYS_STATE_RESERVED = 7,
-};
-
-enum hdmi_ddc_read_write {
-	DDC_WRITE = 0,
-	DDC_READ = 1,
-};
-
-enum hdmi_acr_cts {
-	ACR_NONE = 0,
-	ACR_32 = 1,
-	ACR_44 = 2,
-	ACR_48 = 3,
-};
-
-#define REG_HDMI_CTRL						0x00000000
-#define HDMI_CTRL_ENABLE					0x00000001
-#define HDMI_CTRL_HDMI						0x00000002
-#define HDMI_CTRL_ENCRYPTED					0x00000004
-
-#define REG_HDMI_AUDIO_PKT_CTRL1				0x00000020
-#define HDMI_AUDIO_PKT_CTRL1_AUDIO_SAMPLE_SEND			0x00000001
-
-#define REG_HDMI_ACR_PKT_CTRL					0x00000024
-#define HDMI_ACR_PKT_CTRL_CONT					0x00000001
-#define HDMI_ACR_PKT_CTRL_SEND					0x00000002
-#define HDMI_ACR_PKT_CTRL_SELECT__MASK				0x00000030
-#define HDMI_ACR_PKT_CTRL_SELECT__SHIFT				4
-static inline uint32_t HDMI_ACR_PKT_CTRL_SELECT(enum hdmi_acr_cts val)
-{
-	return ((val) << HDMI_ACR_PKT_CTRL_SELECT__SHIFT) & HDMI_ACR_PKT_CTRL_SELECT__MASK;
-}
-#define HDMI_ACR_PKT_CTRL_SOURCE				0x00000100
-#define HDMI_ACR_PKT_CTRL_N_MULTIPLIER__MASK			0x00070000
-#define HDMI_ACR_PKT_CTRL_N_MULTIPLIER__SHIFT			16
-static inline uint32_t HDMI_ACR_PKT_CTRL_N_MULTIPLIER(uint32_t val)
-{
-	return ((val) << HDMI_ACR_PKT_CTRL_N_MULTIPLIER__SHIFT) & HDMI_ACR_PKT_CTRL_N_MULTIPLIER__MASK;
-}
-#define HDMI_ACR_PKT_CTRL_AUDIO_PRIORITY			0x80000000
-
-#define REG_HDMI_VBI_PKT_CTRL					0x00000028
-#define HDMI_VBI_PKT_CTRL_GC_ENABLE				0x00000010
-#define HDMI_VBI_PKT_CTRL_GC_EVERY_FRAME			0x00000020
-#define HDMI_VBI_PKT_CTRL_ISRC_SEND				0x00000100
-#define HDMI_VBI_PKT_CTRL_ISRC_CONTINUOUS			0x00000200
-#define HDMI_VBI_PKT_CTRL_ACP_SEND				0x00001000
-#define HDMI_VBI_PKT_CTRL_ACP_SRC_SW				0x00002000
-
-#define REG_HDMI_INFOFRAME_CTRL0				0x0000002c
-#define HDMI_INFOFRAME_CTRL0_AVI_SEND				0x00000001
-#define HDMI_INFOFRAME_CTRL0_AVI_CONT				0x00000002
-#define HDMI_INFOFRAME_CTRL0_AUDIO_INFO_SEND			0x00000010
-#define HDMI_INFOFRAME_CTRL0_AUDIO_INFO_CONT			0x00000020
-#define HDMI_INFOFRAME_CTRL0_AUDIO_INFO_SOURCE			0x00000040
-#define HDMI_INFOFRAME_CTRL0_AUDIO_INFO_UPDATE			0x00000080
-
-#define REG_HDMI_INFOFRAME_CTRL1				0x00000030
-#define HDMI_INFOFRAME_CTRL1_AVI_INFO_LINE__MASK		0x0000003f
-#define HDMI_INFOFRAME_CTRL1_AVI_INFO_LINE__SHIFT		0
-static inline uint32_t HDMI_INFOFRAME_CTRL1_AVI_INFO_LINE(uint32_t val)
-{
-	return ((val) << HDMI_INFOFRAME_CTRL1_AVI_INFO_LINE__SHIFT) & HDMI_INFOFRAME_CTRL1_AVI_INFO_LINE__MASK;
-}
-#define HDMI_INFOFRAME_CTRL1_AUDIO_INFO_LINE__MASK		0x00003f00
-#define HDMI_INFOFRAME_CTRL1_AUDIO_INFO_LINE__SHIFT		8
-static inline uint32_t HDMI_INFOFRAME_CTRL1_AUDIO_INFO_LINE(uint32_t val)
-{
-	return ((val) << HDMI_INFOFRAME_CTRL1_AUDIO_INFO_LINE__SHIFT) & HDMI_INFOFRAME_CTRL1_AUDIO_INFO_LINE__MASK;
-}
-#define HDMI_INFOFRAME_CTRL1_MPEG_INFO_LINE__MASK		0x003f0000
-#define HDMI_INFOFRAME_CTRL1_MPEG_INFO_LINE__SHIFT		16
-static inline uint32_t HDMI_INFOFRAME_CTRL1_MPEG_INFO_LINE(uint32_t val)
-{
-	return ((val) << HDMI_INFOFRAME_CTRL1_MPEG_INFO_LINE__SHIFT) & HDMI_INFOFRAME_CTRL1_MPEG_INFO_LINE__MASK;
-}
-#define HDMI_INFOFRAME_CTRL1_VENSPEC_INFO_LINE__MASK		0x3f000000
-#define HDMI_INFOFRAME_CTRL1_VENSPEC_INFO_LINE__SHIFT		24
-static inline uint32_t HDMI_INFOFRAME_CTRL1_VENSPEC_INFO_LINE(uint32_t val)
-{
-	return ((val) << HDMI_INFOFRAME_CTRL1_VENSPEC_INFO_LINE__SHIFT) & HDMI_INFOFRAME_CTRL1_VENSPEC_INFO_LINE__MASK;
-}
-
-#define REG_HDMI_GEN_PKT_CTRL					0x00000034
-#define HDMI_GEN_PKT_CTRL_GENERIC0_SEND				0x00000001
-#define HDMI_GEN_PKT_CTRL_GENERIC0_CONT				0x00000002
-#define HDMI_GEN_PKT_CTRL_GENERIC0_UPDATE__MASK			0x0000000c
-#define HDMI_GEN_PKT_CTRL_GENERIC0_UPDATE__SHIFT		2
-static inline uint32_t HDMI_GEN_PKT_CTRL_GENERIC0_UPDATE(uint32_t val)
-{
-	return ((val) << HDMI_GEN_PKT_CTRL_GENERIC0_UPDATE__SHIFT) & HDMI_GEN_PKT_CTRL_GENERIC0_UPDATE__MASK;
-}
-#define HDMI_GEN_PKT_CTRL_GENERIC1_SEND				0x00000010
-#define HDMI_GEN_PKT_CTRL_GENERIC1_CONT				0x00000020
-#define HDMI_GEN_PKT_CTRL_GENERIC0_LINE__MASK			0x003f0000
-#define HDMI_GEN_PKT_CTRL_GENERIC0_LINE__SHIFT			16
-static inline uint32_t HDMI_GEN_PKT_CTRL_GENERIC0_LINE(uint32_t val)
-{
-	return ((val) << HDMI_GEN_PKT_CTRL_GENERIC0_LINE__SHIFT) & HDMI_GEN_PKT_CTRL_GENERIC0_LINE__MASK;
-}
-#define HDMI_GEN_PKT_CTRL_GENERIC1_LINE__MASK			0x3f000000
-#define HDMI_GEN_PKT_CTRL_GENERIC1_LINE__SHIFT			24
-static inline uint32_t HDMI_GEN_PKT_CTRL_GENERIC1_LINE(uint32_t val)
-{
-	return ((val) << HDMI_GEN_PKT_CTRL_GENERIC1_LINE__SHIFT) & HDMI_GEN_PKT_CTRL_GENERIC1_LINE__MASK;
-}
-
-#define REG_HDMI_GC						0x00000040
-#define HDMI_GC_MUTE						0x00000001
-
-#define REG_HDMI_AUDIO_PKT_CTRL2				0x00000044
-#define HDMI_AUDIO_PKT_CTRL2_OVERRIDE				0x00000001
-#define HDMI_AUDIO_PKT_CTRL2_LAYOUT				0x00000002
-
-static inline uint32_t REG_HDMI_AVI_INFO(uint32_t i0) { return 0x0000006c + 0x4*i0; }
-
-#define REG_HDMI_GENERIC0_HDR					0x00000084
-
-static inline uint32_t REG_HDMI_GENERIC0(uint32_t i0) { return 0x00000088 + 0x4*i0; }
-
-#define REG_HDMI_GENERIC1_HDR					0x000000a4
-
-static inline uint32_t REG_HDMI_GENERIC1(uint32_t i0) { return 0x000000a8 + 0x4*i0; }
-
-static inline uint32_t REG_HDMI_ACR(enum hdmi_acr_cts i0) { return 0x000000c4 + 0x8*i0; }
-
-static inline uint32_t REG_HDMI_ACR_0(enum hdmi_acr_cts i0) { return 0x000000c4 + 0x8*i0; }
-#define HDMI_ACR_0_CTS__MASK					0xfffff000
-#define HDMI_ACR_0_CTS__SHIFT					12
-static inline uint32_t HDMI_ACR_0_CTS(uint32_t val)
-{
-	return ((val) << HDMI_ACR_0_CTS__SHIFT) & HDMI_ACR_0_CTS__MASK;
-}
-
-static inline uint32_t REG_HDMI_ACR_1(enum hdmi_acr_cts i0) { return 0x000000c8 + 0x8*i0; }
-#define HDMI_ACR_1_N__MASK					0xffffffff
-#define HDMI_ACR_1_N__SHIFT					0
-static inline uint32_t HDMI_ACR_1_N(uint32_t val)
-{
-	return ((val) << HDMI_ACR_1_N__SHIFT) & HDMI_ACR_1_N__MASK;
-}
-
-#define REG_HDMI_AUDIO_INFO0					0x000000e4
-#define HDMI_AUDIO_INFO0_CHECKSUM__MASK				0x000000ff
-#define HDMI_AUDIO_INFO0_CHECKSUM__SHIFT			0
-static inline uint32_t HDMI_AUDIO_INFO0_CHECKSUM(uint32_t val)
-{
-	return ((val) << HDMI_AUDIO_INFO0_CHECKSUM__SHIFT) & HDMI_AUDIO_INFO0_CHECKSUM__MASK;
-}
-#define HDMI_AUDIO_INFO0_CC__MASK				0x00000700
-#define HDMI_AUDIO_INFO0_CC__SHIFT				8
-static inline uint32_t HDMI_AUDIO_INFO0_CC(uint32_t val)
-{
-	return ((val) << HDMI_AUDIO_INFO0_CC__SHIFT) & HDMI_AUDIO_INFO0_CC__MASK;
-}
-
-#define REG_HDMI_AUDIO_INFO1					0x000000e8
-#define HDMI_AUDIO_INFO1_CA__MASK				0x000000ff
-#define HDMI_AUDIO_INFO1_CA__SHIFT				0
-static inline uint32_t HDMI_AUDIO_INFO1_CA(uint32_t val)
-{
-	return ((val) << HDMI_AUDIO_INFO1_CA__SHIFT) & HDMI_AUDIO_INFO1_CA__MASK;
-}
-#define HDMI_AUDIO_INFO1_LSV__MASK				0x00007800
-#define HDMI_AUDIO_INFO1_LSV__SHIFT				11
-static inline uint32_t HDMI_AUDIO_INFO1_LSV(uint32_t val)
-{
-	return ((val) << HDMI_AUDIO_INFO1_LSV__SHIFT) & HDMI_AUDIO_INFO1_LSV__MASK;
-}
-#define HDMI_AUDIO_INFO1_DM_INH					0x00008000
-
-#define REG_HDMI_HDCP_CTRL					0x00000110
-#define HDMI_HDCP_CTRL_ENABLE					0x00000001
-#define HDMI_HDCP_CTRL_ENCRYPTION_ENABLE			0x00000100
-
-#define REG_HDMI_HDCP_DEBUG_CTRL				0x00000114
-#define HDMI_HDCP_DEBUG_CTRL_RNG_CIPHER				0x00000004
-
-#define REG_HDMI_HDCP_INT_CTRL					0x00000118
-#define HDMI_HDCP_INT_CTRL_AUTH_SUCCESS_INT			0x00000001
-#define HDMI_HDCP_INT_CTRL_AUTH_SUCCESS_ACK			0x00000002
-#define HDMI_HDCP_INT_CTRL_AUTH_SUCCESS_MASK			0x00000004
-#define HDMI_HDCP_INT_CTRL_AUTH_FAIL_INT			0x00000010
-#define HDMI_HDCP_INT_CTRL_AUTH_FAIL_ACK			0x00000020
-#define HDMI_HDCP_INT_CTRL_AUTH_FAIL_MASK			0x00000040
-#define HDMI_HDCP_INT_CTRL_AUTH_FAIL_INFO_ACK			0x00000080
-#define HDMI_HDCP_INT_CTRL_AUTH_XFER_REQ_INT			0x00000100
-#define HDMI_HDCP_INT_CTRL_AUTH_XFER_REQ_ACK			0x00000200
-#define HDMI_HDCP_INT_CTRL_AUTH_XFER_REQ_MASK			0x00000400
-#define HDMI_HDCP_INT_CTRL_AUTH_XFER_DONE_INT			0x00001000
-#define HDMI_HDCP_INT_CTRL_AUTH_XFER_DONE_ACK			0x00002000
-#define HDMI_HDCP_INT_CTRL_AUTH_XFER_DONE_MASK			0x00004000
-
-#define REG_HDMI_HDCP_LINK0_STATUS				0x0000011c
-#define HDMI_HDCP_LINK0_STATUS_AN_0_READY			0x00000100
-#define HDMI_HDCP_LINK0_STATUS_AN_1_READY			0x00000200
-#define HDMI_HDCP_LINK0_STATUS_RI_MATCHES			0x00001000
-#define HDMI_HDCP_LINK0_STATUS_V_MATCHES			0x00100000
-#define HDMI_HDCP_LINK0_STATUS_KEY_STATE__MASK			0x70000000
-#define HDMI_HDCP_LINK0_STATUS_KEY_STATE__SHIFT			28
-static inline uint32_t HDMI_HDCP_LINK0_STATUS_KEY_STATE(enum hdmi_hdcp_key_state val)
-{
-	return ((val) << HDMI_HDCP_LINK0_STATUS_KEY_STATE__SHIFT) & HDMI_HDCP_LINK0_STATUS_KEY_STATE__MASK;
-}
-
-#define REG_HDMI_HDCP_DDC_CTRL_0				0x00000120
-#define HDMI_HDCP_DDC_CTRL_0_DISABLE				0x00000001
-
-#define REG_HDMI_HDCP_DDC_CTRL_1				0x00000124
-#define HDMI_HDCP_DDC_CTRL_1_FAILED_ACK				0x00000001
-
-#define REG_HDMI_HDCP_DDC_STATUS				0x00000128
-#define HDMI_HDCP_DDC_STATUS_XFER_REQ				0x00000010
-#define HDMI_HDCP_DDC_STATUS_XFER_DONE				0x00000400
-#define HDMI_HDCP_DDC_STATUS_ABORTED				0x00001000
-#define HDMI_HDCP_DDC_STATUS_TIMEOUT				0x00002000
-#define HDMI_HDCP_DDC_STATUS_NACK0				0x00004000
-#define HDMI_HDCP_DDC_STATUS_NACK1				0x00008000
-#define HDMI_HDCP_DDC_STATUS_FAILED				0x00010000
-
-#define REG_HDMI_HDCP_ENTROPY_CTRL0				0x0000012c
-
-#define REG_HDMI_HDCP_ENTROPY_CTRL1				0x0000025c
-
-#define REG_HDMI_HDCP_RESET					0x00000130
-#define HDMI_HDCP_RESET_LINK0_DEAUTHENTICATE			0x00000001
-
-#define REG_HDMI_HDCP_RCVPORT_DATA0				0x00000134
-
-#define REG_HDMI_HDCP_RCVPORT_DATA1				0x00000138
-
-#define REG_HDMI_HDCP_RCVPORT_DATA2_0				0x0000013c
-
-#define REG_HDMI_HDCP_RCVPORT_DATA2_1				0x00000140
-
-#define REG_HDMI_HDCP_RCVPORT_DATA3				0x00000144
-
-#define REG_HDMI_HDCP_RCVPORT_DATA4				0x00000148
-
-#define REG_HDMI_HDCP_RCVPORT_DATA5				0x0000014c
-
-#define REG_HDMI_HDCP_RCVPORT_DATA6				0x00000150
-
-#define REG_HDMI_HDCP_RCVPORT_DATA7				0x00000154
-
-#define REG_HDMI_HDCP_RCVPORT_DATA8				0x00000158
-
-#define REG_HDMI_HDCP_RCVPORT_DATA9				0x0000015c
-
-#define REG_HDMI_HDCP_RCVPORT_DATA10				0x00000160
-
-#define REG_HDMI_HDCP_RCVPORT_DATA11				0x00000164
-
-#define REG_HDMI_HDCP_RCVPORT_DATA12				0x00000168
-
-#define REG_HDMI_VENSPEC_INFO0					0x0000016c
-
-#define REG_HDMI_VENSPEC_INFO1					0x00000170
-
-#define REG_HDMI_VENSPEC_INFO2					0x00000174
-
-#define REG_HDMI_VENSPEC_INFO3					0x00000178
-
-#define REG_HDMI_VENSPEC_INFO4					0x0000017c
-
-#define REG_HDMI_VENSPEC_INFO5					0x00000180
-
-#define REG_HDMI_VENSPEC_INFO6					0x00000184
-
-#define REG_HDMI_AUDIO_CFG					0x000001d0
-#define HDMI_AUDIO_CFG_ENGINE_ENABLE				0x00000001
-#define HDMI_AUDIO_CFG_FIFO_WATERMARK__MASK			0x000000f0
-#define HDMI_AUDIO_CFG_FIFO_WATERMARK__SHIFT			4
-static inline uint32_t HDMI_AUDIO_CFG_FIFO_WATERMARK(uint32_t val)
-{
-	return ((val) << HDMI_AUDIO_CFG_FIFO_WATERMARK__SHIFT) & HDMI_AUDIO_CFG_FIFO_WATERMARK__MASK;
-}
-
-#define REG_HDMI_USEC_REFTIMER					0x00000208
-
-#define REG_HDMI_DDC_CTRL					0x0000020c
-#define HDMI_DDC_CTRL_GO					0x00000001
-#define HDMI_DDC_CTRL_SOFT_RESET				0x00000002
-#define HDMI_DDC_CTRL_SEND_RESET				0x00000004
-#define HDMI_DDC_CTRL_SW_STATUS_RESET				0x00000008
-#define HDMI_DDC_CTRL_TRANSACTION_CNT__MASK			0x00300000
-#define HDMI_DDC_CTRL_TRANSACTION_CNT__SHIFT			20
-static inline uint32_t HDMI_DDC_CTRL_TRANSACTION_CNT(uint32_t val)
-{
-	return ((val) << HDMI_DDC_CTRL_TRANSACTION_CNT__SHIFT) & HDMI_DDC_CTRL_TRANSACTION_CNT__MASK;
-}
-
-#define REG_HDMI_DDC_ARBITRATION				0x00000210
-#define HDMI_DDC_ARBITRATION_HW_ARBITRATION			0x00000010
-
-#define REG_HDMI_DDC_INT_CTRL					0x00000214
-#define HDMI_DDC_INT_CTRL_SW_DONE_INT				0x00000001
-#define HDMI_DDC_INT_CTRL_SW_DONE_ACK				0x00000002
-#define HDMI_DDC_INT_CTRL_SW_DONE_MASK				0x00000004
-
-#define REG_HDMI_DDC_SW_STATUS					0x00000218
-#define HDMI_DDC_SW_STATUS_NACK0				0x00001000
-#define HDMI_DDC_SW_STATUS_NACK1				0x00002000
-#define HDMI_DDC_SW_STATUS_NACK2				0x00004000
-#define HDMI_DDC_SW_STATUS_NACK3				0x00008000
-
-#define REG_HDMI_DDC_HW_STATUS					0x0000021c
-#define HDMI_DDC_HW_STATUS_DONE					0x00000008
-
-#define REG_HDMI_DDC_SPEED					0x00000220
-#define HDMI_DDC_SPEED_THRESHOLD__MASK				0x00000003
-#define HDMI_DDC_SPEED_THRESHOLD__SHIFT				0
-static inline uint32_t HDMI_DDC_SPEED_THRESHOLD(uint32_t val)
-{
-	return ((val) << HDMI_DDC_SPEED_THRESHOLD__SHIFT) & HDMI_DDC_SPEED_THRESHOLD__MASK;
-}
-#define HDMI_DDC_SPEED_PRESCALE__MASK				0xffff0000
-#define HDMI_DDC_SPEED_PRESCALE__SHIFT				16
-static inline uint32_t HDMI_DDC_SPEED_PRESCALE(uint32_t val)
-{
-	return ((val) << HDMI_DDC_SPEED_PRESCALE__SHIFT) & HDMI_DDC_SPEED_PRESCALE__MASK;
-}
-
-#define REG_HDMI_DDC_SETUP					0x00000224
-#define HDMI_DDC_SETUP_TIMEOUT__MASK				0xff000000
-#define HDMI_DDC_SETUP_TIMEOUT__SHIFT				24
-static inline uint32_t HDMI_DDC_SETUP_TIMEOUT(uint32_t val)
-{
-	return ((val) << HDMI_DDC_SETUP_TIMEOUT__SHIFT) & HDMI_DDC_SETUP_TIMEOUT__MASK;
-}
-
-static inline uint32_t REG_HDMI_I2C_TRANSACTION(uint32_t i0) { return 0x00000228 + 0x4*i0; }
-
-static inline uint32_t REG_HDMI_I2C_TRANSACTION_REG(uint32_t i0) { return 0x00000228 + 0x4*i0; }
-#define HDMI_I2C_TRANSACTION_REG_RW__MASK			0x00000001
-#define HDMI_I2C_TRANSACTION_REG_RW__SHIFT			0
-static inline uint32_t HDMI_I2C_TRANSACTION_REG_RW(enum hdmi_ddc_read_write val)
-{
-	return ((val) << HDMI_I2C_TRANSACTION_REG_RW__SHIFT) & HDMI_I2C_TRANSACTION_REG_RW__MASK;
-}
-#define HDMI_I2C_TRANSACTION_REG_STOP_ON_NACK			0x00000100
-#define HDMI_I2C_TRANSACTION_REG_START				0x00001000
-#define HDMI_I2C_TRANSACTION_REG_STOP				0x00002000
-#define HDMI_I2C_TRANSACTION_REG_CNT__MASK			0x00ff0000
-#define HDMI_I2C_TRANSACTION_REG_CNT__SHIFT			16
-static inline uint32_t HDMI_I2C_TRANSACTION_REG_CNT(uint32_t val)
-{
-	return ((val) << HDMI_I2C_TRANSACTION_REG_CNT__SHIFT) & HDMI_I2C_TRANSACTION_REG_CNT__MASK;
-}
-
-#define REG_HDMI_DDC_DATA					0x00000238
-#define HDMI_DDC_DATA_DATA_RW__MASK				0x00000001
-#define HDMI_DDC_DATA_DATA_RW__SHIFT				0
-static inline uint32_t HDMI_DDC_DATA_DATA_RW(enum hdmi_ddc_read_write val)
-{
-	return ((val) << HDMI_DDC_DATA_DATA_RW__SHIFT) & HDMI_DDC_DATA_DATA_RW__MASK;
-}
-#define HDMI_DDC_DATA_DATA__MASK				0x0000ff00
-#define HDMI_DDC_DATA_DATA__SHIFT				8
-static inline uint32_t HDMI_DDC_DATA_DATA(uint32_t val)
-{
-	return ((val) << HDMI_DDC_DATA_DATA__SHIFT) & HDMI_DDC_DATA_DATA__MASK;
-}
-#define HDMI_DDC_DATA_INDEX__MASK				0x00ff0000
-#define HDMI_DDC_DATA_INDEX__SHIFT				16
-static inline uint32_t HDMI_DDC_DATA_INDEX(uint32_t val)
-{
-	return ((val) << HDMI_DDC_DATA_INDEX__SHIFT) & HDMI_DDC_DATA_INDEX__MASK;
-}
-#define HDMI_DDC_DATA_INDEX_WRITE				0x80000000
-
-#define REG_HDMI_HDCP_SHA_CTRL					0x0000023c
-
-#define REG_HDMI_HDCP_SHA_STATUS				0x00000240
-#define HDMI_HDCP_SHA_STATUS_BLOCK_DONE				0x00000001
-#define HDMI_HDCP_SHA_STATUS_COMP_DONE				0x00000010
-
-#define REG_HDMI_HDCP_SHA_DATA					0x00000244
-#define HDMI_HDCP_SHA_DATA_DONE					0x00000001
-
-#define REG_HDMI_HPD_INT_STATUS					0x00000250
-#define HDMI_HPD_INT_STATUS_INT					0x00000001
-#define HDMI_HPD_INT_STATUS_CABLE_DETECTED			0x00000002
-
-#define REG_HDMI_HPD_INT_CTRL					0x00000254
-#define HDMI_HPD_INT_CTRL_INT_ACK				0x00000001
-#define HDMI_HPD_INT_CTRL_INT_CONNECT				0x00000002
-#define HDMI_HPD_INT_CTRL_INT_EN				0x00000004
-#define HDMI_HPD_INT_CTRL_RX_INT_ACK				0x00000010
-#define HDMI_HPD_INT_CTRL_RX_INT_EN				0x00000020
-#define HDMI_HPD_INT_CTRL_RCV_PLUGIN_DET_MASK			0x00000200
-
-#define REG_HDMI_HPD_CTRL					0x00000258
-#define HDMI_HPD_CTRL_TIMEOUT__MASK				0x00001fff
-#define HDMI_HPD_CTRL_TIMEOUT__SHIFT				0
-static inline uint32_t HDMI_HPD_CTRL_TIMEOUT(uint32_t val)
-{
-	return ((val) << HDMI_HPD_CTRL_TIMEOUT__SHIFT) & HDMI_HPD_CTRL_TIMEOUT__MASK;
-}
-#define HDMI_HPD_CTRL_ENABLE					0x10000000
-
-#define REG_HDMI_DDC_REF					0x0000027c
-#define HDMI_DDC_REF_REFTIMER_ENABLE				0x00010000
-#define HDMI_DDC_REF_REFTIMER__MASK				0x0000ffff
-#define HDMI_DDC_REF_REFTIMER__SHIFT				0
-static inline uint32_t HDMI_DDC_REF_REFTIMER(uint32_t val)
-{
-	return ((val) << HDMI_DDC_REF_REFTIMER__SHIFT) & HDMI_DDC_REF_REFTIMER__MASK;
-}
-
-#define REG_HDMI_HDCP_SW_UPPER_AKSV				0x00000284
-
-#define REG_HDMI_HDCP_SW_LOWER_AKSV				0x00000288
-
-#define REG_HDMI_CEC_CTRL					0x0000028c
-
-#define REG_HDMI_CEC_WR_DATA					0x00000290
-
-#define REG_HDMI_CEC_CEC_RETRANSMIT				0x00000294
-
-#define REG_HDMI_CEC_STATUS					0x00000298
-
-#define REG_HDMI_CEC_INT					0x0000029c
-
-#define REG_HDMI_CEC_ADDR					0x000002a0
-
-#define REG_HDMI_CEC_TIME					0x000002a4
-
-#define REG_HDMI_CEC_REFTIMER					0x000002a8
-
-#define REG_HDMI_CEC_RD_DATA					0x000002ac
-
-#define REG_HDMI_CEC_RD_FILTER					0x000002b0
-
-#define REG_HDMI_ACTIVE_HSYNC					0x000002b4
-#define HDMI_ACTIVE_HSYNC_START__MASK				0x00001fff
-#define HDMI_ACTIVE_HSYNC_START__SHIFT				0
-static inline uint32_t HDMI_ACTIVE_HSYNC_START(uint32_t val)
-{
-	return ((val) << HDMI_ACTIVE_HSYNC_START__SHIFT) & HDMI_ACTIVE_HSYNC_START__MASK;
-}
-#define HDMI_ACTIVE_HSYNC_END__MASK				0x0fff0000
-#define HDMI_ACTIVE_HSYNC_END__SHIFT				16
-static inline uint32_t HDMI_ACTIVE_HSYNC_END(uint32_t val)
-{
-	return ((val) << HDMI_ACTIVE_HSYNC_END__SHIFT) & HDMI_ACTIVE_HSYNC_END__MASK;
-}
-
-#define REG_HDMI_ACTIVE_VSYNC					0x000002b8
-#define HDMI_ACTIVE_VSYNC_START__MASK				0x00001fff
-#define HDMI_ACTIVE_VSYNC_START__SHIFT				0
-static inline uint32_t HDMI_ACTIVE_VSYNC_START(uint32_t val)
-{
-	return ((val) << HDMI_ACTIVE_VSYNC_START__SHIFT) & HDMI_ACTIVE_VSYNC_START__MASK;
-}
-#define HDMI_ACTIVE_VSYNC_END__MASK				0x1fff0000
-#define HDMI_ACTIVE_VSYNC_END__SHIFT				16
-static inline uint32_t HDMI_ACTIVE_VSYNC_END(uint32_t val)
-{
-	return ((val) << HDMI_ACTIVE_VSYNC_END__SHIFT) & HDMI_ACTIVE_VSYNC_END__MASK;
-}
-
-#define REG_HDMI_VSYNC_ACTIVE_F2				0x000002bc
-#define HDMI_VSYNC_ACTIVE_F2_START__MASK			0x00001fff
-#define HDMI_VSYNC_ACTIVE_F2_START__SHIFT			0
-static inline uint32_t HDMI_VSYNC_ACTIVE_F2_START(uint32_t val)
-{
-	return ((val) << HDMI_VSYNC_ACTIVE_F2_START__SHIFT) & HDMI_VSYNC_ACTIVE_F2_START__MASK;
-}
-#define HDMI_VSYNC_ACTIVE_F2_END__MASK				0x1fff0000
-#define HDMI_VSYNC_ACTIVE_F2_END__SHIFT				16
-static inline uint32_t HDMI_VSYNC_ACTIVE_F2_END(uint32_t val)
-{
-	return ((val) << HDMI_VSYNC_ACTIVE_F2_END__SHIFT) & HDMI_VSYNC_ACTIVE_F2_END__MASK;
-}
-
-#define REG_HDMI_TOTAL						0x000002c0
-#define HDMI_TOTAL_H_TOTAL__MASK				0x00001fff
-#define HDMI_TOTAL_H_TOTAL__SHIFT				0
-static inline uint32_t HDMI_TOTAL_H_TOTAL(uint32_t val)
-{
-	return ((val) << HDMI_TOTAL_H_TOTAL__SHIFT) & HDMI_TOTAL_H_TOTAL__MASK;
-}
-#define HDMI_TOTAL_V_TOTAL__MASK				0x1fff0000
-#define HDMI_TOTAL_V_TOTAL__SHIFT				16
-static inline uint32_t HDMI_TOTAL_V_TOTAL(uint32_t val)
-{
-	return ((val) << HDMI_TOTAL_V_TOTAL__SHIFT) & HDMI_TOTAL_V_TOTAL__MASK;
-}
-
-#define REG_HDMI_VSYNC_TOTAL_F2					0x000002c4
-#define HDMI_VSYNC_TOTAL_F2_V_TOTAL__MASK			0x00001fff
-#define HDMI_VSYNC_TOTAL_F2_V_TOTAL__SHIFT			0
-static inline uint32_t HDMI_VSYNC_TOTAL_F2_V_TOTAL(uint32_t val)
-{
-	return ((val) << HDMI_VSYNC_TOTAL_F2_V_TOTAL__SHIFT) & HDMI_VSYNC_TOTAL_F2_V_TOTAL__MASK;
-}
-
-#define REG_HDMI_FRAME_CTRL					0x000002c8
-#define HDMI_FRAME_CTRL_RGB_MUX_SEL_BGR				0x00001000
-#define HDMI_FRAME_CTRL_VSYNC_LOW				0x10000000
-#define HDMI_FRAME_CTRL_HSYNC_LOW				0x20000000
-#define HDMI_FRAME_CTRL_INTERLACED_EN				0x80000000
-
-#define REG_HDMI_AUD_INT					0x000002cc
-#define HDMI_AUD_INT_AUD_FIFO_URUN_INT				0x00000001
-#define HDMI_AUD_INT_AUD_FIFO_URAN_MASK				0x00000002
-#define HDMI_AUD_INT_AUD_SAM_DROP_INT				0x00000004
-#define HDMI_AUD_INT_AUD_SAM_DROP_MASK				0x00000008
-
-#define REG_HDMI_PHY_CTRL					0x000002d4
-#define HDMI_PHY_CTRL_SW_RESET_PLL				0x00000001
-#define HDMI_PHY_CTRL_SW_RESET_PLL_LOW				0x00000002
-#define HDMI_PHY_CTRL_SW_RESET					0x00000004
-#define HDMI_PHY_CTRL_SW_RESET_LOW				0x00000008
-
-#define REG_HDMI_CEC_WR_RANGE					0x000002dc
-
-#define REG_HDMI_CEC_RD_RANGE					0x000002e0
-
-#define REG_HDMI_VERSION					0x000002e4
-
-#define REG_HDMI_CEC_COMPL_CTL					0x00000360
-
-#define REG_HDMI_CEC_RD_START_RANGE				0x00000364
-
-#define REG_HDMI_CEC_RD_TOTAL_RANGE				0x00000368
-
-#define REG_HDMI_CEC_RD_ERR_RESP_LO				0x0000036c
-
-#define REG_HDMI_CEC_WR_CHECK_CONFIG				0x00000370
-
-#define REG_HDMI_8x60_PHY_REG0					0x00000000
-#define HDMI_8x60_PHY_REG0_DESER_DEL_CTRL__MASK			0x0000001c
-#define HDMI_8x60_PHY_REG0_DESER_DEL_CTRL__SHIFT		2
-static inline uint32_t HDMI_8x60_PHY_REG0_DESER_DEL_CTRL(uint32_t val)
-{
-	return ((val) << HDMI_8x60_PHY_REG0_DESER_DEL_CTRL__SHIFT) & HDMI_8x60_PHY_REG0_DESER_DEL_CTRL__MASK;
-}
-
-#define REG_HDMI_8x60_PHY_REG1					0x00000004
-#define HDMI_8x60_PHY_REG1_DTEST_MUX_SEL__MASK			0x000000f0
-#define HDMI_8x60_PHY_REG1_DTEST_MUX_SEL__SHIFT			4
-static inline uint32_t HDMI_8x60_PHY_REG1_DTEST_MUX_SEL(uint32_t val)
-{
-	return ((val) << HDMI_8x60_PHY_REG1_DTEST_MUX_SEL__SHIFT) & HDMI_8x60_PHY_REG1_DTEST_MUX_SEL__MASK;
-}
-#define HDMI_8x60_PHY_REG1_OUTVOL_SWING_CTRL__MASK		0x0000000f
-#define HDMI_8x60_PHY_REG1_OUTVOL_SWING_CTRL__SHIFT		0
-static inline uint32_t HDMI_8x60_PHY_REG1_OUTVOL_SWING_CTRL(uint32_t val)
-{
-	return ((val) << HDMI_8x60_PHY_REG1_OUTVOL_SWING_CTRL__SHIFT) & HDMI_8x60_PHY_REG1_OUTVOL_SWING_CTRL__MASK;
-}
-
-#define REG_HDMI_8x60_PHY_REG2					0x00000008
-#define HDMI_8x60_PHY_REG2_PD_DESER				0x00000001
-#define HDMI_8x60_PHY_REG2_PD_DRIVE_1				0x00000002
-#define HDMI_8x60_PHY_REG2_PD_DRIVE_2				0x00000004
-#define HDMI_8x60_PHY_REG2_PD_DRIVE_3				0x00000008
-#define HDMI_8x60_PHY_REG2_PD_DRIVE_4				0x00000010
-#define HDMI_8x60_PHY_REG2_PD_PLL				0x00000020
-#define HDMI_8x60_PHY_REG2_PD_PWRGEN				0x00000040
-#define HDMI_8x60_PHY_REG2_RCV_SENSE_EN				0x00000080
-
-#define REG_HDMI_8x60_PHY_REG3					0x0000000c
-#define HDMI_8x60_PHY_REG3_PLL_ENABLE				0x00000001
-
-#define REG_HDMI_8x60_PHY_REG4					0x00000010
-
-#define REG_HDMI_8x60_PHY_REG5					0x00000014
-
-#define REG_HDMI_8x60_PHY_REG6					0x00000018
-
-#define REG_HDMI_8x60_PHY_REG7					0x0000001c
-
-#define REG_HDMI_8x60_PHY_REG8					0x00000020
-
-#define REG_HDMI_8x60_PHY_REG9					0x00000024
-
-#define REG_HDMI_8x60_PHY_REG10					0x00000028
-
-#define REG_HDMI_8x60_PHY_REG11					0x0000002c
-
-#define REG_HDMI_8x60_PHY_REG12					0x00000030
-#define HDMI_8x60_PHY_REG12_RETIMING_EN				0x00000001
-#define HDMI_8x60_PHY_REG12_PLL_LOCK_DETECT_EN			0x00000002
-#define HDMI_8x60_PHY_REG12_FORCE_LOCK				0x00000010
-
-#define REG_HDMI_8960_PHY_REG0					0x00000000
-
-#define REG_HDMI_8960_PHY_REG1					0x00000004
-
-#define REG_HDMI_8960_PHY_REG2					0x00000008
-
-#define REG_HDMI_8960_PHY_REG3					0x0000000c
-
-#define REG_HDMI_8960_PHY_REG4					0x00000010
-
-#define REG_HDMI_8960_PHY_REG5					0x00000014
-
-#define REG_HDMI_8960_PHY_REG6					0x00000018
-
-#define REG_HDMI_8960_PHY_REG7					0x0000001c
-
-#define REG_HDMI_8960_PHY_REG8					0x00000020
-
-#define REG_HDMI_8960_PHY_REG9					0x00000024
-
-#define REG_HDMI_8960_PHY_REG10					0x00000028
-
-#define REG_HDMI_8960_PHY_REG11					0x0000002c
-
-#define REG_HDMI_8960_PHY_REG12					0x00000030
-#define HDMI_8960_PHY_REG12_SW_RESET				0x00000020
-#define HDMI_8960_PHY_REG12_PWRDN_B				0x00000080
-
-#define REG_HDMI_8960_PHY_REG_BIST_CFG				0x00000034
-
-#define REG_HDMI_8960_PHY_DEBUG_BUS_SEL				0x00000038
-
-#define REG_HDMI_8960_PHY_REG_MISC0				0x0000003c
-
-#define REG_HDMI_8960_PHY_REG13					0x00000040
-
-#define REG_HDMI_8960_PHY_REG14					0x00000044
-
-#define REG_HDMI_8960_PHY_REG15					0x00000048
-
-#define REG_HDMI_8960_PHY_PLL_REFCLK_CFG			0x00000000
-
-#define REG_HDMI_8960_PHY_PLL_CHRG_PUMP_CFG			0x00000004
-
-#define REG_HDMI_8960_PHY_PLL_LOOP_FLT_CFG0			0x00000008
-
-#define REG_HDMI_8960_PHY_PLL_LOOP_FLT_CFG1			0x0000000c
-
-#define REG_HDMI_8960_PHY_PLL_IDAC_ADJ_CFG			0x00000010
-
-#define REG_HDMI_8960_PHY_PLL_I_VI_KVCO_CFG			0x00000014
-
-#define REG_HDMI_8960_PHY_PLL_PWRDN_B				0x00000018
-#define HDMI_8960_PHY_PLL_PWRDN_B_PD_PLL			0x00000002
-#define HDMI_8960_PHY_PLL_PWRDN_B_PLL_PWRDN_B			0x00000008
-
-#define REG_HDMI_8960_PHY_PLL_SDM_CFG0				0x0000001c
-
-#define REG_HDMI_8960_PHY_PLL_SDM_CFG1				0x00000020
-
-#define REG_HDMI_8960_PHY_PLL_SDM_CFG2				0x00000024
-
-#define REG_HDMI_8960_PHY_PLL_SDM_CFG3				0x00000028
-
-#define REG_HDMI_8960_PHY_PLL_SDM_CFG4				0x0000002c
-
-#define REG_HDMI_8960_PHY_PLL_SSC_CFG0				0x00000030
-
-#define REG_HDMI_8960_PHY_PLL_SSC_CFG1				0x00000034
-
-#define REG_HDMI_8960_PHY_PLL_SSC_CFG2				0x00000038
-
-#define REG_HDMI_8960_PHY_PLL_SSC_CFG3				0x0000003c
-
-#define REG_HDMI_8960_PHY_PLL_LOCKDET_CFG0			0x00000040
-
-#define REG_HDMI_8960_PHY_PLL_LOCKDET_CFG1			0x00000044
-
-#define REG_HDMI_8960_PHY_PLL_LOCKDET_CFG2			0x00000048
-
-#define REG_HDMI_8960_PHY_PLL_VCOCAL_CFG0			0x0000004c
-
-#define REG_HDMI_8960_PHY_PLL_VCOCAL_CFG1			0x00000050
-
-#define REG_HDMI_8960_PHY_PLL_VCOCAL_CFG2			0x00000054
-
-#define REG_HDMI_8960_PHY_PLL_VCOCAL_CFG3			0x00000058
-
-#define REG_HDMI_8960_PHY_PLL_VCOCAL_CFG4			0x0000005c
-
-#define REG_HDMI_8960_PHY_PLL_VCOCAL_CFG5			0x00000060
-
-#define REG_HDMI_8960_PHY_PLL_VCOCAL_CFG6			0x00000064
-
-#define REG_HDMI_8960_PHY_PLL_VCOCAL_CFG7			0x00000068
-
-#define REG_HDMI_8960_PHY_PLL_DEBUG_SEL				0x0000006c
-
-#define REG_HDMI_8960_PHY_PLL_MISC0				0x00000070
-
-#define REG_HDMI_8960_PHY_PLL_MISC1				0x00000074
-
-#define REG_HDMI_8960_PHY_PLL_MISC2				0x00000078
-
-#define REG_HDMI_8960_PHY_PLL_MISC3				0x0000007c
-
-#define REG_HDMI_8960_PHY_PLL_MISC4				0x00000080
-
-#define REG_HDMI_8960_PHY_PLL_MISC5				0x00000084
-
-#define REG_HDMI_8960_PHY_PLL_MISC6				0x00000088
-
-#define REG_HDMI_8960_PHY_PLL_DEBUG_BUS0			0x0000008c
-
-#define REG_HDMI_8960_PHY_PLL_DEBUG_BUS1			0x00000090
-
-#define REG_HDMI_8960_PHY_PLL_DEBUG_BUS2			0x00000094
-
-#define REG_HDMI_8960_PHY_PLL_STATUS0				0x00000098
-#define HDMI_8960_PHY_PLL_STATUS0_PLL_LOCK			0x00000001
-
-#define REG_HDMI_8960_PHY_PLL_STATUS1				0x0000009c
-
-#define REG_HDMI_8x74_ANA_CFG0					0x00000000
-
-#define REG_HDMI_8x74_ANA_CFG1					0x00000004
-
-#define REG_HDMI_8x74_ANA_CFG2					0x00000008
-
-#define REG_HDMI_8x74_ANA_CFG3					0x0000000c
-
-#define REG_HDMI_8x74_PD_CTRL0					0x00000010
-
-#define REG_HDMI_8x74_PD_CTRL1					0x00000014
-
-#define REG_HDMI_8x74_GLB_CFG					0x00000018
-
-#define REG_HDMI_8x74_DCC_CFG0					0x0000001c
-
-#define REG_HDMI_8x74_DCC_CFG1					0x00000020
-
-#define REG_HDMI_8x74_TXCAL_CFG0				0x00000024
-
-#define REG_HDMI_8x74_TXCAL_CFG1				0x00000028
-
-#define REG_HDMI_8x74_TXCAL_CFG2				0x0000002c
-
-#define REG_HDMI_8x74_TXCAL_CFG3				0x00000030
-
-#define REG_HDMI_8x74_BIST_CFG0					0x00000034
-
-#define REG_HDMI_8x74_BIST_PATN0				0x0000003c
-
-#define REG_HDMI_8x74_BIST_PATN1				0x00000040
-
-#define REG_HDMI_8x74_BIST_PATN2				0x00000044
-
-#define REG_HDMI_8x74_BIST_PATN3				0x00000048
-
-#define REG_HDMI_8x74_STATUS					0x0000005c
-
-#define REG_HDMI_28nm_PHY_PLL_REFCLK_CFG			0x00000000
-
-#define REG_HDMI_28nm_PHY_PLL_POSTDIV1_CFG			0x00000004
-
-#define REG_HDMI_28nm_PHY_PLL_CHGPUMP_CFG			0x00000008
-
-#define REG_HDMI_28nm_PHY_PLL_VCOLPF_CFG			0x0000000c
-
-#define REG_HDMI_28nm_PHY_PLL_VREG_CFG				0x00000010
-
-#define REG_HDMI_28nm_PHY_PLL_PWRGEN_CFG			0x00000014
-
-#define REG_HDMI_28nm_PHY_PLL_DMUX_CFG				0x00000018
-
-#define REG_HDMI_28nm_PHY_PLL_AMUX_CFG				0x0000001c
-
-#define REG_HDMI_28nm_PHY_PLL_GLB_CFG				0x00000020
-#define HDMI_28nm_PHY_PLL_GLB_CFG_PLL_PWRDN_B			0x00000001
-#define HDMI_28nm_PHY_PLL_GLB_CFG_PLL_LDO_PWRDN_B		0x00000002
-#define HDMI_28nm_PHY_PLL_GLB_CFG_PLL_PWRGEN_PWRDN_B		0x00000004
-#define HDMI_28nm_PHY_PLL_GLB_CFG_PLL_ENABLE			0x00000008
-
-#define REG_HDMI_28nm_PHY_PLL_POSTDIV2_CFG			0x00000024
-
-#define REG_HDMI_28nm_PHY_PLL_POSTDIV3_CFG			0x00000028
-
-#define REG_HDMI_28nm_PHY_PLL_LPFR_CFG				0x0000002c
-
-#define REG_HDMI_28nm_PHY_PLL_LPFC1_CFG				0x00000030
-
-#define REG_HDMI_28nm_PHY_PLL_LPFC2_CFG				0x00000034
-
-#define REG_HDMI_28nm_PHY_PLL_SDM_CFG0				0x00000038
-
-#define REG_HDMI_28nm_PHY_PLL_SDM_CFG1				0x0000003c
-
-#define REG_HDMI_28nm_PHY_PLL_SDM_CFG2				0x00000040
-
-#define REG_HDMI_28nm_PHY_PLL_SDM_CFG3				0x00000044
-
-#define REG_HDMI_28nm_PHY_PLL_SDM_CFG4				0x00000048
-
-#define REG_HDMI_28nm_PHY_PLL_SSC_CFG0				0x0000004c
-
-#define REG_HDMI_28nm_PHY_PLL_SSC_CFG1				0x00000050
-
-#define REG_HDMI_28nm_PHY_PLL_SSC_CFG2				0x00000054
-
-#define REG_HDMI_28nm_PHY_PLL_SSC_CFG3				0x00000058
-
-#define REG_HDMI_28nm_PHY_PLL_LKDET_CFG0			0x0000005c
-
-#define REG_HDMI_28nm_PHY_PLL_LKDET_CFG1			0x00000060
-
-#define REG_HDMI_28nm_PHY_PLL_LKDET_CFG2			0x00000064
-
-#define REG_HDMI_28nm_PHY_PLL_TEST_CFG				0x00000068
-#define HDMI_28nm_PHY_PLL_TEST_CFG_PLL_SW_RESET			0x00000001
-
-#define REG_HDMI_28nm_PHY_PLL_CAL_CFG0				0x0000006c
-
-#define REG_HDMI_28nm_PHY_PLL_CAL_CFG1				0x00000070
-
-#define REG_HDMI_28nm_PHY_PLL_CAL_CFG2				0x00000074
-
-#define REG_HDMI_28nm_PHY_PLL_CAL_CFG3				0x00000078
-
-#define REG_HDMI_28nm_PHY_PLL_CAL_CFG4				0x0000007c
-
-#define REG_HDMI_28nm_PHY_PLL_CAL_CFG5				0x00000080
-
-#define REG_HDMI_28nm_PHY_PLL_CAL_CFG6				0x00000084
-
-#define REG_HDMI_28nm_PHY_PLL_CAL_CFG7				0x00000088
-
-#define REG_HDMI_28nm_PHY_PLL_CAL_CFG8				0x0000008c
-
-#define REG_HDMI_28nm_PHY_PLL_CAL_CFG9				0x00000090
-
-#define REG_HDMI_28nm_PHY_PLL_CAL_CFG10				0x00000094
-
-#define REG_HDMI_28nm_PHY_PLL_CAL_CFG11				0x00000098
-
-#define REG_HDMI_28nm_PHY_PLL_EFUSE_CFG				0x0000009c
-
-#define REG_HDMI_28nm_PHY_PLL_DEBUG_BUS_SEL			0x000000a0
-
-#define REG_HDMI_28nm_PHY_PLL_STATUS				0x000000c0
-
-#define REG_HDMI_8996_PHY_CFG					0x00000000
-
-#define REG_HDMI_8996_PHY_PD_CTL				0x00000004
-
-#define REG_HDMI_8996_PHY_MODE					0x00000008
-
-#define REG_HDMI_8996_PHY_MISR_CLEAR				0x0000000c
-
-#define REG_HDMI_8996_PHY_TX0_TX1_BIST_CFG0			0x00000010
-
-#define REG_HDMI_8996_PHY_TX0_TX1_BIST_CFG1			0x00000014
-
-#define REG_HDMI_8996_PHY_TX0_TX1_PRBS_SEED_BYTE0		0x00000018
-
-#define REG_HDMI_8996_PHY_TX0_TX1_PRBS_SEED_BYTE1		0x0000001c
-
-#define REG_HDMI_8996_PHY_TX0_TX1_BIST_PATTERN0			0x00000020
-
-#define REG_HDMI_8996_PHY_TX0_TX1_BIST_PATTERN1			0x00000024
-
-#define REG_HDMI_8996_PHY_TX2_TX3_BIST_CFG0			0x00000028
-
-#define REG_HDMI_8996_PHY_TX2_TX3_BIST_CFG1			0x0000002c
-
-#define REG_HDMI_8996_PHY_TX2_TX3_PRBS_SEED_BYTE0		0x00000030
-
-#define REG_HDMI_8996_PHY_TX2_TX3_PRBS_SEED_BYTE1		0x00000034
-
-#define REG_HDMI_8996_PHY_TX2_TX3_BIST_PATTERN0			0x00000038
-
-#define REG_HDMI_8996_PHY_TX2_TX3_BIST_PATTERN1			0x0000003c
-
-#define REG_HDMI_8996_PHY_DEBUG_BUS_SEL				0x00000040
-
-#define REG_HDMI_8996_PHY_TXCAL_CFG0				0x00000044
-
-#define REG_HDMI_8996_PHY_TXCAL_CFG1				0x00000048
-
-#define REG_HDMI_8996_PHY_TX0_TX1_LANE_CTL			0x0000004c
-
-#define REG_HDMI_8996_PHY_TX2_TX3_LANE_CTL			0x00000050
-
-#define REG_HDMI_8996_PHY_LANE_BIST_CONFIG			0x00000054
-
-#define REG_HDMI_8996_PHY_CLOCK					0x00000058
-
-#define REG_HDMI_8996_PHY_MISC1					0x0000005c
-
-#define REG_HDMI_8996_PHY_MISC2					0x00000060
-
-#define REG_HDMI_8996_PHY_TX0_TX1_BIST_STATUS0			0x00000064
-
-#define REG_HDMI_8996_PHY_TX0_TX1_BIST_STATUS1			0x00000068
-
-#define REG_HDMI_8996_PHY_TX0_TX1_BIST_STATUS2			0x0000006c
-
-#define REG_HDMI_8996_PHY_TX2_TX3_BIST_STATUS0			0x00000070
-
-#define REG_HDMI_8996_PHY_TX2_TX3_BIST_STATUS1			0x00000074
-
-#define REG_HDMI_8996_PHY_TX2_TX3_BIST_STATUS2			0x00000078
-
-#define REG_HDMI_8996_PHY_PRE_MISR_STATUS0			0x0000007c
-
-#define REG_HDMI_8996_PHY_PRE_MISR_STATUS1			0x00000080
-
-#define REG_HDMI_8996_PHY_PRE_MISR_STATUS2			0x00000084
-
-#define REG_HDMI_8996_PHY_PRE_MISR_STATUS3			0x00000088
-
-#define REG_HDMI_8996_PHY_POST_MISR_STATUS0			0x0000008c
-
-#define REG_HDMI_8996_PHY_POST_MISR_STATUS1			0x00000090
-
-#define REG_HDMI_8996_PHY_POST_MISR_STATUS2			0x00000094
-
-#define REG_HDMI_8996_PHY_POST_MISR_STATUS3			0x00000098
-
-#define REG_HDMI_8996_PHY_STATUS				0x0000009c
-
-#define REG_HDMI_8996_PHY_MISC3_STATUS				0x000000a0
-
-#define REG_HDMI_8996_PHY_MISC4_STATUS				0x000000a4
-
-#define REG_HDMI_8996_PHY_DEBUG_BUS0				0x000000a8
-
-#define REG_HDMI_8996_PHY_DEBUG_BUS1				0x000000ac
-
-#define REG_HDMI_8996_PHY_DEBUG_BUS2				0x000000b0
-
-#define REG_HDMI_8996_PHY_DEBUG_BUS3				0x000000b4
-
-#define REG_HDMI_8996_PHY_PHY_REVISION_ID0			0x000000b8
-
-#define REG_HDMI_8996_PHY_PHY_REVISION_ID1			0x000000bc
-
-#define REG_HDMI_8996_PHY_PHY_REVISION_ID2			0x000000c0
-
-#define REG_HDMI_8996_PHY_PHY_REVISION_ID3			0x000000c4
-
-#define REG_HDMI_PHY_QSERDES_COM_ATB_SEL1			0x00000000
-
-#define REG_HDMI_PHY_QSERDES_COM_ATB_SEL2			0x00000004
-
-#define REG_HDMI_PHY_QSERDES_COM_FREQ_UPDATE			0x00000008
-
-#define REG_HDMI_PHY_QSERDES_COM_BG_TIMER			0x0000000c
-
-#define REG_HDMI_PHY_QSERDES_COM_SSC_EN_CENTER			0x00000010
-
-#define REG_HDMI_PHY_QSERDES_COM_SSC_ADJ_PER1			0x00000014
-
-#define REG_HDMI_PHY_QSERDES_COM_SSC_ADJ_PER2			0x00000018
-
-#define REG_HDMI_PHY_QSERDES_COM_SSC_PER1			0x0000001c
-
-#define REG_HDMI_PHY_QSERDES_COM_SSC_PER2			0x00000020
-
-#define REG_HDMI_PHY_QSERDES_COM_SSC_STEP_SIZE1			0x00000024
-
-#define REG_HDMI_PHY_QSERDES_COM_SSC_STEP_SIZE2			0x00000028
-
-#define REG_HDMI_PHY_QSERDES_COM_POST_DIV			0x0000002c
-
-#define REG_HDMI_PHY_QSERDES_COM_POST_DIV_MUX			0x00000030
-
-#define REG_HDMI_PHY_QSERDES_COM_BIAS_EN_CLKBUFLR_EN		0x00000034
-
-#define REG_HDMI_PHY_QSERDES_COM_CLK_ENABLE1			0x00000038
-
-#define REG_HDMI_PHY_QSERDES_COM_SYS_CLK_CTRL			0x0000003c
-
-#define REG_HDMI_PHY_QSERDES_COM_SYSCLK_BUF_ENABLE		0x00000040
-
-#define REG_HDMI_PHY_QSERDES_COM_PLL_EN				0x00000044
-
-#define REG_HDMI_PHY_QSERDES_COM_PLL_IVCO			0x00000048
-
-#define REG_HDMI_PHY_QSERDES_COM_LOCK_CMP1_MODE0		0x0000004c
-
-#define REG_HDMI_PHY_QSERDES_COM_LOCK_CMP2_MODE0		0x00000050
-
-#define REG_HDMI_PHY_QSERDES_COM_LOCK_CMP3_MODE0		0x00000054
-
-#define REG_HDMI_PHY_QSERDES_COM_LOCK_CMP1_MODE1		0x00000058
-
-#define REG_HDMI_PHY_QSERDES_COM_LOCK_CMP2_MODE1		0x0000005c
-
-#define REG_HDMI_PHY_QSERDES_COM_LOCK_CMP3_MODE1		0x00000060
-
-#define REG_HDMI_PHY_QSERDES_COM_LOCK_CMP1_MODE2		0x00000064
-
-#define REG_HDMI_PHY_QSERDES_COM_CMN_RSVD0			0x00000064
-
-#define REG_HDMI_PHY_QSERDES_COM_LOCK_CMP2_MODE2		0x00000068
-
-#define REG_HDMI_PHY_QSERDES_COM_EP_CLOCK_DETECT_CTRL		0x00000068
-
-#define REG_HDMI_PHY_QSERDES_COM_LOCK_CMP3_MODE2		0x0000006c
-
-#define REG_HDMI_PHY_QSERDES_COM_SYSCLK_DET_COMP_STATUS		0x0000006c
-
-#define REG_HDMI_PHY_QSERDES_COM_BG_TRIM			0x00000070
-
-#define REG_HDMI_PHY_QSERDES_COM_CLK_EP_DIV			0x00000074
-
-#define REG_HDMI_PHY_QSERDES_COM_CP_CTRL_MODE0			0x00000078
-
-#define REG_HDMI_PHY_QSERDES_COM_CP_CTRL_MODE1			0x0000007c
-
-#define REG_HDMI_PHY_QSERDES_COM_CP_CTRL_MODE2			0x00000080
-
-#define REG_HDMI_PHY_QSERDES_COM_CMN_RSVD1			0x00000080
-
-#define REG_HDMI_PHY_QSERDES_COM_PLL_RCTRL_MODE0		0x00000084
-
-#define REG_HDMI_PHY_QSERDES_COM_PLL_RCTRL_MODE1		0x00000088
-
-#define REG_HDMI_PHY_QSERDES_COM_PLL_RCTRL_MODE2		0x0000008c
-
-#define REG_HDMI_PHY_QSERDES_COM_CMN_RSVD2			0x0000008c
-
-#define REG_HDMI_PHY_QSERDES_COM_PLL_CCTRL_MODE0		0x00000090
-
-#define REG_HDMI_PHY_QSERDES_COM_PLL_CCTRL_MODE1		0x00000094
-
-#define REG_HDMI_PHY_QSERDES_COM_PLL_CCTRL_MODE2		0x00000098
-
-#define REG_HDMI_PHY_QSERDES_COM_CMN_RSVD3			0x00000098
-
-#define REG_HDMI_PHY_QSERDES_COM_PLL_CNTRL			0x0000009c
-
-#define REG_HDMI_PHY_QSERDES_COM_PHASE_SEL_CTRL			0x000000a0
-
-#define REG_HDMI_PHY_QSERDES_COM_PHASE_SEL_DC			0x000000a4
-
-#define REG_HDMI_PHY_QSERDES_COM_CORE_CLK_IN_SYNC_SEL		0x000000a8
-
-#define REG_HDMI_PHY_QSERDES_COM_BIAS_EN_CTRL_BY_PSM		0x000000a8
-
-#define REG_HDMI_PHY_QSERDES_COM_SYSCLK_EN_SEL			0x000000ac
-
-#define REG_HDMI_PHY_QSERDES_COM_CML_SYSCLK_SEL			0x000000b0
-
-#define REG_HDMI_PHY_QSERDES_COM_RESETSM_CNTRL			0x000000b4
-
-#define REG_HDMI_PHY_QSERDES_COM_RESETSM_CNTRL2			0x000000b8
-
-#define REG_HDMI_PHY_QSERDES_COM_RESTRIM_CTRL			0x000000bc
-
-#define REG_HDMI_PHY_QSERDES_COM_RESTRIM_CTRL2			0x000000c0
-
-#define REG_HDMI_PHY_QSERDES_COM_RESCODE_DIV_NUM		0x000000c4
-
-#define REG_HDMI_PHY_QSERDES_COM_LOCK_CMP_EN			0x000000c8
-
-#define REG_HDMI_PHY_QSERDES_COM_LOCK_CMP_CFG			0x000000cc
-
-#define REG_HDMI_PHY_QSERDES_COM_DEC_START_MODE0		0x000000d0
-
-#define REG_HDMI_PHY_QSERDES_COM_DEC_START_MODE1		0x000000d4
-
-#define REG_HDMI_PHY_QSERDES_COM_DEC_START_MODE2		0x000000d8
-
-#define REG_HDMI_PHY_QSERDES_COM_VCOCAL_DEADMAN_CTRL		0x000000d8
-
-#define REG_HDMI_PHY_QSERDES_COM_DIV_FRAC_START1_MODE0		0x000000dc
-
-#define REG_HDMI_PHY_QSERDES_COM_DIV_FRAC_START2_MODE0		0x000000e0
-
-#define REG_HDMI_PHY_QSERDES_COM_DIV_FRAC_START3_MODE0		0x000000e4
-
-#define REG_HDMI_PHY_QSERDES_COM_DIV_FRAC_START1_MODE1		0x000000e8
-
-#define REG_HDMI_PHY_QSERDES_COM_DIV_FRAC_START2_MODE1		0x000000ec
-
-#define REG_HDMI_PHY_QSERDES_COM_DIV_FRAC_START3_MODE1		0x000000f0
-
-#define REG_HDMI_PHY_QSERDES_COM_DIV_FRAC_START1_MODE2		0x000000f4
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE_MINVAL1		0x000000f4
-
-#define REG_HDMI_PHY_QSERDES_COM_DIV_FRAC_START2_MODE2		0x000000f8
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE_MINVAL2		0x000000f8
-
-#define REG_HDMI_PHY_QSERDES_COM_DIV_FRAC_START3_MODE2		0x000000fc
-
-#define REG_HDMI_PHY_QSERDES_COM_CMN_RSVD4			0x000000fc
-
-#define REG_HDMI_PHY_QSERDES_COM_INTEGLOOP_INITVAL		0x00000100
-
-#define REG_HDMI_PHY_QSERDES_COM_INTEGLOOP_EN			0x00000104
-
-#define REG_HDMI_PHY_QSERDES_COM_INTEGLOOP_GAIN0_MODE0		0x00000108
-
-#define REG_HDMI_PHY_QSERDES_COM_INTEGLOOP_GAIN1_MODE0		0x0000010c
-
-#define REG_HDMI_PHY_QSERDES_COM_INTEGLOOP_GAIN0_MODE1		0x00000110
-
-#define REG_HDMI_PHY_QSERDES_COM_INTEGLOOP_GAIN1_MODE1		0x00000114
-
-#define REG_HDMI_PHY_QSERDES_COM_INTEGLOOP_GAIN0_MODE2		0x00000118
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE_MAXVAL1		0x00000118
-
-#define REG_HDMI_PHY_QSERDES_COM_INTEGLOOP_GAIN1_MODE2		0x0000011c
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE_MAXVAL2		0x0000011c
-
-#define REG_HDMI_PHY_QSERDES_COM_RES_TRIM_CONTROL2		0x00000120
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE_CTRL			0x00000124
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE_MAP			0x00000128
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE1_MODE0		0x0000012c
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE2_MODE0		0x00000130
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE1_MODE1		0x00000134
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE2_MODE1		0x00000138
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE1_MODE2		0x0000013c
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE_INITVAL1		0x0000013c
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE2_MODE2		0x00000140
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE_INITVAL2		0x00000140
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE_TIMER1		0x00000144
-
-#define REG_HDMI_PHY_QSERDES_COM_VCO_TUNE_TIMER2		0x00000148
-
-#define REG_HDMI_PHY_QSERDES_COM_SAR				0x0000014c
-
-#define REG_HDMI_PHY_QSERDES_COM_SAR_CLK			0x00000150
-
-#define REG_HDMI_PHY_QSERDES_COM_SAR_CODE_OUT_STATUS		0x00000154
-
-#define REG_HDMI_PHY_QSERDES_COM_SAR_CODE_READY_STATUS		0x00000158
-
-#define REG_HDMI_PHY_QSERDES_COM_CMN_STATUS			0x0000015c
-
-#define REG_HDMI_PHY_QSERDES_COM_RESET_SM_STATUS		0x00000160
-
-#define REG_HDMI_PHY_QSERDES_COM_RESTRIM_CODE_STATUS		0x00000164
-
-#define REG_HDMI_PHY_QSERDES_COM_PLLCAL_CODE1_STATUS		0x00000168
-
-#define REG_HDMI_PHY_QSERDES_COM_PLLCAL_CODE2_STATUS		0x0000016c
-
-#define REG_HDMI_PHY_QSERDES_COM_BG_CTRL			0x00000170
-
-#define REG_HDMI_PHY_QSERDES_COM_CLK_SELECT			0x00000174
-
-#define REG_HDMI_PHY_QSERDES_COM_HSCLK_SEL			0x00000178
-
-#define REG_HDMI_PHY_QSERDES_COM_INTEGLOOP_BINCODE_STATUS	0x0000017c
-
-#define REG_HDMI_PHY_QSERDES_COM_PLL_ANALOG			0x00000180
-
-#define REG_HDMI_PHY_QSERDES_COM_CORECLK_DIV			0x00000184
-
-#define REG_HDMI_PHY_QSERDES_COM_SW_RESET			0x00000188
-
-#define REG_HDMI_PHY_QSERDES_COM_CORE_CLK_EN			0x0000018c
-
-#define REG_HDMI_PHY_QSERDES_COM_C_READY_STATUS			0x00000190
-
-#define REG_HDMI_PHY_QSERDES_COM_CMN_CONFIG			0x00000194
-
-#define REG_HDMI_PHY_QSERDES_COM_CMN_RATE_OVERRIDE		0x00000198
-
-#define REG_HDMI_PHY_QSERDES_COM_SVS_MODE_CLK_SEL		0x0000019c
-
-#define REG_HDMI_PHY_QSERDES_COM_DEBUG_BUS0			0x000001a0
-
-#define REG_HDMI_PHY_QSERDES_COM_DEBUG_BUS1			0x000001a4
-
-#define REG_HDMI_PHY_QSERDES_COM_DEBUG_BUS2			0x000001a8
-
-#define REG_HDMI_PHY_QSERDES_COM_DEBUG_BUS3			0x000001ac
-
-#define REG_HDMI_PHY_QSERDES_COM_DEBUG_BUS_SEL			0x000001b0
-
-#define REG_HDMI_PHY_QSERDES_COM_CMN_MISC1			0x000001b4
-
-#define REG_HDMI_PHY_QSERDES_COM_CMN_MISC2			0x000001b8
-
-#define REG_HDMI_PHY_QSERDES_COM_CORECLK_DIV_MODE1		0x000001bc
-
-#define REG_HDMI_PHY_QSERDES_COM_CORECLK_DIV_MODE2		0x000001c0
-
-#define REG_HDMI_PHY_QSERDES_COM_CMN_RSVD5			0x000001c4
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_MODE_LANENO		0x00000000
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_INVERT			0x00000004
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_CLKBUF_ENABLE		0x00000008
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_CMN_CONTROL_ONE		0x0000000c
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_CMN_CONTROL_TWO		0x00000010
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_CMN_CONTROL_THREE		0x00000014
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_TX_EMP_POST1_LVL		0x00000018
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_TX_POST2_EMPH		0x0000001c
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_TX_BOOST_LVL_UP_DN		0x00000020
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_HP_PD_ENABLES		0x00000024
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_TX_IDLE_LVL_LARGE_AMP	0x00000028
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_TX_DRV_LVL			0x0000002c
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_TX_DRV_LVL_OFFSET		0x00000030
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_RESET_TSYNC_EN		0x00000034
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PRE_STALL_LDO_BOOST_EN	0x00000038
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_TX_BAND			0x0000003c
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_SLEW_CNTL			0x00000040
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_INTERFACE_SELECT		0x00000044
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_LPB_EN			0x00000048
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_RES_CODE_LANE_TX		0x0000004c
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_RES_CODE_LANE_RX		0x00000050
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_RES_CODE_LANE_OFFSET		0x00000054
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PERL_LENGTH1			0x00000058
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PERL_LENGTH2			0x0000005c
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_SERDES_BYP_EN_OUT		0x00000060
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_DEBUG_BUS_SEL		0x00000064
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN	0x00000068
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_TX_POL_INV			0x0000006c
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PARRATE_REC_DETECT_IDLE_EN	0x00000070
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_PATTERN1		0x00000074
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_PATTERN2		0x00000078
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_PATTERN3		0x0000007c
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_PATTERN4		0x00000080
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_PATTERN5		0x00000084
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_PATTERN6		0x00000088
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_PATTERN7		0x0000008c
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_PATTERN8		0x00000090
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_LANE_MODE			0x00000094
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_IDAC_CAL_LANE_MODE		0x00000098
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_IDAC_CAL_LANE_MODE_CONFIGURATION	0x0000009c
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_ATB_SEL1			0x000000a0
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_ATB_SEL2			0x000000a4
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_RCV_DETECT_LVL		0x000000a8
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_RCV_DETECT_LVL_2		0x000000ac
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PRBS_SEED1			0x000000b0
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PRBS_SEED2			0x000000b4
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PRBS_SEED3			0x000000b8
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PRBS_SEED4			0x000000bc
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_RESET_GEN			0x000000c0
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_RESET_GEN_MUXES		0x000000c4
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_TRAN_DRVR_EMP_EN		0x000000c8
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_TX_INTERFACE_MODE		0x000000cc
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PWM_CTRL			0x000000d0
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PWM_ENCODED_OR_DATA		0x000000d4
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PWM_GEAR_1_DIVIDER_BAND2	0x000000d8
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PWM_GEAR_2_DIVIDER_BAND2	0x000000dc
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PWM_GEAR_3_DIVIDER_BAND2	0x000000e0
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PWM_GEAR_4_DIVIDER_BAND2	0x000000e4
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PWM_GEAR_1_DIVIDER_BAND0_1	0x000000e8
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PWM_GEAR_2_DIVIDER_BAND0_1	0x000000ec
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PWM_GEAR_3_DIVIDER_BAND0_1	0x000000f0
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_PWM_GEAR_4_DIVIDER_BAND0_1	0x000000f4
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_VMODE_CTRL1			0x000000f8
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_VMODE_CTRL2			0x000000fc
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_TX_ALOG_INTF_OBSV_CNTL	0x00000100
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_STATUS			0x00000104
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_ERROR_COUNT1		0x00000108
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_BIST_ERROR_COUNT2		0x0000010c
-
-#define REG_HDMI_PHY_QSERDES_TX_LX_TX_ALOG_INTF_OBSV		0x00000110
-
-
-#endif /* HDMI_XML */
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c b/drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c
index 4dd055416620..8c8d80b59573 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c
@@ -86,18 +86,18 @@ static inline struct hdmi_phy *pll_get_phy(struct hdmi_pll_8996 *pll)
 static inline void hdmi_pll_write(struct hdmi_pll_8996 *pll, int offset,
 				  u32 data)
 {
-	msm_writel(data, pll->mmio_qserdes_com + offset);
+	writel(data, pll->mmio_qserdes_com + offset);
 }
 
 static inline u32 hdmi_pll_read(struct hdmi_pll_8996 *pll, int offset)
 {
-	return msm_readl(pll->mmio_qserdes_com + offset);
+	return readl(pll->mmio_qserdes_com + offset);
 }
 
 static inline void hdmi_tx_chan_write(struct hdmi_pll_8996 *pll, int channel,
 				      int offset, int data)
 {
-	 msm_writel(data, pll->mmio_qserdes_tx[channel] + offset);
+	 writel(data, pll->mmio_qserdes_tx[channel] + offset);
 }
 
 static inline u32 pll_get_cpctrl(u64 frac_start, unsigned long ref_clk,
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_pll_8960.c b/drivers/gpu/drm/msm/hdmi/hdmi_pll_8960.c
index cb35a297afbd..83c8781fcc3f 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi_pll_8960.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi_pll_8960.c
@@ -236,12 +236,12 @@ static const struct pll_rate freqtbl[] = {
 
 static inline void pll_write(struct hdmi_pll_8960 *pll, u32 reg, u32 data)
 {
-	msm_writel(data, pll->mmio + reg);
+	writel(data, pll->mmio + reg);
 }
 
 static inline u32 pll_read(struct hdmi_pll_8960 *pll, u32 reg)
 {
-	return msm_readl(pll->mmio + reg);
+	return readl(pll->mmio + reg);
 }
 
 static inline struct hdmi_phy *pll_get_phy(struct hdmi_pll_8960 *pll)
diff --git a/drivers/gpu/drm/msm/hdmi/qfprom.xml.h b/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
deleted file mode 100644
index 498801526695..000000000000
--- a/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
+++ /dev/null
@@ -1,61 +0,0 @@
-#ifndef QFPROM_XML
-#define QFPROM_XML
-
-/* Autogenerated file, DO NOT EDIT manually!
-
-This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://github.com/freedreno/envytools/
-git clone https://github.com/freedreno/envytools.git
-
-The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/msm.xml                   (    944 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/freedreno_copyright.xml   (   1572 bytes, from 2022-07-23 20:21:46)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp4.xml              (  20912 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp_common.xml        (   2849 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/mdp/mdp5.xml              (  37461 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi.xml               (  18746 bytes, from 2022-04-28 17:29:36)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_v2.xml        (   3236 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm_8960.xml (   4935 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_28nm.xml      (   7004 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_20nm.xml      (   3712 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_14nm.xml      (   5381 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_10nm.xml      (   4499 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/dsi_phy_7nm.xml       (  11007 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/sfpb.xml              (    602 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/dsi/mmss_cc.xml           (   1686 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/qfprom.xml           (    600 bytes, from 2022-03-08 17:40:42)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/hdmi/hdmi.xml             (  42350 bytes, from 2022-09-20 17:45:56)
-- /home/robclark/src/mesa/mesa/src/freedreno/registers/edp/edp.xml               (  10416 bytes, from 2022-03-08 17:40:42)
-
-Copyright (C) 2013-2022 by the following authors:
-- Rob Clark <robdclark@gmail.com> (robclark)
-- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice (including the
-next paragraph) shall be included in all copies or substantial
-portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-
-#define REG_QFPROM_CONFIG_ROW0_LSB				0x00000238
-#define QFPROM_CONFIG_ROW0_LSB_HDMI_DISABLE			0x00200000
-#define QFPROM_CONFIG_ROW0_LSB_HDCP_DISABLE			0x00400000
-
-
-#endif /* QFPROM_XML */
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 97790faffd23..9c33f4e3f822 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -17,8 +17,9 @@
 
 #include "msm_drv.h"
 #include "msm_debugfs.h"
+#include "msm_gem.h"
+#include "msm_gpu.h"
 #include "msm_kms.h"
-#include "adreno/adreno_gpu.h"
 
 /*
  * MSM driver version:
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index 65f213660452..912ebaa5df84 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -239,9 +239,7 @@ struct msm_drm_private {
 	bool disable_err_irq;
 };
 
-struct msm_format {
-	uint32_t pixel_format;
-};
+const struct msm_format *mdp_get_format(struct msm_kms *kms, uint32_t format, uint64_t modifier);
 
 struct msm_pending_timer;
 
@@ -488,15 +486,12 @@ void __iomem *msm_ioremap_mdss(struct platform_device *mdss_pdev,
 
 struct icc_path *msm_icc_get(struct device *dev, const char *name);
 
-#define msm_writel(data, addr) writel((data), (addr))
-#define msm_readl(addr) readl((addr))
-
 static inline void msm_rmw(void __iomem *addr, u32 mask, u32 or)
 {
-	u32 val = msm_readl(addr);
+	u32 val = readl(addr);
 
 	val &= ~mask;
-	msm_writel(val | or, addr);
+	writel(val | or, addr);
 }
 
 /**
diff --git a/drivers/gpu/drm/msm/msm_fb.c b/drivers/gpu/drm/msm/msm_fb.c
index 80166f702a0d..09268e416843 100644
--- a/drivers/gpu/drm/msm/msm_fb.c
+++ b/drivers/gpu/drm/msm/msm_fb.c
@@ -176,16 +176,16 @@ static struct drm_framebuffer *msm_framebuffer_init(struct drm_device *dev,
 	const struct msm_format *format;
 	int ret, i, n;
 
-	drm_dbg_state(dev, "create framebuffer: mode_cmd=%p (%dx%d@%4.4s)\n",
-			mode_cmd, mode_cmd->width, mode_cmd->height,
-			(char *)&mode_cmd->pixel_format);
+	drm_dbg_state(dev, "create framebuffer: mode_cmd=%p (%dx%d@%p4cc)\n",
+		      mode_cmd, mode_cmd->width, mode_cmd->height,
+		      &mode_cmd->pixel_format);
 
 	n = info->num_planes;
-	format = kms->funcs->get_format(kms, mode_cmd->pixel_format,
+	format = mdp_get_format(kms, mode_cmd->pixel_format,
 			mode_cmd->modifier[0]);
 	if (!format) {
-		DRM_DEV_ERROR(dev->dev, "unsupported pixel format: %4.4s\n",
-				(char *)&mode_cmd->pixel_format);
+		DRM_DEV_ERROR(dev->dev, "unsupported pixel format: %p4cc\n",
+			      &mode_cmd->pixel_format);
 		ret = -EINVAL;
 		goto fail;
 	}
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 175ee4ab8a6f..a5c6498a43f0 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -219,7 +219,7 @@ static void put_pages(struct drm_gem_object *obj)
 	}
 }
 
-static struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj,
+static struct page **msm_gem_get_pages_locked(struct drm_gem_object *obj,
 					      unsigned madv)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
@@ -257,24 +257,24 @@ static void pin_obj_locked(struct drm_gem_object *obj)
 	mutex_unlock(&priv->lru.lock);
 }
 
-struct page **msm_gem_pin_pages(struct drm_gem_object *obj)
+struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj)
 {
 	struct page **p;
 
-	msm_gem_lock(obj);
-	p = msm_gem_pin_pages_locked(obj, MSM_MADV_WILLNEED);
+	msm_gem_assert_locked(obj);
+
+	p = msm_gem_get_pages_locked(obj, MSM_MADV_WILLNEED);
 	if (!IS_ERR(p))
 		pin_obj_locked(obj);
-	msm_gem_unlock(obj);
 
 	return p;
 }
 
-void msm_gem_unpin_pages(struct drm_gem_object *obj)
+void msm_gem_unpin_pages_locked(struct drm_gem_object *obj)
 {
-	msm_gem_lock(obj);
+	msm_gem_assert_locked(obj);
+
 	msm_gem_unpin_locked(obj);
-	msm_gem_unlock(obj);
 }
 
 static pgprot_t msm_gem_pgprot(struct msm_gem_object *msm_obj, pgprot_t prot)
@@ -489,7 +489,7 @@ int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma)
 
 	msm_gem_assert_locked(obj);
 
-	pages = msm_gem_pin_pages_locked(obj, MSM_MADV_WILLNEED);
+	pages = msm_gem_get_pages_locked(obj, MSM_MADV_WILLNEED);
 	if (IS_ERR(pages))
 		return PTR_ERR(pages);
 
@@ -703,7 +703,7 @@ static void *get_vaddr(struct drm_gem_object *obj, unsigned madv)
 	if (obj->import_attach)
 		return ERR_PTR(-ENODEV);
 
-	pages = msm_gem_pin_pages_locked(obj, madv);
+	pages = msm_gem_get_pages_locked(obj, madv);
 	if (IS_ERR(pages))
 		return ERR_CAST(pages);
 
diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h
index 8d414b072c29..85f0257e83da 100644
--- a/drivers/gpu/drm/msm/msm_gem.h
+++ b/drivers/gpu/drm/msm/msm_gem.h
@@ -140,8 +140,8 @@ int msm_gem_get_and_pin_iova(struct drm_gem_object *obj,
 void msm_gem_unpin_iova(struct drm_gem_object *obj,
 		struct msm_gem_address_space *aspace);
 void msm_gem_pin_obj_locked(struct drm_gem_object *obj);
-struct page **msm_gem_pin_pages(struct drm_gem_object *obj);
-void msm_gem_unpin_pages(struct drm_gem_object *obj);
+struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj);
+void msm_gem_unpin_pages_locked(struct drm_gem_object *obj);
 int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
 		struct drm_mode_create_dumb *args);
 int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
diff --git a/drivers/gpu/drm/msm/msm_gem_prime.c b/drivers/gpu/drm/msm/msm_gem_prime.c
index 0915f3b68752..ee267490c935 100644
--- a/drivers/gpu/drm/msm/msm_gem_prime.c
+++ b/drivers/gpu/drm/msm/msm_gem_prime.c
@@ -47,13 +47,23 @@ struct drm_gem_object *msm_gem_prime_import_sg_table(struct drm_device *dev,
 
 int msm_gem_prime_pin(struct drm_gem_object *obj)
 {
-	if (!obj->import_attach)
-		msm_gem_pin_pages(obj);
-	return 0;
+	struct page **pages;
+	int ret = 0;
+
+	if (obj->import_attach)
+		return 0;
+
+	pages = msm_gem_pin_pages_locked(obj);
+	if (IS_ERR(pages))
+		ret = PTR_ERR(pages);
+
+	return ret;
 }
 
 void msm_gem_prime_unpin(struct drm_gem_object *obj)
 {
-	if (!obj->import_attach)
-		msm_gem_unpin_pages(obj);
+	if (obj->import_attach)
+		return;
+
+	msm_gem_unpin_pages_locked(obj);
 }
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 655002b21b0d..cd185b9636d2 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -11,7 +11,7 @@
 #include "msm_mmu.h"
 #include "msm_fence.h"
 #include "msm_gpu_trace.h"
-#include "adreno/adreno_gpu.h"
+//#include "adreno/adreno_gpu.h"
 
 #include <generated/utsrelease.h>
 #include <linux/string_helpers.h>
diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index 2bfcb222e353..a0c1bd6d1d5b 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -555,12 +555,12 @@ struct msm_gpu_state {
 
 static inline void gpu_write(struct msm_gpu *gpu, u32 reg, u32 data)
 {
-	msm_writel(data, gpu->mmio + (reg << 2));
+	writel(data, gpu->mmio + (reg << 2));
 }
 
 static inline u32 gpu_read(struct msm_gpu *gpu, u32 reg)
 {
-	return msm_readl(gpu->mmio + (reg << 2));
+	return readl(gpu->mmio + (reg << 2));
 }
 
 static inline void gpu_rmw(struct msm_gpu *gpu, u32 reg, u32 mask, u32 or)
@@ -586,8 +586,8 @@ static inline u64 gpu_read64(struct msm_gpu *gpu, u32 reg)
 	 * when the lo is read, so make sure to read the lo first to trigger
 	 * that
 	 */
-	val = (u64) msm_readl(gpu->mmio + (reg << 2));
-	val |= ((u64) msm_readl(gpu->mmio + ((reg + 1) << 2)) << 32);
+	val = (u64) readl(gpu->mmio + (reg << 2));
+	val |= ((u64) readl(gpu->mmio + ((reg + 1) << 2)) << 32);
 
 	return val;
 }
@@ -595,8 +595,8 @@ static inline u64 gpu_read64(struct msm_gpu *gpu, u32 reg)
 static inline void gpu_write64(struct msm_gpu *gpu, u32 reg, u64 val)
 {
 	/* Why not a writeq here? Read the screed above */
-	msm_writel(lower_32_bits(val), gpu->mmio + (reg << 2));
-	msm_writel(upper_32_bits(val), gpu->mmio + ((reg + 1) << 2));
+	writel(lower_32_bits(val), gpu->mmio + (reg << 2));
+	writel(upper_32_bits(val), gpu->mmio + ((reg + 1) << 2));
 }
 
 int msm_gpu_pm_suspend(struct msm_gpu *gpu);
diff --git a/drivers/gpu/drm/msm/msm_kms.h b/drivers/gpu/drm/msm/msm_kms.h
index 0641f6111b93..1e0c54de3716 100644
--- a/drivers/gpu/drm/msm/msm_kms.h
+++ b/drivers/gpu/drm/msm/msm_kms.h
@@ -92,10 +92,6 @@ struct msm_kms_funcs {
 	 * Format handling:
 	 */
 
-	/* get msm_format w/ optional format modifiers from drm_mode_fb_cmd2 */
-	const struct msm_format *(*get_format)(struct msm_kms *kms,
-					const uint32_t format,
-					const uint64_t modifiers);
 	/* do format checking on format modified through fb_cmd2 modifiers */
 	int (*check_modified_format)(const struct msm_kms *kms,
 			const struct msm_format *msm_fmt,
diff --git a/drivers/gpu/drm/msm/msm_mmu.h b/drivers/gpu/drm/msm/msm_mmu.h
index eb72d3645c1d..88af4f490881 100644
--- a/drivers/gpu/drm/msm/msm_mmu.h
+++ b/drivers/gpu/drm/msm/msm_mmu.h
@@ -42,7 +42,6 @@ static inline void msm_mmu_init(struct msm_mmu *mmu, struct device *dev,
 
 struct msm_mmu *msm_iommu_new(struct device *dev, unsigned long quirks);
 struct msm_mmu *msm_iommu_gpu_new(struct device *dev, struct msm_gpu *gpu, unsigned long quirks);
-struct msm_mmu *msm_gpummu_new(struct device *dev, struct msm_gpu *gpu);
 
 static inline void msm_mmu_set_fault_handler(struct msm_mmu *mmu, void *arg,
 		int (*handler)(void *arg, unsigned long iova, int flags, void *data))
@@ -53,10 +52,6 @@ static inline void msm_mmu_set_fault_handler(struct msm_mmu *mmu, void *arg,
 
 struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent);
 
-void msm_gpummu_params(struct msm_mmu *mmu, dma_addr_t *pt_base,
-		dma_addr_t *tran_error);
-
-
 int msm_iommu_pagetable_params(struct msm_mmu *mmu, phys_addr_t *ttbr,
 		int *asid);
 struct iommu_domain_geometry *msm_iommu_get_geometry(struct msm_mmu *mmu);
diff --git a/drivers/gpu/drm/msm/registers/.gitignore b/drivers/gpu/drm/msm/registers/.gitignore
new file mode 100644
index 000000000000..848e0e3efbcb
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/.gitignore
@@ -0,0 +1,4 @@
+# ignore XML files present at Mesa but not used by the kernel
+adreno/adreno_control_regs.xml
+adreno/adreno_pipe_regs.xml
+adreno/ocmem.xml
diff --git a/drivers/gpu/drm/msm/registers/adreno/a2xx.xml b/drivers/gpu/drm/msm/registers/adreno/a2xx.xml
new file mode 100644
index 000000000000..22caddaa0db9
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/adreno/a2xx.xml
@@ -0,0 +1,1865 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+<import file="adreno/adreno_common.xml"/>
+<import file="adreno/adreno_pm4.xml"/>
+
+
+<enum name="a2xx_rb_dither_type">
+	<value name="DITHER_PIXEL" value="0"/>
+	<value name="DITHER_SUBPIXEL" value="1"/>
+</enum>
+
+<enum name="a2xx_colorformatx">
+	<value name="COLORX_4_4_4_4" value="0"/>
+	<value name="COLORX_1_5_5_5" value="1"/>
+	<value name="COLORX_5_6_5" value="2"/>
+	<value name="COLORX_8" value="3"/>
+	<value name="COLORX_8_8" value="4"/>
+	<value name="COLORX_8_8_8_8" value="5"/>
+	<value name="COLORX_S8_8_8_8" value="6"/>
+	<value name="COLORX_16_FLOAT" value="7"/>
+	<value name="COLORX_16_16_FLOAT" value="8"/>
+	<value name="COLORX_16_16_16_16_FLOAT" value="9"/>
+	<value name="COLORX_32_FLOAT" value="10"/>
+	<value name="COLORX_32_32_FLOAT" value="11"/>
+	<value name="COLORX_32_32_32_32_FLOAT" value="12"/>
+	<value name="COLORX_2_3_3" value="13"/>
+	<value name="COLORX_8_8_8" value="14"/>
+</enum>
+
+<enum name="a2xx_sq_surfaceformat">
+	<value name="FMT_1_REVERSE" value="0"/>
+	<value name="FMT_1" value="1"/>
+	<value name="FMT_8" value="2"/>
+	<value name="FMT_1_5_5_5" value="3"/>
+	<value name="FMT_5_6_5" value="4"/>
+	<value name="FMT_6_5_5" value="5"/>
+	<value name="FMT_8_8_8_8" value="6"/>
+	<value name="FMT_2_10_10_10" value="7"/>
+	<value name="FMT_8_A" value="8"/>
+	<value name="FMT_8_B" value="9"/>
+	<value name="FMT_8_8" value="10"/>
+	<value name="FMT_Cr_Y1_Cb_Y0" value="11"/>
+	<value name="FMT_Y1_Cr_Y0_Cb" value="12"/>
+	<value name="FMT_5_5_5_1" value="13"/>
+	<value name="FMT_8_8_8_8_A" value="14"/>
+	<value name="FMT_4_4_4_4" value="15"/>
+	<value name="FMT_8_8_8" value="16"/>
+	<value name="FMT_DXT1" value="18"/>
+	<value name="FMT_DXT2_3" value="19"/>
+	<value name="FMT_DXT4_5" value="20"/>
+	<value name="FMT_10_10_10_2" value="21"/>
+	<value name="FMT_24_8" value="22"/>
+	<value name="FMT_16" value="24"/>
+	<value name="FMT_16_16" value="25"/>
+	<value name="FMT_16_16_16_16" value="26"/>
+	<value name="FMT_16_EXPAND" value="27"/>
+	<value name="FMT_16_16_EXPAND" value="28"/>
+	<value name="FMT_16_16_16_16_EXPAND" value="29"/>
+	<value name="FMT_16_FLOAT" value="30"/>
+	<value name="FMT_16_16_FLOAT" value="31"/>
+	<value name="FMT_16_16_16_16_FLOAT" value="32"/>
+	<value name="FMT_32" value="33"/>
+	<value name="FMT_32_32" value="34"/>
+	<value name="FMT_32_32_32_32" value="35"/>
+	<value name="FMT_32_FLOAT" value="36"/>
+	<value name="FMT_32_32_FLOAT" value="37"/>
+	<value name="FMT_32_32_32_32_FLOAT" value="38"/>
+	<value name="FMT_ATI_TC_RGB" value="39"/>
+	<value name="FMT_ATI_TC_RGBA" value="40"/>
+	<value name="FMT_ATI_TC_555_565_RGB" value="41"/>
+	<value name="FMT_ATI_TC_555_565_RGBA" value="42"/>
+	<value name="FMT_ATI_TC_RGBA_INTERP" value="43"/>
+	<value name="FMT_ATI_TC_555_565_RGBA_INTERP" value="44"/>
+	<value name="FMT_ETC1_RGBA_INTERP" value="46"/>
+	<value name="FMT_ETC1_RGB" value="47"/>
+	<value name="FMT_ETC1_RGBA" value="48"/>
+	<value name="FMT_DXN" value="49"/>
+	<value name="FMT_2_3_3" value="51"/>
+	<value name="FMT_2_10_10_10_AS_16_16_16_16" value="54"/>
+	<value name="FMT_10_10_10_2_AS_16_16_16_16" value="55"/>
+	<value name="FMT_32_32_32_FLOAT" value="57"/>
+	<value name="FMT_DXT3A" value="58"/>
+	<value name="FMT_DXT5A" value="59"/>
+	<value name="FMT_CTX1" value="60"/>
+</enum>
+
+<enum name="a2xx_sq_ps_vtx_mode">
+	<value name="POSITION_1_VECTOR" value="0"/>
+	<value name="POSITION_2_VECTORS_UNUSED" value="1"/>
+	<value name="POSITION_2_VECTORS_SPRITE" value="2"/>
+	<value name="POSITION_2_VECTORS_EDGE" value="3"/>
+	<value name="POSITION_2_VECTORS_KILL" value="4"/>
+	<value name="POSITION_2_VECTORS_SPRITE_KILL" value="5"/>
+	<value name="POSITION_2_VECTORS_EDGE_KILL" value="6"/>
+	<value name="MULTIPASS" value="7"/>
+</enum>
+
+<enum name="a2xx_sq_sample_cntl">
+	<value name="CENTROIDS_ONLY" value="0"/>
+	<value name="CENTERS_ONLY" value="1"/>
+	<value name="CENTROIDS_AND_CENTERS" value="2"/>
+</enum>
+
+<enum name="a2xx_dx_clip_space">
+	<value name="DXCLIP_OPENGL" value="0"/>
+	<value name="DXCLIP_DIRECTX" value="1"/>
+</enum>
+
+<enum name="a2xx_pa_su_sc_polymode">
+	<value name="POLY_DISABLED" value="0"/>
+	<value name="POLY_DUALMODE" value="1"/>
+</enum>
+
+<enum name="a2xx_rb_edram_mode">
+	<value name="EDRAM_NOP" value="0"/>
+	<value name="COLOR_DEPTH" value="4"/>
+	<value name="DEPTH_ONLY" value="5"/>
+	<value name="EDRAM_COPY" value="6"/>
+</enum>
+
+<enum name="a2xx_pa_sc_pattern_bit_order">
+	<value name="LITTLE" value="0"/>
+	<value name="BIG" value="1"/>
+</enum>
+
+<enum name="a2xx_pa_sc_auto_reset_cntl">
+	<value name="NEVER" value="0"/>
+	<value name="EACH_PRIMITIVE" value="1"/>
+	<value name="EACH_PACKET" value="2"/>
+</enum>
+
+<enum name="a2xx_pa_pixcenter">
+	<value name="PIXCENTER_D3D" value="0"/>
+	<value name="PIXCENTER_OGL" value="1"/>
+</enum>
+
+<enum name="a2xx_pa_roundmode">
+	<value name="TRUNCATE" value="0"/>
+	<value name="ROUND" value="1"/>
+	<value name="ROUNDTOEVEN" value="2"/>
+	<value name="ROUNDTOODD" value="3"/>
+</enum>
+
+<enum name="a2xx_pa_quantmode">
+	<value name="ONE_SIXTEENTH" value="0"/>
+	<value name="ONE_EIGTH" value="1"/>
+	<value name="ONE_QUARTER" value="2"/>
+	<value name="ONE_HALF" value="3"/>
+	<value name="ONE" value="4"/>
+</enum>
+
+<enum name="a2xx_rb_copy_sample_select">
+	<value name="SAMPLE_0" value="0"/>
+	<value name="SAMPLE_1" value="1"/>
+	<value name="SAMPLE_2" value="2"/>
+	<value name="SAMPLE_3" value="3"/>
+	<value name="SAMPLE_01" value="4"/>
+	<value name="SAMPLE_23" value="5"/>
+	<value name="SAMPLE_0123" value="6"/>
+</enum>
+
+<enum name="a2xx_rb_blend_opcode">
+	<value name="BLEND2_DST_PLUS_SRC" value="0"/>
+	<value name="BLEND2_SRC_MINUS_DST" value="1"/>
+	<value name="BLEND2_MIN_DST_SRC" value="2"/>
+	<value name="BLEND2_MAX_DST_SRC" value="3"/>
+	<value name="BLEND2_DST_MINUS_SRC" value="4"/>
+	<value name="BLEND2_DST_PLUS_SRC_BIAS" value="5"/>
+</enum>
+
+<enum name="a2xx_su_perfcnt_select">
+	<value value="0" name="PERF_PAPC_PASX_REQ"/>
+	<value value="2" name="PERF_PAPC_PASX_FIRST_VECTOR"/>
+	<value value="3" name="PERF_PAPC_PASX_SECOND_VECTOR"/>
+	<value value="4" name="PERF_PAPC_PASX_FIRST_DEAD"/>
+	<value value="5" name="PERF_PAPC_PASX_SECOND_DEAD"/>
+	<value value="6" name="PERF_PAPC_PASX_VTX_KILL_DISCARD"/>
+	<value value="7" name="PERF_PAPC_PASX_VTX_NAN_DISCARD"/>
+	<value value="8" name="PERF_PAPC_PA_INPUT_PRIM"/>
+	<value value="9" name="PERF_PAPC_PA_INPUT_NULL_PRIM"/>
+	<value value="10" name="PERF_PAPC_PA_INPUT_EVENT_FLAG"/>
+	<value value="11" name="PERF_PAPC_PA_INPUT_FIRST_PRIM_SLOT"/>
+	<value value="12" name="PERF_PAPC_PA_INPUT_END_OF_PACKET"/>
+	<value value="13" name="PERF_PAPC_CLPR_CULL_PRIM"/>
+	<value value="15" name="PERF_PAPC_CLPR_VV_CULL_PRIM"/>
+	<value value="17" name="PERF_PAPC_CLPR_VTX_KILL_CULL_PRIM"/>
+	<value value="18" name="PERF_PAPC_CLPR_VTX_NAN_CULL_PRIM"/>
+	<value value="19" name="PERF_PAPC_CLPR_CULL_TO_NULL_PRIM"/>
+	<value value="21" name="PERF_PAPC_CLPR_VV_CLIP_PRIM"/>
+	<value value="23" name="PERF_PAPC_CLPR_POINT_CLIP_CANDIDATE"/>
+	<value value="24" name="PERF_PAPC_CLPR_CLIP_PLANE_CNT_1"/>
+	<value value="25" name="PERF_PAPC_CLPR_CLIP_PLANE_CNT_2"/>
+	<value value="26" name="PERF_PAPC_CLPR_CLIP_PLANE_CNT_3"/>
+	<value value="27" name="PERF_PAPC_CLPR_CLIP_PLANE_CNT_4"/>
+	<value value="28" name="PERF_PAPC_CLPR_CLIP_PLANE_CNT_5"/>
+	<value value="29" name="PERF_PAPC_CLPR_CLIP_PLANE_CNT_6"/>
+	<value value="30" name="PERF_PAPC_CLPR_CLIP_PLANE_NEAR"/>
+	<value value="31" name="PERF_PAPC_CLPR_CLIP_PLANE_FAR"/>
+	<value value="32" name="PERF_PAPC_CLPR_CLIP_PLANE_LEFT"/>
+	<value value="33" name="PERF_PAPC_CLPR_CLIP_PLANE_RIGHT"/>
+	<value value="34" name="PERF_PAPC_CLPR_CLIP_PLANE_TOP"/>
+	<value value="35" name="PERF_PAPC_CLPR_CLIP_PLANE_BOTTOM"/>
+	<value value="36" name="PERF_PAPC_CLSM_NULL_PRIM"/>
+	<value value="37" name="PERF_PAPC_CLSM_TOTALLY_VISIBLE_PRIM"/>
+	<value value="38" name="PERF_PAPC_CLSM_CLIP_PRIM"/>
+	<value value="39" name="PERF_PAPC_CLSM_CULL_TO_NULL_PRIM"/>
+	<value value="40" name="PERF_PAPC_CLSM_OUT_PRIM_CNT_1"/>
+	<value value="41" name="PERF_PAPC_CLSM_OUT_PRIM_CNT_2"/>
+	<value value="42" name="PERF_PAPC_CLSM_OUT_PRIM_CNT_3"/>
+	<value value="43" name="PERF_PAPC_CLSM_OUT_PRIM_CNT_4"/>
+	<value value="44" name="PERF_PAPC_CLSM_OUT_PRIM_CNT_5"/>
+	<value value="45" name="PERF_PAPC_CLSM_OUT_PRIM_CNT_6_7"/>
+	<value value="46" name="PERF_PAPC_CLSM_NON_TRIVIAL_CULL"/>
+	<value value="47" name="PERF_PAPC_SU_INPUT_PRIM"/>
+	<value value="48" name="PERF_PAPC_SU_INPUT_CLIP_PRIM"/>
+	<value value="49" name="PERF_PAPC_SU_INPUT_NULL_PRIM"/>
+	<value value="50" name="PERF_PAPC_SU_ZERO_AREA_CULL_PRIM"/>
+	<value value="51" name="PERF_PAPC_SU_BACK_FACE_CULL_PRIM"/>
+	<value value="52" name="PERF_PAPC_SU_FRONT_FACE_CULL_PRIM"/>
+	<value value="53" name="PERF_PAPC_SU_POLYMODE_FACE_CULL"/>
+	<value value="54" name="PERF_PAPC_SU_POLYMODE_BACK_CULL"/>
+	<value value="55" name="PERF_PAPC_SU_POLYMODE_FRONT_CULL"/>
+	<value value="56" name="PERF_PAPC_SU_POLYMODE_INVALID_FILL"/>
+	<value value="57" name="PERF_PAPC_SU_OUTPUT_PRIM"/>
+	<value value="58" name="PERF_PAPC_SU_OUTPUT_CLIP_PRIM"/>
+	<value value="59" name="PERF_PAPC_SU_OUTPUT_NULL_PRIM"/>
+	<value value="60" name="PERF_PAPC_SU_OUTPUT_EVENT_FLAG"/>
+	<value value="61" name="PERF_PAPC_SU_OUTPUT_FIRST_PRIM_SLOT"/>
+	<value value="62" name="PERF_PAPC_SU_OUTPUT_END_OF_PACKET"/>
+	<value value="63" name="PERF_PAPC_SU_OUTPUT_POLYMODE_FACE"/>
+	<value value="64" name="PERF_PAPC_SU_OUTPUT_POLYMODE_BACK"/>
+	<value value="65" name="PERF_PAPC_SU_OUTPUT_POLYMODE_FRONT"/>
+	<value value="66" name="PERF_PAPC_SU_OUT_CLIP_POLYMODE_FACE"/>
+	<value value="67" name="PERF_PAPC_SU_OUT_CLIP_POLYMODE_BACK"/>
+	<value value="68" name="PERF_PAPC_SU_OUT_CLIP_POLYMODE_FRONT"/>
+	<value value="69" name="PERF_PAPC_PASX_REQ_IDLE"/>
+	<value value="70" name="PERF_PAPC_PASX_REQ_BUSY"/>
+	<value value="71" name="PERF_PAPC_PASX_REQ_STALLED"/>
+	<value value="72" name="PERF_PAPC_PASX_REC_IDLE"/>
+	<value value="73" name="PERF_PAPC_PASX_REC_BUSY"/>
+	<value value="74" name="PERF_PAPC_PASX_REC_STARVED_SX"/>
+	<value value="75" name="PERF_PAPC_PASX_REC_STALLED"/>
+	<value value="76" name="PERF_PAPC_PASX_REC_STALLED_POS_MEM"/>
+	<value value="77" name="PERF_PAPC_PASX_REC_STALLED_CCGSM_IN"/>
+	<value value="78" name="PERF_PAPC_CCGSM_IDLE"/>
+	<value value="79" name="PERF_PAPC_CCGSM_BUSY"/>
+	<value value="80" name="PERF_PAPC_CCGSM_STALLED"/>
+	<value value="81" name="PERF_PAPC_CLPRIM_IDLE"/>
+	<value value="82" name="PERF_PAPC_CLPRIM_BUSY"/>
+	<value value="83" name="PERF_PAPC_CLPRIM_STALLED"/>
+	<value value="84" name="PERF_PAPC_CLPRIM_STARVED_CCGSM"/>
+	<value value="85" name="PERF_PAPC_CLIPSM_IDLE"/>
+	<value value="86" name="PERF_PAPC_CLIPSM_BUSY"/>
+	<value value="87" name="PERF_PAPC_CLIPSM_WAIT_CLIP_VERT_ENGH"/>
+	<value value="88" name="PERF_PAPC_CLIPSM_WAIT_HIGH_PRI_SEQ"/>
+	<value value="89" name="PERF_PAPC_CLIPSM_WAIT_CLIPGA"/>
+	<value value="90" name="PERF_PAPC_CLIPSM_WAIT_AVAIL_VTE_CLIP"/>
+	<value value="91" name="PERF_PAPC_CLIPSM_WAIT_CLIP_OUTSM"/>
+	<value value="92" name="PERF_PAPC_CLIPGA_IDLE"/>
+	<value value="93" name="PERF_PAPC_CLIPGA_BUSY"/>
+	<value value="94" name="PERF_PAPC_CLIPGA_STARVED_VTE_CLIP"/>
+	<value value="95" name="PERF_PAPC_CLIPGA_STALLED"/>
+	<value value="96" name="PERF_PAPC_CLIP_IDLE"/>
+	<value value="97" name="PERF_PAPC_CLIP_BUSY"/>
+	<value value="98" name="PERF_PAPC_SU_IDLE"/>
+	<value value="99" name="PERF_PAPC_SU_BUSY"/>
+	<value value="100" name="PERF_PAPC_SU_STARVED_CLIP"/>
+	<value value="101" name="PERF_PAPC_SU_STALLED_SC"/>
+	<value value="102" name="PERF_PAPC_SU_FACENESS_CULL"/>
+</enum>
+
+<enum name="a2xx_sc_perfcnt_select">
+	<value value="0" name="SC_SR_WINDOW_VALID"/>
+	<value value="1" name="SC_CW_WINDOW_VALID"/>
+	<value value="2" name="SC_QM_WINDOW_VALID"/>
+	<value value="3" name="SC_FW_WINDOW_VALID"/>
+	<value value="4" name="SC_EZ_WINDOW_VALID"/>
+	<value value="5" name="SC_IT_WINDOW_VALID"/>
+	<value value="6" name="SC_STARVED_BY_PA"/>
+	<value value="7" name="SC_STALLED_BY_RB_TILE"/>
+	<value value="8" name="SC_STALLED_BY_RB_SAMP"/>
+	<value value="9" name="SC_STARVED_BY_RB_EZ"/>
+	<value value="10" name="SC_STALLED_BY_SAMPLE_FF"/>
+	<value value="11" name="SC_STALLED_BY_SQ"/>
+	<value value="12" name="SC_STALLED_BY_SP"/>
+	<value value="13" name="SC_TOTAL_NO_PRIMS"/>
+	<value value="14" name="SC_NON_EMPTY_PRIMS"/>
+	<value value="15" name="SC_NO_TILES_PASSING_QM"/>
+	<value value="16" name="SC_NO_PIXELS_PRE_EZ"/>
+	<value value="17" name="SC_NO_PIXELS_POST_EZ"/>
+</enum>
+
+<enum name="a2xx_vgt_perfcount_select">
+	<value value="0" name="VGT_SQ_EVENT_WINDOW_ACTIVE"/>
+	<value value="1" name="VGT_SQ_SEND"/>
+	<value value="2" name="VGT_SQ_STALLED"/>
+	<value value="3" name="VGT_SQ_STARVED_BUSY"/>
+	<value value="4" name="VGT_SQ_STARVED_IDLE"/>
+	<value value="5" name="VGT_SQ_STATIC"/>
+	<value value="6" name="VGT_PA_EVENT_WINDOW_ACTIVE"/>
+	<value value="7" name="VGT_PA_CLIP_V_SEND"/>
+	<value value="8" name="VGT_PA_CLIP_V_STALLED"/>
+	<value value="9" name="VGT_PA_CLIP_V_STARVED_BUSY"/>
+	<value value="10" name="VGT_PA_CLIP_V_STARVED_IDLE"/>
+	<value value="11" name="VGT_PA_CLIP_V_STATIC"/>
+	<value value="12" name="VGT_PA_CLIP_P_SEND"/>
+	<value value="13" name="VGT_PA_CLIP_P_STALLED"/>
+	<value value="14" name="VGT_PA_CLIP_P_STARVED_BUSY"/>
+	<value value="15" name="VGT_PA_CLIP_P_STARVED_IDLE"/>
+	<value value="16" name="VGT_PA_CLIP_P_STATIC"/>
+	<value value="17" name="VGT_PA_CLIP_S_SEND"/>
+	<value value="18" name="VGT_PA_CLIP_S_STALLED"/>
+	<value value="19" name="VGT_PA_CLIP_S_STARVED_BUSY"/>
+	<value value="20" name="VGT_PA_CLIP_S_STARVED_IDLE"/>
+	<value value="21" name="VGT_PA_CLIP_S_STATIC"/>
+	<value value="22" name="RBIU_FIFOS_EVENT_WINDOW_ACTIVE"/>
+	<value value="23" name="RBIU_IMMED_DATA_FIFO_STARVED"/>
+	<value value="24" name="RBIU_IMMED_DATA_FIFO_STALLED"/>
+	<value value="25" name="RBIU_DMA_REQUEST_FIFO_STARVED"/>
+	<value value="26" name="RBIU_DMA_REQUEST_FIFO_STALLED"/>
+	<value value="27" name="RBIU_DRAW_INITIATOR_FIFO_STARVED"/>
+	<value value="28" name="RBIU_DRAW_INITIATOR_FIFO_STALLED"/>
+	<value value="29" name="BIN_PRIM_NEAR_CULL"/>
+	<value value="30" name="BIN_PRIM_ZERO_CULL"/>
+	<value value="31" name="BIN_PRIM_FAR_CULL"/>
+	<value value="32" name="BIN_PRIM_BIN_CULL"/>
+	<value value="33" name="BIN_PRIM_FACE_CULL"/>
+	<value value="34" name="SPARE34"/>
+	<value value="35" name="SPARE35"/>
+	<value value="36" name="SPARE36"/>
+	<value value="37" name="SPARE37"/>
+	<value value="38" name="SPARE38"/>
+	<value value="39" name="SPARE39"/>
+	<value value="40" name="TE_SU_IN_VALID"/>
+	<value value="41" name="TE_SU_IN_READ"/>
+	<value value="42" name="TE_SU_IN_PRIM"/>
+	<value value="43" name="TE_SU_IN_EOP"/>
+	<value value="44" name="TE_SU_IN_NULL_PRIM"/>
+	<value value="45" name="TE_WK_IN_VALID"/>
+	<value value="46" name="TE_WK_IN_READ"/>
+	<value value="47" name="TE_OUT_PRIM_VALID"/>
+	<value value="48" name="TE_OUT_PRIM_READ"/>
+</enum>
+
+<enum name="a2xx_tcr_perfcount_select">
+	<value value="0" name="DGMMPD_IPMUX0_STALL"/>
+	<value value="4" name="DGMMPD_IPMUX_ALL_STALL"/>
+	<value value="5" name="OPMUX0_L2_WRITES"/>
+</enum>
+
+<enum name="a2xx_tp_perfcount_select">
+	<value value="0" name="POINT_QUADS"/>
+	<value value="1" name="BILIN_QUADS"/>
+	<value value="2" name="ANISO_QUADS"/>
+	<value value="3" name="MIP_QUADS"/>
+	<value value="4" name="VOL_QUADS"/>
+	<value value="5" name="MIP_VOL_QUADS"/>
+	<value value="6" name="MIP_ANISO_QUADS"/>
+	<value value="7" name="VOL_ANISO_QUADS"/>
+	<value value="8" name="ANISO_2_1_QUADS"/>
+	<value value="9" name="ANISO_4_1_QUADS"/>
+	<value value="10" name="ANISO_6_1_QUADS"/>
+	<value value="11" name="ANISO_8_1_QUADS"/>
+	<value value="12" name="ANISO_10_1_QUADS"/>
+	<value value="13" name="ANISO_12_1_QUADS"/>
+	<value value="14" name="ANISO_14_1_QUADS"/>
+	<value value="15" name="ANISO_16_1_QUADS"/>
+	<value value="16" name="MIP_VOL_ANISO_QUADS"/>
+	<value value="17" name="ALIGN_2_QUADS"/>
+	<value value="18" name="ALIGN_4_QUADS"/>
+	<value value="19" name="PIX_0_QUAD"/>
+	<value value="20" name="PIX_1_QUAD"/>
+	<value value="21" name="PIX_2_QUAD"/>
+	<value value="22" name="PIX_3_QUAD"/>
+	<value value="23" name="PIX_4_QUAD"/>
+	<value value="24" name="TP_MIPMAP_LOD0"/>
+	<value value="25" name="TP_MIPMAP_LOD1"/>
+	<value value="26" name="TP_MIPMAP_LOD2"/>
+	<value value="27" name="TP_MIPMAP_LOD3"/>
+	<value value="28" name="TP_MIPMAP_LOD4"/>
+	<value value="29" name="TP_MIPMAP_LOD5"/>
+	<value value="30" name="TP_MIPMAP_LOD6"/>
+	<value value="31" name="TP_MIPMAP_LOD7"/>
+	<value value="32" name="TP_MIPMAP_LOD8"/>
+	<value value="33" name="TP_MIPMAP_LOD9"/>
+	<value value="34" name="TP_MIPMAP_LOD10"/>
+	<value value="35" name="TP_MIPMAP_LOD11"/>
+	<value value="36" name="TP_MIPMAP_LOD12"/>
+	<value value="37" name="TP_MIPMAP_LOD13"/>
+	<value value="38" name="TP_MIPMAP_LOD14"/>
+</enum>
+
+<enum name="a2xx_tcm_perfcount_select">
+	<value value="0" name="QUAD0_RD_LAT_FIFO_EMPTY"/>
+	<value value="3" name="QUAD0_RD_LAT_FIFO_4TH_FULL"/>
+	<value value="4" name="QUAD0_RD_LAT_FIFO_HALF_FULL"/>
+	<value value="5" name="QUAD0_RD_LAT_FIFO_FULL"/>
+	<value value="6" name="QUAD0_RD_LAT_FIFO_LT_4TH_FULL"/>
+	<value value="28" name="READ_STARVED_QUAD0"/>
+	<value value="32" name="READ_STARVED"/>
+	<value value="33" name="READ_STALLED_QUAD0"/>
+	<value value="37" name="READ_STALLED"/>
+	<value value="38" name="VALID_READ_QUAD0"/>
+	<value value="42" name="TC_TP_STARVED_QUAD0"/>
+	<value value="46" name="TC_TP_STARVED"/>
+</enum>
+
+<enum name="a2xx_tcf_perfcount_select">
+	<value value="0" name="VALID_CYCLES"/>
+	<value value="1" name="SINGLE_PHASES"/>
+	<value value="2" name="ANISO_PHASES"/>
+	<value value="3" name="MIP_PHASES"/>
+	<value value="4" name="VOL_PHASES"/>
+	<value value="5" name="MIP_VOL_PHASES"/>
+	<value value="6" name="MIP_ANISO_PHASES"/>
+	<value value="7" name="VOL_ANISO_PHASES"/>
+	<value value="8" name="ANISO_2_1_PHASES"/>
+	<value value="9" name="ANISO_4_1_PHASES"/>
+	<value value="10" name="ANISO_6_1_PHASES"/>
+	<value value="11" name="ANISO_8_1_PHASES"/>
+	<value value="12" name="ANISO_10_1_PHASES"/>
+	<value value="13" name="ANISO_12_1_PHASES"/>
+	<value value="14" name="ANISO_14_1_PHASES"/>
+	<value value="15" name="ANISO_16_1_PHASES"/>
+	<value value="16" name="MIP_VOL_ANISO_PHASES"/>
+	<value value="17" name="ALIGN_2_PHASES"/>
+	<value value="18" name="ALIGN_4_PHASES"/>
+	<value value="19" name="TPC_BUSY"/>
+	<value value="20" name="TPC_STALLED"/>
+	<value value="21" name="TPC_STARVED"/>
+	<value value="22" name="TPC_WORKING"/>
+	<value value="23" name="TPC_WALKER_BUSY"/>
+	<value value="24" name="TPC_WALKER_STALLED"/>
+	<value value="25" name="TPC_WALKER_WORKING"/>
+	<value value="26" name="TPC_ALIGNER_BUSY"/>
+	<value value="27" name="TPC_ALIGNER_STALLED"/>
+	<value value="28" name="TPC_ALIGNER_STALLED_BY_BLEND"/>
+	<value value="29" name="TPC_ALIGNER_STALLED_BY_CACHE"/>
+	<value value="30" name="TPC_ALIGNER_WORKING"/>
+	<value value="31" name="TPC_BLEND_BUSY"/>
+	<value value="32" name="TPC_BLEND_SYNC"/>
+	<value value="33" name="TPC_BLEND_STARVED"/>
+	<value value="34" name="TPC_BLEND_WORKING"/>
+	<value value="35" name="OPCODE_0x00"/>
+	<value value="36" name="OPCODE_0x01"/>
+	<value value="37" name="OPCODE_0x04"/>
+	<value value="38" name="OPCODE_0x10"/>
+	<value value="39" name="OPCODE_0x11"/>
+	<value value="40" name="OPCODE_0x12"/>
+	<value value="41" name="OPCODE_0x13"/>
+	<value value="42" name="OPCODE_0x18"/>
+	<value value="43" name="OPCODE_0x19"/>
+	<value value="44" name="OPCODE_0x1A"/>
+	<value value="45" name="OPCODE_OTHER"/>
+	<value value="56" name="IN_FIFO_0_EMPTY"/>
+	<value value="57" name="IN_FIFO_0_LT_HALF_FULL"/>
+	<value value="58" name="IN_FIFO_0_HALF_FULL"/>
+	<value value="59" name="IN_FIFO_0_FULL"/>
+	<value value="72" name="IN_FIFO_TPC_EMPTY"/>
+	<value value="73" name="IN_FIFO_TPC_LT_HALF_FULL"/>
+	<value value="74" name="IN_FIFO_TPC_HALF_FULL"/>
+	<value value="75" name="IN_FIFO_TPC_FULL"/>
+	<value value="76" name="TPC_TC_XFC"/>
+	<value value="77" name="TPC_TC_STATE"/>
+	<value value="78" name="TC_STALL"/>
+	<value value="79" name="QUAD0_TAPS"/>
+	<value value="83" name="QUADS"/>
+	<value value="84" name="TCA_SYNC_STALL"/>
+	<value value="85" name="TAG_STALL"/>
+	<value value="88" name="TCB_SYNC_STALL"/>
+	<value value="89" name="TCA_VALID"/>
+	<value value="90" name="PROBES_VALID"/>
+	<value value="91" name="MISS_STALL"/>
+	<value value="92" name="FETCH_FIFO_STALL"/>
+	<value value="93" name="TCO_STALL"/>
+	<value value="94" name="ANY_STALL"/>
+	<value value="95" name="TAG_MISSES"/>
+	<value value="96" name="TAG_HITS"/>
+	<value value="97" name="SUB_TAG_MISSES"/>
+	<value value="98" name="SET0_INVALIDATES"/>
+	<value value="99" name="SET1_INVALIDATES"/>
+	<value value="100" name="SET2_INVALIDATES"/>
+	<value value="101" name="SET3_INVALIDATES"/>
+	<value value="102" name="SET0_TAG_MISSES"/>
+	<value value="103" name="SET1_TAG_MISSES"/>
+	<value value="104" name="SET2_TAG_MISSES"/>
+	<value value="105" name="SET3_TAG_MISSES"/>
+	<value value="106" name="SET0_TAG_HITS"/>
+	<value value="107" name="SET1_TAG_HITS"/>
+	<value value="108" name="SET2_TAG_HITS"/>
+	<value value="109" name="SET3_TAG_HITS"/>
+	<value value="110" name="SET0_SUB_TAG_MISSES"/>
+	<value value="111" name="SET1_SUB_TAG_MISSES"/>
+	<value value="112" name="SET2_SUB_TAG_MISSES"/>
+	<value value="113" name="SET3_SUB_TAG_MISSES"/>
+	<value value="114" name="SET0_EVICT1"/>
+	<value value="115" name="SET0_EVICT2"/>
+	<value value="116" name="SET0_EVICT3"/>
+	<value value="117" name="SET0_EVICT4"/>
+	<value value="118" name="SET0_EVICT5"/>
+	<value value="119" name="SET0_EVICT6"/>
+	<value value="120" name="SET0_EVICT7"/>
+	<value value="121" name="SET0_EVICT8"/>
+	<value value="130" name="SET1_EVICT1"/>
+	<value value="131" name="SET1_EVICT2"/>
+	<value value="132" name="SET1_EVICT3"/>
+	<value value="133" name="SET1_EVICT4"/>
+	<value value="134" name="SET1_EVICT5"/>
+	<value value="135" name="SET1_EVICT6"/>
+	<value value="136" name="SET1_EVICT7"/>
+	<value value="137" name="SET1_EVICT8"/>
+	<value value="146" name="SET2_EVICT1"/>
+	<value value="147" name="SET2_EVICT2"/>
+	<value value="148" name="SET2_EVICT3"/>
+	<value value="149" name="SET2_EVICT4"/>
+	<value value="150" name="SET2_EVICT5"/>
+	<value value="151" name="SET2_EVICT6"/>
+	<value value="152" name="SET2_EVICT7"/>
+	<value value="153" name="SET2_EVICT8"/>
+	<value value="162" name="SET3_EVICT1"/>
+	<value value="163" name="SET3_EVICT2"/>
+	<value value="164" name="SET3_EVICT3"/>
+	<value value="165" name="SET3_EVICT4"/>
+	<value value="166" name="SET3_EVICT5"/>
+	<value value="167" name="SET3_EVICT6"/>
+	<value value="168" name="SET3_EVICT7"/>
+	<value value="169" name="SET3_EVICT8"/>
+	<value value="178" name="FF_EMPTY"/>
+	<value value="179" name="FF_LT_HALF_FULL"/>
+	<value value="180" name="FF_HALF_FULL"/>
+	<value value="181" name="FF_FULL"/>
+	<value value="182" name="FF_XFC"/>
+	<value value="183" name="FF_STALLED"/>
+	<value value="184" name="FG_MASKS"/>
+	<value value="185" name="FG_LEFT_MASKS"/>
+	<value value="186" name="FG_LEFT_MASK_STALLED"/>
+	<value value="187" name="FG_LEFT_NOT_DONE_STALL"/>
+	<value value="188" name="FG_LEFT_FG_STALL"/>
+	<value value="189" name="FG_LEFT_SECTORS"/>
+	<value value="195" name="FG0_REQUESTS"/>
+	<value value="196" name="FG0_STALLED"/>
+	<value value="199" name="MEM_REQ512"/>
+	<value value="200" name="MEM_REQ_SENT"/>
+	<value value="202" name="MEM_LOCAL_READ_REQ"/>
+	<value value="203" name="TC0_MH_STALLED"/>
+</enum>
+
+<enum name="a2xx_sq_perfcnt_select">
+	<value value="0" name="SQ_PIXEL_VECTORS_SUB"/>
+	<value value="1" name="SQ_VERTEX_VECTORS_SUB"/>
+	<value value="2" name="SQ_ALU0_ACTIVE_VTX_SIMD0"/>
+	<value value="3" name="SQ_ALU1_ACTIVE_VTX_SIMD0"/>
+	<value value="4" name="SQ_ALU0_ACTIVE_PIX_SIMD0"/>
+	<value value="5" name="SQ_ALU1_ACTIVE_PIX_SIMD0"/>
+	<value value="6" name="SQ_ALU0_ACTIVE_VTX_SIMD1"/>
+	<value value="7" name="SQ_ALU1_ACTIVE_VTX_SIMD1"/>
+	<value value="8" name="SQ_ALU0_ACTIVE_PIX_SIMD1"/>
+	<value value="9" name="SQ_ALU1_ACTIVE_PIX_SIMD1"/>
+	<value value="10" name="SQ_EXPORT_CYCLES"/>
+	<value value="11" name="SQ_ALU_CST_WRITTEN"/>
+	<value value="12" name="SQ_TEX_CST_WRITTEN"/>
+	<value value="13" name="SQ_ALU_CST_STALL"/>
+	<value value="14" name="SQ_ALU_TEX_STALL"/>
+	<value value="15" name="SQ_INST_WRITTEN"/>
+	<value value="16" name="SQ_BOOLEAN_WRITTEN"/>
+	<value value="17" name="SQ_LOOPS_WRITTEN"/>
+	<value value="18" name="SQ_PIXEL_SWAP_IN"/>
+	<value value="19" name="SQ_PIXEL_SWAP_OUT"/>
+	<value value="20" name="SQ_VERTEX_SWAP_IN"/>
+	<value value="21" name="SQ_VERTEX_SWAP_OUT"/>
+	<value value="22" name="SQ_ALU_VTX_INST_ISSUED"/>
+	<value value="23" name="SQ_TEX_VTX_INST_ISSUED"/>
+	<value value="24" name="SQ_VC_VTX_INST_ISSUED"/>
+	<value value="25" name="SQ_CF_VTX_INST_ISSUED"/>
+	<value value="26" name="SQ_ALU_PIX_INST_ISSUED"/>
+	<value value="27" name="SQ_TEX_PIX_INST_ISSUED"/>
+	<value value="28" name="SQ_VC_PIX_INST_ISSUED"/>
+	<value value="29" name="SQ_CF_PIX_INST_ISSUED"/>
+	<value value="30" name="SQ_ALU0_FIFO_EMPTY_SIMD0"/>
+	<value value="31" name="SQ_ALU1_FIFO_EMPTY_SIMD0"/>
+	<value value="32" name="SQ_ALU0_FIFO_EMPTY_SIMD1"/>
+	<value value="33" name="SQ_ALU1_FIFO_EMPTY_SIMD1"/>
+	<value value="34" name="SQ_ALU_NOPS"/>
+	<value value="35" name="SQ_PRED_SKIP"/>
+	<value value="36" name="SQ_SYNC_ALU_STALL_SIMD0_VTX"/>
+	<value value="37" name="SQ_SYNC_ALU_STALL_SIMD1_VTX"/>
+	<value value="38" name="SQ_SYNC_TEX_STALL_VTX"/>
+	<value value="39" name="SQ_SYNC_VC_STALL_VTX"/>
+	<value value="40" name="SQ_CONSTANTS_USED_SIMD0"/>
+	<value value="41" name="SQ_CONSTANTS_SENT_SP_SIMD0"/>
+	<value value="42" name="SQ_GPR_STALL_VTX"/>
+	<value value="43" name="SQ_GPR_STALL_PIX"/>
+	<value value="44" name="SQ_VTX_RS_STALL"/>
+	<value value="45" name="SQ_PIX_RS_STALL"/>
+	<value value="46" name="SQ_SX_PC_FULL"/>
+	<value value="47" name="SQ_SX_EXP_BUFF_FULL"/>
+	<value value="48" name="SQ_SX_POS_BUFF_FULL"/>
+	<value value="49" name="SQ_INTERP_QUADS"/>
+	<value value="50" name="SQ_INTERP_ACTIVE"/>
+	<value value="51" name="SQ_IN_PIXEL_STALL"/>
+	<value value="52" name="SQ_IN_VTX_STALL"/>
+	<value value="53" name="SQ_VTX_CNT"/>
+	<value value="54" name="SQ_VTX_VECTOR2"/>
+	<value value="55" name="SQ_VTX_VECTOR3"/>
+	<value value="56" name="SQ_VTX_VECTOR4"/>
+	<value value="57" name="SQ_PIXEL_VECTOR1"/>
+	<value value="58" name="SQ_PIXEL_VECTOR23"/>
+	<value value="59" name="SQ_PIXEL_VECTOR4"/>
+	<value value="60" name="SQ_CONSTANTS_USED_SIMD1"/>
+	<value value="61" name="SQ_CONSTANTS_SENT_SP_SIMD1"/>
+	<value value="62" name="SQ_SX_MEM_EXP_FULL"/>
+	<value value="63" name="SQ_ALU0_ACTIVE_VTX_SIMD2"/>
+	<value value="64" name="SQ_ALU1_ACTIVE_VTX_SIMD2"/>
+	<value value="65" name="SQ_ALU0_ACTIVE_PIX_SIMD2"/>
+	<value value="66" name="SQ_ALU1_ACTIVE_PIX_SIMD2"/>
+	<value value="67" name="SQ_ALU0_ACTIVE_VTX_SIMD3"/>
+	<value value="68" name="SQ_PERFCOUNT_VTX_QUAL_TP_DONE"/>
+	<value value="69" name="SQ_ALU0_ACTIVE_PIX_SIMD3"/>
+	<value value="70" name="SQ_PERFCOUNT_PIX_QUAL_TP_DONE"/>
+	<value value="71" name="SQ_ALU0_FIFO_EMPTY_SIMD2"/>
+	<value value="72" name="SQ_ALU1_FIFO_EMPTY_SIMD2"/>
+	<value value="73" name="SQ_ALU0_FIFO_EMPTY_SIMD3"/>
+	<value value="74" name="SQ_ALU1_FIFO_EMPTY_SIMD3"/>
+	<value value="75" name="SQ_SYNC_ALU_STALL_SIMD2_VTX"/>
+	<value value="76" name="SQ_PERFCOUNT_VTX_POP_THREAD"/>
+	<value value="77" name="SQ_SYNC_ALU_STALL_SIMD0_PIX"/>
+	<value value="78" name="SQ_SYNC_ALU_STALL_SIMD1_PIX"/>
+	<value value="79" name="SQ_SYNC_ALU_STALL_SIMD2_PIX"/>
+	<value value="80" name="SQ_PERFCOUNT_PIX_POP_THREAD"/>
+	<value value="81" name="SQ_SYNC_TEX_STALL_PIX"/>
+	<value value="82" name="SQ_SYNC_VC_STALL_PIX"/>
+	<value value="83" name="SQ_CONSTANTS_USED_SIMD2"/>
+	<value value="84" name="SQ_CONSTANTS_SENT_SP_SIMD2"/>
+	<value value="85" name="SQ_PERFCOUNT_VTX_DEALLOC_ACK"/>
+	<value value="86" name="SQ_PERFCOUNT_PIX_DEALLOC_ACK"/>
+	<value value="87" name="SQ_ALU0_FIFO_FULL_SIMD0"/>
+	<value value="88" name="SQ_ALU1_FIFO_FULL_SIMD0"/>
+	<value value="89" name="SQ_ALU0_FIFO_FULL_SIMD1"/>
+	<value value="90" name="SQ_ALU1_FIFO_FULL_SIMD1"/>
+	<value value="91" name="SQ_ALU0_FIFO_FULL_SIMD2"/>
+	<value value="92" name="SQ_ALU1_FIFO_FULL_SIMD2"/>
+	<value value="93" name="SQ_ALU0_FIFO_FULL_SIMD3"/>
+	<value value="94" name="SQ_ALU1_FIFO_FULL_SIMD3"/>
+	<value value="95" name="VC_PERF_STATIC"/>
+	<value value="96" name="VC_PERF_STALLED"/>
+	<value value="97" name="VC_PERF_STARVED"/>
+	<value value="98" name="VC_PERF_SEND"/>
+	<value value="99" name="VC_PERF_ACTUAL_STARVED"/>
+	<value value="100" name="PIXEL_THREAD_0_ACTIVE"/>
+	<value value="101" name="VERTEX_THREAD_0_ACTIVE"/>
+	<value value="102" name="PIXEL_THREAD_0_NUMBER"/>
+	<value value="103" name="VERTEX_THREAD_0_NUMBER"/>
+	<value value="104" name="VERTEX_EVENT_NUMBER"/>
+	<value value="105" name="PIXEL_EVENT_NUMBER"/>
+	<value value="106" name="PTRBUFF_EF_PUSH"/>
+	<value value="107" name="PTRBUFF_EF_POP_EVENT"/>
+	<value value="108" name="PTRBUFF_EF_POP_NEW_VTX"/>
+	<value value="109" name="PTRBUFF_EF_POP_DEALLOC"/>
+	<value value="110" name="PTRBUFF_EF_POP_PVECTOR"/>
+	<value value="111" name="PTRBUFF_EF_POP_PVECTOR_X"/>
+	<value value="112" name="PTRBUFF_EF_POP_PVECTOR_VNZ"/>
+	<value value="113" name="PTRBUFF_PB_DEALLOC"/>
+	<value value="114" name="PTRBUFF_PI_STATE_PPB_POP"/>
+	<value value="115" name="PTRBUFF_PI_RTR"/>
+	<value value="116" name="PTRBUFF_PI_READ_EN"/>
+	<value value="117" name="PTRBUFF_PI_BUFF_SWAP"/>
+	<value value="118" name="PTRBUFF_SQ_FREE_BUFF"/>
+	<value value="119" name="PTRBUFF_SQ_DEC"/>
+	<value value="120" name="PTRBUFF_SC_VALID_CNTL_EVENT"/>
+	<value value="121" name="PTRBUFF_SC_VALID_IJ_XFER"/>
+	<value value="122" name="PTRBUFF_SC_NEW_VECTOR_1_Q"/>
+	<value value="123" name="PTRBUFF_QUAL_NEW_VECTOR"/>
+	<value value="124" name="PTRBUFF_QUAL_EVENT"/>
+	<value value="125" name="PTRBUFF_END_BUFFER"/>
+	<value value="126" name="PTRBUFF_FILL_QUAD"/>
+	<value value="127" name="VERTS_WRITTEN_SPI"/>
+	<value value="128" name="TP_FETCH_INSTR_EXEC"/>
+	<value value="129" name="TP_FETCH_INSTR_REQ"/>
+	<value value="130" name="TP_DATA_RETURN"/>
+	<value value="131" name="SPI_WRITE_CYCLES_SP"/>
+	<value value="132" name="SPI_WRITES_SP"/>
+	<value value="133" name="SP_ALU_INSTR_EXEC"/>
+	<value value="134" name="SP_CONST_ADDR_TO_SQ"/>
+	<value value="135" name="SP_PRED_KILLS_TO_SQ"/>
+	<value value="136" name="SP_EXPORT_CYCLES_TO_SX"/>
+	<value value="137" name="SP_EXPORTS_TO_SX"/>
+	<value value="138" name="SQ_CYCLES_ELAPSED"/>
+	<value value="139" name="SQ_TCFS_OPT_ALLOC_EXEC"/>
+	<value value="140" name="SQ_TCFS_NO_OPT_ALLOC"/>
+	<value value="141" name="SQ_ALU0_NO_OPT_ALLOC"/>
+	<value value="142" name="SQ_ALU1_NO_OPT_ALLOC"/>
+	<value value="143" name="SQ_TCFS_ARB_XFC_CNT"/>
+	<value value="144" name="SQ_ALU0_ARB_XFC_CNT"/>
+	<value value="145" name="SQ_ALU1_ARB_XFC_CNT"/>
+	<value value="146" name="SQ_TCFS_CFS_UPDATE_CNT"/>
+	<value value="147" name="SQ_ALU0_CFS_UPDATE_CNT"/>
+	<value value="148" name="SQ_ALU1_CFS_UPDATE_CNT"/>
+	<value value="149" name="SQ_VTX_PUSH_THREAD_CNT"/>
+	<value value="150" name="SQ_VTX_POP_THREAD_CNT"/>
+	<value value="151" name="SQ_PIX_PUSH_THREAD_CNT"/>
+	<value value="152" name="SQ_PIX_POP_THREAD_CNT"/>
+	<value value="153" name="SQ_PIX_TOTAL"/>
+	<value value="154" name="SQ_PIX_KILLED"/>
+</enum>
+
+<enum name="a2xx_sx_perfcnt_select">
+	<value value="0" name="SX_EXPORT_VECTORS"/>
+	<value value="1" name="SX_DUMMY_QUADS"/>
+	<value value="2" name="SX_ALPHA_FAIL"/>
+	<value value="3" name="SX_RB_QUAD_BUSY"/>
+	<value value="4" name="SX_RB_COLOR_BUSY"/>
+	<value value="5" name="SX_RB_QUAD_STALL"/>
+	<value value="6" name="SX_RB_COLOR_STALL"/>
+</enum>
+
+<enum name="a2xx_rbbm_perfcount1_sel">
+	<value value="0" name="RBBM1_COUNT"/>
+	<value value="1" name="RBBM1_NRT_BUSY"/>
+	<value value="2" name="RBBM1_RB_BUSY"/>
+	<value value="3" name="RBBM1_SQ_CNTX0_BUSY"/>
+	<value value="4" name="RBBM1_SQ_CNTX17_BUSY"/>
+	<value value="5" name="RBBM1_VGT_BUSY"/>
+	<value value="6" name="RBBM1_VGT_NODMA_BUSY"/>
+	<value value="7" name="RBBM1_PA_BUSY"/>
+	<value value="8" name="RBBM1_SC_CNTX_BUSY"/>
+	<value value="9" name="RBBM1_TPC_BUSY"/>
+	<value value="10" name="RBBM1_TC_BUSY"/>
+	<value value="11" name="RBBM1_SX_BUSY"/>
+	<value value="12" name="RBBM1_CP_COHER_BUSY"/>
+	<value value="13" name="RBBM1_CP_NRT_BUSY"/>
+	<value value="14" name="RBBM1_GFX_IDLE_STALL"/>
+	<value value="15" name="RBBM1_INTERRUPT"/>
+</enum>
+
+<enum name="a2xx_cp_perfcount_sel">
+	<value value="0" name="ALWAYS_COUNT"/>
+	<value value="1" name="TRANS_FIFO_FULL"/>
+	<value value="2" name="TRANS_FIFO_AF"/>
+	<value value="3" name="RCIU_PFPTRANS_WAIT"/>
+	<value value="6" name="RCIU_NRTTRANS_WAIT"/>
+	<value value="8" name="CSF_NRT_READ_WAIT"/>
+	<value value="9" name="CSF_I1_FIFO_FULL"/>
+	<value value="10" name="CSF_I2_FIFO_FULL"/>
+	<value value="11" name="CSF_ST_FIFO_FULL"/>
+	<value value="13" name="CSF_RING_ROQ_FULL"/>
+	<value value="14" name="CSF_I1_ROQ_FULL"/>
+	<value value="15" name="CSF_I2_ROQ_FULL"/>
+	<value value="16" name="CSF_ST_ROQ_FULL"/>
+	<value value="18" name="MIU_TAG_MEM_FULL"/>
+	<value value="19" name="MIU_WRITECLEAN"/>
+	<value value="22" name="MIU_NRT_WRITE_STALLED"/>
+	<value value="23" name="MIU_NRT_READ_STALLED"/>
+	<value value="24" name="ME_WRITE_CONFIRM_FIFO_FULL"/>
+	<value value="25" name="ME_VS_DEALLOC_FIFO_FULL"/>
+	<value value="26" name="ME_PS_DEALLOC_FIFO_FULL"/>
+	<value value="27" name="ME_REGS_VS_EVENT_FIFO_FULL"/>
+	<value value="28" name="ME_REGS_PS_EVENT_FIFO_FULL"/>
+	<value value="29" name="ME_REGS_CF_EVENT_FIFO_FULL"/>
+	<value value="30" name="ME_MICRO_RB_STARVED"/>
+	<value value="31" name="ME_MICRO_I1_STARVED"/>
+	<value value="32" name="ME_MICRO_I2_STARVED"/>
+	<value value="33" name="ME_MICRO_ST_STARVED"/>
+	<value value="40" name="RCIU_RBBM_DWORD_SENT"/>
+	<value value="41" name="ME_BUSY_CLOCKS"/>
+	<value value="42" name="ME_WAIT_CONTEXT_AVAIL"/>
+	<value value="43" name="PFP_TYPE0_PACKET"/>
+	<value value="44" name="PFP_TYPE3_PACKET"/>
+	<value value="45" name="CSF_RB_WPTR_NEQ_RPTR"/>
+	<value value="46" name="CSF_I1_SIZE_NEQ_ZERO"/>
+	<value value="47" name="CSF_I2_SIZE_NEQ_ZERO"/>
+	<value value="48" name="CSF_RBI1I2_FETCHING"/>
+</enum>
+
+<enum name="a2xx_rb_perfcnt_select">
+	<value value="0" name="RBPERF_CNTX_BUSY"/>
+	<value value="1" name="RBPERF_CNTX_BUSY_MAX"/>
+	<value value="2" name="RBPERF_SX_QUAD_STARVED"/>
+	<value value="3" name="RBPERF_SX_QUAD_STARVED_MAX"/>
+	<value value="4" name="RBPERF_GA_GC_CH0_SYS_REQ"/>
+	<value value="5" name="RBPERF_GA_GC_CH0_SYS_REQ_MAX"/>
+	<value value="6" name="RBPERF_GA_GC_CH1_SYS_REQ"/>
+	<value value="7" name="RBPERF_GA_GC_CH1_SYS_REQ_MAX"/>
+	<value value="8" name="RBPERF_MH_STARVED"/>
+	<value value="9" name="RBPERF_MH_STARVED_MAX"/>
+	<value value="10" name="RBPERF_AZ_BC_COLOR_BUSY"/>
+	<value value="11" name="RBPERF_AZ_BC_COLOR_BUSY_MAX"/>
+	<value value="12" name="RBPERF_AZ_BC_Z_BUSY"/>
+	<value value="13" name="RBPERF_AZ_BC_Z_BUSY_MAX"/>
+	<value value="14" name="RBPERF_RB_SC_TILE_RTR_N"/>
+	<value value="15" name="RBPERF_RB_SC_TILE_RTR_N_MAX"/>
+	<value value="16" name="RBPERF_RB_SC_SAMP_RTR_N"/>
+	<value value="17" name="RBPERF_RB_SC_SAMP_RTR_N_MAX"/>
+	<value value="18" name="RBPERF_RB_SX_QUAD_RTR_N"/>
+	<value value="19" name="RBPERF_RB_SX_QUAD_RTR_N_MAX"/>
+	<value value="20" name="RBPERF_RB_SX_COLOR_RTR_N"/>
+	<value value="21" name="RBPERF_RB_SX_COLOR_RTR_N_MAX"/>
+	<value value="22" name="RBPERF_RB_SC_SAMP_LZ_BUSY"/>
+	<value value="23" name="RBPERF_RB_SC_SAMP_LZ_BUSY_MAX"/>
+	<value value="24" name="RBPERF_ZXP_STALL"/>
+	<value value="25" name="RBPERF_ZXP_STALL_MAX"/>
+	<value value="26" name="RBPERF_EVENT_PENDING"/>
+	<value value="27" name="RBPERF_EVENT_PENDING_MAX"/>
+	<value value="28" name="RBPERF_RB_MH_VALID"/>
+	<value value="29" name="RBPERF_RB_MH_VALID_MAX"/>
+	<value value="30" name="RBPERF_SX_RB_QUAD_SEND"/>
+	<value value="31" name="RBPERF_SX_RB_COLOR_SEND"/>
+	<value value="32" name="RBPERF_SC_RB_TILE_SEND"/>
+	<value value="33" name="RBPERF_SC_RB_SAMPLE_SEND"/>
+	<value value="34" name="RBPERF_SX_RB_MEM_EXPORT"/>
+	<value value="35" name="RBPERF_SX_RB_QUAD_EVENT"/>
+	<value value="36" name="RBPERF_SC_RB_TILE_EVENT_FILTERED"/>
+	<value value="37" name="RBPERF_SC_RB_TILE_EVENT_ALL"/>
+	<value value="38" name="RBPERF_RB_SC_EZ_SEND"/>
+	<value value="39" name="RBPERF_RB_SX_INDEX_SEND"/>
+	<value value="40" name="RBPERF_GMEM_INTFO_RD"/>
+	<value value="41" name="RBPERF_GMEM_INTF1_RD"/>
+	<value value="42" name="RBPERF_GMEM_INTFO_WR"/>
+	<value value="43" name="RBPERF_GMEM_INTF1_WR"/>
+	<value value="44" name="RBPERF_RB_CP_CONTEXT_DONE"/>
+	<value value="45" name="RBPERF_RB_CP_CACHE_FLUSH"/>
+	<value value="46" name="RBPERF_ZPASS_DONE"/>
+	<value value="47" name="RBPERF_ZCMD_VALID"/>
+	<value value="48" name="RBPERF_CCMD_VALID"/>
+	<value value="49" name="RBPERF_ACCUM_GRANT"/>
+	<value value="50" name="RBPERF_ACCUM_C0_GRANT"/>
+	<value value="51" name="RBPERF_ACCUM_C1_GRANT"/>
+	<value value="52" name="RBPERF_ACCUM_FULL_BE_WR"/>
+	<value value="53" name="RBPERF_ACCUM_REQUEST_NO_GRANT"/>
+	<value value="54" name="RBPERF_ACCUM_TIMEOUT_PULSE"/>
+	<value value="55" name="RBPERF_ACCUM_LIN_TIMEOUT_PULSE"/>
+	<value value="56" name="RBPERF_ACCUM_CAM_HIT_FLUSHING"/>
+</enum>
+
+<enum name="a2xx_mh_perfcnt_select">
+	<value value="0" name="CP_R0_REQUESTS"/>
+	<value value="1" name="CP_R1_REQUESTS"/>
+	<value value="2" name="CP_R2_REQUESTS"/>
+	<value value="3" name="CP_R3_REQUESTS"/>
+	<value value="4" name="CP_R4_REQUESTS"/>
+	<value value="5" name="CP_TOTAL_READ_REQUESTS"/>
+	<value value="6" name="CP_TOTAL_WRITE_REQUESTS"/>
+	<value value="7" name="CP_TOTAL_REQUESTS"/>
+	<value value="8" name="CP_DATA_BYTES_WRITTEN"/>
+	<value value="9" name="CP_WRITE_CLEAN_RESPONSES"/>
+	<value value="10" name="CP_R0_READ_BURSTS_RECEIVED"/>
+	<value value="11" name="CP_R1_READ_BURSTS_RECEIVED"/>
+	<value value="12" name="CP_R2_READ_BURSTS_RECEIVED"/>
+	<value value="13" name="CP_R3_READ_BURSTS_RECEIVED"/>
+	<value value="14" name="CP_R4_READ_BURSTS_RECEIVED"/>
+	<value value="15" name="CP_TOTAL_READ_BURSTS_RECEIVED"/>
+	<value value="16" name="CP_R0_DATA_BEATS_READ"/>
+	<value value="17" name="CP_R1_DATA_BEATS_READ"/>
+	<value value="18" name="CP_R2_DATA_BEATS_READ"/>
+	<value value="19" name="CP_R3_DATA_BEATS_READ"/>
+	<value value="20" name="CP_R4_DATA_BEATS_READ"/>
+	<value value="21" name="CP_TOTAL_DATA_BEATS_READ"/>
+	<value value="22" name="VGT_R0_REQUESTS"/>
+	<value value="23" name="VGT_R1_REQUESTS"/>
+	<value value="24" name="VGT_TOTAL_REQUESTS"/>
+	<value value="25" name="VGT_R0_READ_BURSTS_RECEIVED"/>
+	<value value="26" name="VGT_R1_READ_BURSTS_RECEIVED"/>
+	<value value="27" name="VGT_TOTAL_READ_BURSTS_RECEIVED"/>
+	<value value="28" name="VGT_R0_DATA_BEATS_READ"/>
+	<value value="29" name="VGT_R1_DATA_BEATS_READ"/>
+	<value value="30" name="VGT_TOTAL_DATA_BEATS_READ"/>
+	<value value="31" name="TC_TOTAL_REQUESTS"/>
+	<value value="32" name="TC_ROQ_REQUESTS"/>
+	<value value="33" name="TC_INFO_SENT"/>
+	<value value="34" name="TC_READ_BURSTS_RECEIVED"/>
+	<value value="35" name="TC_DATA_BEATS_READ"/>
+	<value value="36" name="TCD_BURSTS_READ"/>
+	<value value="37" name="RB_REQUESTS"/>
+	<value value="38" name="RB_DATA_BYTES_WRITTEN"/>
+	<value value="39" name="RB_WRITE_CLEAN_RESPONSES"/>
+	<value value="40" name="AXI_READ_REQUESTS_ID_0"/>
+	<value value="41" name="AXI_READ_REQUESTS_ID_1"/>
+	<value value="42" name="AXI_READ_REQUESTS_ID_2"/>
+	<value value="43" name="AXI_READ_REQUESTS_ID_3"/>
+	<value value="44" name="AXI_READ_REQUESTS_ID_4"/>
+	<value value="45" name="AXI_READ_REQUESTS_ID_5"/>
+	<value value="46" name="AXI_READ_REQUESTS_ID_6"/>
+	<value value="47" name="AXI_READ_REQUESTS_ID_7"/>
+	<value value="48" name="AXI_TOTAL_READ_REQUESTS"/>
+	<value value="49" name="AXI_WRITE_REQUESTS_ID_0"/>
+	<value value="50" name="AXI_WRITE_REQUESTS_ID_1"/>
+	<value value="51" name="AXI_WRITE_REQUESTS_ID_2"/>
+	<value value="52" name="AXI_WRITE_REQUESTS_ID_3"/>
+	<value value="53" name="AXI_WRITE_REQUESTS_ID_4"/>
+	<value value="54" name="AXI_WRITE_REQUESTS_ID_5"/>
+	<value value="55" name="AXI_WRITE_REQUESTS_ID_6"/>
+	<value value="56" name="AXI_WRITE_REQUESTS_ID_7"/>
+	<value value="57" name="AXI_TOTAL_WRITE_REQUESTS"/>
+	<value value="58" name="AXI_TOTAL_REQUESTS_ID_0"/>
+	<value value="59" name="AXI_TOTAL_REQUESTS_ID_1"/>
+	<value value="60" name="AXI_TOTAL_REQUESTS_ID_2"/>
+	<value value="61" name="AXI_TOTAL_REQUESTS_ID_3"/>
+	<value value="62" name="AXI_TOTAL_REQUESTS_ID_4"/>
+	<value value="63" name="AXI_TOTAL_REQUESTS_ID_5"/>
+	<value value="64" name="AXI_TOTAL_REQUESTS_ID_6"/>
+	<value value="65" name="AXI_TOTAL_REQUESTS_ID_7"/>
+	<value value="66" name="AXI_TOTAL_REQUESTS"/>
+	<value value="67" name="AXI_READ_CHANNEL_BURSTS_ID_0"/>
+	<value value="68" name="AXI_READ_CHANNEL_BURSTS_ID_1"/>
+	<value value="69" name="AXI_READ_CHANNEL_BURSTS_ID_2"/>
+	<value value="70" name="AXI_READ_CHANNEL_BURSTS_ID_3"/>
+	<value value="71" name="AXI_READ_CHANNEL_BURSTS_ID_4"/>
+	<value value="72" name="AXI_READ_CHANNEL_BURSTS_ID_5"/>
+	<value value="73" name="AXI_READ_CHANNEL_BURSTS_ID_6"/>
+	<value value="74" name="AXI_READ_CHANNEL_BURSTS_ID_7"/>
+	<value value="75" name="AXI_READ_CHANNEL_TOTAL_BURSTS"/>
+	<value value="76" name="AXI_READ_CHANNEL_DATA_BEATS_READ_ID_0"/>
+	<value value="77" name="AXI_READ_CHANNEL_DATA_BEATS_READ_ID_1"/>
+	<value value="78" name="AXI_READ_CHANNEL_DATA_BEATS_READ_ID_2"/>
+	<value value="79" name="AXI_READ_CHANNEL_DATA_BEATS_READ_ID_3"/>
+	<value value="80" name="AXI_READ_CHANNEL_DATA_BEATS_READ_ID_4"/>
+	<value value="81" name="AXI_READ_CHANNEL_DATA_BEATS_READ_ID_5"/>
+	<value value="82" name="AXI_READ_CHANNEL_DATA_BEATS_READ_ID_6"/>
+	<value value="83" name="AXI_READ_CHANNEL_DATA_BEATS_READ_ID_7"/>
+	<value value="84" name="AXI_READ_CHANNEL_TOTAL_DATA_BEATS_READ"/>
+	<value value="85" name="AXI_WRITE_CHANNEL_BURSTS_ID_0"/>
+	<value value="86" name="AXI_WRITE_CHANNEL_BURSTS_ID_1"/>
+	<value value="87" name="AXI_WRITE_CHANNEL_BURSTS_ID_2"/>
+	<value value="88" name="AXI_WRITE_CHANNEL_BURSTS_ID_3"/>
+	<value value="89" name="AXI_WRITE_CHANNEL_BURSTS_ID_4"/>
+	<value value="90" name="AXI_WRITE_CHANNEL_BURSTS_ID_5"/>
+	<value value="91" name="AXI_WRITE_CHANNEL_BURSTS_ID_6"/>
+	<value value="92" name="AXI_WRITE_CHANNEL_BURSTS_ID_7"/>
+	<value value="93" name="AXI_WRITE_CHANNEL_TOTAL_BURSTS"/>
+	<value value="94" name="AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_0"/>
+	<value value="95" name="AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_1"/>
+	<value value="96" name="AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_2"/>
+	<value value="97" name="AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_3"/>
+	<value value="98" name="AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_4"/>
+	<value value="99" name="AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_5"/>
+	<value value="100" name="AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_6"/>
+	<value value="101" name="AXI_WRITE_CHANNEL_DATA_BYTES_WRITTEN_ID_7"/>
+	<value value="102" name="AXI_WRITE_CHANNEL_TOTAL_DATA_BYTES_WRITTEN"/>
+	<value value="103" name="AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_0"/>
+	<value value="104" name="AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_1"/>
+	<value value="105" name="AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_2"/>
+	<value value="106" name="AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_3"/>
+	<value value="107" name="AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_4"/>
+	<value value="108" name="AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_5"/>
+	<value value="109" name="AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_6"/>
+	<value value="110" name="AXI_WRITE_RESPONSE_CHANNEL_RESPONSES_ID_7"/>
+	<value value="111" name="AXI_WRITE_RESPONSE_CHANNEL_TOTAL_RESPONSES"/>
+	<value value="112" name="TOTAL_MMU_MISSES"/>
+	<value value="113" name="MMU_READ_MISSES"/>
+	<value value="114" name="MMU_WRITE_MISSES"/>
+	<value value="115" name="TOTAL_MMU_HITS"/>
+	<value value="116" name="MMU_READ_HITS"/>
+	<value value="117" name="MMU_WRITE_HITS"/>
+	<value value="118" name="SPLIT_MODE_TC_HITS"/>
+	<value value="119" name="SPLIT_MODE_TC_MISSES"/>
+	<value value="120" name="SPLIT_MODE_NON_TC_HITS"/>
+	<value value="121" name="SPLIT_MODE_NON_TC_MISSES"/>
+	<value value="122" name="STALL_AWAITING_TLB_MISS_FETCH"/>
+	<value value="123" name="MMU_TLB_MISS_READ_BURSTS_RECEIVED"/>
+	<value value="124" name="MMU_TLB_MISS_DATA_BEATS_READ"/>
+	<value value="125" name="CP_CYCLES_HELD_OFF"/>
+	<value value="126" name="VGT_CYCLES_HELD_OFF"/>
+	<value value="127" name="TC_CYCLES_HELD_OFF"/>
+	<value value="128" name="TC_ROQ_CYCLES_HELD_OFF"/>
+	<value value="129" name="TC_CYCLES_HELD_OFF_TCD_FULL"/>
+	<value value="130" name="RB_CYCLES_HELD_OFF"/>
+	<value value="131" name="TOTAL_CYCLES_ANY_CLNT_HELD_OFF"/>
+	<value value="132" name="TLB_MISS_CYCLES_HELD_OFF"/>
+	<value value="133" name="AXI_READ_REQUEST_HELD_OFF"/>
+	<value value="134" name="AXI_WRITE_REQUEST_HELD_OFF"/>
+	<value value="135" name="AXI_REQUEST_HELD_OFF"/>
+	<value value="136" name="AXI_REQUEST_HELD_OFF_INFLIGHT_LIMIT"/>
+	<value value="137" name="AXI_WRITE_DATA_HELD_OFF"/>
+	<value value="138" name="CP_SAME_PAGE_BANK_REQUESTS"/>
+	<value value="139" name="VGT_SAME_PAGE_BANK_REQUESTS"/>
+	<value value="140" name="TC_SAME_PAGE_BANK_REQUESTS"/>
+	<value value="141" name="TC_ARB_HOLD_SAME_PAGE_BANK_REQUESTS"/>
+	<value value="142" name="RB_SAME_PAGE_BANK_REQUESTS"/>
+	<value value="143" name="TOTAL_SAME_PAGE_BANK_REQUESTS"/>
+	<value value="144" name="CP_SAME_PAGE_BANK_REQUESTS_KILLED_FAIRNESS_LIMIT"/>
+	<value value="145" name="VGT_SAME_PAGE_BANK_REQUESTS_KILLED_FAIRNESS_LIMIT"/>
+	<value value="146" name="TC_SAME_PAGE_BANK_REQUESTS_KILLED_FAIRNESS_LIMIT"/>
+	<value value="147" name="RB_SAME_PAGE_BANK_REQUESTS_KILLED_FAIRNESS_LIMIT"/>
+	<value value="148" name="TOTAL_SAME_PAGE_BANK_KILLED_FAIRNESS_LIMIT"/>
+	<value value="149" name="TOTAL_MH_READ_REQUESTS"/>
+	<value value="150" name="TOTAL_MH_WRITE_REQUESTS"/>
+	<value value="151" name="TOTAL_MH_REQUESTS"/>
+	<value value="152" name="MH_BUSY"/>
+	<value value="153" name="CP_NTH_ACCESS_SAME_PAGE_BANK_SEQUENCE"/>
+	<value value="154" name="VGT_NTH_ACCESS_SAME_PAGE_BANK_SEQUENCE"/>
+	<value value="155" name="TC_NTH_ACCESS_SAME_PAGE_BANK_SEQUENCE"/>
+	<value value="156" name="RB_NTH_ACCESS_SAME_PAGE_BANK_SEQUENCE"/>
+	<value value="157" name="TC_ROQ_N_VALID_ENTRIES"/>
+	<value value="158" name="ARQ_N_ENTRIES"/>
+	<value value="159" name="WDB_N_ENTRIES"/>
+	<value value="160" name="MH_READ_LATENCY_OUTST_REQ_SUM"/>
+	<value value="161" name="MC_READ_LATENCY_OUTST_REQ_SUM"/>
+	<value value="162" name="MC_TOTAL_READ_REQUESTS"/>
+	<value value="163" name="ELAPSED_CYCLES_MH_GATED_CLK"/>
+	<value value="164" name="ELAPSED_CLK_CYCLES"/>
+	<value value="165" name="CP_W_16B_REQUESTS"/>
+	<value value="166" name="CP_W_32B_REQUESTS"/>
+	<value value="167" name="TC_16B_REQUESTS"/>
+	<value value="168" name="TC_32B_REQUESTS"/>
+	<value value="169" name="PA_REQUESTS"/>
+	<value value="170" name="PA_DATA_BYTES_WRITTEN"/>
+	<value value="171" name="PA_WRITE_CLEAN_RESPONSES"/>
+	<value value="172" name="PA_CYCLES_HELD_OFF"/>
+	<value value="173" name="AXI_READ_REQUEST_DATA_BEATS_ID_0"/>
+	<value value="174" name="AXI_READ_REQUEST_DATA_BEATS_ID_1"/>
+	<value value="175" name="AXI_READ_REQUEST_DATA_BEATS_ID_2"/>
+	<value value="176" name="AXI_READ_REQUEST_DATA_BEATS_ID_3"/>
+	<value value="177" name="AXI_READ_REQUEST_DATA_BEATS_ID_4"/>
+	<value value="178" name="AXI_READ_REQUEST_DATA_BEATS_ID_5"/>
+	<value value="179" name="AXI_READ_REQUEST_DATA_BEATS_ID_6"/>
+	<value value="180" name="AXI_READ_REQUEST_DATA_BEATS_ID_7"/>
+	<value value="181" name="AXI_TOTAL_READ_REQUEST_DATA_BEATS"/>
+</enum>
+
+<enum name="perf_mode_cnt">
+	<value name="PERF_STATE_RESET" value="0"/>
+	<value name="PERF_STATE_ENABLE" value="1"/>
+	<value name="PERF_STATE_FREEZE" value="2"/>
+</enum>
+
+<domain name="A2XX" width="32">
+
+	<bitset name="a2xx_vgt_current_bin_id_min_max" inline="yes">
+		<bitfield name="COLUMN" low="0" high="2" type="uint"/>
+		<bitfield name="ROW" low="3" high="5" type="uint"/>
+		<bitfield name="GUARD_BAND_MASK" low="6" high="8" type="uint"/>
+	</bitset>
+
+	<reg32 offset="0x0001" name="RBBM_PATCH_RELEASE"/>
+	<reg32 offset="0x003b" name="RBBM_CNTL"/>
+	<reg32 offset="0x003c" name="RBBM_SOFT_RESET"/>
+	<reg32 offset="0x00c0" name="CP_PFP_UCODE_ADDR"/>
+	<reg32 offset="0x00c1" name="CP_PFP_UCODE_DATA"/>
+
+	<enum name="adreno_mmu_clnt_beh">
+		<value name="BEH_NEVR" value="0"/>
+		<value name="BEH_TRAN_RNG" value="1"/>
+		<value name="BEH_TRAN_FLT" value="2"/>
+	</enum>
+
+	<!--
+		Note: these seem applicable only for a2xx devices with gpummu?  At
+		any rate, MH_MMU_CONFIG shows up in places in a3xx firmware where
+		it doesn't make sense, so I think offset 0x40 must be a different
+		register on a3xx.. so moving this back into A2XX domain:
+	 -->
+	<reg32 offset="0x0040" name="MH_MMU_CONFIG">
+		<bitfield name="MMU_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="SPLIT_MODE_ENABLE" pos="1" type="boolean"/>
+		<bitfield name="RB_W_CLNT_BEHAVIOR" low="4" high="5" type="adreno_mmu_clnt_beh"/>
+		<bitfield name="CP_W_CLNT_BEHAVIOR" low="6" high="7" type="adreno_mmu_clnt_beh"/>
+		<bitfield name="CP_R0_CLNT_BEHAVIOR" low="8" high="9" type="adreno_mmu_clnt_beh"/>
+		<bitfield name="CP_R1_CLNT_BEHAVIOR" low="10" high="11" type="adreno_mmu_clnt_beh"/>
+		<bitfield name="CP_R2_CLNT_BEHAVIOR" low="12" high="13" type="adreno_mmu_clnt_beh"/>
+		<bitfield name="CP_R3_CLNT_BEHAVIOR" low="14" high="15" type="adreno_mmu_clnt_beh"/>
+		<bitfield name="CP_R4_CLNT_BEHAVIOR" low="16" high="17" type="adreno_mmu_clnt_beh"/>
+		<bitfield name="VGT_R0_CLNT_BEHAVIOR" low="18" high="19" type="adreno_mmu_clnt_beh"/>
+		<bitfield name="VGT_R1_CLNT_BEHAVIOR" low="20" high="21" type="adreno_mmu_clnt_beh"/>
+		<bitfield name="TC_R_CLNT_BEHAVIOR" low="22" high="23" type="adreno_mmu_clnt_beh"/>
+		<bitfield name="PA_W_CLNT_BEHAVIOR" low="24" high="25" type="adreno_mmu_clnt_beh"/>
+	</reg32>
+	<reg32 offset="0x0041" name="MH_MMU_VA_RANGE">
+		<bitfield name="NUM_64KB_REGIONS" low="0" high="11" type="uint"/>
+		<bitfield name="VA_BASE" low="12" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0042" name="MH_MMU_PT_BASE"/>
+	<reg32 offset="0x0043" name="MH_MMU_PAGE_FAULT"/>
+	<reg32 offset="0x0044" name="MH_MMU_TRAN_ERROR"/>
+	<reg32 offset="0x0045" name="MH_MMU_INVALIDATE">
+		<bitfield name="INVALIDATE_ALL" pos="0" type="boolean"/>
+		<bitfield name="INVALIDATE_TC" pos="1" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0046" name="MH_MMU_MPU_BASE"/>
+	<reg32 offset="0x0047" name="MH_MMU_MPU_END"/>
+
+	<reg32 offset="0x0394" name="NQWAIT_UNTIL"/>
+	<reg32 offset="0x0395" name="RBBM_PERFCOUNTER0_SELECT"/>
+	<reg32 offset="0x0396" name="RBBM_PERFCOUNTER1_SELECT"/>
+	<reg32 offset="0x0397" name="RBBM_PERFCOUNTER0_LO"/>
+	<reg32 offset="0x0398" name="RBBM_PERFCOUNTER0_HI"/>
+	<reg32 offset="0x0399" name="RBBM_PERFCOUNTER1_LO"/>
+	<reg32 offset="0x039a" name="RBBM_PERFCOUNTER1_HI"/>
+	<reg32 offset="0x039b" name="RBBM_DEBUG"/>
+	<reg32 offset="0x039c" name="RBBM_PM_OVERRIDE1">
+		<bitfield name="RBBM_AHBCLK_PM_OVERRIDE" pos="0" type="boolean"/>
+		<bitfield name="SC_REG_SCLK_PM_OVERRIDE" pos="1" type="boolean"/>
+		<bitfield name="SC_SCLK_PM_OVERRIDE" pos="2" type="boolean"/>
+		<bitfield name="SP_TOP_SCLK_PM_OVERRIDE" pos="3" type="boolean"/>
+		<bitfield name="SP_V0_SCLK_PM_OVERRIDE" pos="4" type="boolean"/>
+		<bitfield name="SQ_REG_SCLK_PM_OVERRIDE" pos="5" type="boolean"/>
+		<bitfield name="SQ_REG_FIFOS_SCLK_PM_OVERRIDE" pos="6" type="boolean"/>
+		<bitfield name="SQ_CONST_MEM_SCLK_PM_OVERRIDE" pos="7" type="boolean"/>
+		<bitfield name="SQ_SQ_SCLK_PM_OVERRIDE" pos="8" type="boolean"/>
+		<bitfield name="SX_SCLK_PM_OVERRIDE" pos="9" type="boolean"/>
+		<bitfield name="SX_REG_SCLK_PM_OVERRIDE" pos="10" type="boolean"/>
+		<bitfield name="TCM_TCO_SCLK_PM_OVERRIDE" pos="11" type="boolean"/>
+		<bitfield name="TCM_TCM_SCLK_PM_OVERRIDE" pos="12" type="boolean"/>
+		<bitfield name="TCM_TCD_SCLK_PM_OVERRIDE" pos="13" type="boolean"/>
+		<bitfield name="TCM_REG_SCLK_PM_OVERRIDE" pos="14" type="boolean"/>
+		<bitfield name="TPC_TPC_SCLK_PM_OVERRIDE" pos="15" type="boolean"/>
+		<bitfield name="TPC_REG_SCLK_PM_OVERRIDE" pos="16" type="boolean"/>
+		<bitfield name="TCF_TCA_SCLK_PM_OVERRIDE" pos="17" type="boolean"/>
+		<bitfield name="TCF_TCB_SCLK_PM_OVERRIDE" pos="18" type="boolean"/>
+		<bitfield name="TCF_TCB_READ_SCLK_PM_OVERRIDE" pos="19" type="boolean"/>
+		<bitfield name="TP_TP_SCLK_PM_OVERRIDE" pos="20" type="boolean"/>
+		<bitfield name="TP_REG_SCLK_PM_OVERRIDE" pos="21" type="boolean"/>
+		<bitfield name="CP_G_SCLK_PM_OVERRIDE" pos="22" type="boolean"/>
+		<bitfield name="CP_REG_SCLK_PM_OVERRIDE" pos="23" type="boolean"/>
+		<bitfield name="CP_G_REG_SCLK_PM_OVERRIDE" pos="24" type="boolean"/>
+		<bitfield name="SPI_SCLK_PM_OVERRIDE" pos="25" type="boolean"/>
+		<bitfield name="RB_REG_SCLK_PM_OVERRIDE" pos="26" type="boolean"/>
+		<bitfield name="RB_SCLK_PM_OVERRIDE" pos="27" type="boolean"/>
+		<bitfield name="MH_MH_SCLK_PM_OVERRIDE" pos="28" type="boolean"/>
+		<bitfield name="MH_REG_SCLK_PM_OVERRIDE" pos="29" type="boolean"/>
+		<bitfield name="MH_MMU_SCLK_PM_OVERRIDE" pos="30" type="boolean"/>
+		<bitfield name="MH_TCROQ_SCLK_PM_OVERRIDE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x039d" name="RBBM_PM_OVERRIDE2">
+		<bitfield name="PA_REG_SCLK_PM_OVERRIDE" pos="0" type="boolean"/>
+		<bitfield name="PA_PA_SCLK_PM_OVERRIDE" pos="1" type="boolean"/>
+		<bitfield name="PA_AG_SCLK_PM_OVERRIDE" pos="2" type="boolean"/>
+		<bitfield name="VGT_REG_SCLK_PM_OVERRIDE" pos="3" type="boolean"/>
+		<bitfield name="VGT_FIFOS_SCLK_PM_OVERRIDE" pos="4" type="boolean"/>
+		<bitfield name="VGT_VGT_SCLK_PM_OVERRIDE" pos="5" type="boolean"/>
+		<bitfield name="DEBUG_PERF_SCLK_PM_OVERRIDE" pos="6" type="boolean"/>
+		<bitfield name="PERM_SCLK_PM_OVERRIDE" pos="7" type="boolean"/>
+		<bitfield name="GC_GA_GMEM0_PM_OVERRIDE" pos="8" type="boolean"/>
+		<bitfield name="GC_GA_GMEM1_PM_OVERRIDE" pos="9" type="boolean"/>
+		<bitfield name="GC_GA_GMEM2_PM_OVERRIDE" pos="10" type="boolean"/>
+		<bitfield name="GC_GA_GMEM3_PM_OVERRIDE" pos="11" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x03a0" name="RBBM_DEBUG_OUT"/>
+	<reg32 offset="0x03a1" name="RBBM_DEBUG_CNTL"/>
+	<reg32 offset="0x03b3" name="RBBM_READ_ERROR"/>
+	<reg32 offset="0x03b4" name="RBBM_INT_CNTL">
+		<bitfield name="RDERR_INT_MASK" pos="0" type="boolean"/>
+		<bitfield name="DISPLAY_UPDATE_INT_MASK" pos="1" type="boolean"/>
+		<bitfield name="GUI_IDLE_INT_MASK" pos="19" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x03b5" name="RBBM_INT_STATUS"/>
+	<reg32 offset="0x03b6" name="RBBM_INT_ACK"/>
+	<reg32 offset="0x03b7" name="MASTER_INT_SIGNAL">
+		<bitfield name="MH_INT_STAT" pos="5" type="boolean"/>
+		<bitfield name="SQ_INT_STAT" pos="26" type="boolean"/>
+		<bitfield name="CP_INT_STAT" pos="30" type="boolean"/>
+		<bitfield name="RBBM_INT_STAT" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x03f9" name="RBBM_PERIPHID1"/>
+	<reg32 offset="0x03fa" name="RBBM_PERIPHID2"/>
+	<reg32 offset="0x0444" name="CP_PERFMON_CNTL">
+		<!-- The width is uncertain -->
+		<bitfield name="PERF_MODE_CNT" low="0" high="2" type="perf_mode_cnt"/>
+	</reg32>
+	<reg32 offset="0x0445" name="CP_PERFCOUNTER_SELECT"/>
+	<reg32 offset="0x0446" name="CP_PERFCOUNTER_LO"/>
+	<reg32 offset="0x0447" name="CP_PERFCOUNTER_HI"/>
+	<reg32 offset="0x05d0" name="RBBM_STATUS">
+		<bitfield name="CMDFIFO_AVAIL" low="0" high="4" type="uint"/>
+		<bitfield name="TC_BUSY" pos="5" type="boolean"/>
+		<bitfield name="HIRQ_PENDING" pos="8" type="boolean"/>
+		<bitfield name="CPRQ_PENDING" pos="9" type="boolean"/>
+		<bitfield name="CFRQ_PENDING" pos="10" type="boolean"/>
+		<bitfield name="PFRQ_PENDING" pos="11" type="boolean"/>
+		<bitfield name="VGT_BUSY_NO_DMA" pos="12" type="boolean"/>
+		<bitfield name="RBBM_WU_BUSY" pos="14" type="boolean"/>
+		<bitfield name="CP_NRT_BUSY" pos="16" type="boolean"/>
+		<bitfield name="MH_BUSY" pos="18" type="boolean"/>
+		<bitfield name="MH_COHERENCY_BUSY" pos="19" type="boolean"/>
+		<bitfield name="SX_BUSY" pos="21" type="boolean"/>
+		<bitfield name="TPC_BUSY" pos="22" type="boolean"/>
+		<bitfield name="SC_CNTX_BUSY" pos="24" type="boolean"/>
+		<bitfield name="PA_BUSY" pos="25" type="boolean"/>
+		<bitfield name="VGT_BUSY" pos="26" type="boolean"/>
+		<bitfield name="SQ_CNTX17_BUSY" pos="27" type="boolean"/>
+		<bitfield name="SQ_CNTX0_BUSY" pos="28" type="boolean"/>
+		<bitfield name="RB_CNTX_BUSY" pos="30" type="boolean"/>
+		<bitfield name="GUI_ACTIVE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0a40" name="MH_ARBITER_CONFIG">
+		<bitfield name="SAME_PAGE_LIMIT" low="0" high="5" type="uint"/>
+		<bitfield name="SAME_PAGE_GRANULARITY" pos="6" type="boolean"/>
+		<bitfield name="L1_ARB_ENABLE" pos="7" type="boolean"/>
+		<bitfield name="L1_ARB_HOLD_ENABLE" pos="8" type="boolean"/>
+		<bitfield name="L2_ARB_CONTROL" pos="9" type="boolean"/>
+		<bitfield name="PAGE_SIZE" low="10" high="12" type="uint"/>
+		<bitfield name="TC_REORDER_ENABLE" pos="13" type="boolean"/>
+		<bitfield name="TC_ARB_HOLD_ENABLE" pos="14" type="boolean"/>
+		<bitfield name="IN_FLIGHT_LIMIT_ENABLE" pos="15" type="boolean"/>
+		<bitfield name="IN_FLIGHT_LIMIT" low="16" high="21" type="uint"/>
+		<bitfield name="CP_CLNT_ENABLE" pos="22" type="boolean"/>
+		<bitfield name="VGT_CLNT_ENABLE" pos="23" type="boolean"/>
+		<bitfield name="TC_CLNT_ENABLE" pos="24" type="boolean"/>
+		<bitfield name="RB_CLNT_ENABLE" pos="25" type="boolean"/>
+		<bitfield name="PA_CLNT_ENABLE" pos="26" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0a42" name="MH_INTERRUPT_MASK">
+		<bitfield name="AXI_READ_ERROR" pos="0" type="boolean"/>
+		<bitfield name="AXI_WRITE_ERROR" pos="1" type="boolean"/>
+		<bitfield name="MMU_PAGE_FAULT" pos="2" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0a43" name="MH_INTERRUPT_STATUS"/>
+	<reg32 offset="0x0a44" name="MH_INTERRUPT_CLEAR"/>
+	<reg32 offset="0x0a54" name="MH_CLNT_INTF_CTRL_CONFIG1"/>
+	<reg32 offset="0x0a55" name="MH_CLNT_INTF_CTRL_CONFIG2"/>
+	<reg32 offset="0x0c01" name="A220_VSC_BIN_SIZE">
+		<bitfield name="WIDTH" low="0" high="4" shr="5" type="uint"/>
+		<bitfield name="HEIGHT" low="5" high="9" shr="5" type="uint"/>
+	</reg32>
+	<array offset="0x0c06" name="VSC_PIPE" stride="3" length="8">
+		<reg32 offset="0x0" name="CONFIG"/>
+		<reg32 offset="0x1" name="DATA_ADDRESS"/>
+		<reg32 offset="0x2" name="DATA_LENGTH"/>
+	</array>
+	<reg32 offset="0x0c38" name="PC_DEBUG_CNTL"/>
+	<reg32 offset="0x0c39" name="PC_DEBUG_DATA"/>
+	<reg32 offset="0x0c44" name="PA_SC_VIZ_QUERY_STATUS"/>
+	<reg32 offset="0x0c80" name="GRAS_DEBUG_CNTL"/>
+	<reg32 offset="0x0c80" name="PA_SU_DEBUG_CNTL"/>
+	<reg32 offset="0x0c81" name="GRAS_DEBUG_DATA"/>
+	<reg32 offset="0x0c81" name="PA_SU_DEBUG_DATA"/>
+	<reg32 offset="0x0c86" name="PA_SU_FACE_DATA">
+		<bitfield name="BASE_ADDR" low="5" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0d00" name="SQ_GPR_MANAGEMENT">
+		<bitfield name="REG_DYNAMIC" pos="0" type="boolean"/>
+		<bitfield name="REG_SIZE_PIX" low="4" high="11" type="uint"/>
+		<bitfield name="REG_SIZE_VTX" low="12" high="19" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0d01" name="SQ_FLOW_CONTROL"/>
+	<reg32 offset="0x0d02" name="SQ_INST_STORE_MANAGMENT">
+		<bitfield name="INST_BASE_PIX" low="0" high="11" type="uint"/>
+		<bitfield name="INST_BASE_VTX" low="16" high="27" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0d05" name="SQ_DEBUG_MISC"/>
+	<reg32 offset="0x0d34" name="SQ_INT_CNTL"/>
+	<reg32 offset="0x0d35" name="SQ_INT_STATUS"/>
+	<reg32 offset="0x0d36" name="SQ_INT_ACK"/>
+	<reg32 offset="0x0dae" name="SQ_DEBUG_INPUT_FSM"/>
+	<reg32 offset="0x0daf" name="SQ_DEBUG_CONST_MGR_FSM"/>
+	<reg32 offset="0x0db0" name="SQ_DEBUG_TP_FSM"/>
+	<reg32 offset="0x0db1" name="SQ_DEBUG_FSM_ALU_0"/>
+	<reg32 offset="0x0db2" name="SQ_DEBUG_FSM_ALU_1"/>
+	<reg32 offset="0x0db3" name="SQ_DEBUG_EXP_ALLOC"/>
+	<reg32 offset="0x0db4" name="SQ_DEBUG_PTR_BUFF"/>
+	<reg32 offset="0x0db5" name="SQ_DEBUG_GPR_VTX"/>
+	<reg32 offset="0x0db6" name="SQ_DEBUG_GPR_PIX"/>
+	<reg32 offset="0x0db7" name="SQ_DEBUG_TB_STATUS_SEL"/>
+	<reg32 offset="0x0db8" name="SQ_DEBUG_VTX_TB_0"/>
+	<reg32 offset="0x0db9" name="SQ_DEBUG_VTX_TB_1"/>
+	<reg32 offset="0x0dba" name="SQ_DEBUG_VTX_TB_STATUS_REG"/>
+	<reg32 offset="0x0dbb" name="SQ_DEBUG_VTX_TB_STATE_MEM"/>
+	<reg32 offset="0x0dbc" name="SQ_DEBUG_PIX_TB_0"/>
+	<reg32 offset="0x0dbd" name="SQ_DEBUG_PIX_TB_STATUS_REG_0"/>
+	<reg32 offset="0x0dbe" name="SQ_DEBUG_PIX_TB_STATUS_REG_1"/>
+	<reg32 offset="0x0dbf" name="SQ_DEBUG_PIX_TB_STATUS_REG_2"/>
+	<reg32 offset="0x0dc0" name="SQ_DEBUG_PIX_TB_STATUS_REG_3"/>
+	<reg32 offset="0x0dc1" name="SQ_DEBUG_PIX_TB_STATE_MEM"/>
+	<reg32 offset="0x0e00" name="TC_CNTL_STATUS">
+		<bitfield name="L2_INVALIDATE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0e1e" name="TP0_CHICKEN"/>
+	<reg32 offset="0x0f01" name="RB_BC_CONTROL">
+		<bitfield name="ACCUM_LINEAR_MODE_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="ACCUM_TIMEOUT_SELECT" low="1" high="2" type="uint"/>
+		<bitfield name="DISABLE_EDRAM_CAM" pos="3" type="boolean"/>
+		<bitfield name="DISABLE_EZ_FAST_CONTEXT_SWITCH" pos="4" type="boolean"/>
+		<bitfield name="DISABLE_EZ_NULL_ZCMD_DROP" pos="5" type="boolean"/>
+		<bitfield name="DISABLE_LZ_NULL_ZCMD_DROP" pos="6" type="boolean"/>
+		<bitfield name="ENABLE_AZ_THROTTLE" pos="7" type="boolean"/>
+		<bitfield name="AZ_THROTTLE_COUNT" low="8" high="12" type="uint"/>
+		<bitfield name="ENABLE_CRC_UPDATE" pos="14" type="boolean"/>
+		<bitfield name="CRC_MODE" pos="15" type="boolean"/>
+		<bitfield name="DISABLE_SAMPLE_COUNTERS" pos="16" type="boolean"/>
+		<bitfield name="DISABLE_ACCUM" pos="17" type="boolean"/>
+		<bitfield name="ACCUM_ALLOC_MASK" low="18" high="21" type="uint"/>
+		<bitfield name="LINEAR_PERFORMANCE_ENABLE" pos="22" type="boolean"/>
+		<bitfield name="ACCUM_DATA_FIFO_LIMIT" low="23" high="26" type="uint"/>
+		<bitfield name="MEM_EXPORT_TIMEOUT_SELECT" low="27" high="28" type="uint"/>
+		<bitfield name="MEM_EXPORT_LINEAR_MODE_ENABLE" pos="29" type="boolean"/>
+		<bitfield name="CRC_SYSTEM" pos="30" type="boolean"/>
+		<bitfield name="RESERVED6" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0f02" name="RB_EDRAM_INFO"/>
+	<reg32 offset="0x0f26" name="RB_DEBUG_CNTL"/>
+	<reg32 offset="0x0f27" name="RB_DEBUG_DATA"/>
+	<reg32 offset="0x2000" name="RB_SURFACE_INFO">
+		<bitfield name="SURFACE_PITCH" low="0" high="13" type="uint"/>
+		<bitfield name="MSAA_SAMPLES" low="14" high="15" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2001" name="RB_COLOR_INFO">
+		<bitfield name="FORMAT" low="0" high="3" type="a2xx_colorformatx"/>
+		<bitfield name="ROUND_MODE" low="4" high="5" type="uint"/>
+		<bitfield name="LINEAR" pos="6" type="boolean"/>
+		<bitfield name="ENDIAN" low="7" high="8" type="uint"/>
+		<bitfield name="SWAP" low="9" high="10" type="uint"/>
+		<bitfield name="BASE" low="12" high="31" shr="12"/>
+	</reg32>
+	<reg32 offset="0x2002" name="RB_DEPTH_INFO">
+		<bitfield name="DEPTH_FORMAT" pos="0" type="adreno_rb_depth_format"/>
+		<bitfield name="DEPTH_BASE" low="12" high="31" type="uint" shr="12"/>
+	</reg32>
+	<reg32 offset="0x2005" name="A225_RB_COLOR_INFO3"/>
+	<reg32 offset="0x2006" name="COHER_DEST_BASE_0"/>
+	<reg32 offset="0x200e" name="PA_SC_SCREEN_SCISSOR_TL" type="adreno_reg_xy"/>
+	<reg32 offset="0x200f" name="PA_SC_SCREEN_SCISSOR_BR" type="adreno_reg_xy"/>
+	<reg32 offset="0x2080" name="PA_SC_WINDOW_OFFSET">
+		<bitfield name="X" low="0" high="14" type="int"/>
+		<bitfield name="Y" low="16" high="30" type="int"/>
+		<bitfield name="DISABLE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2081" name="PA_SC_WINDOW_SCISSOR_TL" type="adreno_reg_xy"/>
+	<reg32 offset="0x2082" name="PA_SC_WINDOW_SCISSOR_BR" type="adreno_reg_xy"/>
+	<reg32 offset="0x2010" name="UNKNOWN_2010"/>
+	<reg32 offset="0x2100" name="VGT_MAX_VTX_INDX"/>
+	<reg32 offset="0x2101" name="VGT_MIN_VTX_INDX"/>
+	<reg32 offset="0x2102" name="VGT_INDX_OFFSET"/>
+	<reg32 offset="0x2103" name="A225_PC_MULTI_PRIM_IB_RESET_INDX"/>
+	<reg32 offset="0x2104" name="RB_COLOR_MASK">
+		<bitfield name="WRITE_RED" pos="0" type="boolean"/>
+		<bitfield name="WRITE_GREEN" pos="1" type="boolean"/>
+		<bitfield name="WRITE_BLUE" pos="2" type="boolean"/>
+		<bitfield name="WRITE_ALPHA" pos="3" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2105" name="RB_BLEND_RED"/>
+	<reg32 offset="0x2106" name="RB_BLEND_GREEN"/>
+	<reg32 offset="0x2107" name="RB_BLEND_BLUE"/>
+	<reg32 offset="0x2108" name="RB_BLEND_ALPHA"/>
+	<reg32 offset="0x2109" name="RB_FOG_COLOR">
+		<bitfield name="FOG_RED" low="0" high="7" type="uint"/>
+		<bitfield name="FOG_GREEN" low="8" high="15" type="uint"/>
+		<bitfield name="FOG_BLUE" low="16" high="23" type="uint"/>
+	</reg32>
+	<reg32 offset="0x210c" name="RB_STENCILREFMASK_BF" type="adreno_rb_stencilrefmask"/>
+	<reg32 offset="0x210d" name="RB_STENCILREFMASK" type="adreno_rb_stencilrefmask"/>
+	<reg32 offset="0x210e" name="RB_ALPHA_REF"/>
+	<reg32 offset="0x210f" name="PA_CL_VPORT_XSCALE" type="float"/>
+	<reg32 offset="0x2110" name="PA_CL_VPORT_XOFFSET" type="float"/>
+	<reg32 offset="0x2111" name="PA_CL_VPORT_YSCALE" type="float"/>
+	<reg32 offset="0x2112" name="PA_CL_VPORT_YOFFSET" type="float"/>
+	<reg32 offset="0x2113" name="PA_CL_VPORT_ZSCALE" type="float"/>
+	<reg32 offset="0x2114" name="PA_CL_VPORT_ZOFFSET" type="float"/>
+	<reg32 offset="0x2180" name="SQ_PROGRAM_CNTL">
+		<doc>
+			note: only 0x3f worth of valid register values for VS_REGS and
+			PS_REGS, but high bit is set to indicate '0 registers used':
+		</doc>
+		<bitfield name="VS_REGS" low="0" high="7" type="uint"/>
+		<bitfield name="PS_REGS" low="8" high="15" type="uint"/>
+		<bitfield name="VS_RESOURCE" pos="16" type="boolean"/>
+		<bitfield name="PS_RESOURCE" pos="17" type="boolean"/>
+		<bitfield name="PARAM_GEN" pos="18" type="boolean"/>
+		<bitfield name="GEN_INDEX_PIX" pos="19" type="boolean"/>
+		<bitfield name="VS_EXPORT_COUNT" low="20" high="23" type="uint"/>
+		<bitfield name="VS_EXPORT_MODE" low="24" high="26" type="a2xx_sq_ps_vtx_mode"/>
+		<bitfield name="PS_EXPORT_MODE" low="27" high="30" type="uint"/>
+		<bitfield name="GEN_INDEX_VTX" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2181" name="SQ_CONTEXT_MISC">
+		<bitfield name="INST_PRED_OPTIMIZE" pos="0" type="boolean"/>
+		<bitfield name="SC_OUTPUT_SCREEN_XY" pos="1" type="boolean"/>
+		<bitfield name="SC_SAMPLE_CNTL" low="2" high="3" type="a2xx_sq_sample_cntl"/>
+		<bitfield name="PARAM_GEN_POS" low="8" high="15" type="uint"/>
+		<bitfield name="PERFCOUNTER_REF" pos="16" type="boolean"/>
+		<bitfield name="YEILD_OPTIMIZE" pos="17" type="boolean"/>
+		<bitfield name="TX_CACHE_SEL" pos="18" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2182" name="SQ_INTERPOLATOR_CNTL">
+		<bitfield name="PARAM_SHADE" low="0" high="15" type="uint"/>
+		<bitfield name="SAMPLING_PATTERN" low="16" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2183" name="SQ_WRAPPING_0">
+		<bitfield name="PARAM_WRAP_0" low="0" high="3" type="uint"/>
+		<bitfield name="PARAM_WRAP_1" low="4" high="7" type="uint"/>
+		<bitfield name="PARAM_WRAP_2" low="8" high="11" type="uint"/>
+		<bitfield name="PARAM_WRAP_3" low="12" high="15" type="uint"/>
+		<bitfield name="PARAM_WRAP_4" low="16" high="19" type="uint"/>
+		<bitfield name="PARAM_WRAP_5" low="20" high="23" type="uint"/>
+		<bitfield name="PARAM_WRAP_6" low="24" high="27" type="uint"/>
+		<bitfield name="PARAM_WRAP_7" low="28" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2184" name="SQ_WRAPPING_1">
+		<bitfield name="PARAM_WRAP_8" low="0" high="3" type="uint"/>
+		<bitfield name="PARAM_WRAP_9" low="4" high="7" type="uint"/>
+		<bitfield name="PARAM_WRAP_10" low="8" high="11" type="uint"/>
+		<bitfield name="PARAM_WRAP_11" low="12" high="15" type="uint"/>
+		<bitfield name="PARAM_WRAP_12" low="16" high="19" type="uint"/>
+		<bitfield name="PARAM_WRAP_13" low="20" high="23" type="uint"/>
+		<bitfield name="PARAM_WRAP_14" low="24" high="27" type="uint"/>
+		<bitfield name="PARAM_WRAP_15" low="28" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x21f6" name="SQ_PS_PROGRAM">
+		<bitfield name="BASE" low="0" high="11" type="uint"/>
+		<bitfield name="SIZE" low="12" high="23" type="uint"/>
+	</reg32>
+	<reg32 offset="0x21f7" name="SQ_VS_PROGRAM">
+		<bitfield name="BASE" low="0" high="11" type="uint"/>
+		<bitfield name="SIZE" low="12" high="23" type="uint"/>
+	</reg32>
+	<reg32 offset="0x21f9" name="VGT_EVENT_INITIATOR"/>
+	<reg32 offset="0x21fc" name="VGT_DRAW_INITIATOR" type="vgt_draw_initiator"/>
+	<reg32 offset="0x21fd" name="VGT_IMMED_DATA"/>
+	<reg32 offset="0x2200" name="RB_DEPTHCONTROL">
+		<bitfield name="STENCIL_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="Z_ENABLE" pos="1" type="boolean"/>
+		<bitfield name="Z_WRITE_ENABLE" pos="2" type="boolean"/>
+		<bitfield name="EARLY_Z_ENABLE" pos="3" type="boolean"/>
+		<bitfield name="ZFUNC" low="4" high="6" type="adreno_compare_func"/>
+		<bitfield name="BACKFACE_ENABLE" pos="7" type="boolean"/>
+		<bitfield name="STENCILFUNC" low="8" high="10" type="adreno_compare_func"/>
+		<bitfield name="STENCILFAIL" low="11" high="13" type="adreno_stencil_op"/>
+		<bitfield name="STENCILZPASS" low="14" high="16" type="adreno_stencil_op"/>
+		<bitfield name="STENCILZFAIL" low="17" high="19" type="adreno_stencil_op"/>
+		<bitfield name="STENCILFUNC_BF" low="20" high="22" type="adreno_compare_func"/>
+		<bitfield name="STENCILFAIL_BF" low="23" high="25" type="adreno_stencil_op"/>
+		<bitfield name="STENCILZPASS_BF" low="26" high="28" type="adreno_stencil_op"/>
+		<bitfield name="STENCILZFAIL_BF" low="29" high="31" type="adreno_stencil_op"/>
+	</reg32>
+	<reg32 offset="0x2201" name="RB_BLEND_CONTROL">
+		<bitfield name="COLOR_SRCBLEND" low="0" high="4" type="adreno_rb_blend_factor"/>
+		<bitfield name="COLOR_COMB_FCN" low="5" high="7" type="a2xx_rb_blend_opcode"/>
+		<bitfield name="COLOR_DESTBLEND" low="8" high="12" type="adreno_rb_blend_factor"/>
+		<bitfield name="ALPHA_SRCBLEND" low="16" high="20" type="adreno_rb_blend_factor"/>
+		<bitfield name="ALPHA_COMB_FCN" low="21" high="23" type="a2xx_rb_blend_opcode"/>
+		<bitfield name="ALPHA_DESTBLEND" low="24" high="28" type="adreno_rb_blend_factor"/>
+		<bitfield name="BLEND_FORCE_ENABLE" pos="29" type="boolean"/>
+		<bitfield name="BLEND_FORCE" pos="30" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2202" name="RB_COLORCONTROL">
+		<bitfield name="ALPHA_FUNC" low="0" high="2" type="adreno_compare_func"/>
+		<bitfield name="ALPHA_TEST_ENABLE" pos="3" type="boolean"/>
+		<bitfield name="ALPHA_TO_MASK_ENABLE" pos="4" type="boolean"/>
+		<bitfield name="BLEND_DISABLE" pos="5" type="boolean"/>
+		<bitfield name="VOB_ENABLE" pos="6" type="boolean"/>
+		<bitfield name="VS_EXPORTS_FOG" pos="7" type="boolean"/>
+		<bitfield name="ROP_CODE" low="8" high="11" type="uint"/>
+		<bitfield name="DITHER_MODE" low="12" high="13" type="adreno_rb_dither_mode"/>
+		<bitfield name="DITHER_TYPE" low="14" high="15" type="a2xx_rb_dither_type"/>
+		<bitfield name="PIXEL_FOG" pos="16" type="boolean"/>
+		<bitfield name="ALPHA_TO_MASK_OFFSET0" low="24" high="25" type="uint"/>
+		<bitfield name="ALPHA_TO_MASK_OFFSET1" low="26" high="27" type="uint"/>
+		<bitfield name="ALPHA_TO_MASK_OFFSET2" low="28" high="29" type="uint"/>
+		<bitfield name="ALPHA_TO_MASK_OFFSET3" low="30" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2203" name="VGT_CURRENT_BIN_ID_MAX" type="a2xx_vgt_current_bin_id_min_max"/>
+	<reg32 offset="0x2204" name="PA_CL_CLIP_CNTL">
+		<bitfield name="CLIP_DISABLE" pos="16" type="boolean"/>
+		<bitfield name="BOUNDARY_EDGE_FLAG_ENA" pos="18" type="boolean"/>
+		<bitfield name="DX_CLIP_SPACE_DEF" pos="19" type="a2xx_dx_clip_space"/>
+		<bitfield name="DIS_CLIP_ERR_DETECT" pos="20" type="boolean"/>
+		<bitfield name="VTX_KILL_OR" pos="21" type="boolean"/>
+		<bitfield name="XY_NAN_RETAIN" pos="22" type="boolean"/>
+		<bitfield name="Z_NAN_RETAIN" pos="23" type="boolean"/>
+		<bitfield name="W_NAN_RETAIN" pos="24" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2205" name="PA_SU_SC_MODE_CNTL">
+		<bitfield name="CULL_FRONT" pos="0" type="boolean"/>
+		<bitfield name="CULL_BACK" pos="1" type="boolean"/>
+		<bitfield name="FACE" pos="2" type="boolean"/>
+		<bitfield name="POLYMODE" low="3" high="4" type="a2xx_pa_su_sc_polymode"/>
+		<bitfield name="FRONT_PTYPE" low="5" high="7" type="adreno_pa_su_sc_draw"/>
+		<bitfield name="BACK_PTYPE" low="8" high="10" type="adreno_pa_su_sc_draw"/>
+		<bitfield name="POLY_OFFSET_FRONT_ENABLE" pos="11" type="boolean"/>
+		<bitfield name="POLY_OFFSET_BACK_ENABLE" pos="12" type="boolean"/>
+		<bitfield name="POLY_OFFSET_PARA_ENABLE" pos="13" type="boolean"/>
+		<bitfield name="MSAA_ENABLE" pos="15" type="boolean"/>
+		<bitfield name="VTX_WINDOW_OFFSET_ENABLE" pos="16" type="boolean"/>
+		<bitfield name="LINE_STIPPLE_ENABLE" pos="18" type="boolean"/>
+		<bitfield name="PROVOKING_VTX_LAST" pos="19" type="boolean"/>
+		<bitfield name="PERSP_CORR_DIS" pos="20" type="boolean"/>
+		<bitfield name="MULTI_PRIM_IB_ENA" pos="21" type="boolean"/>
+		<bitfield name="QUAD_ORDER_ENABLE" pos="23" type="boolean"/>
+		<bitfield name="WAIT_RB_IDLE_ALL_TRI" pos="25" type="boolean"/>
+		<bitfield name="WAIT_RB_IDLE_FIRST_TRI_NEW_STATE" pos="26" type="boolean"/>
+		<bitfield name="CLAMPED_FACENESS" pos="28" type="boolean"/>
+		<bitfield name="ZERO_AREA_FACENESS" pos="29" type="boolean"/>
+		<bitfield name="FACE_KILL_ENABLE" pos="30" type="boolean"/>
+		<bitfield name="FACE_WRITE_ENABLE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2206" name="PA_CL_VTE_CNTL">
+		<bitfield name="VPORT_X_SCALE_ENA" pos="0" type="boolean"/>
+		<bitfield name="VPORT_X_OFFSET_ENA" pos="1" type="boolean"/>
+		<bitfield name="VPORT_Y_SCALE_ENA" pos="2" type="boolean"/>
+		<bitfield name="VPORT_Y_OFFSET_ENA" pos="3" type="boolean"/>
+		<bitfield name="VPORT_Z_SCALE_ENA" pos="4" type="boolean"/>
+		<bitfield name="VPORT_Z_OFFSET_ENA" pos="5" type="boolean"/>
+		<bitfield name="VTX_XY_FMT" pos="8" type="boolean"/>
+		<bitfield name="VTX_Z_FMT" pos="9" type="boolean"/>
+		<bitfield name="VTX_W0_FMT" pos="10" type="boolean"/>
+		<bitfield name="PERFCOUNTER_REF" pos="11" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2207" name="VGT_CURRENT_BIN_ID_MIN" type="a2xx_vgt_current_bin_id_min_max"/>
+	<reg32 offset="0x2208" name="RB_MODECONTROL">
+		<bitfield name="EDRAM_MODE" low="0" high="2" type="a2xx_rb_edram_mode"/>
+	</reg32>
+	<reg32 offset="0x2209" name="A220_RB_LRZ_VSC_CONTROL"/>
+	<reg32 offset="0x220a" name="RB_SAMPLE_POS"/>
+	<reg32 offset="0x220b" name="CLEAR_COLOR">
+		<bitfield name="RED" low="0" high="7"/>
+		<bitfield name="GREEN" low="8" high="15"/>
+		<bitfield name="BLUE" low="16" high="23"/>
+		<bitfield name="ALPHA" low="24" high="31"/>
+	</reg32>
+	<reg32 offset="0x2210" name="A220_GRAS_CONTROL"/>
+	<reg32 offset="0x2280" name="PA_SU_POINT_SIZE">
+		<bitfield name="HEIGHT" low="0" high="15" type="ufixed" radix="4"/>
+		<bitfield name="WIDTH" low="16" high="31" type="ufixed" radix="4"/>
+	</reg32>
+	<reg32 offset="0x2281" name="PA_SU_POINT_MINMAX">
+		<bitfield name="MIN" low="0" high="15" type="ufixed" radix="4"/>
+		<bitfield name="MAX" low="16" high="31" type="ufixed" radix="4"/>
+	</reg32>
+	<reg32 offset="0x2282" name="PA_SU_LINE_CNTL">
+		<bitfield name="WIDTH" low="0" high="15" type="ufixed" radix="4"/>
+	</reg32>
+	<reg32 offset="0x2283" name="PA_SC_LINE_STIPPLE">
+		<bitfield name="LINE_PATTERN" low="0" high="15" type="hex"/>
+		<bitfield name="REPEAT_COUNT" low="16" high="23" type="uint"/>
+		<bitfield name="PATTERN_BIT_ORDER" pos="28" type="a2xx_pa_sc_pattern_bit_order"/>
+		<bitfield name="AUTO_RESET_CNTL" low="29" high="30" type="a2xx_pa_sc_auto_reset_cntl"/>
+	</reg32>
+	<reg32 offset="0x2293" name="PA_SC_VIZ_QUERY">
+		<bitfield name="VIZ_QUERY_ENA" pos="0" type="boolean"/>
+		<bitfield name="VIZ_QUERY_ID" low="1" high="6" type="uint"/>
+		<bitfield name="KILL_PIX_POST_EARLY_Z" pos="8" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2294" name="VGT_ENHANCE"/>
+	<reg32 offset="0x2300" name="PA_SC_LINE_CNTL">
+		<bitfield name="BRES_CNTL" low="0" high="15" type="uint"/>
+		<bitfield name="USE_BRES_CNTL" pos="8" type="boolean"/>
+		<bitfield name="EXPAND_LINE_WIDTH" pos="9" type="boolean"/>
+		<bitfield name="LAST_PIXEL" pos="10" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2301" name="PA_SC_AA_CONFIG">
+		<bitfield name="MSAA_NUM_SAMPLES" low="0" high="2" type="uint"/>
+		<bitfield name="MAX_SAMPLE_DIST" low="13" high="16" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2302" name="PA_SU_VTX_CNTL">
+		<bitfield name="PIX_CENTER" pos="0" type="a2xx_pa_pixcenter"/>
+		<bitfield name="ROUND_MODE" low="1" high="2" type="a2xx_pa_roundmode"/>
+		<bitfield name="QUANT_MODE" low="7" high="9" type="a2xx_pa_quantmode"/>
+	</reg32>
+	<reg32 offset="0x2303" name="PA_CL_GB_VERT_CLIP_ADJ" type="float"/>
+	<reg32 offset="0x2304" name="PA_CL_GB_VERT_DISC_ADJ" type="float"/>
+	<reg32 offset="0x2305" name="PA_CL_GB_HORZ_CLIP_ADJ" type="float"/>
+	<reg32 offset="0x2306" name="PA_CL_GB_HORZ_DISC_ADJ" type="float"/>
+	<reg32 offset="0x2307" name="SQ_VS_CONST">
+		<bitfield name="BASE" low="0" high="8" type="uint"/>
+		<bitfield name="SIZE" low="12" high="20" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2308" name="SQ_PS_CONST">
+		<bitfield name="BASE" low="0" high="8" type="uint"/>
+		<bitfield name="SIZE" low="12" high="20" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2309" name="SQ_DEBUG_MISC_0"/>
+	<reg32 offset="0x230a" name="SQ_DEBUG_MISC_1"/>
+	<reg32 offset="0x2312" name="PA_SC_AA_MASK"/>
+	<reg32 offset="0x2316" name="VGT_VERTEX_REUSE_BLOCK_CNTL">
+		<bitfield name="VTX_REUSE_DEPTH" low="0" high="2" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2317" name="VGT_OUT_DEALLOC_CNTL">
+		<bitfield name="DEALLOC_DIST" low="0" high="1" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2318" name="RB_COPY_CONTROL">
+		<bitfield name="COPY_SAMPLE_SELECT" low="0" high="2" type="a2xx_rb_copy_sample_select"/>
+		<bitfield name="DEPTH_CLEAR_ENABLE" pos="3" type="boolean"/>
+		<bitfield name="CLEAR_MASK" low="4" high="7" type="hex"/>
+	</reg32>
+	<reg32 offset="0x2319" name="RB_COPY_DEST_BASE"/>
+	<reg32 offset="0x231a" name="RB_COPY_DEST_PITCH" shr="5" type="uint"/>
+	<reg32 offset="0x231b" name="RB_COPY_DEST_INFO">
+		<bitfield name="DEST_ENDIAN" low="0" high="2" type="adreno_rb_surface_endian"/>
+		<bitfield name="LINEAR" pos="3" type="boolean"/>
+		<bitfield name="FORMAT" low="4" high="7" type="a2xx_colorformatx"/>
+		<bitfield name="SWAP" low="8" high="9" type="uint"/>
+		<bitfield name="DITHER_MODE" low="10" high="11" type="adreno_rb_dither_mode"/>
+		<bitfield name="DITHER_TYPE" low="12" high="13" type="a2xx_rb_dither_type"/>
+		<bitfield name="WRITE_RED" pos="14" type="boolean"/>
+		<bitfield name="WRITE_GREEN" pos="15" type="boolean"/>
+		<bitfield name="WRITE_BLUE" pos="16" type="boolean"/>
+		<bitfield name="WRITE_ALPHA" pos="17" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x231c" name="RB_COPY_DEST_OFFSET">
+		<bitfield name="X" low="0" high="12" type="uint"/>
+		<bitfield name="Y" low="13" high="25" type="uint"/>
+	</reg32>
+	<reg32 offset="0x231d" name="RB_DEPTH_CLEAR"/>
+	<reg32 offset="0x2324" name="RB_SAMPLE_COUNT_CTL"/>
+	<reg32 offset="0x2326" name="RB_COLOR_DEST_MASK"/>
+	<reg32 offset="0x2340" name="A225_GRAS_UCP0X"/>
+	<reg32 offset="0x2357" name="A225_GRAS_UCP5W"/>
+	<reg32 offset="0x2360" name="A225_GRAS_UCP_ENABLED"/>
+	<reg32 offset="0x2380" name="PA_SU_POLY_OFFSET_FRONT_SCALE"/>
+	<reg32 offset="0x2381" name="PA_SU_POLY_OFFSET_FRONT_OFFSET"/>
+	<reg32 offset="0x2382" name="PA_SU_POLY_OFFSET_BACK_SCALE"/>
+	<reg32 offset="0x2383" name="PA_SU_POLY_OFFSET_BACK_OFFSET"/>
+	<reg32 offset="0x4000" name="SQ_CONSTANT_0"/>
+	<reg32 offset="0x4800" name="SQ_FETCH_0"/>
+	<reg32 offset="0x4900" name="SQ_CF_BOOLEANS"/>
+	<reg32 offset="0x4908" name="SQ_CF_LOOP"/>
+	<reg32 offset="0xa29" name="COHER_SIZE_PM4"/>
+	<reg32 offset="0xa2a" name="COHER_BASE_PM4"/>
+	<reg32 offset="0xa2b" name="COHER_STATUS_PM4"/>
+
+	<reg32 offset="0x0c88" name="PA_SU_PERFCOUNTER0_SELECT"/>
+	<reg32 offset="0x0c89" name="PA_SU_PERFCOUNTER1_SELECT"/>
+	<reg32 offset="0x0c8a" name="PA_SU_PERFCOUNTER2_SELECT"/>
+	<reg32 offset="0x0c8b" name="PA_SU_PERFCOUNTER3_SELECT"/>
+	<reg32 offset="0x0c8c" name="PA_SU_PERFCOUNTER0_LOW"/>
+	<reg32 offset="0x0c8d" name="PA_SU_PERFCOUNTER0_HI"/>
+	<reg32 offset="0x0c8e" name="PA_SU_PERFCOUNTER1_LOW"/>
+	<reg32 offset="0x0c8f" name="PA_SU_PERFCOUNTER1_HI"/>
+	<reg32 offset="0x0c90" name="PA_SU_PERFCOUNTER2_LOW"/>
+	<reg32 offset="0x0c91" name="PA_SU_PERFCOUNTER2_HI"/>
+	<reg32 offset="0x0c92" name="PA_SU_PERFCOUNTER3_LOW"/>
+	<reg32 offset="0x0c93" name="PA_SU_PERFCOUNTER3_HI"/>
+	<reg32 offset="0x0c98" name="PA_SC_PERFCOUNTER0_SELECT"/>
+	<reg32 offset="0x0c99" name="PA_SC_PERFCOUNTER0_LOW"/>
+	<reg32 offset="0x0c9a" name="PA_SC_PERFCOUNTER0_HI"/>
+	<reg32 offset="0x0c48" name="VGT_PERFCOUNTER0_SELECT"/>
+	<reg32 offset="0x0c49" name="VGT_PERFCOUNTER1_SELECT"/>
+	<reg32 offset="0x0c4a" name="VGT_PERFCOUNTER2_SELECT"/>
+	<reg32 offset="0x0c4b" name="VGT_PERFCOUNTER3_SELECT"/>
+	<reg32 offset="0x0c4c" name="VGT_PERFCOUNTER0_LOW"/>
+	<reg32 offset="0x0c4e" name="VGT_PERFCOUNTER1_LOW"/>
+	<reg32 offset="0x0c50" name="VGT_PERFCOUNTER2_LOW"/>
+	<reg32 offset="0x0c52" name="VGT_PERFCOUNTER3_LOW"/>
+	<reg32 offset="0x0c4d" name="VGT_PERFCOUNTER0_HI"/>
+	<reg32 offset="0x0c4f" name="VGT_PERFCOUNTER1_HI"/>
+	<reg32 offset="0x0c51" name="VGT_PERFCOUNTER2_HI"/>
+	<reg32 offset="0x0c53" name="VGT_PERFCOUNTER3_HI"/>
+	<reg32 offset="0x0e05" name="TCR_PERFCOUNTER0_SELECT"/>
+	<reg32 offset="0x0e08" name="TCR_PERFCOUNTER1_SELECT"/>
+	<reg32 offset="0x0e06" name="TCR_PERFCOUNTER0_HI"/>
+	<reg32 offset="0x0e09" name="TCR_PERFCOUNTER1_HI"/>
+	<reg32 offset="0x0e07" name="TCR_PERFCOUNTER0_LOW"/>
+	<reg32 offset="0x0e0a" name="TCR_PERFCOUNTER1_LOW"/>
+	<reg32 offset="0x0e1f" name="TP0_PERFCOUNTER0_SELECT"/>
+	<reg32 offset="0x0e20" name="TP0_PERFCOUNTER0_HI"/>
+	<reg32 offset="0x0e21" name="TP0_PERFCOUNTER0_LOW"/>
+	<reg32 offset="0x0e22" name="TP0_PERFCOUNTER1_SELECT"/>
+	<reg32 offset="0x0e23" name="TP0_PERFCOUNTER1_HI"/>
+	<reg32 offset="0x0e24" name="TP0_PERFCOUNTER1_LOW"/>
+	<reg32 offset="0x0e54" name="TCM_PERFCOUNTER0_SELECT"/>
+	<reg32 offset="0x0e57" name="TCM_PERFCOUNTER1_SELECT"/>
+	<reg32 offset="0x0e55" name="TCM_PERFCOUNTER0_HI"/>
+	<reg32 offset="0x0e58" name="TCM_PERFCOUNTER1_HI"/>
+	<reg32 offset="0x0e56" name="TCM_PERFCOUNTER0_LOW"/>
+	<reg32 offset="0x0e59" name="TCM_PERFCOUNTER1_LOW"/>
+	<reg32 offset="0x0e5a" name="TCF_PERFCOUNTER0_SELECT"/>
+	<reg32 offset="0x0e5d" name="TCF_PERFCOUNTER1_SELECT"/>
+	<reg32 offset="0x0e60" name="TCF_PERFCOUNTER2_SELECT"/>
+	<reg32 offset="0x0e63" name="TCF_PERFCOUNTER3_SELECT"/>
+	<reg32 offset="0x0e66" name="TCF_PERFCOUNTER4_SELECT"/>
+	<reg32 offset="0x0e69" name="TCF_PERFCOUNTER5_SELECT"/>
+	<reg32 offset="0x0e6c" name="TCF_PERFCOUNTER6_SELECT"/>
+	<reg32 offset="0x0e6f" name="TCF_PERFCOUNTER7_SELECT"/>
+	<reg32 offset="0x0e72" name="TCF_PERFCOUNTER8_SELECT"/>
+	<reg32 offset="0x0e75" name="TCF_PERFCOUNTER9_SELECT"/>
+	<reg32 offset="0x0e78" name="TCF_PERFCOUNTER10_SELECT"/>
+	<reg32 offset="0x0e7b" name="TCF_PERFCOUNTER11_SELECT"/>
+	<reg32 offset="0x0e5b" name="TCF_PERFCOUNTER0_HI"/>
+	<reg32 offset="0x0e5e" name="TCF_PERFCOUNTER1_HI"/>
+	<reg32 offset="0x0e61" name="TCF_PERFCOUNTER2_HI"/>
+	<reg32 offset="0x0e64" name="TCF_PERFCOUNTER3_HI"/>
+	<reg32 offset="0x0e67" name="TCF_PERFCOUNTER4_HI"/>
+	<reg32 offset="0x0e6a" name="TCF_PERFCOUNTER5_HI"/>
+	<reg32 offset="0x0e6d" name="TCF_PERFCOUNTER6_HI"/>
+	<reg32 offset="0x0e70" name="TCF_PERFCOUNTER7_HI"/>
+	<reg32 offset="0x0e73" name="TCF_PERFCOUNTER8_HI"/>
+	<reg32 offset="0x0e76" name="TCF_PERFCOUNTER9_HI"/>
+	<reg32 offset="0x0e79" name="TCF_PERFCOUNTER10_HI"/>
+	<reg32 offset="0x0e7c" name="TCF_PERFCOUNTER11_HI"/>
+	<reg32 offset="0x0e5c" name="TCF_PERFCOUNTER0_LOW"/>
+	<reg32 offset="0x0e5f" name="TCF_PERFCOUNTER1_LOW"/>
+	<reg32 offset="0x0e62" name="TCF_PERFCOUNTER2_LOW"/>
+	<reg32 offset="0x0e65" name="TCF_PERFCOUNTER3_LOW"/>
+	<reg32 offset="0x0e68" name="TCF_PERFCOUNTER4_LOW"/>
+	<reg32 offset="0x0e6b" name="TCF_PERFCOUNTER5_LOW"/>
+	<reg32 offset="0x0e6e" name="TCF_PERFCOUNTER6_LOW"/>
+	<reg32 offset="0x0e71" name="TCF_PERFCOUNTER7_LOW"/>
+	<reg32 offset="0x0e74" name="TCF_PERFCOUNTER8_LOW"/>
+	<reg32 offset="0x0e77" name="TCF_PERFCOUNTER9_LOW"/>
+	<reg32 offset="0x0e7a" name="TCF_PERFCOUNTER10_LOW"/>
+	<reg32 offset="0x0e7d" name="TCF_PERFCOUNTER11_LOW"/>
+	<reg32 offset="0x0dc8" name="SQ_PERFCOUNTER0_SELECT"/>
+	<reg32 offset="0x0dc9" name="SQ_PERFCOUNTER1_SELECT"/>
+	<reg32 offset="0x0dca" name="SQ_PERFCOUNTER2_SELECT"/>
+	<reg32 offset="0x0dcb" name="SQ_PERFCOUNTER3_SELECT"/>
+	<reg32 offset="0x0dcc" name="SQ_PERFCOUNTER0_LOW"/>
+	<reg32 offset="0x0dcd" name="SQ_PERFCOUNTER0_HI"/>
+	<reg32 offset="0x0dce" name="SQ_PERFCOUNTER1_LOW"/>
+	<reg32 offset="0x0dcf" name="SQ_PERFCOUNTER1_HI"/>
+	<reg32 offset="0x0dd0" name="SQ_PERFCOUNTER2_LOW"/>
+	<reg32 offset="0x0dd1" name="SQ_PERFCOUNTER2_HI"/>
+	<reg32 offset="0x0dd2" name="SQ_PERFCOUNTER3_LOW"/>
+	<reg32 offset="0x0dd3" name="SQ_PERFCOUNTER3_HI"/>
+	<reg32 offset="0x0dd4" name="SX_PERFCOUNTER0_SELECT"/>
+	<reg32 offset="0x0dd8" name="SX_PERFCOUNTER0_LOW"/>
+	<reg32 offset="0x0dd9" name="SX_PERFCOUNTER0_HI"/>
+	<reg32 offset="0x0a46" name="MH_PERFCOUNTER0_SELECT"/>
+	<reg32 offset="0x0a4a" name="MH_PERFCOUNTER1_SELECT"/>
+	<reg32 offset="0x0a47" name="MH_PERFCOUNTER0_CONFIG"/>
+	<reg32 offset="0x0a4b" name="MH_PERFCOUNTER1_CONFIG"/>
+	<reg32 offset="0x0a48" name="MH_PERFCOUNTER0_LOW"/>
+	<reg32 offset="0x0a4c" name="MH_PERFCOUNTER1_LOW"/>
+	<reg32 offset="0x0a49" name="MH_PERFCOUNTER0_HI"/>
+	<reg32 offset="0x0a4d" name="MH_PERFCOUNTER1_HI"/>
+	<reg32 offset="0x0f04" name="RB_PERFCOUNTER0_SELECT"/>
+	<reg32 offset="0x0f05" name="RB_PERFCOUNTER1_SELECT"/>
+	<reg32 offset="0x0f06" name="RB_PERFCOUNTER2_SELECT"/>
+	<reg32 offset="0x0f07" name="RB_PERFCOUNTER3_SELECT"/>
+	<reg32 offset="0x0f08" name="RB_PERFCOUNTER0_LOW"/>
+	<reg32 offset="0x0f09" name="RB_PERFCOUNTER0_HI"/>
+	<reg32 offset="0x0f0a" name="RB_PERFCOUNTER1_LOW"/>
+	<reg32 offset="0x0f0b" name="RB_PERFCOUNTER1_HI"/>
+	<reg32 offset="0x0f0c" name="RB_PERFCOUNTER2_LOW"/>
+	<reg32 offset="0x0f0d" name="RB_PERFCOUNTER2_HI"/>
+	<reg32 offset="0x0f0e" name="RB_PERFCOUNTER3_LOW"/>
+	<reg32 offset="0x0f0f" name="RB_PERFCOUNTER3_HI"/>
+</domain>
+
+<domain name="A2XX_SQ_TEX" width="32">
+	<doc>Texture state dwords</doc>
+	<enum name="sq_tex_clamp">
+		<value name="SQ_TEX_WRAP" value="0"/>
+		<value name="SQ_TEX_MIRROR" value="1"/>
+		<value name="SQ_TEX_CLAMP_LAST_TEXEL" value="2"/>
+		<value name="SQ_TEX_MIRROR_ONCE_LAST_TEXEL" value="3"/>
+		<value name="SQ_TEX_CLAMP_HALF_BORDER" value="4"/>
+		<value name="SQ_TEX_MIRROR_ONCE_HALF_BORDER" value="5"/>
+		<value name="SQ_TEX_CLAMP_BORDER" value="6"/>
+		<value name="SQ_TEX_MIRROR_ONCE_BORDER" value="7"/>
+	</enum>
+	<enum name="sq_tex_swiz">
+		<value name="SQ_TEX_X" value="0"/>
+		<value name="SQ_TEX_Y" value="1"/>
+		<value name="SQ_TEX_Z" value="2"/>
+		<value name="SQ_TEX_W" value="3"/>
+		<value name="SQ_TEX_ZERO" value="4"/>
+		<value name="SQ_TEX_ONE" value="5"/>
+	</enum>
+	<enum name="sq_tex_filter">
+		<value name="SQ_TEX_FILTER_POINT" value="0"/>
+		<value name="SQ_TEX_FILTER_BILINEAR" value="1"/>
+		<value name="SQ_TEX_FILTER_BASEMAP" value="2"/>
+		<value name="SQ_TEX_FILTER_USE_FETCH_CONST" value="3"/>
+	</enum>
+	<enum name="sq_tex_aniso_filter">
+		<value name="SQ_TEX_ANISO_FILTER_DISABLED" value="0"/>
+		<value name="SQ_TEX_ANISO_FILTER_MAX_1_1" value="1"/>
+		<value name="SQ_TEX_ANISO_FILTER_MAX_2_1" value="2"/>
+		<value name="SQ_TEX_ANISO_FILTER_MAX_4_1" value="3"/>
+		<value name="SQ_TEX_ANISO_FILTER_MAX_8_1" value="4"/>
+		<value name="SQ_TEX_ANISO_FILTER_MAX_16_1" value="5"/>
+		<value name="SQ_TEX_ANISO_FILTER_USE_FETCH_CONST" value="7"/>
+	</enum>
+	<enum name="sq_tex_dimension">
+		<value name="SQ_TEX_DIMENSION_1D" value="0"/>
+		<value name="SQ_TEX_DIMENSION_2D" value="1"/>
+		<value name="SQ_TEX_DIMENSION_3D" value="2"/>
+		<value name="SQ_TEX_DIMENSION_CUBE" value="3"/>
+	</enum>
+	<enum name="sq_tex_border_color">
+		<value name="SQ_TEX_BORDER_COLOR_BLACK" value="0"/>
+		<value name="SQ_TEX_BORDER_COLOR_WHITE" value="1"/>
+		<value name="SQ_TEX_BORDER_COLOR_ACBYCR_BLACK" value="2"/>
+		<value name="SQ_TEX_BORDER_COLOR_ACBCRY_BLACK" value="3"/>
+	</enum>
+	<enum name="sq_tex_sign">
+		<value name="SQ_TEX_SIGN_UNSIGNED" value="0"/>
+		<value name="SQ_TEX_SIGN_SIGNED" value="1"/>
+		<!-- biased: 2*color-1 (range -1,1 when sampling) -->
+		<value name="SQ_TEX_SIGN_UNSIGNED_BIASED" value="2"/>
+		<!-- gamma: sRGB to linear - doesn't seem to work on adreno? -->
+		<value name="SQ_TEX_SIGN_GAMMA" value="3"/>
+	</enum>
+	<enum name="sq_tex_endian">
+		<value name="SQ_TEX_ENDIAN_NONE" value="0"/>
+		<value name="SQ_TEX_ENDIAN_8IN16" value="1"/>
+		<value name="SQ_TEX_ENDIAN_8IN32" value="2"/>
+		<value name="SQ_TEX_ENDIAN_16IN32" value="3"/>
+	</enum>
+	<enum name="sq_tex_clamp_policy">
+		<value name="SQ_TEX_CLAMP_POLICY_D3D" value="0"/>
+		<value name="SQ_TEX_CLAMP_POLICY_OGL" value="1"/>
+	</enum>
+	<enum name="sq_tex_num_format">
+		<value name="SQ_TEX_NUM_FORMAT_FRAC" value="0"/>
+		<value name="SQ_TEX_NUM_FORMAT_INT" value="1"/>
+	</enum>
+	<enum name="sq_tex_type">
+		<value name="SQ_TEX_TYPE_0" value="0"/>
+		<value name="SQ_TEX_TYPE_1" value="1"/>
+		<value name="SQ_TEX_TYPE_2" value="2"/>
+		<value name="SQ_TEX_TYPE_3" value="3"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="TYPE" low="0" high="1" type="sq_tex_type"/>
+		<bitfield name="SIGN_X" low="2" high="3" type="sq_tex_sign"/>
+		<bitfield name="SIGN_Y" low="4" high="5" type="sq_tex_sign"/>
+		<bitfield name="SIGN_Z" low="6" high="7" type="sq_tex_sign"/>
+		<bitfield name="SIGN_W" low="8" high="9" type="sq_tex_sign"/>
+		<bitfield name="CLAMP_X" low="10" high="12" type="sq_tex_clamp"/>
+		<bitfield name="CLAMP_Y" low="13" high="15" type="sq_tex_clamp"/>
+		<bitfield name="CLAMP_Z" low="16" high="18" type="sq_tex_clamp"/>
+		<bitfield name="PITCH" low="22" high="30" shr="5" type="uint"/>
+		<bitfield name="TILED" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="FORMAT" low="0" high="5" type="a2xx_sq_surfaceformat"/>
+		<bitfield name="ENDIANNESS" low="6" high="7" type="sq_tex_endian"/>
+		<bitfield name="REQUEST_SIZE" low="8" high="9" type="uint"/>
+		<bitfield name="STACKED" pos="10" type="boolean"/>
+		<bitfield name="CLAMP_POLICY" pos="11" type="sq_tex_clamp_policy"/>
+		<bitfield name="BASE_ADDRESS" low="12" high="31" type="uint" shr="12"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="WIDTH" low="0" high="12" type="uint"/>
+		<bitfield name="HEIGHT" low="13" high="25" type="uint"/>
+		<bitfield name="DEPTH" low="26" high="31" type="uint"/>
+		<!-- 1d/3d have different bit configurations -->
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="NUM_FORMAT" pos="0" type="sq_tex_num_format"/>
+		<bitfield name="SWIZ_X" low="1" high="3" type="sq_tex_swiz"/>
+		<bitfield name="SWIZ_Y" low="4" high="6" type="sq_tex_swiz"/>
+		<bitfield name="SWIZ_Z" low="7" high="9" type="sq_tex_swiz"/>
+		<bitfield name="SWIZ_W" low="10" high="12" type="sq_tex_swiz"/>
+		<bitfield name="EXP_ADJUST" low="13" high="18" type="int"/>
+		<bitfield name="XY_MAG_FILTER" low="19" high="20" type="sq_tex_filter"/>
+		<bitfield name="XY_MIN_FILTER" low="21" high="22" type="sq_tex_filter"/>
+		<bitfield name="MIP_FILTER" low="23" high="24" type="sq_tex_filter"/>
+		<bitfield name="ANISO_FILTER" low="25" high="27" type="sq_tex_aniso_filter"/>
+		<bitfield name="BORDER_SIZE" pos="31" type="uint"/>
+	</reg32>
+	<reg32 offset="4" name="4">
+		<bitfield name="VOL_MAG_FILTER" pos="0" type="sq_tex_filter"/>
+		<bitfield name="VOL_MIN_FILTER" pos="1" type="sq_tex_filter"/>
+		<bitfield name="MIP_MIN_LEVEL" low="2" high="5" type="uint"/>
+		<bitfield name="MIP_MAX_LEVEL" low="6" high="9" type="uint"/>
+		<bitfield name="MAX_ANISO_WALK" pos="10" type="boolean"/>
+		<bitfield name="MIN_ANISO_WALK" pos="11" type="boolean"/>
+		<bitfield name="LOD_BIAS" low="12" high="21" type="fixed" radix="5"/>
+		<bitfield name="GRAD_EXP_ADJUST_H" low="22" high="26" type="uint"/>
+		<bitfield name="GRAD_EXP_ADJUST_V" low="27" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="5" name="5">
+		<bitfield name="BORDER_COLOR" low="0" high="1" type="sq_tex_border_color"/>
+		<bitfield name="FORCE_BCW_MAX" pos="2" type="boolean"/>
+		<bitfield name="TRI_CLAMP" low="3" high="4" type="uint"/>
+		<bitfield name="ANISO_BIAS" low="5" high="8" type="fixed" radix="0"/> <!-- radix unknown -->
+		<bitfield name="DIMENSION" low="9" high="10" type="sq_tex_dimension"/>
+		<bitfield name="PACKED_MIPS" pos="11" type="boolean"/>
+		<bitfield name="MIP_ADDRESS" low="12" high="31" type="uint" shr="12"/>
+	</reg32>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/adreno/a3xx.xml b/drivers/gpu/drm/msm/registers/adreno/a3xx.xml
new file mode 100644
index 000000000000..6717abc0a897
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/adreno/a3xx.xml
@@ -0,0 +1,1751 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+<import file="adreno/adreno_common.xml"/>
+<import file="adreno/adreno_pm4.xml"/>
+
+<enum name="a3xx_tile_mode">
+	<value name="LINEAR" value="0"/>
+	<value name="TILE_4X4" value="1"/>    <!-- "normal" case for textures -->
+	<value name="TILE_32X32" value="2"/>  <!-- only used in GMEM -->
+	<value name="TILE_4X2" value="3"/>    <!-- only used for CrCb -->
+</enum>
+
+<enum name="a3xx_state_block_id">
+	<value name="HLSQ_BLOCK_ID_TP_TEX" value="2"/>
+	<value name="HLSQ_BLOCK_ID_TP_MIPMAP" value="3"/>
+	<value name="HLSQ_BLOCK_ID_SP_VS" value="4"/>
+	<value name="HLSQ_BLOCK_ID_SP_FS" value="6"/>
+</enum>
+
+<enum name="a3xx_cache_opcode">
+	<value name="INVALIDATE" value="1"/>
+</enum>
+
+<enum name="a3xx_vtx_fmt">
+	<value name="VFMT_32_FLOAT" value="0x0"/>
+	<value name="VFMT_32_32_FLOAT" value="0x1"/>
+	<value name="VFMT_32_32_32_FLOAT" value="0x2"/>
+	<value name="VFMT_32_32_32_32_FLOAT" value="0x3"/>
+
+	<value name="VFMT_16_FLOAT" value="0x4"/>
+	<value name="VFMT_16_16_FLOAT" value="0x5"/>
+	<value name="VFMT_16_16_16_FLOAT" value="0x6"/>
+	<value name="VFMT_16_16_16_16_FLOAT" value="0x7"/>
+
+	<value name="VFMT_32_FIXED" value="0x8"/>
+	<value name="VFMT_32_32_FIXED" value="0x9"/>
+	<value name="VFMT_32_32_32_FIXED" value="0xa"/>
+	<value name="VFMT_32_32_32_32_FIXED" value="0xb"/>
+
+	<value name="VFMT_16_SINT" value="0x10"/>
+	<value name="VFMT_16_16_SINT" value="0x11"/>
+	<value name="VFMT_16_16_16_SINT" value="0x12"/>
+	<value name="VFMT_16_16_16_16_SINT" value="0x13"/>
+	<value name="VFMT_16_UINT" value="0x14"/>
+	<value name="VFMT_16_16_UINT" value="0x15"/>
+	<value name="VFMT_16_16_16_UINT" value="0x16"/>
+	<value name="VFMT_16_16_16_16_UINT" value="0x17"/>
+	<value name="VFMT_16_SNORM" value="0x18"/>
+	<value name="VFMT_16_16_SNORM" value="0x19"/>
+	<value name="VFMT_16_16_16_SNORM" value="0x1a"/>
+	<value name="VFMT_16_16_16_16_SNORM" value="0x1b"/>
+	<value name="VFMT_16_UNORM" value="0x1c"/>
+	<value name="VFMT_16_16_UNORM" value="0x1d"/>
+	<value name="VFMT_16_16_16_UNORM" value="0x1e"/>
+	<value name="VFMT_16_16_16_16_UNORM" value="0x1f"/>
+
+	<!-- seems to be no NORM variants for 32bit.. -->
+	<value name="VFMT_32_UINT" value="0x20"/>
+	<value name="VFMT_32_32_UINT" value="0x21"/>
+	<value name="VFMT_32_32_32_UINT" value="0x22"/>
+	<value name="VFMT_32_32_32_32_UINT" value="0x23"/>
+	<value name="VFMT_32_SINT" value="0x24"/>
+	<value name="VFMT_32_32_SINT" value="0x25"/>
+	<value name="VFMT_32_32_32_SINT" value="0x26"/>
+	<value name="VFMT_32_32_32_32_SINT" value="0x27"/>
+
+	<value name="VFMT_8_UINT" value="0x28"/>
+	<value name="VFMT_8_8_UINT" value="0x29"/>
+	<value name="VFMT_8_8_8_UINT" value="0x2a"/>
+	<value name="VFMT_8_8_8_8_UINT" value="0x2b"/>
+	<value name="VFMT_8_UNORM" value="0x2c"/>
+	<value name="VFMT_8_8_UNORM" value="0x2d"/>
+	<value name="VFMT_8_8_8_UNORM" value="0x2e"/>
+	<value name="VFMT_8_8_8_8_UNORM" value="0x2f"/>
+	<value name="VFMT_8_SINT" value="0x30"/>
+	<value name="VFMT_8_8_SINT" value="0x31"/>
+	<value name="VFMT_8_8_8_SINT" value="0x32"/>
+	<value name="VFMT_8_8_8_8_SINT" value="0x33"/>
+	<value name="VFMT_8_SNORM" value="0x34"/>
+	<value name="VFMT_8_8_SNORM" value="0x35"/>
+	<value name="VFMT_8_8_8_SNORM" value="0x36"/>
+	<value name="VFMT_8_8_8_8_SNORM" value="0x37"/>
+	<value name="VFMT_10_10_10_2_UINT" value="0x38"/>
+	<value name="VFMT_10_10_10_2_UNORM" value="0x39"/>
+	<value name="VFMT_10_10_10_2_SINT" value="0x3a"/>
+	<value name="VFMT_10_10_10_2_SNORM" value="0x3b"/>
+	<value name="VFMT_2_10_10_10_UINT" value="0x3c"/>
+	<value name="VFMT_2_10_10_10_UNORM" value="0x3d"/>
+	<value name="VFMT_2_10_10_10_SINT" value="0x3e"/>
+	<value name="VFMT_2_10_10_10_SNORM" value="0x3f"/>
+
+	<value name="VFMT_NONE" value="0xff"/>
+</enum>
+
+<enum name="a3xx_tex_fmt">
+	<value name="TFMT_5_6_5_UNORM" value="0x4"/>
+	<value name="TFMT_5_5_5_1_UNORM" value="0x5"/>
+	<value name="TFMT_4_4_4_4_UNORM" value="0x7"/>
+	<value name="TFMT_Z16_UNORM" value="0x9"/>
+	<value name="TFMT_X8Z24_UNORM" value="0xa"/>
+	<value name="TFMT_Z32_FLOAT" value="0xb"/>
+
+	<!--
+		The NV12 tiled/linear formats seem to require gang'd sampler
+		slots (ie. sampler state N plus N+1) for Y and UV planes.
+		They fetch yuv in single sam instruction, but still require
+		colorspace conversion in the shader.
+	 -->
+	<value name="TFMT_UV_64X32" value="0x10"/>
+	<value name="TFMT_VU_64X32" value="0x11"/>
+	<value name="TFMT_Y_64X32" value="0x12"/>
+	<value name="TFMT_NV12_64X32" value="0x13"/>
+	<value name="TFMT_UV_LINEAR" value="0x14"/>
+	<value name="TFMT_VU_LINEAR" value="0x15"/>
+	<value name="TFMT_Y_LINEAR" value="0x16"/>
+	<value name="TFMT_NV12_LINEAR" value="0x17"/>
+	<value name="TFMT_I420_Y" value="0x18"/>
+	<value name="TFMT_I420_U" value="0x1a"/>
+	<value name="TFMT_I420_V" value="0x1b"/>
+
+	<value name="TFMT_ATC_RGB" value="0x20"/>
+	<value name="TFMT_ATC_RGBA_EXPLICIT" value="0x21"/>
+	<value name="TFMT_ETC1" value="0x22"/>
+	<value name="TFMT_ATC_RGBA_INTERPOLATED" value="0x23"/>
+
+	<value name="TFMT_DXT1" value="0x24"/>
+	<value name="TFMT_DXT3" value="0x25"/>
+	<value name="TFMT_DXT5" value="0x26"/>
+
+	<value name="TFMT_2_10_10_10_UNORM" value="0x28"/>
+	<value name="TFMT_10_10_10_2_UNORM" value="0x29"/>
+	<value name="TFMT_9_9_9_E5_FLOAT" value="0x2a"/>
+	<value name="TFMT_11_11_10_FLOAT" value="0x2b"/>
+	<value name="TFMT_A8_UNORM" value="0x2c"/>    <!-- GL_ALPHA -->
+	<value name="TFMT_L8_UNORM" value="0x2d"/>
+	<value name="TFMT_L8_A8_UNORM" value="0x2f"/> <!-- GL_LUMINANCE_ALPHA -->
+
+	<!--
+		NOTE: GL_ALPHA and GL_LUMINANCE_ALPHA aren't handled in a similar way
+		to float16, float32.. but they seem to use non-standard swizzle too..
+		perhaps we can ditch that if the pattern follows of 0xn0, 0xn1, 0xn2,
+		0xn3 for 1, 2, 3, 4 components respectively..
+
+		Only formats filled in below are the ones that have been observed by
+		the blob or tested.. you can guess what the missing ones are..
+	 -->
+
+	<value name="TFMT_8_UNORM" value="0x30"/>     <!-- GL_LUMINANCE -->
+	<value name="TFMT_8_8_UNORM" value="0x31"/>
+	<value name="TFMT_8_8_8_UNORM" value="0x32"/>
+	<value name="TFMT_8_8_8_8_UNORM" value="0x33"/>
+
+	<value name="TFMT_8_SNORM" value="0x34"/>
+	<value name="TFMT_8_8_SNORM" value="0x35"/>
+	<value name="TFMT_8_8_8_SNORM" value="0x36"/>
+	<value name="TFMT_8_8_8_8_SNORM" value="0x37"/>
+
+	<value name="TFMT_8_UINT" value="0x38"/>
+	<value name="TFMT_8_8_UINT" value="0x39"/>
+	<value name="TFMT_8_8_8_UINT" value="0x3a"/>
+	<value name="TFMT_8_8_8_8_UINT" value="0x3b"/>
+
+	<value name="TFMT_8_SINT" value="0x3c"/>
+	<value name="TFMT_8_8_SINT" value="0x3d"/>
+	<value name="TFMT_8_8_8_SINT" value="0x3e"/>
+	<value name="TFMT_8_8_8_8_SINT" value="0x3f"/>
+
+	<value name="TFMT_16_FLOAT" value="0x40"/>
+	<value name="TFMT_16_16_FLOAT" value="0x41"/>
+	<!-- TFMT_FLOAT_16_16_16 -->
+	<value name="TFMT_16_16_16_16_FLOAT" value="0x43"/>
+
+	<value name="TFMT_16_UINT" value="0x44"/>
+	<value name="TFMT_16_16_UINT" value="0x45"/>
+	<value name="TFMT_16_16_16_16_UINT" value="0x47"/>
+
+	<value name="TFMT_16_SINT" value="0x48"/>
+	<value name="TFMT_16_16_SINT" value="0x49"/>
+	<value name="TFMT_16_16_16_16_SINT" value="0x4b"/>
+
+	<value name="TFMT_16_UNORM" value="0x4c"/>
+	<value name="TFMT_16_16_UNORM" value="0x4d"/>
+	<value name="TFMT_16_16_16_16_UNORM" value="0x4f"/>
+
+	<value name="TFMT_16_SNORM" value="0x50"/>
+	<value name="TFMT_16_16_SNORM" value="0x51"/>
+	<value name="TFMT_16_16_16_16_SNORM" value="0x53"/>
+
+	<value name="TFMT_32_FLOAT" value="0x54"/>
+	<value name="TFMT_32_32_FLOAT" value="0x55"/>
+	<!-- TFMT_32_32_32_FLOAT -->
+	<value name="TFMT_32_32_32_32_FLOAT" value="0x57"/>
+
+	<value name="TFMT_32_UINT" value="0x58"/>
+	<value name="TFMT_32_32_UINT" value="0x59"/>
+	<value name="TFMT_32_32_32_32_UINT" value="0x5b"/>
+
+	<value name="TFMT_32_SINT" value="0x5c"/>
+	<value name="TFMT_32_32_SINT" value="0x5d"/>
+	<value name="TFMT_32_32_32_32_SINT" value="0x5f"/>
+
+	<value name="TFMT_2_10_10_10_UINT" value="0x60"/>
+	<value name="TFMT_10_10_10_2_UINT" value="0x61"/>
+
+	<value name="TFMT_ETC2_RG11_SNORM" value="0x70"/>
+	<value name="TFMT_ETC2_RG11_UNORM" value="0x71"/>
+	<value name="TFMT_ETC2_R11_SNORM" value="0x72"/>
+	<value name="TFMT_ETC2_R11_UNORM" value="0x73"/>
+	<value name="TFMT_ETC2_RGBA8" value="0x74"/>
+	<value name="TFMT_ETC2_RGB8A1" value="0x75"/>
+	<value name="TFMT_ETC2_RGB8" value="0x76"/>
+
+	<value name="TFMT_NONE" value="0xff"/>
+</enum>
+
+<enum name="a3xx_color_fmt">
+	<value name="RB_R5G6B5_UNORM"       value="0x00"/>
+	<value name="RB_R5G5B5A1_UNORM"     value="0x01"/>
+	<value name="RB_R4G4B4A4_UNORM"     value="0x03"/>
+	<value name="RB_R8G8B8_UNORM"	    value="0x04"/>
+	<value name="RB_R8G8B8A8_UNORM"	    value="0x08"/>
+	<value name="RB_R8G8B8A8_SNORM"	    value="0x09"/>
+	<value name="RB_R8G8B8A8_UINT"	    value="0x0a"/>
+	<value name="RB_R8G8B8A8_SINT"	    value="0x0b"/>
+	<value name="RB_R8G8_UNORM"	    value="0x0c"/>
+	<value name="RB_R8G8_SNORM"	    value="0x0d"/>
+	<value name="RB_R8G8_UINT"	    value="0x0e"/>
+	<value name="RB_R8G8_SINT"	    value="0x0f"/>
+	<value name="RB_R10G10B10A2_UNORM"  value="0x10"/>
+	<value name="RB_A2R10G10B10_UNORM"  value="0x11"/>
+	<value name="RB_R10G10B10A2_UINT"   value="0x12"/>
+	<value name="RB_A2R10G10B10_UINT"   value="0x13"/>
+
+	<value name="RB_A8_UNORM"	    value="0x14"/>
+	<value name="RB_R8_UNORM"	    value="0x15"/>
+
+	<value name="RB_R16_FLOAT"          value="0x18"/>
+	<value name="RB_R16G16_FLOAT"       value="0x19"/>
+	<value name="RB_R16G16B16A16_FLOAT" value="0x1b"/> <!-- GL_HALF_FLOAT_OES -->
+	<value name="RB_R11G11B10_FLOAT"    value="0x1c"/>
+
+	<value name="RB_R16_SNORM"          value="0x20"/>
+	<value name="RB_R16G16_SNORM"       value="0x21"/>
+	<value name="RB_R16G16B16A16_SNORM" value="0x23"/>
+
+	<value name="RB_R16_UNORM"          value="0x24"/>
+	<value name="RB_R16G16_UNORM"       value="0x25"/>
+	<value name="RB_R16G16B16A16_UNORM" value="0x27"/>
+
+	<value name="RB_R16_SINT"	    value="0x28"/>
+	<value name="RB_R16G16_SINT"	    value="0x29"/>
+	<value name="RB_R16G16B16A16_SINT"  value="0x2b"/>
+
+	<value name="RB_R16_UINT"	    value="0x2c"/>
+	<value name="RB_R16G16_UINT"	    value="0x2d"/>
+	<value name="RB_R16G16B16A16_UINT"  value="0x2f"/>
+
+	<value name="RB_R32_FLOAT"          value="0x30"/>
+	<value name="RB_R32G32_FLOAT"       value="0x31"/>
+	<value name="RB_R32G32B32A32_FLOAT" value="0x33"/> <!-- GL_FLOAT -->
+
+	<value name="RB_R32_SINT"	    value="0x34"/>
+	<value name="RB_R32G32_SINT"	    value="0x35"/>
+	<value name="RB_R32G32B32A32_SINT"  value="0x37"/>
+
+	<value name="RB_R32_UINT"	    value="0x38"/>
+	<value name="RB_R32G32_UINT"	    value="0x39"/>
+	<value name="RB_R32G32B32A32_UINT"  value="0x3b"/>
+
+	<value name="RB_NONE"               value="0xff"/>
+</enum>
+
+<enum name="a3xx_cp_perfcounter_select">
+	<value value="0x00" name="CP_ALWAYS_COUNT"/>
+	<value value="0x03" name="CP_AHB_PFPTRANS_WAIT"/>
+	<value value="0x06" name="CP_AHB_NRTTRANS_WAIT"/>
+	<value value="0x08" name="CP_CSF_NRT_READ_WAIT"/>
+	<value value="0x09" name="CP_CSF_I1_FIFO_FULL"/>
+	<value value="0x0a" name="CP_CSF_I2_FIFO_FULL"/>
+	<value value="0x0b" name="CP_CSF_ST_FIFO_FULL"/>
+	<value value="0x0c" name="CP_RESERVED_12"/>
+	<value value="0x0d" name="CP_CSF_RING_ROQ_FULL"/>
+	<value value="0x0e" name="CP_CSF_I1_ROQ_FULL"/>
+	<value value="0x0f" name="CP_CSF_I2_ROQ_FULL"/>
+	<value value="0x10" name="CP_CSF_ST_ROQ_FULL"/>
+	<value value="0x11" name="CP_RESERVED_17"/>
+	<value value="0x12" name="CP_MIU_TAG_MEM_FULL"/>
+	<value value="0x16" name="CP_MIU_NRT_WRITE_STALLED"/>
+	<value value="0x17" name="CP_MIU_NRT_READ_STALLED"/>
+	<value value="0x1a" name="CP_ME_REGS_RB_DONE_FIFO_FULL"/>
+	<value value="0x1b" name="CP_ME_REGS_VS_EVENT_FIFO_FULL"/>
+	<value value="0x1c" name="CP_ME_REGS_PS_EVENT_FIFO_FULL"/>
+	<value value="0x1d" name="CP_ME_REGS_CF_EVENT_FIFO_FULL"/>
+	<value value="0x1e" name="CP_ME_MICRO_RB_STARVED"/>
+	<value value="0x28" name="CP_AHB_RBBM_DWORD_SENT"/>
+	<value value="0x29" name="CP_ME_BUSY_CLOCKS"/>
+	<value value="0x2a" name="CP_ME_WAIT_CONTEXT_AVAIL"/>
+	<value value="0x2b" name="CP_PFP_TYPE0_PACKET"/>
+	<value value="0x2c" name="CP_PFP_TYPE3_PACKET"/>
+	<value value="0x2d" name="CP_CSF_RB_WPTR_NEQ_RPTR"/>
+	<value value="0x2e" name="CP_CSF_I1_SIZE_NEQ_ZERO"/>
+	<value value="0x2f" name="CP_CSF_I2_SIZE_NEQ_ZERO"/>
+	<value value="0x30" name="CP_CSF_RBI1I2_FETCHING"/>
+</enum>
+
+<enum name="a3xx_gras_tse_perfcounter_select">
+	<value value="0x00" name="GRAS_TSEPERF_INPUT_PRIM"/>
+	<value value="0x01" name="GRAS_TSEPERF_INPUT_NULL_PRIM"/>
+	<value value="0x02" name="GRAS_TSEPERF_TRIVAL_REJ_PRIM"/>
+	<value value="0x03" name="GRAS_TSEPERF_CLIPPED_PRIM"/>
+	<value value="0x04" name="GRAS_TSEPERF_NEW_PRIM"/>
+	<value value="0x05" name="GRAS_TSEPERF_ZERO_AREA_PRIM"/>
+	<value value="0x06" name="GRAS_TSEPERF_FACENESS_CULLED_PRIM"/>
+	<value value="0x07" name="GRAS_TSEPERF_ZERO_PIXEL_PRIM"/>
+	<value value="0x08" name="GRAS_TSEPERF_OUTPUT_NULL_PRIM"/>
+	<value value="0x09" name="GRAS_TSEPERF_OUTPUT_VISIBLE_PRIM"/>
+	<value value="0x0a" name="GRAS_TSEPERF_PRE_CLIP_PRIM"/>
+	<value value="0x0b" name="GRAS_TSEPERF_POST_CLIP_PRIM"/>
+	<value value="0x0c" name="GRAS_TSEPERF_WORKING_CYCLES"/>
+	<value value="0x0d" name="GRAS_TSEPERF_PC_STARVE"/>
+	<value value="0x0e" name="GRAS_TSERASPERF_STALL"/>
+</enum>
+
+<enum name="a3xx_gras_ras_perfcounter_select">
+	<value value="0x00" name="GRAS_RASPERF_16X16_TILES"/>
+	<value value="0x01" name="GRAS_RASPERF_8X8_TILES"/>
+	<value value="0x02" name="GRAS_RASPERF_4X4_TILES"/>
+	<value value="0x03" name="GRAS_RASPERF_WORKING_CYCLES"/>
+	<value value="0x04" name="GRAS_RASPERF_STALL_CYCLES_BY_RB"/>
+	<value value="0x05" name="GRAS_RASPERF_STALL_CYCLES_BY_VSC"/>
+	<value value="0x06" name="GRAS_RASPERF_STARVE_CYCLES_BY_TSE"/>
+</enum>
+
+<enum name="a3xx_hlsq_perfcounter_select">
+	<value value="0x00" name="HLSQ_PERF_SP_VS_CONSTANT"/>
+	<value value="0x01" name="HLSQ_PERF_SP_VS_INSTRUCTIONS"/>
+	<value value="0x02" name="HLSQ_PERF_SP_FS_CONSTANT"/>
+	<value value="0x03" name="HLSQ_PERF_SP_FS_INSTRUCTIONS"/>
+	<value value="0x04" name="HLSQ_PERF_TP_STATE"/>
+	<value value="0x05" name="HLSQ_PERF_QUADS"/>
+	<value value="0x06" name="HLSQ_PERF_PIXELS"/>
+	<value value="0x07" name="HLSQ_PERF_VERTICES"/>
+	<value value="0x08" name="HLSQ_PERF_FS8_THREADS"/>
+	<value value="0x09" name="HLSQ_PERF_FS16_THREADS"/>
+	<value value="0x0a" name="HLSQ_PERF_FS32_THREADS"/>
+	<value value="0x0b" name="HLSQ_PERF_VS8_THREADS"/>
+	<value value="0x0c" name="HLSQ_PERF_VS16_THREADS"/>
+	<value value="0x0d" name="HLSQ_PERF_SP_VS_DATA_BYTES"/>
+	<value value="0x0e" name="HLSQ_PERF_SP_FS_DATA_BYTES"/>
+	<value value="0x0f" name="HLSQ_PERF_ACTIVE_CYCLES"/>
+	<value value="0x10" name="HLSQ_PERF_STALL_CYCLES_SP_STATE"/>
+	<value value="0x11" name="HLSQ_PERF_STALL_CYCLES_SP_VS"/>
+	<value value="0x12" name="HLSQ_PERF_STALL_CYCLES_SP_FS"/>
+	<value value="0x13" name="HLSQ_PERF_STALL_CYCLES_UCHE"/>
+	<value value="0x14" name="HLSQ_PERF_RBBM_LOAD_CYCLES"/>
+	<value value="0x15" name="HLSQ_PERF_DI_TO_VS_START_SP0"/>
+	<value value="0x16" name="HLSQ_PERF_DI_TO_FS_START_SP0"/>
+	<value value="0x17" name="HLSQ_PERF_VS_START_TO_DONE_SP0"/>
+	<value value="0x18" name="HLSQ_PERF_FS_START_TO_DONE_SP0"/>
+	<value value="0x19" name="HLSQ_PERF_SP_STATE_COPY_CYCLES_VS"/>
+	<value value="0x1a" name="HLSQ_PERF_SP_STATE_COPY_CYCLES_FS"/>
+	<value value="0x1b" name="HLSQ_PERF_UCHE_LATENCY_CYCLES"/>
+	<value value="0x1c" name="HLSQ_PERF_UCHE_LATENCY_COUNT"/>
+</enum>
+
+<enum name="a3xx_pc_perfcounter_select">
+	<value value="0x00" name="PC_PCPERF_VISIBILITY_STREAMS"/>
+	<value value="0x01" name="PC_PCPERF_TOTAL_INSTANCES"/>
+	<value value="0x02" name="PC_PCPERF_PRIMITIVES_PC_VPC"/>
+	<value value="0x03" name="PC_PCPERF_PRIMITIVES_KILLED_BY_VS"/>
+	<value value="0x04" name="PC_PCPERF_PRIMITIVES_VISIBLE_BY_VS"/>
+	<value value="0x05" name="PC_PCPERF_DRAWCALLS_KILLED_BY_VS"/>
+	<value value="0x06" name="PC_PCPERF_DRAWCALLS_VISIBLE_BY_VS"/>
+	<value value="0x07" name="PC_PCPERF_VERTICES_TO_VFD"/>
+	<value value="0x08" name="PC_PCPERF_REUSED_VERTICES"/>
+	<value value="0x09" name="PC_PCPERF_CYCLES_STALLED_BY_VFD"/>
+	<value value="0x0a" name="PC_PCPERF_CYCLES_STALLED_BY_TSE"/>
+	<value value="0x0b" name="PC_PCPERF_CYCLES_STALLED_BY_VBIF"/>
+	<value value="0x0c" name="PC_PCPERF_CYCLES_IS_WORKING"/>
+</enum>
+
+<enum name="a3xx_rb_perfcounter_select">
+	<value value="0x00" name="RB_RBPERF_ACTIVE_CYCLES_ANY"/>
+	<value value="0x01" name="RB_RBPERF_ACTIVE_CYCLES_ALL"/>
+	<value value="0x02" name="RB_RBPERF_STARVE_CYCLES_BY_SP"/>
+	<value value="0x03" name="RB_RBPERF_STARVE_CYCLES_BY_RAS"/>
+	<value value="0x04" name="RB_RBPERF_STARVE_CYCLES_BY_MARB"/>
+	<value value="0x05" name="RB_RBPERF_STALL_CYCLES_BY_MARB"/>
+	<value value="0x06" name="RB_RBPERF_STALL_CYCLES_BY_HLSQ"/>
+	<value value="0x07" name="RB_RBPERF_RB_MARB_DATA"/>
+	<value value="0x08" name="RB_RBPERF_SP_RB_QUAD"/>
+	<value value="0x09" name="RB_RBPERF_RAS_EARLY_Z_QUADS"/>
+	<value value="0x0a" name="RB_RBPERF_GMEM_CH0_READ"/>
+	<value value="0x0b" name="RB_RBPERF_GMEM_CH1_READ"/>
+	<value value="0x0c" name="RB_RBPERF_GMEM_CH0_WRITE"/>
+	<value value="0x0d" name="RB_RBPERF_GMEM_CH1_WRITE"/>
+	<value value="0x0e" name="RB_RBPERF_CP_CONTEXT_DONE"/>
+	<value value="0x0f" name="RB_RBPERF_CP_CACHE_FLUSH"/>
+	<value value="0x10" name="RB_RBPERF_CP_ZPASS_DONE"/>
+</enum>
+
+<enum name="a3xx_rbbm_perfcounter_select">
+	<value value="0" name="RBBM_ALAWYS_ON"/>
+	<value value="1" name="RBBM_VBIF_BUSY"/>
+	<value value="2" name="RBBM_TSE_BUSY"/>
+	<value value="3" name="RBBM_RAS_BUSY"/>
+	<value value="4" name="RBBM_PC_DCALL_BUSY"/>
+	<value value="5" name="RBBM_PC_VSD_BUSY"/>
+	<value value="6" name="RBBM_VFD_BUSY"/>
+	<value value="7" name="RBBM_VPC_BUSY"/>
+	<value value="8" name="RBBM_UCHE_BUSY"/>
+	<value value="9" name="RBBM_VSC_BUSY"/>
+	<value value="10" name="RBBM_HLSQ_BUSY"/>
+	<value value="11" name="RBBM_ANY_RB_BUSY"/>
+	<value value="12" name="RBBM_ANY_TEX_BUSY"/>
+	<value value="13" name="RBBM_ANY_USP_BUSY"/>
+	<value value="14" name="RBBM_ANY_MARB_BUSY"/>
+	<value value="15" name="RBBM_ANY_ARB_BUSY"/>
+	<value value="16" name="RBBM_AHB_STATUS_BUSY"/>
+	<value value="17" name="RBBM_AHB_STATUS_STALLED"/>
+	<value value="18" name="RBBM_AHB_STATUS_TXFR"/>
+	<value value="19" name="RBBM_AHB_STATUS_TXFR_SPLIT"/>
+	<value value="20" name="RBBM_AHB_STATUS_TXFR_ERROR"/>
+	<value value="21" name="RBBM_AHB_STATUS_LONG_STALL"/>
+	<value value="22" name="RBBM_RBBM_STATUS_MASKED"/>
+</enum>
+
+<enum name="a3xx_sp_perfcounter_select">
+	<value value="0x00" name="SP_LM_LOAD_INSTRUCTIONS"/>
+	<value value="0x01" name="SP_LM_STORE_INSTRUCTIONS"/>
+	<value value="0x02" name="SP_LM_ATOMICS"/>
+	<value value="0x03" name="SP_UCHE_LOAD_INSTRUCTIONS"/>
+	<value value="0x04" name="SP_UCHE_STORE_INSTRUCTIONS"/>
+	<value value="0x05" name="SP_UCHE_ATOMICS"/>
+	<value value="0x06" name="SP_VS_TEX_INSTRUCTIONS"/>
+	<value value="0x07" name="SP_VS_CFLOW_INSTRUCTIONS"/>
+	<value value="0x08" name="SP_VS_EFU_INSTRUCTIONS"/>
+	<value value="0x09" name="SP_VS_FULL_ALU_INSTRUCTIONS"/>
+	<value value="0x0a" name="SP_VS_HALF_ALU_INSTRUCTIONS"/>
+	<value value="0x0b" name="SP_FS_TEX_INSTRUCTIONS"/>
+	<value value="0x0c" name="SP_FS_CFLOW_INSTRUCTIONS"/>
+	<value value="0x0d" name="SP_FS_EFU_INSTRUCTIONS"/>
+	<value value="0x0e" name="SP_FS_FULL_ALU_INSTRUCTIONS"/>
+	<value value="0x0f" name="SP_FS_HALF_ALU_INSTRUCTIONS"/>
+	<value value="0x10" name="SP_FS_BARY_INSTRUCTIONS"/>
+	<value value="0x11" name="SP_VS_INSTRUCTIONS"/>
+	<value value="0x12" name="SP_FS_INSTRUCTIONS"/>
+	<value value="0x13" name="SP_ADDR_LOCK_COUNT"/>
+	<value value="0x14" name="SP_UCHE_READ_TRANS"/>
+	<value value="0x15" name="SP_UCHE_WRITE_TRANS"/>
+	<value value="0x16" name="SP_EXPORT_VPC_TRANS"/>
+	<value value="0x17" name="SP_EXPORT_RB_TRANS"/>
+	<value value="0x18" name="SP_PIXELS_KILLED"/>
+	<value value="0x19" name="SP_ICL1_REQUESTS"/>
+	<value value="0x1a" name="SP_ICL1_MISSES"/>
+	<value value="0x1b" name="SP_ICL0_REQUESTS"/>
+	<value value="0x1c" name="SP_ICL0_MISSES"/>
+	<value value="0x1d" name="SP_ALU_ACTIVE_CYCLES"/>
+	<value value="0x1e" name="SP_EFU_ACTIVE_CYCLES"/>
+	<value value="0x1f" name="SP_STALL_CYCLES_BY_VPC"/>
+	<value value="0x20" name="SP_STALL_CYCLES_BY_TP"/>
+	<value value="0x21" name="SP_STALL_CYCLES_BY_UCHE"/>
+	<value value="0x22" name="SP_STALL_CYCLES_BY_RB"/>
+	<value value="0x23" name="SP_ACTIVE_CYCLES_ANY"/>
+	<value value="0x24" name="SP_ACTIVE_CYCLES_ALL"/>
+</enum>
+
+<enum name="a3xx_tp_perfcounter_select">
+	<value value="0x00" name="TPL1_TPPERF_L1_REQUESTS"/>
+	<value value="0x01" name="TPL1_TPPERF_TP0_L1_REQUESTS"/>
+	<value value="0x02" name="TPL1_TPPERF_TP0_L1_MISSES"/>
+	<value value="0x03" name="TPL1_TPPERF_TP1_L1_REQUESTS"/>
+	<value value="0x04" name="TPL1_TPPERF_TP1_L1_MISSES"/>
+	<value value="0x05" name="TPL1_TPPERF_TP2_L1_REQUESTS"/>
+	<value value="0x06" name="TPL1_TPPERF_TP2_L1_MISSES"/>
+	<value value="0x07" name="TPL1_TPPERF_TP3_L1_REQUESTS"/>
+	<value value="0x08" name="TPL1_TPPERF_TP3_L1_MISSES"/>
+	<value value="0x09" name="TPL1_TPPERF_OUTPUT_TEXELS_POINT"/>
+	<value value="0x0a" name="TPL1_TPPERF_OUTPUT_TEXELS_BILINEAR"/>
+	<value value="0x0b" name="TPL1_TPPERF_OUTPUT_TEXELS_MIP"/>
+	<value value="0x0c" name="TPL1_TPPERF_OUTPUT_TEXELS_ANISO"/>
+	<value value="0x0d" name="TPL1_TPPERF_BILINEAR_OPS"/>
+	<value value="0x0e" name="TPL1_TPPERF_QUADSQUADS_OFFSET"/>
+	<value value="0x0f" name="TPL1_TPPERF_QUADQUADS_SHADOW"/>
+	<value value="0x10" name="TPL1_TPPERF_QUADS_ARRAY"/>
+	<value value="0x11" name="TPL1_TPPERF_QUADS_PROJECTION"/>
+	<value value="0x12" name="TPL1_TPPERF_QUADS_GRADIENT"/>
+	<value value="0x13" name="TPL1_TPPERF_QUADS_1D2D"/>
+	<value value="0x14" name="TPL1_TPPERF_QUADS_3DCUBE"/>
+	<value value="0x15" name="TPL1_TPPERF_ZERO_LOD"/>
+	<value value="0x16" name="TPL1_TPPERF_OUTPUT_TEXELS"/>
+	<value value="0x17" name="TPL1_TPPERF_ACTIVE_CYCLES_ANY"/>
+	<value value="0x18" name="TPL1_TPPERF_ACTIVE_CYCLES_ALL"/>
+	<value value="0x19" name="TPL1_TPPERF_STALL_CYCLES_BY_ARB"/>
+	<value value="0x1a" name="TPL1_TPPERF_LATENCY"/>
+	<value value="0x1b" name="TPL1_TPPERF_LATENCY_TRANS"/>
+</enum>
+
+<enum name="a3xx_vfd_perfcounter_select">
+	<value value="0" name="VFD_PERF_UCHE_BYTE_FETCHED"/>
+	<value value="1" name="VFD_PERF_UCHE_TRANS"/>
+	<value value="2" name="VFD_PERF_VPC_BYPASS_COMPONENTS"/>
+	<value value="3" name="VFD_PERF_FETCH_INSTRUCTIONS"/>
+	<value value="4" name="VFD_PERF_DECODE_INSTRUCTIONS"/>
+	<value value="5" name="VFD_PERF_ACTIVE_CYCLES"/>
+	<value value="6" name="VFD_PERF_STALL_CYCLES_UCHE"/>
+	<value value="7" name="VFD_PERF_STALL_CYCLES_HLSQ"/>
+	<value value="8" name="VFD_PERF_STALL_CYCLES_VPC_BYPASS"/>
+	<value value="9" name="VFD_PERF_STALL_CYCLES_VPC_ALLOC"/>
+</enum>
+
+<enum name="a3xx_vpc_perfcounter_select">
+	<value value="0" name="VPC_PERF_SP_LM_PRIMITIVES"/>
+	<value value="1" name="VPC_PERF_COMPONENTS_FROM_SP"/>
+	<value value="2" name="VPC_PERF_SP_LM_COMPONENTS"/>
+	<value value="3" name="VPC_PERF_ACTIVE_CYCLES"/>
+	<value value="4" name="VPC_PERF_STALL_CYCLES_LM"/>
+	<value value="5" name="VPC_PERF_STALL_CYCLES_RAS"/>
+</enum>
+
+<enum name="a3xx_uche_perfcounter_select">
+	<value value="0x00" name="UCHE_UCHEPERF_VBIF_READ_BEATS_TP"/>
+	<value value="0x01" name="UCHE_UCHEPERF_VBIF_READ_BEATS_VFD"/>
+	<value value="0x02" name="UCHE_UCHEPERF_VBIF_READ_BEATS_HLSQ"/>
+	<value value="0x03" name="UCHE_UCHEPERF_VBIF_READ_BEATS_MARB"/>
+	<value value="0x04" name="UCHE_UCHEPERF_VBIF_READ_BEATS_SP"/>
+	<value value="0x08" name="UCHE_UCHEPERF_READ_REQUESTS_TP"/>
+	<value value="0x09" name="UCHE_UCHEPERF_READ_REQUESTS_VFD"/>
+	<value value="0x0a" name="UCHE_UCHEPERF_READ_REQUESTS_HLSQ"/>
+	<value value="0x0b" name="UCHE_UCHEPERF_READ_REQUESTS_MARB"/>
+	<value value="0x0c" name="UCHE_UCHEPERF_READ_REQUESTS_SP"/>
+	<value value="0x0d" name="UCHE_UCHEPERF_WRITE_REQUESTS_MARB"/>
+	<value value="0x0e" name="UCHE_UCHEPERF_WRITE_REQUESTS_SP"/>
+	<value value="0x0f" name="UCHE_UCHEPERF_TAG_CHECK_FAILS"/>
+	<value value="0x10" name="UCHE_UCHEPERF_EVICTS"/>
+	<value value="0x11" name="UCHE_UCHEPERF_FLUSHES"/>
+	<value value="0x12" name="UCHE_UCHEPERF_VBIF_LATENCY_CYCLES"/>
+	<value value="0x13" name="UCHE_UCHEPERF_VBIF_LATENCY_SAMPLES"/>
+	<value value="0x14" name="UCHE_UCHEPERF_ACTIVE_CYCLES"/>
+</enum>
+
+<enum name="a3xx_intp_mode">
+	<value name="SMOOTH" value="0"/>
+	<value name="FLAT" value="1"/>
+	<value name="ZERO" value="2"/>
+	<value name="ONE" value="3"/>
+</enum>
+
+<enum name="a3xx_repl_mode">
+	<value name="S" value="1"/>
+	<value name="T" value="2"/>
+	<value name="ONE_T" value="3"/>
+</enum>
+
+<domain name="A3XX" width="32">
+	<!-- RBBM registers -->
+	<reg32 offset="0x0000" name="RBBM_HW_VERSION"/>
+	<reg32 offset="0x0001" name="RBBM_HW_RELEASE"/>
+	<reg32 offset="0x0002" name="RBBM_HW_CONFIGURATION"/>
+	<reg32 offset="0x0010" name="RBBM_CLOCK_CTL"/>
+	<reg32 offset="0x0012" name="RBBM_SP_HYST_CNT"/>
+	<reg32 offset="0x0018" name="RBBM_SW_RESET_CMD"/>
+	<reg32 offset="0x0020" name="RBBM_AHB_CTL0"/>
+	<reg32 offset="0x0021" name="RBBM_AHB_CTL1"/>
+	<reg32 offset="0x0022" name="RBBM_AHB_CMD"/>
+	<reg32 offset="0x0027" name="RBBM_AHB_ERROR_STATUS"/>
+	<reg32 offset="0x002e" name="RBBM_GPR0_CTL"/>
+	<reg32 offset="0x0030" name="RBBM_STATUS">
+		<bitfield name="HI_BUSY" pos="0" type="boolean"/>
+		<bitfield name="CP_ME_BUSY" pos="1" type="boolean"/>
+		<bitfield name="CP_PFP_BUSY" pos="2" type="boolean"/>
+		<bitfield name="CP_NRT_BUSY" pos="14" type="boolean"/>
+		<bitfield name="VBIF_BUSY" pos="15" type="boolean"/>
+		<bitfield name="TSE_BUSY" pos="16" type="boolean"/>
+		<bitfield name="RAS_BUSY" pos="17" type="boolean"/>
+		<bitfield name="RB_BUSY" pos="18" type="boolean"/>
+		<bitfield name="PC_DCALL_BUSY" pos="19" type="boolean"/>
+		<bitfield name="PC_VSD_BUSY" pos="20" type="boolean"/>
+		<bitfield name="VFD_BUSY" pos="21" type="boolean"/>
+		<bitfield name="VPC_BUSY" pos="22" type="boolean"/>
+		<bitfield name="UCHE_BUSY" pos="23" type="boolean"/>
+		<bitfield name="SP_BUSY" pos="24" type="boolean"/>
+		<bitfield name="TPL1_BUSY" pos="25" type="boolean"/>
+		<bitfield name="MARB_BUSY" pos="26" type="boolean"/>
+		<bitfield name="VSC_BUSY" pos="27" type="boolean"/>
+		<bitfield name="ARB_BUSY" pos="28" type="boolean"/>
+		<bitfield name="HLSQ_BUSY" pos="29" type="boolean"/>
+		<bitfield name="GPU_BUSY_NOHC" pos="30" type="boolean"/>
+		<bitfield name="GPU_BUSY" pos="31" type="boolean"/>
+	</reg32>
+	<!-- used in fw CP_WAIT_FOR_IDLE, similar to NQWAIT_UNTIL on a2xx: -->
+	<reg32 offset="0x0040" name="RBBM_NQWAIT_UNTIL"/>
+	<reg32 offset="0x0033" name="RBBM_WAIT_IDLE_CLOCKS_CTL"/>
+	<reg32 offset="0x0050" name="RBBM_INTERFACE_HANG_INT_CTL"/>
+	<reg32 offset="0x0051" name="RBBM_INTERFACE_HANG_MASK_CTL0"/>
+	<reg32 offset="0x0054" name="RBBM_INTERFACE_HANG_MASK_CTL1"/>
+	<reg32 offset="0x0057" name="RBBM_INTERFACE_HANG_MASK_CTL2"/>
+	<reg32 offset="0x005a" name="RBBM_INTERFACE_HANG_MASK_CTL3"/>
+
+	<bitset name="A3XX_INT0">
+		<bitfield name="RBBM_GPU_IDLE" pos="0" type="boolean"/>
+		<bitfield name="RBBM_AHB_ERROR" pos="1" type="boolean"/>
+		<bitfield name="RBBM_REG_TIMEOUT" pos="2" type="boolean"/>
+		<bitfield name="RBBM_ME_MS_TIMEOUT" pos="3" type="boolean"/>
+		<bitfield name="RBBM_PFP_MS_TIMEOUT" pos="4" type="boolean"/>
+		<bitfield name="RBBM_ATB_BUS_OVERFLOW" pos="5" type="boolean"/>
+		<bitfield name="VFD_ERROR" pos="6" type="boolean"/>
+		<bitfield name="CP_SW_INT" pos="7" type="boolean"/>
+		<bitfield name="CP_T0_PACKET_IN_IB" pos="8" type="boolean"/>
+		<bitfield name="CP_OPCODE_ERROR" pos="9" type="boolean"/>
+		<bitfield name="CP_RESERVED_BIT_ERROR" pos="10" type="boolean"/>
+		<bitfield name="CP_HW_FAULT" pos="11" type="boolean"/>
+		<bitfield name="CP_DMA" pos="12" type="boolean"/>
+		<bitfield name="CP_IB2_INT" pos="13" type="boolean"/>
+		<bitfield name="CP_IB1_INT" pos="14" type="boolean"/>
+		<bitfield name="CP_RB_INT" pos="15" type="boolean"/>
+		<bitfield name="CP_REG_PROTECT_FAULT" pos="16" type="boolean"/>
+		<bitfield name="CP_RB_DONE_TS" pos="17" type="boolean"/>
+		<bitfield name="CP_VS_DONE_TS" pos="18" type="boolean"/>
+		<bitfield name="CP_PS_DONE_TS" pos="19" type="boolean"/>
+		<bitfield name="CACHE_FLUSH_TS" pos="20" type="boolean"/>
+		<bitfield name="CP_AHB_ERROR_HALT" pos="21" type="boolean"/>
+		<bitfield name="MISC_HANG_DETECT" pos="24" type="boolean"/>
+		<bitfield name="UCHE_OOB_ACCESS" pos="25" type="boolean"/>
+	</bitset>
+
+
+	<!--
+		set in pm4 fw INVALID_JUMP_TABLE_ENTRY and CP_INTERRUPT (compare
+		to CP_INT_STATUS in a2xx firmware), so this seems to be the a3xx
+		way for fw to raise and irq:
+	 -->
+	<reg32 offset="0x0060" name="RBBM_INT_SET_CMD" type="A3XX_INT0"/>
+	<reg32 offset="0x0061" name="RBBM_INT_CLEAR_CMD" type="A3XX_INT0"/>
+	<reg32 offset="0x0063" name="RBBM_INT_0_MASK" type="A3XX_INT0"/>
+	<reg32 offset="0x0064" name="RBBM_INT_0_STATUS" type="A3XX_INT0"/>
+	<reg32 offset="0x0080" name="RBBM_PERFCTR_CTL">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0081" name="RBBM_PERFCTR_LOAD_CMD0"/>
+	<reg32 offset="0x0082" name="RBBM_PERFCTR_LOAD_CMD1"/>
+	<reg32 offset="0x0084" name="RBBM_PERFCTR_LOAD_VALUE_LO"/>
+	<reg32 offset="0x0085" name="RBBM_PERFCTR_LOAD_VALUE_HI"/>
+	<reg32 offset="0x0086" name="RBBM_PERFCOUNTER0_SELECT" type="a3xx_rbbm_perfcounter_select"/>
+	<reg32 offset="0x0087" name="RBBM_PERFCOUNTER1_SELECT" type="a3xx_rbbm_perfcounter_select"/>
+	<reg32 offset="0x0088" name="RBBM_GPU_BUSY_MASKED"/>
+	<reg32 offset="0x0090" name="RBBM_PERFCTR_CP_0_LO"/>
+	<reg32 offset="0x0091" name="RBBM_PERFCTR_CP_0_HI"/>
+	<reg32 offset="0x0092" name="RBBM_PERFCTR_RBBM_0_LO"/>
+	<reg32 offset="0x0093" name="RBBM_PERFCTR_RBBM_0_HI"/>
+	<reg32 offset="0x0094" name="RBBM_PERFCTR_RBBM_1_LO"/>
+	<reg32 offset="0x0095" name="RBBM_PERFCTR_RBBM_1_HI"/>
+	<reg32 offset="0x0096" name="RBBM_PERFCTR_PC_0_LO"/>
+	<reg32 offset="0x0097" name="RBBM_PERFCTR_PC_0_HI"/>
+	<reg32 offset="0x0098" name="RBBM_PERFCTR_PC_1_LO"/>
+	<reg32 offset="0x0099" name="RBBM_PERFCTR_PC_1_HI"/>
+	<reg32 offset="0x009a" name="RBBM_PERFCTR_PC_2_LO"/>
+	<reg32 offset="0x009b" name="RBBM_PERFCTR_PC_2_HI"/>
+	<reg32 offset="0x009c" name="RBBM_PERFCTR_PC_3_LO"/>
+	<reg32 offset="0x009d" name="RBBM_PERFCTR_PC_3_HI"/>
+	<reg32 offset="0x009e" name="RBBM_PERFCTR_VFD_0_LO"/>
+	<reg32 offset="0x009f" name="RBBM_PERFCTR_VFD_0_HI"/>
+	<reg32 offset="0x00a0" name="RBBM_PERFCTR_VFD_1_LO"/>
+	<reg32 offset="0x00a1" name="RBBM_PERFCTR_VFD_1_HI"/>
+	<reg32 offset="0x00a2" name="RBBM_PERFCTR_HLSQ_0_LO"/>
+	<reg32 offset="0x00a3" name="RBBM_PERFCTR_HLSQ_0_HI"/>
+	<reg32 offset="0x00a4" name="RBBM_PERFCTR_HLSQ_1_LO"/>
+	<reg32 offset="0x00a5" name="RBBM_PERFCTR_HLSQ_1_HI"/>
+	<reg32 offset="0x00a6" name="RBBM_PERFCTR_HLSQ_2_LO"/>
+	<reg32 offset="0x00a7" name="RBBM_PERFCTR_HLSQ_2_HI"/>
+	<reg32 offset="0x00a8" name="RBBM_PERFCTR_HLSQ_3_LO"/>
+	<reg32 offset="0x00a9" name="RBBM_PERFCTR_HLSQ_3_HI"/>
+	<reg32 offset="0x00aa" name="RBBM_PERFCTR_HLSQ_4_LO"/>
+	<reg32 offset="0x00ab" name="RBBM_PERFCTR_HLSQ_4_HI"/>
+	<reg32 offset="0x00ac" name="RBBM_PERFCTR_HLSQ_5_LO"/>
+	<reg32 offset="0x00ad" name="RBBM_PERFCTR_HLSQ_5_HI"/>
+	<reg32 offset="0x00ae" name="RBBM_PERFCTR_VPC_0_LO"/>
+	<reg32 offset="0x00af" name="RBBM_PERFCTR_VPC_0_HI"/>
+	<reg32 offset="0x00b0" name="RBBM_PERFCTR_VPC_1_LO"/>
+	<reg32 offset="0x00b1" name="RBBM_PERFCTR_VPC_1_HI"/>
+	<reg32 offset="0x00b2" name="RBBM_PERFCTR_TSE_0_LO"/>
+	<reg32 offset="0x00b3" name="RBBM_PERFCTR_TSE_0_HI"/>
+	<reg32 offset="0x00b4" name="RBBM_PERFCTR_TSE_1_LO"/>
+	<reg32 offset="0x00b5" name="RBBM_PERFCTR_TSE_1_HI"/>
+	<reg32 offset="0x00b6" name="RBBM_PERFCTR_RAS_0_LO"/>
+	<reg32 offset="0x00b7" name="RBBM_PERFCTR_RAS_0_HI"/>
+	<reg32 offset="0x00b8" name="RBBM_PERFCTR_RAS_1_LO"/>
+	<reg32 offset="0x00b9" name="RBBM_PERFCTR_RAS_1_HI"/>
+	<reg32 offset="0x00ba" name="RBBM_PERFCTR_UCHE_0_LO"/>
+	<reg32 offset="0x00bb" name="RBBM_PERFCTR_UCHE_0_HI"/>
+	<reg32 offset="0x00bc" name="RBBM_PERFCTR_UCHE_1_LO"/>
+	<reg32 offset="0x00bd" name="RBBM_PERFCTR_UCHE_1_HI"/>
+	<reg32 offset="0x00be" name="RBBM_PERFCTR_UCHE_2_LO"/>
+	<reg32 offset="0x00bf" name="RBBM_PERFCTR_UCHE_2_HI"/>
+	<reg32 offset="0x00c0" name="RBBM_PERFCTR_UCHE_3_LO"/>
+	<reg32 offset="0x00c1" name="RBBM_PERFCTR_UCHE_3_HI"/>
+	<reg32 offset="0x00c2" name="RBBM_PERFCTR_UCHE_4_LO"/>
+	<reg32 offset="0x00c3" name="RBBM_PERFCTR_UCHE_4_HI"/>
+	<reg32 offset="0x00c4" name="RBBM_PERFCTR_UCHE_5_LO"/>
+	<reg32 offset="0x00c5" name="RBBM_PERFCTR_UCHE_5_HI"/>
+	<reg32 offset="0x00c6" name="RBBM_PERFCTR_TP_0_LO"/>
+	<reg32 offset="0x00c7" name="RBBM_PERFCTR_TP_0_HI"/>
+	<reg32 offset="0x00c8" name="RBBM_PERFCTR_TP_1_LO"/>
+	<reg32 offset="0x00c9" name="RBBM_PERFCTR_TP_1_HI"/>
+	<reg32 offset="0x00ca" name="RBBM_PERFCTR_TP_2_LO"/>
+	<reg32 offset="0x00cb" name="RBBM_PERFCTR_TP_2_HI"/>
+	<reg32 offset="0x00cc" name="RBBM_PERFCTR_TP_3_LO"/>
+	<reg32 offset="0x00cd" name="RBBM_PERFCTR_TP_3_HI"/>
+	<reg32 offset="0x00ce" name="RBBM_PERFCTR_TP_4_LO"/>
+	<reg32 offset="0x00cf" name="RBBM_PERFCTR_TP_4_HI"/>
+	<reg32 offset="0x00d0" name="RBBM_PERFCTR_TP_5_LO"/>
+	<reg32 offset="0x00d1" name="RBBM_PERFCTR_TP_5_HI"/>
+	<reg32 offset="0x00d2" name="RBBM_PERFCTR_SP_0_LO"/>
+	<reg32 offset="0x00d3" name="RBBM_PERFCTR_SP_0_HI"/>
+	<reg32 offset="0x00d4" name="RBBM_PERFCTR_SP_1_LO"/>
+	<reg32 offset="0x00d5" name="RBBM_PERFCTR_SP_1_HI"/>
+	<reg32 offset="0x00d6" name="RBBM_PERFCTR_SP_2_LO"/>
+	<reg32 offset="0x00d7" name="RBBM_PERFCTR_SP_2_HI"/>
+	<reg32 offset="0x00d8" name="RBBM_PERFCTR_SP_3_LO"/>
+	<reg32 offset="0x00d9" name="RBBM_PERFCTR_SP_3_HI"/>
+	<reg32 offset="0x00da" name="RBBM_PERFCTR_SP_4_LO"/>
+	<reg32 offset="0x00db" name="RBBM_PERFCTR_SP_4_HI"/>
+	<reg32 offset="0x00dc" name="RBBM_PERFCTR_SP_5_LO"/>
+	<reg32 offset="0x00dd" name="RBBM_PERFCTR_SP_5_HI"/>
+	<reg32 offset="0x00de" name="RBBM_PERFCTR_SP_6_LO"/>
+	<reg32 offset="0x00df" name="RBBM_PERFCTR_SP_6_HI"/>
+	<reg32 offset="0x00e0" name="RBBM_PERFCTR_SP_7_LO"/>
+	<reg32 offset="0x00e1" name="RBBM_PERFCTR_SP_7_HI"/>
+	<reg32 offset="0x00e2" name="RBBM_PERFCTR_RB_0_LO"/>
+	<reg32 offset="0x00e3" name="RBBM_PERFCTR_RB_0_HI"/>
+	<reg32 offset="0x00e4" name="RBBM_PERFCTR_RB_1_LO"/>
+	<reg32 offset="0x00e5" name="RBBM_PERFCTR_RB_1_HI"/>
+	<reg32 offset="0x00ea" name="RBBM_PERFCTR_PWR_0_LO"/>
+	<reg32 offset="0x00eb" name="RBBM_PERFCTR_PWR_0_HI"/>
+	<reg32 offset="0x00ec" name="RBBM_PERFCTR_PWR_1_LO"/>
+	<reg32 offset="0x00ed" name="RBBM_PERFCTR_PWR_1_HI"/>
+	<reg32 offset="0x0100" name="RBBM_RBBM_CTL"/>
+	<reg32 offset="0x0111" name="RBBM_DEBUG_BUS_CTL"/>
+	<reg32 offset="0x0112" name="RBBM_DEBUG_BUS_DATA_STATUS"/>
+
+	<!-- CP registers -->
+	<reg32 offset="0x01c9" name="CP_PFP_UCODE_ADDR"/>
+	<reg32 offset="0x01ca" name="CP_PFP_UCODE_DATA"/>
+	<reg32 offset="0x01cc" name="CP_ROQ_ADDR"/>
+	<reg32 offset="0x01cd" name="CP_ROQ_DATA"/>
+	<reg32 offset="0x01d1" name="CP_MERCIU_ADDR"/>
+	<reg32 offset="0x01d2" name="CP_MERCIU_DATA"/>
+	<reg32 offset="0x01d3" name="CP_MERCIU_DATA2"/>
+	<!-- see a3xx_snapshot_cp_meq().. looks like the way to dump queue between pfp and pm4 -->
+	<reg32 offset="0x01da" name="CP_MEQ_ADDR"/>
+	<reg32 offset="0x01db" name="CP_MEQ_DATA"/>
+	<reg32 offset="0x01f5" name="CP_WFI_PEND_CTR"/>
+	<reg32 offset="0x039d" name="RBBM_PM_OVERRIDE2"/>
+
+	<reg32 offset="0x0445" name="CP_PERFCOUNTER_SELECT" type="a3xx_cp_perfcounter_select"/>
+	<reg32 offset="0x045c" name="CP_HW_FAULT"/>
+	<reg32 offset="0x045e" name="CP_PROTECT_CTRL"/>
+	<reg32 offset="0x045f" name="CP_PROTECT_STATUS"/>
+	<array offset="0x0460" name="CP_PROTECT" stride="1" length="16">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<reg32 offset="0x054d" name="CP_AHB_FAULT"/>
+
+	<reg32 offset="0x0d00" name="SQ_GPR_MANAGEMENT"/>
+	<reg32 offset="0x0d02" name="SQ_INST_STORE_MANAGMENT"/>
+	<reg32 offset="0x0e1e" name="TP0_CHICKEN"/>
+
+	<!-- these I guess or either SP or HLSQ since related to shader core setup: -->
+	<reg32 offset="0x0e22" name="SP_GLOBAL_MEM_SIZE" type="uint">
+		<doc>
+			The pair of MEM_SIZE/ADDR registers get programmed
+			in sequence with the size/addr of each buffer.
+		</doc>
+	</reg32>
+	<reg32 offset="0x0e23" name="SP_GLOBAL_MEM_ADDR"/>
+
+	<!-- GRAS registers -->
+	<reg32 offset="0x2040" name="GRAS_CL_CLIP_CNTL">
+		<bitfield name="IJ_PERSP_CENTER" pos="12" type="boolean"/>
+		<bitfield name="IJ_NON_PERSP_CENTER" pos="13" type="boolean"/>
+		<bitfield name="IJ_PERSP_CENTROID" pos="14" type="boolean"/>
+		<bitfield name="IJ_NON_PERSP_CENTROID" pos="15" type="boolean"/>
+		<bitfield name="CLIP_DISABLE" pos="16" type="boolean"/>
+		<bitfield name="ZFAR_CLIP_DISABLE" pos="17" type="boolean"/>
+		<bitfield name="VP_CLIP_CODE_IGNORE" pos="19" type="boolean"/>
+		<bitfield name="VP_XFORM_DISABLE" pos="20" type="boolean"/>
+		<bitfield name="PERSP_DIVISION_DISABLE" pos="21" type="boolean"/>
+		<bitfield name="ZERO_GB_SCALE_Z" pos="22" type="boolean">
+			<doc>aka clip_halfz</doc>
+		</bitfield>
+		<!-- set when gl_FragCoord.z is enabled in frag shader: -->
+		<bitfield name="ZCOORD" pos="23" type="boolean"/>
+		<bitfield name="WCOORD" pos="24" type="boolean"/>
+		<!-- set when frag shader writes z (so early z test disabled: -->
+		<bitfield name="ZCLIP_DISABLE" pos="25" type="boolean"/>
+		<bitfield name="NUM_USER_CLIP_PLANES" low="26" high="28" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2044" name="GRAS_CL_GB_CLIP_ADJ">
+		<bitfield name="HORZ" low="0" high="9" type="uint"/>
+		<bitfield name="VERT" low="10" high="19" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2048" name="GRAS_CL_VPORT_XOFFSET" type="float"/>
+	<reg32 offset="0x2049" name="GRAS_CL_VPORT_XSCALE" type="float"/>
+	<reg32 offset="0x204a" name="GRAS_CL_VPORT_YOFFSET" type="float"/>
+	<reg32 offset="0x204b" name="GRAS_CL_VPORT_YSCALE" type="float"/>
+	<reg32 offset="0x204c" name="GRAS_CL_VPORT_ZOFFSET" type="float"/>
+	<reg32 offset="0x204d" name="GRAS_CL_VPORT_ZSCALE" type="float"/>
+	<reg32 offset="0x2068" name="GRAS_SU_POINT_MINMAX">
+		<bitfield name="MIN" low="0" high="15" type="ufixed" radix="4"/>
+		<bitfield name="MAX" low="16" high="31" type="ufixed" radix="4"/>
+	</reg32>
+	<reg32 offset="0x2069" name="GRAS_SU_POINT_SIZE" type="fixed" radix="4"/>
+	<reg32 offset="0x206c" name="GRAS_SU_POLY_OFFSET_SCALE">
+		<bitfield name="VAL" low="0" high="23" type="fixed" radix="20"/>
+		<doc>range of -8.0 to 8.0</doc>
+	</reg32>
+	<reg32 offset="0x206d" name="GRAS_SU_POLY_OFFSET_OFFSET" radix="6" type="fixed">
+		<doc>range of -512.0 to 512.0</doc>
+	</reg32>
+	<reg32 offset="0x2070" name="GRAS_SU_MODE_CONTROL">
+		<bitfield name="CULL_FRONT" pos="0" type="boolean"/>
+		<bitfield name="CULL_BACK" pos="1" type="boolean"/>
+		<bitfield name="FRONT_CW" pos="2" type="boolean"/>
+		<bitfield name="LINEHALFWIDTH" low="3" high="10" radix="2" type="fixed"/>
+		<bitfield name="POLY_OFFSET" pos="11" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2072" name="GRAS_SC_CONTROL">
+		<!-- complete wild-ass-guess for sizes of these bitfields.. -->
+		<bitfield name="RENDER_MODE" low="4" high="7" type="a3xx_render_mode"/>
+		<bitfield name="MSAA_SAMPLES" low="8" high="11" type="a3xx_msaa_samples"/>
+		<bitfield name="RASTER_MODE" low="12" high="15"/>
+	</reg32>
+
+	<reg32 offset="0x2074" name="GRAS_SC_SCREEN_SCISSOR_TL" type="adreno_reg_xy"/>
+	<reg32 offset="0x2075" name="GRAS_SC_SCREEN_SCISSOR_BR" type="adreno_reg_xy"/>
+	<reg32 offset="0x2079" name="GRAS_SC_WINDOW_SCISSOR_TL" type="adreno_reg_xy"/>
+	<reg32 offset="0x207a" name="GRAS_SC_WINDOW_SCISSOR_BR" type="adreno_reg_xy"/>
+
+	<!-- RB registers -->
+	<reg32 offset="0x20c0" name="RB_MODE_CONTROL">
+		<!-- guess on the # of bits here.. -->
+		<bitfield name="GMEM_BYPASS" pos="7" type="boolean"/>
+		<doc>
+			RENDER_MODE is RB_RESOLVE_PASS for gmem->mem, otherwise RB_RENDER_PASS
+		</doc>
+		<bitfield name="RENDER_MODE" low="8" high="10" type="a3xx_render_mode"/>
+		<bitfield name="MRT" low="12" high="13" type="uint">
+			<doc>render targets - 1</doc>
+		</bitfield>
+		<bitfield name="MARB_CACHE_SPLIT_MODE" pos="15" type="boolean"/>
+		<bitfield name="PACKER_TIMER_ENABLE" pos="16" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x20c1" name="RB_RENDER_CONTROL">
+		<bitfield name="DUAL_COLOR_IN_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="YUV_IN_ENABLE" pos="1" type="boolean"/>
+		<bitfield name="COV_VALUE_INPUT_ENABLE" pos="2" type="boolean"/>
+		<!-- set when gl_FrontFacing is accessed in frag shader: -->
+		<bitfield name="FACENESS" pos="3" type="boolean"/>
+		<bitfield name="BIN_WIDTH" low="4" high="11" shr="5" type="uint"/>
+		<bitfield name="DISABLE_COLOR_PIPE" pos="12" type="boolean"/>
+		<!--
+			ENABLE_GMEM not set on mem2gmem..  so possibly it is actually
+			controlling blend or readback from GMEM??
+		 -->
+		<bitfield name="ENABLE_GMEM" pos="13" type="boolean"/>
+		<bitfield name="COORD_MASK" low="14" high="17" type="hex"/>
+		<bitfield name="I_CLAMP_ENABLE" pos="19" type="boolean"/>
+		<bitfield name="COV_VALUE_OUTPUT_ENABLE" pos="20" type="boolean"/>
+		<bitfield name="ALPHA_TEST" pos="22" type="boolean"/>
+		<bitfield name="ALPHA_TEST_FUNC" low="24" high="26" type="adreno_compare_func"/>
+		<bitfield name="ALPHA_TO_COVERAGE" pos="30" type="boolean"/>
+		<bitfield name="ALPHA_TO_ONE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x20c2" name="RB_MSAA_CONTROL">
+		<bitfield name="DISABLE" pos="10" type="boolean"/>
+		<bitfield name="SAMPLES" low="12" high="15" type="a3xx_msaa_samples"/>
+		<bitfield name="SAMPLE_MASK" low="16" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="0x20c3" name="RB_ALPHA_REF">
+		<bitfield name="UINT" low="8" high="15" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+	<array offset="0x20c4" name="RB_MRT" stride="4" length="4">
+		<reg32 offset="0x0" name="CONTROL">
+			<bitfield name="READ_DEST_ENABLE" pos="3" type="boolean"/>
+			<!-- both these bits seem to get set when enabling GL_BLEND.. -->
+			<bitfield name="BLEND" pos="4" type="boolean"/>
+			<bitfield name="BLEND2" pos="5" type="boolean"/>
+			<bitfield name="ROP_CODE" low="8" high="11" type="a3xx_rop_code"/>
+			<bitfield name="DITHER_MODE" low="12" high="13" type="adreno_rb_dither_mode"/>
+			<bitfield name="COMPONENT_ENABLE" low="24" high="27" type="hex"/>
+		</reg32>
+		<reg32 offset="0x1" name="BUF_INFO">
+			<bitfield name="COLOR_FORMAT" low="0" high="5" type="a3xx_color_fmt"/>
+			<bitfield name="COLOR_TILE_MODE" low="6" high="7" type="a3xx_tile_mode"/>
+			<bitfield name="COLOR_SWAP" low="10" high="11" type="a3xx_color_swap"/>
+			<bitfield name="COLOR_SRGB" pos="14" type="boolean"/>
+			<doc>
+				Pitch (actually, appears to be pitch in bytes, so really is a stride)
+				in GMEM, so pitch of the current tile.
+			</doc>
+			<bitfield name="COLOR_BUF_PITCH" low="17" high="31" shr="5" type="uint"/>
+		</reg32>
+		<reg32 offset="0x2" name="BUF_BASE">
+			<doc>offset into GMEM (or system memory address in bypass mode)</doc>
+			<bitfield name="COLOR_BUF_BASE" low="4" high="31" shr="5" type="hex"/>
+		</reg32>
+		<reg32 offset="0x3" name="BLEND_CONTROL">
+			<bitfield name="RGB_SRC_FACTOR" low="0" high="4" type="adreno_rb_blend_factor"/>
+			<bitfield name="RGB_BLEND_OPCODE" low="5" high="7" type="a3xx_rb_blend_opcode"/>
+			<bitfield name="RGB_DEST_FACTOR" low="8" high="12" type="adreno_rb_blend_factor"/>
+			<bitfield name="ALPHA_SRC_FACTOR" low="16" high="20" type="adreno_rb_blend_factor"/>
+			<bitfield name="ALPHA_BLEND_OPCODE" low="21" high="23" type="a3xx_rb_blend_opcode"/>
+			<bitfield name="ALPHA_DEST_FACTOR" low="24" high="28" type="adreno_rb_blend_factor"/>
+			<bitfield name="CLAMP_ENABLE" pos="29" type="boolean"/>
+		</reg32>
+	</array>
+
+	<reg32 offset="0x20e4" name="RB_BLEND_RED">
+		<bitfield name="UINT" low="0" high="7" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+	<reg32 offset="0x20e5" name="RB_BLEND_GREEN">
+		<bitfield name="UINT" low="0" high="7" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+	<reg32 offset="0x20e6" name="RB_BLEND_BLUE">
+		<bitfield name="UINT" low="0" high="7" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+	<reg32 offset="0x20e7" name="RB_BLEND_ALPHA">
+		<bitfield name="UINT" low="0" high="7" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+
+	<reg32 offset="0x20e8" name="RB_CLEAR_COLOR_DW0"/>
+	<reg32 offset="0x20e9" name="RB_CLEAR_COLOR_DW1"/>
+	<reg32 offset="0x20ea" name="RB_CLEAR_COLOR_DW2"/>
+	<reg32 offset="0x20eb" name="RB_CLEAR_COLOR_DW3"/>
+	<reg32 offset="0x20ec" name="RB_COPY_CONTROL">
+		<!-- not sure # of bits -->
+		<bitfield name="MSAA_RESOLVE" low="0" high="1" type="a3xx_msaa_samples"/>
+		<bitfield name="DEPTHCLEAR" pos="3" type="boolean"/>
+		<bitfield name="MODE" low="4" high="6" type="adreno_rb_copy_control_mode"/>
+		<bitfield name="MSAA_SRGB_DOWNSAMPLE" pos="7" type="boolean"/>
+		<bitfield name="FASTCLEAR" low="8" high="11" type="hex"/>
+		<bitfield name="DEPTH32_RESOLVE" pos="12" type="boolean"/> <!-- enabled on a Z32F copy -->
+		<bitfield name="GMEM_BASE" low="14" high="31" shr="14" type="hex"/>
+	</reg32>
+	<reg32 offset="0x20ed" name="RB_COPY_DEST_BASE">
+		<bitfield name="BASE" low="4" high="31" shr="5" type="hex"/>
+	</reg32>
+	<reg32 offset="0x20ee" name="RB_COPY_DEST_PITCH">
+		<doc>actually, appears to be pitch in bytes, so really is a stride</doc>
+		<!-- not actually sure about max pitch... -->
+		<bitfield name="PITCH" low="0" high="31" shr="5" type="uint"/>
+	</reg32>
+	<reg32 offset="0x20ef" name="RB_COPY_DEST_INFO">
+		<bitfield name="TILE" low="0" high="1" type="a3xx_tile_mode"/>
+		<bitfield name="FORMAT" low="2" high="7" type="a3xx_color_fmt"/>
+		<bitfield name="SWAP" low="8" high="9" type="a3xx_color_swap"/>
+		<bitfield name="DITHER_MODE" low="10" high="11" type="adreno_rb_dither_mode"/>
+		<bitfield name="COMPONENT_ENABLE" low="14" high="17" type="hex"/>
+		<bitfield name="ENDIAN" low="18" high="20" type="adreno_rb_surface_endian"/>
+	</reg32>
+	<reg32 offset="0x2100" name="RB_DEPTH_CONTROL">
+		<!--
+			guessing that this matches a2xx with the stencil fields
+			moved out into RB_STENCIL_CONTROL?
+		 -->
+		<bitfield name="FRAG_WRITES_Z" pos="0" type="boolean"/>
+		<bitfield name="Z_TEST_ENABLE" pos="1" type="boolean"/>
+		<bitfield name="Z_WRITE_ENABLE" pos="2" type="boolean"/>
+		<bitfield name="EARLY_Z_DISABLE" pos="3" type="boolean"/>
+		<bitfield name="ZFUNC" low="4" high="6" type="adreno_compare_func"/>
+		<bitfield name="Z_CLAMP_ENABLE" pos="7" type="boolean"/>
+		<doc>Z_READ_ENABLE bit is set for zfunc other than GL_ALWAYS or GL_NEVER</doc>
+		<bitfield name="Z_READ_ENABLE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2101" name="RB_DEPTH_CLEAR">
+		<doc>seems to be always set to 0x00000000</doc>
+	</reg32>
+	<reg32 offset="0x2102" name="RB_DEPTH_INFO">
+		<bitfield name="DEPTH_FORMAT" low="0" high="1" type="adreno_rb_depth_format"/>
+		<doc>
+			DEPTH_BASE is offset in GMEM to depth/stencil buffer, ie
+			bin_w * bin_h / 1024 (possible rounded up to multiple of
+			something??  ie. 39 becomes 40, 78 becomes 80.. 75 becomes
+			80.. so maybe it needs to be multiple of 8??
+		</doc>
+		<bitfield name="DEPTH_BASE" low="11" high="31" shr="12" type="hex"/>
+	</reg32>
+	<reg32 offset="0x2103" name="RB_DEPTH_PITCH" shr="3" type="uint">
+		<doc>
+			Pitch of depth buffer or combined depth+stencil buffer
+			in z24s8 cases.
+		</doc>
+	</reg32>
+	<reg32 offset="0x2104" name="RB_STENCIL_CONTROL">
+		<bitfield name="STENCIL_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="STENCIL_ENABLE_BF" pos="1" type="boolean"/>
+		<!--
+			set for stencil operations that require read from stencil
+			buffer, but not for example for stencil clear (which does
+			not require read).. so guessing this is analogous to
+			READ_DEST_ENABLE for color buffer..
+		 -->
+		<bitfield name="STENCIL_READ" pos="2" type="boolean"/>
+		<bitfield name="FUNC" low="8" high="10" type="adreno_compare_func"/>
+		<bitfield name="FAIL" low="11" high="13" type="adreno_stencil_op"/>
+		<bitfield name="ZPASS" low="14" high="16" type="adreno_stencil_op"/>
+		<bitfield name="ZFAIL" low="17" high="19" type="adreno_stencil_op"/>
+		<bitfield name="FUNC_BF" low="20" high="22" type="adreno_compare_func"/>
+		<bitfield name="FAIL_BF" low="23" high="25" type="adreno_stencil_op"/>
+		<bitfield name="ZPASS_BF" low="26" high="28" type="adreno_stencil_op"/>
+		<bitfield name="ZFAIL_BF" low="29" high="31" type="adreno_stencil_op"/>
+	</reg32>
+	<reg32 offset="0x2105" name="RB_STENCIL_CLEAR">
+		<doc>seems to be always set to 0x00000000</doc>
+	</reg32>
+	<reg32 offset="0x2106" name="RB_STENCIL_INFO">
+		<doc>Base address for stencil when not using interleaved depth/stencil</doc>
+		<bitfield name="STENCIL_BASE" low="11" high="31" shr="12" type="hex"/>
+	</reg32>
+	<reg32 offset="0x2107" name="RB_STENCIL_PITCH" shr="3" type="uint">
+		<doc>pitch of stencil buffer when not using interleaved depth/stencil</doc>
+	</reg32>
+	<reg32 offset="0x2108" name="RB_STENCILREFMASK" type="adreno_rb_stencilrefmask"/>
+	<reg32 offset="0x2109" name="RB_STENCILREFMASK_BF" type="adreno_rb_stencilrefmask"/>
+	<!-- VSC == visibility stream c?? -->
+	<reg32 offset="0x210c" name="RB_LRZ_VSC_CONTROL">
+		<doc>seems to be set to 0x00000002 during binning pass</doc>
+		<bitfield name="BINNING_ENABLE" pos="1" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x210e" name="RB_WINDOW_OFFSET">
+		<doc>X/Y offset of current bin</doc>
+		<bitfield name="X" low="0" high="15" type="uint"/>
+		<bitfield name="Y" low="16" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2110" name="RB_SAMPLE_COUNT_CONTROL">
+		<bitfield name="RESET" pos="0" type="boolean"/>
+		<bitfield name="COPY" pos="1" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2111" name="RB_SAMPLE_COUNT_ADDR"/>
+	<reg32 offset="0x2114" name="RB_Z_CLAMP_MIN"/>
+	<reg32 offset="0x2115" name="RB_Z_CLAMP_MAX"/>
+
+	<!-- PC registers -->
+	<reg32 offset="0x21e1" name="VGT_BIN_BASE">
+		<doc>
+			seems to be where firmware writes BIN_DATA_ADDR from
+			CP_SET_BIN_DATA packet..  probably should be called
+			PC_BIN_BASE (just using name from yamato for now)
+		</doc>
+	</reg32>
+	<reg32 offset="0x21e2" name="VGT_BIN_SIZE">
+		<doc>probably should be PC_BIN_SIZE</doc>
+	</reg32>
+	<reg32 offset="0x21e4" name="PC_VSTREAM_CONTROL">
+		<doc>SIZE is current pipe width * height (in tiles)</doc>
+		<bitfield name="SIZE" low="16" high="21" type="uint"/>
+		<doc>
+			N is some sort of slot # between 0..(SIZE-1).  In case
+			multiple tiles use same pipe, each tile gets unique slot #
+		</doc>
+		<bitfield name="N" low="22" high="26" type="uint"/>
+	</reg32>
+	<reg32 offset="0x21ea" name="PC_VERTEX_REUSE_BLOCK_CNTL"/>
+	<reg32 offset="0x21ec" name="PC_PRIM_VTX_CNTL">
+		<doc>
+			STRIDE_IN_VPC: ALIGN(next_outloc - 8, 4) / 4
+			(but, in cases where you'd expect 1, the blob driver uses
+			2, so possibly 0 (no varying) or minimum of 2)
+		</doc>
+		<bitfield name="STRIDE_IN_VPC" low="0" high="4" type="uint"/>
+		<bitfield name="POLYMODE_FRONT_PTYPE" low="5" high="7" type="adreno_pa_su_sc_draw"/>
+		<bitfield name="POLYMODE_BACK_PTYPE" low="8" high="10" type="adreno_pa_su_sc_draw"/>
+		<bitfield name="POLYMODE_ENABLE" pos="12" type="boolean"/>
+		<bitfield name="PRIMITIVE_RESTART" pos="20" type="boolean"/>
+		<bitfield name="PROVOKING_VTX_LAST" pos="25" type="boolean"/>
+		<!-- PSIZE bit set if gl_PointSize written: -->
+		<bitfield name="PSIZE" pos="26" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x21ed" name="PC_RESTART_INDEX"/>
+
+	<!-- HLSQ registers -->
+	<bitset name="a3xx_hlsq_vs_fs_control_reg" inline="yes">
+		<bitfield name="CONSTLENGTH" low="0" high="9" type="uint"/>
+		<bitfield name="CONSTSTARTOFFSET" low="12" high="20" type="uint"/>
+		<bitfield name="INSTRLENGTH" low="24" high="31" type="uint"/>
+	</bitset>
+	<bitset name="a3xx_hlsq_const_vs_fs_presv_range_reg" inline="yes">
+		<!-- are these a3xx_regid?? -->
+		<bitfield name="STARTENTRY" low="0" high="8"/>
+		<bitfield name="ENDENTRY" low="16" high="24"/>
+	</bitset>
+
+	<reg32 offset="0x2200" name="HLSQ_CONTROL_0_REG">
+		<bitfield name="FSTHREADSIZE" low="4" high="5" type="a3xx_threadsize"/>
+		<bitfield name="FSSUPERTHREADENABLE" pos="6" type="boolean"/>
+		<bitfield name="COMPUTEMODE" pos="8" type="boolean"/>
+		<bitfield name="SPSHADERRESTART" pos="9" type="boolean"/>
+		<bitfield name="RESERVED2" pos="10" type="boolean"/>
+		<bitfield name="CYCLETIMEOUTLIMITVPC" low="12" high="23" type="uint"/>
+		<bitfield name="FSONLYTEX" pos="25" type="boolean"/>
+		<bitfield name="CHUNKDISABLE" pos="26" type="boolean"/>
+		<bitfield name="CONSTMODE" pos="27" type="uint"/>
+		<bitfield name="LAZYUPDATEDISABLE" pos="28" type="boolean"/>
+		<bitfield name="SPCONSTFULLUPDATE" pos="29" type="boolean"/>
+		<bitfield name="TPFULLUPDATE" pos="30" type="boolean"/>
+		<bitfield name="SINGLECONTEXT" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2201" name="HLSQ_CONTROL_1_REG">
+		<bitfield name="VSTHREADSIZE" low="6" high="7" type="a3xx_threadsize"/>
+		<bitfield name="VSSUPERTHREADENABLE" pos="8" type="boolean"/>
+		<bitfield name="FRAGCOORDXYREGID" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="FRAGCOORDZWREGID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0x2202" name="HLSQ_CONTROL_2_REG">
+		<bitfield name="FACENESSREGID" low="2" high="9" type="a3xx_regid"/>
+		<bitfield name="COVVALUEREGID" low="18" high="25" type="a3xx_regid"/>
+		<bitfield name="PRIMALLOCTHRESHOLD" low="26" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2203" name="HLSQ_CONTROL_3_REG">
+		<bitfield name="IJPERSPCENTERREGID" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="IJNONPERSPCENTERREGID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="IJPERSPCENTROIDREGID" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="IJNONPERSPCENTROIDREGID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0x2204" name="HLSQ_VS_CONTROL_REG" type="a3xx_hlsq_vs_fs_control_reg"/>
+	<reg32 offset="0x2205" name="HLSQ_FS_CONTROL_REG" type="a3xx_hlsq_vs_fs_control_reg"/>
+	<reg32 offset="0x2206" name="HLSQ_CONST_VSPRESV_RANGE_REG" type="a3xx_hlsq_const_vs_fs_presv_range_reg"/>
+	<reg32 offset="0x2207" name="HLSQ_CONST_FSPRESV_RANGE_REG" type="a3xx_hlsq_const_vs_fs_presv_range_reg"/>
+	<reg32 offset="0x220a" name="HLSQ_CL_NDRANGE_0_REG">
+		<bitfield name="WORKDIM" low="0" high="1" type="uint"/>
+		<bitfield name="LOCALSIZE0" low="2" high="11" type="uint"/>
+		<bitfield name="LOCALSIZE1" low="12" high="21" type="uint"/>
+		<bitfield name="LOCALSIZE2" low="22" high="31" type="uint"/>
+	</reg32>
+	<array offset="0x220b" name="HLSQ_CL_GLOBAL_WORK" stride="2" length="3">
+		<doc>indexed by dimension</doc>
+		<reg32 offset="0" name="SIZE" type="uint"/>
+		<reg32 offset="1" name="OFFSET" type="uint"/>
+	</array>
+	<reg32 offset="0x2211" name="HLSQ_CL_CONTROL_0_REG"/>
+	<reg32 offset="0x2212" name="HLSQ_CL_CONTROL_1_REG"/>
+	<reg32 offset="0x2214" name="HLSQ_CL_KERNEL_CONST_REG"/>
+	<array offset="0x2215" name="HLSQ_CL_KERNEL_GROUP" stride="1" length="3">
+		<doc>indexed by dimension, global_size / local_size</doc>
+		<reg32 offset="0" name="RATIO" type="uint"/>
+	</array>
+	<reg32 offset="0x2216" name="HLSQ_CL_KERNEL_GROUP_Y_REG" type="uint"/>
+	<reg32 offset="0x2217" name="HLSQ_CL_KERNEL_GROUP_Z_REG" type="uint"/>
+	<reg32 offset="0x221a" name="HLSQ_CL_WG_OFFSET_REG"/>
+
+	<!-- VFD registers -->
+	<reg32 offset="0x2240" name="VFD_CONTROL_0">
+		<doc>
+			TOTALATTRTOVS is # of attributes to vertex shader, in register
+			slots (ie. vec4+vec3 -> 7)
+		</doc>
+		<bitfield name="TOTALATTRTOVS" low="0" high="17" type="uint"/>
+		<bitfield name="PACKETSIZE" low="18" high="21" type="uint"/>
+		<doc>STRMDECINSTRCNT is # of VFD_DECODE_INSTR registers valid</doc>
+		<bitfield name="STRMDECINSTRCNT" low="22" high="26" type="uint"/>
+		<doc>STRMFETCHINSTRCNT is # of VFD_FETCH_INSTR registers valid</doc>
+		<bitfield name="STRMFETCHINSTRCNT" low="27" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2241" name="VFD_CONTROL_1">
+		<doc>MAXSTORAGE could be # of attributes/vbo's</doc>
+		<bitfield name="MAXSTORAGE" low="0" high="3" type="uint"/>
+		<bitfield name="MAXTHRESHOLD" low="4" high="7" type="uint"/>
+		<bitfield name="MINTHRESHOLD" low="8" high="11" type="uint"/>
+		<bitfield name="REGID4VTX" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="REGID4INST" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0x2242" name="VFD_INDEX_MIN" type="uint"/>
+	<reg32 offset="0x2243" name="VFD_INDEX_MAX" type="uint"/>
+	<reg32 offset="0x2244" name="VFD_INSTANCEID_OFFSET" type="uint"/>
+	<reg32 offset="0x2245" name="VFD_INDEX_OFFSET" type="uint"/>
+	<array offset="0x2246" name="VFD_FETCH" stride="2" length="16">
+		<reg32 offset="0x0" name="INSTR_0">
+			<bitfield name="FETCHSIZE" low="0" high="6" type="uint"/>
+			<bitfield name="BUFSTRIDE" low="7" high="15" type="uint"/>
+			<bitfield name="INSTANCED" pos="16" type="boolean"/>
+			<bitfield name="SWITCHNEXT" pos="17" type="boolean"/>
+			<bitfield name="INDEXCODE" low="18" high="23" type="uint"/>
+			<bitfield name="STEPRATE" low="24" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x1" name="INSTR_1"/>
+	</array>
+	<array offset="0x2266" name="VFD_DECODE" stride="1" length="16">
+		<reg32 offset="0x0" name="INSTR">
+			<bitfield name="WRITEMASK" low="0" high="3" type="hex"/>
+			<!-- not sure if this is a bit flag and another flag above it, or?? -->
+			<bitfield name="CONSTFILL" pos="4" type="boolean"/>
+			<bitfield name="FORMAT" low="6" high="11" type="a3xx_vtx_fmt"/>
+			<bitfield name="REGID" low="12" high="19" type="a3xx_regid"/>
+			<bitfield name="INT" pos="20" type="boolean"/>
+			<doc>SHIFTCNT appears to be size, ie. FLOAT_32_32_32 is 12, and BYTE_8 is 1</doc>
+			<bitfield name="SWAP" low="22" high="23" type="a3xx_color_swap"/>
+			<bitfield name="SHIFTCNT" low="24" high="28" type="uint"/>
+			<bitfield name="LASTCOMPVALID" pos="29" type="boolean"/>
+			<bitfield name="SWITCHNEXT" pos="30" type="boolean"/>
+		</reg32>
+	</array>
+	<reg32 offset="0x227e" name="VFD_VS_THREADING_THRESHOLD">
+		<bitfield name="REGID_THRESHOLD" low="0" high="3" type="uint"/>
+		<!-- <bitfield name="RESERVED6" low="4" high="7" type="uint"/> -->
+		<bitfield name="REGID_VTXCNT" low="8" high="15" type="a3xx_regid"/>
+	</reg32>
+
+	<!-- VPC registers -->
+	<reg32 offset="0x2280" name="VPC_ATTR">
+		<bitfield name="TOTALATTR" low="0" high="8" type="uint"/>
+		<!-- PSIZE bit set if gl_PointSize written: -->
+		<bitfield name="PSIZE" pos="9" type="boolean"/>
+		<bitfield name="THRDASSIGN" low="12" high="27" type="uint"/>
+		<bitfield name="LMSIZE" low="28" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2281" name="VPC_PACK">
+		<!-- these are always seem to be set to same as TOTALATTR -->
+		<bitfield name="NUMFPNONPOSVAR" low="8" high="15" type="uint"/>
+		<bitfield name="NUMNONPOSVSVAR" low="16" high="23" type="uint"/>
+	</reg32>
+	<!--
+		varying interpolate mode.  One field per scalar/component
+		(since varying slots are scalar, so things don't have to
+		be aligned to vec4).
+		4 regs * 16 scalar components each => 16 vec4
+	 -->
+	<array offset="0x2282" name="VPC_VARYING_INTERP" stride="1" length="4">
+		<reg32 offset="0x0" name="MODE">
+			<bitfield name="C0" low="0"  high="1"  type="a3xx_intp_mode"/>
+			<bitfield name="C1" low="2"  high="3"  type="a3xx_intp_mode"/>
+			<bitfield name="C2" low="4"  high="5"  type="a3xx_intp_mode"/>
+			<bitfield name="C3" low="6"  high="7"  type="a3xx_intp_mode"/>
+			<bitfield name="C4" low="8"  high="9"  type="a3xx_intp_mode"/>
+			<bitfield name="C5" low="10" high="11" type="a3xx_intp_mode"/>
+			<bitfield name="C6" low="12" high="13" type="a3xx_intp_mode"/>
+			<bitfield name="C7" low="14" high="15" type="a3xx_intp_mode"/>
+			<bitfield name="C8" low="16" high="17" type="a3xx_intp_mode"/>
+			<bitfield name="C9" low="18" high="19" type="a3xx_intp_mode"/>
+			<bitfield name="CA" low="20" high="21" type="a3xx_intp_mode"/>
+			<bitfield name="CB" low="22" high="23" type="a3xx_intp_mode"/>
+			<bitfield name="CC" low="24" high="25" type="a3xx_intp_mode"/>
+			<bitfield name="CD" low="26" high="27" type="a3xx_intp_mode"/>
+			<bitfield name="CE" low="28" high="29" type="a3xx_intp_mode"/>
+			<bitfield name="CF" low="30" high="31" type="a3xx_intp_mode"/>
+		</reg32>
+	</array>
+	<array offset="0x2286" name="VPC_VARYING_PS_REPL" stride="1" length="4">
+		<reg32 offset="0x0" name="MODE">
+			<bitfield name="C0" low="0"  high="1"  type="a3xx_repl_mode"/>
+			<bitfield name="C1" low="2"  high="3"  type="a3xx_repl_mode"/>
+			<bitfield name="C2" low="4"  high="5"  type="a3xx_repl_mode"/>
+			<bitfield name="C3" low="6"  high="7"  type="a3xx_repl_mode"/>
+			<bitfield name="C4" low="8"  high="9"  type="a3xx_repl_mode"/>
+			<bitfield name="C5" low="10" high="11" type="a3xx_repl_mode"/>
+			<bitfield name="C6" low="12" high="13" type="a3xx_repl_mode"/>
+			<bitfield name="C7" low="14" high="15" type="a3xx_repl_mode"/>
+			<bitfield name="C8" low="16" high="17" type="a3xx_repl_mode"/>
+			<bitfield name="C9" low="18" high="19" type="a3xx_repl_mode"/>
+			<bitfield name="CA" low="20" high="21" type="a3xx_repl_mode"/>
+			<bitfield name="CB" low="22" high="23" type="a3xx_repl_mode"/>
+			<bitfield name="CC" low="24" high="25" type="a3xx_repl_mode"/>
+			<bitfield name="CD" low="26" high="27" type="a3xx_repl_mode"/>
+			<bitfield name="CE" low="28" high="29" type="a3xx_repl_mode"/>
+			<bitfield name="CF" low="30" high="31" type="a3xx_repl_mode"/>
+		</reg32>
+	</array>
+	<reg32 offset="0x228a" name="VPC_VARY_CYLWRAP_ENABLE_0"/>
+	<reg32 offset="0x228b" name="VPC_VARY_CYLWRAP_ENABLE_1"/>
+
+	<!-- SP registers -->
+	<bitset name="a3xx_vs_fs_length_reg" inline="yes">
+		<bitfield name="SHADERLENGTH" low="0" high="31" type="uint"/>
+	</bitset>
+
+	<bitset name="sp_vs_fs_obj_offset_reg" inline="yes">
+		<bitfield name="FIRSTEXECINSTROFFSET" low="0" high="15" type="uint"/>
+		<doc>
+			From register spec:
+			SP_FS_OBJ_OFFSET_REG.CONSTOBJECTSTARTOFFSET [16:24]: Constant object
+			start offset in on chip RAM,
+			128bit aligned
+		</doc>
+		<bitfield name="CONSTOBJECTOFFSET" low="16" high="24" type="uint"/>
+		<bitfield name="SHADEROBJOFFSET" low="25" high="31" type="uint"/>
+	</bitset>
+
+	<reg32 offset="0x22c0" name="SP_SP_CTRL_REG">
+		<!-- this bit is set during resolve pass: -->
+		<bitfield name="RESOLVE" pos="16" type="boolean"/>
+		<bitfield name="CONSTMODE" pos="18" type="uint"/>
+		<bitfield name="BINNING" pos="19" type="boolean"/>
+		<bitfield name="SLEEPMODE" low="20" high="21" type="uint"/>
+		<!-- L0MODE==1 when oxiliForceSpL0ModeBuffer=1 -->
+		<bitfield name="L0MODE" low="22" high="23" type="uint"/>
+	</reg32>
+	<reg32 offset="0x22c4" name="SP_VS_CTRL_REG0">
+		<bitfield name="THREADMODE" pos="0" type="a3xx_threadmode"/>
+		<bitfield name="INSTRBUFFERMODE" pos="1" type="a3xx_instrbuffermode"/>
+		<!-- maybe CACHEINVALID is two bits?? -->
+		<bitfield name="CACHEINVALID" pos="2" type="boolean"/>
+		<bitfield name="ALUSCHMODE" pos="3" type="boolean"/>
+		<doc>
+			The full/half register footprint is in units of four components,
+			so if r0.x is used, that counts as all of r0.[xyzw] as used.
+			There are separate full/half register footprint values as the
+			full and half registers are independent (not overlapping).
+			Presumably the thread scheduler hardware allocates the full/half
+			register names from the actual physical register file and
+			handles the register renaming.
+		</doc>
+		<bitfield name="HALFREGFOOTPRINT" low="4" high="9" type="uint"/>
+		<bitfield name="FULLREGFOOTPRINT" low="10" high="15" type="uint"/>
+		<bitfield name="THREADSIZE" pos="20" type="a3xx_threadsize"/>
+		<bitfield name="SUPERTHREADMODE" pos="21" type="boolean"/>
+		<doc>
+			From regspec:
+			SP_FS_CTRL_REG0.FS_LENGTH [31:24]: FS length, unit = 256bits.
+			If bit31 is 1, it means overflow
+			or any long shader.
+		</doc>
+		<bitfield name="LENGTH" low="24" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x22c5" name="SP_VS_CTRL_REG1">
+		<bitfield name="CONSTLENGTH" low="0" high="9" type="uint"/>
+		<!--
+			not sure about full vs half const.. I can't get blob generate
+			something with a mediump/lowp uniform.
+		 -->
+		<bitfield name="CONSTFOOTPRINT" low="10" high="19" type="uint"/>
+		<bitfield name="INITIALOUTSTANDING" low="24" high="30" type="uint"/>
+	</reg32>
+	<reg32 offset="0x22c6" name="SP_VS_PARAM_REG">
+		<bitfield name="POSREGID" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="PSIZEREGID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="POS2DMODE" pos="16" type="boolean"/>
+		<bitfield name="TOTALVSOUTVAR" low="20" high="24" type="uint"/>
+	</reg32>
+	<array offset="0x22c7" name="SP_VS_OUT" stride="1" length="8">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="A_REGID" low="0" high="7" type="a3xx_regid"/>
+			<bitfield name="A_HALF" pos="8" type="boolean"/>
+			<bitfield name="A_COMPMASK" low="9" high="12" type="hex"/>
+			<bitfield name="B_REGID" low="16" high="23" type="a3xx_regid"/>
+			<bitfield name="B_HALF" pos="24" type="boolean"/>
+			<bitfield name="B_COMPMASK" low="25" high="28" type="hex"/>
+		</reg32>
+	</array>
+	<array offset="0x22d0" name="SP_VS_VPC_DST" stride="1" length="4">
+		<reg32 offset="0x0" name="REG">
+			<doc>
+				These seem to be offsets for storage of the varyings.
+				Always seems to start from 8, possibly loc 0 and 4
+				are for gl_Position and gl_PointSize?
+			</doc>
+			<bitfield name="OUTLOC0" low="0" high="6" type="uint"/>
+			<bitfield name="OUTLOC1" low="8" high="14" type="uint"/>
+			<bitfield name="OUTLOC2" low="16" high="22" type="uint"/>
+			<bitfield name="OUTLOC3" low="24" high="30" type="uint"/>
+		</reg32>
+	</array>
+	<reg32 offset="0x22d4" name="SP_VS_OBJ_OFFSET_REG" type="sp_vs_fs_obj_offset_reg"/>
+	<doc>
+		SP_VS_OBJ_START_REG contains pointer to the vertex shader program,
+		immediately followed by the binning shader program (although I
+		guess that is probably just re-using the same gpu buffer)
+	</doc>
+	<reg32 offset="0x22d5" name="SP_VS_OBJ_START_REG"/>
+	<reg32 offset="0x22d6" name="SP_VS_PVT_MEM_PARAM_REG">
+		<bitfield name="MEMSIZEPERITEM" low="0" high="7" shr="7">
+			<doc>The size of memory that ldp/stp can address, in 128 byte increments.</doc>
+		</bitfield>
+		<bitfield name="HWSTACKOFFSET" low="8" high="23" type="uint"/>
+		<bitfield name="HWSTACKSIZEPERTHREAD" low="24" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x22d7" name="SP_VS_PVT_MEM_ADDR_REG">
+		<bitfield name="BURSTLEN" low="0" high="4"/>
+		<bitfield name="SHADERSTARTADDRESS" shr="5" low="5" high="31"/>
+	</reg32>
+	<reg32 offset="0x22d8" name="SP_VS_PVT_MEM_SIZE_REG"/>
+	<reg32 offset="0x22df" name="SP_VS_LENGTH_REG" type="a3xx_vs_fs_length_reg"/>
+	<reg32 offset="0x22e0" name="SP_FS_CTRL_REG0">
+		<bitfield name="THREADMODE" pos="0" type="a3xx_threadmode"/>
+		<bitfield name="INSTRBUFFERMODE" pos="1" type="a3xx_instrbuffermode"/>
+		<!-- maybe CACHEINVALID is two bits?? -->
+		<bitfield name="CACHEINVALID" pos="2" type="boolean"/>
+		<bitfield name="ALUSCHMODE" pos="3" type="boolean"/>
+		<doc>
+			The full/half register footprint is in units of four components,
+			so if r0.x is used, that counts as all of r0.[xyzw] as used.
+			There are separate full/half register footprint values as the
+			full and half registers are independent (not overlapping).
+			Presumably the thread scheduler hardware allocates the full/half
+			register names from the actual physical register file and
+			handles the register renaming.
+		</doc>
+		<bitfield name="HALFREGFOOTPRINT" low="4" high="9" type="uint"/>
+		<bitfield name="FULLREGFOOTPRINT" low="10" high="15" type="uint"/>
+		<bitfield name="FSBYPASSENABLE" pos="17" type="boolean"/>
+		<bitfield name="INOUTREGOVERLAP" pos="18" type="boolean"/>
+		<bitfield name="OUTORDERED" pos="19" type="boolean"/>
+		<bitfield name="THREADSIZE" pos="20" type="a3xx_threadsize"/>
+		<bitfield name="SUPERTHREADMODE" pos="21" type="boolean"/>
+		<bitfield name="PIXLODENABLE" pos="22" type="boolean"/>
+		<bitfield name="COMPUTEMODE" pos="23" type="boolean"/>
+		<doc>
+			From regspec:
+			SP_FS_CTRL_REG0.FS_LENGTH [31:24]: FS length, unit = 256bits.
+			If bit31 is 1, it means overflow
+			or any long shader.
+		</doc>
+		<bitfield name="LENGTH" low="24" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x22e1" name="SP_FS_CTRL_REG1">
+		<bitfield name="CONSTLENGTH" low="0" high="9" type="uint"/>
+		<bitfield name="CONSTFOOTPRINT" low="10" high="19" type="uint"/>
+		<bitfield name="INITIALOUTSTANDING" low="20" high="23" type="uint"/>
+		<bitfield name="HALFPRECVAROFFSET" low="24" high="30" type="uint"/>
+	</reg32>
+	<reg32 offset="0x22e2" name="SP_FS_OBJ_OFFSET_REG" type="sp_vs_fs_obj_offset_reg"/>
+	<doc>SP_FS_OBJ_START_REG contains pointer to fragment shader program</doc>
+	<reg32 offset="0x22e3" name="SP_FS_OBJ_START_REG"/>
+	<reg32 offset="0x22e4" name="SP_FS_PVT_MEM_PARAM_REG">
+		<bitfield name="MEMSIZEPERITEM" low="0" high="7" type="uint"/>
+		<bitfield name="HWSTACKOFFSET" low="8" high="23" type="uint"/>
+		<bitfield name="HWSTACKSIZEPERTHREAD" low="24" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x22e5" name="SP_FS_PVT_MEM_ADDR_REG">
+		<bitfield name="BURSTLEN" low="0" high="4"/>
+		<bitfield name="SHADERSTARTADDRESS" shr="5" low="5" high="31"/>
+	</reg32>
+	<reg32 offset="0x22e6" name="SP_FS_PVT_MEM_SIZE_REG"/>
+	<reg32 offset="0x22e8" name="SP_FS_FLAT_SHAD_MODE_REG_0">
+		<doc>seems to be one bit per scalar, '1' for flat, '0' for smooth</doc>
+	</reg32>
+	<reg32 offset="0x22e9" name="SP_FS_FLAT_SHAD_MODE_REG_1">
+		<doc>seems to be one bit per scalar, '1' for flat, '0' for smooth</doc>
+	</reg32>
+	<reg32 offset="0x22ec" name="SP_FS_OUTPUT_REG">
+		<bitfield name="MRT" low="0" high="1" type="uint">
+			<doc>render targets - 1</doc>
+		</bitfield>
+		<bitfield name="DEPTH_ENABLE" pos="7" type="boolean"/>
+		<bitfield name="DEPTH_REGID" low="8" high="15" type="a3xx_regid"/>
+	</reg32>
+	<array offset="0x22f0" name="SP_FS_MRT" stride="1" length="4">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="REGID" low="0" high="7" type="a3xx_regid"/>
+			<bitfield name="HALF_PRECISION" pos="8" type="boolean"/>
+			<bitfield name="SINT" pos="10" type="boolean"/>
+			<bitfield name="UINT" pos="11" type="boolean"/>
+		</reg32>
+	</array>
+	<array offset="0x22f4" name="SP_FS_IMAGE_OUTPUT" stride="1" length="4">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="MRTFORMAT" low="0" high="5" type="a3xx_color_fmt"/>
+		</reg32>
+	</array>
+	<reg32 offset="0x22ff" name="SP_FS_LENGTH_REG" type="a3xx_vs_fs_length_reg"/>
+
+	<reg32 offset="0x2301" name="PA_SC_AA_CONFIG"/>
+	<!-- TPL1 registers -->
+	<!-- assume VS/FS_TEX_OFFSET is same -->
+	<bitset name="a3xx_tpl1_tp_vs_fs_tex_offset" inline="yes">
+		<bitfield name="SAMPLEROFFSET" low="0" high="7" type="uint"/>
+		<bitfield name="MEMOBJOFFSET" low="8" high="15" type="uint"/>
+		<!-- not sure the size of this: -->
+		<bitfield name="BASETABLEPTR" low="16" high="31" type="uint"/>
+	</bitset>
+	<reg32 offset="0x2340" name="TPL1_TP_VS_TEX_OFFSET" type="a3xx_tpl1_tp_vs_fs_tex_offset"/>
+	<reg32 offset="0x2341" name="TPL1_TP_VS_BORDER_COLOR_BASE_ADDR"/>
+	<reg32 offset="0x2342" name="TPL1_TP_FS_TEX_OFFSET" type="a3xx_tpl1_tp_vs_fs_tex_offset"/>
+	<reg32 offset="0x2343" name="TPL1_TP_FS_BORDER_COLOR_BASE_ADDR"/>
+
+	<!-- VBIF registers -->
+	<reg32 offset="0x3001" name="VBIF_CLKON"/>
+	<reg32 offset="0x300c" name="VBIF_FIXED_SORT_EN"/>
+	<reg32 offset="0x300d" name="VBIF_FIXED_SORT_SEL0"/>
+	<reg32 offset="0x300e" name="VBIF_FIXED_SORT_SEL1"/>
+	<reg32 offset="0x301c" name="VBIF_ABIT_SORT"/>
+	<reg32 offset="0x301d" name="VBIF_ABIT_SORT_CONF"/>
+	<reg32 offset="0x302a" name="VBIF_GATE_OFF_WRREQ_EN"/>
+	<reg32 offset="0x302c" name="VBIF_IN_RD_LIM_CONF0"/>
+	<reg32 offset="0x302d" name="VBIF_IN_RD_LIM_CONF1"/>
+	<reg32 offset="0x3030" name="VBIF_IN_WR_LIM_CONF0"/>
+	<reg32 offset="0x3031" name="VBIF_IN_WR_LIM_CONF1"/>
+	<reg32 offset="0x3034" name="VBIF_OUT_RD_LIM_CONF0"/>
+	<reg32 offset="0x3035" name="VBIF_OUT_WR_LIM_CONF0"/>
+	<reg32 offset="0x3036" name="VBIF_DDR_OUT_MAX_BURST"/>
+	<reg32 offset="0x303c" name="VBIF_ARB_CTL"/>
+	<reg32 offset="0x3049" name="VBIF_ROUND_ROBIN_QOS_ARB"/>
+	<reg32 offset="0x3058" name="VBIF_OUT_AXI_AMEMTYPE_CONF0"/>
+	<reg32 offset="0x305e" name="VBIF_OUT_AXI_AOOO_EN"/>
+	<reg32 offset="0x305f" name="VBIF_OUT_AXI_AOOO"/>
+
+	<bitset name="a3xx_vbif_perf_cnt" inline="yes">
+		<bitfield name="CNT0" pos="0" type="boolean"/>
+		<bitfield name="CNT1" pos="1" type="boolean"/>
+		<bitfield name="PWRCNT0" pos="2" type="boolean"/>
+		<bitfield name="PWRCNT1" pos="3" type="boolean"/>
+		<bitfield name="PWRCNT2" pos="4" type="boolean"/>
+	</bitset>
+
+	<reg32 offset="0x3070" name="VBIF_PERF_CNT_EN" type="a3xx_vbif_perf_cnt"/>
+	<reg32 offset="0x3071" name="VBIF_PERF_CNT_CLR" type="a3xx_vbif_perf_cnt"/>
+	<reg32 offset="0x3072" name="VBIF_PERF_CNT_SEL"/>
+	<reg32 offset="0x3073" name="VBIF_PERF_CNT0_LO"/>
+	<reg32 offset="0x3074" name="VBIF_PERF_CNT0_HI"/>
+	<reg32 offset="0x3075" name="VBIF_PERF_CNT1_LO"/>
+	<reg32 offset="0x3076" name="VBIF_PERF_CNT1_HI"/>
+	<reg32 offset="0x3077" name="VBIF_PERF_PWR_CNT0_LO"/>
+	<reg32 offset="0x3078" name="VBIF_PERF_PWR_CNT0_HI"/>
+	<reg32 offset="0x3079" name="VBIF_PERF_PWR_CNT1_LO"/>
+	<reg32 offset="0x307a" name="VBIF_PERF_PWR_CNT1_HI"/>
+	<reg32 offset="0x307b" name="VBIF_PERF_PWR_CNT2_LO"/>
+	<reg32 offset="0x307c" name="VBIF_PERF_PWR_CNT2_HI"/>
+
+
+	<reg32 offset="0x0c01" name="VSC_BIN_SIZE">
+		<bitfield name="WIDTH" low="0" high="4" shr="5" type="uint"/>
+		<bitfield name="HEIGHT" low="5" high="9" shr="5" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x0c02" name="VSC_SIZE_ADDRESS"/>
+	<array offset="0x0c06" name="VSC_PIPE" stride="3" length="8">
+		<reg32 offset="0x0" name="CONFIG">
+			<doc>
+				Configures the mapping between VSC_PIPE buffer and
+				bin, X/Y specify the bin index in the horiz/vert
+				direction (0,0 is upper left, 0,1 is leftmost bin
+				on second row, and so on).  W/H specify the number
+				of bins assigned to this VSC_PIPE in the horiz/vert
+				dimension.
+			</doc>
+			<bitfield name="X" low="0" high="9" type="uint"/>
+			<bitfield name="Y" low="10" high="19" type="uint"/>
+			<bitfield name="W" low="20" high="23" type="uint"/>
+			<bitfield name="H" low="24" high="27" type="uint"/>
+		</reg32>
+		<reg32 offset="0x1" name="DATA_ADDRESS"/>
+		<reg32 offset="0x2" name="DATA_LENGTH"/>
+	</array>
+	<reg32 offset="0x0c3c" name="VSC_BIN_CONTROL">
+		<doc>seems to be set to 0x00000001 during binning pass</doc>
+		<bitfield name="BINNING_ENABLE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0c3d" name="UNKNOWN_0C3D">
+		<doc>seems to be always set to 0x00000001</doc>
+	</reg32>
+	<reg32 offset="0x0c48" name="PC_PERFCOUNTER0_SELECT" type="a3xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0c49" name="PC_PERFCOUNTER1_SELECT" type="a3xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0c4a" name="PC_PERFCOUNTER2_SELECT" type="a3xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0c4b" name="PC_PERFCOUNTER3_SELECT" type="a3xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0c81" name="GRAS_TSE_DEBUG_ECO">
+		<doc>seems to be always set to 0x00000001</doc>
+	</reg32>
+
+	<reg32 offset="0x0c88" name="GRAS_PERFCOUNTER0_SELECT" type="a3xx_gras_tse_perfcounter_select"/>
+	<reg32 offset="0x0c89" name="GRAS_PERFCOUNTER1_SELECT" type="a3xx_gras_tse_perfcounter_select"/>
+	<reg32 offset="0x0c8a" name="GRAS_PERFCOUNTER2_SELECT" type="a3xx_gras_ras_perfcounter_select"/>
+	<reg32 offset="0x0c8b" name="GRAS_PERFCOUNTER3_SELECT" type="a3xx_gras_ras_perfcounter_select"/>
+	<array offset="0x0ca0" name="GRAS_CL_USER_PLANE" stride="4" length="6">
+		<reg32 offset="0x0" name="X"/>
+		<reg32 offset="0x1" name="Y"/>
+		<reg32 offset="0x2" name="Z"/>
+		<reg32 offset="0x3" name="W"/>
+	</array>
+	<reg32 offset="0x0cc0" name="RB_GMEM_BASE_ADDR"/>
+	<reg32 offset="0x0cc1" name="RB_DEBUG_ECO_CONTROLS_ADDR"/>
+	<reg32 offset="0x0cc6" name="RB_PERFCOUNTER0_SELECT" type="a3xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cc7" name="RB_PERFCOUNTER1_SELECT" type="a3xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0ce0" name="RB_FRAME_BUFFER_DIMENSION">
+		<bitfield name="WIDTH" low="0" high="13" type="uint"/>
+		<bitfield name="HEIGHT" low="14" high="27" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0e00" name="HLSQ_PERFCOUNTER0_SELECT" type="a3xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e01" name="HLSQ_PERFCOUNTER1_SELECT" type="a3xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e02" name="HLSQ_PERFCOUNTER2_SELECT" type="a3xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e03" name="HLSQ_PERFCOUNTER3_SELECT" type="a3xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e04" name="HLSQ_PERFCOUNTER4_SELECT" type="a3xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e05" name="HLSQ_PERFCOUNTER5_SELECT" type="a3xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e43" name="UNKNOWN_0E43">
+		<doc>seems to be always set to 0x00000001</doc>
+	</reg32>
+	<reg32 offset="0x0e44" name="VFD_PERFCOUNTER0_SELECT" type="a3xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e45" name="VFD_PERFCOUNTER1_SELECT" type="a3xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e61" name="VPC_VPC_DEBUG_RAM_SEL"/>
+	<reg32 offset="0x0e62" name="VPC_VPC_DEBUG_RAM_READ"/>
+	<reg32 offset="0x0e64" name="VPC_PERFCOUNTER0_SELECT" type="a3xx_vpc_perfcounter_select"/>
+	<reg32 offset="0x0e65" name="VPC_PERFCOUNTER1_SELECT" type="a3xx_vpc_perfcounter_select"/>
+	<reg32 offset="0x0e82" name="UCHE_CACHE_MODE_CONTROL_REG"/>
+	<reg32 offset="0x0e84" name="UCHE_PERFCOUNTER0_SELECT" type="a3xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0e85" name="UCHE_PERFCOUNTER1_SELECT" type="a3xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0e86" name="UCHE_PERFCOUNTER2_SELECT" type="a3xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0e87" name="UCHE_PERFCOUNTER3_SELECT" type="a3xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0e88" name="UCHE_PERFCOUNTER4_SELECT" type="a3xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0e89" name="UCHE_PERFCOUNTER5_SELECT" type="a3xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0ea0" name="UCHE_CACHE_INVALIDATE0_REG">
+		<!-- might be shifted right by 5, assuming 32byte cache line size.. -->
+		<bitfield name="ADDR" low="0" high="27" type="hex"/>
+	</reg32>
+	<reg32 offset="0x0ea1" name="UCHE_CACHE_INVALIDATE1_REG">
+		<!-- might be shifted right by 5, assuming 32byte cache line size.. -->
+		<bitfield name="ADDR" low="0" high="27" type="hex"/>
+		<!-- I'd assume 2 bits, for FLUSH/INVALIDATE/CLEAN? -->
+		<bitfield name="OPCODE" low="28" high="29" type="a3xx_cache_opcode"/>
+		<bitfield name="ENTIRE_CACHE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0ea6" name="UNKNOWN_0EA6"/>
+	<reg32 offset="0x0ec4" name="SP_PERFCOUNTER0_SELECT" type="a3xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ec5" name="SP_PERFCOUNTER1_SELECT" type="a3xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ec6" name="SP_PERFCOUNTER2_SELECT" type="a3xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ec7" name="SP_PERFCOUNTER3_SELECT" type="a3xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ec8" name="SP_PERFCOUNTER4_SELECT" type="a3xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ec9" name="SP_PERFCOUNTER5_SELECT" type="a3xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0eca" name="SP_PERFCOUNTER6_SELECT" type="a3xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ecb" name="SP_PERFCOUNTER7_SELECT" type="a3xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ee0" name="UNKNOWN_0EE0">
+		<doc>seems to be always set to 0x00000003</doc>
+	</reg32>
+	<reg32 offset="0x0f03" name="UNKNOWN_0F03">
+		<doc>seems to be always set to 0x00000001</doc>
+	</reg32>
+	<reg32 offset="0x0f04" name="TP_PERFCOUNTER0_SELECT" type="a3xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f05" name="TP_PERFCOUNTER1_SELECT" type="a3xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f06" name="TP_PERFCOUNTER2_SELECT" type="a3xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f07" name="TP_PERFCOUNTER3_SELECT" type="a3xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f08" name="TP_PERFCOUNTER4_SELECT" type="a3xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f09" name="TP_PERFCOUNTER5_SELECT" type="a3xx_tp_perfcounter_select"/>
+
+	<!-- this seems to be the register that CP_RUN_OPENCL writes: -->
+	<reg32 offset="0x21f0" name="VGT_CL_INITIATOR"/>
+
+	<!-- seems to be same as a2xx according to fwdump.. -->
+	<reg32 offset="0x21f9" name="VGT_EVENT_INITIATOR"/>
+	<reg32 offset="0x21fc" name="VGT_DRAW_INITIATOR" type="vgt_draw_initiator"/>
+	<reg32 offset="0x21fd" name="VGT_IMMED_DATA"/>
+</domain>
+
+<domain name="A3XX_TEX_SAMP" width="32">
+	<doc>Texture sampler dwords</doc>
+	<enum name="a3xx_tex_filter">
+		<value name="A3XX_TEX_NEAREST" value="0"/>
+		<value name="A3XX_TEX_LINEAR" value="1"/>
+		<value name="A3XX_TEX_ANISO" value="2"/>
+	</enum>
+	<enum name="a3xx_tex_clamp">
+		<value name="A3XX_TEX_REPEAT" value="0"/>
+		<value name="A3XX_TEX_CLAMP_TO_EDGE" value="1"/>
+		<value name="A3XX_TEX_MIRROR_REPEAT" value="2"/>
+		<value name="A3XX_TEX_CLAMP_TO_BORDER" value="3"/>
+		<value name="A3XX_TEX_MIRROR_CLAMP" value="4"/>
+	</enum>
+	<enum name="a3xx_tex_aniso">
+		<value name="A3XX_TEX_ANISO_1" value="0"/>
+		<value name="A3XX_TEX_ANISO_2" value="1"/>
+		<value name="A3XX_TEX_ANISO_4" value="2"/>
+		<value name="A3XX_TEX_ANISO_8" value="3"/>
+		<value name="A3XX_TEX_ANISO_16" value="4"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="CLAMPENABLE" pos="0" type="boolean"/>
+		<bitfield name="MIPFILTER_LINEAR" pos="1" type="boolean"/>
+		<bitfield name="XY_MAG" low="2" high="3" type="a3xx_tex_filter"/>
+		<bitfield name="XY_MIN" low="4" high="5" type="a3xx_tex_filter"/>
+		<bitfield name="WRAP_S" low="6" high="8" type="a3xx_tex_clamp"/>
+		<bitfield name="WRAP_T" low="9" high="11" type="a3xx_tex_clamp"/>
+		<bitfield name="WRAP_R" low="12" high="14" type="a3xx_tex_clamp"/>
+		<bitfield name="ANISO" low="15" high="17" type="a3xx_tex_aniso"/>
+		<bitfield name="COMPARE_FUNC" low="20" high="22" type="adreno_compare_func"/>
+		<bitfield name="CUBEMAPSEAMLESSFILTOFF" pos="24" type="boolean"/>
+		<!-- UNNORM_COORDS == CLK_NORMALIZED_COORDS_FALSE -->
+		<bitfield name="UNNORM_COORDS" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="LOD_BIAS" low="0" high="10" type="fixed" radix="6"/>
+		<bitfield name="MAX_LOD" low="12" high="21" type="ufixed" radix="6"/>
+		<bitfield name="MIN_LOD" low="22" high="31" type="ufixed" radix="6"/>
+	</reg32>
+</domain>
+
+<domain name="A3XX_TEX_CONST" width="32">
+	<doc>Texture constant dwords</doc>
+	<enum name="a3xx_tex_swiz">
+		<!-- same as a2xx? -->
+		<value name="A3XX_TEX_X" value="0"/>
+		<value name="A3XX_TEX_Y" value="1"/>
+		<value name="A3XX_TEX_Z" value="2"/>
+		<value name="A3XX_TEX_W" value="3"/>
+		<value name="A3XX_TEX_ZERO" value="4"/>
+		<value name="A3XX_TEX_ONE" value="5"/>
+	</enum>
+	<enum name="a3xx_tex_type">
+		<value name="A3XX_TEX_1D" value="0"/>
+		<value name="A3XX_TEX_2D" value="1"/>
+		<value name="A3XX_TEX_CUBE" value="2"/>
+		<value name="A3XX_TEX_3D" value="3"/>
+	</enum>
+	<enum name="a3xx_tex_msaa">
+		<value name="A3XX_TPL1_MSAA1X" value="0"/>
+		<value name="A3XX_TPL1_MSAA2X" value="1"/>
+		<value name="A3XX_TPL1_MSAA4X" value="2"/>
+		<value name="A3XX_TPL1_MSAA8X" value="3"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="TILE_MODE" low="0" high="1" type="a3xx_tile_mode"/>
+		<bitfield name="SRGB" pos="2" type="boolean"/>
+		<bitfield name="SWIZ_X" low="4" high="6" type="a3xx_tex_swiz"/>
+		<bitfield name="SWIZ_Y" low="7" high="9" type="a3xx_tex_swiz"/>
+		<bitfield name="SWIZ_Z" low="10" high="12" type="a3xx_tex_swiz"/>
+		<bitfield name="SWIZ_W" low="13" high="15" type="a3xx_tex_swiz"/>
+		<bitfield name="MIPLVLS" low="16" high="19" type="uint"/>
+		<bitfield name="MSAATEX" low="20" high="21" type="a3xx_tex_msaa"/>
+		<bitfield name="FMT" low="22" high="28" type="a3xx_tex_fmt"/>
+		<bitfield name="NOCONVERT" pos="29" type="boolean"/>
+		<bitfield name="TYPE" low="30" high="31" type="a3xx_tex_type"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="HEIGHT" low="0" high="13" type="uint"/>
+		<bitfield name="WIDTH" low="14" high="27" type="uint"/>
+		<!-- minimum pitch (for mipmap levels): log2(pitchalign / 16) -->
+		<bitfield name="PITCHALIGN" low="28" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<doc>INDX is index of texture address(es) in MIPMAP state block</doc>
+		<bitfield name="INDX" low="0" high="8" type="uint"/>
+		<doc>Pitch in bytes (so actually stride)</doc>
+		<bitfield name="PITCH" low="12" high="29" type="uint"/>
+		<doc>SWAP bit is set for BGRA instead of RGBA</doc>
+		<bitfield name="SWAP" low="30" high="31" type="a3xx_color_swap"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<!--
+		Update: the two LAYERSZn seem not to be the same thing.
+		According to Ilia's experimentation the first one goes up
+		to at *least* bit 14..
+		 -->
+		<bitfield name="LAYERSZ1" low="0" high="16" shr="12" type="uint"/>
+		<bitfield name="DEPTH" low="17" high="27" type="uint"/>
+		<bitfield name="LAYERSZ2" low="28" high="31" shr="12" type="uint"/>
+	</reg32>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/adreno/a4xx.xml b/drivers/gpu/drm/msm/registers/adreno/a4xx.xml
new file mode 100644
index 000000000000..69a9f9b02bc9
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/adreno/a4xx.xml
@@ -0,0 +1,2409 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+<import file="adreno/adreno_common.xml"/>
+<import file="adreno/adreno_pm4.xml"/>
+
+<enum name="a4xx_color_fmt">
+	<value name="RB4_A8_UNORM" value="0x01"/>
+	<value name="RB4_R8_UNORM" value="0x02"/>
+	<value name="RB4_R8_SNORM" value="0x03"/>
+	<value name="RB4_R8_UINT" value="0x04"/>
+	<value name="RB4_R8_SINT" value="0x05"/>
+
+	<value name="RB4_R4G4B4A4_UNORM" value="0x08"/>
+	<value name="RB4_R5G5B5A1_UNORM" value="0x0a"/>
+	<value name="RB4_R5G6B5_UNORM" value="0x0e"/>
+	<value name="RB4_R8G8_UNORM" value="0x0f"/>
+	<value name="RB4_R8G8_SNORM" value="0x10"/>
+	<value name="RB4_R8G8_UINT" value="0x11"/>
+	<value name="RB4_R8G8_SINT" value="0x12"/>
+	<value name="RB4_R16_UNORM" value="0x13"/>
+	<value name="RB4_R16_SNORM" value="0x14"/>
+	<value name="RB4_R16_FLOAT" value="0x15"/>
+	<value name="RB4_R16_UINT" value="0x16"/>
+	<value name="RB4_R16_SINT" value="0x17"/>
+
+	<value name="RB4_R8G8B8_UNORM" value="0x19"/>
+
+	<value name="RB4_R8G8B8A8_UNORM" value="0x1a"/>
+	<value name="RB4_R8G8B8A8_SNORM" value="0x1c"/>
+	<value name="RB4_R8G8B8A8_UINT" value="0x1d"/>
+	<value name="RB4_R8G8B8A8_SINT" value="0x1e"/>
+	<value name="RB4_R10G10B10A2_UNORM" value="0x1f"/>
+	<value name="RB4_R10G10B10A2_UINT" value="0x22"/>
+	<value name="RB4_R11G11B10_FLOAT" value="0x27"/>
+	<value name="RB4_R16G16_UNORM" value="0x28"/>
+	<value name="RB4_R16G16_SNORM" value="0x29"/>
+	<value name="RB4_R16G16_FLOAT" value="0x2a"/>
+	<value name="RB4_R16G16_UINT" value="0x2b"/>
+	<value name="RB4_R16G16_SINT" value="0x2c"/>
+	<value name="RB4_R32_FLOAT" value="0x2d"/>
+	<value name="RB4_R32_UINT" value="0x2e"/>
+	<value name="RB4_R32_SINT" value="0x2f"/>
+
+	<value name="RB4_R16G16B16A16_UNORM" value="0x34"/>
+	<value name="RB4_R16G16B16A16_SNORM" value="0x35"/>
+	<value name="RB4_R16G16B16A16_FLOAT" value="0x36"/>
+	<value name="RB4_R16G16B16A16_UINT" value="0x37"/>
+	<value name="RB4_R16G16B16A16_SINT" value="0x38"/>
+	<value name="RB4_R32G32_FLOAT" value="0x39"/>
+	<value name="RB4_R32G32_UINT" value="0x3a"/>
+	<value name="RB4_R32G32_SINT" value="0x3b"/>
+
+	<value name="RB4_R32G32B32A32_FLOAT" value="0x3c"/>
+	<value name="RB4_R32G32B32A32_UINT" value="0x3d"/>
+	<value name="RB4_R32G32B32A32_SINT" value="0x3e"/>
+
+	<value name="RB4_NONE" value="0xff"/>
+</enum>
+
+<enum name="a4xx_tile_mode">
+	<value name="TILE4_LINEAR" value="0"/>
+	<value name="TILE4_2" value="2"/>
+	<value name="TILE4_3" value="3"/>
+</enum>
+
+<enum name="a4xx_vtx_fmt" prefix="chipset">
+	<!-- hmm, shifted one compared to a3xx?!?  -->
+	<value name="VFMT4_32_FLOAT" value="0x1"/>
+	<value name="VFMT4_32_32_FLOAT" value="0x2"/>
+	<value name="VFMT4_32_32_32_FLOAT" value="0x3"/>
+	<value name="VFMT4_32_32_32_32_FLOAT" value="0x4"/>
+
+	<value name="VFMT4_16_FLOAT" value="0x5"/>
+	<value name="VFMT4_16_16_FLOAT" value="0x6"/>
+	<value name="VFMT4_16_16_16_FLOAT" value="0x7"/>
+	<value name="VFMT4_16_16_16_16_FLOAT" value="0x8"/>
+
+	<value name="VFMT4_32_FIXED" value="0x9"/>
+	<value name="VFMT4_32_32_FIXED" value="0xa"/>
+	<value name="VFMT4_32_32_32_FIXED" value="0xb"/>
+	<value name="VFMT4_32_32_32_32_FIXED" value="0xc"/>
+
+	<value name="VFMT4_11_11_10_FLOAT" value="0xd"/>
+
+	<!-- beyond here it does not appear to be shifted -->
+	<value name="VFMT4_16_SINT" value="0x10"/>
+	<value name="VFMT4_16_16_SINT" value="0x11"/>
+	<value name="VFMT4_16_16_16_SINT" value="0x12"/>
+	<value name="VFMT4_16_16_16_16_SINT" value="0x13"/>
+	<value name="VFMT4_16_UINT" value="0x14"/>
+	<value name="VFMT4_16_16_UINT" value="0x15"/>
+	<value name="VFMT4_16_16_16_UINT" value="0x16"/>
+	<value name="VFMT4_16_16_16_16_UINT" value="0x17"/>
+	<value name="VFMT4_16_SNORM" value="0x18"/>
+	<value name="VFMT4_16_16_SNORM" value="0x19"/>
+	<value name="VFMT4_16_16_16_SNORM" value="0x1a"/>
+	<value name="VFMT4_16_16_16_16_SNORM" value="0x1b"/>
+	<value name="VFMT4_16_UNORM" value="0x1c"/>
+	<value name="VFMT4_16_16_UNORM" value="0x1d"/>
+	<value name="VFMT4_16_16_16_UNORM" value="0x1e"/>
+	<value name="VFMT4_16_16_16_16_UNORM" value="0x1f"/>
+
+	<value name="VFMT4_32_UINT" value="0x20"/>
+	<value name="VFMT4_32_32_UINT" value="0x21"/>
+	<value name="VFMT4_32_32_32_UINT" value="0x22"/>
+	<value name="VFMT4_32_32_32_32_UINT" value="0x23"/>
+	<value name="VFMT4_32_SINT" value="0x24"/>
+	<value name="VFMT4_32_32_SINT" value="0x25"/>
+	<value name="VFMT4_32_32_32_SINT" value="0x26"/>
+	<value name="VFMT4_32_32_32_32_SINT" value="0x27"/>
+
+	<value name="VFMT4_8_UINT" value="0x28"/>
+	<value name="VFMT4_8_8_UINT" value="0x29"/>
+	<value name="VFMT4_8_8_8_UINT" value="0x2a"/>
+	<value name="VFMT4_8_8_8_8_UINT" value="0x2b"/>
+	<value name="VFMT4_8_UNORM" value="0x2c"/>
+	<value name="VFMT4_8_8_UNORM" value="0x2d"/>
+	<value name="VFMT4_8_8_8_UNORM" value="0x2e"/>
+	<value name="VFMT4_8_8_8_8_UNORM" value="0x2f"/>
+	<value name="VFMT4_8_SINT" value="0x30"/>
+	<value name="VFMT4_8_8_SINT" value="0x31"/>
+	<value name="VFMT4_8_8_8_SINT" value="0x32"/>
+	<value name="VFMT4_8_8_8_8_SINT" value="0x33"/>
+	<value name="VFMT4_8_SNORM" value="0x34"/>
+	<value name="VFMT4_8_8_SNORM" value="0x35"/>
+	<value name="VFMT4_8_8_8_SNORM" value="0x36"/>
+	<value name="VFMT4_8_8_8_8_SNORM" value="0x37"/>
+
+	<value name="VFMT4_10_10_10_2_UINT" value="0x38"/>
+	<value name="VFMT4_10_10_10_2_UNORM" value="0x39"/>
+	<value name="VFMT4_10_10_10_2_SINT" value="0x3a"/>
+	<value name="VFMT4_10_10_10_2_SNORM" value="0x3b"/>
+	<value name="VFMT4_2_10_10_10_UINT" value="0x3c"/>
+	<value name="VFMT4_2_10_10_10_UNORM" value="0x3d"/>
+	<value name="VFMT4_2_10_10_10_SINT" value="0x3e"/>
+	<value name="VFMT4_2_10_10_10_SNORM" value="0x3f"/>
+
+	<value name="VFMT4_NONE" value="0xff"/>
+</enum>
+
+<enum name="a4xx_tex_fmt">
+	<!-- 0x00 .. 0x02 -->
+
+	<!-- 8-bit formats -->
+	<value name="TFMT4_A8_UNORM" value="0x03"/>
+	<value name="TFMT4_8_UNORM"  value="0x04"/>
+	<value name="TFMT4_8_SNORM"  value="0x05"/>
+	<value name="TFMT4_8_UINT"   value="0x06"/>
+	<value name="TFMT4_8_SINT"   value="0x07"/>
+
+	<!-- 16-bit formats -->
+	<value name="TFMT4_4_4_4_4_UNORM" value="0x08"/>
+	<value name="TFMT4_5_5_5_1_UNORM" value="0x09"/>
+	<!-- 0x0a -->
+	<value name="TFMT4_5_6_5_UNORM"   value="0x0b"/>
+
+	<!-- 0x0c -->
+
+	<value name="TFMT4_L8_A8_UNORM" value="0x0d"/>
+	<value name="TFMT4_8_8_UNORM"   value="0x0e"/>
+	<value name="TFMT4_8_8_SNORM"   value="0x0f"/>
+	<value name="TFMT4_8_8_UINT"    value="0x10"/>
+	<value name="TFMT4_8_8_SINT"    value="0x11"/>
+
+	<value name="TFMT4_16_UNORM" value="0x12"/>
+	<value name="TFMT4_16_SNORM" value="0x13"/>
+	<value name="TFMT4_16_FLOAT" value="0x14"/>
+	<value name="TFMT4_16_UINT"  value="0x15"/>
+	<value name="TFMT4_16_SINT"  value="0x16"/>
+
+	<!-- 0x17 .. 0x1b -->
+
+	<!-- 32-bit formats -->
+	<value name="TFMT4_8_8_8_8_UNORM" value="0x1c"/>
+	<value name="TFMT4_8_8_8_8_SNORM" value="0x1d"/>
+	<value name="TFMT4_8_8_8_8_UINT"  value="0x1e"/>
+	<value name="TFMT4_8_8_8_8_SINT"  value="0x1f"/>
+
+	<value name="TFMT4_9_9_9_E5_FLOAT"   value="0x20"/>
+	<value name="TFMT4_10_10_10_2_UNORM" value="0x21"/>
+	<value name="TFMT4_10_10_10_2_UINT"  value="0x22"/>
+	<!-- 0x23 .. 0x24 -->
+	<value name="TFMT4_11_11_10_FLOAT"   value="0x25"/>
+
+	<value name="TFMT4_16_16_UNORM" value="0x26"/>
+	<value name="TFMT4_16_16_SNORM" value="0x27"/>
+	<value name="TFMT4_16_16_FLOAT" value="0x28"/>
+	<value name="TFMT4_16_16_UINT"  value="0x29"/>
+	<value name="TFMT4_16_16_SINT"  value="0x2a"/>
+
+	<value name="TFMT4_32_FLOAT" value="0x2b"/>
+	<value name="TFMT4_32_UINT"  value="0x2c"/>
+	<value name="TFMT4_32_SINT"  value="0x2d"/>
+
+	<!-- 0x2e .. 0x32 -->
+
+	<!-- 64-bit formats -->
+	<value name="TFMT4_16_16_16_16_UNORM" value="0x33"/>
+	<value name="TFMT4_16_16_16_16_SNORM" value="0x34"/>
+	<value name="TFMT4_16_16_16_16_FLOAT" value="0x35"/>
+	<value name="TFMT4_16_16_16_16_UINT"  value="0x36"/>
+	<value name="TFMT4_16_16_16_16_SINT"  value="0x37"/>
+
+	<value name="TFMT4_32_32_FLOAT" value="0x38"/>
+	<value name="TFMT4_32_32_UINT"  value="0x39"/>
+	<value name="TFMT4_32_32_SINT"  value="0x3a"/>
+
+	<!-- 96-bit formats -->
+	<value name="TFMT4_32_32_32_FLOAT" value="0x3b"/>
+	<value name="TFMT4_32_32_32_UINT"  value="0x3c"/>
+	<value name="TFMT4_32_32_32_SINT"  value="0x3d"/>
+
+	<!-- 0x3e -->
+
+	<!-- 128-bit formats -->
+	<value name="TFMT4_32_32_32_32_FLOAT" value="0x3f"/>
+	<value name="TFMT4_32_32_32_32_UINT"  value="0x40"/>
+	<value name="TFMT4_32_32_32_32_SINT"  value="0x41"/>
+
+	<!-- 0x42 .. 0x46 -->
+	<value name="TFMT4_X8Z24_UNORM" value="0x47"/>
+	<!-- 0x48 .. 0x55 -->
+
+	<!-- compressed formats -->
+	<value name="TFMT4_DXT1"                  value="0x56"/>
+	<value name="TFMT4_DXT3"                  value="0x57"/>
+	<value name="TFMT4_DXT5"                  value="0x58"/>
+	<!-- 0x59 -->
+	<value name="TFMT4_RGTC1_UNORM"           value="0x5a"/>
+	<value name="TFMT4_RGTC1_SNORM"           value="0x5b"/>
+	<!-- 0x5c .. 0x5d -->
+	<value name="TFMT4_RGTC2_UNORM"           value="0x5e"/>
+	<value name="TFMT4_RGTC2_SNORM"           value="0x5f"/>
+	<!-- 0x60 -->
+	<value name="TFMT4_BPTC_UFLOAT"           value="0x61"/>
+	<value name="TFMT4_BPTC_FLOAT"            value="0x62"/>
+	<value name="TFMT4_BPTC"                  value="0x63"/>
+	<value name="TFMT4_ATC_RGB"               value="0x64"/>
+	<value name="TFMT4_ATC_RGBA_EXPLICIT"     value="0x65"/>
+	<value name="TFMT4_ATC_RGBA_INTERPOLATED" value="0x66"/>
+	<value name="TFMT4_ETC2_RG11_UNORM"       value="0x67"/>
+	<value name="TFMT4_ETC2_RG11_SNORM"       value="0x68"/>
+	<value name="TFMT4_ETC2_R11_UNORM"        value="0x69"/>
+	<value name="TFMT4_ETC2_R11_SNORM"        value="0x6a"/>
+	<value name="TFMT4_ETC1"                  value="0x6b"/>
+	<value name="TFMT4_ETC2_RGB8"             value="0x6c"/>
+	<value name="TFMT4_ETC2_RGBA8"            value="0x6d"/>
+	<value name="TFMT4_ETC2_RGB8A1"           value="0x6e"/>
+	<value name="TFMT4_ASTC_4x4"              value="0x6f"/>
+	<value name="TFMT4_ASTC_5x4"              value="0x70"/>
+	<value name="TFMT4_ASTC_5x5"              value="0x71"/>
+	<value name="TFMT4_ASTC_6x5"              value="0x72"/>
+	<value name="TFMT4_ASTC_6x6"              value="0x73"/>
+	<value name="TFMT4_ASTC_8x5"              value="0x74"/>
+	<value name="TFMT4_ASTC_8x6"              value="0x75"/>
+	<value name="TFMT4_ASTC_8x8"              value="0x76"/>
+	<value name="TFMT4_ASTC_10x5"             value="0x77"/>
+	<value name="TFMT4_ASTC_10x6"             value="0x78"/>
+	<value name="TFMT4_ASTC_10x8"             value="0x79"/>
+	<value name="TFMT4_ASTC_10x10"            value="0x7a"/>
+	<value name="TFMT4_ASTC_12x10"            value="0x7b"/>
+	<value name="TFMT4_ASTC_12x12"            value="0x7c"/>
+	<!-- 0x7d .. 0x7f -->
+
+	<value name="TFMT4_NONE"                  value="0xff"/>
+</enum>
+
+<enum name="a4xx_depth_format">
+	<value name="DEPTH4_NONE" value="0"/>
+	<value name="DEPTH4_16" value="1"/>
+	<value name="DEPTH4_24_8" value="2"/>
+	<value name="DEPTH4_32" value="3"/>
+</enum>
+
+<!--
+NOTE counters extracted from test-perf log with the following awful
+script:
+##################
+#!/bin/bash
+
+log=$1
+
+grep -F "counter
+countable
+group" $log | grep -v gl > shortlist.txt
+
+countable=""
+IFS=$'\n'; for line in $(cat shortlist.txt); do
+	# parse ######### group[$n]: $name
+	l=${line########### group}
+	if [ $l != $line ];  then
+		group=`echo $line | awk '{print $3}'`
+		echo "Group: $group"
+		continue
+	fi
+	# parse #########   counter[$n]: $name
+	l=${line###########   counter}
+	if [ $l != $line ]; then
+		countable=`echo $line | awk '{print $3}'`
+		#echo "  Countable: $countable"
+		continue
+	fi
+	# parse 		countable:
+	l=${line##		countable:}
+	if [ $l != $line ]; then
+		val=`echo $line | awk '{print $2}'`
+		echo "<value value=\"$val\" name=\"$countable\"/>"
+	fi
+
+done
+##################
+ -->
+<enum name="a4xx_ccu_perfcounter_select">
+	<value value="0" name="CCU_BUSY_CYCLES"/>
+	<value value="2" name="CCU_RB_DEPTH_RETURN_STALL"/>
+	<value value="3" name="CCU_RB_COLOR_RETURN_STALL"/>
+	<value value="6" name="CCU_DEPTH_BLOCKS"/>
+	<value value="7" name="CCU_COLOR_BLOCKS"/>
+	<value value="8" name="CCU_DEPTH_BLOCK_HIT"/>
+	<value value="9" name="CCU_COLOR_BLOCK_HIT"/>
+	<value value="10" name="CCU_DEPTH_FLAG1_COUNT"/>
+	<value value="11" name="CCU_DEPTH_FLAG2_COUNT"/>
+	<value value="12" name="CCU_DEPTH_FLAG3_COUNT"/>
+	<value value="13" name="CCU_DEPTH_FLAG4_COUNT"/>
+	<value value="14" name="CCU_COLOR_FLAG1_COUNT"/>
+	<value value="15" name="CCU_COLOR_FLAG2_COUNT"/>
+	<value value="16" name="CCU_COLOR_FLAG3_COUNT"/>
+	<value value="17" name="CCU_COLOR_FLAG4_COUNT"/>
+	<value value="18" name="CCU_PARTIAL_BLOCK_READ"/>
+</enum>
+
+<!--
+NOTE other than CP_ALWAYS_COUNT (which is the only one we use so far),
+on a3xx the countable #'s from AMD_performance_monitor disagreed with
+TRM.  All these #'s for a4xx come from AMD_performance_monitor, so
+perhaps they should be taken with a grain of salt
+-->
+<enum name="a4xx_cp_perfcounter_select">
+	<!-- first ctr at least seems same as a3xx, so we can measure freq -->
+	<value value="0" name="CP_ALWAYS_COUNT"/>
+	<value value="1" name="CP_BUSY"/>
+	<value value="2" name="CP_PFP_IDLE"/>
+	<value value="3" name="CP_PFP_BUSY_WORKING"/>
+	<value value="4" name="CP_PFP_STALL_CYCLES_ANY"/>
+	<value value="5" name="CP_PFP_STARVE_CYCLES_ANY"/>
+	<value value="6" name="CP_PFP_STARVED_PER_LOAD_ADDR"/>
+	<value value="7" name="CP_PFP_STALLED_PER_STORE_ADDR"/>
+	<value value="8" name="CP_PFP_PC_PROFILE"/>
+	<value value="9" name="CP_PFP_MATCH_PM4_PKT_PROFILE"/>
+	<value value="10" name="CP_PFP_COND_INDIRECT_DISCARDED"/>
+	<value value="11" name="CP_LONG_RESUMPTIONS"/>
+	<value value="12" name="CP_RESUME_CYCLES"/>
+	<value value="13" name="CP_RESUME_TO_BOUNDARY_CYCLES"/>
+	<value value="14" name="CP_LONG_PREEMPTIONS"/>
+	<value value="15" name="CP_PREEMPT_CYCLES"/>
+	<value value="16" name="CP_PREEMPT_TO_BOUNDARY_CYCLES"/>
+	<value value="17" name="CP_ME_FIFO_EMPTY_PFP_IDLE"/>
+	<value value="18" name="CP_ME_FIFO_EMPTY_PFP_BUSY"/>
+	<value value="19" name="CP_ME_FIFO_NOT_EMPTY_NOT_FULL"/>
+	<value value="20" name="CP_ME_FIFO_FULL_ME_BUSY"/>
+	<value value="21" name="CP_ME_FIFO_FULL_ME_NON_WORKING"/>
+	<value value="22" name="CP_ME_WAITING_FOR_PACKETS"/>
+	<value value="23" name="CP_ME_BUSY_WORKING"/>
+	<value value="24" name="CP_ME_STARVE_CYCLES_ANY"/>
+	<value value="25" name="CP_ME_STARVE_CYCLES_PER_PROFILE"/>
+	<value value="26" name="CP_ME_STALL_CYCLES_PER_PROFILE"/>
+	<value value="27" name="CP_ME_PC_PROFILE"/>
+	<value value="28" name="CP_RCIU_FIFO_EMPTY"/>
+	<value value="29" name="CP_RCIU_FIFO_NOT_EMPTY_NOT_FULL"/>
+	<value value="30" name="CP_RCIU_FIFO_FULL"/>
+	<value value="31" name="CP_RCIU_FIFO_FULL_NO_CONTEXT"/>
+	<value value="32" name="CP_RCIU_FIFO_FULL_AHB_MASTER"/>
+	<value value="33" name="CP_RCIU_FIFO_FULL_OTHER"/>
+	<value value="34" name="CP_AHB_IDLE"/>
+	<value value="35" name="CP_AHB_STALL_ON_GRANT_NO_SPLIT"/>
+	<value value="36" name="CP_AHB_STALL_ON_GRANT_SPLIT"/>
+	<value value="37" name="CP_AHB_STALL_ON_GRANT_SPLIT_PROFILE"/>
+	<value value="38" name="CP_AHB_BUSY_WORKING"/>
+	<value value="39" name="CP_AHB_BUSY_STALL_ON_HRDY"/>
+	<value value="40" name="CP_AHB_BUSY_STALL_ON_HRDY_PROFILE"/>
+</enum>
+
+<enum name="a4xx_gras_ras_perfcounter_select">
+	<value value="0" name="RAS_SUPER_TILES"/>
+	<value value="1" name="RAS_8X8_TILES"/>
+	<value value="2" name="RAS_4X4_TILES"/>
+	<value value="3" name="RAS_BUSY_CYCLES"/>
+	<value value="4" name="RAS_STALL_CYCLES_BY_RB"/>
+	<value value="5" name="RAS_STALL_CYCLES_BY_VSC"/>
+	<value value="6" name="RAS_STARVE_CYCLES_BY_TSE"/>
+	<value value="7" name="RAS_SUPERTILE_CYCLES"/>
+	<value value="8" name="RAS_TILE_CYCLES"/>
+	<value value="9" name="RAS_FULLY_COVERED_SUPER_TILES"/>
+	<value value="10" name="RAS_FULLY_COVERED_8X8_TILES"/>
+	<value value="11" name="RAS_4X4_PRIM"/>
+	<value value="12" name="RAS_8X4_4X8_PRIM"/>
+	<value value="13" name="RAS_8X8_PRIM"/>
+</enum>
+
+<enum name="a4xx_gras_tse_perfcounter_select">
+	<value value="0" name="TSE_INPUT_PRIM"/>
+	<value value="1" name="TSE_INPUT_NULL_PRIM"/>
+	<value value="2" name="TSE_TRIVAL_REJ_PRIM"/>
+	<value value="3" name="TSE_CLIPPED_PRIM"/>
+	<value value="4" name="TSE_NEW_PRIM"/>
+	<value value="5" name="TSE_ZERO_AREA_PRIM"/>
+	<value value="6" name="TSE_FACENESS_CULLED_PRIM"/>
+	<value value="7" name="TSE_ZERO_PIXEL_PRIM"/>
+	<value value="8" name="TSE_OUTPUT_NULL_PRIM"/>
+	<value value="9" name="TSE_OUTPUT_VISIBLE_PRIM"/>
+	<value value="10" name="TSE_PRE_CLIP_PRIM"/>
+	<value value="11" name="TSE_POST_CLIP_PRIM"/>
+	<value value="12" name="TSE_BUSY_CYCLES"/>
+	<value value="13" name="TSE_PC_STARVE"/>
+	<value value="14" name="TSE_RAS_STALL"/>
+	<value value="15" name="TSE_STALL_BARYPLANE_FIFO_FULL"/>
+	<value value="16" name="TSE_STALL_ZPLANE_FIFO_FULL"/>
+</enum>
+
+<enum name="a4xx_hlsq_perfcounter_select">
+	<value value="0" name="HLSQ_SP_VS_STAGE_CONSTANT"/>
+	<value value="1" name="HLSQ_SP_VS_STAGE_INSTRUCTIONS"/>
+	<value value="2" name="HLSQ_SP_FS_STAGE_CONSTANT"/>
+	<value value="3" name="HLSQ_SP_FS_STAGE_INSTRUCTIONS"/>
+	<value value="4" name="HLSQ_TP_STATE"/>
+	<value value="5" name="HLSQ_QUADS"/>
+	<value value="6" name="HLSQ_PIXELS"/>
+	<value value="7" name="HLSQ_VERTICES"/>
+	<value value="13" name="HLSQ_SP_VS_STAGE_DATA_BYTES"/>
+	<value value="14" name="HLSQ_SP_FS_STAGE_DATA_BYTES"/>
+	<value value="15" name="HLSQ_BUSY_CYCLES"/>
+	<value value="16" name="HLSQ_STALL_CYCLES_SP_STATE"/>
+	<value value="17" name="HLSQ_STALL_CYCLES_SP_VS_STAGE"/>
+	<value value="18" name="HLSQ_STALL_CYCLES_SP_FS_STAGE"/>
+	<value value="19" name="HLSQ_STALL_CYCLES_UCHE"/>
+	<value value="20" name="HLSQ_RBBM_LOAD_CYCLES"/>
+	<value value="21" name="HLSQ_DI_TO_VS_START_SP"/>
+	<value value="22" name="HLSQ_DI_TO_FS_START_SP"/>
+	<value value="23" name="HLSQ_VS_STAGE_START_TO_DONE_SP"/>
+	<value value="24" name="HLSQ_FS_STAGE_START_TO_DONE_SP"/>
+	<value value="25" name="HLSQ_SP_STATE_COPY_CYCLES_VS_STAGE"/>
+	<value value="26" name="HLSQ_SP_STATE_COPY_CYCLES_FS_STAGE"/>
+	<value value="27" name="HLSQ_UCHE_LATENCY_CYCLES"/>
+	<value value="28" name="HLSQ_UCHE_LATENCY_COUNT"/>
+	<value value="29" name="HLSQ_STARVE_CYCLES_VFD"/>
+</enum>
+
+<enum name="a4xx_pc_perfcounter_select">
+	<value value="0" name="PC_VIS_STREAMS_LOADED"/>
+	<value value="2" name="PC_VPC_PRIMITIVES"/>
+	<value value="3" name="PC_DEAD_PRIM"/>
+	<value value="4" name="PC_LIVE_PRIM"/>
+	<value value="5" name="PC_DEAD_DRAWCALLS"/>
+	<value value="6" name="PC_LIVE_DRAWCALLS"/>
+	<value value="7" name="PC_VERTEX_MISSES"/>
+	<value value="9" name="PC_STALL_CYCLES_VFD"/>
+	<value value="10" name="PC_STALL_CYCLES_TSE"/>
+	<value value="11" name="PC_STALL_CYCLES_UCHE"/>
+	<value value="12" name="PC_WORKING_CYCLES"/>
+	<value value="13" name="PC_IA_VERTICES"/>
+	<value value="14" name="PC_GS_PRIMITIVES"/>
+	<value value="15" name="PC_HS_INVOCATIONS"/>
+	<value value="16" name="PC_DS_INVOCATIONS"/>
+	<value value="17" name="PC_DS_PRIMITIVES"/>
+	<value value="20" name="PC_STARVE_CYCLES_FOR_INDEX"/>
+	<value value="21" name="PC_STARVE_CYCLES_FOR_TESS_FACTOR"/>
+	<value value="22" name="PC_STARVE_CYCLES_FOR_VIZ_STREAM"/>
+	<value value="23" name="PC_STALL_CYCLES_TESS"/>
+	<value value="24" name="PC_STARVE_CYCLES_FOR_POSITION"/>
+	<value value="25" name="PC_MODE0_DRAWCALL"/>
+	<value value="26" name="PC_MODE1_DRAWCALL"/>
+	<value value="27" name="PC_MODE2_DRAWCALL"/>
+	<value value="28" name="PC_MODE3_DRAWCALL"/>
+	<value value="29" name="PC_MODE4_DRAWCALL"/>
+	<value value="30" name="PC_PREDICATED_DEAD_DRAWCALL"/>
+	<value value="31" name="PC_STALL_CYCLES_BY_TSE_ONLY"/>
+	<value value="32" name="PC_STALL_CYCLES_BY_VPC_ONLY"/>
+	<value value="33" name="PC_VPC_POS_DATA_TRANSACTION"/>
+	<value value="34" name="PC_BUSY_CYCLES"/>
+	<value value="35" name="PC_STARVE_CYCLES_DI"/>
+	<value value="36" name="PC_STALL_CYCLES_VPC"/>
+	<value value="37" name="TESS_WORKING_CYCLES"/>
+	<value value="38" name="TESS_NUM_CYCLES_SETUP_WORKING"/>
+	<value value="39" name="TESS_NUM_CYCLES_PTGEN_WORKING"/>
+	<value value="40" name="TESS_NUM_CYCLES_CONNGEN_WORKING"/>
+	<value value="41" name="TESS_BUSY_CYCLES"/>
+	<value value="42" name="TESS_STARVE_CYCLES_PC"/>
+	<value value="43" name="TESS_STALL_CYCLES_PC"/>
+</enum>
+
+<enum name="a4xx_pwr_perfcounter_select">
+	<!-- NOTE not actually used.. see RBBM_RBBM_CTL.RESET_PWR_CTR0/1 -->
+	<value value="0" name="PWR_CORE_CLOCK_CYCLES"/>
+	<value value="1" name="PWR_BUSY_CLOCK_CYCLES"/>
+</enum>
+
+<enum name="a4xx_rb_perfcounter_select">
+	<value value="0" name="RB_BUSY_CYCLES"/>
+	<value value="1" name="RB_BUSY_CYCLES_BINNING"/>
+	<value value="2" name="RB_BUSY_CYCLES_RENDERING"/>
+	<value value="3" name="RB_BUSY_CYCLES_RESOLVE"/>
+	<value value="4" name="RB_STARVE_CYCLES_BY_SP"/>
+	<value value="5" name="RB_STARVE_CYCLES_BY_RAS"/>
+	<value value="6" name="RB_STARVE_CYCLES_BY_MARB"/>
+	<value value="7" name="RB_STALL_CYCLES_BY_MARB"/>
+	<value value="8" name="RB_STALL_CYCLES_BY_HLSQ"/>
+	<value value="9" name="RB_RB_RB_MARB_DATA"/>
+	<value value="10" name="RB_SP_RB_QUAD"/>
+	<value value="11" name="RB_RAS_RB_Z_QUADS"/>
+	<value value="12" name="RB_GMEM_CH0_READ"/>
+	<value value="13" name="RB_GMEM_CH1_READ"/>
+	<value value="14" name="RB_GMEM_CH0_WRITE"/>
+	<value value="15" name="RB_GMEM_CH1_WRITE"/>
+	<value value="16" name="RB_CP_CONTEXT_DONE"/>
+	<value value="17" name="RB_CP_CACHE_FLUSH"/>
+	<value value="18" name="RB_CP_ZPASS_DONE"/>
+	<value value="19" name="RB_STALL_FIFO0_FULL"/>
+	<value value="20" name="RB_STALL_FIFO1_FULL"/>
+	<value value="21" name="RB_STALL_FIFO2_FULL"/>
+	<value value="22" name="RB_STALL_FIFO3_FULL"/>
+	<value value="23" name="RB_RB_HLSQ_TRANSACTIONS"/>
+	<value value="24" name="RB_Z_READ"/>
+	<value value="25" name="RB_Z_WRITE"/>
+	<value value="26" name="RB_C_READ"/>
+	<value value="27" name="RB_C_WRITE"/>
+	<value value="28" name="RB_C_READ_LATENCY"/>
+	<value value="29" name="RB_Z_READ_LATENCY"/>
+	<value value="30" name="RB_STALL_BY_UCHE"/>
+	<value value="31" name="RB_MARB_UCHE_TRANSACTIONS"/>
+	<value value="32" name="RB_CACHE_STALL_MISS"/>
+	<value value="33" name="RB_CACHE_STALL_FIFO_FULL"/>
+	<value value="34" name="RB_8BIT_BLENDER_UNITS_ACTIVE"/>
+	<value value="35" name="RB_16BIT_BLENDER_UNITS_ACTIVE"/>
+	<value value="36" name="RB_SAMPLER_UNITS_ACTIVE"/>
+	<value value="38" name="RB_TOTAL_PASS"/>
+	<value value="39" name="RB_Z_PASS"/>
+	<value value="40" name="RB_Z_FAIL"/>
+	<value value="41" name="RB_S_FAIL"/>
+	<value value="42" name="RB_POWER0"/>
+	<value value="43" name="RB_POWER1"/>
+	<value value="44" name="RB_POWER2"/>
+	<value value="45" name="RB_POWER3"/>
+	<value value="46" name="RB_POWER4"/>
+	<value value="47" name="RB_POWER5"/>
+	<value value="48" name="RB_POWER6"/>
+	<value value="49" name="RB_POWER7"/>
+</enum>
+
+<enum name="a4xx_rbbm_perfcounter_select">
+	<value value="0" name="RBBM_ALWAYS_ON"/>
+	<value value="1" name="RBBM_VBIF_BUSY"/>
+	<value value="2" name="RBBM_TSE_BUSY"/>
+	<value value="3" name="RBBM_RAS_BUSY"/>
+	<value value="4" name="RBBM_PC_DCALL_BUSY"/>
+	<value value="5" name="RBBM_PC_VSD_BUSY"/>
+	<value value="6" name="RBBM_VFD_BUSY"/>
+	<value value="7" name="RBBM_VPC_BUSY"/>
+	<value value="8" name="RBBM_UCHE_BUSY"/>
+	<value value="9" name="RBBM_VSC_BUSY"/>
+	<value value="10" name="RBBM_HLSQ_BUSY"/>
+	<value value="11" name="RBBM_ANY_RB_BUSY"/>
+	<value value="12" name="RBBM_ANY_TPL1_BUSY"/>
+	<value value="13" name="RBBM_ANY_SP_BUSY"/>
+	<value value="14" name="RBBM_ANY_MARB_BUSY"/>
+	<value value="15" name="RBBM_ANY_ARB_BUSY"/>
+	<value value="16" name="RBBM_AHB_STATUS_BUSY"/>
+	<value value="17" name="RBBM_AHB_STATUS_STALLED"/>
+	<value value="18" name="RBBM_AHB_STATUS_TXFR"/>
+	<value value="19" name="RBBM_AHB_STATUS_TXFR_SPLIT"/>
+	<value value="20" name="RBBM_AHB_STATUS_TXFR_ERROR"/>
+	<value value="21" name="RBBM_AHB_STATUS_LONG_STALL"/>
+	<value value="22" name="RBBM_STATUS_MASKED"/>
+	<value value="23" name="RBBM_CP_BUSY_GFX_CORE_IDLE"/>
+	<value value="24" name="RBBM_TESS_BUSY"/>
+	<value value="25" name="RBBM_COM_BUSY"/>
+	<value value="32" name="RBBM_DCOM_BUSY"/>
+	<value value="33" name="RBBM_ANY_CCU_BUSY"/>
+	<value value="34" name="RBBM_DPM_BUSY"/>
+</enum>
+
+<enum name="a4xx_sp_perfcounter_select">
+	<value value="0" name="SP_LM_LOAD_INSTRUCTIONS"/>
+	<value value="1" name="SP_LM_STORE_INSTRUCTIONS"/>
+	<value value="2" name="SP_LM_ATOMICS"/>
+	<value value="3" name="SP_GM_LOAD_INSTRUCTIONS"/>
+	<value value="4" name="SP_GM_STORE_INSTRUCTIONS"/>
+	<value value="5" name="SP_GM_ATOMICS"/>
+	<value value="6" name="SP_VS_STAGE_TEX_INSTRUCTIONS"/>
+	<value value="7" name="SP_VS_STAGE_CFLOW_INSTRUCTIONS"/>
+	<value value="8" name="SP_VS_STAGE_EFU_INSTRUCTIONS"/>
+	<value value="9" name="SP_VS_STAGE_FULL_ALU_INSTRUCTIONS"/>
+	<value value="10" name="SP_VS_STAGE_HALF_ALU_INSTRUCTIONS"/>
+	<value value="11" name="SP_FS_STAGE_TEX_INSTRUCTIONS"/>
+	<value value="12" name="SP_FS_STAGE_CFLOW_INSTRUCTIONS"/>
+	<value value="13" name="SP_FS_STAGE_EFU_INSTRUCTIONS"/>
+	<value value="14" name="SP_FS_STAGE_FULL_ALU_INSTRUCTIONS"/>
+	<value value="15" name="SP_FS_STAGE_HALF_ALU_INSTRUCTIONS"/>
+	<value value="17" name="SP_VS_INSTRUCTIONS"/>
+	<value value="18" name="SP_FS_INSTRUCTIONS"/>
+	<value value="19" name="SP_ADDR_LOCK_COUNT"/>
+	<value value="20" name="SP_UCHE_READ_TRANS"/>
+	<value value="21" name="SP_UCHE_WRITE_TRANS"/>
+	<value value="22" name="SP_EXPORT_VPC_TRANS"/>
+	<value value="23" name="SP_EXPORT_RB_TRANS"/>
+	<value value="24" name="SP_PIXELS_KILLED"/>
+	<value value="25" name="SP_ICL1_REQUESTS"/>
+	<value value="26" name="SP_ICL1_MISSES"/>
+	<value value="27" name="SP_ICL0_REQUESTS"/>
+	<value value="28" name="SP_ICL0_MISSES"/>
+	<value value="29" name="SP_ALU_WORKING_CYCLES"/>
+	<value value="30" name="SP_EFU_WORKING_CYCLES"/>
+	<value value="31" name="SP_STALL_CYCLES_BY_VPC"/>
+	<value value="32" name="SP_STALL_CYCLES_BY_TP"/>
+	<value value="33" name="SP_STALL_CYCLES_BY_UCHE"/>
+	<value value="34" name="SP_STALL_CYCLES_BY_RB"/>
+	<value value="35" name="SP_BUSY_CYCLES"/>
+	<value value="36" name="SP_HS_INSTRUCTIONS"/>
+	<value value="37" name="SP_DS_INSTRUCTIONS"/>
+	<value value="38" name="SP_GS_INSTRUCTIONS"/>
+	<value value="39" name="SP_CS_INSTRUCTIONS"/>
+	<value value="40" name="SP_SCHEDULER_NON_WORKING"/>
+	<value value="41" name="SP_WAVE_CONTEXTS"/>
+	<value value="42" name="SP_WAVE_CONTEXT_CYCLES"/>
+	<value value="43" name="SP_POWER0"/>
+	<value value="44" name="SP_POWER1"/>
+	<value value="45" name="SP_POWER2"/>
+	<value value="46" name="SP_POWER3"/>
+	<value value="47" name="SP_POWER4"/>
+	<value value="48" name="SP_POWER5"/>
+	<value value="49" name="SP_POWER6"/>
+	<value value="50" name="SP_POWER7"/>
+	<value value="51" name="SP_POWER8"/>
+	<value value="52" name="SP_POWER9"/>
+	<value value="53" name="SP_POWER10"/>
+	<value value="54" name="SP_POWER11"/>
+	<value value="55" name="SP_POWER12"/>
+	<value value="56" name="SP_POWER13"/>
+	<value value="57" name="SP_POWER14"/>
+	<value value="58" name="SP_POWER15"/>
+</enum>
+
+<enum name="a4xx_tp_perfcounter_select">
+	<value value="0" name="TP_L1_REQUESTS"/>
+	<value value="1" name="TP_L1_MISSES"/>
+	<value value="8" name="TP_QUADS_OFFSET"/>
+	<value value="9" name="TP_QUAD_SHADOW"/>
+	<value value="10" name="TP_QUADS_ARRAY"/>
+	<value value="11" name="TP_QUADS_GRADIENT"/>
+	<value value="12" name="TP_QUADS_1D2D"/>
+	<value value="13" name="TP_QUADS_3DCUBE"/>
+	<value value="16" name="TP_BUSY_CYCLES"/>
+	<value value="17" name="TP_STALL_CYCLES_BY_ARB"/>
+	<value value="20" name="TP_STATE_CACHE_REQUESTS"/>
+	<value value="21" name="TP_STATE_CACHE_MISSES"/>
+	<value value="22" name="TP_POWER0"/>
+	<value value="23" name="TP_POWER1"/>
+	<value value="24" name="TP_POWER2"/>
+	<value value="25" name="TP_POWER3"/>
+	<value value="26" name="TP_POWER4"/>
+	<value value="27" name="TP_POWER5"/>
+	<value value="28" name="TP_POWER6"/>
+	<value value="29" name="TP_POWER7"/>
+</enum>
+
+<enum name="a4xx_uche_perfcounter_select">
+	<value value="0" name="UCHE_VBIF_READ_BEATS_TP"/>
+	<value value="1" name="UCHE_VBIF_READ_BEATS_VFD"/>
+	<value value="2" name="UCHE_VBIF_READ_BEATS_HLSQ"/>
+	<value value="3" name="UCHE_VBIF_READ_BEATS_MARB"/>
+	<value value="4" name="UCHE_VBIF_READ_BEATS_SP"/>
+	<value value="5" name="UCHE_READ_REQUESTS_TP"/>
+	<value value="6" name="UCHE_READ_REQUESTS_VFD"/>
+	<value value="7" name="UCHE_READ_REQUESTS_HLSQ"/>
+	<value value="8" name="UCHE_READ_REQUESTS_MARB"/>
+	<value value="9" name="UCHE_READ_REQUESTS_SP"/>
+	<value value="10" name="UCHE_WRITE_REQUESTS_MARB"/>
+	<value value="11" name="UCHE_WRITE_REQUESTS_SP"/>
+	<value value="12" name="UCHE_TAG_CHECK_FAILS"/>
+	<value value="13" name="UCHE_EVICTS"/>
+	<value value="14" name="UCHE_FLUSHES"/>
+	<value value="15" name="UCHE_VBIF_LATENCY_CYCLES"/>
+	<value value="16" name="UCHE_VBIF_LATENCY_SAMPLES"/>
+	<value value="17" name="UCHE_BUSY_CYCLES"/>
+	<value value="18" name="UCHE_VBIF_READ_BEATS_PC"/>
+	<value value="19" name="UCHE_READ_REQUESTS_PC"/>
+	<value value="20" name="UCHE_WRITE_REQUESTS_VPC"/>
+	<value value="21" name="UCHE_STALL_BY_VBIF"/>
+	<value value="22" name="UCHE_WRITE_REQUESTS_VSC"/>
+	<value value="23" name="UCHE_POWER0"/>
+	<value value="24" name="UCHE_POWER1"/>
+	<value value="25" name="UCHE_POWER2"/>
+	<value value="26" name="UCHE_POWER3"/>
+	<value value="27" name="UCHE_POWER4"/>
+	<value value="28" name="UCHE_POWER5"/>
+	<value value="29" name="UCHE_POWER6"/>
+	<value value="30" name="UCHE_POWER7"/>
+</enum>
+
+<enum name="a4xx_vbif_perfcounter_select">
+	<value value="0" name="AXI_READ_REQUESTS_ID_0"/>
+	<value value="1" name="AXI_READ_REQUESTS_ID_1"/>
+	<value value="2" name="AXI_READ_REQUESTS_ID_2"/>
+	<value value="3" name="AXI_READ_REQUESTS_ID_3"/>
+	<value value="4" name="AXI_READ_REQUESTS_ID_4"/>
+	<value value="5" name="AXI_READ_REQUESTS_ID_5"/>
+	<value value="6" name="AXI_READ_REQUESTS_ID_6"/>
+	<value value="7" name="AXI_READ_REQUESTS_ID_7"/>
+	<value value="8" name="AXI_READ_REQUESTS_ID_8"/>
+	<value value="9" name="AXI_READ_REQUESTS_ID_9"/>
+	<value value="10" name="AXI_READ_REQUESTS_ID_10"/>
+	<value value="11" name="AXI_READ_REQUESTS_ID_11"/>
+	<value value="12" name="AXI_READ_REQUESTS_ID_12"/>
+	<value value="13" name="AXI_READ_REQUESTS_ID_13"/>
+	<value value="14" name="AXI_READ_REQUESTS_ID_14"/>
+	<value value="15" name="AXI_READ_REQUESTS_ID_15"/>
+	<value value="16" name="AXI0_READ_REQUESTS_TOTAL"/>
+	<value value="17" name="AXI1_READ_REQUESTS_TOTAL"/>
+	<value value="18" name="AXI2_READ_REQUESTS_TOTAL"/>
+	<value value="19" name="AXI3_READ_REQUESTS_TOTAL"/>
+	<value value="20" name="AXI_READ_REQUESTS_TOTAL"/>
+	<value value="21" name="AXI_WRITE_REQUESTS_ID_0"/>
+	<value value="22" name="AXI_WRITE_REQUESTS_ID_1"/>
+	<value value="23" name="AXI_WRITE_REQUESTS_ID_2"/>
+	<value value="24" name="AXI_WRITE_REQUESTS_ID_3"/>
+	<value value="25" name="AXI_WRITE_REQUESTS_ID_4"/>
+	<value value="26" name="AXI_WRITE_REQUESTS_ID_5"/>
+	<value value="27" name="AXI_WRITE_REQUESTS_ID_6"/>
+	<value value="28" name="AXI_WRITE_REQUESTS_ID_7"/>
+	<value value="29" name="AXI_WRITE_REQUESTS_ID_8"/>
+	<value value="30" name="AXI_WRITE_REQUESTS_ID_9"/>
+	<value value="31" name="AXI_WRITE_REQUESTS_ID_10"/>
+	<value value="32" name="AXI_WRITE_REQUESTS_ID_11"/>
+	<value value="33" name="AXI_WRITE_REQUESTS_ID_12"/>
+	<value value="34" name="AXI_WRITE_REQUESTS_ID_13"/>
+	<value value="35" name="AXI_WRITE_REQUESTS_ID_14"/>
+	<value value="36" name="AXI_WRITE_REQUESTS_ID_15"/>
+	<value value="37" name="AXI0_WRITE_REQUESTS_TOTAL"/>
+	<value value="38" name="AXI1_WRITE_REQUESTS_TOTAL"/>
+	<value value="39" name="AXI2_WRITE_REQUESTS_TOTAL"/>
+	<value value="40" name="AXI3_WRITE_REQUESTS_TOTAL"/>
+	<value value="41" name="AXI_WRITE_REQUESTS_TOTAL"/>
+	<value value="42" name="AXI_TOTAL_REQUESTS"/>
+	<value value="43" name="AXI_READ_DATA_BEATS_ID_0"/>
+	<value value="44" name="AXI_READ_DATA_BEATS_ID_1"/>
+	<value value="45" name="AXI_READ_DATA_BEATS_ID_2"/>
+	<value value="46" name="AXI_READ_DATA_BEATS_ID_3"/>
+	<value value="47" name="AXI_READ_DATA_BEATS_ID_4"/>
+	<value value="48" name="AXI_READ_DATA_BEATS_ID_5"/>
+	<value value="49" name="AXI_READ_DATA_BEATS_ID_6"/>
+	<value value="50" name="AXI_READ_DATA_BEATS_ID_7"/>
+	<value value="51" name="AXI_READ_DATA_BEATS_ID_8"/>
+	<value value="52" name="AXI_READ_DATA_BEATS_ID_9"/>
+	<value value="53" name="AXI_READ_DATA_BEATS_ID_10"/>
+	<value value="54" name="AXI_READ_DATA_BEATS_ID_11"/>
+	<value value="55" name="AXI_READ_DATA_BEATS_ID_12"/>
+	<value value="56" name="AXI_READ_DATA_BEATS_ID_13"/>
+	<value value="57" name="AXI_READ_DATA_BEATS_ID_14"/>
+	<value value="58" name="AXI_READ_DATA_BEATS_ID_15"/>
+	<value value="59" name="AXI0_READ_DATA_BEATS_TOTAL"/>
+	<value value="60" name="AXI1_READ_DATA_BEATS_TOTAL"/>
+	<value value="61" name="AXI2_READ_DATA_BEATS_TOTAL"/>
+	<value value="62" name="AXI3_READ_DATA_BEATS_TOTAL"/>
+	<value value="63" name="AXI_READ_DATA_BEATS_TOTAL"/>
+	<value value="64" name="AXI_WRITE_DATA_BEATS_ID_0"/>
+	<value value="65" name="AXI_WRITE_DATA_BEATS_ID_1"/>
+	<value value="66" name="AXI_WRITE_DATA_BEATS_ID_2"/>
+	<value value="67" name="AXI_WRITE_DATA_BEATS_ID_3"/>
+	<value value="68" name="AXI_WRITE_DATA_BEATS_ID_4"/>
+	<value value="69" name="AXI_WRITE_DATA_BEATS_ID_5"/>
+	<value value="70" name="AXI_WRITE_DATA_BEATS_ID_6"/>
+	<value value="71" name="AXI_WRITE_DATA_BEATS_ID_7"/>
+	<value value="72" name="AXI_WRITE_DATA_BEATS_ID_8"/>
+	<value value="73" name="AXI_WRITE_DATA_BEATS_ID_9"/>
+	<value value="74" name="AXI_WRITE_DATA_BEATS_ID_10"/>
+	<value value="75" name="AXI_WRITE_DATA_BEATS_ID_11"/>
+	<value value="76" name="AXI_WRITE_DATA_BEATS_ID_12"/>
+	<value value="77" name="AXI_WRITE_DATA_BEATS_ID_13"/>
+	<value value="78" name="AXI_WRITE_DATA_BEATS_ID_14"/>
+	<value value="79" name="AXI_WRITE_DATA_BEATS_ID_15"/>
+	<value value="80" name="AXI0_WRITE_DATA_BEATS_TOTAL"/>
+	<value value="81" name="AXI1_WRITE_DATA_BEATS_TOTAL"/>
+	<value value="82" name="AXI2_WRITE_DATA_BEATS_TOTAL"/>
+	<value value="83" name="AXI3_WRITE_DATA_BEATS_TOTAL"/>
+	<value value="84" name="AXI_WRITE_DATA_BEATS_TOTAL"/>
+	<value value="85" name="AXI_DATA_BEATS_TOTAL"/>
+	<value value="86" name="CYCLES_HELD_OFF_ID_0"/>
+	<value value="87" name="CYCLES_HELD_OFF_ID_1"/>
+	<value value="88" name="CYCLES_HELD_OFF_ID_2"/>
+	<value value="89" name="CYCLES_HELD_OFF_ID_3"/>
+	<value value="90" name="CYCLES_HELD_OFF_ID_4"/>
+	<value value="91" name="CYCLES_HELD_OFF_ID_5"/>
+	<value value="92" name="CYCLES_HELD_OFF_ID_6"/>
+	<value value="93" name="CYCLES_HELD_OFF_ID_7"/>
+	<value value="94" name="CYCLES_HELD_OFF_ID_8"/>
+	<value value="95" name="CYCLES_HELD_OFF_ID_9"/>
+	<value value="96" name="CYCLES_HELD_OFF_ID_10"/>
+	<value value="97" name="CYCLES_HELD_OFF_ID_11"/>
+	<value value="98" name="CYCLES_HELD_OFF_ID_12"/>
+	<value value="99" name="CYCLES_HELD_OFF_ID_13"/>
+	<value value="100" name="CYCLES_HELD_OFF_ID_14"/>
+	<value value="101" name="CYCLES_HELD_OFF_ID_15"/>
+	<value value="102" name="AXI_READ_REQUEST_HELD_OFF"/>
+	<value value="103" name="AXI_WRITE_REQUEST_HELD_OFF"/>
+	<value value="104" name="AXI_REQUEST_HELD_OFF"/>
+	<value value="105" name="AXI_WRITE_DATA_HELD_OFF"/>
+	<value value="106" name="OCMEM_AXI_READ_REQUEST_HELD_OFF"/>
+	<value value="107" name="OCMEM_AXI_WRITE_REQUEST_HELD_OFF"/>
+	<value value="108" name="OCMEM_AXI_REQUEST_HELD_OFF"/>
+	<value value="109" name="OCMEM_AXI_WRITE_DATA_HELD_OFF"/>
+	<value value="110" name="ELAPSED_CYCLES_DDR"/>
+	<value value="111" name="ELAPSED_CYCLES_OCMEM"/>
+</enum>
+
+<enum name="a4xx_vfd_perfcounter_select">
+	<value value="0" name="VFD_UCHE_BYTE_FETCHED"/>
+	<value value="1" name="VFD_UCHE_TRANS"/>
+	<value value="3" name="VFD_FETCH_INSTRUCTIONS"/>
+	<value value="5" name="VFD_BUSY_CYCLES"/>
+	<value value="6" name="VFD_STALL_CYCLES_UCHE"/>
+	<value value="7" name="VFD_STALL_CYCLES_HLSQ"/>
+	<value value="8" name="VFD_STALL_CYCLES_VPC_BYPASS"/>
+	<value value="9" name="VFD_STALL_CYCLES_VPC_ALLOC"/>
+	<value value="13" name="VFD_MODE_0_FIBERS"/>
+	<value value="14" name="VFD_MODE_1_FIBERS"/>
+	<value value="15" name="VFD_MODE_2_FIBERS"/>
+	<value value="16" name="VFD_MODE_3_FIBERS"/>
+	<value value="17" name="VFD_MODE_4_FIBERS"/>
+	<value value="18" name="VFD_BFIFO_STALL"/>
+	<value value="19" name="VFD_NUM_VERTICES_TOTAL"/>
+	<value value="20" name="VFD_PACKER_FULL"/>
+	<value value="21" name="VFD_UCHE_REQUEST_FIFO_FULL"/>
+	<value value="22" name="VFD_STARVE_CYCLES_PC"/>
+	<value value="23" name="VFD_STARVE_CYCLES_UCHE"/>
+</enum>
+
+<enum name="a4xx_vpc_perfcounter_select">
+	<value value="2" name="VPC_SP_LM_COMPONENTS"/>
+	<value value="3" name="VPC_SP0_LM_BYTES"/>
+	<value value="4" name="VPC_SP1_LM_BYTES"/>
+	<value value="5" name="VPC_SP2_LM_BYTES"/>
+	<value value="6" name="VPC_SP3_LM_BYTES"/>
+	<value value="7" name="VPC_WORKING_CYCLES"/>
+	<value value="8" name="VPC_STALL_CYCLES_LM"/>
+	<value value="9" name="VPC_STARVE_CYCLES_RAS"/>
+	<value value="10" name="VPC_STREAMOUT_CYCLES"/>
+	<value value="12" name="VPC_UCHE_TRANSACTIONS"/>
+	<value value="13" name="VPC_STALL_CYCLES_UCHE"/>
+	<value value="14" name="VPC_BUSY_CYCLES"/>
+	<value value="15" name="VPC_STARVE_CYCLES_SP"/>
+</enum>
+
+<enum name="a4xx_vsc_perfcounter_select">
+	<value value="0" name="VSC_BUSY_CYCLES"/>
+	<value value="1" name="VSC_WORKING_CYCLES"/>
+	<value value="2" name="VSC_STALL_CYCLES_UCHE"/>
+	<value value="3" name="VSC_STARVE_CYCLES_RAS"/>
+	<value value="4" name="VSC_EOT_NUM"/>
+</enum>
+
+<domain name="A4XX" width="32">
+	<!-- RB registers -->
+	<reg32 offset="0x0cc0" name="RB_GMEM_BASE_ADDR"/>
+	<reg32 offset="0x0cc7" name="RB_PERFCTR_RB_SEL_0" type="a4xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cc8" name="RB_PERFCTR_RB_SEL_1" type="a4xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cc9" name="RB_PERFCTR_RB_SEL_2" type="a4xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cca" name="RB_PERFCTR_RB_SEL_3" type="a4xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0ccb" name="RB_PERFCTR_RB_SEL_4" type="a4xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0ccc" name="RB_PERFCTR_RB_SEL_5" type="a4xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0ccd" name="RB_PERFCTR_RB_SEL_6" type="a4xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cce" name="RB_PERFCTR_RB_SEL_7" type="a4xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0ccf" name="RB_PERFCTR_CCU_SEL_0" type="a4xx_ccu_perfcounter_select"/>
+	<reg32 offset="0x0cd0" name="RB_PERFCTR_CCU_SEL_1" type="a4xx_ccu_perfcounter_select"/>
+	<reg32 offset="0x0cd1" name="RB_PERFCTR_CCU_SEL_2" type="a4xx_ccu_perfcounter_select"/>
+	<reg32 offset="0x0cd2" name="RB_PERFCTR_CCU_SEL_3" type="a4xx_ccu_perfcounter_select"/>
+	<reg32 offset="0x0ce0" name="RB_FRAME_BUFFER_DIMENSION">
+		<bitfield name="WIDTH" low="0" high="13" type="uint"/>
+		<bitfield name="HEIGHT" low="16" high="29" type="uint"/>
+	</reg32>
+	<reg32 offset="0x20cc" name="RB_CLEAR_COLOR_DW0"/>
+	<reg32 offset="0x20cd" name="RB_CLEAR_COLOR_DW1"/>
+	<reg32 offset="0x20ce" name="RB_CLEAR_COLOR_DW2"/>
+	<reg32 offset="0x20cf" name="RB_CLEAR_COLOR_DW3"/>
+	<reg32 offset="0x20a0" name="RB_MODE_CONTROL">
+		<!--
+		for non-bypass mode, these are bin width/height..  although
+		possibly bigger bitfields to hold entire width/height for
+		gmem-bypass??  Either way, it appears to need to be multiple
+		of 32..
+		-->
+		<bitfield name="WIDTH" low="0" high="5" shr="5" type="uint"/>
+		<bitfield name="HEIGHT" low="8" high="13" shr="5" type="uint"/>
+		<bitfield name="ENABLE_GMEM" pos="16" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x20a1" name="RB_RENDER_CONTROL">
+		<bitfield name="BINNING_PASS" pos="0" type="boolean"/>
+		<!-- nearly everything has bit3 set.. -->
+		<!-- bit5 set on resolve and tiling pass -->
+		<bitfield name="DISABLE_COLOR_PIPE" pos="5" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x20a2" name="RB_MSAA_CONTROL">
+		<bitfield name="DISABLE" pos="12" type="boolean"/>
+		<bitfield name="SAMPLES" low="13" high="15" type="uint"/>
+	</reg32>
+	<reg32 offset="0x20a3" name="RB_RENDER_CONTROL2">
+		<bitfield name="COORD_MASK" low="0" high="3" type="hex"/>
+		<bitfield name="SAMPLEMASK" pos="4" type="boolean"/>
+		<bitfield name="FACENESS" pos="5" type="boolean"/>
+		<bitfield name="SAMPLEID" pos="6" type="boolean"/>
+		<bitfield name="MSAA_SAMPLES" low="7" high="9" type="uint"/>
+		<bitfield name="SAMPLEID_HR" pos="11" type="boolean"/>
+		<bitfield name="IJ_PERSP_PIXEL" pos="12" type="boolean"/>
+		<!-- the 2 below are just educated guesses -->
+		<bitfield name="IJ_PERSP_CENTROID" pos="13" type="boolean"/>
+		<bitfield name="IJ_PERSP_SAMPLE" pos="14" type="boolean"/>
+		<!-- needs to be enabled to get nopersp values,
+		     perhaps other cases too? -->
+		<bitfield name="SIZE" pos="15" type="boolean"/>
+	</reg32>
+	<array offset="0x20a4" name="RB_MRT" stride="5" length="8">
+		<reg32 offset="0x0" name="CONTROL">
+			<bitfield name="READ_DEST_ENABLE" pos="3" type="boolean"/>
+			<!-- both these bits seem to get set when enabling GL_BLEND.. -->
+			<bitfield name="BLEND" pos="4" type="boolean"/>
+			<bitfield name="BLEND2" pos="5" type="boolean"/>
+			<bitfield name="ROP_ENABLE" pos="6" type="boolean"/>
+			<bitfield name="ROP_CODE" low="8" high="11" type="a3xx_rop_code"/>
+			<bitfield name="COMPONENT_ENABLE" low="24" high="27" type="hex"/>
+		</reg32>
+		<reg32 offset="0x1" name="BUF_INFO">
+			<bitfield name="COLOR_FORMAT" low="0" high="5" type="a4xx_color_fmt"/>
+			<!--
+			    guestimate position of COLOR_TILE_MODE..  this works out if
+			    common value is 2, like on a3xx..
+			 -->
+			<bitfield name="COLOR_TILE_MODE" low="6" high="7" type="a4xx_tile_mode"/>
+			<bitfield name="DITHER_MODE" low="9" high="10" type="adreno_rb_dither_mode"/>
+			<bitfield name="COLOR_SWAP" low="11" high="12" type="a3xx_color_swap"/>
+			<bitfield name="COLOR_SRGB" pos="13" type="boolean"/>
+			<!-- note: possibly some # of lsb's aren't there: -->
+			<doc>
+				Pitch (actually, appears to be pitch in bytes, so really is a stride)
+				in GMEM, so pitch of the current tile.
+			</doc>
+			<bitfield name="COLOR_BUF_PITCH" low="14" high="31" shr="4" type="uint"/>
+		</reg32>
+		<reg32 offset="0x2" name="BASE"/>
+		<reg32 offset="0x3" name="CONTROL3">
+			<!-- probably missing some lsb's.. and guessing upper size -->
+			<!-- pitch * cpp * msaa: -->
+			<bitfield name="STRIDE" low="3" high="25" type="uint"/>
+		</reg32>
+		<reg32 offset="0x4" name="BLEND_CONTROL">
+			<bitfield name="RGB_SRC_FACTOR" low="0" high="4" type="adreno_rb_blend_factor"/>
+			<bitfield name="RGB_BLEND_OPCODE" low="5" high="7" type="a3xx_rb_blend_opcode"/>
+			<bitfield name="RGB_DEST_FACTOR" low="8" high="12" type="adreno_rb_blend_factor"/>
+			<bitfield name="ALPHA_SRC_FACTOR" low="16" high="20" type="adreno_rb_blend_factor"/>
+			<bitfield name="ALPHA_BLEND_OPCODE" low="21" high="23" type="a3xx_rb_blend_opcode"/>
+			<bitfield name="ALPHA_DEST_FACTOR" low="24" high="28" type="adreno_rb_blend_factor"/>
+		</reg32>
+	</array>
+
+	<reg32 offset="0x20f0" name="RB_BLEND_RED">
+		<bitfield name="UINT" low="0" high="7" type="hex"/>
+		<bitfield name="SINT" low="8" high="15" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+	<reg32 offset="0x20f1" name="RB_BLEND_RED_F32" type="float"/>
+
+	<reg32 offset="0x20f2" name="RB_BLEND_GREEN">
+		<bitfield name="UINT" low="0" high="7" type="hex"/>
+		<bitfield name="SINT" low="8" high="15" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+	<reg32 offset="0x20f3" name="RB_BLEND_GREEN_F32" type="float"/>
+
+	<reg32 offset="0x20f4" name="RB_BLEND_BLUE">
+		<bitfield name="UINT" low="0" high="7" type="hex"/>
+		<bitfield name="SINT" low="8" high="15" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+	<reg32 offset="0x20f5" name="RB_BLEND_BLUE_F32" type="float"/>
+
+	<reg32 offset="0x20f6" name="RB_BLEND_ALPHA">
+		<bitfield name="UINT" low="0" high="7" type="hex"/>
+		<bitfield name="SINT" low="8" high="15" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+	<reg32 offset="0x20f7" name="RB_BLEND_ALPHA_F32" type="float"/>
+
+	<reg32 offset="0x20f8" name="RB_ALPHA_CONTROL">
+		<bitfield name="ALPHA_REF" low="0" high="7" type="hex"/>
+		<bitfield name="ALPHA_TEST" pos="8" type="boolean"/>
+		<bitfield name="ALPHA_TEST_FUNC" low="9" high="11" type="adreno_compare_func"/>
+	</reg32>
+	<reg32 offset="0x20f9" name="RB_FS_OUTPUT">
+		<!-- per-mrt enable bit -->
+		<bitfield name="ENABLE_BLEND" low="0" high="7"/>
+		<bitfield name="INDEPENDENT_BLEND" pos="8" type="boolean"/>
+		<!-- a guess? -->
+		<bitfield name="SAMPLE_MASK" low="16" high="31"/>
+	</reg32>
+	<reg32 offset="0x20fa" name="RB_SAMPLE_COUNT_CONTROL">
+		<bitfield name="COPY" pos="1" type="boolean"/>
+		<bitfield name="ADDR" low="2" high="31" shr="2"/>
+	</reg32>
+	<!-- always 00000000 for binning pass, else 0000000f: -->
+	<reg32 offset="0x20fb" name="RB_RENDER_COMPONENTS">
+		<bitfield name="RT0" low="0" high="3"/>
+		<bitfield name="RT1" low="4" high="7"/>
+		<bitfield name="RT2" low="8" high="11"/>
+		<bitfield name="RT3" low="12" high="15"/>
+		<bitfield name="RT4" low="16" high="19"/>
+		<bitfield name="RT5" low="20" high="23"/>
+		<bitfield name="RT6" low="24" high="27"/>
+		<bitfield name="RT7" low="28" high="31"/>
+	</reg32>
+
+	<reg32 offset="0x20fc" name="RB_COPY_CONTROL">
+		<!-- not sure # of bits -->
+		<bitfield name="MSAA_RESOLVE" low="0" high="1" type="a3xx_msaa_samples"/>
+		<bitfield name="MODE" low="4" high="6" type="adreno_rb_copy_control_mode"/>
+		<bitfield name="FASTCLEAR" low="8" high="11" type="hex"/>
+		<bitfield name="GMEM_BASE" low="14" high="31" shr="14" type="hex"/>
+	</reg32>
+	<reg32 offset="0x20fd" name="RB_COPY_DEST_BASE">
+		<bitfield name="BASE" low="5" high="31" shr="5" type="hex"/>
+	</reg32>
+	<reg32 offset="0x20fe" name="RB_COPY_DEST_PITCH">
+		<doc>actually, appears to be pitch in bytes, so really is a stride</doc>
+		<!-- not actually sure about max pitch... -->
+		<bitfield name="PITCH" low="0" high="31" shr="5" type="uint"/>
+	</reg32>
+	<reg32 offset="0x20ff" name="RB_COPY_DEST_INFO">
+		<bitfield name="FORMAT" low="2" high="7" type="a4xx_color_fmt"/>
+		<bitfield name="SWAP" low="8" high="9" type="a3xx_color_swap"/>
+		<bitfield name="DITHER_MODE" low="10" high="11" type="adreno_rb_dither_mode"/>
+		<bitfield name="COMPONENT_ENABLE" low="14" high="17" type="hex"/>
+		<bitfield name="ENDIAN" low="18" high="20" type="adreno_rb_surface_endian"/>
+		<bitfield name="TILE" low="24" high="25" type="a4xx_tile_mode"/>
+	</reg32>
+	<reg32 offset="0x2100" name="RB_FS_OUTPUT_REG">
+		<!-- bit0 set except for binning pass.. -->
+		<bitfield name="MRT" low="0" high="3" type="uint"/>
+		<bitfield name="FRAG_WRITES_Z" pos="5" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2101" name="RB_DEPTH_CONTROL">
+		<!--
+			guessing that this matches a2xx with the stencil fields
+			moved out into RB_STENCIL_CONTROL?
+		 -->
+		<bitfield name="FRAG_WRITES_Z" pos="0" type="boolean"/>
+		<bitfield name="Z_TEST_ENABLE" pos="1" type="boolean"/>
+		<bitfield name="Z_WRITE_ENABLE" pos="2" type="boolean"/>
+		<bitfield name="ZFUNC" low="4" high="6" type="adreno_compare_func"/>
+		<bitfield name="Z_CLAMP_ENABLE" pos="7" type="boolean"/>
+		<bitfield name="EARLY_Z_DISABLE" pos="16" type="boolean"/>
+		<bitfield name="FORCE_FRAGZ_TO_FS" pos="17" type="boolean"/>
+		<doc>Z_READ_ENABLE bit is set for zfunc other than GL_ALWAYS or GL_NEVER</doc>
+		<bitfield name="Z_READ_ENABLE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2102" name="RB_DEPTH_CLEAR"/>
+	<reg32 offset="0x2103" name="RB_DEPTH_INFO">
+		<bitfield name="DEPTH_FORMAT" low="0" high="1" type="a4xx_depth_format"/>
+		<doc>
+			DEPTH_BASE is offset in GMEM to depth/stencil buffer, ie
+			bin_w * bin_h / 1024 (possible rounded up to multiple of
+			something??  ie. 39 becomes 40, 78 becomes 80.. 75 becomes
+			80.. so maybe it needs to be multiple of 8??
+		</doc>
+		<bitfield name="DEPTH_BASE" low="12" high="31" shr="12" type="hex"/>
+	</reg32>
+	<reg32 offset="0x2104" name="RB_DEPTH_PITCH" shr="5" type="uint">
+		<doc>stride of depth/stencil buffer</doc>
+	</reg32>
+	<reg32 offset="0x2105" name="RB_DEPTH_PITCH2" shr="5" type="uint">
+		<doc>???</doc>
+	</reg32>
+	<reg32 offset="0x2106" name="RB_STENCIL_CONTROL">
+		<bitfield name="STENCIL_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="STENCIL_ENABLE_BF" pos="1" type="boolean"/>
+		<!--
+			set for stencil operations that require read from stencil
+			buffer, but not for example for stencil clear (which does
+			not require read).. so guessing this is analogous to
+			READ_DEST_ENABLE for color buffer..
+		 -->
+		<bitfield name="STENCIL_READ" pos="2" type="boolean"/>
+		<bitfield name="FUNC" low="8" high="10" type="adreno_compare_func"/>
+		<bitfield name="FAIL" low="11" high="13" type="adreno_stencil_op"/>
+		<bitfield name="ZPASS" low="14" high="16" type="adreno_stencil_op"/>
+		<bitfield name="ZFAIL" low="17" high="19" type="adreno_stencil_op"/>
+		<bitfield name="FUNC_BF" low="20" high="22" type="adreno_compare_func"/>
+		<bitfield name="FAIL_BF" low="23" high="25" type="adreno_stencil_op"/>
+		<bitfield name="ZPASS_BF" low="26" high="28" type="adreno_stencil_op"/>
+		<bitfield name="ZFAIL_BF" low="29" high="31" type="adreno_stencil_op"/>
+	</reg32>
+	<reg32 offset="0x2107" name="RB_STENCIL_CONTROL2">
+		<!--
+		This seems to be set by blob if there is a stencil buffer
+		at all in GMEM, regardless of whether it is enabled for
+		a particular draw (ie. RB_STENCIL_CONTROL).  Not really
+		sure if that is required or just a quirk of the blob
+		-->
+		<bitfield name="STENCIL_BUFFER" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2108" name="RB_STENCIL_INFO">
+		<bitfield name="SEPARATE_STENCIL" pos="0" type="boolean"/>
+		<doc>Base address for stencil when not using interleaved depth/stencil</doc>
+		<bitfield name="STENCIL_BASE" low="12" high="31" shr="12" type="hex"/>
+	</reg32>
+	<reg32 offset="0x2109" name="RB_STENCIL_PITCH" shr="5" type="uint">
+		<doc>pitch of stencil buffer when not using interleaved depth/stencil</doc>
+	</reg32>
+
+	<reg32 offset="0x210b" name="RB_STENCILREFMASK" type="adreno_rb_stencilrefmask"/>
+	<reg32 offset="0x210c" name="RB_STENCILREFMASK_BF" type="adreno_rb_stencilrefmask"/>
+	<reg32 offset="0x210d" name="RB_BIN_OFFSET" type="adreno_reg_xy"/>
+	<array offset="0x2120" name="RB_VPORT_Z_CLAMP" stride="2" length="16">
+		<reg32 offset="0x0" name="MIN"/>
+		<reg32 offset="0x1" name="MAX"/>
+	</array>
+
+	<!-- RBBM registers -->
+	<reg32 offset="0x0000" name="RBBM_HW_VERSION"/>
+	<reg32 offset="0x0002" name="RBBM_HW_CONFIGURATION"/>
+	<array offset="0x4" name="RBBM_CLOCK_CTL_TP" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<array offset="0x8" name="RBBM_CLOCK_CTL2_TP" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<array offset="0xc" name="RBBM_CLOCK_HYST_TP" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<array offset="0x10" name="RBBM_CLOCK_DELAY_TP" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<reg32 offset="0x0014" name="RBBM_CLOCK_CTL_UCHE "/>
+	<reg32 offset="0x0015" name="RBBM_CLOCK_CTL2_UCHE"/>
+	<reg32 offset="0x0016" name="RBBM_CLOCK_CTL3_UCHE"/>
+	<reg32 offset="0x0017" name="RBBM_CLOCK_CTL4_UCHE"/>
+	<reg32 offset="0x0018" name="RBBM_CLOCK_HYST_UCHE"/>
+	<reg32 offset="0x0019" name="RBBM_CLOCK_DELAY_UCHE"/>
+	<reg32 offset="0x001a" name="RBBM_CLOCK_MODE_GPC"/>
+	<reg32 offset="0x001b" name="RBBM_CLOCK_DELAY_GPC"/>
+	<reg32 offset="0x001c" name="RBBM_CLOCK_HYST_GPC"/>
+	<reg32 offset="0x001d" name="RBBM_CLOCK_CTL_TSE_RAS_RBBM"/>
+	<reg32 offset="0x001e" name="RBBM_CLOCK_HYST_TSE_RAS_RBBM"/>
+	<reg32 offset="0x001f" name="RBBM_CLOCK_DELAY_TSE_RAS_RBBM"/>
+	<reg32 offset="0x0020" name="RBBM_CLOCK_CTL"/>
+	<reg32 offset="0x0021" name="RBBM_SP_HYST_CNT"/>
+	<reg32 offset="0x0022" name="RBBM_SW_RESET_CMD"/>
+	<reg32 offset="0x0023" name="RBBM_AHB_CTL0"/>
+	<reg32 offset="0x0024" name="RBBM_AHB_CTL1"/>
+	<reg32 offset="0x0025" name="RBBM_AHB_CMD"/>
+	<reg32 offset="0x0026" name="RBBM_RB_SUB_BLOCK_SEL_CTL"/>
+	<reg32 offset="0x0028" name="RBBM_RAM_ACC_63_32"/>
+	<reg32 offset="0x002b" name="RBBM_WAIT_IDLE_CLOCKS_CTL"/>
+	<reg32 offset="0x002f" name="RBBM_INTERFACE_HANG_INT_CTL"/>
+	<reg32 offset="0x0034" name="RBBM_INTERFACE_HANG_MASK_CTL4"/>
+	<reg32 offset="0x0036" name="RBBM_INT_CLEAR_CMD"/>
+	<reg32 offset="0x0037" name="RBBM_INT_0_MASK"/>
+	<reg32 offset="0x003e" name="RBBM_RBBM_CTL"/>
+	<reg32 offset="0x003f" name="RBBM_AHB_DEBUG_CTL"/>
+	<reg32 offset="0x0041" name="RBBM_VBIF_DEBUG_CTL"/>
+	<reg32 offset="0x0042" name="RBBM_CLOCK_CTL2"/>
+	<reg32 offset="0x0045" name="RBBM_BLOCK_SW_RESET_CMD"/>
+	<reg32 offset="0x0047" name="RBBM_RESET_CYCLES"/>
+	<reg32 offset="0x0049" name="RBBM_EXT_TRACE_BUS_CTL"/>
+	<reg32 offset="0x004a" name="RBBM_CFG_DEBBUS_SEL_A"/>
+	<reg32 offset="0x004b" name="RBBM_CFG_DEBBUS_SEL_B"/>
+	<reg32 offset="0x004c" name="RBBM_CFG_DEBBUS_SEL_C"/>
+	<reg32 offset="0x004d" name="RBBM_CFG_DEBBUS_SEL_D"/>
+	<reg32 offset="0x0098" name="RBBM_POWER_CNTL_IP">
+		<bitfield name="SW_COLLAPSE" pos="0" type="boolean"/>
+		<bitfield name="SP_TP_PWR_ON" pos="20" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x009c" name="RBBM_PERFCTR_CP_0_LO"/>
+	<reg32 offset="0x009d" name="RBBM_PERFCTR_CP_0_HI"/>
+	<reg32 offset="0x009e" name="RBBM_PERFCTR_CP_1_LO"/>
+	<reg32 offset="0x009f" name="RBBM_PERFCTR_CP_1_HI"/>
+	<reg32 offset="0x00a0" name="RBBM_PERFCTR_CP_2_LO"/>
+	<reg32 offset="0x00a1" name="RBBM_PERFCTR_CP_2_HI"/>
+	<reg32 offset="0x00a2" name="RBBM_PERFCTR_CP_3_LO"/>
+	<reg32 offset="0x00a3" name="RBBM_PERFCTR_CP_3_HI"/>
+	<reg32 offset="0x00a4" name="RBBM_PERFCTR_CP_4_LO"/>
+	<reg32 offset="0x00a5" name="RBBM_PERFCTR_CP_4_HI"/>
+	<reg32 offset="0x00a6" name="RBBM_PERFCTR_CP_5_LO"/>
+	<reg32 offset="0x00a7" name="RBBM_PERFCTR_CP_5_HI"/>
+	<reg32 offset="0x00a8" name="RBBM_PERFCTR_CP_6_LO"/>
+	<reg32 offset="0x00a9" name="RBBM_PERFCTR_CP_6_HI"/>
+	<reg32 offset="0x00aa" name="RBBM_PERFCTR_CP_7_LO"/>
+	<reg32 offset="0x00ab" name="RBBM_PERFCTR_CP_7_HI"/>
+	<reg32 offset="0x00ac" name="RBBM_PERFCTR_RBBM_0_LO"/>
+	<reg32 offset="0x00ad" name="RBBM_PERFCTR_RBBM_0_HI"/>
+	<reg32 offset="0x00ae" name="RBBM_PERFCTR_RBBM_1_LO"/>
+	<reg32 offset="0x00af" name="RBBM_PERFCTR_RBBM_1_HI"/>
+	<reg32 offset="0x00b0" name="RBBM_PERFCTR_RBBM_2_LO"/>
+	<reg32 offset="0x00b1" name="RBBM_PERFCTR_RBBM_2_HI"/>
+	<reg32 offset="0x00b2" name="RBBM_PERFCTR_RBBM_3_LO"/>
+	<reg32 offset="0x00b3" name="RBBM_PERFCTR_RBBM_3_HI"/>
+	<reg32 offset="0x00b4" name="RBBM_PERFCTR_PC_0_LO"/>
+	<reg32 offset="0x00b5" name="RBBM_PERFCTR_PC_0_HI"/>
+	<reg32 offset="0x00b6" name="RBBM_PERFCTR_PC_1_LO"/>
+	<reg32 offset="0x00b7" name="RBBM_PERFCTR_PC_1_HI"/>
+	<reg32 offset="0x00b8" name="RBBM_PERFCTR_PC_2_LO"/>
+	<reg32 offset="0x00b9" name="RBBM_PERFCTR_PC_2_HI"/>
+	<reg32 offset="0x00ba" name="RBBM_PERFCTR_PC_3_LO"/>
+	<reg32 offset="0x00bb" name="RBBM_PERFCTR_PC_3_HI"/>
+	<reg32 offset="0x00bc" name="RBBM_PERFCTR_PC_4_LO"/>
+	<reg32 offset="0x00bd" name="RBBM_PERFCTR_PC_4_HI"/>
+	<reg32 offset="0x00be" name="RBBM_PERFCTR_PC_5_LO"/>
+	<reg32 offset="0x00bf" name="RBBM_PERFCTR_PC_5_HI"/>
+	<reg32 offset="0x00c0" name="RBBM_PERFCTR_PC_6_LO"/>
+	<reg32 offset="0x00c1" name="RBBM_PERFCTR_PC_6_HI"/>
+	<reg32 offset="0x00c2" name="RBBM_PERFCTR_PC_7_LO"/>
+	<reg32 offset="0x00c3" name="RBBM_PERFCTR_PC_7_HI"/>
+	<reg32 offset="0x00c4" name="RBBM_PERFCTR_VFD_0_LO"/>
+	<reg32 offset="0x00c5" name="RBBM_PERFCTR_VFD_0_HI"/>
+	<reg32 offset="0x00c6" name="RBBM_PERFCTR_VFD_1_LO"/>
+	<reg32 offset="0x00c7" name="RBBM_PERFCTR_VFD_1_HI"/>
+	<reg32 offset="0x00c8" name="RBBM_PERFCTR_VFD_2_LO"/>
+	<reg32 offset="0x00c9" name="RBBM_PERFCTR_VFD_2_HI"/>
+	<reg32 offset="0x00ca" name="RBBM_PERFCTR_VFD_3_LO"/>
+	<reg32 offset="0x00cb" name="RBBM_PERFCTR_VFD_3_HI"/>
+	<reg32 offset="0x00cc" name="RBBM_PERFCTR_VFD_4_LO"/>
+	<reg32 offset="0x00cd" name="RBBM_PERFCTR_VFD_4_HI"/>
+	<reg32 offset="0x00ce" name="RBBM_PERFCTR_VFD_5_LO"/>
+	<reg32 offset="0x00cf" name="RBBM_PERFCTR_VFD_5_HI"/>
+	<reg32 offset="0x00d0" name="RBBM_PERFCTR_VFD_6_LO"/>
+	<reg32 offset="0x00d1" name="RBBM_PERFCTR_VFD_6_HI"/>
+	<reg32 offset="0x00d2" name="RBBM_PERFCTR_VFD_7_LO"/>
+	<reg32 offset="0x00d3" name="RBBM_PERFCTR_VFD_7_HI"/>
+	<reg32 offset="0x00d4" name="RBBM_PERFCTR_HLSQ_0_LO"/>
+	<reg32 offset="0x00d5" name="RBBM_PERFCTR_HLSQ_0_HI"/>
+	<reg32 offset="0x00d6" name="RBBM_PERFCTR_HLSQ_1_LO"/>
+	<reg32 offset="0x00d7" name="RBBM_PERFCTR_HLSQ_1_HI"/>
+	<reg32 offset="0x00d8" name="RBBM_PERFCTR_HLSQ_2_LO"/>
+	<reg32 offset="0x00d9" name="RBBM_PERFCTR_HLSQ_2_HI"/>
+	<reg32 offset="0x00da" name="RBBM_PERFCTR_HLSQ_3_LO"/>
+	<reg32 offset="0x00db" name="RBBM_PERFCTR_HLSQ_3_HI"/>
+	<reg32 offset="0x00dc" name="RBBM_PERFCTR_HLSQ_4_LO"/>
+	<reg32 offset="0x00dd" name="RBBM_PERFCTR_HLSQ_4_HI"/>
+	<reg32 offset="0x00de" name="RBBM_PERFCTR_HLSQ_5_LO"/>
+	<reg32 offset="0x00df" name="RBBM_PERFCTR_HLSQ_5_HI"/>
+	<reg32 offset="0x00e0" name="RBBM_PERFCTR_HLSQ_6_LO"/>
+	<reg32 offset="0x00e1" name="RBBM_PERFCTR_HLSQ_6_HI"/>
+	<reg32 offset="0x00e2" name="RBBM_PERFCTR_HLSQ_7_LO"/>
+	<reg32 offset="0x00e3" name="RBBM_PERFCTR_HLSQ_7_HI"/>
+	<reg32 offset="0x00e4" name="RBBM_PERFCTR_VPC_0_LO"/>
+	<reg32 offset="0x00e5" name="RBBM_PERFCTR_VPC_0_HI"/>
+	<reg32 offset="0x00e6" name="RBBM_PERFCTR_VPC_1_LO"/>
+	<reg32 offset="0x00e7" name="RBBM_PERFCTR_VPC_1_HI"/>
+	<reg32 offset="0x00e8" name="RBBM_PERFCTR_VPC_2_LO"/>
+	<reg32 offset="0x00e9" name="RBBM_PERFCTR_VPC_2_HI"/>
+	<reg32 offset="0x00ea" name="RBBM_PERFCTR_VPC_3_LO"/>
+	<reg32 offset="0x00eb" name="RBBM_PERFCTR_VPC_3_HI"/>
+	<reg32 offset="0x00ec" name="RBBM_PERFCTR_CCU_0_LO"/>
+	<reg32 offset="0x00ed" name="RBBM_PERFCTR_CCU_0_HI"/>
+	<reg32 offset="0x00ee" name="RBBM_PERFCTR_CCU_1_LO"/>
+	<reg32 offset="0x00ef" name="RBBM_PERFCTR_CCU_1_HI"/>
+	<reg32 offset="0x00f0" name="RBBM_PERFCTR_CCU_2_LO"/>
+	<reg32 offset="0x00f1" name="RBBM_PERFCTR_CCU_2_HI"/>
+	<reg32 offset="0x00f2" name="RBBM_PERFCTR_CCU_3_LO"/>
+	<reg32 offset="0x00f3" name="RBBM_PERFCTR_CCU_3_HI"/>
+	<reg32 offset="0x00f4" name="RBBM_PERFCTR_TSE_0_LO"/>
+	<reg32 offset="0x00f5" name="RBBM_PERFCTR_TSE_0_HI"/>
+	<reg32 offset="0x00f6" name="RBBM_PERFCTR_TSE_1_LO"/>
+	<reg32 offset="0x00f7" name="RBBM_PERFCTR_TSE_1_HI"/>
+	<reg32 offset="0x00f8" name="RBBM_PERFCTR_TSE_2_LO"/>
+	<reg32 offset="0x00f9" name="RBBM_PERFCTR_TSE_2_HI"/>
+	<reg32 offset="0x00fa" name="RBBM_PERFCTR_TSE_3_LO"/>
+	<reg32 offset="0x00fb" name="RBBM_PERFCTR_TSE_3_HI"/>
+	<reg32 offset="0x00fc" name="RBBM_PERFCTR_RAS_0_LO"/>
+	<reg32 offset="0x00fd" name="RBBM_PERFCTR_RAS_0_HI"/>
+	<reg32 offset="0x00fe" name="RBBM_PERFCTR_RAS_1_LO"/>
+	<reg32 offset="0x00ff" name="RBBM_PERFCTR_RAS_1_HI"/>
+	<reg32 offset="0x0100" name="RBBM_PERFCTR_RAS_2_LO"/>
+	<reg32 offset="0x0101" name="RBBM_PERFCTR_RAS_2_HI"/>
+	<reg32 offset="0x0102" name="RBBM_PERFCTR_RAS_3_LO"/>
+	<reg32 offset="0x0103" name="RBBM_PERFCTR_RAS_3_HI"/>
+	<reg32 offset="0x0104" name="RBBM_PERFCTR_UCHE_0_LO"/>
+	<reg32 offset="0x0105" name="RBBM_PERFCTR_UCHE_0_HI"/>
+	<reg32 offset="0x0106" name="RBBM_PERFCTR_UCHE_1_LO"/>
+	<reg32 offset="0x0107" name="RBBM_PERFCTR_UCHE_1_HI"/>
+	<reg32 offset="0x0108" name="RBBM_PERFCTR_UCHE_2_LO"/>
+	<reg32 offset="0x0109" name="RBBM_PERFCTR_UCHE_2_HI"/>
+	<reg32 offset="0x010a" name="RBBM_PERFCTR_UCHE_3_LO"/>
+	<reg32 offset="0x010b" name="RBBM_PERFCTR_UCHE_3_HI"/>
+	<reg32 offset="0x010c" name="RBBM_PERFCTR_UCHE_4_LO"/>
+	<reg32 offset="0x010d" name="RBBM_PERFCTR_UCHE_4_HI"/>
+	<reg32 offset="0x010e" name="RBBM_PERFCTR_UCHE_5_LO"/>
+	<reg32 offset="0x010f" name="RBBM_PERFCTR_UCHE_5_HI"/>
+	<reg32 offset="0x0110" name="RBBM_PERFCTR_UCHE_6_LO"/>
+	<reg32 offset="0x0111" name="RBBM_PERFCTR_UCHE_6_HI"/>
+	<reg32 offset="0x0112" name="RBBM_PERFCTR_UCHE_7_LO"/>
+	<reg32 offset="0x0113" name="RBBM_PERFCTR_UCHE_7_HI"/>
+	<reg32 offset="0x0114" name="RBBM_PERFCTR_TP_0_LO"/>
+	<reg32 offset="0x0115" name="RBBM_PERFCTR_TP_0_HI"/>
+	<reg32 offset="0x0116" name="RBBM_PERFCTR_TP_1_LO"/>
+	<reg32 offset="0x0117" name="RBBM_PERFCTR_TP_1_HI"/>
+	<reg32 offset="0x0118" name="RBBM_PERFCTR_TP_2_LO"/>
+	<reg32 offset="0x0119" name="RBBM_PERFCTR_TP_2_HI"/>
+	<reg32 offset="0x011a" name="RBBM_PERFCTR_TP_3_LO"/>
+	<reg32 offset="0x011b" name="RBBM_PERFCTR_TP_3_HI"/>
+	<reg32 offset="0x011c" name="RBBM_PERFCTR_TP_4_LO"/>
+	<reg32 offset="0x011d" name="RBBM_PERFCTR_TP_4_HI"/>
+	<reg32 offset="0x011e" name="RBBM_PERFCTR_TP_5_LO"/>
+	<reg32 offset="0x011f" name="RBBM_PERFCTR_TP_5_HI"/>
+	<reg32 offset="0x0120" name="RBBM_PERFCTR_TP_6_LO"/>
+	<reg32 offset="0x0121" name="RBBM_PERFCTR_TP_6_HI"/>
+	<reg32 offset="0x0122" name="RBBM_PERFCTR_TP_7_LO"/>
+	<reg32 offset="0x0123" name="RBBM_PERFCTR_TP_7_HI"/>
+	<reg32 offset="0x0124" name="RBBM_PERFCTR_SP_0_LO"/>
+	<reg32 offset="0x0125" name="RBBM_PERFCTR_SP_0_HI"/>
+	<reg32 offset="0x0126" name="RBBM_PERFCTR_SP_1_LO"/>
+	<reg32 offset="0x0127" name="RBBM_PERFCTR_SP_1_HI"/>
+	<reg32 offset="0x0128" name="RBBM_PERFCTR_SP_2_LO"/>
+	<reg32 offset="0x0129" name="RBBM_PERFCTR_SP_2_HI"/>
+	<reg32 offset="0x012a" name="RBBM_PERFCTR_SP_3_LO"/>
+	<reg32 offset="0x012b" name="RBBM_PERFCTR_SP_3_HI"/>
+	<reg32 offset="0x012c" name="RBBM_PERFCTR_SP_4_LO"/>
+	<reg32 offset="0x012d" name="RBBM_PERFCTR_SP_4_HI"/>
+	<reg32 offset="0x012e" name="RBBM_PERFCTR_SP_5_LO"/>
+	<reg32 offset="0x012f" name="RBBM_PERFCTR_SP_5_HI"/>
+	<reg32 offset="0x0130" name="RBBM_PERFCTR_SP_6_LO"/>
+	<reg32 offset="0x0131" name="RBBM_PERFCTR_SP_6_HI"/>
+	<reg32 offset="0x0132" name="RBBM_PERFCTR_SP_7_LO"/>
+	<reg32 offset="0x0133" name="RBBM_PERFCTR_SP_7_HI"/>
+	<reg32 offset="0x0134" name="RBBM_PERFCTR_SP_8_LO"/>
+	<reg32 offset="0x0135" name="RBBM_PERFCTR_SP_8_HI"/>
+	<reg32 offset="0x0136" name="RBBM_PERFCTR_SP_9_LO"/>
+	<reg32 offset="0x0137" name="RBBM_PERFCTR_SP_9_HI"/>
+	<reg32 offset="0x0138" name="RBBM_PERFCTR_SP_10_LO"/>
+	<reg32 offset="0x0139" name="RBBM_PERFCTR_SP_10_HI"/>
+	<reg32 offset="0x013a" name="RBBM_PERFCTR_SP_11_LO"/>
+	<reg32 offset="0x013b" name="RBBM_PERFCTR_SP_11_HI"/>
+	<reg32 offset="0x013c" name="RBBM_PERFCTR_RB_0_LO"/>
+	<reg32 offset="0x013d" name="RBBM_PERFCTR_RB_0_HI"/>
+	<reg32 offset="0x013e" name="RBBM_PERFCTR_RB_1_LO"/>
+	<reg32 offset="0x013f" name="RBBM_PERFCTR_RB_1_HI"/>
+	<reg32 offset="0x0140" name="RBBM_PERFCTR_RB_2_LO"/>
+	<reg32 offset="0x0141" name="RBBM_PERFCTR_RB_2_HI"/>
+	<reg32 offset="0x0142" name="RBBM_PERFCTR_RB_3_LO"/>
+	<reg32 offset="0x0143" name="RBBM_PERFCTR_RB_3_HI"/>
+	<reg32 offset="0x0144" name="RBBM_PERFCTR_RB_4_LO"/>
+	<reg32 offset="0x0145" name="RBBM_PERFCTR_RB_4_HI"/>
+	<reg32 offset="0x0146" name="RBBM_PERFCTR_RB_5_LO"/>
+	<reg32 offset="0x0147" name="RBBM_PERFCTR_RB_5_HI"/>
+	<reg32 offset="0x0148" name="RBBM_PERFCTR_RB_6_LO"/>
+	<reg32 offset="0x0149" name="RBBM_PERFCTR_RB_6_HI"/>
+	<reg32 offset="0x014a" name="RBBM_PERFCTR_RB_7_LO"/>
+	<reg32 offset="0x014b" name="RBBM_PERFCTR_RB_7_HI"/>
+	<reg32 offset="0x014c" name="RBBM_PERFCTR_VSC_0_LO"/>
+	<reg32 offset="0x014d" name="RBBM_PERFCTR_VSC_0_HI"/>
+	<reg32 offset="0x014e" name="RBBM_PERFCTR_VSC_1_LO"/>
+	<reg32 offset="0x014f" name="RBBM_PERFCTR_VSC_1_HI"/>
+	<reg32 offset="0x0166" name="RBBM_PERFCTR_PWR_0_LO"/>
+	<reg32 offset="0x0167" name="RBBM_PERFCTR_PWR_0_HI"/>
+	<reg32 offset="0x0168" name="RBBM_PERFCTR_PWR_1_LO"/>
+	<reg32 offset="0x0169" name="RBBM_PERFCTR_PWR_1_HI"/>
+	<reg32 offset="0x016e" name="RBBM_ALWAYSON_COUNTER_LO"/>
+	<reg32 offset="0x016f" name="RBBM_ALWAYSON_COUNTER_HI"/>
+	<array offset="0x0068" name="RBBM_CLOCK_CTL_SP" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<array offset="0x006c" name="RBBM_CLOCK_CTL2_SP" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<array offset="0x0070" name="RBBM_CLOCK_HYST_SP" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<array offset="0x0074" name="RBBM_CLOCK_DELAY_SP" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<array offset="0x0078" name="RBBM_CLOCK_CTL_RB" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<array offset="0x007c" name="RBBM_CLOCK_CTL2_RB" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<array offset="0x0082" name="RBBM_CLOCK_CTL_MARB_CCU" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<array offset="0x0086" name="RBBM_CLOCK_HYST_RB_MARB_CCU" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<reg32 offset="0x0080" name="RBBM_CLOCK_HYST_COM_DCOM"/>
+	<reg32 offset="0x0081" name="RBBM_CLOCK_CTL_COM_DCOM"/>
+	<reg32 offset="0x008a" name="RBBM_CLOCK_CTL_HLSQ"/>
+	<reg32 offset="0x008b" name="RBBM_CLOCK_HYST_HLSQ"/>
+	<reg32 offset="0x008c" name="RBBM_CLOCK_DELAY_HLSQ"/>
+	<bitset name="A4XX_CGC_HLSQ">
+		<bitfield name="EARLY_CYC" low="20" high="22" type="uint"/>
+	</bitset>
+	<reg32 offset="0x008d" name="RBBM_CLOCK_DELAY_COM_DCOM"/>
+	<array offset="0x008e" name="RBBM_CLOCK_DELAY_RB_MARB_CCU_L1" stride="1" length="4">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<bitset name="A4XX_INT0">
+		<bitfield name="RBBM_GPU_IDLE" pos="0" type="boolean"/>
+		<bitfield name="RBBM_AHB_ERROR" pos="1" type="boolean"/>
+		<bitfield name="RBBM_REG_TIMEOUT" pos="2" type="boolean"/>
+		<bitfield name="RBBM_ME_MS_TIMEOUT" pos="3" type="boolean"/>
+		<bitfield name="RBBM_PFP_MS_TIMEOUT" pos="4" type="boolean"/>
+		<bitfield name="RBBM_ATB_BUS_OVERFLOW" pos="5" type="boolean"/>
+		<bitfield name="VFD_ERROR" pos="6" type="boolean"/>
+		<bitfield name="CP_SW_INT" pos="7" type="boolean"/>
+		<bitfield name="CP_T0_PACKET_IN_IB" pos="8" type="boolean"/>
+		<bitfield name="CP_OPCODE_ERROR" pos="9" type="boolean"/>
+		<bitfield name="CP_RESERVED_BIT_ERROR" pos="10" type="boolean"/>
+		<bitfield name="CP_HW_FAULT" pos="11" type="boolean"/>
+		<bitfield name="CP_DMA" pos="12" type="boolean"/>
+		<bitfield name="CP_IB2_INT" pos="13" type="boolean"/>
+		<bitfield name="CP_IB1_INT" pos="14" type="boolean"/>
+		<bitfield name="CP_RB_INT" pos="15" type="boolean"/>
+		<bitfield name="CP_REG_PROTECT_FAULT" pos="16" type="boolean"/>
+		<bitfield name="CP_RB_DONE_TS" pos="17" type="boolean"/>
+		<bitfield name="CP_VS_DONE_TS" pos="18" type="boolean"/>
+		<bitfield name="CP_PS_DONE_TS" pos="19" type="boolean"/>
+		<bitfield name="CACHE_FLUSH_TS" pos="20" type="boolean"/>
+		<bitfield name="CP_AHB_ERROR_HALT" pos="21" type="boolean"/>
+		<bitfield name="MISC_HANG_DETECT" pos="24" type="boolean"/>
+		<bitfield name="UCHE_OOB_ACCESS" pos="25" type="boolean"/>
+	</bitset>
+
+	<reg32 offset="0x0099" name="RBBM_SP_REGFILE_SLEEP_CNTL_0"/>
+	<reg32 offset="0x009a" name="RBBM_SP_REGFILE_SLEEP_CNTL_1"/>
+	<reg32 offset="0x0170" name="RBBM_PERFCTR_CTL"/>
+	<reg32 offset="0x0171" name="RBBM_PERFCTR_LOAD_CMD0"/>
+	<reg32 offset="0x0172" name="RBBM_PERFCTR_LOAD_CMD1"/>
+	<reg32 offset="0x0173" name="RBBM_PERFCTR_LOAD_CMD2"/>
+	<reg32 offset="0x0174" name="RBBM_PERFCTR_LOAD_VALUE_LO"/>
+	<reg32 offset="0x0175" name="RBBM_PERFCTR_LOAD_VALUE_HI"/>
+	<reg32 offset="0x0176" name="RBBM_PERFCTR_RBBM_SEL_0" type="a4xx_rbbm_perfcounter_select"/>
+	<reg32 offset="0x0177" name="RBBM_PERFCTR_RBBM_SEL_1" type="a4xx_rbbm_perfcounter_select"/>
+	<reg32 offset="0x0178" name="RBBM_PERFCTR_RBBM_SEL_2" type="a4xx_rbbm_perfcounter_select"/>
+	<reg32 offset="0x0179" name="RBBM_PERFCTR_RBBM_SEL_3" type="a4xx_rbbm_perfcounter_select"/>
+	<reg32 offset="0x017a" name="RBBM_GPU_BUSY_MASKED"/>
+	<reg32 offset="0x017d" name="RBBM_INT_0_STATUS"/>
+	<reg32 offset="0x0182" name="RBBM_CLOCK_STATUS"/>
+	<reg32 offset="0x0189" name="RBBM_AHB_STATUS"/>
+	<reg32 offset="0x018c" name="RBBM_AHB_ME_SPLIT_STATUS"/>
+	<reg32 offset="0x018d" name="RBBM_AHB_PFP_SPLIT_STATUS"/>
+	<reg32 offset="0x018f" name="RBBM_AHB_ERROR_STATUS"/>
+	<reg32 offset="0x0191" name="RBBM_STATUS">
+		<bitfield name="HI_BUSY" pos="0" type="boolean"/>
+		<bitfield name="CP_ME_BUSY" pos="1" type="boolean"/>
+		<bitfield name="CP_PFP_BUSY" pos="2" type="boolean"/>
+		<bitfield name="CP_NRT_BUSY" pos="14" type="boolean"/>
+		<bitfield name="VBIF_BUSY" pos="15" type="boolean"/>
+		<bitfield name="TSE_BUSY" pos="16" type="boolean"/>
+		<bitfield name="RAS_BUSY" pos="17" type="boolean"/>
+		<bitfield name="RB_BUSY" pos="18" type="boolean"/>
+		<bitfield name="PC_DCALL_BUSY" pos="19" type="boolean"/>
+		<bitfield name="PC_VSD_BUSY" pos="20" type="boolean"/>
+		<bitfield name="VFD_BUSY" pos="21" type="boolean"/>
+		<bitfield name="VPC_BUSY" pos="22" type="boolean"/>
+		<bitfield name="UCHE_BUSY" pos="23" type="boolean"/>
+		<bitfield name="SP_BUSY" pos="24" type="boolean"/>
+		<bitfield name="TPL1_BUSY" pos="25" type="boolean"/>
+		<bitfield name="MARB_BUSY" pos="26" type="boolean"/>
+		<bitfield name="VSC_BUSY" pos="27" type="boolean"/>
+		<bitfield name="ARB_BUSY" pos="28" type="boolean"/>
+		<bitfield name="HLSQ_BUSY" pos="29" type="boolean"/>
+		<bitfield name="GPU_BUSY_NOHC" pos="30" type="boolean"/>
+		<bitfield name="GPU_BUSY" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x019f" name="RBBM_INTERFACE_RRDY_STATUS5"/>
+	<reg32 offset="0x01b0" name="RBBM_POWER_STATUS">
+		<bitfield name="SP_TP_PWR_ON" pos="20" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x01b8" name="RBBM_WAIT_IDLE_CLOCKS_CTL2"/>
+
+	<!-- CP registers -->
+	<reg32 offset="0x0228" name="CP_SCRATCH_UMASK"/>
+	<reg32 offset="0x0229" name="CP_SCRATCH_ADDR"/>
+	<reg32 offset="0x0200" name="CP_RB_BASE"/>
+	<reg32 offset="0x0201" name="CP_RB_CNTL"/>
+	<reg32 offset="0x0205" name="CP_RB_WPTR"/>
+	<reg32 offset="0x0203" name="CP_RB_RPTR_ADDR"/>
+	<reg32 offset="0x0204" name="CP_RB_RPTR"/>
+	<reg32 offset="0x0206" name="CP_IB1_BASE"/>
+	<reg32 offset="0x0207" name="CP_IB1_BUFSZ"/>
+	<reg32 offset="0x0208" name="CP_IB2_BASE"/>
+	<reg32 offset="0x0209" name="CP_IB2_BUFSZ"/>
+	<reg32 offset="0x020c" name="CP_ME_NRT_ADDR"/>
+	<reg32 offset="0x020d" name="CP_ME_NRT_DATA"/>
+	<reg32 offset="0x0217" name="CP_ME_RB_DONE_DATA"/>
+	<reg32 offset="0x0219" name="CP_QUEUE_THRESH2"/>
+	<reg32 offset="0x021b" name="CP_MERCIU_SIZE"/>
+	<reg32 offset="0x021c" name="CP_ROQ_ADDR"/>
+	<reg32 offset="0x021d" name="CP_ROQ_DATA"/>
+	<reg32 offset="0x021e" name="CP_MEQ_ADDR"/>
+	<reg32 offset="0x021f" name="CP_MEQ_DATA"/>
+	<reg32 offset="0x0220" name="CP_MERCIU_ADDR"/>
+	<reg32 offset="0x0221" name="CP_MERCIU_DATA"/>
+	<reg32 offset="0x0222" name="CP_MERCIU_DATA2"/>
+	<reg32 offset="0x0223" name="CP_PFP_UCODE_ADDR"/>
+	<reg32 offset="0x0224" name="CP_PFP_UCODE_DATA"/>
+	<reg32 offset="0x0225" name="CP_ME_RAM_WADDR"/>
+	<reg32 offset="0x0226" name="CP_ME_RAM_RADDR"/>
+	<reg32 offset="0x0227" name="CP_ME_RAM_DATA"/>
+	<reg32 offset="0x022a" name="CP_PREEMPT"/>
+	<reg32 offset="0x022c" name="CP_CNTL"/>
+	<reg32 offset="0x022d" name="CP_ME_CNTL"/>
+	<reg32 offset="0x022e" name="CP_DEBUG"/>
+	<reg32 offset="0x0231" name="CP_DEBUG_ECO_CONTROL"/>
+	<reg32 offset="0x0232" name="CP_DRAW_STATE_ADDR"/>
+	<array offset="0x0240" name="CP_PROTECT" stride="1" length="16">
+		<reg32 offset="0x0" name="REG" type="adreno_cp_protect"/>
+	</array>
+	<reg32 offset="0x0250" name="CP_PROTECT_CTRL"/>
+	<reg32 offset="0x04c0" name="CP_ST_BASE"/>
+	<reg32 offset="0x04ce" name="CP_STQ_AVAIL"/>
+	<reg32 offset="0x04d0" name="CP_MERCIU_STAT"/>
+	<reg32 offset="0x04d2" name="CP_WFI_PEND_CTR"/>
+	<reg32 offset="0x04d8" name="CP_HW_FAULT"/>
+	<reg32 offset="0x04da" name="CP_PROTECT_STATUS"/>
+	<reg32 offset="0x04dd" name="CP_EVENTS_IN_FLIGHT"/>
+	<reg32 offset="0x0500" name="CP_PERFCTR_CP_SEL_0" type="a4xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0501" name="CP_PERFCTR_CP_SEL_1" type="a4xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0502" name="CP_PERFCTR_CP_SEL_2" type="a4xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0503" name="CP_PERFCTR_CP_SEL_3" type="a4xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0504" name="CP_PERFCTR_CP_SEL_4" type="a4xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0505" name="CP_PERFCTR_CP_SEL_5" type="a4xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0506" name="CP_PERFCTR_CP_SEL_6" type="a4xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0507" name="CP_PERFCTR_CP_SEL_7" type="a4xx_cp_perfcounter_select"/>
+	<reg32 offset="0x050b" name="CP_PERFCOMBINER_SELECT"/>
+	<array offset="0x0578" name="CP_SCRATCH" stride="1" length="23">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+
+
+	<!-- SP registers -->
+	<reg32 offset="0x0ec0" name="SP_VS_STATUS"/>
+	<reg32 offset="0x0ec3" name="SP_MODE_CONTROL"/>
+
+	<reg32 offset="0x0ec4" name="SP_PERFCTR_SP_SEL_0" type="a4xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ec5" name="SP_PERFCTR_SP_SEL_1" type="a4xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ec6" name="SP_PERFCTR_SP_SEL_2" type="a4xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ec7" name="SP_PERFCTR_SP_SEL_3" type="a4xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ec8" name="SP_PERFCTR_SP_SEL_4" type="a4xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ec9" name="SP_PERFCTR_SP_SEL_5" type="a4xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0eca" name="SP_PERFCTR_SP_SEL_6" type="a4xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ecb" name="SP_PERFCTR_SP_SEL_7" type="a4xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ecc" name="SP_PERFCTR_SP_SEL_8" type="a4xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ecd" name="SP_PERFCTR_SP_SEL_9" type="a4xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ece" name="SP_PERFCTR_SP_SEL_10" type="a4xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ecf" name="SP_PERFCTR_SP_SEL_11" type="a4xx_sp_perfcounter_select"/>
+
+	<reg32 offset="0x22c0" name="SP_SP_CTRL_REG">
+		<bitfield name="BINNING_PASS" pos="19" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x22c1" name="SP_INSTR_CACHE_CTRL">
+		<!-- set when VS in buffer mode: -->
+		<bitfield name="VS_BUFFER" pos="7" type="boolean"/>
+		<!-- set when FS in buffer mode: -->
+		<bitfield name="FS_BUFFER" pos="8" type="boolean"/>
+		<!-- set when both VS or FS in buffer mode: -->
+		<bitfield name="INSTR_BUFFER" pos="10" type="boolean"/>
+		<!-- TODO other bits probably matter when other stages active? -->
+	</reg32>
+
+	<bitset name="a4xx_sp_vs_fs_ctrl_reg0" inline="yes">
+		<!--
+			NOTE that SP_{VS,FS}_CTRL_REG1 are different, but so far REG0
+			appears to be the same..
+		-->
+		<bitfield name="THREADMODE" pos="0" type="a3xx_threadmode"/>
+		<!-- VARYING bit only for FS.. think it controls emitting (ei) flag? -->
+		<bitfield name="VARYING" pos="1" type="boolean"/>
+		<!-- maybe CACHEINVALID is two bits?? -->
+		<bitfield name="CACHEINVALID" pos="2" type="boolean"/>
+		<doc>
+			The full/half register footprint is in units of four components,
+			so if r0.x is used, that counts as all of r0.[xyzw] as used.
+			There are separate full/half register footprint values as the
+			full and half registers are independent (not overlapping).
+			Presumably the thread scheduler hardware allocates the full/half
+			register names from the actual physical register file and
+			handles the register renaming.
+		</doc>
+		<bitfield name="HALFREGFOOTPRINT" low="4" high="9" type="uint"/>
+		<bitfield name="FULLREGFOOTPRINT" low="10" high="15" type="uint"/>
+		<!-- maybe INOUTREGOVERLAP is a bitflag? -->
+		<bitfield name="INOUTREGOVERLAP" low="18" high="19" type="uint"/>
+		<bitfield name="THREADSIZE" pos="20" type="a3xx_threadsize"/>
+		<bitfield name="SUPERTHREADMODE" pos="21" type="boolean"/>
+		<bitfield name="PIXLODENABLE" pos="22" type="boolean"/>
+	</bitset>
+
+	<reg32 offset="0x22c4" name="SP_VS_CTRL_REG0" type="a4xx_sp_vs_fs_ctrl_reg0"/>
+	<reg32 offset="0x22c5" name="SP_VS_CTRL_REG1">
+		<bitfield name="CONSTLENGTH" low="0" high="7" type="uint"/>
+		<bitfield name="INITIALOUTSTANDING" low="24" high="30" type="uint"/>
+	</reg32>
+	<reg32 offset="0x22c6" name="SP_VS_PARAM_REG">
+		<bitfield name="POSREGID" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="PSIZEREGID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="TOTALVSOUTVAR" low="20" high="31" type="uint"/>
+	</reg32>
+	<array offset="0x22c7" name="SP_VS_OUT" stride="1" length="16">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="A_REGID" low="0" high="8" type="a3xx_regid"/>
+			<bitfield name="A_COMPMASK" low="9" high="12" type="hex"/>
+			<bitfield name="B_REGID" low="16" high="24" type="a3xx_regid"/>
+			<bitfield name="B_COMPMASK" low="25" high="28" type="hex"/>
+		</reg32>
+	</array>
+	<array offset="0x22d8" name="SP_VS_VPC_DST" stride="1" length="8">
+		<reg32 offset="0x0" name="REG">
+			<doc>
+				These seem to be offsets for storage of the varyings.
+				Always seems to start from 8, possibly loc 0 and 4
+				are for gl_Position and gl_PointSize?
+			</doc>
+			<bitfield name="OUTLOC0" low="0" high="7" type="uint"/>
+			<bitfield name="OUTLOC1" low="8" high="15" type="uint"/>
+			<bitfield name="OUTLOC2" low="16" high="23" type="uint"/>
+			<bitfield name="OUTLOC3" low="24" high="31" type="uint"/>
+		</reg32>
+	</array>
+
+	<reg32 offset="0x22e0" name="SP_VS_OBJ_OFFSET_REG">
+		<!-- always 00000000: -->
+		<doc>
+			From register spec:
+			SP_FS_OBJ_OFFSET_REG.CONSTOBJECTSTARTOFFSET [16:24]: Constant object
+			start offset in on chip RAM,
+			128bit aligned
+		</doc>
+		<bitfield name="CONSTOBJECTOFFSET" low="16" high="24" type="uint"/>
+		<bitfield name="SHADEROBJOFFSET" low="25" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x22e1" name="SP_VS_OBJ_START"/>
+	<reg32 offset="0x22e2" name="SP_VS_PVT_MEM_PARAM"/>
+	<reg32 offset="0x22e3" name="SP_VS_PVT_MEM_ADDR"/>
+	<reg32 offset="0x22e5" name="SP_VS_LENGTH_REG" type="uint"/>
+	<reg32 offset="0x22e8" name="SP_FS_CTRL_REG0" type="a4xx_sp_vs_fs_ctrl_reg0"/>
+	<reg32 offset="0x22e9" name="SP_FS_CTRL_REG1">
+		<bitfield name="CONSTLENGTH" low="0" high="7" type="uint"/>
+		<bitfield name="FACENESS" pos="19" type="boolean"/>
+		<bitfield name="VARYING" pos="20" type="boolean"/>
+		<bitfield name="FRAGCOORD" pos="21" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x22ea" name="SP_FS_OBJ_OFFSET_REG">
+		<bitfield name="CONSTOBJECTOFFSET" low="16" high="24" type="uint"/>
+		<bitfield name="SHADEROBJOFFSET" low="25" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x22eb" name="SP_FS_OBJ_START"/>
+	<reg32 offset="0x22ec" name="SP_FS_PVT_MEM_PARAM"/>
+	<reg32 offset="0x22ed" name="SP_FS_PVT_MEM_ADDR"/>
+	<reg32 offset="0x22ef" name="SP_FS_LENGTH_REG" type="uint"/>
+	<reg32 offset="0x22f0" name="SP_FS_OUTPUT_REG">
+		<bitfield name="MRT" low="0" high="3" type="uint"/>
+		<bitfield name="DEPTH_ENABLE" pos="7" type="boolean"/>
+		<!-- TODO double check.. for now assume same as a3xx -->
+		<bitfield name="DEPTH_REGID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="SAMPLEMASK_REGID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<array offset="0x22f1" name="SP_FS_MRT" stride="1" length="8">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="REGID" low="0" high="7" type="a3xx_regid"/>
+			<bitfield name="HALF_PRECISION" pos="8" type="boolean"/>
+			<bitfield name="COLOR_SINT" pos="10" type="boolean"/>
+			<bitfield name="COLOR_UINT" pos="11" type="boolean"/>
+			<bitfield name="MRTFORMAT" low="12" high="17" type="a4xx_color_fmt"/>
+			<bitfield name="COLOR_SRGB" pos="18" type="boolean"/>
+		</reg32>
+	</array>
+	<reg32 offset="0x2300" name="SP_CS_CTRL_REG0" type="a4xx_sp_vs_fs_ctrl_reg0"/>
+	<reg32 offset="0x2301" name="SP_CS_OBJ_OFFSET_REG"/>
+	<reg32 offset="0x2302" name="SP_CS_OBJ_START"/>
+	<reg32 offset="0x2303" name="SP_CS_PVT_MEM_PARAM"/>
+	<reg32 offset="0x2304" name="SP_CS_PVT_MEM_ADDR"/>
+	<reg32 offset="0x2305" name="SP_CS_PVT_MEM_SIZE"/>
+	<reg32 offset="0x2306" name="SP_CS_LENGTH_REG" type="uint"/>
+	<reg32 offset="0x230d" name="SP_HS_OBJ_OFFSET_REG">
+		<bitfield name="CONSTOBJECTOFFSET" low="16" high="24" type="uint"/>
+		<bitfield name="SHADEROBJOFFSET" low="25" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x230e" name="SP_HS_OBJ_START"/>
+	<reg32 offset="0x230f" name="SP_HS_PVT_MEM_PARAM"/>
+	<reg32 offset="0x2310" name="SP_HS_PVT_MEM_ADDR"/>
+	<reg32 offset="0x2312" name="SP_HS_LENGTH_REG" type="uint"/>
+
+	<reg32 offset="0x231a" name="SP_DS_PARAM_REG">
+		<bitfield name="POSREGID" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="TOTALGSOUTVAR" low="20" high="31" type="uint"/>
+	</reg32>
+	<array offset="0x231b" name="SP_DS_OUT" stride="1" length="16">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="A_REGID" low="0" high="8" type="a3xx_regid"/>
+			<bitfield name="A_COMPMASK" low="9" high="12" type="hex"/>
+			<bitfield name="B_REGID" low="16" high="24" type="a3xx_regid"/>
+			<bitfield name="B_COMPMASK" low="25" high="28" type="hex"/>
+		</reg32>
+	</array>
+	<array offset="0x232c" name="SP_DS_VPC_DST" stride="1" length="8">
+		<reg32 offset="0x0" name="REG">
+			<doc>
+				These seem to be offsets for storage of the varyings.
+				Always seems to start from 8, possibly loc 0 and 4
+				are for gl_Position and gl_PointSize?
+			</doc>
+			<bitfield name="OUTLOC0" low="0" high="7" type="uint"/>
+			<bitfield name="OUTLOC1" low="8" high="15" type="uint"/>
+			<bitfield name="OUTLOC2" low="16" high="23" type="uint"/>
+			<bitfield name="OUTLOC3" low="24" high="31" type="uint"/>
+		</reg32>
+	</array>
+	<reg32 offset="0x2334" name="SP_DS_OBJ_OFFSET_REG">
+		<bitfield name="CONSTOBJECTOFFSET" low="16" high="24" type="uint"/>
+		<bitfield name="SHADEROBJOFFSET" low="25" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2335" name="SP_DS_OBJ_START"/>
+	<reg32 offset="0x2336" name="SP_DS_PVT_MEM_PARAM"/>
+	<reg32 offset="0x2337" name="SP_DS_PVT_MEM_ADDR"/>
+	<reg32 offset="0x2339" name="SP_DS_LENGTH_REG" type="uint"/>
+
+	<reg32 offset="0x2341" name="SP_GS_PARAM_REG">
+		<bitfield name="POSREGID" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="PRIMREGID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="TOTALGSOUTVAR" low="20" high="31" type="uint"/>
+	</reg32>
+	<array offset="0x2342" name="SP_GS_OUT" stride="1" length="16">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="A_REGID" low="0" high="8" type="a3xx_regid"/>
+			<bitfield name="A_COMPMASK" low="9" high="12" type="hex"/>
+			<bitfield name="B_REGID" low="16" high="24" type="a3xx_regid"/>
+			<bitfield name="B_COMPMASK" low="25" high="28" type="hex"/>
+		</reg32>
+	</array>
+	<array offset="0x2353" name="SP_GS_VPC_DST" stride="1" length="8">
+		<reg32 offset="0x0" name="REG">
+			<doc>
+				These seem to be offsets for storage of the varyings.
+				Always seems to start from 8, possibly loc 0 and 4
+				are for gl_Position and gl_PointSize?
+			</doc>
+			<bitfield name="OUTLOC0" low="0" high="7" type="uint"/>
+			<bitfield name="OUTLOC1" low="8" high="15" type="uint"/>
+			<bitfield name="OUTLOC2" low="16" high="23" type="uint"/>
+			<bitfield name="OUTLOC3" low="24" high="31" type="uint"/>
+		</reg32>
+	</array>
+	<reg32 offset="0x235b" name="SP_GS_OBJ_OFFSET_REG">
+		<bitfield name="CONSTOBJECTOFFSET" low="16" high="24" type="uint"/>
+		<bitfield name="SHADEROBJOFFSET" low="25" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x235c" name="SP_GS_OBJ_START"/>
+	<reg32 offset="0x235d" name="SP_GS_PVT_MEM_PARAM"/>
+	<reg32 offset="0x235e" name="SP_GS_PVT_MEM_ADDR"/>
+	<reg32 offset="0x2360" name="SP_GS_LENGTH_REG" type="uint"/>
+
+	<!-- VPC registers -->
+	<reg32 offset="0x0e60" name="VPC_DEBUG_RAM_SEL"/>
+	<reg32 offset="0x0e61" name="VPC_DEBUG_RAM_READ"/>
+	<reg32 offset="0x0e64" name="VPC_DEBUG_ECO_CONTROL"/>
+	<reg32 offset="0x0e65" name="VPC_PERFCTR_VPC_SEL_0" type="a4xx_vpc_perfcounter_select"/>
+	<reg32 offset="0x0e66" name="VPC_PERFCTR_VPC_SEL_1" type="a4xx_vpc_perfcounter_select"/>
+	<reg32 offset="0x0e67" name="VPC_PERFCTR_VPC_SEL_2" type="a4xx_vpc_perfcounter_select"/>
+	<reg32 offset="0x0e68" name="VPC_PERFCTR_VPC_SEL_3" type="a4xx_vpc_perfcounter_select"/>
+	<reg32 offset="0x2140" name="VPC_ATTR">
+		<bitfield name="TOTALATTR" low="0" high="8" type="uint"/>
+		<!-- PSIZE bit set if gl_PointSize written: -->
+		<bitfield name="PSIZE" pos="9" type="boolean"/>
+		<bitfield name="THRDASSIGN" low="12" high="13" type="uint"/>
+		<bitfield name="ENABLE" pos="25" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2141" name="VPC_PACK">
+		<bitfield name="NUMBYPASSVAR" low="0" high="7" type="uint"/>
+		<bitfield name="NUMFPNONPOSVAR" low="8" high="15" type="uint"/>
+		<bitfield name="NUMNONPOSVSVAR" low="16" high="23" type="uint"/>
+	</reg32>
+	<array offset="0x2142" name="VPC_VARYING_INTERP" stride="1" length="8">
+		<reg32 offset="0x0" name="MODE"/>
+	</array>
+	<array offset="0x214a" name="VPC_VARYING_PS_REPL" stride="1" length="8">
+		<reg32 offset="0x0" name="MODE"/>
+	</array>
+
+	<reg32 offset="0x216e" name="VPC_SO_FLUSH_WADDR_3"/>
+
+	<!-- VSC registers -->
+	<reg32 offset="0x0c00" name="VSC_BIN_SIZE">
+		<bitfield name="WIDTH" low="0" high="4" shr="5" type="uint"/>
+		<bitfield name="HEIGHT" low="5" high="9" shr="5" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0c01" name="VSC_SIZE_ADDRESS"/>
+	<reg32 offset="0x0c02" name="VSC_SIZE_ADDRESS2"/>
+	<reg32 offset="0x0c03" name="VSC_DEBUG_ECO_CONTROL"/>
+	<array offset="0x0c08" name="VSC_PIPE_CONFIG" stride="1" length="8">
+		<reg32 offset="0x0" name="REG">
+			<doc>
+				Configures the mapping between VSC_PIPE buffer and
+				bin, X/Y specify the bin index in the horiz/vert
+				direction (0,0 is upper left, 0,1 is leftmost bin
+				on second row, and so on).  W/H specify the number
+				of bins assigned to this VSC_PIPE in the horiz/vert
+				dimension.
+			</doc>
+			<bitfield name="X" low="0" high="9" type="uint"/>
+			<bitfield name="Y" low="10" high="19" type="uint"/>
+			<bitfield name="W" low="20" high="23" type="uint"/>
+			<bitfield name="H" low="24" high="27" type="uint"/>
+		</reg32>
+	</array>
+	<array offset="0x0c10" name="VSC_PIPE_DATA_ADDRESS" stride="1" length="8">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<array offset="0x0c18" name="VSC_PIPE_DATA_LENGTH" stride="1" length="8">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<reg32 offset="0x0c41" name="VSC_PIPE_PARTIAL_POSN_1"/>
+	<reg32 offset="0x0c50" name="VSC_PERFCTR_VSC_SEL_0" type="a4xx_vsc_perfcounter_select"/>
+	<reg32 offset="0x0c51" name="VSC_PERFCTR_VSC_SEL_1" type="a4xx_vsc_perfcounter_select"/>
+
+	<!-- VFD registers -->
+	<reg32 offset="0x0e40" name="VFD_DEBUG_CONTROL"/>
+	<reg32 offset="0x0e43" name="VFD_PERFCTR_VFD_SEL_0" type="a4xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e44" name="VFD_PERFCTR_VFD_SEL_1" type="a4xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e45" name="VFD_PERFCTR_VFD_SEL_2" type="a4xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e46" name="VFD_PERFCTR_VFD_SEL_3" type="a4xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e47" name="VFD_PERFCTR_VFD_SEL_4" type="a4xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e48" name="VFD_PERFCTR_VFD_SEL_5" type="a4xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e49" name="VFD_PERFCTR_VFD_SEL_6" type="a4xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e4a" name="VFD_PERFCTR_VFD_SEL_7" type="a4xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x21d0" name="VGT_CL_INITIATOR"/>
+	<reg32 offset="0x21d9" name="VGT_EVENT_INITIATOR"/>
+	<reg32 offset="0x2200" name="VFD_CONTROL_0">
+		<doc>
+			TOTALATTRTOVS is # of attributes to vertex shader, in register
+			slots (ie. vec4+vec3 -> 7)
+		</doc>
+		<bitfield name="TOTALATTRTOVS" low="0" high="7" type="uint"/>
+		<doc>
+		BYPASSATTROVS seems to count varyings that are just directly
+		assigned from attributes (ie, "vFoo = aFoo;")
+		</doc>
+		<bitfield name="BYPASSATTROVS" low="9" high="16" type="uint"/>
+		<doc>STRMDECINSTRCNT is # of VFD_DECODE_INSTR registers valid</doc>
+		<bitfield name="STRMDECINSTRCNT" low="20" high="25" type="uint"/>
+		<doc>STRMFETCHINSTRCNT is # of VFD_FETCH_INSTR registers valid</doc>
+		<bitfield name="STRMFETCHINSTRCNT" low="26" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2201" name="VFD_CONTROL_1">
+		<doc>MAXSTORAGE could be # of attributes/vbo's</doc>
+		<bitfield name="MAXSTORAGE" low="0" high="15" type="uint"/>
+		<bitfield name="REGID4VTX" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="REGID4INST" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0x2202" name="VFD_CONTROL_2"/>
+	<reg32 offset="0x2203" name="VFD_CONTROL_3">
+		<bitfield name="REGID_VTXCNT" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="REGID_TESSX" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="REGID_TESSY" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0x2204" name="VFD_CONTROL_4"/>
+	<reg32 offset="0x2208" name="VFD_INDEX_OFFSET"/>
+	<array offset="0x220a" name="VFD_FETCH" stride="4" length="32">
+		<reg32 offset="0x0" name="INSTR_0">
+			<bitfield name="FETCHSIZE" low="0" high="6" type="uint"/>
+			<bitfield name="BUFSTRIDE" low="7" high="16" type="uint"/>
+			<bitfield name="SWITCHNEXT" pos="19" type="boolean"/>
+			<bitfield name="INSTANCED" pos="20" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x1" name="INSTR_1"/>
+		<reg32 offset="0x2" name="INSTR_2">
+			<bitfield name="SIZE" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="0x3" name="INSTR_3">
+			<!-- might well be bigger.. -->
+			<bitfield name="STEPRATE" low="0" high="8" type="uint"/>
+		</reg32>
+	</array>
+	<array offset="0x228a" name="VFD_DECODE" stride="1" length="32">
+		<reg32 offset="0x0" name="INSTR">
+			<bitfield name="WRITEMASK" low="0" high="3" type="hex"/>
+			<!-- not sure if this is a bit flag and another flag above it, or?? -->
+			<bitfield name="CONSTFILL" pos="4" type="boolean"/>
+			<bitfield name="FORMAT" low="6" high="11" type="a4xx_vtx_fmt"/>
+			<bitfield name="REGID" low="12" high="19" type="a3xx_regid"/>
+			<bitfield name="INT" pos="20" type="boolean"/>
+			<doc>SHIFTCNT appears to be size, ie. FLOAT_32_32_32 is 12, and BYTE_8 is 1</doc>
+			<bitfield name="SWAP" low="22" high="23" type="a3xx_color_swap"/>
+			<bitfield name="SHIFTCNT" low="24" high="28" type="uint"/>
+			<bitfield name="LASTCOMPVALID" pos="29" type="boolean"/>
+			<bitfield name="SWITCHNEXT" pos="30" type="boolean"/>
+		</reg32>
+	</array>
+
+	<!-- TPL1 registers -->
+	<reg32 offset="0x0f00" name="TPL1_DEBUG_ECO_CONTROL"/>
+	<!-- always 0000003a: -->
+	<reg32 offset="0x0f03" name="TPL1_TP_MODE_CONTROL"/>
+	<reg32 offset="0x0f04" name="TPL1_PERFCTR_TP_SEL_0" type="a4xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f05" name="TPL1_PERFCTR_TP_SEL_1" type="a4xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f06" name="TPL1_PERFCTR_TP_SEL_2" type="a4xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f07" name="TPL1_PERFCTR_TP_SEL_3" type="a4xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f08" name="TPL1_PERFCTR_TP_SEL_4" type="a4xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f09" name="TPL1_PERFCTR_TP_SEL_5" type="a4xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f0a" name="TPL1_PERFCTR_TP_SEL_6" type="a4xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f0b" name="TPL1_PERFCTR_TP_SEL_7" type="a4xx_tp_perfcounter_select"/>
+	<reg32 offset="0x2380" name="TPL1_TP_TEX_OFFSET"/>
+	<reg32 offset="0x2381" name="TPL1_TP_TEX_COUNT">
+		<bitfield name="VS" low="0" high="7" type="uint"/>
+		<bitfield name="HS" low="8" high="15" type="uint"/>
+		<bitfield name="DS" low="16" high="23" type="uint"/>
+		<bitfield name="GS" low="24" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2384" name="TPL1_TP_VS_BORDER_COLOR_BASE_ADDR"/>
+	<reg32 offset="0x2387" name="TPL1_TP_HS_BORDER_COLOR_BASE_ADDR"/>
+	<reg32 offset="0x238a" name="TPL1_TP_DS_BORDER_COLOR_BASE_ADDR"/>
+	<reg32 offset="0x238d" name="TPL1_TP_GS_BORDER_COLOR_BASE_ADDR"/>
+	<reg32 offset="0x23a0" name="TPL1_TP_FS_TEX_COUNT">
+		<bitfield name="FS" low="0" high="7" type="uint"/>
+		<bitfield name="CS" low="8" high="15" type="uint"/>
+	</reg32>
+	<reg32 offset="0x23a1" name="TPL1_TP_FS_BORDER_COLOR_BASE_ADDR"/>
+	<reg32 offset="0x23a4" name="TPL1_TP_CS_BORDER_COLOR_BASE_ADDR"/>
+	<reg32 offset="0x23a5" name="TPL1_TP_CS_SAMPLER_BASE_ADDR"/>
+	<reg32 offset="0x23a6" name="TPL1_TP_CS_TEXMEMOBJ_BASE_ADDR"/>
+
+	<!-- GRAS registers -->
+	<reg32 offset="0x0c80" name="GRAS_TSE_STATUS"/>
+	<reg32 offset="0x0c81" name="GRAS_DEBUG_ECO_CONTROL"/>
+	<reg32 offset="0x0c88" name="GRAS_PERFCTR_TSE_SEL_0" type="a4xx_gras_tse_perfcounter_select"/>
+	<reg32 offset="0x0c89" name="GRAS_PERFCTR_TSE_SEL_1" type="a4xx_gras_tse_perfcounter_select"/>
+	<reg32 offset="0x0c8a" name="GRAS_PERFCTR_TSE_SEL_2" type="a4xx_gras_tse_perfcounter_select"/>
+	<reg32 offset="0x0c8b" name="GRAS_PERFCTR_TSE_SEL_3" type="a4xx_gras_tse_perfcounter_select"/>
+	<reg32 offset="0x0c8c" name="GRAS_PERFCTR_RAS_SEL_0" type="a4xx_gras_ras_perfcounter_select"/>
+	<reg32 offset="0x0c8d" name="GRAS_PERFCTR_RAS_SEL_1" type="a4xx_gras_ras_perfcounter_select"/>
+	<reg32 offset="0x0c8e" name="GRAS_PERFCTR_RAS_SEL_2" type="a4xx_gras_ras_perfcounter_select"/>
+	<reg32 offset="0x0c8f" name="GRAS_PERFCTR_RAS_SEL_3" type="a4xx_gras_ras_perfcounter_select"/>
+	<reg32 offset="0x2000" name="GRAS_CL_CLIP_CNTL">
+		<bitfield name="CLIP_DISABLE" pos="15" type="boolean"/>
+		<bitfield name="ZNEAR_CLIP_DISABLE" pos="16" type="boolean"/>
+		<bitfield name="ZFAR_CLIP_DISABLE" pos="17" type="boolean"/>
+		<bitfield name="ZERO_GB_SCALE_Z" pos="22" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2003" name="GRAS_CNTL">
+		<bitfield name="IJ_PERSP" pos="0" type="boolean"/>
+		<bitfield name="IJ_LINEAR" pos="1" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2004" name="GRAS_CL_GB_CLIP_ADJ">
+		<bitfield name="HORZ" low="0" high="9" type="uint"/>
+		<bitfield name="VERT" low="10" high="19" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2008" name="GRAS_CL_VPORT_XOFFSET_0" type="float"/>
+	<reg32 offset="0x2009" name="GRAS_CL_VPORT_XSCALE_0" type="float"/>
+	<reg32 offset="0x200a" name="GRAS_CL_VPORT_YOFFSET_0" type="float"/>
+	<reg32 offset="0x200b" name="GRAS_CL_VPORT_YSCALE_0" type="float"/>
+	<reg32 offset="0x200c" name="GRAS_CL_VPORT_ZOFFSET_0" type="float"/>
+	<reg32 offset="0x200d" name="GRAS_CL_VPORT_ZSCALE_0" type="float"/>
+	<reg32 offset="0x2070" name="GRAS_SU_POINT_MINMAX">
+		<bitfield name="MIN" low="0" high="15" type="ufixed" radix="4"/>
+		<bitfield name="MAX" low="16" high="31" type="ufixed" radix="4"/>
+	</reg32>
+	<reg32 offset="0x2071" name="GRAS_SU_POINT_SIZE" type="fixed" radix="4"/>
+	<reg32 offset="0x2073" name="GRAS_ALPHA_CONTROL">
+		<bitfield name="ALPHA_TEST_ENABLE" pos="2" type="boolean"/>
+		<bitfield name="FORCE_FRAGZ_TO_FS" pos="3" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x2074" name="GRAS_SU_POLY_OFFSET_SCALE" type="float"/>
+	<reg32 offset="0x2075" name="GRAS_SU_POLY_OFFSET_OFFSET" type="float"/>
+	<reg32 offset="0x2076" name="GRAS_SU_POLY_OFFSET_CLAMP" type="float"/>
+	<reg32 offset="0x2077" name="GRAS_DEPTH_CONTROL">
+		<!-- guestimating that this is GRAS based on addr -->
+		<bitfield name="FORMAT" low="0" high="1" type="a4xx_depth_format"/>
+	</reg32>
+	<reg32 offset="0x2078" name="GRAS_SU_MODE_CONTROL">
+		<bitfield name="CULL_FRONT" pos="0" type="boolean"/>
+		<bitfield name="CULL_BACK" pos="1" type="boolean"/>
+		<bitfield name="FRONT_CW" pos="2" type="boolean"/>
+		<bitfield name="LINEHALFWIDTH" low="3" high="10" radix="2" type="fixed"/>
+		<bitfield name="POLY_OFFSET" pos="11" type="boolean"/>
+		<bitfield name="MSAA_ENABLE" pos="13" type="boolean"/>
+		<!-- bit20 set whenever RENDER_MODE = RB_RENDERING_PASS -->
+		<bitfield name="RENDERING_PASS" pos="20" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x207b" name="GRAS_SC_CONTROL">
+		<!-- complete wild-ass-guess for sizes of these bitfields.. -->
+		<bitfield name="RENDER_MODE" low="2" high="3" type="a3xx_render_mode"/>
+		<bitfield name="MSAA_SAMPLES" low="7" high="9" type="uint"/>
+		<bitfield name="MSAA_DISABLE" pos="11" type="boolean"/>
+		<bitfield name="RASTER_MODE" low="12" high="15"/>
+	</reg32>
+	<reg32 offset="0x207c" name="GRAS_SC_SCREEN_SCISSOR_TL" type="adreno_reg_xy"/>
+	<reg32 offset="0x207d" name="GRAS_SC_SCREEN_SCISSOR_BR" type="adreno_reg_xy"/>
+	<reg32 offset="0x209c" name="GRAS_SC_WINDOW_SCISSOR_BR" type="adreno_reg_xy"/>
+	<reg32 offset="0x209d" name="GRAS_SC_WINDOW_SCISSOR_TL" type="adreno_reg_xy"/>
+	<reg32 offset="0x209e" name="GRAS_SC_EXTENT_WINDOW_BR" type="adreno_reg_xy"/>
+	<reg32 offset="0x209f" name="GRAS_SC_EXTENT_WINDOW_TL" type="adreno_reg_xy"/>
+
+	<!-- UCHE registers -->
+	<reg32 offset="0x0e80" name="UCHE_CACHE_MODE_CONTROL"/>
+	<reg32 offset="0x0e83" name="UCHE_TRAP_BASE_LO"/>
+	<reg32 offset="0x0e84" name="UCHE_TRAP_BASE_HI"/>
+	<reg32 offset="0x0e88" name="UCHE_CACHE_STATUS"/>
+	<reg32 offset="0x0e8a" name="UCHE_INVALIDATE0"/>
+	<reg32 offset="0x0e8b" name="UCHE_INVALIDATE1"/>
+	<reg32 offset="0x0e8c" name="UCHE_CACHE_WAYS_VFD"/>
+	<reg32 offset="0x0e8e" name="UCHE_PERFCTR_UCHE_SEL_0" type="a4xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0e8f" name="UCHE_PERFCTR_UCHE_SEL_1" type="a4xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0e90" name="UCHE_PERFCTR_UCHE_SEL_2" type="a4xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0e91" name="UCHE_PERFCTR_UCHE_SEL_3" type="a4xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0e92" name="UCHE_PERFCTR_UCHE_SEL_4" type="a4xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0e93" name="UCHE_PERFCTR_UCHE_SEL_5" type="a4xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0e94" name="UCHE_PERFCTR_UCHE_SEL_6" type="a4xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0e95" name="UCHE_PERFCTR_UCHE_SEL_7" type="a4xx_uche_perfcounter_select"/>
+
+	<!-- HLSQ registers -->
+	<reg32 offset="0x0e00" name="HLSQ_TIMEOUT_THRESHOLD"/>
+	<reg32 offset="0x0e04" name="HLSQ_DEBUG_ECO_CONTROL"/>
+	<!-- always 00000000: -->
+	<reg32 offset="0x0e05" name="HLSQ_MODE_CONTROL"/>
+	<reg32 offset="0x0e0e" name="HLSQ_PERF_PIPE_MASK"/>
+	<reg32 offset="0x0e06" name="HLSQ_PERFCTR_HLSQ_SEL_0" type="a4xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e07" name="HLSQ_PERFCTR_HLSQ_SEL_1" type="a4xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e08" name="HLSQ_PERFCTR_HLSQ_SEL_2" type="a4xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e09" name="HLSQ_PERFCTR_HLSQ_SEL_3" type="a4xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e0a" name="HLSQ_PERFCTR_HLSQ_SEL_4" type="a4xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e0b" name="HLSQ_PERFCTR_HLSQ_SEL_5" type="a4xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e0c" name="HLSQ_PERFCTR_HLSQ_SEL_6" type="a4xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e0d" name="HLSQ_PERFCTR_HLSQ_SEL_7" type="a4xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x23c0" name="HLSQ_CONTROL_0_REG">
+		<!-- I guess same as a3xx, but so far only seen 08000050 -->
+		<bitfield name="FSTHREADSIZE" pos="4" type="a3xx_threadsize"/>
+		<bitfield name="FSSUPERTHREADENABLE" pos="6" type="boolean"/>
+		<bitfield name="SPSHADERRESTART" pos="9" type="boolean"/>
+		<bitfield name="RESERVED2" pos="10" type="boolean"/>
+		<bitfield name="CHUNKDISABLE" pos="26" type="boolean"/>
+		<bitfield name="CONSTMODE" pos="27" type="uint"/>
+		<bitfield name="LAZYUPDATEDISABLE" pos="28" type="boolean"/>
+		<bitfield name="SPCONSTFULLUPDATE" pos="29" type="boolean"/>
+		<bitfield name="TPFULLUPDATE" pos="30" type="boolean"/>
+		<bitfield name="SINGLECONTEXT" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x23c1" name="HLSQ_CONTROL_1_REG">
+		<bitfield name="VSTHREADSIZE" pos="6" type="a3xx_threadsize"/>
+		<bitfield name="VSSUPERTHREADENABLE" pos="8" type="boolean"/>
+		<bitfield name="RESERVED1" pos="9" type="boolean"/>
+		<bitfield name="COORDREGID" low="16" high="23" type="a3xx_regid"/>
+		<!-- set if gl_FragCoord.[zw] used in frag shader: -->
+		<bitfield name="ZWCOORDREGID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0x23c2" name="HLSQ_CONTROL_2_REG">
+		<bitfield name="PRIMALLOCTHRESHOLD" low="26" high="31" type="uint"/>
+		<bitfield name="FACEREGID" low="2" high="9" type="a3xx_regid"/>
+		<bitfield name="SAMPLEID_REGID" low="10" high="17" type="a3xx_regid"/>
+		<bitfield name="SAMPLEMASK_REGID" low="18" high="25" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0x23c3" name="HLSQ_CONTROL_3_REG">
+		<!-- register loaded with position (bary.f) -->
+		<bitfield name="IJ_PERSP_PIXEL" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="IJ_LINEAR_PIXEL" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="IJ_PERSP_CENTROID" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="IJ_LINEAR_CENTROID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<!-- 0x23c4 3 regids, lowest one goes to 0 when *not* per-sample shading -->
+	<reg32 offset="0x23c4" name="HLSQ_CONTROL_4_REG">
+		<bitfield name="IJ_PERSP_SAMPLE" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="IJ_LINEAR_SAMPLE" low="8" high="15" type="a3xx_regid"/>
+	</reg32>
+
+	<bitset name="a4xx_xs_control_reg" inline="yes">
+		<bitfield name="CONSTLENGTH" low="0" high="7" type="uint"/>
+		<bitfield name="CONSTOBJECTOFFSET" low="8" high="14" type="uint"/>
+		<bitfield name="SSBO_ENABLE" pos="15" type="boolean"/>
+		<bitfield name="ENABLED" pos="16" type="boolean"/>
+		<bitfield name="SHADEROBJOFFSET" low="17" high="23" type="uint"/>
+		<bitfield name="INSTRLENGTH" low="24" high="31" type="uint"/>
+	</bitset>
+	<reg32 offset="0x23c5" name="HLSQ_VS_CONTROL_REG" type="a4xx_xs_control_reg"/>
+	<reg32 offset="0x23c6" name="HLSQ_FS_CONTROL_REG" type="a4xx_xs_control_reg"/>
+	<reg32 offset="0x23c7" name="HLSQ_HS_CONTROL_REG" type="a4xx_xs_control_reg"/>
+	<reg32 offset="0x23c8" name="HLSQ_DS_CONTROL_REG" type="a4xx_xs_control_reg"/>
+	<reg32 offset="0x23c9" name="HLSQ_GS_CONTROL_REG" type="a4xx_xs_control_reg"/>
+	<reg32 offset="0x23ca" name="HLSQ_CS_CONTROL_REG" type="a4xx_xs_control_reg"/>
+	<reg32 offset="0x23cd" name="HLSQ_CL_NDRANGE_0">
+		<bitfield name="KERNELDIM" low="0" high="1" type="uint"/>
+		<!-- localsize is value minus one: -->
+		<bitfield name="LOCALSIZEX" low="2" high="11" type="uint"/>
+		<bitfield name="LOCALSIZEY" low="12" high="21" type="uint"/>
+		<bitfield name="LOCALSIZEZ" low="22" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x23ce" name="HLSQ_CL_NDRANGE_1">
+		<bitfield name="SIZE_X" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x23cf" name="HLSQ_CL_NDRANGE_2"/>
+	<reg32 offset="0x23d0" name="HLSQ_CL_NDRANGE_3">
+		<bitfield name="SIZE_Y" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x23d1" name="HLSQ_CL_NDRANGE_4"/>
+	<reg32 offset="0x23d2" name="HLSQ_CL_NDRANGE_5">
+		<bitfield name="SIZE_Z" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x23d3" name="HLSQ_CL_NDRANGE_6"/>
+	<reg32 offset="0x23d4" name="HLSQ_CL_CONTROL_0">
+		<bitfield name="WGIDCONSTID" low="0" high="11" type="a3xx_regid"/>
+		<bitfield name="KERNELDIMCONSTID" low="12" high="23" type="a3xx_regid"/>
+		<bitfield name="LOCALIDREGID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0x23d5" name="HLSQ_CL_CONTROL_1">
+		<!-- GLOBALSIZECONSTID? "kernel size" -->
+		<bitfield name="UNK0CONSTID" low="0" high="11" type="a3xx_regid"/>
+		<bitfield name="WORKGROUPSIZECONSTID" low="12" high="23" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0x23d6" name="HLSQ_CL_KERNEL_CONST">
+		<!-- GLOBALOFFSETCONSTID -->
+		<bitfield name="UNK0CONSTID" low="0" high="11" type="a3xx_regid"/>
+		<bitfield name="NUMWGCONSTID" low="12" high="23" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0x23d7" name="HLSQ_CL_KERNEL_GROUP_X"/>
+	<reg32 offset="0x23d8" name="HLSQ_CL_KERNEL_GROUP_Y"/>
+	<reg32 offset="0x23d9" name="HLSQ_CL_KERNEL_GROUP_Z"/>
+	<reg32 offset="0x23da" name="HLSQ_CL_WG_OFFSET">
+		<!-- WGOFFSETCONSTID -->
+		<bitfield name="UNK0CONSTID" low="0" high="11" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0x23db" name="HLSQ_UPDATE_CONTROL"/>
+
+	<!-- PC registers -->
+	<reg32 offset="0x0d00" name="PC_BINNING_COMMAND">
+		<bitfield name="BINNING_ENABLE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0d08" name="PC_TESSFACTOR_ADDR"/>
+	<reg32 offset="0x0d0c" name="PC_DRAWCALL_SETUP_OVERRIDE"/>
+	<reg32 offset="0x0d10" name="PC_PERFCTR_PC_SEL_0" type="a4xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d11" name="PC_PERFCTR_PC_SEL_1" type="a4xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d12" name="PC_PERFCTR_PC_SEL_2" type="a4xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d13" name="PC_PERFCTR_PC_SEL_3" type="a4xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d14" name="PC_PERFCTR_PC_SEL_4" type="a4xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d15" name="PC_PERFCTR_PC_SEL_5" type="a4xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d16" name="PC_PERFCTR_PC_SEL_6" type="a4xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d17" name="PC_PERFCTR_PC_SEL_7" type="a4xx_pc_perfcounter_select"/>
+	<reg32 offset="0x21c0" name="PC_BIN_BASE"/>
+	<reg32 offset="0x21c2" name="PC_VSTREAM_CONTROL">
+		<doc>SIZE is current pipe width * height (in tiles)</doc>
+		<bitfield name="SIZE" low="16" high="21" type="uint"/>
+		<doc>
+			N is some sort of slot # between 0..(SIZE-1).  In case
+			multiple tiles use same pipe, each tile gets unique slot #
+		</doc>
+		<bitfield name="N" low="22" high="26" type="uint"/>
+	</reg32>
+	<reg32 offset="0x21c4" name="PC_PRIM_VTX_CNTL">
+		<!-- bit0 set if there is >= 1 varying (actually used by FS) -->
+		<bitfield name="VAROUT" low="0" high="3" type="uint">
+			<doc>in groups of 4x vec4, blob only uses values
+			0, 1, 2, 4, 6, 8</doc>
+		</bitfield>
+		<bitfield name="PRIMITIVE_RESTART" pos="20" type="boolean"/>
+		<bitfield name="PROVOKING_VTX_LAST" pos="25" type="boolean"/>
+		<!-- PSIZE bit set if gl_PointSize written: -->
+		<bitfield name="PSIZE" pos="26" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x21c5" name="PC_PRIM_VTX_CNTL2">
+		<bitfield name="POLYMODE_FRONT_PTYPE" low="0" high="2" type="adreno_pa_su_sc_draw"/>
+		<bitfield name="POLYMODE_BACK_PTYPE" low="3" high="5" type="adreno_pa_su_sc_draw"/>
+		<bitfield name="POLYMODE_ENABLE" pos="6" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x21c6" name="PC_RESTART_INDEX"/>
+	<reg32 offset="0x21e5" name="PC_GS_PARAM">
+		<bitfield name="MAX_VERTICES" low="0" high="9" type="uint"/><!-- +1, i.e. max is 1024 -->
+		<bitfield name="INVOCATIONS" low="11" high="15" type="uint"/><!-- +1, i.e. max is 32 -->
+		<bitfield name="PRIMTYPE" low="23" high="24" type="adreno_pa_su_sc_draw"/>
+		<bitfield name="LAYER" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x21e7" name="PC_HS_PARAM">
+		<bitfield name="VERTICES_OUT" low="0" high="5" type="uint"/>
+		<bitfield name="SPACING" low="21" high="22" type="a4xx_tess_spacing"/>
+		<bitfield name="CW" pos="23" type="boolean"/>
+		<bitfield name="CONNECTED" pos="24" type="boolean"/>
+	</reg32>
+
+	<!-- VBIF registers -->
+	<reg32 offset="0x3000" name="VBIF_VERSION"/>
+	<reg32 offset="0x3001" name="VBIF_CLKON">
+		<bitfield name="FORCE_ON_TESTBUS" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x301c" name="VBIF_ABIT_SORT"/>
+	<reg32 offset="0x301d" name="VBIF_ABIT_SORT_CONF"/>
+	<reg32 offset="0x302a" name="VBIF_GATE_OFF_WRREQ_EN"/>
+	<reg32 offset="0x302c" name="VBIF_IN_RD_LIM_CONF0"/>
+	<reg32 offset="0x302d" name="VBIF_IN_RD_LIM_CONF1"/>
+	<reg32 offset="0x3030" name="VBIF_IN_WR_LIM_CONF0"/>
+	<reg32 offset="0x3031" name="VBIF_IN_WR_LIM_CONF1"/>
+	<reg32 offset="0x3049" name="VBIF_ROUND_ROBIN_QOS_ARB"/>
+	<reg32 offset="0x30c0" name="VBIF_PERF_CNT_EN0"/>
+	<reg32 offset="0x30c1" name="VBIF_PERF_CNT_EN1"/>
+	<reg32 offset="0x30c2" name="VBIF_PERF_CNT_EN2"/>
+	<reg32 offset="0x30c3" name="VBIF_PERF_CNT_EN3"/>
+	<reg32 offset="0x30d0" name="VBIF_PERF_CNT_SEL0" type="a4xx_vbif_perfcounter_select"/>
+	<reg32 offset="0x30d1" name="VBIF_PERF_CNT_SEL1" type="a4xx_vbif_perfcounter_select"/>
+	<reg32 offset="0x30d2" name="VBIF_PERF_CNT_SEL2" type="a4xx_vbif_perfcounter_select"/>
+	<reg32 offset="0x30d3" name="VBIF_PERF_CNT_SEL3" type="a4xx_vbif_perfcounter_select"/>
+	<reg32 offset="0x30d8" name="VBIF_PERF_CNT_LOW0"/>
+	<reg32 offset="0x30d9" name="VBIF_PERF_CNT_LOW1"/>
+	<reg32 offset="0x30da" name="VBIF_PERF_CNT_LOW2"/>
+	<reg32 offset="0x30db" name="VBIF_PERF_CNT_LOW3"/>
+	<reg32 offset="0x30e0" name="VBIF_PERF_CNT_HIGH0"/>
+	<reg32 offset="0x30e1" name="VBIF_PERF_CNT_HIGH1"/>
+	<reg32 offset="0x30e2" name="VBIF_PERF_CNT_HIGH2"/>
+	<reg32 offset="0x30e3" name="VBIF_PERF_CNT_HIGH3"/>
+	<reg32 offset="0x3100" name="VBIF_PERF_PWR_CNT_EN0"/>
+	<reg32 offset="0x3101" name="VBIF_PERF_PWR_CNT_EN1"/>
+	<reg32 offset="0x3102" name="VBIF_PERF_PWR_CNT_EN2"/>
+
+	<!--
+	Unknown registers:
+	(mostly related to DX11 features not used yet, I guess?)
+	-->
+
+	<!-- always 00000006: -->
+	<reg32 offset="0x0cc5" name="UNKNOWN_0CC5"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x0cc6" name="UNKNOWN_0CC6"/>
+
+	<!-- always 00000001: -->
+	<reg32 offset="0x0d01" name="UNKNOWN_0D01"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x0e42" name="UNKNOWN_0E42"/>
+
+	<!-- always 00040000: -->
+	<reg32 offset="0x0ec2" name="UNKNOWN_0EC2"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x2001" name="UNKNOWN_2001"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x209b" name="UNKNOWN_209B"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x20ef" name="UNKNOWN_20EF"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x2152" name="UNKNOWN_2152"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x2153" name="UNKNOWN_2153"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x2154" name="UNKNOWN_2154"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x2155" name="UNKNOWN_2155"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x2156" name="UNKNOWN_2156"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x2157" name="UNKNOWN_2157"/>
+
+	<!-- always 0000000b: -->
+	<reg32 offset="0x21c3" name="UNKNOWN_21C3"/>
+
+	<!-- always 00000001: -->
+	<reg32 offset="0x21e6" name="UNKNOWN_21E6"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x2209" name="UNKNOWN_2209"/>
+
+	<!-- always 00000000: -->
+	<reg32 offset="0x22d7" name="UNKNOWN_22D7"/>
+
+        <!-- always 00fcfc00: -->
+        <reg32 offset="0x2352" name="UNKNOWN_2352"/>
+
+</domain>
+
+
+<domain name="A4XX_TEX_SAMP" width="32">
+	<doc>Texture sampler dwords</doc>
+	<enum name="a4xx_tex_filter">
+		<value name="A4XX_TEX_NEAREST" value="0"/>
+		<value name="A4XX_TEX_LINEAR" value="1"/>
+		<value name="A4XX_TEX_ANISO" value="2"/>
+	</enum>
+	<enum name="a4xx_tex_clamp">
+		<value name="A4XX_TEX_REPEAT" value="0"/>
+		<value name="A4XX_TEX_CLAMP_TO_EDGE" value="1"/>
+		<value name="A4XX_TEX_MIRROR_REPEAT" value="2"/>
+		<value name="A4XX_TEX_CLAMP_TO_BORDER" value="3"/>
+		<value name="A4XX_TEX_MIRROR_CLAMP" value="4"/>
+	</enum>
+	<enum name="a4xx_tex_aniso">
+		<value name="A4XX_TEX_ANISO_1" value="0"/>
+		<value name="A4XX_TEX_ANISO_2" value="1"/>
+		<value name="A4XX_TEX_ANISO_4" value="2"/>
+		<value name="A4XX_TEX_ANISO_8" value="3"/>
+		<value name="A4XX_TEX_ANISO_16" value="4"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="MIPFILTER_LINEAR_NEAR" pos="0" type="boolean"/>
+		<bitfield name="XY_MAG" low="1" high="2" type="a4xx_tex_filter"/>
+		<bitfield name="XY_MIN" low="3" high="4" type="a4xx_tex_filter"/>
+		<bitfield name="WRAP_S" low="5" high="7" type="a4xx_tex_clamp"/>
+		<bitfield name="WRAP_T" low="8" high="10" type="a4xx_tex_clamp"/>
+		<bitfield name="WRAP_R" low="11" high="13" type="a4xx_tex_clamp"/>
+		<bitfield name="ANISO" low="14" high="16" type="a4xx_tex_aniso"/>
+		<bitfield name="LOD_BIAS" low="19" high="31" type="fixed" radix="8"/><!-- no idea how many bits for real -->
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="COMPARE_FUNC" low="1" high="3" type="adreno_compare_func"/>
+		<bitfield name="CUBEMAPSEAMLESSFILTOFF" pos="4" type="boolean"/>
+		<bitfield name="UNNORM_COORDS" pos="5" type="boolean"/>
+		<bitfield name="MIPFILTER_LINEAR_FAR" pos="6" type="boolean"/>
+		<bitfield name="MAX_LOD" low="8" high="19" type="ufixed" radix="8"/>
+		<bitfield name="MIN_LOD" low="20" high="31" type="ufixed" radix="8"/>
+	</reg32>
+</domain>
+
+<domain name="A4XX_TEX_CONST" width="32">
+	<doc>Texture constant dwords</doc>
+	<enum name="a4xx_tex_swiz">
+		<!-- same as a2xx? -->
+		<value name="A4XX_TEX_X" value="0"/>
+		<value name="A4XX_TEX_Y" value="1"/>
+		<value name="A4XX_TEX_Z" value="2"/>
+		<value name="A4XX_TEX_W" value="3"/>
+		<value name="A4XX_TEX_ZERO" value="4"/>
+		<value name="A4XX_TEX_ONE" value="5"/>
+	</enum>
+	<enum name="a4xx_tex_type">
+		<value name="A4XX_TEX_1D" value="0"/>
+		<value name="A4XX_TEX_2D" value="1"/>
+		<value name="A4XX_TEX_CUBE" value="2"/>
+		<value name="A4XX_TEX_3D" value="3"/>
+		<value name="A4XX_TEX_BUFFER" value="4"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="TILED" pos="0" type="boolean"/>
+		<bitfield name="SRGB" pos="2" type="boolean"/>
+		<bitfield name="SWIZ_X" low="4" high="6" type="a4xx_tex_swiz"/>
+		<bitfield name="SWIZ_Y" low="7" high="9" type="a4xx_tex_swiz"/>
+		<bitfield name="SWIZ_Z" low="10" high="12" type="a4xx_tex_swiz"/>
+		<bitfield name="SWIZ_W" low="13" high="15" type="a4xx_tex_swiz"/>
+		<bitfield name="MIPLVLS" low="16" high="19" type="uint"/>
+		<bitfield name="FMT" low="22" high="28" type="a4xx_tex_fmt"/>
+		<bitfield name="TYPE" low="29" high="31" type="a4xx_tex_type"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="HEIGHT" low="0" high="14" type="uint"/>
+		<bitfield name="WIDTH" low="15" high="29" type="uint"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<!-- minimum pitch (for mipmap levels): log2(pitchalign / 32) -->
+		<bitfield name="PITCHALIGN" low="0" high="3" type="uint"/>
+		<bitfield name="BUFFER" pos="6" type="boolean"/>
+		<doc>Pitch in bytes (so actually stride)</doc>
+		<bitfield name="PITCH" low="9" high="29" type="uint"/>
+		<bitfield name="SWAP" low="30" high="31" type="a3xx_color_swap"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="LAYERSZ" low="0" high="13" shr="12" type="uint"/>
+		<bitfield name="DEPTH" low="18" high="30" type="uint"/>
+	</reg32>
+	<reg32 offset="4" name="4">
+		<!--
+		like a3xx we seem to have two LAYERSZ's.. although this one
+		seems too small to be useful, and when it overflows blob just
+		sets it to zero..
+		 -->
+		<bitfield name="LAYERSZ" low="0" high="3" shr="12" type="uint"/>
+		<bitfield name="BASE" low="5" high="31" shr="5"/>
+	</reg32>
+	<reg32 offset="5" name="5"/>
+	<reg32 offset="6" name="6"/>
+	<reg32 offset="7" name="7"/>
+</domain>
+
+<domain name="A4XX_SSBO_0" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="BASE" low="5" high="31" shr="5"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<doc>Pitch in bytes (so actually stride)</doc>
+		<bitfield name="PITCH" low="0" high="21" type="uint"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="ARRAY_PITCH" low="12" high="25" shr="12" type="uint"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<!-- bytes per pixel: -->
+		<bitfield name="CPP" low="0" high="5" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="A4XX_SSBO_1" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="CPP" low="0" high="4" type="uint"/>
+		<bitfield name="FMT" low="8" high="15" type="a4xx_color_fmt"/>
+		<bitfield name="WIDTH" low="16" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="HEIGHT" low="0" high="15" type="uint"/>
+		<bitfield name="DEPTH" low="16" high="31" type="uint"/>
+	</reg32>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/adreno/a5xx.xml b/drivers/gpu/drm/msm/registers/adreno/a5xx.xml
new file mode 100644
index 000000000000..bd8df5945166
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/adreno/a5xx.xml
@@ -0,0 +1,3039 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+<import file="adreno/adreno_common.xml"/>
+<import file="adreno/adreno_pm4.xml"/>
+
+<enum name="a5xx_color_fmt">
+	<value value="0x02" name="RB5_A8_UNORM"/>
+	<value value="0x03" name="RB5_R8_UNORM"/>
+	<value value="0x04" name="RB5_R8_SNORM"/>
+	<value value="0x05" name="RB5_R8_UINT"/>
+	<value value="0x06" name="RB5_R8_SINT"/>
+	<value value="0x08" name="RB5_R4G4B4A4_UNORM"/>
+	<value value="0x0a" name="RB5_R5G5B5A1_UNORM"/>
+	<value value="0x0e" name="RB5_R5G6B5_UNORM"/>
+	<value value="0x0f" name="RB5_R8G8_UNORM"/>
+	<value value="0x10" name="RB5_R8G8_SNORM"/>
+	<value value="0x11" name="RB5_R8G8_UINT"/>
+	<value value="0x12" name="RB5_R8G8_SINT"/>
+	<value value="0x15" name="RB5_R16_UNORM"/>
+	<value value="0x16" name="RB5_R16_SNORM"/>
+	<value value="0x17" name="RB5_R16_FLOAT"/>
+	<value value="0x18" name="RB5_R16_UINT"/>
+	<value value="0x19" name="RB5_R16_SINT"/>
+	<value value="0x30" name="RB5_R8G8B8A8_UNORM"/>
+	<value value="0x31" name="RB5_R8G8B8_UNORM"/>
+	<value value="0x32" name="RB5_R8G8B8A8_SNORM"/>
+	<value value="0x33" name="RB5_R8G8B8A8_UINT"/>
+	<value value="0x34" name="RB5_R8G8B8A8_SINT"/>
+	<value value="0x37" name="RB5_R10G10B10A2_UNORM"/>  <!-- GL_RGB10_A2 -->
+	<value value="0x3a" name="RB5_R10G10B10A2_UINT"/>   <!-- GL_RGB10_A2UI -->
+	<value value="0x42" name="RB5_R11G11B10_FLOAT"/>    <!-- GL_R11F_G11F_B10F -->
+	<value value="0x43" name="RB5_R16G16_UNORM"/>
+	<value value="0x44" name="RB5_R16G16_SNORM"/>
+	<value value="0x45" name="RB5_R16G16_FLOAT"/>
+	<value value="0x46" name="RB5_R16G16_UINT"/>
+	<value value="0x47" name="RB5_R16G16_SINT"/>
+	<value value="0x4a" name="RB5_R32_FLOAT"/>
+	<value value="0x4b" name="RB5_R32_UINT"/>
+	<value value="0x4c" name="RB5_R32_SINT"/>
+	<value value="0x60" name="RB5_R16G16B16A16_UNORM"/>
+	<value value="0x61" name="RB5_R16G16B16A16_SNORM"/>
+	<value value="0x62" name="RB5_R16G16B16A16_FLOAT"/>
+	<value value="0x63" name="RB5_R16G16B16A16_UINT"/>
+	<value value="0x64" name="RB5_R16G16B16A16_SINT"/>
+	<value value="0x67" name="RB5_R32G32_FLOAT"/>
+	<value value="0x68" name="RB5_R32G32_UINT"/>
+	<value value="0x69" name="RB5_R32G32_SINT"/>
+	<value value="0x82" name="RB5_R32G32B32A32_FLOAT"/>
+	<value value="0x83" name="RB5_R32G32B32A32_UINT"/>
+	<value value="0x84" name="RB5_R32G32B32A32_SINT"/>
+
+	<value value="0xff" name="RB5_NONE"/>
+</enum>
+
+<enum name="a5xx_tile_mode">
+	<value name="TILE5_LINEAR" value="0"/>
+	<value name="TILE5_2" value="2"/>
+	<value name="TILE5_3" value="3"/>
+</enum>
+
+<enum name="a5xx_vtx_fmt" prefix="chipset">
+	<value value="0x03" name="VFMT5_8_UNORM"/>
+	<value value="0x04" name="VFMT5_8_SNORM"/>
+	<value value="0x05" name="VFMT5_8_UINT"/>
+	<value value="0x06" name="VFMT5_8_SINT"/>
+
+	<value value="0x0f" name="VFMT5_8_8_UNORM"/>
+	<value value="0x10" name="VFMT5_8_8_SNORM"/>
+	<value value="0x11" name="VFMT5_8_8_UINT"/>
+	<value value="0x12" name="VFMT5_8_8_SINT"/>
+
+	<value value="0x15" name="VFMT5_16_UNORM"/>
+	<value value="0x16" name="VFMT5_16_SNORM"/>
+	<value value="0x17" name="VFMT5_16_FLOAT"/>
+	<value value="0x18" name="VFMT5_16_UINT"/>
+	<value value="0x19" name="VFMT5_16_SINT"/>
+
+	<value value="0x21" name="VFMT5_8_8_8_UNORM"/>
+	<value value="0x22" name="VFMT5_8_8_8_SNORM"/>
+	<value value="0x23" name="VFMT5_8_8_8_UINT"/>
+	<value value="0x24" name="VFMT5_8_8_8_SINT"/>
+
+	<value value="0x30" name="VFMT5_8_8_8_8_UNORM"/>
+	<value value="0x32" name="VFMT5_8_8_8_8_SNORM"/>
+	<value value="0x33" name="VFMT5_8_8_8_8_UINT"/>
+	<value value="0x34" name="VFMT5_8_8_8_8_SINT"/>
+
+	<value value="0x36" name="VFMT5_10_10_10_2_UNORM"/>
+	<value value="0x39" name="VFMT5_10_10_10_2_SNORM"/>
+	<value value="0x3a" name="VFMT5_10_10_10_2_UINT"/>
+	<value value="0x3b" name="VFMT5_10_10_10_2_SINT"/>
+
+	<value value="0x42" name="VFMT5_11_11_10_FLOAT"/>
+
+	<value value="0x43" name="VFMT5_16_16_UNORM"/>
+	<value value="0x44" name="VFMT5_16_16_SNORM"/>
+	<value value="0x45" name="VFMT5_16_16_FLOAT"/>
+	<value value="0x46" name="VFMT5_16_16_UINT"/>
+	<value value="0x47" name="VFMT5_16_16_SINT"/>
+
+	<value value="0x48" name="VFMT5_32_UNORM"/>
+	<value value="0x49" name="VFMT5_32_SNORM"/>
+	<value value="0x4a" name="VFMT5_32_FLOAT"/>
+	<value value="0x4b" name="VFMT5_32_UINT"/>
+	<value value="0x4c" name="VFMT5_32_SINT"/>
+	<value value="0x4d" name="VFMT5_32_FIXED"/>
+
+	<value value="0x58" name="VFMT5_16_16_16_UNORM"/>
+	<value value="0x59" name="VFMT5_16_16_16_SNORM"/>
+	<value value="0x5a" name="VFMT5_16_16_16_FLOAT"/>
+	<value value="0x5b" name="VFMT5_16_16_16_UINT"/>
+	<value value="0x5c" name="VFMT5_16_16_16_SINT"/>
+
+	<value value="0x60" name="VFMT5_16_16_16_16_UNORM"/>
+	<value value="0x61" name="VFMT5_16_16_16_16_SNORM"/>
+	<value value="0x62" name="VFMT5_16_16_16_16_FLOAT"/>
+	<value value="0x63" name="VFMT5_16_16_16_16_UINT"/>
+	<value value="0x64" name="VFMT5_16_16_16_16_SINT"/>
+
+	<value value="0x65" name="VFMT5_32_32_UNORM"/>
+	<value value="0x66" name="VFMT5_32_32_SNORM"/>
+	<value value="0x67" name="VFMT5_32_32_FLOAT"/>
+	<value value="0x68" name="VFMT5_32_32_UINT"/>
+	<value value="0x69" name="VFMT5_32_32_SINT"/>
+	<value value="0x6a" name="VFMT5_32_32_FIXED"/>
+
+	<value value="0x70" name="VFMT5_32_32_32_UNORM"/>
+	<value value="0x71" name="VFMT5_32_32_32_SNORM"/>
+	<value value="0x72" name="VFMT5_32_32_32_UINT"/>
+	<value value="0x73" name="VFMT5_32_32_32_SINT"/>
+	<value value="0x74" name="VFMT5_32_32_32_FLOAT"/>
+	<value value="0x75" name="VFMT5_32_32_32_FIXED"/>
+
+	<value value="0x80" name="VFMT5_32_32_32_32_UNORM"/>
+	<value value="0x81" name="VFMT5_32_32_32_32_SNORM"/>
+	<value value="0x82" name="VFMT5_32_32_32_32_FLOAT"/>
+	<value value="0x83" name="VFMT5_32_32_32_32_UINT"/>
+	<value value="0x84" name="VFMT5_32_32_32_32_SINT"/>
+	<value value="0x85" name="VFMT5_32_32_32_32_FIXED"/>
+
+	<value value="0xff" name="VFMT5_NONE"/>
+</enum>
+
+<enum name="a5xx_tex_fmt">
+	<value value="0x02" name="TFMT5_A8_UNORM"/>
+	<value value="0x03" name="TFMT5_8_UNORM"/>
+	<value value="0x04" name="TFMT5_8_SNORM"/>
+	<value value="0x05" name="TFMT5_8_UINT"/>
+	<value value="0x06" name="TFMT5_8_SINT"/>
+	<value value="0x08" name="TFMT5_4_4_4_4_UNORM"/>
+	<value value="0x0a" name="TFMT5_5_5_5_1_UNORM"/>
+	<value value="0x0e" name="TFMT5_5_6_5_UNORM"/>
+	<value value="0x0f" name="TFMT5_8_8_UNORM"/>
+	<value value="0x10" name="TFMT5_8_8_SNORM"/>
+	<value value="0x11" name="TFMT5_8_8_UINT"/>
+	<value value="0x12" name="TFMT5_8_8_SINT"/>
+	<value value="0x13" name="TFMT5_L8_A8_UNORM"/>
+	<value value="0x15" name="TFMT5_16_UNORM"/>
+	<value value="0x16" name="TFMT5_16_SNORM"/>
+	<value value="0x17" name="TFMT5_16_FLOAT"/>
+	<value value="0x18" name="TFMT5_16_UINT"/>
+	<value value="0x19" name="TFMT5_16_SINT"/>
+	<value value="0x30" name="TFMT5_8_8_8_8_UNORM"/>
+	<value value="0x31" name="TFMT5_8_8_8_UNORM"/>
+	<value value="0x32" name="TFMT5_8_8_8_8_SNORM"/>
+	<value value="0x33" name="TFMT5_8_8_8_8_UINT"/>
+	<value value="0x34" name="TFMT5_8_8_8_8_SINT"/>
+	<value value="0x35" name="TFMT5_9_9_9_E5_FLOAT"/>
+	<value value="0x36" name="TFMT5_10_10_10_2_UNORM"/>
+	<value value="0x3a" name="TFMT5_10_10_10_2_UINT"/>
+	<value value="0x42" name="TFMT5_11_11_10_FLOAT"/>
+	<value value="0x43" name="TFMT5_16_16_UNORM"/>
+	<value value="0x44" name="TFMT5_16_16_SNORM"/>
+	<value value="0x45" name="TFMT5_16_16_FLOAT"/>
+	<value value="0x46" name="TFMT5_16_16_UINT"/>
+	<value value="0x47" name="TFMT5_16_16_SINT"/>
+	<value value="0x4a" name="TFMT5_32_FLOAT"/>
+	<value value="0x4b" name="TFMT5_32_UINT"/>
+	<value value="0x4c" name="TFMT5_32_SINT"/>
+	<value value="0x60" name="TFMT5_16_16_16_16_UNORM"/>
+	<value value="0x61" name="TFMT5_16_16_16_16_SNORM"/>
+	<value value="0x62" name="TFMT5_16_16_16_16_FLOAT"/>
+	<value value="0x63" name="TFMT5_16_16_16_16_UINT"/>
+	<value value="0x64" name="TFMT5_16_16_16_16_SINT"/>
+	<value value="0x67" name="TFMT5_32_32_FLOAT"/>
+	<value value="0x68" name="TFMT5_32_32_UINT"/>
+	<value value="0x69" name="TFMT5_32_32_SINT"/>
+	<value value="0x72" name="TFMT5_32_32_32_UINT"/>
+	<value value="0x73" name="TFMT5_32_32_32_SINT"/>
+	<value value="0x74" name="TFMT5_32_32_32_FLOAT"/>
+	<value value="0x82" name="TFMT5_32_32_32_32_FLOAT"/>
+	<value value="0x83" name="TFMT5_32_32_32_32_UINT"/>
+	<value value="0x84" name="TFMT5_32_32_32_32_SINT"/>
+	<value value="0xa0" name="TFMT5_X8Z24_UNORM"/>
+
+	<value value="0xab" name="TFMT5_ETC2_RG11_UNORM"/>
+	<value value="0xac" name="TFMT5_ETC2_RG11_SNORM"/>
+	<value value="0xad" name="TFMT5_ETC2_R11_UNORM"/>
+	<value value="0xae" name="TFMT5_ETC2_R11_SNORM"/>
+	<value value="0xaf" name="TFMT5_ETC1"/>
+	<value value="0xb0" name="TFMT5_ETC2_RGB8"/>
+	<value value="0xb1" name="TFMT5_ETC2_RGBA8"/>
+	<value value="0xb2" name="TFMT5_ETC2_RGB8A1"/>
+	<value value="0xb3" name="TFMT5_DXT1"/>
+	<value value="0xb4" name="TFMT5_DXT3"/>
+	<value value="0xb5" name="TFMT5_DXT5"/>
+	<value value="0xb7" name="TFMT5_RGTC1_UNORM"/>
+	<value value="0xb8" name="TFMT5_RGTC1_SNORM"/>
+	<value value="0xbb" name="TFMT5_RGTC2_UNORM"/>
+	<value value="0xbc" name="TFMT5_RGTC2_SNORM"/>
+	<value value="0xbe" name="TFMT5_BPTC_UFLOAT"/>
+	<value value="0xbf" name="TFMT5_BPTC_FLOAT"/>
+	<value value="0xc0" name="TFMT5_BPTC"/>
+	<value value="0xc1" name="TFMT5_ASTC_4x4"/>
+	<value value="0xc2" name="TFMT5_ASTC_5x4"/>
+	<value value="0xc3" name="TFMT5_ASTC_5x5"/>
+	<value value="0xc4" name="TFMT5_ASTC_6x5"/>
+	<value value="0xc5" name="TFMT5_ASTC_6x6"/>
+	<value value="0xc6" name="TFMT5_ASTC_8x5"/>
+	<value value="0xc7" name="TFMT5_ASTC_8x6"/>
+	<value value="0xc8" name="TFMT5_ASTC_8x8"/>
+	<value value="0xc9" name="TFMT5_ASTC_10x5"/>
+	<value value="0xca" name="TFMT5_ASTC_10x6"/>
+	<value value="0xcb" name="TFMT5_ASTC_10x8"/>
+	<value value="0xcc" name="TFMT5_ASTC_10x10"/>
+	<value value="0xcd" name="TFMT5_ASTC_12x10"/>
+	<value value="0xce" name="TFMT5_ASTC_12x12"/>
+
+	<value value="0xff" name="TFMT5_NONE"/>
+</enum>
+
+<enum name="a5xx_depth_format">
+	<value name="DEPTH5_NONE" value="0"/>
+	<value name="DEPTH5_16" value="1"/>
+	<value name="DEPTH5_24_8" value="2"/>
+	<value name="DEPTH5_32" value="4"/>
+</enum>
+
+<enum name="a5xx_blit_buf">
+	<value value="0" name="BLIT_MRT0"/>
+	<value value="1" name="BLIT_MRT1"/>
+	<value value="2" name="BLIT_MRT2"/>
+	<value value="3" name="BLIT_MRT3"/>
+	<value value="4" name="BLIT_MRT4"/>
+	<value value="5" name="BLIT_MRT5"/>
+	<value value="6" name="BLIT_MRT6"/>
+	<value value="7" name="BLIT_MRT7"/>
+	<value value="8" name="BLIT_ZS"/>       <!-- depth or combined depth+stencil -->
+	<value value="9" name="BLIT_S"/>        <!-- separate stencil -->
+</enum>
+
+<!-- see comment in a4xx.xml about script to extract countables from test-perf output -->
+<enum name="a5xx_cp_perfcounter_select">
+	<value value="0" name="PERF_CP_ALWAYS_COUNT"/>
+	<value value="1" name="PERF_CP_BUSY_GFX_CORE_IDLE"/>
+	<value value="2" name="PERF_CP_BUSY_CYCLES"/>
+	<value value="3" name="PERF_CP_PFP_IDLE"/>
+	<value value="4" name="PERF_CP_PFP_BUSY_WORKING"/>
+	<value value="5" name="PERF_CP_PFP_STALL_CYCLES_ANY"/>
+	<value value="6" name="PERF_CP_PFP_STARVE_CYCLES_ANY"/>
+	<value value="7" name="PERF_CP_PFP_ICACHE_MISS"/>
+	<value value="8" name="PERF_CP_PFP_ICACHE_HIT"/>
+	<value value="9" name="PERF_CP_PFP_MATCH_PM4_PKT_PROFILE"/>
+	<value value="10" name="PERF_CP_ME_BUSY_WORKING"/>
+	<value value="11" name="PERF_CP_ME_IDLE"/>
+	<value value="12" name="PERF_CP_ME_STARVE_CYCLES_ANY"/>
+	<value value="13" name="PERF_CP_ME_FIFO_EMPTY_PFP_IDLE"/>
+	<value value="14" name="PERF_CP_ME_FIFO_EMPTY_PFP_BUSY"/>
+	<value value="15" name="PERF_CP_ME_FIFO_FULL_ME_BUSY"/>
+	<value value="16" name="PERF_CP_ME_FIFO_FULL_ME_NON_WORKING"/>
+	<value value="17" name="PERF_CP_ME_STALL_CYCLES_ANY"/>
+	<value value="18" name="PERF_CP_ME_ICACHE_MISS"/>
+	<value value="19" name="PERF_CP_ME_ICACHE_HIT"/>
+	<value value="20" name="PERF_CP_NUM_PREEMPTIONS"/>
+	<value value="21" name="PERF_CP_PREEMPTION_REACTION_DELAY"/>
+	<value value="22" name="PERF_CP_PREEMPTION_SWITCH_OUT_TIME"/>
+	<value value="23" name="PERF_CP_PREEMPTION_SWITCH_IN_TIME"/>
+	<value value="24" name="PERF_CP_DEAD_DRAWS_IN_BIN_RENDER"/>
+	<value value="25" name="PERF_CP_PREDICATED_DRAWS_KILLED"/>
+	<value value="26" name="PERF_CP_MODE_SWITCH"/>
+	<value value="27" name="PERF_CP_ZPASS_DONE"/>
+	<value value="28" name="PERF_CP_CONTEXT_DONE"/>
+	<value value="29" name="PERF_CP_CACHE_FLUSH"/>
+	<value value="30" name="PERF_CP_LONG_PREEMPTIONS"/>
+</enum>
+
+<enum name="a5xx_rbbm_perfcounter_select">
+	<value value="0" name="PERF_RBBM_ALWAYS_COUNT"/>
+	<value value="1" name="PERF_RBBM_ALWAYS_ON"/>
+	<value value="2" name="PERF_RBBM_TSE_BUSY"/>
+	<value value="3" name="PERF_RBBM_RAS_BUSY"/>
+	<value value="4" name="PERF_RBBM_PC_DCALL_BUSY"/>
+	<value value="5" name="PERF_RBBM_PC_VSD_BUSY"/>
+	<value value="6" name="PERF_RBBM_STATUS_MASKED"/>
+	<value value="7" name="PERF_RBBM_COM_BUSY"/>
+	<value value="8" name="PERF_RBBM_DCOM_BUSY"/>
+	<value value="9" name="PERF_RBBM_VBIF_BUSY"/>
+	<value value="10" name="PERF_RBBM_VSC_BUSY"/>
+	<value value="11" name="PERF_RBBM_TESS_BUSY"/>
+	<value value="12" name="PERF_RBBM_UCHE_BUSY"/>
+	<value value="13" name="PERF_RBBM_HLSQ_BUSY"/>
+</enum>
+
+<enum name="a5xx_pc_perfcounter_select">
+	<value value="0" name="PERF_PC_BUSY_CYCLES"/>
+	<value value="1" name="PERF_PC_WORKING_CYCLES"/>
+	<value value="2" name="PERF_PC_STALL_CYCLES_VFD"/>
+	<value value="3" name="PERF_PC_STALL_CYCLES_TSE"/>
+	<value value="4" name="PERF_PC_STALL_CYCLES_VPC"/>
+	<value value="5" name="PERF_PC_STALL_CYCLES_UCHE"/>
+	<value value="6" name="PERF_PC_STALL_CYCLES_TESS"/>
+	<value value="7" name="PERF_PC_STALL_CYCLES_TSE_ONLY"/>
+	<value value="8" name="PERF_PC_STALL_CYCLES_VPC_ONLY"/>
+	<value value="9" name="PERF_PC_PASS1_TF_STALL_CYCLES"/>
+	<value value="10" name="PERF_PC_STARVE_CYCLES_FOR_INDEX"/>
+	<value value="11" name="PERF_PC_STARVE_CYCLES_FOR_TESS_FACTOR"/>
+	<value value="12" name="PERF_PC_STARVE_CYCLES_FOR_VIZ_STREAM"/>
+	<value value="13" name="PERF_PC_STARVE_CYCLES_FOR_POSITION"/>
+	<value value="14" name="PERF_PC_STARVE_CYCLES_DI"/>
+	<value value="15" name="PERF_PC_VIS_STREAMS_LOADED"/>
+	<value value="16" name="PERF_PC_INSTANCES"/>
+	<value value="17" name="PERF_PC_VPC_PRIMITIVES"/>
+	<value value="18" name="PERF_PC_DEAD_PRIM"/>
+	<value value="19" name="PERF_PC_LIVE_PRIM"/>
+	<value value="20" name="PERF_PC_VERTEX_HITS"/>
+	<value value="21" name="PERF_PC_IA_VERTICES"/>
+	<value value="22" name="PERF_PC_IA_PRIMITIVES"/>
+	<value value="23" name="PERF_PC_GS_PRIMITIVES"/>
+	<value value="24" name="PERF_PC_HS_INVOCATIONS"/>
+	<value value="25" name="PERF_PC_DS_INVOCATIONS"/>
+	<value value="26" name="PERF_PC_VS_INVOCATIONS"/>
+	<value value="27" name="PERF_PC_GS_INVOCATIONS"/>
+	<value value="28" name="PERF_PC_DS_PRIMITIVES"/>
+	<value value="29" name="PERF_PC_VPC_POS_DATA_TRANSACTION"/>
+	<value value="30" name="PERF_PC_3D_DRAWCALLS"/>
+	<value value="31" name="PERF_PC_2D_DRAWCALLS"/>
+	<value value="32" name="PERF_PC_NON_DRAWCALL_GLOBAL_EVENTS"/>
+	<value value="33" name="PERF_TESS_BUSY_CYCLES"/>
+	<value value="34" name="PERF_TESS_WORKING_CYCLES"/>
+	<value value="35" name="PERF_TESS_STALL_CYCLES_PC"/>
+	<value value="36" name="PERF_TESS_STARVE_CYCLES_PC"/>
+</enum>
+
+<enum name="a5xx_vfd_perfcounter_select">
+	<value value="0" name="PERF_VFD_BUSY_CYCLES"/>
+	<value value="1" name="PERF_VFD_STALL_CYCLES_UCHE"/>
+	<value value="2" name="PERF_VFD_STALL_CYCLES_VPC_ALLOC"/>
+	<value value="3" name="PERF_VFD_STALL_CYCLES_MISS_VB"/>
+	<value value="4" name="PERF_VFD_STALL_CYCLES_MISS_Q"/>
+	<value value="5" name="PERF_VFD_STALL_CYCLES_SP_INFO"/>
+	<value value="6" name="PERF_VFD_STALL_CYCLES_SP_ATTR"/>
+	<value value="7" name="PERF_VFD_STALL_CYCLES_VFDP_VB"/>
+	<value value="8" name="PERF_VFD_STALL_CYCLES_VFDP_Q"/>
+	<value value="9" name="PERF_VFD_DECODER_PACKER_STALL"/>
+	<value value="10" name="PERF_VFD_STARVE_CYCLES_UCHE"/>
+	<value value="11" name="PERF_VFD_RBUFFER_FULL"/>
+	<value value="12" name="PERF_VFD_ATTR_INFO_FIFO_FULL"/>
+	<value value="13" name="PERF_VFD_DECODED_ATTRIBUTE_BYTES"/>
+	<value value="14" name="PERF_VFD_NUM_ATTRIBUTES"/>
+	<value value="15" name="PERF_VFD_INSTRUCTIONS"/>
+	<value value="16" name="PERF_VFD_UPPER_SHADER_FIBERS"/>
+	<value value="17" name="PERF_VFD_LOWER_SHADER_FIBERS"/>
+	<value value="18" name="PERF_VFD_MODE_0_FIBERS"/>
+	<value value="19" name="PERF_VFD_MODE_1_FIBERS"/>
+	<value value="20" name="PERF_VFD_MODE_2_FIBERS"/>
+	<value value="21" name="PERF_VFD_MODE_3_FIBERS"/>
+	<value value="22" name="PERF_VFD_MODE_4_FIBERS"/>
+	<value value="23" name="PERF_VFD_TOTAL_VERTICES"/>
+	<value value="24" name="PERF_VFD_NUM_ATTR_MISS"/>
+	<value value="25" name="PERF_VFD_1_BURST_REQ"/>
+	<value value="26" name="PERF_VFDP_STALL_CYCLES_VFD"/>
+	<value value="27" name="PERF_VFDP_STALL_CYCLES_VFD_INDEX"/>
+	<value value="28" name="PERF_VFDP_STALL_CYCLES_VFD_PROG"/>
+	<value value="29" name="PERF_VFDP_STARVE_CYCLES_PC"/>
+	<value value="30" name="PERF_VFDP_VS_STAGE_32_WAVES"/>
+</enum>
+
+<enum name="a5xx_hlsq_perfcounter_select">
+	<value value="0" name="PERF_HLSQ_BUSY_CYCLES"/>
+	<value value="1" name="PERF_HLSQ_STALL_CYCLES_UCHE"/>
+	<value value="2" name="PERF_HLSQ_STALL_CYCLES_SP_STATE"/>
+	<value value="3" name="PERF_HLSQ_STALL_CYCLES_SP_FS_STAGE"/>
+	<value value="4" name="PERF_HLSQ_UCHE_LATENCY_CYCLES"/>
+	<value value="5" name="PERF_HLSQ_UCHE_LATENCY_COUNT"/>
+	<value value="6" name="PERF_HLSQ_FS_STAGE_32_WAVES"/>
+	<value value="7" name="PERF_HLSQ_FS_STAGE_64_WAVES"/>
+	<value value="8" name="PERF_HLSQ_QUADS"/>
+	<value value="9" name="PERF_HLSQ_SP_STATE_COPY_TRANS_FS_STAGE"/>
+	<value value="10" name="PERF_HLSQ_SP_STATE_COPY_TRANS_VS_STAGE"/>
+	<value value="11" name="PERF_HLSQ_TP_STATE_COPY_TRANS_FS_STAGE"/>
+	<value value="12" name="PERF_HLSQ_TP_STATE_COPY_TRANS_VS_STAGE"/>
+	<value value="13" name="PERF_HLSQ_CS_INVOCATIONS"/>
+	<value value="14" name="PERF_HLSQ_COMPUTE_DRAWCALLS"/>
+</enum>
+
+<enum name="a5xx_vpc_perfcounter_select">
+	<value value="0" name="PERF_VPC_BUSY_CYCLES"/>
+	<value value="1" name="PERF_VPC_WORKING_CYCLES"/>
+	<value value="2" name="PERF_VPC_STALL_CYCLES_UCHE"/>
+	<value value="3" name="PERF_VPC_STALL_CYCLES_VFD_WACK"/>
+	<value value="4" name="PERF_VPC_STALL_CYCLES_HLSQ_PRIM_ALLOC"/>
+	<value value="5" name="PERF_VPC_STALL_CYCLES_PC"/>
+	<value value="6" name="PERF_VPC_STALL_CYCLES_SP_LM"/>
+	<value value="7" name="PERF_VPC_POS_EXPORT_STALL_CYCLES"/>
+	<value value="8" name="PERF_VPC_STARVE_CYCLES_SP"/>
+	<value value="9" name="PERF_VPC_STARVE_CYCLES_LRZ"/>
+	<value value="10" name="PERF_VPC_PC_PRIMITIVES"/>
+	<value value="11" name="PERF_VPC_SP_COMPONENTS"/>
+	<value value="12" name="PERF_VPC_SP_LM_PRIMITIVES"/>
+	<value value="13" name="PERF_VPC_SP_LM_COMPONENTS"/>
+	<value value="14" name="PERF_VPC_SP_LM_DWORDS"/>
+	<value value="15" name="PERF_VPC_STREAMOUT_COMPONENTS"/>
+	<value value="16" name="PERF_VPC_GRANT_PHASES"/>
+</enum>
+
+<enum name="a5xx_tse_perfcounter_select">
+	<value value="0" name="PERF_TSE_BUSY_CYCLES"/>
+	<value value="1" name="PERF_TSE_CLIPPING_CYCLES"/>
+	<value value="2" name="PERF_TSE_STALL_CYCLES_RAS"/>
+	<value value="3" name="PERF_TSE_STALL_CYCLES_LRZ_BARYPLANE"/>
+	<value value="4" name="PERF_TSE_STALL_CYCLES_LRZ_ZPLANE"/>
+	<value value="5" name="PERF_TSE_STARVE_CYCLES_PC"/>
+	<value value="6" name="PERF_TSE_INPUT_PRIM"/>
+	<value value="7" name="PERF_TSE_INPUT_NULL_PRIM"/>
+	<value value="8" name="PERF_TSE_TRIVAL_REJ_PRIM"/>
+	<value value="9" name="PERF_TSE_CLIPPED_PRIM"/>
+	<value value="10" name="PERF_TSE_ZERO_AREA_PRIM"/>
+	<value value="11" name="PERF_TSE_FACENESS_CULLED_PRIM"/>
+	<value value="12" name="PERF_TSE_ZERO_PIXEL_PRIM"/>
+	<value value="13" name="PERF_TSE_OUTPUT_NULL_PRIM"/>
+	<value value="14" name="PERF_TSE_OUTPUT_VISIBLE_PRIM"/>
+	<value value="15" name="PERF_TSE_CINVOCATION"/>
+	<value value="16" name="PERF_TSE_CPRIMITIVES"/>
+	<value value="17" name="PERF_TSE_2D_INPUT_PRIM"/>
+	<value value="18" name="PERF_TSE_2D_ALIVE_CLCLES"/>
+</enum>
+
+<enum name="a5xx_ras_perfcounter_select">
+	<value value="0" name="PERF_RAS_BUSY_CYCLES"/>
+	<value value="1" name="PERF_RAS_SUPERTILE_ACTIVE_CYCLES"/>
+	<value value="2" name="PERF_RAS_STALL_CYCLES_LRZ"/>
+	<value value="3" name="PERF_RAS_STARVE_CYCLES_TSE"/>
+	<value value="4" name="PERF_RAS_SUPER_TILES"/>
+	<value value="5" name="PERF_RAS_8X4_TILES"/>
+	<value value="6" name="PERF_RAS_MASKGEN_ACTIVE"/>
+	<value value="7" name="PERF_RAS_FULLY_COVERED_SUPER_TILES"/>
+	<value value="8" name="PERF_RAS_FULLY_COVERED_8X4_TILES"/>
+	<value value="9" name="PERF_RAS_PRIM_KILLED_INVISILBE"/>
+</enum>
+
+<enum name="a5xx_lrz_perfcounter_select">
+	<value value="0" name="PERF_LRZ_BUSY_CYCLES"/>
+	<value value="1" name="PERF_LRZ_STARVE_CYCLES_RAS"/>
+	<value value="2" name="PERF_LRZ_STALL_CYCLES_RB"/>
+	<value value="3" name="PERF_LRZ_STALL_CYCLES_VSC"/>
+	<value value="4" name="PERF_LRZ_STALL_CYCLES_VPC"/>
+	<value value="5" name="PERF_LRZ_STALL_CYCLES_FLAG_PREFETCH"/>
+	<value value="6" name="PERF_LRZ_STALL_CYCLES_UCHE"/>
+	<value value="7" name="PERF_LRZ_LRZ_READ"/>
+	<value value="8" name="PERF_LRZ_LRZ_WRITE"/>
+	<value value="9" name="PERF_LRZ_READ_LATENCY"/>
+	<value value="10" name="PERF_LRZ_MERGE_CACHE_UPDATING"/>
+	<value value="11" name="PERF_LRZ_PRIM_KILLED_BY_MASKGEN"/>
+	<value value="12" name="PERF_LRZ_PRIM_KILLED_BY_LRZ"/>
+	<value value="13" name="PERF_LRZ_VISIBLE_PRIM_AFTER_LRZ"/>
+	<value value="14" name="PERF_LRZ_FULL_8X8_TILES"/>
+	<value value="15" name="PERF_LRZ_PARTIAL_8X8_TILES"/>
+	<value value="16" name="PERF_LRZ_TILE_KILLED"/>
+	<value value="17" name="PERF_LRZ_TOTAL_PIXEL"/>
+	<value value="18" name="PERF_LRZ_VISIBLE_PIXEL_AFTER_LRZ"/>
+</enum>
+
+<enum name="a5xx_uche_perfcounter_select">
+	<value value="0" name="PERF_UCHE_BUSY_CYCLES"/>
+	<value value="1" name="PERF_UCHE_STALL_CYCLES_VBIF"/>
+	<value value="2" name="PERF_UCHE_VBIF_LATENCY_CYCLES"/>
+	<value value="3" name="PERF_UCHE_VBIF_LATENCY_SAMPLES"/>
+	<value value="4" name="PERF_UCHE_VBIF_READ_BEATS_TP"/>
+	<value value="5" name="PERF_UCHE_VBIF_READ_BEATS_VFD"/>
+	<value value="6" name="PERF_UCHE_VBIF_READ_BEATS_HLSQ"/>
+	<value value="7" name="PERF_UCHE_VBIF_READ_BEATS_LRZ"/>
+	<value value="8" name="PERF_UCHE_VBIF_READ_BEATS_SP"/>
+	<value value="9" name="PERF_UCHE_READ_REQUESTS_TP"/>
+	<value value="10" name="PERF_UCHE_READ_REQUESTS_VFD"/>
+	<value value="11" name="PERF_UCHE_READ_REQUESTS_HLSQ"/>
+	<value value="12" name="PERF_UCHE_READ_REQUESTS_LRZ"/>
+	<value value="13" name="PERF_UCHE_READ_REQUESTS_SP"/>
+	<value value="14" name="PERF_UCHE_WRITE_REQUESTS_LRZ"/>
+	<value value="15" name="PERF_UCHE_WRITE_REQUESTS_SP"/>
+	<value value="16" name="PERF_UCHE_WRITE_REQUESTS_VPC"/>
+	<value value="17" name="PERF_UCHE_WRITE_REQUESTS_VSC"/>
+	<value value="18" name="PERF_UCHE_EVICTS"/>
+	<value value="19" name="PERF_UCHE_BANK_REQ0"/>
+	<value value="20" name="PERF_UCHE_BANK_REQ1"/>
+	<value value="21" name="PERF_UCHE_BANK_REQ2"/>
+	<value value="22" name="PERF_UCHE_BANK_REQ3"/>
+	<value value="23" name="PERF_UCHE_BANK_REQ4"/>
+	<value value="24" name="PERF_UCHE_BANK_REQ5"/>
+	<value value="25" name="PERF_UCHE_BANK_REQ6"/>
+	<value value="26" name="PERF_UCHE_BANK_REQ7"/>
+	<value value="27" name="PERF_UCHE_VBIF_READ_BEATS_CH0"/>
+	<value value="28" name="PERF_UCHE_VBIF_READ_BEATS_CH1"/>
+	<value value="29" name="PERF_UCHE_GMEM_READ_BEATS"/>
+	<value value="30" name="PERF_UCHE_FLAG_COUNT"/>
+</enum>
+
+<enum name="a5xx_tp_perfcounter_select">
+	<value value="0" name="PERF_TP_BUSY_CYCLES"/>
+	<value value="1" name="PERF_TP_STALL_CYCLES_UCHE"/>
+	<value value="2" name="PERF_TP_LATENCY_CYCLES"/>
+	<value value="3" name="PERF_TP_LATENCY_TRANS"/>
+	<value value="4" name="PERF_TP_FLAG_CACHE_REQUEST_SAMPLES"/>
+	<value value="5" name="PERF_TP_FLAG_CACHE_REQUEST_LATENCY"/>
+	<value value="6" name="PERF_TP_L1_CACHELINE_REQUESTS"/>
+	<value value="7" name="PERF_TP_L1_CACHELINE_MISSES"/>
+	<value value="8" name="PERF_TP_SP_TP_TRANS"/>
+	<value value="9" name="PERF_TP_TP_SP_TRANS"/>
+	<value value="10" name="PERF_TP_OUTPUT_PIXELS"/>
+	<value value="11" name="PERF_TP_FILTER_WORKLOAD_16BIT"/>
+	<value value="12" name="PERF_TP_FILTER_WORKLOAD_32BIT"/>
+	<value value="13" name="PERF_TP_QUADS_RECEIVED"/>
+	<value value="14" name="PERF_TP_QUADS_OFFSET"/>
+	<value value="15" name="PERF_TP_QUADS_SHADOW"/>
+	<value value="16" name="PERF_TP_QUADS_ARRAY"/>
+	<value value="17" name="PERF_TP_QUADS_GRADIENT"/>
+	<value value="18" name="PERF_TP_QUADS_1D"/>
+	<value value="19" name="PERF_TP_QUADS_2D"/>
+	<value value="20" name="PERF_TP_QUADS_BUFFER"/>
+	<value value="21" name="PERF_TP_QUADS_3D"/>
+	<value value="22" name="PERF_TP_QUADS_CUBE"/>
+	<value value="23" name="PERF_TP_STATE_CACHE_REQUESTS"/>
+	<value value="24" name="PERF_TP_STATE_CACHE_MISSES"/>
+	<value value="25" name="PERF_TP_DIVERGENT_QUADS_RECEIVED"/>
+	<value value="26" name="PERF_TP_BINDLESS_STATE_CACHE_REQUESTS"/>
+	<value value="27" name="PERF_TP_BINDLESS_STATE_CACHE_MISSES"/>
+	<value value="28" name="PERF_TP_PRT_NON_RESIDENT_EVENTS"/>
+	<value value="29" name="PERF_TP_OUTPUT_PIXELS_POINT"/>
+	<value value="30" name="PERF_TP_OUTPUT_PIXELS_BILINEAR"/>
+	<value value="31" name="PERF_TP_OUTPUT_PIXELS_MIP"/>
+	<value value="32" name="PERF_TP_OUTPUT_PIXELS_ANISO"/>
+	<value value="33" name="PERF_TP_OUTPUT_PIXELS_ZERO_LOD"/>
+	<value value="34" name="PERF_TP_FLAG_CACHE_REQUESTS"/>
+	<value value="35" name="PERF_TP_FLAG_CACHE_MISSES"/>
+	<value value="36" name="PERF_TP_L1_5_L2_REQUESTS"/>
+	<value value="37" name="PERF_TP_2D_OUTPUT_PIXELS"/>
+	<value value="38" name="PERF_TP_2D_OUTPUT_PIXELS_POINT"/>
+	<value value="39" name="PERF_TP_2D_OUTPUT_PIXELS_BILINEAR"/>
+	<value value="40" name="PERF_TP_2D_FILTER_WORKLOAD_16BIT"/>
+	<value value="41" name="PERF_TP_2D_FILTER_WORKLOAD_32BIT"/>
+</enum>
+
+<enum name="a5xx_sp_perfcounter_select">
+	<value value="0" name="PERF_SP_BUSY_CYCLES"/>
+	<value value="1" name="PERF_SP_ALU_WORKING_CYCLES"/>
+	<value value="2" name="PERF_SP_EFU_WORKING_CYCLES"/>
+	<value value="3" name="PERF_SP_STALL_CYCLES_VPC"/>
+	<value value="4" name="PERF_SP_STALL_CYCLES_TP"/>
+	<value value="5" name="PERF_SP_STALL_CYCLES_UCHE"/>
+	<value value="6" name="PERF_SP_STALL_CYCLES_RB"/>
+	<value value="7" name="PERF_SP_SCHEDULER_NON_WORKING"/>
+	<value value="8" name="PERF_SP_WAVE_CONTEXTS"/>
+	<value value="9" name="PERF_SP_WAVE_CONTEXT_CYCLES"/>
+	<value value="10" name="PERF_SP_FS_STAGE_WAVE_CYCLES"/>
+	<value value="11" name="PERF_SP_FS_STAGE_WAVE_SAMPLES"/>
+	<value value="12" name="PERF_SP_VS_STAGE_WAVE_CYCLES"/>
+	<value value="13" name="PERF_SP_VS_STAGE_WAVE_SAMPLES"/>
+	<value value="14" name="PERF_SP_FS_STAGE_DURATION_CYCLES"/>
+	<value value="15" name="PERF_SP_VS_STAGE_DURATION_CYCLES"/>
+	<value value="16" name="PERF_SP_WAVE_CTRL_CYCLES"/>
+	<value value="17" name="PERF_SP_WAVE_LOAD_CYCLES"/>
+	<value value="18" name="PERF_SP_WAVE_EMIT_CYCLES"/>
+	<value value="19" name="PERF_SP_WAVE_NOP_CYCLES"/>
+	<value value="20" name="PERF_SP_WAVE_WAIT_CYCLES"/>
+	<value value="21" name="PERF_SP_WAVE_FETCH_CYCLES"/>
+	<value value="22" name="PERF_SP_WAVE_IDLE_CYCLES"/>
+	<value value="23" name="PERF_SP_WAVE_END_CYCLES"/>
+	<value value="24" name="PERF_SP_WAVE_LONG_SYNC_CYCLES"/>
+	<value value="25" name="PERF_SP_WAVE_SHORT_SYNC_CYCLES"/>
+	<value value="26" name="PERF_SP_WAVE_JOIN_CYCLES"/>
+	<value value="27" name="PERF_SP_LM_LOAD_INSTRUCTIONS"/>
+	<value value="28" name="PERF_SP_LM_STORE_INSTRUCTIONS"/>
+	<value value="29" name="PERF_SP_LM_ATOMICS"/>
+	<value value="30" name="PERF_SP_GM_LOAD_INSTRUCTIONS"/>
+	<value value="31" name="PERF_SP_GM_STORE_INSTRUCTIONS"/>
+	<value value="32" name="PERF_SP_GM_ATOMICS"/>
+	<value value="33" name="PERF_SP_VS_STAGE_TEX_INSTRUCTIONS"/>
+	<value value="34" name="PERF_SP_VS_STAGE_CFLOW_INSTRUCTIONS"/>
+	<value value="35" name="PERF_SP_VS_STAGE_EFU_INSTRUCTIONS"/>
+	<value value="36" name="PERF_SP_VS_STAGE_FULL_ALU_INSTRUCTIONS"/>
+	<value value="37" name="PERF_SP_VS_STAGE_HALF_ALU_INSTRUCTIONS"/>
+	<value value="38" name="PERF_SP_FS_STAGE_TEX_INSTRUCTIONS"/>
+	<value value="39" name="PERF_SP_FS_STAGE_CFLOW_INSTRUCTIONS"/>
+	<value value="40" name="PERF_SP_FS_STAGE_EFU_INSTRUCTIONS"/>
+	<value value="41" name="PERF_SP_FS_STAGE_FULL_ALU_INSTRUCTIONS"/>
+	<value value="42" name="PERF_SP_FS_STAGE_HALF_ALU_INSTRUCTIONS"/>
+	<value value="43" name="PERF_SP_FS_STAGE_BARY_INSTRUCTIONS"/>
+	<value value="44" name="PERF_SP_VS_INSTRUCTIONS"/>
+	<value value="45" name="PERF_SP_FS_INSTRUCTIONS"/>
+	<value value="46" name="PERF_SP_ADDR_LOCK_COUNT"/>
+	<value value="47" name="PERF_SP_UCHE_READ_TRANS"/>
+	<value value="48" name="PERF_SP_UCHE_WRITE_TRANS"/>
+	<value value="49" name="PERF_SP_EXPORT_VPC_TRANS"/>
+	<value value="50" name="PERF_SP_EXPORT_RB_TRANS"/>
+	<value value="51" name="PERF_SP_PIXELS_KILLED"/>
+	<value value="52" name="PERF_SP_ICL1_REQUESTS"/>
+	<value value="53" name="PERF_SP_ICL1_MISSES"/>
+	<value value="54" name="PERF_SP_ICL0_REQUESTS"/>
+	<value value="55" name="PERF_SP_ICL0_MISSES"/>
+	<value value="56" name="PERF_SP_HS_INSTRUCTIONS"/>
+	<value value="57" name="PERF_SP_DS_INSTRUCTIONS"/>
+	<value value="58" name="PERF_SP_GS_INSTRUCTIONS"/>
+	<value value="59" name="PERF_SP_CS_INSTRUCTIONS"/>
+	<value value="60" name="PERF_SP_GPR_READ"/>
+	<value value="61" name="PERF_SP_GPR_WRITE"/>
+	<value value="62" name="PERF_SP_LM_CH0_REQUESTS"/>
+	<value value="63" name="PERF_SP_LM_CH1_REQUESTS"/>
+	<value value="64" name="PERF_SP_LM_BANK_CONFLICTS"/>
+</enum>
+
+<enum name="a5xx_rb_perfcounter_select">
+	<value value="0" name="PERF_RB_BUSY_CYCLES"/>
+	<value value="1" name="PERF_RB_STALL_CYCLES_CCU"/>
+	<value value="2" name="PERF_RB_STALL_CYCLES_HLSQ"/>
+	<value value="3" name="PERF_RB_STALL_CYCLES_FIFO0_FULL"/>
+	<value value="4" name="PERF_RB_STALL_CYCLES_FIFO1_FULL"/>
+	<value value="5" name="PERF_RB_STALL_CYCLES_FIFO2_FULL"/>
+	<value value="6" name="PERF_RB_STARVE_CYCLES_SP"/>
+	<value value="7" name="PERF_RB_STARVE_CYCLES_LRZ_TILE"/>
+	<value value="8" name="PERF_RB_STARVE_CYCLES_CCU"/>
+	<value value="9" name="PERF_RB_STARVE_CYCLES_Z_PLANE"/>
+	<value value="10" name="PERF_RB_STARVE_CYCLES_BARY_PLANE"/>
+	<value value="11" name="PERF_RB_Z_WORKLOAD"/>
+	<value value="12" name="PERF_RB_HLSQ_ACTIVE"/>
+	<value value="13" name="PERF_RB_Z_READ"/>
+	<value value="14" name="PERF_RB_Z_WRITE"/>
+	<value value="15" name="PERF_RB_C_READ"/>
+	<value value="16" name="PERF_RB_C_WRITE"/>
+	<value value="17" name="PERF_RB_TOTAL_PASS"/>
+	<value value="18" name="PERF_RB_Z_PASS"/>
+	<value value="19" name="PERF_RB_Z_FAIL"/>
+	<value value="20" name="PERF_RB_S_FAIL"/>
+	<value value="21" name="PERF_RB_BLENDED_FXP_COMPONENTS"/>
+	<value value="22" name="PERF_RB_BLENDED_FP16_COMPONENTS"/>
+	<value value="23" name="RB_RESERVED"/>
+	<value value="24" name="PERF_RB_2D_ALIVE_CYCLES"/>
+	<value value="25" name="PERF_RB_2D_STALL_CYCLES_A2D"/>
+	<value value="26" name="PERF_RB_2D_STARVE_CYCLES_SRC"/>
+	<value value="27" name="PERF_RB_2D_STARVE_CYCLES_SP"/>
+	<value value="28" name="PERF_RB_2D_STARVE_CYCLES_DST"/>
+	<value value="29" name="PERF_RB_2D_VALID_PIXELS"/>
+</enum>
+
+<enum name="a5xx_rb_samples_perfcounter_select">
+	<value value="0" name="TOTAL_SAMPLES"/>
+	<value value="1" name="ZPASS_SAMPLES"/>
+	<value value="2" name="ZFAIL_SAMPLES"/>
+	<value value="3" name="SFAIL_SAMPLES"/>
+</enum>
+
+<enum name="a5xx_vsc_perfcounter_select">
+	<value value="0" name="PERF_VSC_BUSY_CYCLES"/>
+	<value value="1" name="PERF_VSC_WORKING_CYCLES"/>
+	<value value="2" name="PERF_VSC_STALL_CYCLES_UCHE"/>
+	<value value="3" name="PERF_VSC_EOT_NUM"/>
+</enum>
+
+<enum name="a5xx_ccu_perfcounter_select">
+	<value value="0" name="PERF_CCU_BUSY_CYCLES"/>
+	<value value="1" name="PERF_CCU_STALL_CYCLES_RB_DEPTH_RETURN"/>
+	<value value="2" name="PERF_CCU_STALL_CYCLES_RB_COLOR_RETURN"/>
+	<value value="3" name="PERF_CCU_STARVE_CYCLES_FLAG_RETURN"/>
+	<value value="4" name="PERF_CCU_DEPTH_BLOCKS"/>
+	<value value="5" name="PERF_CCU_COLOR_BLOCKS"/>
+	<value value="6" name="PERF_CCU_DEPTH_BLOCK_HIT"/>
+	<value value="7" name="PERF_CCU_COLOR_BLOCK_HIT"/>
+	<value value="8" name="PERF_CCU_PARTIAL_BLOCK_READ"/>
+	<value value="9" name="PERF_CCU_GMEM_READ"/>
+	<value value="10" name="PERF_CCU_GMEM_WRITE"/>
+	<value value="11" name="PERF_CCU_DEPTH_READ_FLAG0_COUNT"/>
+	<value value="12" name="PERF_CCU_DEPTH_READ_FLAG1_COUNT"/>
+	<value value="13" name="PERF_CCU_DEPTH_READ_FLAG2_COUNT"/>
+	<value value="14" name="PERF_CCU_DEPTH_READ_FLAG3_COUNT"/>
+	<value value="15" name="PERF_CCU_DEPTH_READ_FLAG4_COUNT"/>
+	<value value="16" name="PERF_CCU_COLOR_READ_FLAG0_COUNT"/>
+	<value value="17" name="PERF_CCU_COLOR_READ_FLAG1_COUNT"/>
+	<value value="18" name="PERF_CCU_COLOR_READ_FLAG2_COUNT"/>
+	<value value="19" name="PERF_CCU_COLOR_READ_FLAG3_COUNT"/>
+	<value value="20" name="PERF_CCU_COLOR_READ_FLAG4_COUNT"/>
+	<value value="21" name="PERF_CCU_2D_BUSY_CYCLES"/>
+	<value value="22" name="PERF_CCU_2D_RD_REQ"/>
+	<value value="23" name="PERF_CCU_2D_WR_REQ"/>
+	<value value="24" name="PERF_CCU_2D_REORDER_STARVE_CYCLES"/>
+	<value value="25" name="PERF_CCU_2D_PIXELS"/>
+</enum>
+
+<enum name="a5xx_cmp_perfcounter_select">
+	<value value="0" name="PERF_CMPDECMP_STALL_CYCLES_VBIF"/>
+	<value value="1" name="PERF_CMPDECMP_VBIF_LATENCY_CYCLES"/>
+	<value value="2" name="PERF_CMPDECMP_VBIF_LATENCY_SAMPLES"/>
+	<value value="3" name="PERF_CMPDECMP_VBIF_READ_DATA_CCU"/>
+	<value value="4" name="PERF_CMPDECMP_VBIF_WRITE_DATA_CCU"/>
+	<value value="5" name="PERF_CMPDECMP_VBIF_READ_REQUEST"/>
+	<value value="6" name="PERF_CMPDECMP_VBIF_WRITE_REQUEST"/>
+	<value value="7" name="PERF_CMPDECMP_VBIF_READ_DATA"/>
+	<value value="8" name="PERF_CMPDECMP_VBIF_WRITE_DATA"/>
+	<value value="9" name="PERF_CMPDECMP_FLAG_FETCH_CYCLES"/>
+	<value value="10" name="PERF_CMPDECMP_FLAG_FETCH_SAMPLES"/>
+	<value value="11" name="PERF_CMPDECMP_DEPTH_WRITE_FLAG1_COUNT"/>
+	<value value="12" name="PERF_CMPDECMP_DEPTH_WRITE_FLAG2_COUNT"/>
+	<value value="13" name="PERF_CMPDECMP_DEPTH_WRITE_FLAG3_COUNT"/>
+	<value value="14" name="PERF_CMPDECMP_DEPTH_WRITE_FLAG4_COUNT"/>
+	<value value="15" name="PERF_CMPDECMP_COLOR_WRITE_FLAG1_COUNT"/>
+	<value value="16" name="PERF_CMPDECMP_COLOR_WRITE_FLAG2_COUNT"/>
+	<value value="17" name="PERF_CMPDECMP_COLOR_WRITE_FLAG3_COUNT"/>
+	<value value="18" name="PERF_CMPDECMP_COLOR_WRITE_FLAG4_COUNT"/>
+	<value value="19" name="PERF_CMPDECMP_2D_STALL_CYCLES_VBIF_REQ"/>
+	<value value="20" name="PERF_CMPDECMP_2D_STALL_CYCLES_VBIF_WR"/>
+	<value value="21" name="PERF_CMPDECMP_2D_STALL_CYCLES_VBIF_RETURN"/>
+	<value value="22" name="PERF_CMPDECMP_2D_RD_DATA"/>
+	<value value="23" name="PERF_CMPDECMP_2D_WR_DATA"/>
+</enum>
+
+<enum name="a5xx_vbif_perfcounter_select">
+	<value value="0" name="AXI_READ_REQUESTS_ID_0"/>
+	<value value="1" name="AXI_READ_REQUESTS_ID_1"/>
+	<value value="2" name="AXI_READ_REQUESTS_ID_2"/>
+	<value value="3" name="AXI_READ_REQUESTS_ID_3"/>
+	<value value="4" name="AXI_READ_REQUESTS_ID_4"/>
+	<value value="5" name="AXI_READ_REQUESTS_ID_5"/>
+	<value value="6" name="AXI_READ_REQUESTS_ID_6"/>
+	<value value="7" name="AXI_READ_REQUESTS_ID_7"/>
+	<value value="8" name="AXI_READ_REQUESTS_ID_8"/>
+	<value value="9" name="AXI_READ_REQUESTS_ID_9"/>
+	<value value="10" name="AXI_READ_REQUESTS_ID_10"/>
+	<value value="11" name="AXI_READ_REQUESTS_ID_11"/>
+	<value value="12" name="AXI_READ_REQUESTS_ID_12"/>
+	<value value="13" name="AXI_READ_REQUESTS_ID_13"/>
+	<value value="14" name="AXI_READ_REQUESTS_ID_14"/>
+	<value value="15" name="AXI_READ_REQUESTS_ID_15"/>
+	<value value="16" name="AXI0_READ_REQUESTS_TOTAL"/>
+	<value value="17" name="AXI1_READ_REQUESTS_TOTAL"/>
+	<value value="18" name="AXI2_READ_REQUESTS_TOTAL"/>
+	<value value="19" name="AXI3_READ_REQUESTS_TOTAL"/>
+	<value value="20" name="AXI_READ_REQUESTS_TOTAL"/>
+	<value value="21" name="AXI_WRITE_REQUESTS_ID_0"/>
+	<value value="22" name="AXI_WRITE_REQUESTS_ID_1"/>
+	<value value="23" name="AXI_WRITE_REQUESTS_ID_2"/>
+	<value value="24" name="AXI_WRITE_REQUESTS_ID_3"/>
+	<value value="25" name="AXI_WRITE_REQUESTS_ID_4"/>
+	<value value="26" name="AXI_WRITE_REQUESTS_ID_5"/>
+	<value value="27" name="AXI_WRITE_REQUESTS_ID_6"/>
+	<value value="28" name="AXI_WRITE_REQUESTS_ID_7"/>
+	<value value="29" name="AXI_WRITE_REQUESTS_ID_8"/>
+	<value value="30" name="AXI_WRITE_REQUESTS_ID_9"/>
+	<value value="31" name="AXI_WRITE_REQUESTS_ID_10"/>
+	<value value="32" name="AXI_WRITE_REQUESTS_ID_11"/>
+	<value value="33" name="AXI_WRITE_REQUESTS_ID_12"/>
+	<value value="34" name="AXI_WRITE_REQUESTS_ID_13"/>
+	<value value="35" name="AXI_WRITE_REQUESTS_ID_14"/>
+	<value value="36" name="AXI_WRITE_REQUESTS_ID_15"/>
+	<value value="37" name="AXI0_WRITE_REQUESTS_TOTAL"/>
+	<value value="38" name="AXI1_WRITE_REQUESTS_TOTAL"/>
+	<value value="39" name="AXI2_WRITE_REQUESTS_TOTAL"/>
+	<value value="40" name="AXI3_WRITE_REQUESTS_TOTAL"/>
+	<value value="41" name="AXI_WRITE_REQUESTS_TOTAL"/>
+	<value value="42" name="AXI_TOTAL_REQUESTS"/>
+	<value value="43" name="AXI_READ_DATA_BEATS_ID_0"/>
+	<value value="44" name="AXI_READ_DATA_BEATS_ID_1"/>
+	<value value="45" name="AXI_READ_DATA_BEATS_ID_2"/>
+	<value value="46" name="AXI_READ_DATA_BEATS_ID_3"/>
+	<value value="47" name="AXI_READ_DATA_BEATS_ID_4"/>
+	<value value="48" name="AXI_READ_DATA_BEATS_ID_5"/>
+	<value value="49" name="AXI_READ_DATA_BEATS_ID_6"/>
+	<value value="50" name="AXI_READ_DATA_BEATS_ID_7"/>
+	<value value="51" name="AXI_READ_DATA_BEATS_ID_8"/>
+	<value value="52" name="AXI_READ_DATA_BEATS_ID_9"/>
+	<value value="53" name="AXI_READ_DATA_BEATS_ID_10"/>
+	<value value="54" name="AXI_READ_DATA_BEATS_ID_11"/>
+	<value value="55" name="AXI_READ_DATA_BEATS_ID_12"/>
+	<value value="56" name="AXI_READ_DATA_BEATS_ID_13"/>
+	<value value="57" name="AXI_READ_DATA_BEATS_ID_14"/>
+	<value value="58" name="AXI_READ_DATA_BEATS_ID_15"/>
+	<value value="59" name="AXI0_READ_DATA_BEATS_TOTAL"/>
+	<value value="60" name="AXI1_READ_DATA_BEATS_TOTAL"/>
+	<value value="61" name="AXI2_READ_DATA_BEATS_TOTAL"/>
+	<value value="62" name="AXI3_READ_DATA_BEATS_TOTAL"/>
+	<value value="63" name="AXI_READ_DATA_BEATS_TOTAL"/>
+	<value value="64" name="AXI_WRITE_DATA_BEATS_ID_0"/>
+	<value value="65" name="AXI_WRITE_DATA_BEATS_ID_1"/>
+	<value value="66" name="AXI_WRITE_DATA_BEATS_ID_2"/>
+	<value value="67" name="AXI_WRITE_DATA_BEATS_ID_3"/>
+	<value value="68" name="AXI_WRITE_DATA_BEATS_ID_4"/>
+	<value value="69" name="AXI_WRITE_DATA_BEATS_ID_5"/>
+	<value value="70" name="AXI_WRITE_DATA_BEATS_ID_6"/>
+	<value value="71" name="AXI_WRITE_DATA_BEATS_ID_7"/>
+	<value value="72" name="AXI_WRITE_DATA_BEATS_ID_8"/>
+	<value value="73" name="AXI_WRITE_DATA_BEATS_ID_9"/>
+	<value value="74" name="AXI_WRITE_DATA_BEATS_ID_10"/>
+	<value value="75" name="AXI_WRITE_DATA_BEATS_ID_11"/>
+	<value value="76" name="AXI_WRITE_DATA_BEATS_ID_12"/>
+	<value value="77" name="AXI_WRITE_DATA_BEATS_ID_13"/>
+	<value value="78" name="AXI_WRITE_DATA_BEATS_ID_14"/>
+	<value value="79" name="AXI_WRITE_DATA_BEATS_ID_15"/>
+	<value value="80" name="AXI0_WRITE_DATA_BEATS_TOTAL"/>
+	<value value="81" name="AXI1_WRITE_DATA_BEATS_TOTAL"/>
+	<value value="82" name="AXI2_WRITE_DATA_BEATS_TOTAL"/>
+	<value value="83" name="AXI3_WRITE_DATA_BEATS_TOTAL"/>
+	<value value="84" name="AXI_WRITE_DATA_BEATS_TOTAL"/>
+	<value value="85" name="AXI_DATA_BEATS_TOTAL"/>
+</enum>
+
+<domain name="A5XX" width="32">
+	<bitset name="A5XX_INT0">
+		<bitfield name="RBBM_GPU_IDLE" pos="0" type="boolean"/>
+		<bitfield name="RBBM_AHB_ERROR" pos="1" type="boolean"/>
+		<bitfield name="RBBM_TRANSFER_TIMEOUT" pos="2" type="boolean"/>
+		<bitfield name="RBBM_ME_MS_TIMEOUT" pos="3" type="boolean"/>
+		<bitfield name="RBBM_PFP_MS_TIMEOUT" pos="4" type="boolean"/>
+		<bitfield name="RBBM_ETS_MS_TIMEOUT" pos="5" type="boolean"/>
+		<bitfield name="RBBM_ATB_ASYNC_OVERFLOW" pos="6" type="boolean"/>
+		<bitfield name="RBBM_GPC_ERROR" pos="7" type="boolean"/>
+		<bitfield name="CP_SW" pos="8" type="boolean"/>
+		<bitfield name="CP_HW_ERROR" pos="9" type="boolean"/>
+		<bitfield name="CP_CCU_FLUSH_DEPTH_TS" pos="10" type="boolean"/>
+		<bitfield name="CP_CCU_FLUSH_COLOR_TS" pos="11" type="boolean"/>
+		<bitfield name="CP_CCU_RESOLVE_TS" pos="12" type="boolean"/>
+		<bitfield name="CP_IB2" pos="13" type="boolean"/>
+		<bitfield name="CP_IB1" pos="14" type="boolean"/>
+		<bitfield name="CP_RB" pos="15" type="boolean"/>
+		<bitfield name="CP_UNUSED_1" pos="16" type="boolean"/>
+		<bitfield name="CP_RB_DONE_TS" pos="17" type="boolean"/>
+		<bitfield name="CP_WT_DONE_TS" pos="18" type="boolean"/>
+		<bitfield name="UNKNOWN_1" pos="19" type="boolean"/>
+		<bitfield name="CP_CACHE_FLUSH_TS" pos="20" type="boolean"/>
+		<bitfield name="UNUSED_2" pos="21" type="boolean"/>
+		<bitfield name="RBBM_ATB_BUS_OVERFLOW" pos="22" type="boolean"/>
+		<bitfield name="MISC_HANG_DETECT" pos="23" type="boolean"/>
+		<bitfield name="UCHE_OOB_ACCESS" pos="24" type="boolean"/>
+		<bitfield name="UCHE_TRAP_INTR" pos="25" type="boolean"/>
+		<bitfield name="DEBBUS_INTR_0" pos="26" type="boolean"/>
+		<bitfield name="DEBBUS_INTR_1" pos="27" type="boolean"/>
+		<bitfield name="GPMU_VOLTAGE_DROOP" pos="28" type="boolean"/>
+		<bitfield name="GPMU_FIRMWARE" pos="29" type="boolean"/>
+		<bitfield name="ISDB_CPU_IRQ" pos="30" type="boolean"/>
+		<bitfield name="ISDB_UNDER_DEBUG" pos="31" type="boolean"/>
+	</bitset>
+
+	<!-- CP Interrupt bits -->
+	<bitset name="A5XX_CP_INT">
+		<bitfield name="CP_OPCODE_ERROR" pos="0" type="boolean"/>
+		<bitfield name="CP_RESERVED_BIT_ERROR" pos="1" type="boolean"/>
+		<bitfield name="CP_HW_FAULT_ERROR" pos="2" type="boolean"/>
+		<bitfield name="CP_DMA_ERROR" pos="3" type="boolean"/>
+		<bitfield name="CP_REGISTER_PROTECTION_ERROR" pos="4" type="boolean"/>
+		<bitfield name="CP_AHB_ERROR" pos="5" type="boolean"/>
+	</bitset>
+
+	<!-- CP registers -->
+	<reg32 offset="0x0800" name="CP_RB_BASE"/>
+	<reg32 offset="0x0801" name="CP_RB_BASE_HI"/>
+	<reg32 offset="0x0802" name="CP_RB_CNTL"/>
+	<reg32 offset="0x0804" name="CP_RB_RPTR_ADDR"/>
+	<reg32 offset="0x0805" name="CP_RB_RPTR_ADDR_HI"/>
+	<reg32 offset="0x0806" name="CP_RB_RPTR"/>
+	<reg32 offset="0x0807" name="CP_RB_WPTR"/>
+	<reg32 offset="0x0808" name="CP_PFP_STAT_ADDR"/>
+	<reg32 offset="0x0809" name="CP_PFP_STAT_DATA"/>
+	<reg32 offset="0x080b" name="CP_DRAW_STATE_ADDR"/>
+	<reg32 offset="0x080c" name="CP_DRAW_STATE_DATA"/>
+	<reg32 offset="0x080d" name="CP_ME_NRT_ADDR_LO"/>
+	<reg32 offset="0x080e" name="CP_ME_NRT_ADDR_HI"/>
+	<reg32 offset="0x0810" name="CP_ME_NRT_DATA"/>
+	<reg32 offset="0x0817" name="CP_CRASH_SCRIPT_BASE_LO"/>
+	<reg32 offset="0x0818" name="CP_CRASH_SCRIPT_BASE_HI"/>
+	<reg32 offset="0x0819" name="CP_CRASH_DUMP_CNTL"/>
+	<reg32 offset="0x081a" name="CP_ME_STAT_ADDR"/>
+	<reg32 offset="0x081f" name="CP_ROQ_THRESHOLDS_1"/>
+	<reg32 offset="0x0820" name="CP_ROQ_THRESHOLDS_2"/>
+	<reg32 offset="0x0821" name="CP_ROQ_DBG_ADDR"/>
+	<reg32 offset="0x0822" name="CP_ROQ_DBG_DATA"/>
+	<reg32 offset="0x0823" name="CP_MEQ_DBG_ADDR"/>
+	<reg32 offset="0x0824" name="CP_MEQ_DBG_DATA"/>
+	<reg32 offset="0x0825" name="CP_MEQ_THRESHOLDS"/>
+	<reg32 offset="0x0826" name="CP_MERCIU_SIZE"/>
+	<reg32 offset="0x0827" name="CP_MERCIU_DBG_ADDR"/>
+	<reg32 offset="0x0828" name="CP_MERCIU_DBG_DATA_1"/>
+	<reg32 offset="0x0829" name="CP_MERCIU_DBG_DATA_2"/>
+	<reg32 offset="0x082a" name="CP_PFP_UCODE_DBG_ADDR"/>
+	<reg32 offset="0x082b" name="CP_PFP_UCODE_DBG_DATA"/>
+	<reg32 offset="0x082f" name="CP_ME_UCODE_DBG_ADDR"/>
+	<reg32 offset="0x0830" name="CP_ME_UCODE_DBG_DATA"/>
+	<reg32 offset="0x0831" name="CP_CNTL"/>
+	<reg32 offset="0x0832" name="CP_PFP_ME_CNTL"/>
+	<reg32 offset="0x0833" name="CP_CHICKEN_DBG"/>
+	<reg32 offset="0x0835" name="CP_PFP_INSTR_BASE_LO"/>
+	<reg32 offset="0x0836" name="CP_PFP_INSTR_BASE_HI"/>
+	<reg32 offset="0x0838" name="CP_ME_INSTR_BASE_LO"/>
+	<reg32 offset="0x0839" name="CP_ME_INSTR_BASE_HI"/>
+	<reg32 offset="0x083b" name="CP_CONTEXT_SWITCH_CNTL"/>
+	<reg32 offset="0x083c" name="CP_CONTEXT_SWITCH_RESTORE_ADDR_LO"/>
+	<reg32 offset="0x083d" name="CP_CONTEXT_SWITCH_RESTORE_ADDR_HI"/>
+	<reg32 offset="0x083e" name="CP_CONTEXT_SWITCH_SAVE_ADDR_LO"/>
+	<reg32 offset="0x083f" name="CP_CONTEXT_SWITCH_SAVE_ADDR_HI"/>
+	<reg32 offset="0x0840" name="CP_CONTEXT_SWITCH_SMMU_INFO_LO"/>
+	<reg32 offset="0x0841" name="CP_CONTEXT_SWITCH_SMMU_INFO_HI"/>
+	<reg32 offset="0x0860" name="CP_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0b14" name="CP_ME_STAT_DATA"/>
+	<reg32 offset="0x0b15" name="CP_WFI_PEND_CTR"/>
+	<reg32 offset="0x0b18" name="CP_INTERRUPT_STATUS"/>
+	<reg32 offset="0x0b1a" name="CP_HW_FAULT"/>
+	<reg32 offset="0x0b1c" name="CP_PROTECT_STATUS"/>
+	<reg32 offset="0x0b1f" name="CP_IB1_BASE"/>
+	<reg32 offset="0x0b20" name="CP_IB1_BASE_HI"/>
+	<reg32 offset="0x0b21" name="CP_IB1_BUFSZ"/>
+	<reg32 offset="0x0b22" name="CP_IB2_BASE"/>
+	<reg32 offset="0x0b23" name="CP_IB2_BASE_HI"/>
+	<reg32 offset="0x0b24" name="CP_IB2_BUFSZ"/>
+	<array offset="0x0b78" name="CP_SCRATCH" stride="1" length="8">
+		<reg32 offset="0x0" name="REG" type="uint"/>
+	</array>
+	<array offset="0x0880" name="CP_PROTECT" stride="1" length="32">
+		<reg32 offset="0x0" name="REG" type="adreno_cp_protect"/>
+	</array>
+	<reg32 offset="0x08a0" name="CP_PROTECT_CNTL"/>
+	<reg32 offset="0x0b1b" name="CP_AHB_FAULT"/>
+	<reg32 offset="0x0bb0" name="CP_PERFCTR_CP_SEL_0" type="a5xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0bb1" name="CP_PERFCTR_CP_SEL_1" type="a5xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0bb2" name="CP_PERFCTR_CP_SEL_2" type="a5xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0bb3" name="CP_PERFCTR_CP_SEL_3" type="a5xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0bb4" name="CP_PERFCTR_CP_SEL_4" type="a5xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0bb5" name="CP_PERFCTR_CP_SEL_5" type="a5xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0bb6" name="CP_PERFCTR_CP_SEL_6" type="a5xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0bb7" name="CP_PERFCTR_CP_SEL_7" type="a5xx_cp_perfcounter_select"/>
+	<reg32 offset="0x0bc1" name="VSC_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0bba" name="CP_POWERCTR_CP_SEL_0"/>
+	<reg32 offset="0x0bbb" name="CP_POWERCTR_CP_SEL_1"/>
+	<reg32 offset="0x0bbc" name="CP_POWERCTR_CP_SEL_2"/>
+	<reg32 offset="0x0bbd" name="CP_POWERCTR_CP_SEL_3"/>
+
+	<!-- RBBM registers -->
+	<reg32 offset="0x0004" name="RBBM_CFG_DBGBUS_SEL_A"/>
+	<reg32 offset="0x0005" name="RBBM_CFG_DBGBUS_SEL_B"/>
+	<reg32 offset="0x0006" name="RBBM_CFG_DBGBUS_SEL_C"/>
+	<reg32 offset="0x0007" name="RBBM_CFG_DBGBUS_SEL_D"/>
+<!--
+#define A5XX_RBBM_CFG_DBGBUS_SEL_PING_INDEX_SHIFT    0x0
+#define A5XX_RBBM_CFG_DBGBUS_SEL_PING_BLK_SEL_SHIFT  0x8
+#define A5XX_RBBM_CFG_DBGBUS_SEL_PONG_INDEX_SHIFT    0x10
+#define A5XX_RBBM_CFG_DBGBUS_SEL_PONG_BLK_SEL_SHIFT  0x18
+ -->
+	<reg32 offset="0x0008" name="RBBM_CFG_DBGBUS_CNTLT"/>
+	<reg32 offset="0x0009" name="RBBM_CFG_DBGBUS_CNTLM"/>
+	<reg32 offset="0x0018" name="RBBM_CFG_DEBBUS_CTLTM_ENABLE_SHIFT"/>
+	<reg32 offset="0x000a" name="RBBM_CFG_DBGBUS_OPL"/>
+	<reg32 offset="0x000b" name="RBBM_CFG_DBGBUS_OPE"/>
+	<reg32 offset="0x000c" name="RBBM_CFG_DBGBUS_IVTL_0"/>
+	<reg32 offset="0x000d" name="RBBM_CFG_DBGBUS_IVTL_1"/>
+	<reg32 offset="0x000e" name="RBBM_CFG_DBGBUS_IVTL_2"/>
+	<reg32 offset="0x000f" name="RBBM_CFG_DBGBUS_IVTL_3"/>
+	<reg32 offset="0x0010" name="RBBM_CFG_DBGBUS_MASKL_0"/>
+	<reg32 offset="0x0011" name="RBBM_CFG_DBGBUS_MASKL_1"/>
+	<reg32 offset="0x0012" name="RBBM_CFG_DBGBUS_MASKL_2"/>
+	<reg32 offset="0x0013" name="RBBM_CFG_DBGBUS_MASKL_3"/>
+	<reg32 offset="0x0014" name="RBBM_CFG_DBGBUS_BYTEL_0"/>
+	<reg32 offset="0x0015" name="RBBM_CFG_DBGBUS_BYTEL_1"/>
+	<reg32 offset="0x0016" name="RBBM_CFG_DBGBUS_IVTE_0"/>
+	<reg32 offset="0x0017" name="RBBM_CFG_DBGBUS_IVTE_1"/>
+	<reg32 offset="0x0018" name="RBBM_CFG_DBGBUS_IVTE_2"/>
+	<reg32 offset="0x0019" name="RBBM_CFG_DBGBUS_IVTE_3"/>
+	<reg32 offset="0x001a" name="RBBM_CFG_DBGBUS_MASKE_0"/>
+	<reg32 offset="0x001b" name="RBBM_CFG_DBGBUS_MASKE_1"/>
+	<reg32 offset="0x001c" name="RBBM_CFG_DBGBUS_MASKE_2"/>
+	<reg32 offset="0x001d" name="RBBM_CFG_DBGBUS_MASKE_3"/>
+	<reg32 offset="0x001e" name="RBBM_CFG_DBGBUS_NIBBLEE"/>
+	<reg32 offset="0x001f" name="RBBM_CFG_DBGBUS_PTRC0"/>
+	<reg32 offset="0x0020" name="RBBM_CFG_DBGBUS_PTRC1"/>
+	<reg32 offset="0x0021" name="RBBM_CFG_DBGBUS_LOADREG"/>
+	<reg32 offset="0x0022" name="RBBM_CFG_DBGBUS_IDX"/>
+	<reg32 offset="0x0023" name="RBBM_CFG_DBGBUS_CLRC"/>
+	<reg32 offset="0x0024" name="RBBM_CFG_DBGBUS_LOADIVT"/>
+	<reg32 offset="0x002f" name="RBBM_INTERFACE_HANG_INT_CNTL"/>
+	<reg32 offset="0x0037" name="RBBM_INT_CLEAR_CMD"/>
+	<reg32 offset="0x0038" name="RBBM_INT_0_MASK">
+		<bitfield name="RBBM_GPU_IDLE" pos="0" type="boolean"/>
+		<bitfield name="RBBM_AHB_ERROR" pos="1" type="boolean"/>
+		<bitfield name="RBBM_TRANSFER_TIMEOUT" pos="2" type="boolean"/>
+		<bitfield name="RBBM_ME_MS_TIMEOUT" pos="3" type="boolean"/>
+		<bitfield name="RBBM_PFP_MS_TIMEOUT" pos="4" type="boolean"/>
+		<bitfield name="RBBM_ETS_MS_TIMEOUT" pos="5" type="boolean"/>
+		<bitfield name="RBBM_ATB_ASYNC_OVERFLOW" pos="6" type="boolean"/>
+		<bitfield name="RBBM_GPC_ERROR" pos="7" type="boolean"/>
+		<bitfield name="CP_SW" pos="8" type="boolean"/>
+		<bitfield name="CP_HW_ERROR" pos="9" type="boolean"/>
+		<bitfield name="CP_CCU_FLUSH_DEPTH_TS" pos="10" type="boolean"/>
+		<bitfield name="CP_CCU_FLUSH_COLOR_TS" pos="11" type="boolean"/>
+		<bitfield name="CP_CCU_RESOLVE_TS" pos="12" type="boolean"/>
+		<bitfield name="CP_IB2" pos="13" type="boolean"/>
+		<bitfield name="CP_IB1" pos="14" type="boolean"/>
+		<bitfield name="CP_RB" pos="15" type="boolean"/>
+		<bitfield name="CP_RB_DONE_TS" pos="17" type="boolean"/>
+		<bitfield name="CP_WT_DONE_TS" pos="18" type="boolean"/>
+		<bitfield name="CP_CACHE_FLUSH_TS" pos="20" type="boolean"/>
+		<bitfield name="RBBM_ATB_BUS_OVERFLOW" pos="22" type="boolean"/>
+		<bitfield name="MISC_HANG_DETECT" pos="23" type="boolean"/>
+		<bitfield name="UCHE_OOB_ACCESS" pos="24" type="boolean"/>
+		<bitfield name="UCHE_TRAP_INTR" pos="25" type="boolean"/>
+		<bitfield name="DEBBUS_INTR_0" pos="26" type="boolean"/>
+		<bitfield name="DEBBUS_INTR_1" pos="27" type="boolean"/>
+		<bitfield name="GPMU_VOLTAGE_DROOP" pos="28" type="boolean"/>
+		<bitfield name="GPMU_FIRMWARE" pos="29" type="boolean"/>
+		<bitfield name="ISDB_CPU_IRQ" pos="30" type="boolean"/>
+		<bitfield name="ISDB_UNDER_DEBUG" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x003f" name="RBBM_AHB_DBG_CNTL"/>
+	<reg32 offset="0x0041" name="RBBM_EXT_VBIF_DBG_CNTL"/>
+	<reg32 offset="0x0043" name="RBBM_SW_RESET_CMD"/>
+	<reg32 offset="0x0045" name="RBBM_BLOCK_SW_RESET_CMD"/>
+	<reg32 offset="0x0046" name="RBBM_BLOCK_SW_RESET_CMD2"/>
+	<reg32 offset="0x0048" name="RBBM_DBG_LO_HI_GPIO"/>
+	<reg32 offset="0x0049" name="RBBM_EXT_TRACE_BUS_CNTL"/>
+	<reg32 offset="0x004a" name="RBBM_CLOCK_CNTL_TP0"/>
+	<reg32 offset="0x004b" name="RBBM_CLOCK_CNTL_TP1"/>
+	<reg32 offset="0x004c" name="RBBM_CLOCK_CNTL_TP2"/>
+	<reg32 offset="0x004d" name="RBBM_CLOCK_CNTL_TP3"/>
+	<reg32 offset="0x004e" name="RBBM_CLOCK_CNTL2_TP0"/>
+	<reg32 offset="0x004f" name="RBBM_CLOCK_CNTL2_TP1"/>
+	<reg32 offset="0x0050" name="RBBM_CLOCK_CNTL2_TP2"/>
+	<reg32 offset="0x0051" name="RBBM_CLOCK_CNTL2_TP3"/>
+	<reg32 offset="0x0052" name="RBBM_CLOCK_CNTL3_TP0"/>
+	<reg32 offset="0x0053" name="RBBM_CLOCK_CNTL3_TP1"/>
+	<reg32 offset="0x0054" name="RBBM_CLOCK_CNTL3_TP2"/>
+	<reg32 offset="0x0055" name="RBBM_CLOCK_CNTL3_TP3"/>
+	<reg32 offset="0x0059" name="RBBM_READ_AHB_THROUGH_DBG"/>
+	<reg32 offset="0x005a" name="RBBM_CLOCK_CNTL_UCHE"/>
+	<reg32 offset="0x005b" name="RBBM_CLOCK_CNTL2_UCHE"/>
+	<reg32 offset="0x005c" name="RBBM_CLOCK_CNTL3_UCHE"/>
+	<reg32 offset="0x005d" name="RBBM_CLOCK_CNTL4_UCHE"/>
+	<reg32 offset="0x005e" name="RBBM_CLOCK_HYST_UCHE"/>
+	<reg32 offset="0x005f" name="RBBM_CLOCK_DELAY_UCHE"/>
+	<reg32 offset="0x0060" name="RBBM_CLOCK_MODE_GPC"/>
+	<reg32 offset="0x0061" name="RBBM_CLOCK_DELAY_GPC"/>
+	<reg32 offset="0x0062" name="RBBM_CLOCK_HYST_GPC"/>
+	<reg32 offset="0x0063" name="RBBM_CLOCK_CNTL_TSE_RAS_RBBM"/>
+	<reg32 offset="0x0064" name="RBBM_CLOCK_HYST_TSE_RAS_RBBM"/>
+	<reg32 offset="0x0065" name="RBBM_CLOCK_DELAY_TSE_RAS_RBBM"/>
+	<reg32 offset="0x0066" name="RBBM_CLOCK_DELAY_HLSQ"/>
+	<reg32 offset="0x0067" name="RBBM_CLOCK_CNTL"/>
+	<reg32 offset="0x0068" name="RBBM_CLOCK_CNTL_SP0"/>
+	<reg32 offset="0x0069" name="RBBM_CLOCK_CNTL_SP1"/>
+	<reg32 offset="0x006a" name="RBBM_CLOCK_CNTL_SP2"/>
+	<reg32 offset="0x006b" name="RBBM_CLOCK_CNTL_SP3"/>
+	<reg32 offset="0x006c" name="RBBM_CLOCK_CNTL2_SP0"/>
+	<reg32 offset="0x006d" name="RBBM_CLOCK_CNTL2_SP1"/>
+	<reg32 offset="0x006e" name="RBBM_CLOCK_CNTL2_SP2"/>
+	<reg32 offset="0x006f" name="RBBM_CLOCK_CNTL2_SP3"/>
+	<reg32 offset="0x0070" name="RBBM_CLOCK_HYST_SP0"/>
+	<reg32 offset="0x0071" name="RBBM_CLOCK_HYST_SP1"/>
+	<reg32 offset="0x0072" name="RBBM_CLOCK_HYST_SP2"/>
+	<reg32 offset="0x0073" name="RBBM_CLOCK_HYST_SP3"/>
+	<reg32 offset="0x0074" name="RBBM_CLOCK_DELAY_SP0"/>
+	<reg32 offset="0x0075" name="RBBM_CLOCK_DELAY_SP1"/>
+	<reg32 offset="0x0076" name="RBBM_CLOCK_DELAY_SP2"/>
+	<reg32 offset="0x0077" name="RBBM_CLOCK_DELAY_SP3"/>
+	<reg32 offset="0x0078" name="RBBM_CLOCK_CNTL_RB0"/>
+	<reg32 offset="0x0079" name="RBBM_CLOCK_CNTL_RB1"/>
+	<reg32 offset="0x007a" name="RBBM_CLOCK_CNTL_RB2"/>
+	<reg32 offset="0x007b" name="RBBM_CLOCK_CNTL_RB3"/>
+	<reg32 offset="0x007c" name="RBBM_CLOCK_CNTL2_RB0"/>
+	<reg32 offset="0x007d" name="RBBM_CLOCK_CNTL2_RB1"/>
+	<reg32 offset="0x007e" name="RBBM_CLOCK_CNTL2_RB2"/>
+	<reg32 offset="0x007f" name="RBBM_CLOCK_CNTL2_RB3"/>
+	<reg32 offset="0x0080" name="RBBM_CLOCK_HYST_RAC"/>
+	<reg32 offset="0x0081" name="RBBM_CLOCK_DELAY_RAC"/>
+	<reg32 offset="0x0082" name="RBBM_CLOCK_CNTL_CCU0"/>
+	<reg32 offset="0x0083" name="RBBM_CLOCK_CNTL_CCU1"/>
+	<reg32 offset="0x0084" name="RBBM_CLOCK_CNTL_CCU2"/>
+	<reg32 offset="0x0085" name="RBBM_CLOCK_CNTL_CCU3"/>
+	<reg32 offset="0x0086" name="RBBM_CLOCK_HYST_RB_CCU0"/>
+	<reg32 offset="0x0087" name="RBBM_CLOCK_HYST_RB_CCU1"/>
+	<reg32 offset="0x0088" name="RBBM_CLOCK_HYST_RB_CCU2"/>
+	<reg32 offset="0x0089" name="RBBM_CLOCK_HYST_RB_CCU3"/>
+	<reg32 offset="0x008a" name="RBBM_CLOCK_CNTL_RAC"/>
+	<reg32 offset="0x008b" name="RBBM_CLOCK_CNTL2_RAC"/>
+	<reg32 offset="0x008c" name="RBBM_CLOCK_DELAY_RB_CCU_L1_0"/>
+	<reg32 offset="0x008d" name="RBBM_CLOCK_DELAY_RB_CCU_L1_1"/>
+	<reg32 offset="0x008e" name="RBBM_CLOCK_DELAY_RB_CCU_L1_2"/>
+	<reg32 offset="0x008f" name="RBBM_CLOCK_DELAY_RB_CCU_L1_3"/>
+	<reg32 offset="0x0090" name="RBBM_CLOCK_HYST_VFD"/>
+	<reg32 offset="0x0091" name="RBBM_CLOCK_MODE_VFD"/>
+	<reg32 offset="0x0092" name="RBBM_CLOCK_DELAY_VFD"/>
+	<reg32 offset="0x0093" name="RBBM_AHB_CNTL0"/>
+	<reg32 offset="0x0094" name="RBBM_AHB_CNTL1"/>
+	<reg32 offset="0x0095" name="RBBM_AHB_CNTL2"/>
+	<reg32 offset="0x0096" name="RBBM_AHB_CMD"/>
+	<reg32 offset="0x009c" name="RBBM_INTERFACE_HANG_MASK_CNTL11"/>
+	<reg32 offset="0x009d" name="RBBM_INTERFACE_HANG_MASK_CNTL12"/>
+	<reg32 offset="0x009e" name="RBBM_INTERFACE_HANG_MASK_CNTL13"/>
+	<reg32 offset="0x009f" name="RBBM_INTERFACE_HANG_MASK_CNTL14"/>
+	<reg32 offset="0x00a0" name="RBBM_INTERFACE_HANG_MASK_CNTL15"/>
+	<reg32 offset="0x00a1" name="RBBM_INTERFACE_HANG_MASK_CNTL16"/>
+	<reg32 offset="0x00a2" name="RBBM_INTERFACE_HANG_MASK_CNTL17"/>
+	<reg32 offset="0x00a3" name="RBBM_INTERFACE_HANG_MASK_CNTL18"/>
+	<reg32 offset="0x00a4" name="RBBM_CLOCK_DELAY_TP0"/>
+	<reg32 offset="0x00a5" name="RBBM_CLOCK_DELAY_TP1"/>
+	<reg32 offset="0x00a6" name="RBBM_CLOCK_DELAY_TP2"/>
+	<reg32 offset="0x00a7" name="RBBM_CLOCK_DELAY_TP3"/>
+	<reg32 offset="0x00a8" name="RBBM_CLOCK_DELAY2_TP0"/>
+	<reg32 offset="0x00a9" name="RBBM_CLOCK_DELAY2_TP1"/>
+	<reg32 offset="0x00aa" name="RBBM_CLOCK_DELAY2_TP2"/>
+	<reg32 offset="0x00ab" name="RBBM_CLOCK_DELAY2_TP3"/>
+	<reg32 offset="0x00ac" name="RBBM_CLOCK_DELAY3_TP0"/>
+	<reg32 offset="0x00ad" name="RBBM_CLOCK_DELAY3_TP1"/>
+	<reg32 offset="0x00ae" name="RBBM_CLOCK_DELAY3_TP2"/>
+	<reg32 offset="0x00af" name="RBBM_CLOCK_DELAY3_TP3"/>
+	<reg32 offset="0x00b0" name="RBBM_CLOCK_HYST_TP0"/>
+	<reg32 offset="0x00b1" name="RBBM_CLOCK_HYST_TP1"/>
+	<reg32 offset="0x00b2" name="RBBM_CLOCK_HYST_TP2"/>
+	<reg32 offset="0x00b3" name="RBBM_CLOCK_HYST_TP3"/>
+	<reg32 offset="0x00b4" name="RBBM_CLOCK_HYST2_TP0"/>
+	<reg32 offset="0x00b5" name="RBBM_CLOCK_HYST2_TP1"/>
+	<reg32 offset="0x00b6" name="RBBM_CLOCK_HYST2_TP2"/>
+	<reg32 offset="0x00b7" name="RBBM_CLOCK_HYST2_TP3"/>
+	<reg32 offset="0x00b8" name="RBBM_CLOCK_HYST3_TP0"/>
+	<reg32 offset="0x00b9" name="RBBM_CLOCK_HYST3_TP1"/>
+	<reg32 offset="0x00ba" name="RBBM_CLOCK_HYST3_TP2"/>
+	<reg32 offset="0x00bb" name="RBBM_CLOCK_HYST3_TP3"/>
+	<reg32 offset="0x00c8" name="RBBM_CLOCK_CNTL_GPMU"/>
+	<reg32 offset="0x00c9" name="RBBM_CLOCK_DELAY_GPMU"/>
+	<reg32 offset="0x00ca" name="RBBM_CLOCK_HYST_GPMU"/>
+	<reg32 offset="0x03a0" name="RBBM_PERFCTR_CP_0_LO"/>
+	<reg32 offset="0x03a1" name="RBBM_PERFCTR_CP_0_HI"/>
+	<reg32 offset="0x03a2" name="RBBM_PERFCTR_CP_1_LO"/>
+	<reg32 offset="0x03a3" name="RBBM_PERFCTR_CP_1_HI"/>
+	<reg32 offset="0x03a4" name="RBBM_PERFCTR_CP_2_LO"/>
+	<reg32 offset="0x03a5" name="RBBM_PERFCTR_CP_2_HI"/>
+	<reg32 offset="0x03a6" name="RBBM_PERFCTR_CP_3_LO"/>
+	<reg32 offset="0x03a7" name="RBBM_PERFCTR_CP_3_HI"/>
+	<reg32 offset="0x03a8" name="RBBM_PERFCTR_CP_4_LO"/>
+	<reg32 offset="0x03a9" name="RBBM_PERFCTR_CP_4_HI"/>
+	<reg32 offset="0x03aa" name="RBBM_PERFCTR_CP_5_LO"/>
+	<reg32 offset="0x03ab" name="RBBM_PERFCTR_CP_5_HI"/>
+	<reg32 offset="0x03ac" name="RBBM_PERFCTR_CP_6_LO"/>
+	<reg32 offset="0x03ad" name="RBBM_PERFCTR_CP_6_HI"/>
+	<reg32 offset="0x03ae" name="RBBM_PERFCTR_CP_7_LO"/>
+	<reg32 offset="0x03af" name="RBBM_PERFCTR_CP_7_HI"/>
+	<reg32 offset="0x03b0" name="RBBM_PERFCTR_RBBM_0_LO"/>
+	<reg32 offset="0x03b1" name="RBBM_PERFCTR_RBBM_0_HI"/>
+	<reg32 offset="0x03b2" name="RBBM_PERFCTR_RBBM_1_LO"/>
+	<reg32 offset="0x03b3" name="RBBM_PERFCTR_RBBM_1_HI"/>
+	<reg32 offset="0x03b4" name="RBBM_PERFCTR_RBBM_2_LO"/>
+	<reg32 offset="0x03b5" name="RBBM_PERFCTR_RBBM_2_HI"/>
+	<reg32 offset="0x03b6" name="RBBM_PERFCTR_RBBM_3_LO"/>
+	<reg32 offset="0x03b7" name="RBBM_PERFCTR_RBBM_3_HI"/>
+	<reg32 offset="0x03b8" name="RBBM_PERFCTR_PC_0_LO"/>
+	<reg32 offset="0x03b9" name="RBBM_PERFCTR_PC_0_HI"/>
+	<reg32 offset="0x03ba" name="RBBM_PERFCTR_PC_1_LO"/>
+	<reg32 offset="0x03bb" name="RBBM_PERFCTR_PC_1_HI"/>
+	<reg32 offset="0x03bc" name="RBBM_PERFCTR_PC_2_LO"/>
+	<reg32 offset="0x03bd" name="RBBM_PERFCTR_PC_2_HI"/>
+	<reg32 offset="0x03be" name="RBBM_PERFCTR_PC_3_LO"/>
+	<reg32 offset="0x03bf" name="RBBM_PERFCTR_PC_3_HI"/>
+	<reg32 offset="0x03c0" name="RBBM_PERFCTR_PC_4_LO"/>
+	<reg32 offset="0x03c1" name="RBBM_PERFCTR_PC_4_HI"/>
+	<reg32 offset="0x03c2" name="RBBM_PERFCTR_PC_5_LO"/>
+	<reg32 offset="0x03c3" name="RBBM_PERFCTR_PC_5_HI"/>
+	<reg32 offset="0x03c4" name="RBBM_PERFCTR_PC_6_LO"/>
+	<reg32 offset="0x03c5" name="RBBM_PERFCTR_PC_6_HI"/>
+	<reg32 offset="0x03c6" name="RBBM_PERFCTR_PC_7_LO"/>
+	<reg32 offset="0x03c7" name="RBBM_PERFCTR_PC_7_HI"/>
+	<reg32 offset="0x03c8" name="RBBM_PERFCTR_VFD_0_LO"/>
+	<reg32 offset="0x03c9" name="RBBM_PERFCTR_VFD_0_HI"/>
+	<reg32 offset="0x03ca" name="RBBM_PERFCTR_VFD_1_LO"/>
+	<reg32 offset="0x03cb" name="RBBM_PERFCTR_VFD_1_HI"/>
+	<reg32 offset="0x03cc" name="RBBM_PERFCTR_VFD_2_LO"/>
+	<reg32 offset="0x03cd" name="RBBM_PERFCTR_VFD_2_HI"/>
+	<reg32 offset="0x03ce" name="RBBM_PERFCTR_VFD_3_LO"/>
+	<reg32 offset="0x03cf" name="RBBM_PERFCTR_VFD_3_HI"/>
+	<reg32 offset="0x03d0" name="RBBM_PERFCTR_VFD_4_LO"/>
+	<reg32 offset="0x03d1" name="RBBM_PERFCTR_VFD_4_HI"/>
+	<reg32 offset="0x03d2" name="RBBM_PERFCTR_VFD_5_LO"/>
+	<reg32 offset="0x03d3" name="RBBM_PERFCTR_VFD_5_HI"/>
+	<reg32 offset="0x03d4" name="RBBM_PERFCTR_VFD_6_LO"/>
+	<reg32 offset="0x03d5" name="RBBM_PERFCTR_VFD_6_HI"/>
+	<reg32 offset="0x03d6" name="RBBM_PERFCTR_VFD_7_LO"/>
+	<reg32 offset="0x03d7" name="RBBM_PERFCTR_VFD_7_HI"/>
+	<reg32 offset="0x03d8" name="RBBM_PERFCTR_HLSQ_0_LO"/>
+	<reg32 offset="0x03d9" name="RBBM_PERFCTR_HLSQ_0_HI"/>
+	<reg32 offset="0x03da" name="RBBM_PERFCTR_HLSQ_1_LO"/>
+	<reg32 offset="0x03db" name="RBBM_PERFCTR_HLSQ_1_HI"/>
+	<reg32 offset="0x03dc" name="RBBM_PERFCTR_HLSQ_2_LO"/>
+	<reg32 offset="0x03dd" name="RBBM_PERFCTR_HLSQ_2_HI"/>
+	<reg32 offset="0x03de" name="RBBM_PERFCTR_HLSQ_3_LO"/>
+	<reg32 offset="0x03df" name="RBBM_PERFCTR_HLSQ_3_HI"/>
+	<reg32 offset="0x03e0" name="RBBM_PERFCTR_HLSQ_4_LO"/>
+	<reg32 offset="0x03e1" name="RBBM_PERFCTR_HLSQ_4_HI"/>
+	<reg32 offset="0x03e2" name="RBBM_PERFCTR_HLSQ_5_LO"/>
+	<reg32 offset="0x03e3" name="RBBM_PERFCTR_HLSQ_5_HI"/>
+	<reg32 offset="0x03e4" name="RBBM_PERFCTR_HLSQ_6_LO"/>
+	<reg32 offset="0x03e5" name="RBBM_PERFCTR_HLSQ_6_HI"/>
+	<reg32 offset="0x03e6" name="RBBM_PERFCTR_HLSQ_7_LO"/>
+	<reg32 offset="0x03e7" name="RBBM_PERFCTR_HLSQ_7_HI"/>
+	<reg32 offset="0x03e8" name="RBBM_PERFCTR_VPC_0_LO"/>
+	<reg32 offset="0x03e9" name="RBBM_PERFCTR_VPC_0_HI"/>
+	<reg32 offset="0x03ea" name="RBBM_PERFCTR_VPC_1_LO"/>
+	<reg32 offset="0x03eb" name="RBBM_PERFCTR_VPC_1_HI"/>
+	<reg32 offset="0x03ec" name="RBBM_PERFCTR_VPC_2_LO"/>
+	<reg32 offset="0x03ed" name="RBBM_PERFCTR_VPC_2_HI"/>
+	<reg32 offset="0x03ee" name="RBBM_PERFCTR_VPC_3_LO"/>
+	<reg32 offset="0x03ef" name="RBBM_PERFCTR_VPC_3_HI"/>
+	<reg32 offset="0x03f0" name="RBBM_PERFCTR_CCU_0_LO"/>
+	<reg32 offset="0x03f1" name="RBBM_PERFCTR_CCU_0_HI"/>
+	<reg32 offset="0x03f2" name="RBBM_PERFCTR_CCU_1_LO"/>
+	<reg32 offset="0x03f3" name="RBBM_PERFCTR_CCU_1_HI"/>
+	<reg32 offset="0x03f4" name="RBBM_PERFCTR_CCU_2_LO"/>
+	<reg32 offset="0x03f5" name="RBBM_PERFCTR_CCU_2_HI"/>
+	<reg32 offset="0x03f6" name="RBBM_PERFCTR_CCU_3_LO"/>
+	<reg32 offset="0x03f7" name="RBBM_PERFCTR_CCU_3_HI"/>
+	<reg32 offset="0x03f8" name="RBBM_PERFCTR_TSE_0_LO"/>
+	<reg32 offset="0x03f9" name="RBBM_PERFCTR_TSE_0_HI"/>
+	<reg32 offset="0x03fa" name="RBBM_PERFCTR_TSE_1_LO"/>
+	<reg32 offset="0x03fb" name="RBBM_PERFCTR_TSE_1_HI"/>
+	<reg32 offset="0x03fc" name="RBBM_PERFCTR_TSE_2_LO"/>
+	<reg32 offset="0x03fd" name="RBBM_PERFCTR_TSE_2_HI"/>
+	<reg32 offset="0x03fe" name="RBBM_PERFCTR_TSE_3_LO"/>
+	<reg32 offset="0x03ff" name="RBBM_PERFCTR_TSE_3_HI"/>
+	<reg32 offset="0x0400" name="RBBM_PERFCTR_RAS_0_LO"/>
+	<reg32 offset="0x0401" name="RBBM_PERFCTR_RAS_0_HI"/>
+	<reg32 offset="0x0402" name="RBBM_PERFCTR_RAS_1_LO"/>
+	<reg32 offset="0x0403" name="RBBM_PERFCTR_RAS_1_HI"/>
+	<reg32 offset="0x0404" name="RBBM_PERFCTR_RAS_2_LO"/>
+	<reg32 offset="0x0405" name="RBBM_PERFCTR_RAS_2_HI"/>
+	<reg32 offset="0x0406" name="RBBM_PERFCTR_RAS_3_LO"/>
+	<reg32 offset="0x0407" name="RBBM_PERFCTR_RAS_3_HI"/>
+	<reg32 offset="0x0408" name="RBBM_PERFCTR_UCHE_0_LO"/>
+	<reg32 offset="0x0409" name="RBBM_PERFCTR_UCHE_0_HI"/>
+	<reg32 offset="0x040a" name="RBBM_PERFCTR_UCHE_1_LO"/>
+	<reg32 offset="0x040b" name="RBBM_PERFCTR_UCHE_1_HI"/>
+	<reg32 offset="0x040c" name="RBBM_PERFCTR_UCHE_2_LO"/>
+	<reg32 offset="0x040d" name="RBBM_PERFCTR_UCHE_2_HI"/>
+	<reg32 offset="0x040e" name="RBBM_PERFCTR_UCHE_3_LO"/>
+	<reg32 offset="0x040f" name="RBBM_PERFCTR_UCHE_3_HI"/>
+	<reg32 offset="0x0410" name="RBBM_PERFCTR_UCHE_4_LO"/>
+	<reg32 offset="0x0411" name="RBBM_PERFCTR_UCHE_4_HI"/>
+	<reg32 offset="0x0412" name="RBBM_PERFCTR_UCHE_5_LO"/>
+	<reg32 offset="0x0413" name="RBBM_PERFCTR_UCHE_5_HI"/>
+	<reg32 offset="0x0414" name="RBBM_PERFCTR_UCHE_6_LO"/>
+	<reg32 offset="0x0415" name="RBBM_PERFCTR_UCHE_6_HI"/>
+	<reg32 offset="0x0416" name="RBBM_PERFCTR_UCHE_7_LO"/>
+	<reg32 offset="0x0417" name="RBBM_PERFCTR_UCHE_7_HI"/>
+	<reg32 offset="0x0418" name="RBBM_PERFCTR_TP_0_LO"/>
+	<reg32 offset="0x0419" name="RBBM_PERFCTR_TP_0_HI"/>
+	<reg32 offset="0x041a" name="RBBM_PERFCTR_TP_1_LO"/>
+	<reg32 offset="0x041b" name="RBBM_PERFCTR_TP_1_HI"/>
+	<reg32 offset="0x041c" name="RBBM_PERFCTR_TP_2_LO"/>
+	<reg32 offset="0x041d" name="RBBM_PERFCTR_TP_2_HI"/>
+	<reg32 offset="0x041e" name="RBBM_PERFCTR_TP_3_LO"/>
+	<reg32 offset="0x041f" name="RBBM_PERFCTR_TP_3_HI"/>
+	<reg32 offset="0x0420" name="RBBM_PERFCTR_TP_4_LO"/>
+	<reg32 offset="0x0421" name="RBBM_PERFCTR_TP_4_HI"/>
+	<reg32 offset="0x0422" name="RBBM_PERFCTR_TP_5_LO"/>
+	<reg32 offset="0x0423" name="RBBM_PERFCTR_TP_5_HI"/>
+	<reg32 offset="0x0424" name="RBBM_PERFCTR_TP_6_LO"/>
+	<reg32 offset="0x0425" name="RBBM_PERFCTR_TP_6_HI"/>
+	<reg32 offset="0x0426" name="RBBM_PERFCTR_TP_7_LO"/>
+	<reg32 offset="0x0427" name="RBBM_PERFCTR_TP_7_HI"/>
+	<reg32 offset="0x0428" name="RBBM_PERFCTR_SP_0_LO"/>
+	<reg32 offset="0x0429" name="RBBM_PERFCTR_SP_0_HI"/>
+	<reg32 offset="0x042a" name="RBBM_PERFCTR_SP_1_LO"/>
+	<reg32 offset="0x042b" name="RBBM_PERFCTR_SP_1_HI"/>
+	<reg32 offset="0x042c" name="RBBM_PERFCTR_SP_2_LO"/>
+	<reg32 offset="0x042d" name="RBBM_PERFCTR_SP_2_HI"/>
+	<reg32 offset="0x042e" name="RBBM_PERFCTR_SP_3_LO"/>
+	<reg32 offset="0x042f" name="RBBM_PERFCTR_SP_3_HI"/>
+	<reg32 offset="0x0430" name="RBBM_PERFCTR_SP_4_LO"/>
+	<reg32 offset="0x0431" name="RBBM_PERFCTR_SP_4_HI"/>
+	<reg32 offset="0x0432" name="RBBM_PERFCTR_SP_5_LO"/>
+	<reg32 offset="0x0433" name="RBBM_PERFCTR_SP_5_HI"/>
+	<reg32 offset="0x0434" name="RBBM_PERFCTR_SP_6_LO"/>
+	<reg32 offset="0x0435" name="RBBM_PERFCTR_SP_6_HI"/>
+	<reg32 offset="0x0436" name="RBBM_PERFCTR_SP_7_LO"/>
+	<reg32 offset="0x0437" name="RBBM_PERFCTR_SP_7_HI"/>
+	<reg32 offset="0x0438" name="RBBM_PERFCTR_SP_8_LO"/>
+	<reg32 offset="0x0439" name="RBBM_PERFCTR_SP_8_HI"/>
+	<reg32 offset="0x043a" name="RBBM_PERFCTR_SP_9_LO"/>
+	<reg32 offset="0x043b" name="RBBM_PERFCTR_SP_9_HI"/>
+	<reg32 offset="0x043c" name="RBBM_PERFCTR_SP_10_LO"/>
+	<reg32 offset="0x043d" name="RBBM_PERFCTR_SP_10_HI"/>
+	<reg32 offset="0x043e" name="RBBM_PERFCTR_SP_11_LO"/>
+	<reg32 offset="0x043f" name="RBBM_PERFCTR_SP_11_HI"/>
+	<reg32 offset="0x0440" name="RBBM_PERFCTR_RB_0_LO"/>
+	<reg32 offset="0x0441" name="RBBM_PERFCTR_RB_0_HI"/>
+	<reg32 offset="0x0442" name="RBBM_PERFCTR_RB_1_LO"/>
+	<reg32 offset="0x0443" name="RBBM_PERFCTR_RB_1_HI"/>
+	<reg32 offset="0x0444" name="RBBM_PERFCTR_RB_2_LO"/>
+	<reg32 offset="0x0445" name="RBBM_PERFCTR_RB_2_HI"/>
+	<reg32 offset="0x0446" name="RBBM_PERFCTR_RB_3_LO"/>
+	<reg32 offset="0x0447" name="RBBM_PERFCTR_RB_3_HI"/>
+	<reg32 offset="0x0448" name="RBBM_PERFCTR_RB_4_LO"/>
+	<reg32 offset="0x0449" name="RBBM_PERFCTR_RB_4_HI"/>
+	<reg32 offset="0x044a" name="RBBM_PERFCTR_RB_5_LO"/>
+	<reg32 offset="0x044b" name="RBBM_PERFCTR_RB_5_HI"/>
+	<reg32 offset="0x044c" name="RBBM_PERFCTR_RB_6_LO"/>
+	<reg32 offset="0x044d" name="RBBM_PERFCTR_RB_6_HI"/>
+	<reg32 offset="0x044e" name="RBBM_PERFCTR_RB_7_LO"/>
+	<reg32 offset="0x044f" name="RBBM_PERFCTR_RB_7_HI"/>
+	<reg32 offset="0x0450" name="RBBM_PERFCTR_VSC_0_LO"/>
+	<reg32 offset="0x0451" name="RBBM_PERFCTR_VSC_0_HI"/>
+	<reg32 offset="0x0452" name="RBBM_PERFCTR_VSC_1_LO"/>
+	<reg32 offset="0x0453" name="RBBM_PERFCTR_VSC_1_HI"/>
+	<reg32 offset="0x0454" name="RBBM_PERFCTR_LRZ_0_LO"/>
+	<reg32 offset="0x0455" name="RBBM_PERFCTR_LRZ_0_HI"/>
+	<reg32 offset="0x0456" name="RBBM_PERFCTR_LRZ_1_LO"/>
+	<reg32 offset="0x0457" name="RBBM_PERFCTR_LRZ_1_HI"/>
+	<reg32 offset="0x0458" name="RBBM_PERFCTR_LRZ_2_LO"/>
+	<reg32 offset="0x0459" name="RBBM_PERFCTR_LRZ_2_HI"/>
+	<reg32 offset="0x045a" name="RBBM_PERFCTR_LRZ_3_LO"/>
+	<reg32 offset="0x045b" name="RBBM_PERFCTR_LRZ_3_HI"/>
+	<reg32 offset="0x045c" name="RBBM_PERFCTR_CMP_0_LO"/>
+	<reg32 offset="0x045d" name="RBBM_PERFCTR_CMP_0_HI"/>
+	<reg32 offset="0x045e" name="RBBM_PERFCTR_CMP_1_LO"/>
+	<reg32 offset="0x045f" name="RBBM_PERFCTR_CMP_1_HI"/>
+	<reg32 offset="0x0460" name="RBBM_PERFCTR_CMP_2_LO"/>
+	<reg32 offset="0x0461" name="RBBM_PERFCTR_CMP_2_HI"/>
+	<reg32 offset="0x0462" name="RBBM_PERFCTR_CMP_3_LO"/>
+	<reg32 offset="0x0463" name="RBBM_PERFCTR_CMP_3_HI"/>
+	<reg32 offset="0x046b" name="RBBM_PERFCTR_RBBM_SEL_0" type="a5xx_rbbm_perfcounter_select"/>
+	<reg32 offset="0x046c" name="RBBM_PERFCTR_RBBM_SEL_1" type="a5xx_rbbm_perfcounter_select"/>
+	<reg32 offset="0x046d" name="RBBM_PERFCTR_RBBM_SEL_2" type="a5xx_rbbm_perfcounter_select"/>
+	<reg32 offset="0x046e" name="RBBM_PERFCTR_RBBM_SEL_3" type="a5xx_rbbm_perfcounter_select"/>
+	<reg32 offset="0x04d2" name="RBBM_ALWAYSON_COUNTER_LO"/>
+	<reg32 offset="0x04d3" name="RBBM_ALWAYSON_COUNTER_HI"/>
+	<reg32 offset="0x04f5" name="RBBM_STATUS">
+		<bitfield high="31" low="31" name="GPU_BUSY_IGN_AHB" />
+		<bitfield high="30" low="30" name="GPU_BUSY_IGN_AHB_CP" />
+		<bitfield high="29" low="29" name="HLSQ_BUSY" />
+		<bitfield high="28" low="28" name="VSC_BUSY" />
+		<bitfield high="27" low="27" name="TPL1_BUSY" />
+		<bitfield high="26" low="26" name="SP_BUSY" />
+		<bitfield high="25" low="25" name="UCHE_BUSY" />
+		<bitfield high="24" low="24" name="VPC_BUSY" />
+		<bitfield high="23" low="23" name="VFDP_BUSY" />
+		<bitfield high="22" low="22" name="VFD_BUSY" />
+		<bitfield high="21" low="21" name="TESS_BUSY" />
+		<bitfield high="20" low="20" name="PC_VSD_BUSY" />
+		<bitfield high="19" low="19" name="PC_DCALL_BUSY" />
+		<bitfield high="18" low="18" name="GPMU_SLAVE_BUSY" />
+		<bitfield high="17" low="17" name="DCOM_BUSY" />
+		<bitfield high="16" low="16" name="COM_BUSY" />
+		<bitfield high="15" low="15" name="LRZ_BUZY" />
+		<bitfield high="14" low="14" name="A2D_DSP_BUSY" />
+		<bitfield high="13" low="13" name="CCUFCHE_BUSY" />
+		<bitfield high="12" low="12" name="RB_BUSY" />
+		<bitfield high="11" low="11" name="RAS_BUSY" />
+		<bitfield high="10" low="10" name="TSE_BUSY" />
+		<bitfield high="9" low="9" name="VBIF_BUSY" />
+		<bitfield high="8" low="8" name="GPU_BUSY_IGN_AHB_HYST" />
+		<bitfield high="7" low="7" name="CP_BUSY_IGN_HYST" />
+		<bitfield high="6" low="6" name="CP_BUSY" />
+		<bitfield high="5" low="5" name="GPMU_MASTER_BUSY" />
+		<bitfield high="4" low="4" name="CP_CRASH_BUSY" />
+		<bitfield high="3" low="3" name="CP_ETS_BUSY" />
+		<bitfield high="2" low="2" name="CP_PFP_BUSY" />
+		<bitfield high="1" low="1" name="CP_ME_BUSY" />
+		<bitfield high="0" low="0" name="HI_BUSY" />
+	</reg32>
+	<reg32 offset="0x0530" name="RBBM_STATUS3">
+		<bitfield pos="24" name="SMMU_STALLED_ON_FAULT" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x04e1" name="RBBM_INT_0_STATUS"/>
+	<reg32 offset="0x04f0" name="RBBM_AHB_ME_SPLIT_STATUS"/>
+	<reg32 offset="0x04f1" name="RBBM_AHB_PFP_SPLIT_STATUS"/>
+	<reg32 offset="0x04f3" name="RBBM_AHB_ETS_SPLIT_STATUS"/>
+	<reg32 offset="0x04f4" name="RBBM_AHB_ERROR_STATUS"/>
+	<reg32 offset="0x0464" name="RBBM_PERFCTR_CNTL"/>
+	<reg32 offset="0x0465" name="RBBM_PERFCTR_LOAD_CMD0"/>
+	<reg32 offset="0x0466" name="RBBM_PERFCTR_LOAD_CMD1"/>
+	<reg32 offset="0x0467" name="RBBM_PERFCTR_LOAD_CMD2"/>
+	<reg32 offset="0x0468" name="RBBM_PERFCTR_LOAD_CMD3"/>
+	<reg32 offset="0x0469" name="RBBM_PERFCTR_LOAD_VALUE_LO"/>
+	<reg32 offset="0x046a" name="RBBM_PERFCTR_LOAD_VALUE_HI"/>
+	<reg32 offset="0x046f" name="RBBM_PERFCTR_GPU_BUSY_MASKED"/>
+	<reg32 offset="0x04ed" name="RBBM_AHB_ERROR"/>
+	<reg32 offset="0x0504" name="RBBM_CFG_DBGBUS_EVENT_LOGIC"/>
+	<reg32 offset="0x0505" name="RBBM_CFG_DBGBUS_OVER"/>
+	<reg32 offset="0x0506" name="RBBM_CFG_DBGBUS_COUNT0"/>
+	<reg32 offset="0x0507" name="RBBM_CFG_DBGBUS_COUNT1"/>
+	<reg32 offset="0x0508" name="RBBM_CFG_DBGBUS_COUNT2"/>
+	<reg32 offset="0x0509" name="RBBM_CFG_DBGBUS_COUNT3"/>
+	<reg32 offset="0x050a" name="RBBM_CFG_DBGBUS_COUNT4"/>
+	<reg32 offset="0x050b" name="RBBM_CFG_DBGBUS_COUNT5"/>
+	<reg32 offset="0x050c" name="RBBM_CFG_DBGBUS_TRACE_ADDR"/>
+	<reg32 offset="0x050d" name="RBBM_CFG_DBGBUS_TRACE_BUF0"/>
+	<reg32 offset="0x050e" name="RBBM_CFG_DBGBUS_TRACE_BUF1"/>
+	<reg32 offset="0x050f" name="RBBM_CFG_DBGBUS_TRACE_BUF2"/>
+	<reg32 offset="0x0510" name="RBBM_CFG_DBGBUS_TRACE_BUF3"/>
+	<reg32 offset="0x0511" name="RBBM_CFG_DBGBUS_TRACE_BUF4"/>
+	<reg32 offset="0x0512" name="RBBM_CFG_DBGBUS_MISR0"/>
+	<reg32 offset="0x0513" name="RBBM_CFG_DBGBUS_MISR1"/>
+	<reg32 offset="0x0533" name="RBBM_ISDB_CNT"/>
+	<reg32 offset="0xf000" name="RBBM_SECVID_TRUST_CONFIG"/>
+	<reg32 offset="0xf400" name="RBBM_SECVID_TRUST_CNTL"/>
+	<reg32 offset="0xf800" name="RBBM_SECVID_TSB_TRUSTED_BASE_LO"/>
+	<reg32 offset="0xf801" name="RBBM_SECVID_TSB_TRUSTED_BASE_HI"/>
+	<reg32 offset="0xf802" name="RBBM_SECVID_TSB_TRUSTED_SIZE"/>
+	<reg32 offset="0xf803" name="RBBM_SECVID_TSB_CNTL"/>
+	<reg32 offset="0xf804" name="RBBM_SECVID_TSB_COMP_STATUS_LO"/>
+	<reg32 offset="0xf805" name="RBBM_SECVID_TSB_COMP_STATUS_HI"/>
+	<reg32 offset="0xf806" name="RBBM_SECVID_TSB_UCHE_STATUS_LO"/>
+	<reg32 offset="0xf807" name="RBBM_SECVID_TSB_UCHE_STATUS_HI"/>
+	<reg32 offset="0xf810" name="RBBM_SECVID_TSB_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+
+	<!-- VSC registers -->
+	<reg32 offset="0x0bc2" name="VSC_BIN_SIZE">
+		<bitfield name="WIDTH" low="0" high="7" shr="5" type="uint"/>
+		<bitfield name="HEIGHT" low="9" high="16" shr="5" type="uint"/>
+		<!-- b17 maybe BYPASS like RB_CNTL, but reg not written for bypass -->
+	</reg32>
+	<reg32 offset="0x0bc3" name="VSC_SIZE_ADDRESS_LO"/>
+	<reg32 offset="0x0bc4" name="VSC_SIZE_ADDRESS_HI"/>
+	<reg32 offset="0x0bc5" name="UNKNOWN_0BC5"/> <!-- always 00000000? -->
+	<reg32 offset="0x0bc6" name="UNKNOWN_0BC6"/> <!-- always 00000000? -->
+	<array offset="0x0bd0" name="VSC_PIPE_CONFIG" stride="1" length="16">
+		<reg32 offset="0x0" name="REG">
+			<doc>
+				Configures the mapping between VSC_PIPE buffer and
+				bin, X/Y specify the bin index in the horiz/vert
+				direction (0,0 is upper left, 0,1 is leftmost bin
+				on second row, and so on).  W/H specify the number
+				of bins assigned to this VSC_PIPE in the horiz/vert
+				dimension.
+			</doc>
+			<bitfield name="X" low="0" high="9" type="uint"/>
+			<bitfield name="Y" low="10" high="19" type="uint"/>
+			<bitfield name="W" low="20" high="23" type="uint"/>
+			<bitfield name="H" low="24" high="27" type="uint"/>
+		</reg32>
+	</array>
+	<array offset="0x0be0" name="VSC_PIPE_DATA_ADDRESS" stride="2" length="16">
+		<reg32 offset="0x0" name="LO"/>
+		<reg32 offset="0x1" name="HI"/>
+	</array>
+	<array offset="0x0c00" name="VSC_PIPE_DATA_LENGTH" stride="1" length="16">
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+	<reg32 offset="0x0c60" name="VSC_PERFCTR_VSC_SEL_0" type="a5xx_vsc_perfcounter_select"/>
+	<reg32 offset="0x0c61" name="VSC_PERFCTR_VSC_SEL_1" type="a5xx_vsc_perfcounter_select"/>
+
+	<!-- used for some blits?? -->
+	<reg32 offset="0x0cdd" name="VSC_RESOLVE_CNTL" type="adreno_reg_xy"/>
+
+	<!-- GRAS registers -->
+	<reg32 offset="0x0c81" name="GRAS_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0c90" name="GRAS_PERFCTR_TSE_SEL_0" type="a5xx_tse_perfcounter_select"/>
+	<reg32 offset="0x0c91" name="GRAS_PERFCTR_TSE_SEL_1" type="a5xx_tse_perfcounter_select"/>
+	<reg32 offset="0x0c92" name="GRAS_PERFCTR_TSE_SEL_2" type="a5xx_tse_perfcounter_select"/>
+	<reg32 offset="0x0c93" name="GRAS_PERFCTR_TSE_SEL_3" type="a5xx_tse_perfcounter_select"/>
+	<reg32 offset="0x0c94" name="GRAS_PERFCTR_RAS_SEL_0" type="a5xx_ras_perfcounter_select"/>
+	<reg32 offset="0x0c95" name="GRAS_PERFCTR_RAS_SEL_1" type="a5xx_ras_perfcounter_select"/>
+	<reg32 offset="0x0c96" name="GRAS_PERFCTR_RAS_SEL_2" type="a5xx_ras_perfcounter_select"/>
+	<reg32 offset="0x0c97" name="GRAS_PERFCTR_RAS_SEL_3" type="a5xx_ras_perfcounter_select"/>
+	<reg32 offset="0x0c98" name="GRAS_PERFCTR_LRZ_SEL_0" type="a5xx_lrz_perfcounter_select"/>
+	<reg32 offset="0x0c99" name="GRAS_PERFCTR_LRZ_SEL_1" type="a5xx_lrz_perfcounter_select"/>
+	<reg32 offset="0x0c9a" name="GRAS_PERFCTR_LRZ_SEL_2" type="a5xx_lrz_perfcounter_select"/>
+	<reg32 offset="0x0c9b" name="GRAS_PERFCTR_LRZ_SEL_3" type="a5xx_lrz_perfcounter_select"/>
+
+	<reg32 offset="0x0cc4" name="RB_DBG_ECO_CNTL"/> <!-- always 00100000? -->
+	<reg32 offset="0x0cc5" name="RB_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0cc6" name="RB_MODE_CNTL"/> <!-- always 00000044? -->
+	<reg32 offset="0x0cc7" name="RB_CCU_CNTL"/> <!-- always b0056080 or 10000000? -->
+	<reg32 offset="0x0cd0" name="RB_PERFCTR_RB_SEL_0" type="a5xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cd1" name="RB_PERFCTR_RB_SEL_1" type="a5xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cd2" name="RB_PERFCTR_RB_SEL_2" type="a5xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cd3" name="RB_PERFCTR_RB_SEL_3" type="a5xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cd4" name="RB_PERFCTR_RB_SEL_4" type="a5xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cd5" name="RB_PERFCTR_RB_SEL_5" type="a5xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cd6" name="RB_PERFCTR_RB_SEL_6" type="a5xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cd7" name="RB_PERFCTR_RB_SEL_7" type="a5xx_rb_perfcounter_select"/>
+	<reg32 offset="0x0cd8" name="RB_PERFCTR_CCU_SEL_0" type="a5xx_ccu_perfcounter_select"/>
+	<reg32 offset="0x0cd9" name="RB_PERFCTR_CCU_SEL_1" type="a5xx_ccu_perfcounter_select"/>
+	<reg32 offset="0x0cda" name="RB_PERFCTR_CCU_SEL_2" type="a5xx_ccu_perfcounter_select"/>
+	<reg32 offset="0x0cdb" name="RB_PERFCTR_CCU_SEL_3" type="a5xx_ccu_perfcounter_select"/>
+	<reg32 offset="0x0ce0" name="RB_POWERCTR_RB_SEL_0"/>
+	<reg32 offset="0x0ce1" name="RB_POWERCTR_RB_SEL_1"/>
+	<reg32 offset="0x0ce2" name="RB_POWERCTR_RB_SEL_2"/>
+	<reg32 offset="0x0ce3" name="RB_POWERCTR_RB_SEL_3"/>
+	<reg32 offset="0x0ce4" name="RB_POWERCTR_CCU_SEL_0"/>
+	<reg32 offset="0x0ce5" name="RB_POWERCTR_CCU_SEL_1"/>
+	<reg32 offset="0x0cec" name="RB_PERFCTR_CMP_SEL_0" type="a5xx_cmp_perfcounter_select"/>
+	<reg32 offset="0x0ced" name="RB_PERFCTR_CMP_SEL_1" type="a5xx_cmp_perfcounter_select"/>
+	<reg32 offset="0x0cee" name="RB_PERFCTR_CMP_SEL_2" type="a5xx_cmp_perfcounter_select"/>
+	<reg32 offset="0x0cef" name="RB_PERFCTR_CMP_SEL_3" type="a5xx_cmp_perfcounter_select"/>
+
+	<reg32 offset="0x0d00" name="PC_DBG_ECO_CNTL">
+		<bitfield name="TWOPASSUSEWFI" pos="8" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0d01" name="PC_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0d02" name="PC_MODE_CNTL"/> <!-- always 0000001f? -->
+	<reg32 offset="0x0d04" name="PC_INDEX_BUF_LO"/>
+	<reg32 offset="0x0d05" name="PC_INDEX_BUF_HI"/>
+	<reg32 offset="0x0d06" name="PC_START_INDEX"/>
+	<reg32 offset="0x0d07" name="PC_MAX_INDEX"/>
+	<reg32 offset="0x0d08" name="PC_TESSFACTOR_ADDR_LO"/>
+	<reg32 offset="0x0d09" name="PC_TESSFACTOR_ADDR_HI"/>
+	<reg32 offset="0x0d10" name="PC_PERFCTR_PC_SEL_0" type="a5xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d11" name="PC_PERFCTR_PC_SEL_1" type="a5xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d12" name="PC_PERFCTR_PC_SEL_2" type="a5xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d13" name="PC_PERFCTR_PC_SEL_3" type="a5xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d14" name="PC_PERFCTR_PC_SEL_4" type="a5xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d15" name="PC_PERFCTR_PC_SEL_5" type="a5xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d16" name="PC_PERFCTR_PC_SEL_6" type="a5xx_pc_perfcounter_select"/>
+	<reg32 offset="0x0d17" name="PC_PERFCTR_PC_SEL_7" type="a5xx_pc_perfcounter_select"/>
+
+	<reg32 offset="0x0e00" name="HLSQ_TIMEOUT_THRESHOLD_0"/>
+	<reg32 offset="0x0e01" name="HLSQ_TIMEOUT_THRESHOLD_1"/>
+	<reg32 offset="0x0e04" name="HLSQ_DBG_ECO_CNTL"/>
+	<reg32 offset="0x0e05" name="HLSQ_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0e06" name="HLSQ_MODE_CNTL"/> <!-- always 00000001? -->
+	<reg32 offset="0x0e10" name="HLSQ_PERFCTR_HLSQ_SEL_0" type="a5xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e11" name="HLSQ_PERFCTR_HLSQ_SEL_1" type="a5xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e12" name="HLSQ_PERFCTR_HLSQ_SEL_2" type="a5xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e13" name="HLSQ_PERFCTR_HLSQ_SEL_3" type="a5xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e14" name="HLSQ_PERFCTR_HLSQ_SEL_4" type="a5xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e15" name="HLSQ_PERFCTR_HLSQ_SEL_5" type="a5xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e16" name="HLSQ_PERFCTR_HLSQ_SEL_6" type="a5xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0e17" name="HLSQ_PERFCTR_HLSQ_SEL_7" type="a5xx_hlsq_perfcounter_select"/>
+	<reg32 offset="0x0f08" name="HLSQ_SPTP_RDSEL"/>
+	<reg32 offset="0xbc00" name="HLSQ_DBG_READ_SEL"/>
+	<reg32 offset="0xa000" name="HLSQ_DBG_AHB_READ_APERTURE"/>
+
+	<reg32 offset="0x0e41" name="VFD_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0e42" name="VFD_MODE_CNTL"/> <!-- always 00000000? -->
+	<reg32 offset="0x0e50" name="VFD_PERFCTR_VFD_SEL_0" type="a5xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e51" name="VFD_PERFCTR_VFD_SEL_1" type="a5xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e52" name="VFD_PERFCTR_VFD_SEL_2" type="a5xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e53" name="VFD_PERFCTR_VFD_SEL_3" type="a5xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e54" name="VFD_PERFCTR_VFD_SEL_4" type="a5xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e55" name="VFD_PERFCTR_VFD_SEL_5" type="a5xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e56" name="VFD_PERFCTR_VFD_SEL_6" type="a5xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e57" name="VFD_PERFCTR_VFD_SEL_7" type="a5xx_vfd_perfcounter_select"/>
+	<reg32 offset="0x0e60" name="VPC_DBG_ECO_CNTL">
+		<bitfield name="ALLFLATOPTDIS" pos="10" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0e61" name="VPC_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0e62" name="VPC_MODE_CNTL">
+		<bitfield name="BINNING_PASS" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0e64" name="VPC_PERFCTR_VPC_SEL_0" type="a5xx_vpc_perfcounter_select"/>
+	<reg32 offset="0x0e65" name="VPC_PERFCTR_VPC_SEL_1" type="a5xx_vpc_perfcounter_select"/>
+	<reg32 offset="0x0e66" name="VPC_PERFCTR_VPC_SEL_2" type="a5xx_vpc_perfcounter_select"/>
+	<reg32 offset="0x0e67" name="VPC_PERFCTR_VPC_SEL_3" type="a5xx_vpc_perfcounter_select"/>
+
+	<reg32 offset="0x0e80" name="UCHE_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0e81" name="UCHE_MODE_CNTL"/>
+	<reg32 offset="0x0e82" name="UCHE_SVM_CNTL"/>
+	<reg32 offset="0x0e87" name="UCHE_WRITE_THRU_BASE_LO"/>
+	<reg32 offset="0x0e88" name="UCHE_WRITE_THRU_BASE_HI"/>
+	<reg32 offset="0x0e89" name="UCHE_TRAP_BASE_LO"/>
+	<reg32 offset="0x0e8a" name="UCHE_TRAP_BASE_HI"/>
+	<reg32 offset="0x0e8b" name="UCHE_GMEM_RANGE_MIN_LO"/>
+	<reg32 offset="0x0e8c" name="UCHE_GMEM_RANGE_MIN_HI"/>
+	<reg32 offset="0x0e8d" name="UCHE_GMEM_RANGE_MAX_LO"/>
+	<reg32 offset="0x0e8e" name="UCHE_GMEM_RANGE_MAX_HI"/>
+	<reg32 offset="0x0e8f" name="UCHE_DBG_ECO_CNTL_2"/>
+	<reg32 offset="0x0e90" name="UCHE_DBG_ECO_CNTL"/>
+	<reg32 offset="0x0e91" name="UCHE_CACHE_INVALIDATE_MIN_LO"/>
+	<reg32 offset="0x0e92" name="UCHE_CACHE_INVALIDATE_MIN_HI"/>
+	<reg32 offset="0x0e93" name="UCHE_CACHE_INVALIDATE_MAX_LO"/>
+	<reg32 offset="0x0e94" name="UCHE_CACHE_INVALIDATE_MAX_HI"/>
+	<reg32 offset="0x0e95" name="UCHE_CACHE_INVALIDATE"/>
+	<reg32 offset="0x0e96" name="UCHE_CACHE_WAYS"/>
+	<reg32 offset="0x0ea0" name="UCHE_PERFCTR_UCHE_SEL_0" type="a5xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0ea1" name="UCHE_PERFCTR_UCHE_SEL_1" type="a5xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0ea2" name="UCHE_PERFCTR_UCHE_SEL_2" type="a5xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0ea3" name="UCHE_PERFCTR_UCHE_SEL_3" type="a5xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0ea4" name="UCHE_PERFCTR_UCHE_SEL_4" type="a5xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0ea5" name="UCHE_PERFCTR_UCHE_SEL_5" type="a5xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0ea6" name="UCHE_PERFCTR_UCHE_SEL_6" type="a5xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0ea7" name="UCHE_PERFCTR_UCHE_SEL_7" type="a5xx_uche_perfcounter_select"/>
+	<reg32 offset="0x0ea8" name="UCHE_POWERCTR_UCHE_SEL_0"/>
+	<reg32 offset="0x0ea9" name="UCHE_POWERCTR_UCHE_SEL_1"/>
+	<reg32 offset="0x0eaa" name="UCHE_POWERCTR_UCHE_SEL_2"/>
+	<reg32 offset="0x0eab" name="UCHE_POWERCTR_UCHE_SEL_3"/>
+	<reg32 offset="0x0eb1" name="UCHE_TRAP_LOG_LO"/>
+	<reg32 offset="0x0eb2" name="UCHE_TRAP_LOG_HI"/>
+
+	<reg32 offset="0x0ec0" name="SP_DBG_ECO_CNTL"/>
+	<reg32 offset="0x0ec1" name="SP_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0ec2" name="SP_MODE_CNTL"/> <!-- always 0000001e? -->
+	<reg32 offset="0x0ed0" name="SP_PERFCTR_SP_SEL_0"  type="a5xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ed1" name="SP_PERFCTR_SP_SEL_1"  type="a5xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ed2" name="SP_PERFCTR_SP_SEL_2"  type="a5xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ed3" name="SP_PERFCTR_SP_SEL_3"  type="a5xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ed4" name="SP_PERFCTR_SP_SEL_4"  type="a5xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ed5" name="SP_PERFCTR_SP_SEL_5"  type="a5xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ed6" name="SP_PERFCTR_SP_SEL_6"  type="a5xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ed7" name="SP_PERFCTR_SP_SEL_7"  type="a5xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ed8" name="SP_PERFCTR_SP_SEL_8"  type="a5xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0ed9" name="SP_PERFCTR_SP_SEL_9"  type="a5xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0eda" name="SP_PERFCTR_SP_SEL_10" type="a5xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0edb" name="SP_PERFCTR_SP_SEL_11" type="a5xx_sp_perfcounter_select"/>
+	<reg32 offset="0x0edc" name="SP_POWERCTR_SP_SEL_0"/>
+	<reg32 offset="0x0edd" name="SP_POWERCTR_SP_SEL_1"/>
+	<reg32 offset="0x0ede" name="SP_POWERCTR_SP_SEL_2"/>
+	<reg32 offset="0x0edf" name="SP_POWERCTR_SP_SEL_3"/>
+
+	<reg32 offset="0x0f01" name="TPL1_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0f02" name="TPL1_MODE_CNTL"/> <!-- always 00000544? -->
+	<reg32 offset="0x0f10" name="TPL1_PERFCTR_TP_SEL_0" type="a5xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f11" name="TPL1_PERFCTR_TP_SEL_1" type="a5xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f12" name="TPL1_PERFCTR_TP_SEL_2" type="a5xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f13" name="TPL1_PERFCTR_TP_SEL_3" type="a5xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f14" name="TPL1_PERFCTR_TP_SEL_4" type="a5xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f15" name="TPL1_PERFCTR_TP_SEL_5" type="a5xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f16" name="TPL1_PERFCTR_TP_SEL_6" type="a5xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f17" name="TPL1_PERFCTR_TP_SEL_7" type="a5xx_tp_perfcounter_select"/>
+	<reg32 offset="0x0f18" name="TPL1_POWERCTR_TP_SEL_0"/>
+	<reg32 offset="0x0f19" name="TPL1_POWERCTR_TP_SEL_1"/>
+	<reg32 offset="0x0f1a" name="TPL1_POWERCTR_TP_SEL_2"/>
+	<reg32 offset="0x0f1b" name="TPL1_POWERCTR_TP_SEL_3"/>
+
+	<reg32 offset="0x3000" name="VBIF_VERSION"/>
+	<reg32 offset="0x3001" name="VBIF_CLKON"/>
+<!--
+#define A5XX_VBIF_CLKON_FORCE_ON_TESTBUS_MASK   0x1
+#define A5XX_VBIF_CLKON_FORCE_ON_TESTBUS_SHIFT  0x1
+ -->
+	<reg32 offset="0x3028" name="VBIF_ABIT_SORT"/>
+	<reg32 offset="0x3029" name="VBIF_ABIT_SORT_CONF"/>
+	<reg32 offset="0x3049" name="VBIF_ROUND_ROBIN_QOS_ARB"/>
+	<reg32 offset="0x302a" name="VBIF_GATE_OFF_WRREQ_EN"/>
+	<reg32 offset="0x302c" name="VBIF_IN_RD_LIM_CONF0"/>
+	<reg32 offset="0x302d" name="VBIF_IN_RD_LIM_CONF1"/>
+	<reg32 offset="0x3080" name="VBIF_XIN_HALT_CTRL0"/>
+<!--
+#define A5XX_VBIF_XIN_HALT_CTRL0_MASK	   0xF
+#define A510_VBIF_XIN_HALT_CTRL0_MASK	   0x7
+ -->
+	<reg32 offset="0x3081" name="VBIF_XIN_HALT_CTRL1"/>
+	<reg32 offset="0x3084" name="VBIF_TEST_BUS_OUT_CTRL"/>
+<!--
+#define A5XX_VBIF_TEST_BUS_OUT_CTRL_EN_MASK    0x1
+#define A5XX_VBIF_TEST_BUS_OUT_CTRL_EN_SHIFT   0x0
+ -->
+	<reg32 offset="0x3085" name="VBIF_TEST_BUS1_CTRL0"/>
+	<reg32 offset="0x3086" name="VBIF_TEST_BUS1_CTRL1"/>
+<!--
+#define A5XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_MASK  0xF
+#define A5XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_SHIFT 0x0
+ -->
+	<reg32 offset="0x3087" name="VBIF_TEST_BUS2_CTRL0"/>
+	<reg32 offset="0x3088" name="VBIF_TEST_BUS2_CTRL1"/>
+<!--
+#define A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK     0xF
+#define A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT    0x0
+ -->
+	<reg32 offset="0x308c" name="VBIF_TEST_BUS_OUT"/>
+	<reg32 offset="0x30c0" name="VBIF_PERF_CNT_EN0"/>
+	<reg32 offset="0x30c1" name="VBIF_PERF_CNT_EN1"/>
+	<reg32 offset="0x30c2" name="VBIF_PERF_CNT_EN2"/>
+	<reg32 offset="0x30c3" name="VBIF_PERF_CNT_EN3"/>
+	<reg32 offset="0x30c8" name="VBIF_PERF_CNT_CLR0"/>
+	<reg32 offset="0x30c9" name="VBIF_PERF_CNT_CLR1"/>
+	<reg32 offset="0x30ca" name="VBIF_PERF_CNT_CLR2"/>
+	<reg32 offset="0x30cb" name="VBIF_PERF_CNT_CLR3"/>
+	<reg32 offset="0x30d0" name="VBIF_PERF_CNT_SEL0" type="a5xx_vbif_perfcounter_select"/>
+	<reg32 offset="0x30d1" name="VBIF_PERF_CNT_SEL1" type="a5xx_vbif_perfcounter_select"/>
+	<reg32 offset="0x30d2" name="VBIF_PERF_CNT_SEL2" type="a5xx_vbif_perfcounter_select"/>
+	<reg32 offset="0x30d3" name="VBIF_PERF_CNT_SEL3" type="a5xx_vbif_perfcounter_select"/>
+	<reg32 offset="0x30d8" name="VBIF_PERF_CNT_LOW0"/>
+	<reg32 offset="0x30d9" name="VBIF_PERF_CNT_LOW1"/>
+	<reg32 offset="0x30da" name="VBIF_PERF_CNT_LOW2"/>
+	<reg32 offset="0x30db" name="VBIF_PERF_CNT_LOW3"/>
+	<reg32 offset="0x30e0" name="VBIF_PERF_CNT_HIGH0"/>
+	<reg32 offset="0x30e1" name="VBIF_PERF_CNT_HIGH1"/>
+	<reg32 offset="0x30e2" name="VBIF_PERF_CNT_HIGH2"/>
+	<reg32 offset="0x30e3" name="VBIF_PERF_CNT_HIGH3"/>
+	<reg32 offset="0x3100" name="VBIF_PERF_PWR_CNT_EN0"/>
+	<reg32 offset="0x3101" name="VBIF_PERF_PWR_CNT_EN1"/>
+	<reg32 offset="0x3102" name="VBIF_PERF_PWR_CNT_EN2"/>
+	<reg32 offset="0x3110" name="VBIF_PERF_PWR_CNT_LOW0"/>
+	<reg32 offset="0x3111" name="VBIF_PERF_PWR_CNT_LOW1"/>
+	<reg32 offset="0x3112" name="VBIF_PERF_PWR_CNT_LOW2"/>
+	<reg32 offset="0x3118" name="VBIF_PERF_PWR_CNT_HIGH0"/>
+	<reg32 offset="0x3119" name="VBIF_PERF_PWR_CNT_HIGH1"/>
+	<reg32 offset="0x311a" name="VBIF_PERF_PWR_CNT_HIGH2"/>
+
+	<reg32 offset="0x8800" name="GPMU_INST_RAM_BASE"/>
+	<reg32 offset="0x9800" name="GPMU_DATA_RAM_BASE"/>
+
+<!--
+/* COUNTABLE FOR SP PERFCOUNTER */
+#define A5XX_SP_ALU_ACTIVE_CYCLES          0x1
+#define A5XX_SP0_ICL1_MISSES               0x35
+#define A5XX_SP_FS_CFLOW_INSTRUCTIONS      0x27
+
+/* COUNTABLE FOR TSE PERFCOUNTER */
+#define A5XX_TSE_INPUT_PRIM_NUM            0x6
+ -->
+	<reg32 offset="0xa840" name="SP_POWER_COUNTER_0_LO"/>
+	<reg32 offset="0xa841" name="SP_POWER_COUNTER_0_HI"/>
+	<reg32 offset="0xa842" name="SP_POWER_COUNTER_1_LO"/>
+	<reg32 offset="0xa843" name="SP_POWER_COUNTER_1_HI"/>
+	<reg32 offset="0xa844" name="SP_POWER_COUNTER_2_LO"/>
+	<reg32 offset="0xa845" name="SP_POWER_COUNTER_2_HI"/>
+	<reg32 offset="0xa846" name="SP_POWER_COUNTER_3_LO"/>
+	<reg32 offset="0xa847" name="SP_POWER_COUNTER_3_HI"/>
+	<reg32 offset="0xa848" name="TP_POWER_COUNTER_0_LO"/>
+	<reg32 offset="0xa849" name="TP_POWER_COUNTER_0_HI"/>
+	<reg32 offset="0xa84a" name="TP_POWER_COUNTER_1_LO"/>
+	<reg32 offset="0xa84b" name="TP_POWER_COUNTER_1_HI"/>
+	<reg32 offset="0xa84c" name="TP_POWER_COUNTER_2_LO"/>
+	<reg32 offset="0xa84d" name="TP_POWER_COUNTER_2_HI"/>
+	<reg32 offset="0xa84e" name="TP_POWER_COUNTER_3_LO"/>
+	<reg32 offset="0xa84f" name="TP_POWER_COUNTER_3_HI"/>
+	<reg32 offset="0xa850" name="RB_POWER_COUNTER_0_LO"/>
+	<reg32 offset="0xa851" name="RB_POWER_COUNTER_0_HI"/>
+	<reg32 offset="0xa852" name="RB_POWER_COUNTER_1_LO"/>
+	<reg32 offset="0xa853" name="RB_POWER_COUNTER_1_HI"/>
+	<reg32 offset="0xa854" name="RB_POWER_COUNTER_2_LO"/>
+	<reg32 offset="0xa855" name="RB_POWER_COUNTER_2_HI"/>
+	<reg32 offset="0xa856" name="RB_POWER_COUNTER_3_LO"/>
+	<reg32 offset="0xa857" name="RB_POWER_COUNTER_3_HI"/>
+	<reg32 offset="0xa858" name="CCU_POWER_COUNTER_0_LO"/>
+	<reg32 offset="0xa859" name="CCU_POWER_COUNTER_0_HI"/>
+	<reg32 offset="0xa85a" name="CCU_POWER_COUNTER_1_LO"/>
+	<reg32 offset="0xa85b" name="CCU_POWER_COUNTER_1_HI"/>
+	<reg32 offset="0xa85c" name="UCHE_POWER_COUNTER_0_LO"/>
+	<reg32 offset="0xa85d" name="UCHE_POWER_COUNTER_0_HI"/>
+	<reg32 offset="0xa85e" name="UCHE_POWER_COUNTER_1_LO"/>
+	<reg32 offset="0xa85f" name="UCHE_POWER_COUNTER_1_HI"/>
+	<reg32 offset="0xa860" name="UCHE_POWER_COUNTER_2_LO"/>
+	<reg32 offset="0xa861" name="UCHE_POWER_COUNTER_2_HI"/>
+	<reg32 offset="0xa862" name="UCHE_POWER_COUNTER_3_LO"/>
+	<reg32 offset="0xa863" name="UCHE_POWER_COUNTER_3_HI"/>
+	<reg32 offset="0xa864" name="CP_POWER_COUNTER_0_LO"/>
+	<reg32 offset="0xa865" name="CP_POWER_COUNTER_0_HI"/>
+	<reg32 offset="0xa866" name="CP_POWER_COUNTER_1_LO"/>
+	<reg32 offset="0xa867" name="CP_POWER_COUNTER_1_HI"/>
+	<reg32 offset="0xa868" name="CP_POWER_COUNTER_2_LO"/>
+	<reg32 offset="0xa869" name="CP_POWER_COUNTER_2_HI"/>
+	<reg32 offset="0xa86a" name="CP_POWER_COUNTER_3_LO"/>
+	<reg32 offset="0xa86b" name="CP_POWER_COUNTER_3_HI"/>
+	<reg32 offset="0xa86c" name="GPMU_POWER_COUNTER_0_LO"/>
+	<reg32 offset="0xa86d" name="GPMU_POWER_COUNTER_0_HI"/>
+	<reg32 offset="0xa86e" name="GPMU_POWER_COUNTER_1_LO"/>
+	<reg32 offset="0xa86f" name="GPMU_POWER_COUNTER_1_HI"/>
+	<reg32 offset="0xa870" name="GPMU_POWER_COUNTER_2_LO"/>
+	<reg32 offset="0xa871" name="GPMU_POWER_COUNTER_2_HI"/>
+	<reg32 offset="0xa872" name="GPMU_POWER_COUNTER_3_LO"/>
+	<reg32 offset="0xa873" name="GPMU_POWER_COUNTER_3_HI"/>
+	<reg32 offset="0xa874" name="GPMU_POWER_COUNTER_4_LO"/>
+	<reg32 offset="0xa875" name="GPMU_POWER_COUNTER_4_HI"/>
+	<reg32 offset="0xa876" name="GPMU_POWER_COUNTER_5_LO"/>
+	<reg32 offset="0xa877" name="GPMU_POWER_COUNTER_5_HI"/>
+	<reg32 offset="0xa878" name="GPMU_POWER_COUNTER_ENABLE"/>
+	<reg32 offset="0xa879" name="GPMU_ALWAYS_ON_COUNTER_LO"/>
+	<reg32 offset="0xa87a" name="GPMU_ALWAYS_ON_COUNTER_HI"/>
+	<reg32 offset="0xa87b" name="GPMU_ALWAYS_ON_COUNTER_RESET"/>
+	<reg32 offset="0xa87c" name="GPMU_POWER_COUNTER_SELECT_0"/>
+	<reg32 offset="0xa87d" name="GPMU_POWER_COUNTER_SELECT_1"/>
+
+	<reg32 offset="0xa880" name="GPMU_GPMU_SP_CLOCK_CONTROL"/>
+	<reg32 offset="0xa881" name="GPMU_SP_POWER_CNTL"/>
+	<reg32 offset="0xa886" name="GPMU_RBCCU_CLOCK_CNTL"/>
+	<reg32 offset="0xa887" name="GPMU_RBCCU_POWER_CNTL"/>
+	<reg32 offset="0xa88b" name="GPMU_SP_PWR_CLK_STATUS">
+		<bitfield name="PWR_ON" pos="20" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xa88d" name="GPMU_RBCCU_PWR_CLK_STATUS">
+		<bitfield name="PWR_ON" pos="20" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xa891" name="GPMU_PWR_COL_STAGGER_DELAY"/>
+	<reg32 offset="0xa892" name="GPMU_PWR_COL_INTER_FRAME_CTRL"/>
+	<reg32 offset="0xa893" name="GPMU_PWR_COL_INTER_FRAME_HYST"/>
+	<reg32 offset="0xa894" name="GPMU_PWR_COL_BINNING_CTRL"/>
+	<reg32 offset="0xa8a3" name="GPMU_CLOCK_THROTTLE_CTRL"/>
+	<reg32 offset="0xa8a8" name="GPMU_THROTTLE_UNMASK_FORCE_CTRL"/>
+	<reg32 offset="0xa8c1" name="GPMU_WFI_CONFIG"/>
+	<reg32 offset="0xa8d6" name="GPMU_RBBM_INTR_INFO"/>
+	<reg32 offset="0xa8d8" name="GPMU_CM3_SYSRESET"/>
+	<reg32 offset="0xa8e0" name="GPMU_GENERAL_0"/>
+	<reg32 offset="0xa8e1" name="GPMU_GENERAL_1"/>
+	<reg32 offset="0xac00" name="GPMU_TEMP_SENSOR_ID"/>
+	<reg32 offset="0xac01" name="GPMU_TEMP_SENSOR_CONFIG"/>
+	<reg32 offset="0xac02" name="GPMU_TEMP_VAL"/>
+	<reg32 offset="0xac03" name="GPMU_DELTA_TEMP_THRESHOLD"/>
+	<reg32 offset="0xac05" name="GPMU_TEMP_THRESHOLD_INTR_STATUS"/>
+	<reg32 offset="0xac06" name="GPMU_TEMP_THRESHOLD_INTR_EN_MASK"/>
+	<reg32 offset="0xac40" name="GPMU_LEAKAGE_TEMP_COEFF_0_1"/>
+	<reg32 offset="0xac41" name="GPMU_LEAKAGE_TEMP_COEFF_2_3"/>
+	<reg32 offset="0xac42" name="GPMU_LEAKAGE_VTG_COEFF_0_1"/>
+	<reg32 offset="0xac43" name="GPMU_LEAKAGE_VTG_COEFF_2_3"/>
+	<reg32 offset="0xac46" name="GPMU_BASE_LEAKAGE"/>
+	<reg32 offset="0xac60" name="GPMU_GPMU_VOLTAGE"/>
+	<reg32 offset="0xac61" name="GPMU_GPMU_VOLTAGE_INTR_STATUS"/>
+	<reg32 offset="0xac62" name="GPMU_GPMU_VOLTAGE_INTR_EN_MASK"/>
+	<reg32 offset="0xac80" name="GPMU_GPMU_PWR_THRESHOLD"/>
+	<reg32 offset="0xacc4" name="GPMU_GPMU_LLM_GLM_SLEEP_CTRL"/>
+	<reg32 offset="0xacc5" name="GPMU_GPMU_LLM_GLM_SLEEP_STATUS"/>
+	<reg32 offset="0xb80c" name="GDPM_CONFIG1"/>
+	<reg32 offset="0xb80d" name="GDPM_CONFIG2"/>
+	<reg32 offset="0xb80f" name="GDPM_INT_EN"/>
+	<reg32 offset="0xb811" name="GDPM_INT_MASK"/>
+	<reg32 offset="0xb9a0" name="GPMU_BEC_ENABLE"/>
+	<reg32 offset="0xc41a" name="GPU_CS_SENSOR_GENERAL_STATUS"/>
+	<reg32 offset="0xc41d" name="GPU_CS_AMP_CALIBRATION_STATUS1_0"/>
+	<reg32 offset="0xc41f" name="GPU_CS_AMP_CALIBRATION_STATUS1_2"/>
+	<reg32 offset="0xc421" name="GPU_CS_AMP_CALIBRATION_STATUS1_4"/>
+	<reg32 offset="0xc520" name="GPU_CS_ENABLE_REG"/>
+	<reg32 offset="0xc557" name="GPU_CS_AMP_CALIBRATION_CONTROL1"/>
+
+
+	<reg32 offset="0xe000" name="GRAS_CL_CNTL">
+		<bitfield name="ZERO_GB_SCALE_Z" pos="6" type="boolean"/>
+	</reg32>
+	<bitset name="a5xx_gras_xs_cl_cntl" inline="yes">
+		<bitfield name="CLIP_MASK" low="0" high="7"/>
+		<bitfield name="CULL_MASK" low="8" high="15"/>
+	</bitset>
+	<reg32 offset="0xe001" name="GRAS_VS_CL_CNTL" type="a5xx_gras_xs_cl_cntl"/>
+	<reg32 offset="0xe004" name="UNKNOWN_E004"/> <!-- always 00000000? -->
+	<reg32 offset="0xe005" name="GRAS_CNTL">
+		<!-- see also RB_RENDER_CONTROL0 -->
+		<bitfield name="IJ_PERSP_PIXEL" pos="0" type="boolean"/>
+		<bitfield name="IJ_PERSP_CENTROID" pos="1" type="boolean"/>
+		<bitfield name="IJ_PERSP_SAMPLE" pos="2" type="boolean"/>
+		<bitfield name="IJ_LINEAR_PIXEL" pos="3" type="boolean"/>
+		<bitfield name="IJ_LINEAR_CENTROID" pos="4" type="boolean"/>
+		<bitfield name="IJ_LINEAR_SAMPLE" pos="5" type="boolean"/>
+		<bitfield name="COORD_MASK" low="6" high="9" type="hex"/>
+	</reg32>
+	<reg32 offset="0xe006" name="GRAS_CL_GUARDBAND_CLIP_ADJ">
+		<bitfield name="HORZ" low="0" high="9" type="uint"/>
+		<bitfield name="VERT" low="10" high="19" type="uint"/>
+	</reg32>
+	<reg32 offset="0xe010" name="GRAS_CL_VPORT_XOFFSET_0" type="float"/>
+	<reg32 offset="0xe011" name="GRAS_CL_VPORT_XSCALE_0" type="float"/>
+	<reg32 offset="0xe012" name="GRAS_CL_VPORT_YOFFSET_0" type="float"/>
+	<reg32 offset="0xe013" name="GRAS_CL_VPORT_YSCALE_0" type="float"/>
+	<reg32 offset="0xe014" name="GRAS_CL_VPORT_ZOFFSET_0" type="float"/>
+	<reg32 offset="0xe015" name="GRAS_CL_VPORT_ZSCALE_0" type="float"/>
+	<reg32 offset="0xe090" name="GRAS_SU_CNTL">
+		<bitfield name="CULL_FRONT" pos="0" type="boolean"/>
+		<bitfield name="CULL_BACK" pos="1" type="boolean"/>
+		<bitfield name="FRONT_CW" pos="2" type="boolean"/>
+		<bitfield name="LINEHALFWIDTH" low="3" high="10" radix="2" type="fixed"/>
+		<bitfield name="POLY_OFFSET" pos="11" type="boolean"/>
+		<bitfield name="LINE_MODE" pos="13" type="a5xx_line_mode"/>
+		<!-- probably LINEHALFWIDTH is the same as a4xx.. -->
+	</reg32>
+	<reg32 offset="0xe091" name="GRAS_SU_POINT_MINMAX">
+		<bitfield name="MIN" low="0" high="15" type="ufixed" radix="4"/>
+		<bitfield name="MAX" low="16" high="31" type="ufixed" radix="4"/>
+	</reg32>
+	<reg32 offset="0xe092" name="GRAS_SU_POINT_SIZE" type="fixed" radix="4"/>
+	<reg32 offset="0xe093" name="GRAS_SU_LAYERED"/>
+	<reg32 offset="0xe094" name="GRAS_SU_DEPTH_PLANE_CNTL">
+		<bitfield name="FRAG_WRITES_Z" pos="0" type="boolean"/>
+		<bitfield name="UNK1" pos="1" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe095" name="GRAS_SU_POLY_OFFSET_SCALE" type="float"/>
+	<reg32 offset="0xe096" name="GRAS_SU_POLY_OFFSET_OFFSET" type="float"/>
+	<reg32 offset="0xe097" name="GRAS_SU_POLY_OFFSET_OFFSET_CLAMP" type="float"/>
+	<!-- duplicates RB_DEPTH_INFO0: -->
+	<reg32 offset="0xe098" name="GRAS_SU_DEPTH_BUFFER_INFO">
+		<bitfield name="DEPTH_FORMAT" low="0" high="2" type="a5xx_depth_format"/>
+	</reg32>
+	<reg32 offset="0xe099" name="GRAS_SU_CONSERVATIVE_RAS_CNTL"/> <!-- always 00000000? -->
+	<!--
+	guessing about window/screen/extent, I think they can in the end be
+	used interchangeably?
+	 -->
+	<reg32 offset="0xe0a0" name="GRAS_SC_CNTL">
+		<bitfield name="BINNING_PASS" pos="0" type="boolean"/>
+		<bitfield name="SAMPLES_PASSED" pos="15" type="boolean"/>
+	</reg32>
+	<!-- note, 0x4 for binning pass when frag writes z?? -->
+	<reg32 offset="0xe0a1" name="GRAS_SC_BIN_CNTL"/> <!-- always 00000000? -->
+	<reg32 offset="0xe0a2" name="GRAS_SC_RAS_MSAA_CNTL">
+		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>
+	</reg32>
+	<reg32 offset="0xe0a3" name="GRAS_SC_DEST_MSAA_CNTL">
+		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>
+		<bitfield name="MSAA_DISABLE" pos="2" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe0a4" name="GRAS_SC_SCREEN_SCISSOR_CNTL"/> <!-- always 00000000? -->
+	<reg32 offset="0xe0aa" name="GRAS_SC_SCREEN_SCISSOR_TL_0" type="adreno_reg_xy"/>
+	<reg32 offset="0xe0ab" name="GRAS_SC_SCREEN_SCISSOR_BR_0" type="adreno_reg_xy"/>
+	<reg32 offset="0xe0ca" name="GRAS_SC_VIEWPORT_SCISSOR_TL_0" type="adreno_reg_xy"/>
+	<reg32 offset="0xe0cb" name="GRAS_SC_VIEWPORT_SCISSOR_BR_0" type="adreno_reg_xy"/>
+	<reg32 offset="0xe0ea" name="GRAS_SC_WINDOW_SCISSOR_TL" type="adreno_reg_xy"/>
+	<reg32 offset="0xe0eb" name="GRAS_SC_WINDOW_SCISSOR_BR" type="adreno_reg_xy"/>
+
+	<doc>
+		LRZ:  (Low Resolution Z ??)
+		----
+
+		I think it serves two functions, early discard of primitives in binning
+		pass without needing full resolution depth buffer, and also functions as
+		a depth-prepass, used during the GMEM draws to discard primitives that
+		would not be visible due to later draws.
+
+		The LRZ buffer always seems to be z16 format, regardless of actual
+		depth buffer format.
+
+		Note that LRZ write should be disabled when blend/stencil/etc is enabled,
+		since the occluded primitive can still contribute to final color value
+		of a fragment.
+
+		Only enabled for GL_LESS/GL_LEQUAL/GL_GREATER/GL_GEQUAL?
+	</doc>
+	<reg32 offset="0xe100" name="GRAS_LRZ_CNTL">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+		<doc>LRZ write also disabled for blend/etc.</doc>
+		<bitfield name="LRZ_WRITE" pos="1" type="boolean"/>
+		<doc>update MAX instead of MIN value, ie. GL_GREATER/GL_GEQUAL</doc>
+		<bitfield name="GREATER" pos="2" type="boolean"/>
+		<!--
+		b3 set sometimes, when depth buffer isn't cleared.. maybe it
+		invalidates the LRZ buffer?  (Or just the covered positions?
+		 -->
+	</reg32>
+	<reg32 offset="0xe101" name="GRAS_LRZ_BUFFER_BASE_LO"/>
+	<reg32 offset="0xe102" name="GRAS_LRZ_BUFFER_BASE_HI"/>
+	<!--
+	lzr pitch is depth pitch (in pixels) / 8 (aligned to 32)..
+	 -->
+	<doc>
+		Pitch is depth width (in pixels) / 8 (aligned to 32).  Height
+		is also divided by 8 (ie. covers 8x8 pixels)
+	</doc>
+	<reg32 offset="0xe103" name="GRAS_LRZ_BUFFER_PITCH" shr="5" type="uint"/>
+	<reg32 offset="0xe104" name="GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_LO"/>
+	<reg32 offset="0xe105" name="GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_HI"/>
+
+	<reg32 offset="0xe140" name="RB_CNTL">
+		<bitfield name="WIDTH" low="0" high="7" shr="5" type="uint"/>
+		<bitfield name="HEIGHT" low="9" high="16" shr="5" type="uint"/>
+		<bitfield name="BYPASS" pos="17" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe141" name="RB_RENDER_CNTL">
+<!--
+bit 3 set for normal draws
+bit 7 for RECTLIST (clear) when z32s8 (used for clear of depth32?  not set
+	for z32 with no stencil, but maybe in that case separate z/s not used?
+	see mrt-fbo-* zs=2)
+ -->
+		<bitfield name="BINNING_PASS" pos="0" type="boolean"/>
+		<bitfield name="SAMPLES_PASSED" pos="6" type="boolean"/>
+		<bitfield name="DISABLE_COLOR_PIPE" pos="7" type="boolean"/>
+		<!-- why everything twice?? maybe read vs write? -->
+		<!-- UBWC flag buffer enabled for depth/stencil: -->
+		<bitfield name="FLAG_DEPTH" pos="14" type="boolean"/>
+		<bitfield name="FLAG_DEPTH2" pos="15" type="boolean"/>
+		<!-- bitmask of MRTs using UBWC flag buffer: -->
+		<bitfield name="FLAG_MRTS" low="16" high="23"/>
+		<bitfield name="FLAG_MRTS2" low="24" high="31"/>
+	</reg32>
+	<reg32 offset="0xe142" name="RB_RAS_MSAA_CNTL">
+		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>
+	</reg32>
+	<reg32 offset="0xe143" name="RB_DEST_MSAA_CNTL">
+		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>
+		<bitfield name="MSAA_DISABLE" pos="2" type="boolean"/>
+	</reg32>
+	<!--
+	note: maybe not actually called RB_RENDER_CONTROLn (since RB_RENDER_CNTL
+	name comes from kernel and is probably right)
+	 -->
+	<reg32 offset="0xe144" name="RB_RENDER_CONTROL0">
+		<!-- see also GRAS_CNTL -->
+		<bitfield name="IJ_PERSP_PIXEL" pos="0" type="boolean"/>
+		<bitfield name="IJ_PERSP_CENTROID" pos="1" type="boolean"/>
+		<bitfield name="IJ_PERSP_SAMPLE" pos="2" type="boolean"/>
+		<bitfield name="IJ_LINEAR_PIXEL" pos="3" type="boolean"/>
+		<bitfield name="IJ_LINEAR_CENTROID" pos="4" type="boolean"/>
+		<bitfield name="IJ_LINEAR_SAMPLE" pos="5" type="boolean"/>
+		<bitfield name="COORD_MASK" low="6" high="9" type="hex"/>
+	</reg32>
+	<reg32 offset="0xe145" name="RB_RENDER_CONTROL1">
+		<bitfield name="SAMPLEMASK" pos="0" type="boolean"/>
+		<bitfield name="FACENESS" pos="1" type="boolean"/>
+		<bitfield name="SAMPLEID" pos="2" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe146" name="RB_FS_OUTPUT_CNTL">
+		<!-- bit0 set except for binning pass.. -->
+		<bitfield name="MRT" low="0" high="3" type="uint"/>
+		<bitfield name="FRAG_WRITES_Z" pos="5" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe147" name="RB_RENDER_COMPONENTS">
+		<bitfield name="RT0" low="0" high="3"/>
+		<bitfield name="RT1" low="4" high="7"/>
+		<bitfield name="RT2" low="8" high="11"/>
+		<bitfield name="RT3" low="12" high="15"/>
+		<bitfield name="RT4" low="16" high="19"/>
+		<bitfield name="RT5" low="20" high="23"/>
+		<bitfield name="RT6" low="24" high="27"/>
+		<bitfield name="RT7" low="28" high="31"/>
+	</reg32>
+	<array offset="0xe150" name="RB_MRT" stride="7" length="8">
+		<reg32 offset="0x0" name="CONTROL">
+			<bitfield name="BLEND" pos="0" type="boolean"/>
+			<bitfield name="BLEND2" pos="1" type="boolean"/>
+			<bitfield name="ROP_ENABLE" pos="2" type="boolean"/>
+			<bitfield name="ROP_CODE" low="3" high="6" type="a3xx_rop_code"/>
+			<bitfield name="COMPONENT_ENABLE" low="7" high="10" type="hex"/>
+		</reg32>
+		<reg32 offset="0x1" name="BLEND_CONTROL">
+			<bitfield name="RGB_SRC_FACTOR" low="0" high="4" type="adreno_rb_blend_factor"/>
+			<bitfield name="RGB_BLEND_OPCODE" low="5" high="7" type="a3xx_rb_blend_opcode"/>
+			<bitfield name="RGB_DEST_FACTOR" low="8" high="12" type="adreno_rb_blend_factor"/>
+			<bitfield name="ALPHA_SRC_FACTOR" low="16" high="20" type="adreno_rb_blend_factor"/>
+			<bitfield name="ALPHA_BLEND_OPCODE" low="21" high="23" type="a3xx_rb_blend_opcode"/>
+			<bitfield name="ALPHA_DEST_FACTOR" low="24" high="28" type="adreno_rb_blend_factor"/>
+		</reg32>
+		<reg32 offset="0x2" name="BUF_INFO">
+			<!--
+			not sure if there is a separate COLOR_SWAP field like on a3xx/a4xx,
+			or if it is inherent in the format.  Will have to play with bits
+			once we get things working and see what happens.  If it is a diff
+			field, it doesn't seem to have the same encoding as a3xx/a4xx.
+			 -->
+			<bitfield name="COLOR_FORMAT" low="0" high="7" type="a5xx_color_fmt"/>
+			<bitfield name="COLOR_TILE_MODE" low="8" high="9" type="a5xx_tile_mode"/>
+			<bitfield name="DITHER_MODE" low="11" high="12" type="adreno_rb_dither_mode"/>
+			<bitfield name="COLOR_SWAP" low="13" high="14" type="a3xx_color_swap"/>
+			<bitfield name="COLOR_SRGB" pos="15" type="boolean"/>
+		</reg32>
+		<!--
+		at least in gmem, things seem to be aligned to pitch of 64..
+		maybe an artifact of tiled format used in gmem?
+		 -->
+		<reg32 offset="0x3" name="PITCH" shr="6" type="uint"/>
+		<reg32 offset="0x4" name="ARRAY_PITCH" shr="6" type="uint"/>
+		<reg32 offset="0x5" name="BASE_LO"/>
+		<reg32 offset="0x6" name="BASE_HI"/>
+	</array>
+	<reg32 offset="0xe1a0" name="RB_BLEND_RED">
+		<bitfield name="UINT" low="0" high="7" type="hex"/>
+		<bitfield name="SINT" low="8" high="15" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+	<reg32 offset="0xe1a1" name="RB_BLEND_RED_F32" type="float"/>
+	<reg32 offset="0xe1a2" name="RB_BLEND_GREEN">
+		<bitfield name="UINT" low="0" high="7" type="hex"/>
+		<bitfield name="SINT" low="8" high="15" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+	<reg32 offset="0xe1a3" name="RB_BLEND_GREEN_F32" type="float"/>
+	<reg32 offset="0xe1a4" name="RB_BLEND_BLUE">
+		<bitfield name="UINT" low="0" high="7" type="hex"/>
+		<bitfield name="SINT" low="8" high="15" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+	<reg32 offset="0xe1a5" name="RB_BLEND_BLUE_F32" type="float"/>
+	<reg32 offset="0xe1a6" name="RB_BLEND_ALPHA">
+		<bitfield name="UINT" low="0" high="7" type="hex"/>
+		<bitfield name="SINT" low="8" high="15" type="hex"/>
+		<bitfield name="FLOAT" low="16" high="31" type="float"/>
+	</reg32>
+	<reg32 offset="0xe1a7" name="RB_BLEND_ALPHA_F32" type="float"/>
+	<reg32 offset="0xe1a8" name="RB_ALPHA_CONTROL">
+		<bitfield name="ALPHA_REF" low="0" high="7" type="hex"/>
+		<bitfield name="ALPHA_TEST" pos="8" type="boolean"/>
+		<bitfield name="ALPHA_TEST_FUNC" low="9" high="11" type="adreno_compare_func"/>
+	</reg32>
+	<reg32 offset="0xe1a9" name="RB_BLEND_CNTL">
+		<!-- per-mrt enable bit -->
+		<bitfield name="ENABLE_BLEND" low="0" high="7"/>
+		<bitfield name="INDEPENDENT_BLEND" pos="8" type="boolean"/>
+		<bitfield name="ALPHA_TO_COVERAGE" pos="10" type="boolean"/>
+		<!-- a guess? -->
+		<bitfield name="SAMPLE_MASK" low="16" high="31"/>
+	</reg32>
+	<reg32 offset="0xe1b0" name="RB_DEPTH_PLANE_CNTL">
+		<bitfield name="FRAG_WRITES_Z" pos="0" type="boolean"/>
+		<bitfield name="UNK1" pos="1" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe1b1" name="RB_DEPTH_CNTL">
+		<bitfield name="Z_TEST_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="Z_WRITE_ENABLE" pos="1" type="boolean"/>
+		<bitfield name="ZFUNC" low="2" high="4" type="adreno_compare_func"/>
+		<doc>Z_READ_ENABLE bit is set for zfunc other than GL_ALWAYS or GL_NEVER</doc>
+		<bitfield name="Z_READ_ENABLE" pos="6" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe1b2" name="RB_DEPTH_BUFFER_INFO">
+		<bitfield name="DEPTH_FORMAT" low="0" high="2" type="a5xx_depth_format"/>
+	</reg32>
+	<reg32 offset="0xe1b3" name="RB_DEPTH_BUFFER_BASE_LO"/>
+	<reg32 offset="0xe1b4" name="RB_DEPTH_BUFFER_BASE_HI"/>
+	<reg32 offset="0xe1b5" name="RB_DEPTH_BUFFER_PITCH" shr="6" type="uint">
+		<doc>stride of depth/stencil buffer</doc>
+	</reg32>
+	<reg32 offset="0xe1b6" name="RB_DEPTH_BUFFER_ARRAY_PITCH" shr="6" type="uint">
+		<doc>size of layer</doc>
+	</reg32>
+	<reg32 offset="0xe1c0" name="RB_STENCIL_CONTROL">
+		<bitfield name="STENCIL_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="STENCIL_ENABLE_BF" pos="1" type="boolean"/>
+		<!--
+			set for stencil operations that require read from stencil
+			buffer, but not for example for stencil clear (which does
+			not require read).. so guessing this is analogous to
+			READ_DEST_ENABLE for color buffer..
+		 -->
+		<bitfield name="STENCIL_READ" pos="2" type="boolean"/>
+		<bitfield name="FUNC" low="8" high="10" type="adreno_compare_func"/>
+		<bitfield name="FAIL" low="11" high="13" type="adreno_stencil_op"/>
+		<bitfield name="ZPASS" low="14" high="16" type="adreno_stencil_op"/>
+		<bitfield name="ZFAIL" low="17" high="19" type="adreno_stencil_op"/>
+		<bitfield name="FUNC_BF" low="20" high="22" type="adreno_compare_func"/>
+		<bitfield name="FAIL_BF" low="23" high="25" type="adreno_stencil_op"/>
+		<bitfield name="ZPASS_BF" low="26" high="28" type="adreno_stencil_op"/>
+		<bitfield name="ZFAIL_BF" low="29" high="31" type="adreno_stencil_op"/>
+	</reg32>
+	<reg32 offset="0xe1c1" name="RB_STENCIL_INFO">
+		<bitfield name="SEPARATE_STENCIL" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe1c2" name="RB_STENCIL_BASE_LO"/>
+	<reg32 offset="0xe1c3" name="RB_STENCIL_BASE_HI"/>
+	<reg32 offset="0xe1c4" name="RB_STENCIL_PITCH" shr="6" type="uint"/>
+	<reg32 offset="0xe1c5" name="RB_STENCIL_ARRAY_PITCH" shr="6" type="uint"/>
+	<reg32 offset="0xe1c6" name="RB_STENCILREFMASK" type="adreno_rb_stencilrefmask"/>
+	<reg32 offset="0xe1c7" name="RB_STENCILREFMASK_BF" type="adreno_rb_stencilrefmask"/>
+	<reg32 offset="0xe1d0" name="RB_WINDOW_OFFSET" type="adreno_reg_xy"/>
+	<reg32 offset="0xe1d1" name="RB_SAMPLE_COUNT_CONTROL">
+		<bitfield name="COPY" pos="1" type="boolean"/>
+	</reg32>
+
+	<doc>
+		Blits:
+		------
+
+		Blits are triggered by CP_EVENT_WRITE:BLIT, compared to previous
+		generations where they shared most of the gl pipeline and were
+		triggered by CP_DRAW_INDX*
+
+		For gmem->mem blob uses RB_BLIT_CNTL.BUF to specify src of
+		blit (ie MRTn, ZS, etc) and RB_BLIT_DST_LO/HI for destination
+		gpuaddr.  The gmem offset is taken from RB_MRT[n].BASE_LO/HI
+
+		For mem->gmem blob uses just MRT0 or ZS and RB_BLIT_DST_LO/HI
+		for the GMEM offset, and gpuaddr from RB_MRT[0].BASE_LO/HI
+		(I suppose this is just to avoid trashing RB_MRT[1..7]??)
+	</doc>
+	<reg32 offset="0xe210" name="RB_BLIT_CNTL">
+		<bitfield name="BUF" low="0" high="3" type="a5xx_blit_buf"/>
+	</reg32>
+	<reg32 offset="0xe211" name="RB_RESOLVE_CNTL_1" type="adreno_reg_xy"/>
+	<reg32 offset="0xe212" name="RB_RESOLVE_CNTL_2" type="adreno_reg_xy"/>
+	<reg32 offset="0xe213" name="RB_RESOLVE_CNTL_3">
+		<!-- if b0 set, output is in TILE5_3 format -->
+		<bitfield name="TILED" pos="0" type="boolean"/>
+	<!--
+		0xe213:
+		        0x0 mem->gmem
+		        0xf gmem->mem with flag buffer (color)
+		        0x4 gmem->mem without flag buffer (color)
+		        0x7 BYPASS mode flag buffer result (ie. on readpix)
+		            also for gmem->mem preserving tiling
+	-->
+	</reg32>
+	<reg32 offset="0xe214" name="RB_BLIT_DST_LO"/>
+	<reg32 offset="0xe215" name="RB_BLIT_DST_HI"/>
+	<reg32 offset="0xe216" name="RB_BLIT_DST_PITCH" shr="6" type="uint"/>
+	<!-- array-pitch is size of layer -->
+	<reg32 offset="0xe217" name="RB_BLIT_DST_ARRAY_PITCH" shr="6" type="uint"/>
+	<reg32 offset="0xe218" name="RB_CLEAR_COLOR_DW0"/>
+	<reg32 offset="0xe219" name="RB_CLEAR_COLOR_DW1"/>
+	<reg32 offset="0xe21a" name="RB_CLEAR_COLOR_DW2"/>
+	<reg32 offset="0xe21b" name="RB_CLEAR_COLOR_DW3"/>
+	<reg32 offset="0xe21c" name="RB_CLEAR_CNTL">
+		<bitfield name="FAST_CLEAR" pos="1" type="boolean"/>
+		<bitfield name="MSAA_RESOLVE" pos="2" type="boolean"/>
+		<doc>
+			For MASK, if RB_BLIT_CNTL.BUF=BLIT_ZS:
+				1 - depth
+				2 - stencil
+				3 - depth+stencil
+			if RB_BLIT_CNTL.BUF=BLIT_MRTn
+				then probably a component mask, I always see 0xf
+		</doc>
+		<bitfield name="MASK" low="4" high="7"/>
+	</reg32>
+
+	<doc>
+		Buffer Metadata (flag buffers):
+		-------------------------------
+
+		Blob seems to stick some metadata at the front of the buffer,
+		both z/s and MRT.  I think this is same as UBWC (bandwidth
+		compression) metadata that mdp 1.7 and later supports.  See
+		1d3fae5698ce5358caab87a15383b690941697e8 in downstream kernel.
+		UBWC seems to stand for "universal bandwidth compression".
+
+		Before glReadPixels() it does a pair of BYPASS blits (at least
+		if metadata is used) presumably to resolve metadata.
+
+		NOTES: see: getUBwcBlockSize(), getUBwcMetaBufferSize() at
+		https://android.googlesource.com/platform/hardware/qcom/display/+/android-6.0.1_r40/msm8994/libgralloc/alloc_controller.cpp
+		(note that bpp in bytes, not bits, so really cpp)
+
+		Example Layout 2d w/ mipmap levels:
+
+			100x2000, ifmt=GL_RG, fmt=GL_RG16F, type=GL_FLOAT, meta=64x512@0x8000 (7x500)
+				base=c072e000, offset=16384, size=1703936
+
+				color		flags
+			0	c073a000	c0732000	- level 0 flags is address
+			1	c0838000	c0834000	  programmed in texture state
+			2	c0879000	c0877000
+			3	c089a000	c0899000
+			4	c08ab000	c08aa000
+			5	c08b4000	c08b3000
+			6	c08b9000	c08b8000
+			7	c08bc000	c08bb000
+			8	c08be000	c08bd000
+			9	c08c0000	c08bf000
+			10	c08c2000	c08c1000
+
+		ARRAY_PITCH is the combined size of all the levels plus flags,
+		so 0xc08c3000 - 0xc0732000 = 0x00191000 (1642496); each level
+		takes up a minimum of 2 pages (since color and flags parts are
+		each page aligned.
+
+			{ TILE_MODE = TILE5_3 | SWIZ_X = A5XX_TEX_X | SWIZ_Y = A5XX_TEX_Y | SWIZ_Z = A5XX_TEX_ZERO | SWIZ_W = A5XX_TEX_ONE | MIPLVLS = 0 | FMT = TFMT5_16_16_FLOAT | SWAP = WZYX }
+			{ WIDTH = 100 | HEIGHT = 2000 }
+			{ FETCHSIZE = TFETCH5_4_BYTE | PITCH = 512 | TYPE = A5XX_TEX_2D }
+			{ ARRAY_PITCH = 1642496 | 0x18800000 }  - NOTE c2dc always has 0x18800000 but
+			{ BASE_LO = 0xc0732000 }                  this varies for blob gles driver..
+			{ BASE_HI = 0 | DEPTH = 1 }               not sure what it is
+
+
+	</doc>
+	<reg32 offset="0xe240" name="RB_DEPTH_FLAG_BUFFER_BASE_LO"/>
+	<reg32 offset="0xe241" name="RB_DEPTH_FLAG_BUFFER_BASE_HI"/>
+	<reg32 offset="0xe242" name="RB_DEPTH_FLAG_BUFFER_PITCH">
+	</reg32>
+	<array offset="0xe243" name="RB_MRT_FLAG_BUFFER" stride="4" length="8">
+		<reg32 offset="0" name="ADDR_LO"/>
+		<reg32 offset="1" name="ADDR_HI"/>
+		<reg32 offset="2" name="PITCH" shr="6" type="uint"/>
+		<!-- array-pitch is size of layer -->
+		<reg32 offset="3" name="ARRAY_PITCH" shr="6" type="uint"/>
+	</array>
+	<reg32 offset="0xe263" name="RB_BLIT_FLAG_DST_LO"/>
+	<reg32 offset="0xe264" name="RB_BLIT_FLAG_DST_HI"/>
+	<reg32 offset="0xe265" name="RB_BLIT_FLAG_DST_PITCH" shr="6" type="uint"/>
+	<!-- array-pitch is size of layer -->
+	<reg32 offset="0xe266" name="RB_BLIT_FLAG_DST_ARRAY_PITCH" shr="6" type="uint"/>
+
+	<reg32 offset="0xe267" name="RB_SAMPLE_COUNT_ADDR_LO"/>
+	<reg32 offset="0xe268" name="RB_SAMPLE_COUNT_ADDR_HI"/>
+
+	<reg32 offset="0xe280" name="VPC_CNTL_0">
+		<doc>
+			num of varyings plus four for gl_Position (plus one if gl_PointSize)
+			plus # of transform-feedback (streamout) varyings if using the
+			hw streamout (rather than stg instructions in shader)
+		</doc>
+		<bitfield name="STRIDE_IN_VPC" low="0" high="6" type="uint"/>
+		<bitfield name="VARYING" pos="11" type="boolean"/>
+	</reg32>
+	<array offset="0xe282" name="VPC_VARYING_INTERP" stride="1" length="8">
+		<reg32 offset="0x0" name="MODE"/>
+	</array>
+	<array offset="0xe28a" name="VPC_VARYING_PS_REPL" stride="1" length="8">
+		<reg32 offset="0x0" name="MODE"/>
+	</array>
+	<reg32 offset="0xe292" name="UNKNOWN_E292"/>
+	<reg32 offset="0xe293" name="UNKNOWN_E293"/>
+	<array offset="0xe294" name="VPC_VAR" stride="1" length="4">
+		<!-- one bit per varying component: -->
+		<reg32 offset="0" name="DISABLE"/>
+	</array>
+	<reg32 offset="0xe298" name="VPC_GS_SIV_CNTL"/>
+	<reg32 offset="0xe29a" name="VPC_CLIP_CNTL">
+		<bitfield name="CLIP_MASK" low="0" high="7" type="uint"/>
+		<!-- there can be up to 8 total clip/cull distance outputs,
+		     but apparenly VPC can only deal with vec4, so when there are
+		     more than 4 outputs a second location needs to be programmed
+		-->
+		<bitfield name="CLIP_DIST_03_LOC" low="8" high="15" type="uint"/>
+		<bitfield name="CLIP_DIST_47_LOC" low="16" high="23" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0xe29d" name="VPC_PACK">
+		<bitfield name="NUMNONPOSVAR" low="0" high="7" type="uint"/>
+		<!--
+		This seems to be the OUTLOC for the psize output.  It could possibly
+		be the max-OUTLOC position, but it is only set when VS writes psize
+		(and blob always puts psize at highest OUTLOC)
+		 -->
+		<bitfield name="PSIZELOC" low="8" high="15" type="uint"/>
+	</reg32>
+	<reg32 offset="0xe2a0" name="VPC_FS_PRIMITIVEID_CNTL"/>
+
+	<doc>
+		Stream-Out:
+		-----------
+
+		VPC_SO[0..3] registers setup details about streamout buffers, and
+		number of components to write to each.
+
+		VPC_SO_PROG provides the mapping between output varyings and the SO
+		buffers.  It is written multiple times (via a CP_CONTEXT_REG_BUNCH
+		packet, not sure if that matters), each write can handle up to two
+		components of stream-out output.  Order matches up to OUTLOC,
+		including padding.  So, if outputting first 3 varyings:
+
+			SP_VS_OUT[0].REG: { A_REGID = r0.w | A_COMPMASK = 0xf | B_REGID = r0.x | B_COMPMASK = 0x7 }
+			SP_VS_OUT[0x1].REG: { A_REGID = r1.w | A_COMPMASK = 0x3 | B_REGID = r2.y | B_COMPMASK = 0xf }
+			SP_VS_VPC_DST[0].REG: { OUTLOC0 = 0 | OUTLOC1 = 4 | OUTLOC2 = 8 | OUTLOC3 = 12 }
+
+		Then:
+
+			VPC_SO_PROG: { A_BUF = 0 | A_OFF = 0 | A_EN | A_BUF = 0 | B_OFF = 4 | B_EN }
+			VPC_SO_PROG: { A_BUF = 0 | A_OFF = 8 | A_EN | A_BUF = 0 | B_OFF = 12 | B_EN }
+			VPC_SO_PROG: { A_BUF = 2 | A_OFF = 0 | A_EN | A_BUF = 2 | B_OFF = 4 | B_EN }
+			VPC_SO_PROG: { A_BUF = 2 | A_OFF = 8 | A_EN | A_BUF = 0 | B_OFF = 0 }
+			VPC_SO_PROG: { A_BUF = 1 | A_OFF = 0 | A_EN | A_BUF = 1 | B_OFF = 4 | B_EN }
+
+		Note that varying order is OUTLOC0, OUTLOC2, OUTLOC1, and note
+		the padding between OUTLOC1 and OUTLOC2.
+
+		The BUF bitfield indicates which of the four streamout buffers
+		to write into at the specified offset.
+
+		The VPC_SO[n].FLUSH_BASE_LO/HI is used for hw to write back next
+		offset which gets loaded back into VPC_SO[n].BUFFER_OFFSET via a
+		CP_MEM_TO_REG.  Probably can be ignored until we have GS/etc, at
+		which point we can't calculate the offset on the CPU.
+	</doc>
+	<reg32 offset="0xe2a1" name="VPC_SO_BUF_CNTL">
+		<bitfield name="BUF0" pos="0" type="boolean"/>
+		<bitfield name="BUF1" pos="3" type="boolean"/>
+		<bitfield name="BUF2" pos="6" type="boolean"/>
+		<bitfield name="BUF3" pos="9" type="boolean"/>
+		<bitfield name="ENABLE" pos="15" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe2a2" name="VPC_SO_OVERRIDE">
+		<bitfield name="SO_DISABLE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe2a3" name="VPC_SO_CNTL">
+		<!-- always 0x10000 when SO enabled.. -->
+		<bitfield name="ENABLE" pos="16" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe2a4" name="VPC_SO_PROG">
+		<bitfield name="A_BUF" low="0" high="1" type="uint"/>
+		<bitfield name="A_OFF" low="2" high="10" shr="2" type="uint"/>
+		<bitfield name="A_EN" pos="11" type="boolean"/>
+		<bitfield name="B_BUF" low="12" high="13" type="uint"/>
+		<bitfield name="B_OFF" low="14" high="22" shr="2" type="uint"/>
+		<bitfield name="B_EN" pos="23" type="boolean"/>
+	</reg32>
+	<array offset="0xe2a7" name="VPC_SO" stride="7" length="4">
+		<reg32 offset="0" name="BUFFER_BASE_LO"/>
+		<reg32 offset="1" name="BUFFER_BASE_HI"/>
+		<reg32 offset="2" name="BUFFER_SIZE"/>
+		<reg32 offset="3" name="NCOMP"/>  <!-- component count -->
+		<reg32 offset="4" name="BUFFER_OFFSET"/>
+		<reg32 offset="5" name="FLUSH_BASE_LO"/>
+		<reg32 offset="6" name="FLUSH_BASE_HI"/>
+	</array>
+
+	<reg32 offset="0xe384" name="PC_PRIMITIVE_CNTL">
+		<!-- # of varyings plus four for gl_Position (plus one if gl_PointSize) -->
+		<bitfield name="STRIDE_IN_VPC" low="0" high="6" type="uint"/>
+		<bitfield name="PRIMITIVE_RESTART" pos="8" type="boolean"/>
+		<bitfield name="COUNT_PRIMITIVES" pos="9" type="boolean"/><!-- enabled when gl_PrimitiveIDIn is used -->
+		<bitfield name="PROVOKING_VTX_LAST" pos="10" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe385" name="PC_PRIM_VTX_CNTL">
+		<bitfield name="PSIZE" pos="11" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe388" name="PC_RASTER_CNTL">
+		<bitfield name="POLYMODE_FRONT_PTYPE" low="0" high="2" type="adreno_pa_su_sc_draw"/>
+		<bitfield name="POLYMODE_BACK_PTYPE" low="3" high="5" type="adreno_pa_su_sc_draw"/>
+		<bitfield name="POLYMODE_ENABLE" pos="6" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe389" name="PC_CLIP_CNTL">
+		<bitfield name="CLIP_MASK" low="0" high="7"/>
+	</reg32>
+	<reg32 offset="0xe38c" name="PC_RESTART_INDEX"/>
+	<reg32 offset="0xe38d" name="PC_GS_LAYERED"/>
+	<reg32 offset="0xe38e" name="PC_GS_PARAM">
+		<bitfield name="MAX_VERTICES" low="0" high="9" type="uint"/><!-- vertices - 1 -->
+		<bitfield name="INVOCATIONS" low="11" high="15" type="uint"/><!-- invoc - 1 -->
+		<bitfield name="PRIMTYPE" low="23" high="24" type="adreno_pa_su_sc_draw"/>
+	</reg32>
+	<reg32 offset="0xe38f" name="PC_HS_PARAM">
+		<bitfield name="VERTICES_OUT" low="0" high="5" type="uint"/>
+		<bitfield name="SPACING" low="21" high="22" type="a4xx_tess_spacing"/>
+		<bitfield name="CW" pos="23" type="boolean"/>
+		<bitfield name="CONNECTED" pos="24" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe3b0" name="PC_POWER_CNTL"/>
+
+	<reg32 offset="0xe400" name="VFD_CONTROL_0">
+		<bitfield name="VTXCNT" low="0" high="5" type="uint"/>
+	</reg32>
+	<reg32 offset="0xe401" name="VFD_CONTROL_1">
+		<bitfield name="REGID4VTX" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="REGID4INST" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="REGID4PRIMID" low="16" high="23" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xe402" name="VFD_CONTROL_2">
+		<bitfield name="REGID_PATCHID" low="0" high="7" type="a3xx_regid"/><!-- same as VFD_CONTROL_3.REGID_PATCHID? -->
+	</reg32>
+	<reg32 offset="0xe403" name="VFD_CONTROL_3">
+		<bitfield name="REGID_PATCHID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="REGID_TESSX" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="REGID_TESSY" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xe404" name="VFD_CONTROL_4">
+	</reg32>
+	<reg32 offset="0xe405" name="VFD_CONTROL_5">
+		<!-- b0 set if gl_PrimitiveID used in fs ?? -->
+	</reg32>
+	<reg32 offset="0xe408" name="VFD_INDEX_OFFSET"/>
+	<reg32 offset="0xe409" name="VFD_INSTANCE_START_OFFSET"/>
+	<array offset="0xe40a" name="VFD_FETCH" stride="4" length="32">
+		<reg32 offset="0x0" name="BASE_LO"/>
+		<reg32 offset="0x1" name="BASE_HI"/>
+		<reg32 offset="0x2" name="SIZE" type="uint"/>
+		<reg32 offset="0x3" name="STRIDE" type="uint"/>
+	</array>
+	<array offset="0xe48a" name="VFD_DECODE" stride="2" length="32">
+		<reg32 offset="0x0" name="INSTR">
+			<!-- IDX appears to index into VFD_FETCH[] -->
+			<bitfield name="IDX" low="0" high="4" type="uint"/>
+			<bitfield name="INSTANCED" pos="17" type="boolean"/>
+			<bitfield name="FORMAT" low="20" high="27" type="a5xx_vtx_fmt"/>
+			<bitfield name="SWAP" low="28" high="29" type="a3xx_color_swap"/>
+			<bitfield name="UNK30" pos="30" type="boolean"/>
+			<bitfield name="FLOAT" pos="31" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x1" name="STEP_RATE"/> <!-- ??? -->
+	</array>
+	<array offset="0xe4ca" name="VFD_DEST_CNTL" stride="1" length="32">
+		<reg32 offset="0x0" name="INSTR">
+			<bitfield name="WRITEMASK" low="0" high="3" type="hex"/>
+			<bitfield name="REGID" low="4" high="11" type="a3xx_regid"/>
+		</reg32>
+	</array>
+	<reg32 offset="0xe4f0" name="VFD_POWER_CNTL"/>
+
+	<!-- 0x0 for compute, 0x10 for 3d? -->
+	<reg32 offset="0xe580" name="SP_SP_CNTL"/>
+
+	<bitset name="a5xx_xs_config" inline="yes">
+		<bitfield name="ENABLED" pos="0" type="boolean"/>
+		<bitfield name="CONSTOBJECTOFFSET" low="1" high="7" type="uint"/>
+		<bitfield name="SHADEROBJOFFSET" low="8" high="14" type="uint"/>
+	</bitset>
+	<bitset name="a5xx_xs_cntl" inline="yes">
+		<bitfield name="SSBO_ENABLE" pos="0" type="boolean"/>
+		<!--
+		no idea high bit.. could be this is amount of on-chip memory used
+		rather than total size?
+		 -->
+		<bitfield name="INSTRLEN" low="1" high="31" type="uint"/>
+	</bitset>
+	<bitset name="a5xx_sp_xs_ctrl_reg0" inline="yes">
+		<!-- bit1 almost always set -->
+		<!-- set for "buffer mode" (ie. shader small enough to fit internally) -->
+		<bitfield name="BUFFER" pos="2" type="boolean"/>
+		<!-- 24 or more (full size) GPRS and blob uses TWO_QUADS instead of FOUR_QUADS -->
+		<bitfield name="THREADSIZE" pos="3" type="a3xx_threadsize"/>
+		<bitfield name="HALFREGFOOTPRINT" low="4" high="9" type="uint"/>
+		<bitfield name="FULLREGFOOTPRINT" low="10" high="15" type="uint"/>
+		<bitfield name="VARYING" pos="16" type="boolean"/>
+		<bitfield name="PIXLODENABLE" pos="20" type="boolean"/>
+		<!-- seems to be nesting level for flow control:.. -->
+		<bitfield name="BRANCHSTACK" low="25" high="31" type="uint"/>
+	</bitset>
+	<!-- assuming things appear in same relative order as a4xx: -->
+	<!-- duplicated exactly w/ corresponding HLSQ_ regs starting at 0xe78b.. -->
+	<reg32 offset="0xe584" name="SP_VS_CONFIG" type="a5xx_xs_config"/>
+	<reg32 offset="0xe585" name="SP_FS_CONFIG" type="a5xx_xs_config"/>
+	<reg32 offset="0xe586" name="SP_HS_CONFIG" type="a5xx_xs_config"/>
+	<reg32 offset="0xe587" name="SP_DS_CONFIG" type="a5xx_xs_config"/>
+	<reg32 offset="0xe588" name="SP_GS_CONFIG" type="a5xx_xs_config"/>
+	<reg32 offset="0xe589" name="SP_CS_CONFIG" type="a5xx_xs_config"/>
+	<reg32 offset="0xe58a" name="SP_VS_CONFIG_MAX_CONST"/>
+	<reg32 offset="0xe58b" name="SP_FS_CONFIG_MAX_CONST"/>
+	<reg32 offset="0xe590" name="SP_VS_CTRL_REG0" type="a5xx_sp_xs_ctrl_reg0"/>
+	<reg32 offset="0xe592" name="SP_PRIMITIVE_CNTL">
+		<!-- # of VS outputs including pos/psize -->
+		<bitfield name="VSOUT" low="0" high="4" type="uint"/>
+	</reg32>
+	<array offset="0xe593" name="SP_VS_OUT" stride="1" length="16">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="A_REGID" low="0" high="7" type="a3xx_regid"/>
+			<bitfield name="A_COMPMASK" low="8" high="11" type="hex"/>
+			<bitfield name="B_REGID" low="16" high="23" type="a3xx_regid"/>
+			<bitfield name="B_COMPMASK" low="24" high="27" type="hex"/>
+		</reg32>
+	</array>
+	<!--
+	Starting with a5xx, position/psize outputs from shader end up in the
+	SP_VS_OUT map, with highest OUTLOCn position.  (Generally they are
+	the last entries too, except when gl_PointCoord is used, blob inserts
+	an extra varying after, but with a lower OUTLOC position.  If present,
+	psize is last, preceded by position.
+	 -->
+	<array offset="0xe5a3" name="SP_VS_VPC_DST" stride="1" length="8">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="OUTLOC0" low="0" high="7" type="uint"/>
+			<bitfield name="OUTLOC1" low="8" high="15" type="uint"/>
+			<bitfield name="OUTLOC2" low="16" high="23" type="uint"/>
+			<bitfield name="OUTLOC3" low="24" high="31" type="uint"/>
+		</reg32>
+	</array>
+	<reg32 offset="0xe5ab" name="UNKNOWN_E5AB"/>
+	<reg32 offset="0xe5ac" name="SP_VS_OBJ_START_LO"/>
+	<reg32 offset="0xe5ad" name="SP_VS_OBJ_START_HI"/>
+
+	<bitset name="a5xx_sp_xs_pvt_mem_param" inline="yes">
+		<bitfield name="MEMSIZEPERITEM" low="0" high="7" shr="9">
+			<doc>The size of memory that ldp/stp can address.</doc>
+		</bitfield>
+		<bitfield name="HWSTACKOFFSET" low="8" high="23" shr="11" type="uint"/>
+		<bitfield name="HWSTACKSIZEPERTHREAD" low="24" high="31">
+                        <doc>Guessing that this is the same as a3xx/a6xx.</doc>
+		</bitfield>
+	</bitset>
+
+	<bitset name="a5xx_sp_xs_pvt_mem_size" inline="yes">
+		<bitfield name="TOTALPVTMEMSIZE" low="0" high="17" shr="12"/>
+	</bitset>
+
+	<reg32 offset="0xe5ae" name="SP_VS_PVT_MEM_PARAM" type="a5xx_sp_xs_pvt_mem_param"/>
+	<reg64 offset="0xe5af" name="SP_VS_PVT_MEM_ADDR" type="waddress" align="32"/>
+	<reg32 offset="0xe5b1" name="SP_VS_PVT_MEM_SIZE" type="a5xx_sp_xs_pvt_mem_size"/>
+	<reg32 offset="0xe5c0" name="SP_FS_CTRL_REG0" type="a5xx_sp_xs_ctrl_reg0"/>
+	<reg32 offset="0xe5c2" name="UNKNOWN_E5C2"/>
+	<reg32 offset="0xe5c3" name="SP_FS_OBJ_START_LO"/>
+	<reg32 offset="0xe5c4" name="SP_FS_OBJ_START_HI"/>
+	<reg32 offset="0xe5c5" name="SP_FS_PVT_MEM_PARAM" type="a5xx_sp_xs_pvt_mem_param"/>
+	<reg64 offset="0xe5c6" name="SP_FS_PVT_MEM_ADDR" type="waddress" align="32"/>
+	<reg32 offset="0xe5c8" name="SP_FS_PVT_MEM_SIZE" type="a5xx_sp_xs_pvt_mem_size"/>
+	<reg32 offset="0xe5c9" name="SP_BLEND_CNTL">
+		<!-- per-mrt enable bit -->
+		<bitfield name="ENABLE_BLEND" low="0" high="7"/>
+		<bitfield name="UNK8" pos="8" type="boolean"/>
+		<bitfield name="ALPHA_TO_COVERAGE" pos="10" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xe5ca" name="SP_FS_OUTPUT_CNTL">
+		<bitfield name="MRT" low="0" high="3" type="uint"/>
+		<bitfield name="DEPTH_REGID" low="5" high="12" type="a3xx_regid"/>
+		<bitfield name="SAMPLEMASK_REGID" low="13" high="20" type="a3xx_regid"/>
+	</reg32>
+	<array offset="0xe5cb" name="SP_FS_OUTPUT" stride="1" length="8">
+		<doc>per MRT</doc>
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="REGID" low="0" high="7" type="a3xx_regid"/>
+			<bitfield name="HALF_PRECISION" pos="8" type="boolean"/>
+		</reg32>
+	</array>
+	<array offset="0xe5d3" name="SP_FS_MRT" stride="1" length="8">
+		<reg32 offset="0" name="REG">
+			<bitfield name="COLOR_FORMAT" low="0" high="7" type="a5xx_color_fmt"/>
+			<bitfield name="COLOR_SINT" pos="8" type="boolean"/>
+			<bitfield name="COLOR_UINT" pos="9" type="boolean"/>
+			<bitfield name="COLOR_SRGB" pos="10" type="boolean"/>
+		</reg32>
+	</array>
+	<!--
+	e5db/e5dc seems to look related to some optimization to do sample from
+	texture using varying value directly before shader thread starts?  I
+	guess that could optimize common simple frag shaders..
+	 -->
+	<reg32 offset="0xe5db" name="UNKNOWN_E5DB"/>
+	<reg32 offset="0xe5f0" name="SP_CS_CTRL_REG0" type="a5xx_sp_xs_ctrl_reg0"/>
+	<reg32 offset="0xe5f2" name="UNKNOWN_E5F2"/>
+	<reg32 offset="0xe5f3" name="SP_CS_OBJ_START_LO"/>
+	<reg32 offset="0xe5f4" name="SP_CS_OBJ_START_HI"/>
+	<reg32 offset="0xe5f5" name="SP_CS_PVT_MEM_PARAM" type="a5xx_sp_xs_pvt_mem_param"/>
+	<reg64 offset="0xe5f6" name="SP_CS_PVT_MEM_ADDR" type="waddress" align="32"/>
+	<reg32 offset="0xe5f8" name="SP_CS_PVT_MEM_SIZE" type="a5xx_sp_xs_pvt_mem_size"/>
+
+	<!-- e5f9 something compute related.. seems to change when HLSQ_CS_CNTL_1 changes -->
+
+	<reg32 offset="0xe600" name="SP_HS_CTRL_REG0" type="a5xx_sp_xs_ctrl_reg0"/>
+	<reg32 offset="0xe602" name="UNKNOWN_E602"/>
+	<reg32 offset="0xe603" name="SP_HS_OBJ_START_LO"/>
+	<reg32 offset="0xe604" name="SP_HS_OBJ_START_HI"/>
+	<reg32 offset="0xe605" name="SP_HS_PVT_MEM_PARAM" type="a5xx_sp_xs_pvt_mem_param"/>
+	<reg64 offset="0xe606" name="SP_HS_PVT_MEM_ADDR" type="waddress" align="32"/>
+	<reg32 offset="0xe608" name="SP_HS_PVT_MEM_SIZE" type="a5xx_sp_xs_pvt_mem_size"/>
+	<reg32 offset="0xe610" name="SP_DS_CTRL_REG0" type="a5xx_sp_xs_ctrl_reg0"/>
+	<reg32 offset="0xe62b" name="UNKNOWN_E62B"/>
+	<reg32 offset="0xe62c" name="SP_DS_OBJ_START_LO"/>
+	<reg32 offset="0xe62d" name="SP_DS_OBJ_START_HI"/>
+	<reg32 offset="0xe62e" name="SP_DS_PVT_MEM_PARAM" type="a5xx_sp_xs_pvt_mem_param"/>
+	<reg64 offset="0xe62f" name="SP_DS_PVT_MEM_ADDR" type="waddress" align="32"/>
+	<reg32 offset="0xe631" name="SP_DS_PVT_MEM_SIZE" type="a5xx_sp_xs_pvt_mem_size"/>
+	<reg32 offset="0xe640" name="SP_GS_CTRL_REG0" type="a5xx_sp_xs_ctrl_reg0"/>
+	<reg32 offset="0xe65b" name="UNKNOWN_E65B"/>
+	<reg32 offset="0xe65c" name="SP_GS_OBJ_START_LO"/>
+	<reg32 offset="0xe65d" name="SP_GS_OBJ_START_HI"/>
+	<reg32 offset="0xe65e" name="SP_GS_PVT_MEM_PARAM" type="a5xx_sp_xs_pvt_mem_param"/>
+	<reg64 offset="0xe65f" name="SP_GS_PVT_MEM_ADDR" type="waddress" align="32"/>
+	<reg32 offset="0xe661" name="SP_GS_PVT_MEM_SIZE" type="a5xx_sp_xs_pvt_mem_size"/>
+
+	<reg32 offset="0xe704" name="TPL1_TP_RAS_MSAA_CNTL">
+		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>
+	</reg32>
+	<reg32 offset="0xe705" name="TPL1_TP_DEST_MSAA_CNTL">
+		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>
+		<bitfield name="MSAA_DISABLE" pos="2" type="boolean"/>
+	</reg32>
+	<!-- either blob is doing it wrong, or this is not per-stage anymore: -->
+	<reg32 offset="0xe706" name="TPL1_TP_BORDER_COLOR_BASE_ADDR_LO"/>
+	<reg32 offset="0xe707" name="TPL1_TP_BORDER_COLOR_BASE_ADDR_HI"/>
+
+	<!--
+	so these have the same info that is normally in the CP_LOAD_STATE
+	packets.. not sure if they are normally written by pm4/me or if the
+	CP_LOAD_STATE mechanism is deprecated?
+	 -->
+	<reg32 offset="0xe700" name="TPL1_VS_TEX_COUNT" type="uint"/>
+	<reg32 offset="0xe701" name="TPL1_HS_TEX_COUNT" type="uint"/>
+	<reg32 offset="0xe702" name="TPL1_DS_TEX_COUNT" type="uint"/>
+	<reg32 offset="0xe703" name="TPL1_GS_TEX_COUNT" type="uint"/>
+
+	<reg32 offset="0xe722" name="TPL1_VS_TEX_SAMP_LO"/>
+	<reg32 offset="0xe723" name="TPL1_VS_TEX_SAMP_HI"/>
+	<reg32 offset="0xe724" name="TPL1_HS_TEX_SAMP_LO"/>
+	<reg32 offset="0xe725" name="TPL1_HS_TEX_SAMP_HI"/>
+	<reg32 offset="0xe726" name="TPL1_DS_TEX_SAMP_LO"/>
+	<reg32 offset="0xe727" name="TPL1_DS_TEX_SAMP_HI"/>
+	<reg32 offset="0xe728" name="TPL1_GS_TEX_SAMP_LO"/>
+	<reg32 offset="0xe729" name="TPL1_GS_TEX_SAMP_HI"/>
+
+	<reg32 offset="0xe72a" name="TPL1_VS_TEX_CONST_LO"/>
+	<reg32 offset="0xe72b" name="TPL1_VS_TEX_CONST_HI"/>
+	<reg32 offset="0xe72c" name="TPL1_HS_TEX_CONST_LO"/>
+	<reg32 offset="0xe72d" name="TPL1_HS_TEX_CONST_HI"/>
+	<reg32 offset="0xe72e" name="TPL1_DS_TEX_CONST_LO"/>
+	<reg32 offset="0xe72f" name="TPL1_DS_TEX_CONST_HI"/>
+	<reg32 offset="0xe730" name="TPL1_GS_TEX_CONST_LO"/>
+	<reg32 offset="0xe731" name="TPL1_GS_TEX_CONST_HI"/>
+
+	<reg32 offset="0xe750" name="TPL1_FS_TEX_COUNT" type="uint"/>
+	<reg32 offset="0xe751" name="TPL1_CS_TEX_COUNT" type="uint"/>
+
+	<reg32 offset="0xe75a" name="TPL1_FS_TEX_SAMP_LO"/>
+	<reg32 offset="0xe75b" name="TPL1_FS_TEX_SAMP_HI"/>
+	<reg32 offset="0xe75c" name="TPL1_CS_TEX_SAMP_LO"/>
+	<reg32 offset="0xe75d" name="TPL1_CS_TEX_SAMP_HI"/>
+	<reg32 offset="0xe75e" name="TPL1_FS_TEX_CONST_LO"/>
+	<reg32 offset="0xe75f" name="TPL1_FS_TEX_CONST_HI"/>
+	<reg32 offset="0xe760" name="TPL1_CS_TEX_CONST_LO"/>
+	<reg32 offset="0xe761" name="TPL1_CS_TEX_CONST_HI"/>
+
+	<reg32 offset="0xe764" name="TPL1_TP_FS_ROTATION_CNTL"/>
+
+	<reg32 offset="0xe784" name="HLSQ_CONTROL_0_REG">
+		<!-- 24 or more (full size) GPRS and blob uses TWO_QUADS instead of FOUR_QUADS -->
+		<bitfield name="FSTHREADSIZE" pos="0" type="a3xx_threadsize"/>
+		<bitfield name="CSTHREADSIZE" pos="2" type="a3xx_threadsize"/>
+	</reg32>
+	<reg32 offset="0xe785" name="HLSQ_CONTROL_1_REG">
+		<!-- I guess.. not set exactly same as a4xx, but similar: -->
+		<bitfield name="PRIMALLOCTHRESHOLD" low="0" high="5" type="uint"/>
+	</reg32>
+	<reg32 offset="0xe786" name="HLSQ_CONTROL_2_REG">
+		<bitfield name="FACEREGID" low="0" high="7" type="a3xx_regid"/>
+		<!-- SAMPLEID is loaded into a half-precision register: -->
+		<bitfield name="SAMPLEID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="SAMPLEMASK" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="CENTERRHW" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xe787" name="HLSQ_CONTROL_3_REG">
+		<!-- register loaded with position (bary.f) -->
+		<bitfield name="IJ_PERSP_PIXEL" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="IJ_LINEAR_PIXEL" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="IJ_PERSP_CENTROID" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="IJ_LINEAR_CENTROID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xe788" name="HLSQ_CONTROL_4_REG">
+		<bitfield name="IJ_PERSP_SAMPLE" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="IJ_LINEAR_SAMPLE" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="XYCOORDREGID" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="ZWCOORDREGID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<!--
+	0x020fffff for normal draws, 0x1f00000 for compute.. maybe what state
+	is enabled?  We could probably try disabling different bits and see
+	what breaks to figure out which is what:
+	 -->
+	<reg32 offset="0xe78a" name="HLSQ_UPDATE_CNTL"/>
+	<reg32 offset="0xe78b" name="HLSQ_VS_CONFIG" type="a5xx_xs_config"/>
+	<reg32 offset="0xe78c" name="HLSQ_FS_CONFIG" type="a5xx_xs_config"/>
+	<reg32 offset="0xe78d" name="HLSQ_HS_CONFIG" type="a5xx_xs_config"/>
+	<reg32 offset="0xe78e" name="HLSQ_DS_CONFIG" type="a5xx_xs_config"/>
+	<reg32 offset="0xe78f" name="HLSQ_GS_CONFIG" type="a5xx_xs_config"/>
+	<reg32 offset="0xe790" name="HLSQ_CS_CONFIG" type="a5xx_xs_config"/>
+	<reg32 offset="0xe791" name="HLSQ_VS_CNTL" type="a5xx_xs_cntl"/>
+	<reg32 offset="0xe792" name="HLSQ_FS_CNTL" type="a5xx_xs_cntl"/>
+	<reg32 offset="0xe793" name="HLSQ_HS_CNTL" type="a5xx_xs_cntl"/>
+	<reg32 offset="0xe794" name="HLSQ_DS_CNTL" type="a5xx_xs_cntl"/>
+	<reg32 offset="0xe795" name="HLSQ_GS_CNTL" type="a5xx_xs_cntl"/>
+	<reg32 offset="0xe796" name="HLSQ_CS_CNTL" type="a5xx_xs_cntl"/>
+	<reg32 offset="0xe7b9" name="HLSQ_CS_KERNEL_GROUP_X"/>
+	<reg32 offset="0xe7ba" name="HLSQ_CS_KERNEL_GROUP_Y"/>
+	<reg32 offset="0xe7bb" name="HLSQ_CS_KERNEL_GROUP_Z"/>
+	<reg32 offset="0xe7b0" name="HLSQ_CS_NDRANGE_0">
+		<bitfield name="KERNELDIM" low="0" high="1" type="uint"/>
+		<!-- localsize is value minus one: -->
+		<bitfield name="LOCALSIZEX" low="2" high="11" type="uint"/>
+		<bitfield name="LOCALSIZEY" low="12" high="21" type="uint"/>
+		<bitfield name="LOCALSIZEZ" low="22" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xe7b1" name="HLSQ_CS_NDRANGE_1">
+		<bitfield name="GLOBALSIZE_X" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xe7b2" name="HLSQ_CS_NDRANGE_2">
+		<bitfield name="GLOBALOFF_X" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xe7b3" name="HLSQ_CS_NDRANGE_3">
+		<bitfield name="GLOBALSIZE_Y" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xe7b4" name="HLSQ_CS_NDRANGE_4">
+		<bitfield name="GLOBALOFF_Y" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xe7b5" name="HLSQ_CS_NDRANGE_5">
+		<bitfield name="GLOBALSIZE_Z" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xe7b6" name="HLSQ_CS_NDRANGE_6">
+		<bitfield name="GLOBALOFF_Z" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xe7b7" name="HLSQ_CS_CNTL_0">
+		<bitfield name="WGIDCONSTID" low="0" high="7" type="a3xx_regid"/>
+		<!-- possibly one of these is KERNELDIMCONSTID? -->
+		<!--
+		UNK0 appears to be NUMWGCONSTID.. but only works in certain
+		cases?  Blob doesn't appear to use it, but instead emits
+		these via const (uniform).  Which requires some shenanigans
+		for indirect draws when the offset is not strongly aligned
+		enough to use as EXT_SRC_ADDR in CP_LOAD_STATE
+		 -->
+		<bitfield name="UNK0" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="UNK1" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="LOCALIDREGID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xe7b8" name="HLSQ_CS_CNTL_1"/>
+	<reg32 offset="0xe7c0" name="UNKNOWN_E7C0"/>
+	<reg32 offset="0xe7c3" name="HLSQ_VS_CONSTLEN" type="uint"/>
+	<reg32 offset="0xe7c4" name="HLSQ_VS_INSTRLEN" type="uint"/>
+	<reg32 offset="0xe7c5" name="UNKNOWN_E7C5"/>
+	<reg32 offset="0xe7c8" name="HLSQ_HS_CONSTLEN" type="uint"/>
+	<reg32 offset="0xe7c9" name="HLSQ_HS_INSTRLEN" type="uint"/>
+	<reg32 offset="0xe7ca" name="UNKNOWN_E7CA"/>
+	<reg32 offset="0xe7cd" name="HLSQ_DS_CONSTLEN" type="uint"/>
+	<reg32 offset="0xe7ce" name="HLSQ_DS_INSTRLEN" type="uint"/>
+	<reg32 offset="0xe7cf" name="UNKNOWN_E7CF"/>
+	<reg32 offset="0xe7d2" name="HLSQ_GS_CONSTLEN" type="uint"/>
+	<reg32 offset="0xe7d3" name="HLSQ_GS_INSTRLEN" type="uint"/>
+	<reg32 offset="0xe7d4" name="UNKNOWN_E7D4"/>
+	<reg32 offset="0xe7d7" name="HLSQ_FS_CONSTLEN" type="uint"/>
+	<reg32 offset="0xe7d8" name="HLSQ_FS_INSTRLEN" type="uint"/>
+	<reg32 offset="0xe7d9" name="UNKNOWN_E7D9"/>
+	<reg32 offset="0xe7dc" name="HLSQ_CS_CONSTLEN" type="uint"/>
+	<reg32 offset="0xe7dd" name="HLSQ_CS_INSTRLEN" type="uint"/>
+
+	<!--
+		Separate blit/2d or dma engine?  Seems to get used sometimes for
+		texture uploads, where a4xx blob would use normal draws.  Used
+		in render-mode 0x5..
+
+		Note seems mostly to be used for small blits, large blits seem
+		to use the CP_EVENT_WRITE:BLIT style of doing things.  See
+		cubemap-0003 (40x40) vs cubemap-0004 (256x256).
+
+		see cube-0000, cubemap-(1..3 but not 4+), quad-textured-10..17
+
+		Other nearby registers are probably color formats, etc.  The
+		blit coords are in CP packet.  Play more w/ glTexSubImage2D()
+		to work it out.
+
+		Separate this into a different domain??  Would that help to
+		restrict which registers we dump based on mode?
+
+		regs 0x2000 to 0x2004 (plus all-zero regs 0x2005-0x2009) look
+		like 2nd source for blending?  Used in mipmap generation.. but
+		maybe layout is a bit different.  (Possibly used for reading
+		src via sampler, to enable scaling??)  0x2040 also used in this
+		case.
+	 -->
+	<reg32 offset="0x2100" name="RB_2D_BLIT_CNTL"/>  <!-- same as 0x2180 -->
+	<reg32 offset="0x2101" name="RB_2D_SRC_SOLID_DW0"/>
+	<reg32 offset="0x2102" name="RB_2D_SRC_SOLID_DW1"/>
+	<reg32 offset="0x2103" name="RB_2D_SRC_SOLID_DW2"/>
+	<reg32 offset="0x2104" name="RB_2D_SRC_SOLID_DW3"/>
+
+	<bitset name="a5xx_2d_surf_info" inline="yes">
+		<bitfield name="COLOR_FORMAT" low="0" high="7" type="a5xx_color_fmt"/>
+		<bitfield name="TILE_MODE" low="8" high="9" type="a5xx_tile_mode"/>
+		<bitfield name="COLOR_SWAP" low="10" high="11" type="a3xx_color_swap"/>
+		<!-- b12 seems to be set when UBWC "FLAGS" buffer enabled -->
+		<bitfield name="FLAGS" pos="12" type="boolean"/>
+		<bitfield name="SRGB" pos="13" type="boolean"/>
+	</bitset>
+
+	<reg32 offset="0x2107" name="RB_2D_SRC_INFO" type="a5xx_2d_surf_info"/>
+	<reg32 offset="0x2108" name="RB_2D_SRC_LO"/>
+	<reg32 offset="0x2109" name="RB_2D_SRC_HI"/>
+	<reg32 offset="0x210a" name="RB_2D_SRC_SIZE">
+		<bitfield name="PITCH" low="0" high="15" shr="6" type="uint"/>
+		<bitfield name="ARRAY_PITCH" low="16" high="31" shr="6" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x2110" name="RB_2D_DST_INFO" type="a5xx_2d_surf_info"/>
+	<reg32 offset="0x2111" name="RB_2D_DST_LO"/>
+	<reg32 offset="0x2112" name="RB_2D_DST_HI"/>
+	<reg32 offset="0x2113" name="RB_2D_DST_SIZE">
+		<bitfield name="PITCH" low="0" high="15" shr="6" type="uint"/>
+		<bitfield name="ARRAY_PITCH" low="16" high="31" shr="6" type="uint"/>
+	</reg32>
+	<reg32 offset="0x2140" name="RB_2D_SRC_FLAGS_LO"/>
+	<reg32 offset="0x2141" name="RB_2D_SRC_FLAGS_HI"/>
+        <reg32 offset="0x2142" name="RB_2D_SRC_FLAGS_PITCH" shr="6" type="uint"/>
+	<reg32 offset="0x2143" name="RB_2D_DST_FLAGS_LO"/>
+	<reg32 offset="0x2144" name="RB_2D_DST_FLAGS_HI"/>
+        <reg32 offset="0x2145" name="RB_2D_DST_FLAGS_PITCH" shr="6" type="uint"/>
+	<reg32 offset="0x2180" name="GRAS_2D_BLIT_CNTL"/> <!-- same as 0x2100 -->
+	<!-- looks same as 0x2107: -->
+	<reg32 offset="0x2181" name="GRAS_2D_SRC_INFO" type="a5xx_2d_surf_info"/>
+	<!-- looks same as 0x2110: -->
+	<reg32 offset="0x2182" name="GRAS_2D_DST_INFO" type="a5xx_2d_surf_info"/>
+<!--
+0x2100 and 0x2180 look like same thing (RB and GRAS versions)..
+   0x86000000 for copy, 0x00000000 for fill?
+
+0x2184 0x9 for copy, 0x1 for blit (maybe bitmask of enabled src/dst???)
+ -->
+	<reg32 offset="0x2184" name="UNKNOWN_2184"/>
+</domain>
+
+<domain name="A5XX_TEX_SAMP" width="32">
+	<doc>Texture sampler dwords</doc>
+	<enum name="a5xx_tex_filter"> <!-- same as a4xx? -->
+		<value name="A5XX_TEX_NEAREST" value="0"/>
+		<value name="A5XX_TEX_LINEAR" value="1"/>
+		<value name="A5XX_TEX_ANISO" value="2"/>
+	</enum>
+	<enum name="a5xx_tex_clamp"> <!-- same as a4xx? -->
+		<value name="A5XX_TEX_REPEAT" value="0"/>
+		<value name="A5XX_TEX_CLAMP_TO_EDGE" value="1"/>
+		<value name="A5XX_TEX_MIRROR_REPEAT" value="2"/>
+		<value name="A5XX_TEX_CLAMP_TO_BORDER" value="3"/>
+		<value name="A5XX_TEX_MIRROR_CLAMP" value="4"/>
+	</enum>
+	<enum name="a5xx_tex_aniso"> <!-- same as a4xx? -->
+		<value name="A5XX_TEX_ANISO_1" value="0"/>
+		<value name="A5XX_TEX_ANISO_2" value="1"/>
+		<value name="A5XX_TEX_ANISO_4" value="2"/>
+		<value name="A5XX_TEX_ANISO_8" value="3"/>
+		<value name="A5XX_TEX_ANISO_16" value="4"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="MIPFILTER_LINEAR_NEAR" pos="0" type="boolean"/>
+		<bitfield name="XY_MAG" low="1" high="2" type="a5xx_tex_filter"/>
+		<bitfield name="XY_MIN" low="3" high="4" type="a5xx_tex_filter"/>
+		<bitfield name="WRAP_S" low="5" high="7" type="a5xx_tex_clamp"/>
+		<bitfield name="WRAP_T" low="8" high="10" type="a5xx_tex_clamp"/>
+		<bitfield name="WRAP_R" low="11" high="13" type="a5xx_tex_clamp"/>
+		<bitfield name="ANISO" low="14" high="16" type="a5xx_tex_aniso"/>
+		<bitfield name="LOD_BIAS" low="19" high="31" type="fixed" radix="8"/><!-- no idea how many bits for real -->
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="COMPARE_FUNC" low="1" high="3" type="adreno_compare_func"/>
+		<bitfield name="CUBEMAPSEAMLESSFILTOFF" pos="4" type="boolean"/>
+		<bitfield name="UNNORM_COORDS" pos="5" type="boolean"/>
+		<bitfield name="MIPFILTER_LINEAR_FAR" pos="6" type="boolean"/>
+		<bitfield name="MAX_LOD" low="8" high="19" type="ufixed" radix="8"/>
+		<bitfield name="MIN_LOD" low="20" high="31" type="ufixed" radix="8"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<!--
+		offset into border-color buffer?  Blob always uses 0x80 for FS state
+		if both VS and FS have border-color.
+Seems like when both VS and FS have bcolor, one starts 0x300 after other..
+and 0x80 in TEX_SAMP.2 ..  blob doesn't seem to be able to cope w/ multiple
+different border-color states per texture..  Looks something like:
+0000: 3f000000 00000000 00000000 3f800000 00008000 ffff0000 00004000 7fff0000
+0020: 00003800 3c000000 80100010 0000f008 ff000080 7f000040 c0000200 00800000
+0040: 00003800 3c000000 00000000 00000000 00000000 00000000 00000000 00000000
+*
+0300: 3f800000 3f800000 3f800000 3f800000 ffffffff ffffffff 7fff7fff 7fff7fff
+0320: 3c003c00 3c003c00 ffffffff 0000ffff ffffffff 7f7f7f7f ffffffff 00ffffff
+0340: 3c003c00 3c003c00 00000000 00000000 00000000 00000000 00000000 00000000
+
+		 -->
+		<bitfield name="BCOLOR_OFFSET" low="7" high="31"/>
+	</reg32>
+	<reg32 offset="3" name="3"/>
+</domain>
+
+<domain name="A5XX_TEX_CONST" width="32">
+	<doc>Texture constant dwords</doc>
+	<enum name="a5xx_tex_swiz"> <!-- same as a4xx? -->
+		<value name="A5XX_TEX_X" value="0"/>
+		<value name="A5XX_TEX_Y" value="1"/>
+		<value name="A5XX_TEX_Z" value="2"/>
+		<value name="A5XX_TEX_W" value="3"/>
+		<value name="A5XX_TEX_ZERO" value="4"/>
+		<value name="A5XX_TEX_ONE" value="5"/>
+	</enum>
+	<enum name="a5xx_tex_type"> <!-- same as a4xx? -->
+		<value name="A5XX_TEX_1D" value="0"/>
+		<value name="A5XX_TEX_2D" value="1"/>
+		<value name="A5XX_TEX_CUBE" value="2"/>
+		<value name="A5XX_TEX_3D" value="3"/>
+		<value name="A5XX_TEX_BUFFER" value="4"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="TILE_MODE" low="0" high="1" type="a5xx_tile_mode"/>
+		<bitfield name="SRGB" pos="2" type="boolean"/>
+		<bitfield name="SWIZ_X" low="4" high="6" type="a5xx_tex_swiz"/>
+		<bitfield name="SWIZ_Y" low="7" high="9" type="a5xx_tex_swiz"/>
+		<bitfield name="SWIZ_Z" low="10" high="12" type="a5xx_tex_swiz"/>
+		<bitfield name="SWIZ_W" low="13" high="15" type="a5xx_tex_swiz"/>
+		<bitfield name="MIPLVLS" low="16" high="19" type="uint"/>
+		<bitfield name="SAMPLES" low="20" high="21" type="a3xx_msaa_samples"/>
+		<bitfield name="FMT" low="22" high="29" type="a5xx_tex_fmt"/>
+		<bitfield name="SWAP" low="30" high="31" type="a3xx_color_swap"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="WIDTH" low="0" high="14" type="uint"/>
+		<bitfield name="HEIGHT" low="15" high="29" type="uint"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<!--
+		b4 and b31 set for buffer/ssbo case, in which case low 15 bits
+		of size encoded in WIDTH, and high 15 bits encoded in HEIGHT
+
+		b31 is probably the 'BUFFER' bit.. it is the one that changes
+		behavior of texture in dEQP-GLES31.functional.texture.texture_buffer.render.as_fragment_texture.buffer_size_131071
+		 -->
+		<bitfield name="BUFFER" pos="4" type="boolean"/>
+		<!-- minimum pitch (for mipmap levels): log2(pitchalign / 64) -->
+		<bitfield name="PITCHALIGN" low="0" high="3" type="uint"/>
+		<doc>Pitch in bytes (so actually stride)</doc>
+		<bitfield name="PITCH" low="7" high="28" type="uint"/>
+		<bitfield name="TYPE" low="29" high="31" type="a5xx_tex_type"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<!--
+		ARRAY_PITCH is basically LAYERSZ for the first mipmap level, and
+		for 3d textures (laid out mipmap level first) MIN_LAYERSZ is the
+		layer size at the point that it stops being reduced moving to
+		higher (smaller) mipmap levels
+		 -->
+		<bitfield name="ARRAY_PITCH" low="0" high="13" shr="12" type="uint"/>
+		<bitfield name="MIN_LAYERSZ" low="23" high="26" shr="12"/>
+		<!--
+		by default levels with w < 16 are linear
+		TILE_ALL makes all levels have tiling
+		seems required when using UBWC, since all levels have UBWC (can possibly be disabled?)
+		 -->
+		<bitfield name="TILE_ALL" pos="27" type="boolean"/>
+		<bitfield name="FLAG" pos="28" type="boolean"/>
+	</reg32>
+	<reg32 offset="4" name="4">
+		<bitfield name="BASE_LO" low="5" high="31" shr="5"/>
+	</reg32>
+	<reg32 offset="5" name="5">
+		<bitfield name="BASE_HI" low="0" high="16"/>
+		<bitfield name="DEPTH" low="17" high="29" type="uint"/>
+	</reg32>
+	<reg32 offset="6" name="6"/>
+	<reg32 offset="7" name="7"/>
+	<reg32 offset="8" name="8"/>
+	<reg32 offset="9" name="9"/>
+	<reg32 offset="10" name="10"/>
+	<reg32 offset="11" name="11"/>
+</domain>
+
+<!--
+Note the "SSBO" state blocks are actually used for both images and SSBOs,
+naming is just because I r/e'd SSBOs first.  I should probably come up
+with a better name.
+-->
+<domain name="A5XX_SSBO_0" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="BASE_LO" low="5" high="31" shr="5"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<!-- no BASE_HI here?  Maybe this is only used for 32b mode? -->
+		<doc>Pitch in bytes (so actually stride)</doc>
+		<bitfield name="PITCH" low="0" high="21" type="uint"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="ARRAY_PITCH" low="12" high="25" shr="12" type="uint"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<!-- bytes per pixel: -->
+		<bitfield name="CPP" low="0" high="5" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="A5XX_SSBO_1" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="FMT" low="8" high="15" type="a5xx_tex_fmt"/>
+		<bitfield name="WIDTH" low="16" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="HEIGHT" low="0" high="15" type="uint"/>
+		<bitfield name="DEPTH" low="16" high="31" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="A5XX_SSBO_2" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="BASE_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="BASE_HI" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain name="A5XX_UBO" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="BASE_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="BASE_HI" low="0" high="16"/>
+		<!-- size probably in high bits -->
+	</reg32>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/adreno/a6xx.xml b/drivers/gpu/drm/msm/registers/adreno/a6xx.xml
new file mode 100644
index 000000000000..2dfe6913ab4f
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/adreno/a6xx.xml
@@ -0,0 +1,5011 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+<import file="adreno/adreno_common.xml"/>
+<import file="adreno/adreno_pm4.xml"/>
+
+<!--
+Each register that is actually being used by driver should have "usage" defined,
+currently there are following usages:
+- "cmd" - the register is used outside of renderpass and blits,
+		roughly corresponds to registers used in ib1 for Freedreno
+- "rp_blit" - the register is used inside renderpass or blits
+		(ib2 for Freedreno)
+
+It is expected that register with "cmd" usage may be written into only at
+the start of the command buffer (ib1), while "rp_blit" usage indicates that register
+is either overwritten by renderpass/blit (ib2) or not used if not overwritten
+by a particular renderpass/blit.
+-->
+
+<!-- these might be same as a5xx -->
+<enum name="a6xx_tile_mode">
+	<value name="TILE6_LINEAR" value="0"/>
+	<value name="TILE6_2" value="2"/>
+	<value name="TILE6_3" value="3"/>
+</enum>
+
+<enum name="a6xx_format">
+	<value value="0x02" name="FMT6_A8_UNORM"/>
+	<value value="0x03" name="FMT6_8_UNORM"/>
+	<value value="0x04" name="FMT6_8_SNORM"/>
+	<value value="0x05" name="FMT6_8_UINT"/>
+	<value value="0x06" name="FMT6_8_SINT"/>
+
+	<value value="0x08" name="FMT6_4_4_4_4_UNORM"/>
+	<value value="0x0a" name="FMT6_5_5_5_1_UNORM"/>
+	<value value="0x0c" name="FMT6_1_5_5_5_UNORM"/> <!-- read only -->
+	<value value="0x0e" name="FMT6_5_6_5_UNORM"/>
+
+	<value value="0x0f" name="FMT6_8_8_UNORM"/>
+	<value value="0x10" name="FMT6_8_8_SNORM"/>
+	<value value="0x11" name="FMT6_8_8_UINT"/>
+	<value value="0x12" name="FMT6_8_8_SINT"/>
+	<value value="0x13" name="FMT6_L8_A8_UNORM"/>
+
+	<value value="0x15" name="FMT6_16_UNORM"/>
+	<value value="0x16" name="FMT6_16_SNORM"/>
+	<value value="0x17" name="FMT6_16_FLOAT"/>
+	<value value="0x18" name="FMT6_16_UINT"/>
+	<value value="0x19" name="FMT6_16_SINT"/>
+
+	<value value="0x21" name="FMT6_8_8_8_UNORM"/>
+	<value value="0x22" name="FMT6_8_8_8_SNORM"/>
+	<value value="0x23" name="FMT6_8_8_8_UINT"/>
+	<value value="0x24" name="FMT6_8_8_8_SINT"/>
+
+	<value value="0x30" name="FMT6_8_8_8_8_UNORM"/>
+	<value value="0x31" name="FMT6_8_8_8_X8_UNORM"/> <!-- samples 1 for alpha -->
+	<value value="0x32" name="FMT6_8_8_8_8_SNORM"/>
+	<value value="0x33" name="FMT6_8_8_8_8_UINT"/>
+	<value value="0x34" name="FMT6_8_8_8_8_SINT"/>
+
+	<value value="0x35" name="FMT6_9_9_9_E5_FLOAT"/>
+
+	<value value="0x36" name="FMT6_10_10_10_2_UNORM"/>
+	<value value="0x37" name="FMT6_10_10_10_2_UNORM_DEST"/>
+	<value value="0x39" name="FMT6_10_10_10_2_SNORM"/>
+	<value value="0x3a" name="FMT6_10_10_10_2_UINT"/>
+	<value value="0x3b" name="FMT6_10_10_10_2_SINT"/>
+
+	<value value="0x42" name="FMT6_11_11_10_FLOAT"/>
+
+	<value value="0x43" name="FMT6_16_16_UNORM"/>
+	<value value="0x44" name="FMT6_16_16_SNORM"/>
+	<value value="0x45" name="FMT6_16_16_FLOAT"/>
+	<value value="0x46" name="FMT6_16_16_UINT"/>
+	<value value="0x47" name="FMT6_16_16_SINT"/>
+
+	<value value="0x48" name="FMT6_32_UNORM"/>
+	<value value="0x49" name="FMT6_32_SNORM"/>
+	<value value="0x4a" name="FMT6_32_FLOAT"/>
+	<value value="0x4b" name="FMT6_32_UINT"/>
+	<value value="0x4c" name="FMT6_32_SINT"/>
+	<value value="0x4d" name="FMT6_32_FIXED"/>
+
+	<value value="0x58" name="FMT6_16_16_16_UNORM"/>
+	<value value="0x59" name="FMT6_16_16_16_SNORM"/>
+	<value value="0x5a" name="FMT6_16_16_16_FLOAT"/>
+	<value value="0x5b" name="FMT6_16_16_16_UINT"/>
+	<value value="0x5c" name="FMT6_16_16_16_SINT"/>
+
+	<value value="0x60" name="FMT6_16_16_16_16_UNORM"/>
+	<value value="0x61" name="FMT6_16_16_16_16_SNORM"/>
+	<value value="0x62" name="FMT6_16_16_16_16_FLOAT"/>
+	<value value="0x63" name="FMT6_16_16_16_16_UINT"/>
+	<value value="0x64" name="FMT6_16_16_16_16_SINT"/>
+
+	<value value="0x65" name="FMT6_32_32_UNORM"/>
+	<value value="0x66" name="FMT6_32_32_SNORM"/>
+	<value value="0x67" name="FMT6_32_32_FLOAT"/>
+	<value value="0x68" name="FMT6_32_32_UINT"/>
+	<value value="0x69" name="FMT6_32_32_SINT"/>
+	<value value="0x6a" name="FMT6_32_32_FIXED"/>
+
+	<value value="0x70" name="FMT6_32_32_32_UNORM"/>
+	<value value="0x71" name="FMT6_32_32_32_SNORM"/>
+	<value value="0x72" name="FMT6_32_32_32_UINT"/>
+	<value value="0x73" name="FMT6_32_32_32_SINT"/>
+	<value value="0x74" name="FMT6_32_32_32_FLOAT"/>
+	<value value="0x75" name="FMT6_32_32_32_FIXED"/>
+
+	<value value="0x80" name="FMT6_32_32_32_32_UNORM"/>
+	<value value="0x81" name="FMT6_32_32_32_32_SNORM"/>
+	<value value="0x82" name="FMT6_32_32_32_32_FLOAT"/>
+	<value value="0x83" name="FMT6_32_32_32_32_UINT"/>
+	<value value="0x84" name="FMT6_32_32_32_32_SINT"/>
+	<value value="0x85" name="FMT6_32_32_32_32_FIXED"/>
+
+	<value value="0x8c" name="FMT6_G8R8B8R8_422_UNORM"/> <!-- UYVY -->
+	<value value="0x8d" name="FMT6_R8G8R8B8_422_UNORM"/> <!-- YUYV -->
+	<value value="0x8e" name="FMT6_R8_G8B8_2PLANE_420_UNORM"/> <!-- NV12 -->
+	<value value="0x8f" name="FMT6_NV21"/>
+	<value value="0x90" name="FMT6_R8_G8_B8_3PLANE_420_UNORM"/> <!-- YV12 -->
+
+	<value value="0x91" name="FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8"/>
+
+	<!-- Note: tiling/UBWC for these may be different from equivalent formats
+	For example FMT6_NV12_Y is not compatible with FMT6_8_UNORM
+	-->
+	<value value="0x94" name="FMT6_NV12_Y"/>
+	<value value="0x95" name="FMT6_NV12_UV"/>
+	<value value="0x96" name="FMT6_NV12_VU"/>
+	<value value="0x97" name="FMT6_NV12_4R"/>
+	<value value="0x98" name="FMT6_NV12_4R_Y"/>
+	<value value="0x99" name="FMT6_NV12_4R_UV"/>
+	<value value="0x9a" name="FMT6_P010"/>
+	<value value="0x9b" name="FMT6_P010_Y"/>
+	<value value="0x9c" name="FMT6_P010_UV"/>
+	<value value="0x9d" name="FMT6_TP10"/>
+	<value value="0x9e" name="FMT6_TP10_Y"/>
+	<value value="0x9f" name="FMT6_TP10_UV"/>
+
+	<value value="0xa0" name="FMT6_Z24_UNORM_S8_UINT"/>
+
+	<value value="0xab" name="FMT6_ETC2_RG11_UNORM"/>
+	<value value="0xac" name="FMT6_ETC2_RG11_SNORM"/>
+	<value value="0xad" name="FMT6_ETC2_R11_UNORM"/>
+	<value value="0xae" name="FMT6_ETC2_R11_SNORM"/>
+	<value value="0xaf" name="FMT6_ETC1"/>
+	<value value="0xb0" name="FMT6_ETC2_RGB8"/>
+	<value value="0xb1" name="FMT6_ETC2_RGBA8"/>
+	<value value="0xb2" name="FMT6_ETC2_RGB8A1"/>
+	<value value="0xb3" name="FMT6_DXT1"/>
+	<value value="0xb4" name="FMT6_DXT3"/>
+	<value value="0xb5" name="FMT6_DXT5"/>
+	<value value="0xb7" name="FMT6_RGTC1_UNORM"/>
+	<value value="0xb8" name="FMT6_RGTC1_SNORM"/>
+	<value value="0xbb" name="FMT6_RGTC2_UNORM"/>
+	<value value="0xbc" name="FMT6_RGTC2_SNORM"/>
+	<value value="0xbe" name="FMT6_BPTC_UFLOAT"/>
+	<value value="0xbf" name="FMT6_BPTC_FLOAT"/>
+	<value value="0xc0" name="FMT6_BPTC"/>
+	<value value="0xc1" name="FMT6_ASTC_4x4"/>
+	<value value="0xc2" name="FMT6_ASTC_5x4"/>
+	<value value="0xc3" name="FMT6_ASTC_5x5"/>
+	<value value="0xc4" name="FMT6_ASTC_6x5"/>
+	<value value="0xc5" name="FMT6_ASTC_6x6"/>
+	<value value="0xc6" name="FMT6_ASTC_8x5"/>
+	<value value="0xc7" name="FMT6_ASTC_8x6"/>
+	<value value="0xc8" name="FMT6_ASTC_8x8"/>
+	<value value="0xc9" name="FMT6_ASTC_10x5"/>
+	<value value="0xca" name="FMT6_ASTC_10x6"/>
+	<value value="0xcb" name="FMT6_ASTC_10x8"/>
+	<value value="0xcc" name="FMT6_ASTC_10x10"/>
+	<value value="0xcd" name="FMT6_ASTC_12x10"/>
+	<value value="0xce" name="FMT6_ASTC_12x12"/>
+
+	<!-- for sampling stencil (integer, 2nd channel), not available on a630 -->
+	<value value="0xea" name="FMT6_Z24_UINT_S8_UINT"/>
+
+	<!-- Not a hw enum, used internally in driver -->
+	<value value="0xff" name="FMT6_NONE"/>
+
+</enum>
+
+<!-- probably same as a5xx -->
+<enum name="a6xx_polygon_mode">
+	<value name="POLYMODE6_POINTS" value="1"/>
+	<value name="POLYMODE6_LINES" value="2"/>
+	<value name="POLYMODE6_TRIANGLES" value="3"/>
+</enum>
+
+<enum name="a6xx_depth_format">
+	<value name="DEPTH6_NONE" value="0"/>
+	<value name="DEPTH6_16" value="1"/>
+	<value name="DEPTH6_24_8" value="2"/>
+	<value name="DEPTH6_32" value="4"/>
+</enum>
+
+<bitset name="a6x_cp_protect" inline="yes">
+	<bitfield name="BASE_ADDR" low="0" high="17"/>
+	<bitfield name="MASK_LEN" low="18" high="30"/>
+	<bitfield name="READ" pos="31" type="boolean"/>
+</bitset>
+
+<enum name="a6xx_shader_id">
+	<value value="0x9" name="A6XX_TP0_TMO_DATA"/>
+	<value value="0xa" name="A6XX_TP0_SMO_DATA"/>
+	<value value="0xb" name="A6XX_TP0_MIPMAP_BASE_DATA"/>
+	<value value="0x19" name="A6XX_TP1_TMO_DATA"/>
+	<value value="0x1a" name="A6XX_TP1_SMO_DATA"/>
+	<value value="0x1b" name="A6XX_TP1_MIPMAP_BASE_DATA"/>
+	<value value="0x29" name="A6XX_SP_INST_DATA"/>
+	<value value="0x2a" name="A6XX_SP_LB_0_DATA"/>
+	<value value="0x2b" name="A6XX_SP_LB_1_DATA"/>
+	<value value="0x2c" name="A6XX_SP_LB_2_DATA"/>
+	<value value="0x2d" name="A6XX_SP_LB_3_DATA"/>
+	<value value="0x2e" name="A6XX_SP_LB_4_DATA"/>
+	<value value="0x2f" name="A6XX_SP_LB_5_DATA"/>
+	<value value="0x30" name="A6XX_SP_CB_BINDLESS_DATA"/>
+	<value value="0x31" name="A6XX_SP_CB_LEGACY_DATA"/>
+	<value value="0x32" name="A6XX_SP_UAV_DATA"/>
+	<value value="0x33" name="A6XX_SP_INST_TAG"/>
+	<value value="0x34" name="A6XX_SP_CB_BINDLESS_TAG"/>
+	<value value="0x35" name="A6XX_SP_TMO_UMO_TAG"/>
+	<value value="0x36" name="A6XX_SP_SMO_TAG"/>
+	<value value="0x37" name="A6XX_SP_STATE_DATA"/>
+	<value value="0x49" name="A6XX_HLSQ_CHUNK_CVS_RAM"/>
+	<value value="0x4a" name="A6XX_HLSQ_CHUNK_CPS_RAM"/>
+	<value value="0x4b" name="A6XX_HLSQ_CHUNK_CVS_RAM_TAG"/>
+	<value value="0x4c" name="A6XX_HLSQ_CHUNK_CPS_RAM_TAG"/>
+	<value value="0x4d" name="A6XX_HLSQ_ICB_CVS_CB_BASE_TAG"/>
+	<value value="0x4e" name="A6XX_HLSQ_ICB_CPS_CB_BASE_TAG"/>
+	<value value="0x50" name="A6XX_HLSQ_CVS_MISC_RAM"/>
+	<value value="0x51" name="A6XX_HLSQ_CPS_MISC_RAM"/>
+	<value value="0x52" name="A6XX_HLSQ_INST_RAM"/>
+	<value value="0x53" name="A6XX_HLSQ_GFX_CVS_CONST_RAM"/>
+	<value value="0x54" name="A6XX_HLSQ_GFX_CPS_CONST_RAM"/>
+	<value value="0x55" name="A6XX_HLSQ_CVS_MISC_RAM_TAG"/>
+	<value value="0x56" name="A6XX_HLSQ_CPS_MISC_RAM_TAG"/>
+	<value value="0x57" name="A6XX_HLSQ_INST_RAM_TAG"/>
+	<value value="0x58" name="A6XX_HLSQ_GFX_CVS_CONST_RAM_TAG"/>
+	<value value="0x59" name="A6XX_HLSQ_GFX_CPS_CONST_RAM_TAG"/>
+	<value value="0x5a" name="A6XX_HLSQ_PWR_REST_RAM"/>
+	<value value="0x5b" name="A6XX_HLSQ_PWR_REST_TAG"/>
+	<value value="0x60" name="A6XX_HLSQ_DATAPATH_META"/>
+	<value value="0x61" name="A6XX_HLSQ_FRONTEND_META"/>
+	<value value="0x62" name="A6XX_HLSQ_INDIRECT_META"/>
+	<value value="0x63" name="A6XX_HLSQ_BACKEND_META"/>
+	<value value="0x70" name="A6XX_SP_LB_6_DATA"/>
+	<value value="0x71" name="A6XX_SP_LB_7_DATA"/>
+	<value value="0x73" name="A6XX_HLSQ_INST_RAM_1"/>
+</enum>
+
+<enum name="a7xx_statetype_id">
+	<value value="0" name="A7XX_TP0_NCTX_REG"/>
+	<value value="1" name="A7XX_TP0_CTX0_3D_CVS_REG"/>
+	<value value="2" name="A7XX_TP0_CTX0_3D_CPS_REG"/>
+	<value value="3" name="A7XX_TP0_CTX1_3D_CVS_REG"/>
+	<value value="4" name="A7XX_TP0_CTX1_3D_CPS_REG"/>
+	<value value="5" name="A7XX_TP0_CTX2_3D_CPS_REG"/>
+	<value value="6" name="A7XX_TP0_CTX3_3D_CPS_REG"/>
+	<value value="9" name="A7XX_TP0_TMO_DATA"/>
+	<value value="10" name="A7XX_TP0_SMO_DATA"/>
+	<value value="11" name="A7XX_TP0_MIPMAP_BASE_DATA"/>
+	<value value="32" name="A7XX_SP_NCTX_REG"/>
+	<value value="33" name="A7XX_SP_CTX0_3D_CVS_REG"/>
+	<value value="34" name="A7XX_SP_CTX0_3D_CPS_REG"/>
+	<value value="35" name="A7XX_SP_CTX1_3D_CVS_REG"/>
+	<value value="36" name="A7XX_SP_CTX1_3D_CPS_REG"/>
+	<value value="37" name="A7XX_SP_CTX2_3D_CPS_REG"/>
+	<value value="38" name="A7XX_SP_CTX3_3D_CPS_REG"/>
+	<value value="39" name="A7XX_SP_INST_DATA"/>
+	<value value="40" name="A7XX_SP_INST_DATA_1"/>
+	<value value="41" name="A7XX_SP_LB_0_DATA"/>
+	<value value="42" name="A7XX_SP_LB_1_DATA"/>
+	<value value="43" name="A7XX_SP_LB_2_DATA"/>
+	<value value="44" name="A7XX_SP_LB_3_DATA"/>
+	<value value="45" name="A7XX_SP_LB_4_DATA"/>
+	<value value="46" name="A7XX_SP_LB_5_DATA"/>
+	<value value="47" name="A7XX_SP_LB_6_DATA"/>
+	<value value="48" name="A7XX_SP_LB_7_DATA"/>
+	<value value="49" name="A7XX_SP_CB_RAM"/>
+	<value value="50" name="A7XX_SP_LB_13_DATA"/>
+	<value value="51" name="A7XX_SP_LB_14_DATA"/>
+	<value value="52" name="A7XX_SP_INST_TAG"/>
+	<value value="53" name="A7XX_SP_INST_DATA_2"/>
+	<value value="54" name="A7XX_SP_TMO_TAG"/>
+	<value value="55" name="A7XX_SP_SMO_TAG"/>
+	<value value="56" name="A7XX_SP_STATE_DATA"/>
+	<value value="57" name="A7XX_SP_HWAVE_RAM"/>
+	<value value="58" name="A7XX_SP_L0_INST_BUF"/>
+	<value value="59" name="A7XX_SP_LB_8_DATA"/>
+	<value value="60" name="A7XX_SP_LB_9_DATA"/>
+	<value value="61" name="A7XX_SP_LB_10_DATA"/>
+	<value value="62" name="A7XX_SP_LB_11_DATA"/>
+	<value value="63" name="A7XX_SP_LB_12_DATA"/>
+	<value value="64" name="A7XX_HLSQ_DATAPATH_DSTR_META"/>
+	<value value="67" name="A7XX_HLSQ_L2STC_TAG_RAM"/>
+	<value value="68" name="A7XX_HLSQ_L2STC_INFO_CMD"/>
+	<value value="69" name="A7XX_HLSQ_CVS_BE_CTXT_BUF_RAM_TAG"/>
+	<value value="70" name="A7XX_HLSQ_CPS_BE_CTXT_BUF_RAM_TAG"/>
+	<value value="71" name="A7XX_HLSQ_GFX_CVS_BE_CTXT_BUF_RAM"/>
+	<value value="72" name="A7XX_HLSQ_GFX_CPS_BE_CTXT_BUF_RAM"/>
+	<value value="73" name="A7XX_HLSQ_CHUNK_CVS_RAM"/>
+	<value value="74" name="A7XX_HLSQ_CHUNK_CPS_RAM"/>
+	<value value="75" name="A7XX_HLSQ_CHUNK_CVS_RAM_TAG"/>
+	<value value="76" name="A7XX_HLSQ_CHUNK_CPS_RAM_TAG"/>
+	<value value="77" name="A7XX_HLSQ_ICB_CVS_CB_BASE_TAG"/>
+	<value value="78" name="A7XX_HLSQ_ICB_CPS_CB_BASE_TAG"/>
+	<value value="79" name="A7XX_HLSQ_CVS_MISC_RAM"/>
+	<value value="80" name="A7XX_HLSQ_CPS_MISC_RAM"/>
+	<value value="81" name="A7XX_HLSQ_CPS_MISC_RAM_1"/>
+	<value value="82" name="A7XX_HLSQ_INST_RAM"/>
+	<value value="83" name="A7XX_HLSQ_GFX_CVS_CONST_RAM"/>
+	<value value="84" name="A7XX_HLSQ_GFX_CPS_CONST_RAM"/>
+	<value value="85" name="A7XX_HLSQ_CVS_MISC_RAM_TAG"/>
+	<value value="86" name="A7XX_HLSQ_CPS_MISC_RAM_TAG"/>
+	<value value="87" name="A7XX_HLSQ_INST_RAM_TAG"/>
+	<value value="88" name="A7XX_HLSQ_GFX_CVS_CONST_RAM_TAG"/>
+	<value value="89" name="A7XX_HLSQ_GFX_CPS_CONST_RAM_TAG"/>
+	<value value="90" name="A7XX_HLSQ_GFX_LOCAL_MISC_RAM"/>
+	<value value="91" name="A7XX_HLSQ_GFX_LOCAL_MISC_RAM_TAG"/>
+	<value value="92" name="A7XX_HLSQ_INST_RAM_1"/>
+	<value value="93" name="A7XX_HLSQ_STPROC_META"/>
+	<value value="94" name="A7XX_HLSQ_BV_BE_META"/>
+	<value value="95" name="A7XX_HLSQ_INST_RAM_2"/>
+	<value value="96" name="A7XX_HLSQ_DATAPATH_META"/>
+	<value value="97" name="A7XX_HLSQ_FRONTEND_META"/>
+	<value value="98" name="A7XX_HLSQ_INDIRECT_META"/>
+	<value value="99" name="A7XX_HLSQ_BACKEND_META"/>
+</enum>
+
+<enum name="a6xx_debugbus_id">
+	<value value="0x1" name="A6XX_DBGBUS_CP"/>
+	<value value="0x2" name="A6XX_DBGBUS_RBBM"/>
+	<value value="0x3" name="A6XX_DBGBUS_VBIF"/>
+	<value value="0x4" name="A6XX_DBGBUS_HLSQ"/>
+	<value value="0x5" name="A6XX_DBGBUS_UCHE"/>
+	<value value="0x6" name="A6XX_DBGBUS_DPM"/>
+	<value value="0x7" name="A6XX_DBGBUS_TESS"/>
+	<value value="0x8" name="A6XX_DBGBUS_PC"/>
+	<value value="0x9" name="A6XX_DBGBUS_VFDP"/>
+	<value value="0xa" name="A6XX_DBGBUS_VPC"/>
+	<value value="0xb" name="A6XX_DBGBUS_TSE"/>
+	<value value="0xc" name="A6XX_DBGBUS_RAS"/>
+	<value value="0xd" name="A6XX_DBGBUS_VSC"/>
+	<value value="0xe" name="A6XX_DBGBUS_COM"/>
+	<value value="0x10" name="A6XX_DBGBUS_LRZ"/>
+	<value value="0x11" name="A6XX_DBGBUS_A2D"/>
+	<value value="0x12" name="A6XX_DBGBUS_CCUFCHE"/>
+	<value value="0x13" name="A6XX_DBGBUS_GMU_CX"/>
+	<value value="0x14" name="A6XX_DBGBUS_RBP"/>
+	<value value="0x15" name="A6XX_DBGBUS_DCS"/>
+	<value value="0x16" name="A6XX_DBGBUS_DBGC"/>
+	<value value="0x17" name="A6XX_DBGBUS_CX"/>
+	<value value="0x18" name="A6XX_DBGBUS_GMU_GX"/>
+	<value value="0x19" name="A6XX_DBGBUS_TPFCHE"/>
+	<value value="0x1a" name="A6XX_DBGBUS_GBIF_GX"/>
+	<value value="0x1d" name="A6XX_DBGBUS_GPC"/>
+	<value value="0x1e" name="A6XX_DBGBUS_LARC"/>
+	<value value="0x1f" name="A6XX_DBGBUS_HLSQ_SPTP"/>
+	<value value="0x20" name="A6XX_DBGBUS_RB_0"/>
+	<value value="0x21" name="A6XX_DBGBUS_RB_1"/>
+	<value value="0x22" name="A6XX_DBGBUS_RB_2"/>
+	<value value="0x24" name="A6XX_DBGBUS_UCHE_WRAPPER"/>
+	<value value="0x28" name="A6XX_DBGBUS_CCU_0"/>
+	<value value="0x29" name="A6XX_DBGBUS_CCU_1"/>
+	<value value="0x2a" name="A6XX_DBGBUS_CCU_2"/>
+	<value value="0x38" name="A6XX_DBGBUS_VFD_0"/>
+	<value value="0x39" name="A6XX_DBGBUS_VFD_1"/>
+	<value value="0x3a" name="A6XX_DBGBUS_VFD_2"/>
+	<value value="0x3b" name="A6XX_DBGBUS_VFD_3"/>
+	<value value="0x3c" name="A6XX_DBGBUS_VFD_4"/>
+	<value value="0x3d" name="A6XX_DBGBUS_VFD_5"/>
+	<value value="0x40" name="A6XX_DBGBUS_SP_0"/>
+	<value value="0x41" name="A6XX_DBGBUS_SP_1"/>
+	<value value="0x42" name="A6XX_DBGBUS_SP_2"/>
+	<value value="0x48" name="A6XX_DBGBUS_TPL1_0"/>
+	<value value="0x49" name="A6XX_DBGBUS_TPL1_1"/>
+	<value value="0x4a" name="A6XX_DBGBUS_TPL1_2"/>
+	<value value="0x4b" name="A6XX_DBGBUS_TPL1_3"/>
+	<value value="0x4c" name="A6XX_DBGBUS_TPL1_4"/>
+	<value value="0x4d" name="A6XX_DBGBUS_TPL1_5"/>
+	<value value="0x58" name="A6XX_DBGBUS_SPTP_0"/>
+	<value value="0x59" name="A6XX_DBGBUS_SPTP_1"/>
+	<value value="0x5a" name="A6XX_DBGBUS_SPTP_2"/>
+	<value value="0x5b" name="A6XX_DBGBUS_SPTP_3"/>
+	<value value="0x5c" name="A6XX_DBGBUS_SPTP_4"/>
+	<value value="0x5d" name="A6XX_DBGBUS_SPTP_5"/>
+</enum>
+
+<enum name="a7xx_state_location">
+	<value value="0" name="A7XX_HLSQ_STATE"/>
+	<value value="1" name="A7XX_HLSQ_DP"/>
+	<value value="2" name="A7XX_SP_TOP"/>
+	<value value="3" name="A7XX_USPTP"/>
+	<value value="4" name="A7XX_HLSQ_DP_STR"/>
+</enum>
+
+<enum name="a7xx_pipe">
+	<value value="0" name="A7XX_PIPE_NONE"/>
+	<value value="1" name="A7XX_PIPE_BR"/>
+	<value value="2" name="A7XX_PIPE_BV"/>
+	<value value="3" name="A7XX_PIPE_LPAC"/>
+</enum>
+
+<enum name="a7xx_cluster">
+	<value value="0" name="A7XX_CLUSTER_NONE"/>
+	<value value="1" name="A7XX_CLUSTER_FE"/>
+	<value value="2" name="A7XX_CLUSTER_SP_VS"/>
+	<value value="3" name="A7XX_CLUSTER_PC_VS"/>
+	<value value="4" name="A7XX_CLUSTER_GRAS"/>
+	<value value="5" name="A7XX_CLUSTER_SP_PS"/>
+	<value value="6" name="A7XX_CLUSTER_VPC_PS"/>
+	<value value="7" name="A7XX_CLUSTER_PS"/>
+</enum>
+
+<enum name="a7xx_debugbus_id">
+	<value value="1" name="A7XX_DBGBUS_CP_0_0"/>
+	<value value="2" name="A7XX_DBGBUS_CP_0_1"/>
+	<value value="3" name="A7XX_DBGBUS_RBBM"/>
+	<value value="5" name="A7XX_DBGBUS_GBIF_GX"/>
+	<value value="6" name="A7XX_DBGBUS_GBIF_CX"/>
+	<value value="7" name="A7XX_DBGBUS_HLSQ"/>
+	<value value="9" name="A7XX_DBGBUS_UCHE_0"/>
+	<value value="10" name="A7XX_DBGBUS_UCHE_1"/>
+	<value value="13" name="A7XX_DBGBUS_TESS_BR"/>
+	<value value="14" name="A7XX_DBGBUS_TESS_BV"/>
+	<value value="17" name="A7XX_DBGBUS_PC_BR"/>
+	<value value="18" name="A7XX_DBGBUS_PC_BV"/>
+	<value value="21" name="A7XX_DBGBUS_VFDP_BR"/>
+	<value value="22" name="A7XX_DBGBUS_VFDP_BV"/>
+	<value value="25" name="A7XX_DBGBUS_VPC_BR"/>
+	<value value="26" name="A7XX_DBGBUS_VPC_BV"/>
+	<value value="29" name="A7XX_DBGBUS_TSE_BR"/>
+	<value value="30" name="A7XX_DBGBUS_TSE_BV"/>
+	<value value="33" name="A7XX_DBGBUS_RAS_BR"/>
+	<value value="34" name="A7XX_DBGBUS_RAS_BV"/>
+	<value value="37" name="A7XX_DBGBUS_VSC"/>
+	<value value="39" name="A7XX_DBGBUS_COM_0"/>
+	<value value="43" name="A7XX_DBGBUS_LRZ_BR"/>
+	<value value="44" name="A7XX_DBGBUS_LRZ_BV"/>
+	<value value="47" name="A7XX_DBGBUS_UFC_0"/>
+	<value value="48" name="A7XX_DBGBUS_UFC_1"/>
+	<value value="55" name="A7XX_DBGBUS_GMU_GX"/>
+	<value value="59" name="A7XX_DBGBUS_DBGC"/>
+	<value value="60" name="A7XX_DBGBUS_CX"/>
+	<value value="61" name="A7XX_DBGBUS_GMU_CX"/>
+	<value value="62" name="A7XX_DBGBUS_GPC_BR"/>
+	<value value="63" name="A7XX_DBGBUS_GPC_BV"/>
+	<value value="66" name="A7XX_DBGBUS_LARC"/>
+	<value value="68" name="A7XX_DBGBUS_HLSQ_SPTP"/>
+	<value value="70" name="A7XX_DBGBUS_RB_0"/>
+	<value value="71" name="A7XX_DBGBUS_RB_1"/>
+	<value value="72" name="A7XX_DBGBUS_RB_2"/>
+	<value value="73" name="A7XX_DBGBUS_RB_3"/>
+	<value value="74" name="A7XX_DBGBUS_RB_4"/>
+	<value value="75" name="A7XX_DBGBUS_RB_5"/>
+	<value value="102" name="A7XX_DBGBUS_UCHE_WRAPPER"/>
+	<value value="106" name="A7XX_DBGBUS_CCU_0"/>
+	<value value="107" name="A7XX_DBGBUS_CCU_1"/>
+	<value value="108" name="A7XX_DBGBUS_CCU_2"/>
+	<value value="109" name="A7XX_DBGBUS_CCU_3"/>
+	<value value="110" name="A7XX_DBGBUS_CCU_4"/>
+	<value value="111" name="A7XX_DBGBUS_CCU_5"/>
+	<value value="138" name="A7XX_DBGBUS_VFD_BR_0"/>
+	<value value="139" name="A7XX_DBGBUS_VFD_BR_1"/>
+	<value value="140" name="A7XX_DBGBUS_VFD_BR_2"/>
+	<value value="141" name="A7XX_DBGBUS_VFD_BR_3"/>
+	<value value="142" name="A7XX_DBGBUS_VFD_BR_4"/>
+	<value value="143" name="A7XX_DBGBUS_VFD_BR_5"/>
+	<value value="144" name="A7XX_DBGBUS_VFD_BR_6"/>
+	<value value="145" name="A7XX_DBGBUS_VFD_BR_7"/>
+	<value value="202" name="A7XX_DBGBUS_VFD_BV_0"/>
+	<value value="203" name="A7XX_DBGBUS_VFD_BV_1"/>
+	<value value="204" name="A7XX_DBGBUS_VFD_BV_2"/>
+	<value value="205" name="A7XX_DBGBUS_VFD_BV_3"/>
+	<value value="234" name="A7XX_DBGBUS_USP_0"/>
+	<value value="235" name="A7XX_DBGBUS_USP_1"/>
+	<value value="236" name="A7XX_DBGBUS_USP_2"/>
+	<value value="237" name="A7XX_DBGBUS_USP_3"/>
+	<value value="238" name="A7XX_DBGBUS_USP_4"/>
+	<value value="239" name="A7XX_DBGBUS_USP_5"/>
+	<value value="266" name="A7XX_DBGBUS_TP_0"/>
+	<value value="267" name="A7XX_DBGBUS_TP_1"/>
+	<value value="268" name="A7XX_DBGBUS_TP_2"/>
+	<value value="269" name="A7XX_DBGBUS_TP_3"/>
+	<value value="270" name="A7XX_DBGBUS_TP_4"/>
+	<value value="271" name="A7XX_DBGBUS_TP_5"/>
+	<value value="272" name="A7XX_DBGBUS_TP_6"/>
+	<value value="273" name="A7XX_DBGBUS_TP_7"/>
+	<value value="274" name="A7XX_DBGBUS_TP_8"/>
+	<value value="275" name="A7XX_DBGBUS_TP_9"/>
+	<value value="276" name="A7XX_DBGBUS_TP_10"/>
+	<value value="277" name="A7XX_DBGBUS_TP_11"/>
+	<value value="330" name="A7XX_DBGBUS_USPTP_0"/>
+	<value value="331" name="A7XX_DBGBUS_USPTP_1"/>
+	<value value="332" name="A7XX_DBGBUS_USPTP_2"/>
+	<value value="333" name="A7XX_DBGBUS_USPTP_3"/>
+	<value value="334" name="A7XX_DBGBUS_USPTP_4"/>
+	<value value="335" name="A7XX_DBGBUS_USPTP_5"/>
+	<value value="336" name="A7XX_DBGBUS_USPTP_6"/>
+	<value value="337" name="A7XX_DBGBUS_USPTP_7"/>
+	<value value="338" name="A7XX_DBGBUS_USPTP_8"/>
+	<value value="339" name="A7XX_DBGBUS_USPTP_9"/>
+	<value value="340" name="A7XX_DBGBUS_USPTP_10"/>
+	<value value="341" name="A7XX_DBGBUS_USPTP_11"/>
+	<value value="396" name="A7XX_DBGBUS_CCHE_0"/>
+	<value value="397" name="A7XX_DBGBUS_CCHE_1"/>
+	<value value="398" name="A7XX_DBGBUS_CCHE_2"/>
+	<value value="408" name="A7XX_DBGBUS_VPC_DSTR_0"/>
+	<value value="409" name="A7XX_DBGBUS_VPC_DSTR_1"/>
+	<value value="410" name="A7XX_DBGBUS_VPC_DSTR_2"/>
+	<value value="411" name="A7XX_DBGBUS_HLSQ_DP_STR_0"/>
+	<value value="412" name="A7XX_DBGBUS_HLSQ_DP_STR_1"/>
+	<value value="413" name="A7XX_DBGBUS_HLSQ_DP_STR_2"/>
+	<value value="414" name="A7XX_DBGBUS_HLSQ_DP_STR_3"/>
+	<value value="415" name="A7XX_DBGBUS_HLSQ_DP_STR_4"/>
+	<value value="416" name="A7XX_DBGBUS_HLSQ_DP_STR_5"/>
+	<value value="443" name="A7XX_DBGBUS_UFC_DSTR_0"/>
+	<value value="444" name="A7XX_DBGBUS_UFC_DSTR_1"/>
+	<value value="445" name="A7XX_DBGBUS_UFC_DSTR_2"/>
+	<value value="446" name="A7XX_DBGBUS_CGC_SUBCORE"/>
+	<value value="447" name="A7XX_DBGBUS_CGC_CORE"/>
+</enum>
+
+<enum name="a6xx_cp_perfcounter_select">
+	<value value="0" name="PERF_CP_ALWAYS_COUNT"/>
+	<value value="1" name="PERF_CP_BUSY_GFX_CORE_IDLE"/>
+	<value value="2" name="PERF_CP_BUSY_CYCLES"/>
+	<value value="3" name="PERF_CP_NUM_PREEMPTIONS"/>
+	<value value="4" name="PERF_CP_PREEMPTION_REACTION_DELAY"/>
+	<value value="5" name="PERF_CP_PREEMPTION_SWITCH_OUT_TIME"/>
+	<value value="6" name="PERF_CP_PREEMPTION_SWITCH_IN_TIME"/>
+	<value value="7" name="PERF_CP_DEAD_DRAWS_IN_BIN_RENDER"/>
+	<value value="8" name="PERF_CP_PREDICATED_DRAWS_KILLED"/>
+	<value value="9" name="PERF_CP_MODE_SWITCH"/>
+	<value value="10" name="PERF_CP_ZPASS_DONE"/>
+	<value value="11" name="PERF_CP_CONTEXT_DONE"/>
+	<value value="12" name="PERF_CP_CACHE_FLUSH"/>
+	<value value="13" name="PERF_CP_LONG_PREEMPTIONS"/>
+	<value value="14" name="PERF_CP_SQE_I_CACHE_STARVE"/>
+	<value value="15" name="PERF_CP_SQE_IDLE"/>
+	<value value="16" name="PERF_CP_SQE_PM4_STARVE_RB_IB"/>
+	<value value="17" name="PERF_CP_SQE_PM4_STARVE_SDS"/>
+	<value value="18" name="PERF_CP_SQE_MRB_STARVE"/>
+	<value value="19" name="PERF_CP_SQE_RRB_STARVE"/>
+	<value value="20" name="PERF_CP_SQE_VSD_STARVE"/>
+	<value value="21" name="PERF_CP_VSD_DECODE_STARVE"/>
+	<value value="22" name="PERF_CP_SQE_PIPE_OUT_STALL"/>
+	<value value="23" name="PERF_CP_SQE_SYNC_STALL"/>
+	<value value="24" name="PERF_CP_SQE_PM4_WFI_STALL"/>
+	<value value="25" name="PERF_CP_SQE_SYS_WFI_STALL"/>
+	<value value="26" name="PERF_CP_SQE_T4_EXEC"/>
+	<value value="27" name="PERF_CP_SQE_LOAD_STATE_EXEC"/>
+	<value value="28" name="PERF_CP_SQE_SAVE_SDS_STATE"/>
+	<value value="29" name="PERF_CP_SQE_DRAW_EXEC"/>
+	<value value="30" name="PERF_CP_SQE_CTXT_REG_BUNCH_EXEC"/>
+	<value value="31" name="PERF_CP_SQE_EXEC_PROFILED"/>
+	<value value="32" name="PERF_CP_MEMORY_POOL_EMPTY"/>
+	<value value="33" name="PERF_CP_MEMORY_POOL_SYNC_STALL"/>
+	<value value="34" name="PERF_CP_MEMORY_POOL_ABOVE_THRESH"/>
+	<value value="35" name="PERF_CP_AHB_WR_STALL_PRE_DRAWS"/>
+	<value value="36" name="PERF_CP_AHB_STALL_SQE_GMU"/>
+	<value value="37" name="PERF_CP_AHB_STALL_SQE_WR_OTHER"/>
+	<value value="38" name="PERF_CP_AHB_STALL_SQE_RD_OTHER"/>
+	<value value="39" name="PERF_CP_CLUSTER0_EMPTY"/>
+	<value value="40" name="PERF_CP_CLUSTER1_EMPTY"/>
+	<value value="41" name="PERF_CP_CLUSTER2_EMPTY"/>
+	<value value="42" name="PERF_CP_CLUSTER3_EMPTY"/>
+	<value value="43" name="PERF_CP_CLUSTER4_EMPTY"/>
+	<value value="44" name="PERF_CP_CLUSTER5_EMPTY"/>
+	<value value="45" name="PERF_CP_PM4_DATA"/>
+	<value value="46" name="PERF_CP_PM4_HEADERS"/>
+	<value value="47" name="PERF_CP_VBIF_READ_BEATS"/>
+	<value value="48" name="PERF_CP_VBIF_WRITE_BEATS"/>
+	<value value="49" name="PERF_CP_SQE_INSTR_COUNTER"/>
+</enum>
+
+<enum name="a6xx_rbbm_perfcounter_select">
+	<value value="0" name="PERF_RBBM_ALWAYS_COUNT"/>
+	<value value="1" name="PERF_RBBM_ALWAYS_ON"/>
+	<value value="2" name="PERF_RBBM_TSE_BUSY"/>
+	<value value="3" name="PERF_RBBM_RAS_BUSY"/>
+	<value value="4" name="PERF_RBBM_PC_DCALL_BUSY"/>
+	<value value="5" name="PERF_RBBM_PC_VSD_BUSY"/>
+	<value value="6" name="PERF_RBBM_STATUS_MASKED"/>
+	<value value="7" name="PERF_RBBM_COM_BUSY"/>
+	<value value="8" name="PERF_RBBM_DCOM_BUSY"/>
+	<value value="9" name="PERF_RBBM_VBIF_BUSY"/>
+	<value value="10" name="PERF_RBBM_VSC_BUSY"/>
+	<value value="11" name="PERF_RBBM_TESS_BUSY"/>
+	<value value="12" name="PERF_RBBM_UCHE_BUSY"/>
+	<value value="13" name="PERF_RBBM_HLSQ_BUSY"/>
+</enum>
+
+<enum name="a6xx_pc_perfcounter_select">
+	<value value="0" name="PERF_PC_BUSY_CYCLES"/>
+	<value value="1" name="PERF_PC_WORKING_CYCLES"/>
+	<value value="2" name="PERF_PC_STALL_CYCLES_VFD"/>
+	<value value="3" name="PERF_PC_STALL_CYCLES_TSE"/>
+	<value value="4" name="PERF_PC_STALL_CYCLES_VPC"/>
+	<value value="5" name="PERF_PC_STALL_CYCLES_UCHE"/>
+	<value value="6" name="PERF_PC_STALL_CYCLES_TESS"/>
+	<value value="7" name="PERF_PC_STALL_CYCLES_TSE_ONLY"/>
+	<value value="8" name="PERF_PC_STALL_CYCLES_VPC_ONLY"/>
+	<value value="9" name="PERF_PC_PASS1_TF_STALL_CYCLES"/>
+	<value value="10" name="PERF_PC_STARVE_CYCLES_FOR_INDEX"/>
+	<value value="11" name="PERF_PC_STARVE_CYCLES_FOR_TESS_FACTOR"/>
+	<value value="12" name="PERF_PC_STARVE_CYCLES_FOR_VIZ_STREAM"/>
+	<value value="13" name="PERF_PC_STARVE_CYCLES_FOR_POSITION"/>
+	<value value="14" name="PERF_PC_STARVE_CYCLES_DI"/>
+	<value value="15" name="PERF_PC_VIS_STREAMS_LOADED"/>
+	<value value="16" name="PERF_PC_INSTANCES"/>
+	<value value="17" name="PERF_PC_VPC_PRIMITIVES"/>
+	<value value="18" name="PERF_PC_DEAD_PRIM"/>
+	<value value="19" name="PERF_PC_LIVE_PRIM"/>
+	<value value="20" name="PERF_PC_VERTEX_HITS"/>
+	<value value="21" name="PERF_PC_IA_VERTICES"/>
+	<value value="22" name="PERF_PC_IA_PRIMITIVES"/>
+	<value value="23" name="PERF_PC_GS_PRIMITIVES"/>
+	<value value="24" name="PERF_PC_HS_INVOCATIONS"/>
+	<value value="25" name="PERF_PC_DS_INVOCATIONS"/>
+	<value value="26" name="PERF_PC_VS_INVOCATIONS"/>
+	<value value="27" name="PERF_PC_GS_INVOCATIONS"/>
+	<value value="28" name="PERF_PC_DS_PRIMITIVES"/>
+	<value value="29" name="PERF_PC_VPC_POS_DATA_TRANSACTION"/>
+	<value value="30" name="PERF_PC_3D_DRAWCALLS"/>
+	<value value="31" name="PERF_PC_2D_DRAWCALLS"/>
+	<value value="32" name="PERF_PC_NON_DRAWCALL_GLOBAL_EVENTS"/>
+	<value value="33" name="PERF_TESS_BUSY_CYCLES"/>
+	<value value="34" name="PERF_TESS_WORKING_CYCLES"/>
+	<value value="35" name="PERF_TESS_STALL_CYCLES_PC"/>
+	<value value="36" name="PERF_TESS_STARVE_CYCLES_PC"/>
+	<value value="37" name="PERF_PC_TSE_TRANSACTION"/>
+	<value value="38" name="PERF_PC_TSE_VERTEX"/>
+	<value value="39" name="PERF_PC_TESS_PC_UV_TRANS"/>
+	<value value="40" name="PERF_PC_TESS_PC_UV_PATCHES"/>
+	<value value="41" name="PERF_PC_TESS_FACTOR_TRANS"/>
+</enum>
+
+<enum name="a6xx_vfd_perfcounter_select">
+	<value value="0" name="PERF_VFD_BUSY_CYCLES"/>
+	<value value="1" name="PERF_VFD_STALL_CYCLES_UCHE"/>
+	<value value="2" name="PERF_VFD_STALL_CYCLES_VPC_ALLOC"/>
+	<value value="3" name="PERF_VFD_STALL_CYCLES_SP_INFO"/>
+	<value value="4" name="PERF_VFD_STALL_CYCLES_SP_ATTR"/>
+	<value value="5" name="PERF_VFD_STARVE_CYCLES_UCHE"/>
+	<value value="6" name="PERF_VFD_RBUFFER_FULL"/>
+	<value value="7" name="PERF_VFD_ATTR_INFO_FIFO_FULL"/>
+	<value value="8" name="PERF_VFD_DECODED_ATTRIBUTE_BYTES"/>
+	<value value="9" name="PERF_VFD_NUM_ATTRIBUTES"/>
+	<value value="10" name="PERF_VFD_UPPER_SHADER_FIBERS"/>
+	<value value="11" name="PERF_VFD_LOWER_SHADER_FIBERS"/>
+	<value value="12" name="PERF_VFD_MODE_0_FIBERS"/>
+	<value value="13" name="PERF_VFD_MODE_1_FIBERS"/>
+	<value value="14" name="PERF_VFD_MODE_2_FIBERS"/>
+	<value value="15" name="PERF_VFD_MODE_3_FIBERS"/>
+	<value value="16" name="PERF_VFD_MODE_4_FIBERS"/>
+	<value value="17" name="PERF_VFD_TOTAL_VERTICES"/>
+	<value value="18" name="PERF_VFDP_STALL_CYCLES_VFD"/>
+	<value value="19" name="PERF_VFDP_STALL_CYCLES_VFD_INDEX"/>
+	<value value="20" name="PERF_VFDP_STALL_CYCLES_VFD_PROG"/>
+	<value value="21" name="PERF_VFDP_STARVE_CYCLES_PC"/>
+	<value value="22" name="PERF_VFDP_VS_STAGE_WAVES"/>
+</enum>
+
+<enum name="a6xx_hlsq_perfcounter_select">
+	<value value="0" name="PERF_HLSQ_BUSY_CYCLES"/>
+	<value value="1" name="PERF_HLSQ_STALL_CYCLES_UCHE"/>
+	<value value="2" name="PERF_HLSQ_STALL_CYCLES_SP_STATE"/>
+	<value value="3" name="PERF_HLSQ_STALL_CYCLES_SP_FS_STAGE"/>
+	<value value="4" name="PERF_HLSQ_UCHE_LATENCY_CYCLES"/>
+	<value value="5" name="PERF_HLSQ_UCHE_LATENCY_COUNT"/>
+	<value value="6" name="PERF_HLSQ_FS_STAGE_1X_WAVES"/>
+	<value value="7" name="PERF_HLSQ_FS_STAGE_2X_WAVES"/>
+	<value value="8" name="PERF_HLSQ_QUADS"/>
+	<value value="9" name="PERF_HLSQ_CS_INVOCATIONS"/>
+	<value value="10" name="PERF_HLSQ_COMPUTE_DRAWCALLS"/>
+	<value value="11" name="PERF_HLSQ_FS_DATA_WAIT_PROGRAMMING"/>
+	<value value="12" name="PERF_HLSQ_DUAL_FS_PROG_ACTIVE"/>
+	<value value="13" name="PERF_HLSQ_DUAL_VS_PROG_ACTIVE"/>
+	<value value="14" name="PERF_HLSQ_FS_BATCH_COUNT_ZERO"/>
+	<value value="15" name="PERF_HLSQ_VS_BATCH_COUNT_ZERO"/>
+	<value value="16" name="PERF_HLSQ_WAVE_PENDING_NO_QUAD"/>
+	<value value="17" name="PERF_HLSQ_WAVE_PENDING_NO_PRIM_BASE"/>
+	<value value="18" name="PERF_HLSQ_STALL_CYCLES_VPC"/>
+	<value value="19" name="PERF_HLSQ_PIXELS"/>
+	<value value="20" name="PERF_HLSQ_DRAW_MODE_SWITCH_VSFS_SYNC"/>
+</enum>
+
+<enum name="a6xx_vpc_perfcounter_select">
+	<value value="0" name="PERF_VPC_BUSY_CYCLES"/>
+	<value value="1" name="PERF_VPC_WORKING_CYCLES"/>
+	<value value="2" name="PERF_VPC_STALL_CYCLES_UCHE"/>
+	<value value="3" name="PERF_VPC_STALL_CYCLES_VFD_WACK"/>
+	<value value="4" name="PERF_VPC_STALL_CYCLES_HLSQ_PRIM_ALLOC"/>
+	<value value="5" name="PERF_VPC_STALL_CYCLES_PC"/>
+	<value value="6" name="PERF_VPC_STALL_CYCLES_SP_LM"/>
+	<value value="7" name="PERF_VPC_STARVE_CYCLES_SP"/>
+	<value value="8" name="PERF_VPC_STARVE_CYCLES_LRZ"/>
+	<value value="9" name="PERF_VPC_PC_PRIMITIVES"/>
+	<value value="10" name="PERF_VPC_SP_COMPONENTS"/>
+	<value value="11" name="PERF_VPC_STALL_CYCLES_VPCRAM_POS"/>
+	<value value="12" name="PERF_VPC_LRZ_ASSIGN_PRIMITIVES"/>
+	<value value="13" name="PERF_VPC_RB_VISIBLE_PRIMITIVES"/>
+	<value value="14" name="PERF_VPC_LM_TRANSACTION"/>
+	<value value="15" name="PERF_VPC_STREAMOUT_TRANSACTION"/>
+	<value value="16" name="PERF_VPC_VS_BUSY_CYCLES"/>
+	<value value="17" name="PERF_VPC_PS_BUSY_CYCLES"/>
+	<value value="18" name="PERF_VPC_VS_WORKING_CYCLES"/>
+	<value value="19" name="PERF_VPC_PS_WORKING_CYCLES"/>
+	<value value="20" name="PERF_VPC_STARVE_CYCLES_RB"/>
+	<value value="21" name="PERF_VPC_NUM_VPCRAM_READ_POS"/>
+	<value value="22" name="PERF_VPC_WIT_FULL_CYCLES"/>
+	<value value="23" name="PERF_VPC_VPCRAM_FULL_CYCLES"/>
+	<value value="24" name="PERF_VPC_LM_FULL_WAIT_FOR_INTP_END"/>
+	<value value="25" name="PERF_VPC_NUM_VPCRAM_WRITE"/>
+	<value value="26" name="PERF_VPC_NUM_VPCRAM_READ_SO"/>
+	<value value="27" name="PERF_VPC_NUM_ATTR_REQ_LM"/>
+</enum>
+
+<enum name="a6xx_tse_perfcounter_select">
+	<value value="0" name="PERF_TSE_BUSY_CYCLES"/>
+	<value value="1" name="PERF_TSE_CLIPPING_CYCLES"/>
+	<value value="2" name="PERF_TSE_STALL_CYCLES_RAS"/>
+	<value value="3" name="PERF_TSE_STALL_CYCLES_LRZ_BARYPLANE"/>
+	<value value="4" name="PERF_TSE_STALL_CYCLES_LRZ_ZPLANE"/>
+	<value value="5" name="PERF_TSE_STARVE_CYCLES_PC"/>
+	<value value="6" name="PERF_TSE_INPUT_PRIM"/>
+	<value value="7" name="PERF_TSE_INPUT_NULL_PRIM"/>
+	<value value="8" name="PERF_TSE_TRIVAL_REJ_PRIM"/>
+	<value value="9" name="PERF_TSE_CLIPPED_PRIM"/>
+	<value value="10" name="PERF_TSE_ZERO_AREA_PRIM"/>
+	<value value="11" name="PERF_TSE_FACENESS_CULLED_PRIM"/>
+	<value value="12" name="PERF_TSE_ZERO_PIXEL_PRIM"/>
+	<value value="13" name="PERF_TSE_OUTPUT_NULL_PRIM"/>
+	<value value="14" name="PERF_TSE_OUTPUT_VISIBLE_PRIM"/>
+	<value value="15" name="PERF_TSE_CINVOCATION"/>
+	<value value="16" name="PERF_TSE_CPRIMITIVES"/>
+	<value value="17" name="PERF_TSE_2D_INPUT_PRIM"/>
+	<value value="18" name="PERF_TSE_2D_ALIVE_CYCLES"/>
+	<value value="19" name="PERF_TSE_CLIP_PLANES"/>
+</enum>
+
+<enum name="a6xx_ras_perfcounter_select">
+	<value value="0" name="PERF_RAS_BUSY_CYCLES"/>
+	<value value="1" name="PERF_RAS_SUPERTILE_ACTIVE_CYCLES"/>
+	<value value="2" name="PERF_RAS_STALL_CYCLES_LRZ"/>
+	<value value="3" name="PERF_RAS_STARVE_CYCLES_TSE"/>
+	<value value="4" name="PERF_RAS_SUPER_TILES"/>
+	<value value="5" name="PERF_RAS_8X4_TILES"/>
+	<value value="6" name="PERF_RAS_MASKGEN_ACTIVE"/>
+	<value value="7" name="PERF_RAS_FULLY_COVERED_SUPER_TILES"/>
+	<value value="8" name="PERF_RAS_FULLY_COVERED_8X4_TILES"/>
+	<value value="9" name="PERF_RAS_PRIM_KILLED_INVISILBE"/>
+	<value value="10" name="PERF_RAS_SUPERTILE_GEN_ACTIVE_CYCLES"/>
+	<value value="11" name="PERF_RAS_LRZ_INTF_WORKING_CYCLES"/>
+	<value value="12" name="PERF_RAS_BLOCKS"/>
+</enum>
+
+<enum name="a6xx_uche_perfcounter_select">
+	<value value="0" name="PERF_UCHE_BUSY_CYCLES"/>
+	<value value="1" name="PERF_UCHE_STALL_CYCLES_ARBITER"/>
+	<value value="2" name="PERF_UCHE_VBIF_LATENCY_CYCLES"/>
+	<value value="3" name="PERF_UCHE_VBIF_LATENCY_SAMPLES"/>
+	<value value="4" name="PERF_UCHE_VBIF_READ_BEATS_TP"/>
+	<value value="5" name="PERF_UCHE_VBIF_READ_BEATS_VFD"/>
+	<value value="6" name="PERF_UCHE_VBIF_READ_BEATS_HLSQ"/>
+	<value value="7" name="PERF_UCHE_VBIF_READ_BEATS_LRZ"/>
+	<value value="8" name="PERF_UCHE_VBIF_READ_BEATS_SP"/>
+	<value value="9" name="PERF_UCHE_READ_REQUESTS_TP"/>
+	<value value="10" name="PERF_UCHE_READ_REQUESTS_VFD"/>
+	<value value="11" name="PERF_UCHE_READ_REQUESTS_HLSQ"/>
+	<value value="12" name="PERF_UCHE_READ_REQUESTS_LRZ"/>
+	<value value="13" name="PERF_UCHE_READ_REQUESTS_SP"/>
+	<value value="14" name="PERF_UCHE_WRITE_REQUESTS_LRZ"/>
+	<value value="15" name="PERF_UCHE_WRITE_REQUESTS_SP"/>
+	<value value="16" name="PERF_UCHE_WRITE_REQUESTS_VPC"/>
+	<value value="17" name="PERF_UCHE_WRITE_REQUESTS_VSC"/>
+	<value value="18" name="PERF_UCHE_EVICTS"/>
+	<value value="19" name="PERF_UCHE_BANK_REQ0"/>
+	<value value="20" name="PERF_UCHE_BANK_REQ1"/>
+	<value value="21" name="PERF_UCHE_BANK_REQ2"/>
+	<value value="22" name="PERF_UCHE_BANK_REQ3"/>
+	<value value="23" name="PERF_UCHE_BANK_REQ4"/>
+	<value value="24" name="PERF_UCHE_BANK_REQ5"/>
+	<value value="25" name="PERF_UCHE_BANK_REQ6"/>
+	<value value="26" name="PERF_UCHE_BANK_REQ7"/>
+	<value value="27" name="PERF_UCHE_VBIF_READ_BEATS_CH0"/>
+	<value value="28" name="PERF_UCHE_VBIF_READ_BEATS_CH1"/>
+	<value value="29" name="PERF_UCHE_GMEM_READ_BEATS"/>
+	<value value="30" name="PERF_UCHE_TPH_REF_FULL"/>
+	<value value="31" name="PERF_UCHE_TPH_VICTIM_FULL"/>
+	<value value="32" name="PERF_UCHE_TPH_EXT_FULL"/>
+	<value value="33" name="PERF_UCHE_VBIF_STALL_WRITE_DATA"/>
+	<value value="34" name="PERF_UCHE_DCMP_LATENCY_SAMPLES"/>
+	<value value="35" name="PERF_UCHE_DCMP_LATENCY_CYCLES"/>
+	<value value="36" name="PERF_UCHE_VBIF_READ_BEATS_PC"/>
+	<value value="37" name="PERF_UCHE_READ_REQUESTS_PC"/>
+	<value value="38" name="PERF_UCHE_RAM_READ_REQ"/>
+	<value value="39" name="PERF_UCHE_RAM_WRITE_REQ"/>
+</enum>
+
+<enum name="a6xx_tp_perfcounter_select">
+	<value value="0" name="PERF_TP_BUSY_CYCLES"/>
+	<value value="1" name="PERF_TP_STALL_CYCLES_UCHE"/>
+	<value value="2" name="PERF_TP_LATENCY_CYCLES"/>
+	<value value="3" name="PERF_TP_LATENCY_TRANS"/>
+	<value value="4" name="PERF_TP_FLAG_CACHE_REQUEST_SAMPLES"/>
+	<value value="5" name="PERF_TP_FLAG_CACHE_REQUEST_LATENCY"/>
+	<value value="6" name="PERF_TP_L1_CACHELINE_REQUESTS"/>
+	<value value="7" name="PERF_TP_L1_CACHELINE_MISSES"/>
+	<value value="8" name="PERF_TP_SP_TP_TRANS"/>
+	<value value="9" name="PERF_TP_TP_SP_TRANS"/>
+	<value value="10" name="PERF_TP_OUTPUT_PIXELS"/>
+	<value value="11" name="PERF_TP_FILTER_WORKLOAD_16BIT"/>
+	<value value="12" name="PERF_TP_FILTER_WORKLOAD_32BIT"/>
+	<value value="13" name="PERF_TP_QUADS_RECEIVED"/>
+	<value value="14" name="PERF_TP_QUADS_OFFSET"/>
+	<value value="15" name="PERF_TP_QUADS_SHADOW"/>
+	<value value="16" name="PERF_TP_QUADS_ARRAY"/>
+	<value value="17" name="PERF_TP_QUADS_GRADIENT"/>
+	<value value="18" name="PERF_TP_QUADS_1D"/>
+	<value value="19" name="PERF_TP_QUADS_2D"/>
+	<value value="20" name="PERF_TP_QUADS_BUFFER"/>
+	<value value="21" name="PERF_TP_QUADS_3D"/>
+	<value value="22" name="PERF_TP_QUADS_CUBE"/>
+	<value value="23" name="PERF_TP_DIVERGENT_QUADS_RECEIVED"/>
+	<value value="24" name="PERF_TP_PRT_NON_RESIDENT_EVENTS"/>
+	<value value="25" name="PERF_TP_OUTPUT_PIXELS_POINT"/>
+	<value value="26" name="PERF_TP_OUTPUT_PIXELS_BILINEAR"/>
+	<value value="27" name="PERF_TP_OUTPUT_PIXELS_MIP"/>
+	<value value="28" name="PERF_TP_OUTPUT_PIXELS_ANISO"/>
+	<value value="29" name="PERF_TP_OUTPUT_PIXELS_ZERO_LOD"/>
+	<value value="30" name="PERF_TP_FLAG_CACHE_REQUESTS"/>
+	<value value="31" name="PERF_TP_FLAG_CACHE_MISSES"/>
+	<value value="32" name="PERF_TP_L1_5_L2_REQUESTS"/>
+	<value value="33" name="PERF_TP_2D_OUTPUT_PIXELS"/>
+	<value value="34" name="PERF_TP_2D_OUTPUT_PIXELS_POINT"/>
+	<value value="35" name="PERF_TP_2D_OUTPUT_PIXELS_BILINEAR"/>
+	<value value="36" name="PERF_TP_2D_FILTER_WORKLOAD_16BIT"/>
+	<value value="37" name="PERF_TP_2D_FILTER_WORKLOAD_32BIT"/>
+	<value value="38" name="PERF_TP_TPA2TPC_TRANS"/>
+	<value value="39" name="PERF_TP_L1_MISSES_ASTC_1TILE"/>
+	<value value="40" name="PERF_TP_L1_MISSES_ASTC_2TILE"/>
+	<value value="41" name="PERF_TP_L1_MISSES_ASTC_4TILE"/>
+	<value value="42" name="PERF_TP_L1_5_L2_COMPRESS_REQS"/>
+	<value value="43" name="PERF_TP_L1_5_L2_COMPRESS_MISS"/>
+	<value value="44" name="PERF_TP_L1_BANK_CONFLICT"/>
+	<value value="45" name="PERF_TP_L1_5_MISS_LATENCY_CYCLES"/>
+	<value value="46" name="PERF_TP_L1_5_MISS_LATENCY_TRANS"/>
+	<value value="47" name="PERF_TP_QUADS_CONSTANT_MULTIPLIED"/>
+	<value value="48" name="PERF_TP_FRONTEND_WORKING_CYCLES"/>
+	<value value="49" name="PERF_TP_L1_TAG_WORKING_CYCLES"/>
+	<value value="50" name="PERF_TP_L1_DATA_WRITE_WORKING_CYCLES"/>
+	<value value="51" name="PERF_TP_PRE_L1_DECOM_WORKING_CYCLES"/>
+	<value value="52" name="PERF_TP_BACKEND_WORKING_CYCLES"/>
+	<value value="53" name="PERF_TP_FLAG_CACHE_WORKING_CYCLES"/>
+	<value value="54" name="PERF_TP_L1_5_CACHE_WORKING_CYCLES"/>
+	<value value="55" name="PERF_TP_STARVE_CYCLES_SP"/>
+	<value value="56" name="PERF_TP_STARVE_CYCLES_UCHE"/>
+</enum>
+
+<enum name="a6xx_sp_perfcounter_select">
+	<value value="0" name="PERF_SP_BUSY_CYCLES"/>
+	<value value="1" name="PERF_SP_ALU_WORKING_CYCLES"/>
+	<value value="2" name="PERF_SP_EFU_WORKING_CYCLES"/>
+	<value value="3" name="PERF_SP_STALL_CYCLES_VPC"/>
+	<value value="4" name="PERF_SP_STALL_CYCLES_TP"/>
+	<value value="5" name="PERF_SP_STALL_CYCLES_UCHE"/>
+	<value value="6" name="PERF_SP_STALL_CYCLES_RB"/>
+	<value value="7" name="PERF_SP_NON_EXECUTION_CYCLES"/>
+	<value value="8" name="PERF_SP_WAVE_CONTEXTS"/>
+	<value value="9" name="PERF_SP_WAVE_CONTEXT_CYCLES"/>
+	<value value="10" name="PERF_SP_FS_STAGE_WAVE_CYCLES"/>
+	<value value="11" name="PERF_SP_FS_STAGE_WAVE_SAMPLES"/>
+	<value value="12" name="PERF_SP_VS_STAGE_WAVE_CYCLES"/>
+	<value value="13" name="PERF_SP_VS_STAGE_WAVE_SAMPLES"/>
+	<value value="14" name="PERF_SP_FS_STAGE_DURATION_CYCLES"/>
+	<value value="15" name="PERF_SP_VS_STAGE_DURATION_CYCLES"/>
+	<value value="16" name="PERF_SP_WAVE_CTRL_CYCLES"/>
+	<value value="17" name="PERF_SP_WAVE_LOAD_CYCLES"/>
+	<value value="18" name="PERF_SP_WAVE_EMIT_CYCLES"/>
+	<value value="19" name="PERF_SP_WAVE_NOP_CYCLES"/>
+	<value value="20" name="PERF_SP_WAVE_WAIT_CYCLES"/>
+	<value value="21" name="PERF_SP_WAVE_FETCH_CYCLES"/>
+	<value value="22" name="PERF_SP_WAVE_IDLE_CYCLES"/>
+	<value value="23" name="PERF_SP_WAVE_END_CYCLES"/>
+	<value value="24" name="PERF_SP_WAVE_LONG_SYNC_CYCLES"/>
+	<value value="25" name="PERF_SP_WAVE_SHORT_SYNC_CYCLES"/>
+	<value value="26" name="PERF_SP_WAVE_JOIN_CYCLES"/>
+	<value value="27" name="PERF_SP_LM_LOAD_INSTRUCTIONS"/>
+	<value value="28" name="PERF_SP_LM_STORE_INSTRUCTIONS"/>
+	<value value="29" name="PERF_SP_LM_ATOMICS"/>
+	<value value="30" name="PERF_SP_GM_LOAD_INSTRUCTIONS"/>
+	<value value="31" name="PERF_SP_GM_STORE_INSTRUCTIONS"/>
+	<value value="32" name="PERF_SP_GM_ATOMICS"/>
+	<value value="33" name="PERF_SP_VS_STAGE_TEX_INSTRUCTIONS"/>
+	<value value="34" name="PERF_SP_VS_STAGE_EFU_INSTRUCTIONS"/>
+	<value value="35" name="PERF_SP_VS_STAGE_FULL_ALU_INSTRUCTIONS"/>
+	<value value="36" name="PERF_SP_VS_STAGE_HALF_ALU_INSTRUCTIONS"/>
+	<value value="37" name="PERF_SP_FS_STAGE_TEX_INSTRUCTIONS"/>
+	<value value="38" name="PERF_SP_FS_STAGE_CFLOW_INSTRUCTIONS"/>
+	<value value="39" name="PERF_SP_FS_STAGE_EFU_INSTRUCTIONS"/>
+	<value value="40" name="PERF_SP_FS_STAGE_FULL_ALU_INSTRUCTIONS"/>
+	<value value="41" name="PERF_SP_FS_STAGE_HALF_ALU_INSTRUCTIONS"/>
+	<value value="42" name="PERF_SP_FS_STAGE_BARY_INSTRUCTIONS"/>
+	<value value="43" name="PERF_SP_VS_INSTRUCTIONS"/>
+	<value value="44" name="PERF_SP_FS_INSTRUCTIONS"/>
+	<value value="45" name="PERF_SP_ADDR_LOCK_COUNT"/>
+	<value value="46" name="PERF_SP_UCHE_READ_TRANS"/>
+	<value value="47" name="PERF_SP_UCHE_WRITE_TRANS"/>
+	<value value="48" name="PERF_SP_EXPORT_VPC_TRANS"/>
+	<value value="49" name="PERF_SP_EXPORT_RB_TRANS"/>
+	<value value="50" name="PERF_SP_PIXELS_KILLED"/>
+	<value value="51" name="PERF_SP_ICL1_REQUESTS"/>
+	<value value="52" name="PERF_SP_ICL1_MISSES"/>
+	<value value="53" name="PERF_SP_HS_INSTRUCTIONS"/>
+	<value value="54" name="PERF_SP_DS_INSTRUCTIONS"/>
+	<value value="55" name="PERF_SP_GS_INSTRUCTIONS"/>
+	<value value="56" name="PERF_SP_CS_INSTRUCTIONS"/>
+	<value value="57" name="PERF_SP_GPR_READ"/>
+	<value value="58" name="PERF_SP_GPR_WRITE"/>
+	<value value="59" name="PERF_SP_FS_STAGE_HALF_EFU_INSTRUCTIONS"/>
+	<value value="60" name="PERF_SP_VS_STAGE_HALF_EFU_INSTRUCTIONS"/>
+	<value value="61" name="PERF_SP_LM_BANK_CONFLICTS"/>
+	<value value="62" name="PERF_SP_TEX_CONTROL_WORKING_CYCLES"/>
+	<value value="63" name="PERF_SP_LOAD_CONTROL_WORKING_CYCLES"/>
+	<value value="64" name="PERF_SP_FLOW_CONTROL_WORKING_CYCLES"/>
+	<value value="65" name="PERF_SP_LM_WORKING_CYCLES"/>
+	<value value="66" name="PERF_SP_DISPATCHER_WORKING_CYCLES"/>
+	<value value="67" name="PERF_SP_SEQUENCER_WORKING_CYCLES"/>
+	<value value="68" name="PERF_SP_LOW_EFFICIENCY_STARVED_BY_TP"/>
+	<value value="69" name="PERF_SP_STARVE_CYCLES_HLSQ"/>
+	<value value="70" name="PERF_SP_NON_EXECUTION_LS_CYCLES"/>
+	<value value="71" name="PERF_SP_WORKING_EU"/>
+	<value value="72" name="PERF_SP_ANY_EU_WORKING"/>
+	<value value="73" name="PERF_SP_WORKING_EU_FS_STAGE"/>
+	<value value="74" name="PERF_SP_ANY_EU_WORKING_FS_STAGE"/>
+	<value value="75" name="PERF_SP_WORKING_EU_VS_STAGE"/>
+	<value value="76" name="PERF_SP_ANY_EU_WORKING_VS_STAGE"/>
+	<value value="77" name="PERF_SP_WORKING_EU_CS_STAGE"/>
+	<value value="78" name="PERF_SP_ANY_EU_WORKING_CS_STAGE"/>
+	<value value="79" name="PERF_SP_GPR_READ_PREFETCH"/>
+	<value value="80" name="PERF_SP_GPR_READ_CONFLICT"/>
+	<value value="81" name="PERF_SP_GPR_WRITE_CONFLICT"/>
+	<value value="82" name="PERF_SP_GM_LOAD_LATENCY_CYCLES"/>
+	<value value="83" name="PERF_SP_GM_LOAD_LATENCY_SAMPLES"/>
+	<value value="84" name="PERF_SP_EXECUTABLE_WAVES"/>
+</enum>
+
+<enum name="a6xx_rb_perfcounter_select">
+	<value value="0" name="PERF_RB_BUSY_CYCLES"/>
+	<value value="1" name="PERF_RB_STALL_CYCLES_HLSQ"/>
+	<value value="2" name="PERF_RB_STALL_CYCLES_FIFO0_FULL"/>
+	<value value="3" name="PERF_RB_STALL_CYCLES_FIFO1_FULL"/>
+	<value value="4" name="PERF_RB_STALL_CYCLES_FIFO2_FULL"/>
+	<value value="5" name="PERF_RB_STARVE_CYCLES_SP"/>
+	<value value="6" name="PERF_RB_STARVE_CYCLES_LRZ_TILE"/>
+	<value value="7" name="PERF_RB_STARVE_CYCLES_CCU"/>
+	<value value="8" name="PERF_RB_STARVE_CYCLES_Z_PLANE"/>
+	<value value="9" name="PERF_RB_STARVE_CYCLES_BARY_PLANE"/>
+	<value value="10" name="PERF_RB_Z_WORKLOAD"/>
+	<value value="11" name="PERF_RB_HLSQ_ACTIVE"/>
+	<value value="12" name="PERF_RB_Z_READ"/>
+	<value value="13" name="PERF_RB_Z_WRITE"/>
+	<value value="14" name="PERF_RB_C_READ"/>
+	<value value="15" name="PERF_RB_C_WRITE"/>
+	<value value="16" name="PERF_RB_TOTAL_PASS"/>
+	<value value="17" name="PERF_RB_Z_PASS"/>
+	<value value="18" name="PERF_RB_Z_FAIL"/>
+	<value value="19" name="PERF_RB_S_FAIL"/>
+	<value value="20" name="PERF_RB_BLENDED_FXP_COMPONENTS"/>
+	<value value="21" name="PERF_RB_BLENDED_FP16_COMPONENTS"/>
+	<value value="22" name="PERF_RB_PS_INVOCATIONS"/>
+	<value value="23" name="PERF_RB_2D_ALIVE_CYCLES"/>
+	<value value="24" name="PERF_RB_2D_STALL_CYCLES_A2D"/>
+	<value value="25" name="PERF_RB_2D_STARVE_CYCLES_SRC"/>
+	<value value="26" name="PERF_RB_2D_STARVE_CYCLES_SP"/>
+	<value value="27" name="PERF_RB_2D_STARVE_CYCLES_DST"/>
+	<value value="28" name="PERF_RB_2D_VALID_PIXELS"/>
+	<value value="29" name="PERF_RB_3D_PIXELS"/>
+	<value value="30" name="PERF_RB_BLENDER_WORKING_CYCLES"/>
+	<value value="31" name="PERF_RB_ZPROC_WORKING_CYCLES"/>
+	<value value="32" name="PERF_RB_CPROC_WORKING_CYCLES"/>
+	<value value="33" name="PERF_RB_SAMPLER_WORKING_CYCLES"/>
+	<value value="34" name="PERF_RB_STALL_CYCLES_CCU_COLOR_READ"/>
+	<value value="35" name="PERF_RB_STALL_CYCLES_CCU_COLOR_WRITE"/>
+	<value value="36" name="PERF_RB_STALL_CYCLES_CCU_DEPTH_READ"/>
+	<value value="37" name="PERF_RB_STALL_CYCLES_CCU_DEPTH_WRITE"/>
+	<value value="38" name="PERF_RB_STALL_CYCLES_VPC"/>
+	<value value="39" name="PERF_RB_2D_INPUT_TRANS"/>
+	<value value="40" name="PERF_RB_2D_OUTPUT_RB_DST_TRANS"/>
+	<value value="41" name="PERF_RB_2D_OUTPUT_RB_SRC_TRANS"/>
+	<value value="42" name="PERF_RB_BLENDED_FP32_COMPONENTS"/>
+	<value value="43" name="PERF_RB_COLOR_PIX_TILES"/>
+	<value value="44" name="PERF_RB_STALL_CYCLES_CCU"/>
+	<value value="45" name="PERF_RB_EARLY_Z_ARB3_GRANT"/>
+	<value value="46" name="PERF_RB_LATE_Z_ARB3_GRANT"/>
+	<value value="47" name="PERF_RB_EARLY_Z_SKIP_GRANT"/>
+</enum>
+
+<enum name="a6xx_vsc_perfcounter_select">
+	<value value="0" name="PERF_VSC_BUSY_CYCLES"/>
+	<value value="1" name="PERF_VSC_WORKING_CYCLES"/>
+	<value value="2" name="PERF_VSC_STALL_CYCLES_UCHE"/>
+	<value value="3" name="PERF_VSC_EOT_NUM"/>
+	<value value="4" name="PERF_VSC_INPUT_TILES"/>
+</enum>
+
+<enum name="a6xx_ccu_perfcounter_select">
+	<value value="0" name="PERF_CCU_BUSY_CYCLES"/>
+	<value value="1" name="PERF_CCU_STALL_CYCLES_RB_DEPTH_RETURN"/>
+	<value value="2" name="PERF_CCU_STALL_CYCLES_RB_COLOR_RETURN"/>
+	<value value="3" name="PERF_CCU_STARVE_CYCLES_FLAG_RETURN"/>
+	<value value="4" name="PERF_CCU_DEPTH_BLOCKS"/>
+	<value value="5" name="PERF_CCU_COLOR_BLOCKS"/>
+	<value value="6" name="PERF_CCU_DEPTH_BLOCK_HIT"/>
+	<value value="7" name="PERF_CCU_COLOR_BLOCK_HIT"/>
+	<value value="8" name="PERF_CCU_PARTIAL_BLOCK_READ"/>
+	<value value="9" name="PERF_CCU_GMEM_READ"/>
+	<value value="10" name="PERF_CCU_GMEM_WRITE"/>
+	<value value="11" name="PERF_CCU_DEPTH_READ_FLAG0_COUNT"/>
+	<value value="12" name="PERF_CCU_DEPTH_READ_FLAG1_COUNT"/>
+	<value value="13" name="PERF_CCU_DEPTH_READ_FLAG2_COUNT"/>
+	<value value="14" name="PERF_CCU_DEPTH_READ_FLAG3_COUNT"/>
+	<value value="15" name="PERF_CCU_DEPTH_READ_FLAG4_COUNT"/>
+	<value value="16" name="PERF_CCU_DEPTH_READ_FLAG5_COUNT"/>
+	<value value="17" name="PERF_CCU_DEPTH_READ_FLAG6_COUNT"/>
+	<value value="18" name="PERF_CCU_DEPTH_READ_FLAG8_COUNT"/>
+	<value value="19" name="PERF_CCU_COLOR_READ_FLAG0_COUNT"/>
+	<value value="20" name="PERF_CCU_COLOR_READ_FLAG1_COUNT"/>
+	<value value="21" name="PERF_CCU_COLOR_READ_FLAG2_COUNT"/>
+	<value value="22" name="PERF_CCU_COLOR_READ_FLAG3_COUNT"/>
+	<value value="23" name="PERF_CCU_COLOR_READ_FLAG4_COUNT"/>
+	<value value="24" name="PERF_CCU_COLOR_READ_FLAG5_COUNT"/>
+	<value value="25" name="PERF_CCU_COLOR_READ_FLAG6_COUNT"/>
+	<value value="26" name="PERF_CCU_COLOR_READ_FLAG8_COUNT"/>
+	<value value="27" name="PERF_CCU_2D_RD_REQ"/>
+	<value value="28" name="PERF_CCU_2D_WR_REQ"/>
+</enum>
+
+<enum name="a6xx_lrz_perfcounter_select">
+	<value value="0" name="PERF_LRZ_BUSY_CYCLES"/>
+	<value value="1" name="PERF_LRZ_STARVE_CYCLES_RAS"/>
+	<value value="2" name="PERF_LRZ_STALL_CYCLES_RB"/>
+	<value value="3" name="PERF_LRZ_STALL_CYCLES_VSC"/>
+	<value value="4" name="PERF_LRZ_STALL_CYCLES_VPC"/>
+	<value value="5" name="PERF_LRZ_STALL_CYCLES_FLAG_PREFETCH"/>
+	<value value="6" name="PERF_LRZ_STALL_CYCLES_UCHE"/>
+	<value value="7" name="PERF_LRZ_LRZ_READ"/>
+	<value value="8" name="PERF_LRZ_LRZ_WRITE"/>
+	<value value="9" name="PERF_LRZ_READ_LATENCY"/>
+	<value value="10" name="PERF_LRZ_MERGE_CACHE_UPDATING"/>
+	<value value="11" name="PERF_LRZ_PRIM_KILLED_BY_MASKGEN"/>
+	<value value="12" name="PERF_LRZ_PRIM_KILLED_BY_LRZ"/>
+	<value value="13" name="PERF_LRZ_VISIBLE_PRIM_AFTER_LRZ"/>
+	<value value="14" name="PERF_LRZ_FULL_8X8_TILES"/>
+	<value value="15" name="PERF_LRZ_PARTIAL_8X8_TILES"/>
+	<value value="16" name="PERF_LRZ_TILE_KILLED"/>
+	<value value="17" name="PERF_LRZ_TOTAL_PIXEL"/>
+	<value value="18" name="PERF_LRZ_VISIBLE_PIXEL_AFTER_LRZ"/>
+	<value value="19" name="PERF_LRZ_FULLY_COVERED_TILES"/>
+	<value value="20" name="PERF_LRZ_PARTIAL_COVERED_TILES"/>
+	<value value="21" name="PERF_LRZ_FEEDBACK_ACCEPT"/>
+	<value value="22" name="PERF_LRZ_FEEDBACK_DISCARD"/>
+	<value value="23" name="PERF_LRZ_FEEDBACK_STALL"/>
+	<value value="24" name="PERF_LRZ_STALL_CYCLES_RB_ZPLANE"/>
+	<value value="25" name="PERF_LRZ_STALL_CYCLES_RB_BPLANE"/>
+	<value value="26" name="PERF_LRZ_STALL_CYCLES_VC"/>
+	<value value="27" name="PERF_LRZ_RAS_MASK_TRANS"/>
+</enum>
+
+<enum name="a6xx_cmp_perfcounter_select">
+	<value value="0" name="PERF_CMPDECMP_STALL_CYCLES_ARB"/>
+	<value value="1" name="PERF_CMPDECMP_VBIF_LATENCY_CYCLES"/>
+	<value value="2" name="PERF_CMPDECMP_VBIF_LATENCY_SAMPLES"/>
+	<value value="3" name="PERF_CMPDECMP_VBIF_READ_DATA_CCU"/>
+	<value value="4" name="PERF_CMPDECMP_VBIF_WRITE_DATA_CCU"/>
+	<value value="5" name="PERF_CMPDECMP_VBIF_READ_REQUEST"/>
+	<value value="6" name="PERF_CMPDECMP_VBIF_WRITE_REQUEST"/>
+	<value value="7" name="PERF_CMPDECMP_VBIF_READ_DATA"/>
+	<value value="8" name="PERF_CMPDECMP_VBIF_WRITE_DATA"/>
+	<value value="9" name="PERF_CMPDECMP_FLAG_FETCH_CYCLES"/>
+	<value value="10" name="PERF_CMPDECMP_FLAG_FETCH_SAMPLES"/>
+	<value value="11" name="PERF_CMPDECMP_DEPTH_WRITE_FLAG1_COUNT"/>
+	<value value="12" name="PERF_CMPDECMP_DEPTH_WRITE_FLAG2_COUNT"/>
+	<value value="13" name="PERF_CMPDECMP_DEPTH_WRITE_FLAG3_COUNT"/>
+	<value value="14" name="PERF_CMPDECMP_DEPTH_WRITE_FLAG4_COUNT"/>
+	<value value="15" name="PERF_CMPDECMP_DEPTH_WRITE_FLAG5_COUNT"/>
+	<value value="16" name="PERF_CMPDECMP_DEPTH_WRITE_FLAG6_COUNT"/>
+	<value value="17" name="PERF_CMPDECMP_DEPTH_WRITE_FLAG8_COUNT"/>
+	<value value="18" name="PERF_CMPDECMP_COLOR_WRITE_FLAG1_COUNT"/>
+	<value value="19" name="PERF_CMPDECMP_COLOR_WRITE_FLAG2_COUNT"/>
+	<value value="20" name="PERF_CMPDECMP_COLOR_WRITE_FLAG3_COUNT"/>
+	<value value="21" name="PERF_CMPDECMP_COLOR_WRITE_FLAG4_COUNT"/>
+	<value value="22" name="PERF_CMPDECMP_COLOR_WRITE_FLAG5_COUNT"/>
+	<value value="23" name="PERF_CMPDECMP_COLOR_WRITE_FLAG6_COUNT"/>
+	<value value="24" name="PERF_CMPDECMP_COLOR_WRITE_FLAG8_COUNT"/>
+	<value value="25" name="PERF_CMPDECMP_2D_STALL_CYCLES_VBIF_REQ"/>
+	<value value="26" name="PERF_CMPDECMP_2D_STALL_CYCLES_VBIF_WR"/>
+	<value value="27" name="PERF_CMPDECMP_2D_STALL_CYCLES_VBIF_RETURN"/>
+	<value value="28" name="PERF_CMPDECMP_2D_RD_DATA"/>
+	<value value="29" name="PERF_CMPDECMP_2D_WR_DATA"/>
+	<value value="30" name="PERF_CMPDECMP_VBIF_READ_DATA_UCHE_CH0"/>
+	<value value="31" name="PERF_CMPDECMP_VBIF_READ_DATA_UCHE_CH1"/>
+	<value value="32" name="PERF_CMPDECMP_2D_OUTPUT_TRANS"/>
+	<value value="33" name="PERF_CMPDECMP_VBIF_WRITE_DATA_UCHE"/>
+	<value value="34" name="PERF_CMPDECMP_DEPTH_WRITE_FLAG0_COUNT"/>
+	<value value="35" name="PERF_CMPDECMP_COLOR_WRITE_FLAG0_COUNT"/>
+	<value value="36" name="PERF_CMPDECMP_COLOR_WRITE_FLAGALPHA_COUNT"/>
+	<value value="37" name="PERF_CMPDECMP_2D_BUSY_CYCLES"/>
+	<value value="38" name="PERF_CMPDECMP_2D_REORDER_STARVE_CYCLES"/>
+	<value value="39" name="PERF_CMPDECMP_2D_PIXELS"/>
+</enum>
+
+<!--
+Used in a6xx_2d_blit_cntl.. the value mostly seems to correlate to the
+component type/size, so I think it relates to internal format used for
+blending?  The one exception is that 16b unorm and 32b float use the
+same value... maybe 16b unorm is uncommon enough that it was just easier
+to upconvert to 32b float internally?
+
+ 8b unorm:  10 (sometimes 0, is the high bit part of something else?)
+16b unorm:   4
+
+32b int:     7
+16b int:     6
+ 8b int:     5
+
+32b float:   4
+16b float:   3
+ -->
+<enum name="a6xx_2d_ifmt">
+	<value value="0x10" name="R2D_UNORM8"/>
+	<value value="0x7"  name="R2D_INT32"/>
+	<value value="0x6"  name="R2D_INT16"/>
+	<value value="0x5"  name="R2D_INT8"/>
+	<value value="0x4"  name="R2D_FLOAT32"/>
+	<value value="0x3"  name="R2D_FLOAT16"/>
+	<value value="0x1"  name="R2D_UNORM8_SRGB"/>
+	<value value="0x0"  name="R2D_RAW"/>
+</enum>
+
+<enum name="a6xx_ztest_mode">
+	<doc>Allow early z-test and early-lrz (if applicable)</doc>
+	<value value="0x0" name="A6XX_EARLY_Z"/>
+	<doc>Disable early z-test and early-lrz test (if applicable)</doc>
+	<value value="0x1" name="A6XX_LATE_Z"/>
+	<doc>
+		A special mode that allows early-lrz test but disables
+		early-z test.  Which might sound a bit funny, since
+		lrz-test happens before z-test.  But as long as a couple
+		conditions are maintained this allows using lrz-test in
+		cases where fragment shader has kill/discard:
+
+		1) Disable lrz-write in cases where it is uncertain during
+		   binning pass that a fragment will pass.  Ie.  if frag
+		   shader has-kill, writes-z, or alpha/stencil test is
+		   enabled.  (For correctness, lrz-write must be disabled
+		   when blend is enabled.)  This is analogous to how a
+		   z-prepass works.
+
+		2) Disable lrz-write and test if a depth-test direction
+		   reversal is detected.  Due to condition (1), the contents
+		   of the lrz buffer are a conservative estimation of the
+		   depth buffer during the draw pass.  Meaning that geometry
+		   that we know for certain will not be visible will not pass
+		   lrz-test.  But geometry which may be (or contributes to
+		   blend) will pass the lrz-test.
+
+		This allows us to keep early-lrz-test in cases where the frag
+		shader does not write-z (ie. we know the z-value before FS)
+		and does not have side-effects (image/ssbo writes, etc), but
+		does have kill/discard.  Which turns out to be a common
+		enough case that it is useful to keep early-lrz test against
+		the conservative lrz buffer to discard fragments that we
+		know will definitely not be visible.
+	</doc>
+	<value value="0x2" name="A6XX_EARLY_LRZ_LATE_Z"/>
+	<doc>Not a real hw value, used internally by mesa</doc>
+	<value value="0x3" name="A6XX_INVALID_ZTEST"/>
+</enum>
+
+<enum name="a6xx_tess_spacing">
+	<value value="0x0" name="TESS_EQUAL"/>
+	<value value="0x2" name="TESS_FRACTIONAL_ODD"/>
+	<value value="0x3" name="TESS_FRACTIONAL_EVEN"/>
+</enum>
+<enum name="a6xx_tess_output">
+	<value value="0x0" name="TESS_POINTS"/>
+	<value value="0x1" name="TESS_LINES"/>
+	<value value="0x2" name="TESS_CW_TRIS"/>
+	<value value="0x3" name="TESS_CCW_TRIS"/>
+</enum>
+
+<domain name="A6XX" width="32" prefix="variant" varset="chip">
+	<bitset name="A6XX_RBBM_INT_0_MASK" inline="no" varset="chip">
+		<bitfield name="RBBM_GPU_IDLE" pos="0" type="boolean"/>
+		<bitfield name="CP_AHB_ERROR" pos="1" type="boolean"/>
+		<bitfield name="CP_IPC_INTR_0" pos="4" type="boolean" variants="A7XX-"/>
+		<bitfield name="CP_IPC_INTR_1" pos="5" type="boolean" variants="A7XX-"/>
+		<bitfield name="RBBM_ATB_ASYNCFIFO_OVERFLOW" pos="6" type="boolean"/>
+		<bitfield name="RBBM_GPC_ERROR" pos="7" type="boolean"/>
+		<bitfield name="CP_SW" pos="8" type="boolean"/>
+		<bitfield name="CP_HW_ERROR" pos="9" type="boolean"/>
+		<bitfield name="CP_CCU_FLUSH_DEPTH_TS" pos="10" type="boolean"/>
+		<bitfield name="CP_CCU_FLUSH_COLOR_TS" pos="11" type="boolean"/>
+		<bitfield name="CP_CCU_RESOLVE_TS" pos="12" type="boolean"/>
+		<bitfield name="CP_IB2" pos="13" type="boolean"/>
+		<bitfield name="CP_IB1" pos="14" type="boolean"/>
+		<bitfield name="CP_RB" pos="15" type="boolean" variants="A6XX"/>
+		<!-- Same as above but different name??: -->
+		<bitfield name="PM4CPINTERRUPT" pos="15" type="boolean" variants="A7XX-"/>
+		<bitfield name="PM4CPINTERRUPTLPAC" pos="16" type="boolean" variants="A7XX-"/>
+		<bitfield name="CP_RB_DONE_TS" pos="17" type="boolean"/>
+		<bitfield name="CP_WT_DONE_TS" pos="18" type="boolean"/>
+		<bitfield name="CP_CACHE_FLUSH_TS" pos="20" type="boolean"/>
+		<bitfield name="CP_CACHE_FLUSH_TS_LPAC" pos="21" type="boolean" variants="A7XX-"/>
+		<bitfield name="RBBM_ATB_BUS_OVERFLOW" pos="22" type="boolean"/>
+		<bitfield name="RBBM_HANG_DETECT" pos="23" type="boolean"/>
+		<bitfield name="UCHE_OOB_ACCESS" pos="24" type="boolean"/>
+		<bitfield name="UCHE_TRAP_INTR" pos="25" type="boolean"/>
+		<bitfield name="DEBBUS_INTR_0" pos="26" type="boolean"/>
+		<bitfield name="DEBBUS_INTR_1" pos="27" type="boolean"/>
+		<bitfield name="TSBWRITEERROR" pos="28" type="boolean" variants="A7XX-"/>
+		<bitfield name="SWFUSEVIOLATION" pos="29" type="boolean" variants="A7XX-"/>
+		<bitfield name="ISDB_CPU_IRQ" pos="30" type="boolean"/>
+		<bitfield name="ISDB_UNDER_DEBUG" pos="31" type="boolean"/>
+	</bitset>
+
+	<!--
+		Note the _LPAC bits probably *actually* first appeared in a660, but the
+		_BV bits are new in a7xx
+	 -->
+	<bitset name="A6XX_CP_INT" varset="chip">
+		<bitfield name="CP_OPCODE_ERROR" pos="0" type="boolean"/>
+		<bitfield name="CP_UCODE_ERROR" pos="1" type="boolean"/>
+		<bitfield name="CP_HW_FAULT_ERROR" pos="2" type="boolean"/>
+		<bitfield name="CP_REGISTER_PROTECTION_ERROR" pos="4" type="boolean"/>
+		<bitfield name="CP_AHB_ERROR" pos="5" type="boolean"/>
+		<bitfield name="CP_VSD_PARITY_ERROR" pos="6" type="boolean"/>
+		<bitfield name="CP_ILLEGAL_INSTR_ERROR" pos="7" type="boolean"/>
+		<bitfield name="CP_OPCODE_ERROR_LPAC" pos="8" type="boolean" variants="A7XX-"/>
+		<bitfield name="CP_UCODE_ERROR_LPAC" pos="9" type="boolean" variants="A7XX-"/>
+		<bitfield name="CP_HW_FAULT_ERROR_LPAC" pos="10" type="boolean" variants="A7XX-"/>
+		<bitfield name="CP_REGISTER_PROTECTION_ERROR_LPAC" pos="11" type="boolean" variants="A7XX-"/>
+		<bitfield name="CP_ILLEGAL_INSTR_ERROR_LPAC" pos="12" type="boolean" variants="A7XX-"/>
+		<bitfield name="CP_OPCODE_ERROR_BV" pos="13" type="boolean" variants="A7XX-"/>
+		<bitfield name="CP_UCODE_ERROR_BV" pos="14" type="boolean" variants="A7XX-"/>
+		<bitfield name="CP_HW_FAULT_ERROR_BV" pos="15" type="boolean" variants="A7XX-"/>
+		<bitfield name="CP_REGISTER_PROTECTION_ERROR_BV" pos="16" type="boolean" variants="A7XX-"/>
+		<bitfield name="CP_ILLEGAL_INSTR_ERROR_BV" pos="17" type="boolean" variants="A7XX-"/>
+	</bitset>
+
+	<reg64 offset="0x0800" name="CP_RB_BASE"/>
+	<reg32 offset="0x0802" name="CP_RB_CNTL"/>
+	<reg64 offset="0x0804" name="CP_RB_RPTR_ADDR"/>
+	<reg32 offset="0x0806" name="CP_RB_RPTR"/>
+	<reg32 offset="0x0807" name="CP_RB_WPTR"/>
+	<reg32 offset="0x0808" name="CP_SQE_CNTL"/>
+	<reg32 offset="0x0812" name="CP_CP2GMU_STATUS">
+		<bitfield name="IFPC" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0821" name="CP_HW_FAULT"/>
+	<reg32 offset="0x0823" name="CP_INTERRUPT_STATUS" type="A6XX_CP_INT"/>
+	<reg32 offset="0x0824" name="CP_PROTECT_STATUS"/>
+	<reg32 offset="0x0825" name="CP_STATUS_1"/>
+	<reg64 offset="0x0830" name="CP_SQE_INSTR_BASE"/>
+	<reg32 offset="0x0840" name="CP_MISC_CNTL"/>
+	<reg32 offset="0x0844" name="CP_APRIV_CNTL">
+		<!-- Crashdumper writes -->
+		<bitfield pos="6" name="CDWRITE" type="boolean"/>
+		<!-- Crashdumper reads -->
+		<bitfield pos="5" name="CDREAD" type="boolean"/>
+
+		<!-- 4 is unknown -->
+
+		<!-- RPTR shadow writes -->
+		<bitfield pos="3" name="RBRPWB" type="boolean"/>
+		<!-- Memory accesses from PM4 packets in the ringbuffer -->
+		<bitfield pos="2" name="RBPRIVLEVEL" type="boolean"/>
+		<!-- Ringbuffer reads -->
+		<bitfield pos="1" name="RBFETCH" type="boolean"/>
+		<!-- Instruction cache fetches -->
+		<bitfield pos="0" name="ICACHE" type="boolean"/>
+	</reg32>
+	<!-- Preemptions taking longer than this threshold increment PERF_CP_LONG_PREEMPTIONS: -->
+	<reg32 offset="0x08C0" name="CP_PREEMPT_THRESHOLD"/>
+	<!-- all the threshold values seem to be in units of quad-dwords: -->
+	<reg32 offset="0x08C1" name="CP_ROQ_THRESHOLDS_1">
+		<doc>
+			b0..7 identifies where MRB data starts (and RB data ends)
+			b8.15 identifies where VSD data starts (and MRB data ends)
+			b16..23 identifies where IB1 data starts (and RB data ends)
+			b24..31 identifies where IB2 data starts (and IB1 data ends)
+		</doc>
+		<bitfield name="MRB_START" low="0" high="7" shr="2"/>
+		<bitfield name="VSD_START" low="8" high="15" shr="2"/>
+		<bitfield name="IB1_START" low="16" high="23" shr="2"/>
+		<bitfield name="IB2_START" low="24" high="31" shr="2"/>
+	</reg32>
+	<reg32 offset="0x08C2" name="CP_ROQ_THRESHOLDS_2">
+		<doc>
+			low bits identify where CP_SET_DRAW_STATE stateobj
+			processing starts (and IB2 data ends). I'm guessing
+			b8 is part of this since (from downstream kgsl):
+
+				/* ROQ sizes are twice as big on a640/a680 than on a630 */
+				if (adreno_is_a640(adreno_dev) || adreno_is_a680(adreno_dev)) {
+					kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_2, 0x02000140);
+					kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362C);
+				} ...
+		</doc>
+		<bitfield name="SDS_START" low="0" high="8" shr="2"/>
+		<!-- total ROQ size: -->
+		<bitfield name="ROQ_SIZE" low="16" high="31" shr="2"/>
+	</reg32>
+	<reg32 offset="0x08C3" name="CP_MEM_POOL_SIZE"/>
+	<reg32 offset="0x0841" name="CP_CHICKEN_DBG"/>
+	<reg32 offset="0x0842" name="CP_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0843" name="CP_DBG_ECO_CNTL"/>
+	<reg32 offset="0x084F" name="CP_PROTECT_CNTL">
+		<bitfield pos="3" name="LAST_SPAN_INF_RANGE" type="boolean"/>
+		<bitfield pos="1" name="ACCESS_FAULT_ON_VIOL_EN" type="boolean"/>
+		<bitfield pos="0" name="ACCESS_PROT_EN" type="boolean"/>
+	</reg32>
+
+	<array offset="0x0883" name="CP_SCRATCH" stride="1" length="8">
+		<reg32 offset="0x0" name="REG" type="uint"/>
+	</array>
+	<array offset="0x0850" name="CP_PROTECT" stride="1" length="32">
+		<reg32 offset="0x0" name="REG" type="a6x_cp_protect"/>
+	</array>
+
+	<reg32 offset="0x08A0" name="CP_CONTEXT_SWITCH_CNTL"/>
+	<reg64 offset="0x08A1" name="CP_CONTEXT_SWITCH_SMMU_INFO"/>
+	<reg64 offset="0x08A3" name="CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR"/>
+	<reg64 offset="0x08A5" name="CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR"/>
+	<reg64 offset="0x08A7" name="CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR"/>
+	<reg32 offset="0x08ab" name="CP_CONTEXT_SWITCH_LEVEL_STATUS" variants="A7XX-"/>
+	<array offset="0x08D0" name="CP_PERFCTR_CP_SEL" stride="1" length="14"/>
+	<array offset="0x08e0" name="CP_BV_PERFCTR_CP_SEL" stride="1" length="7" variants="A7XX-"/>
+	<reg64 offset="0x0900" name="CP_CRASH_SCRIPT_BASE"/>
+	<reg32 offset="0x0902" name="CP_CRASH_DUMP_CNTL"/>
+	<reg32 offset="0x0903" name="CP_CRASH_DUMP_STATUS"/>
+	<reg32 offset="0x0908" name="CP_SQE_STAT_ADDR"/>
+	<reg32 offset="0x0909" name="CP_SQE_STAT_DATA"/>
+	<reg32 offset="0x090A" name="CP_DRAW_STATE_ADDR"/>
+	<reg32 offset="0x090B" name="CP_DRAW_STATE_DATA"/>
+	<reg32 offset="0x090C" name="CP_ROQ_DBG_ADDR"/>
+	<reg32 offset="0x090D" name="CP_ROQ_DBG_DATA"/>
+	<reg32 offset="0x090E" name="CP_MEM_POOL_DBG_ADDR"/>
+	<reg32 offset="0x090F" name="CP_MEM_POOL_DBG_DATA"/>
+	<reg32 offset="0x0910" name="CP_SQE_UCODE_DBG_ADDR"/>
+	<reg32 offset="0x0911" name="CP_SQE_UCODE_DBG_DATA"/>
+	<reg64 offset="0x0928" name="CP_IB1_BASE"/>
+	<reg32 offset="0x092A" name="CP_IB1_REM_SIZE"/>
+	<reg64 offset="0x092B" name="CP_IB2_BASE"/>
+	<reg32 offset="0x092D" name="CP_IB2_REM_SIZE"/>
+	<!-- SDS == CP_SET_DRAW_STATE: -->
+	<reg64 offset="0x092e" name="CP_SDS_BASE"/>
+	<reg32 offset="0x0930" name="CP_SDS_REM_SIZE"/>
+	<!-- MRB == MEM_READ_ADDR/$addr in SQE firmware -->
+	<reg64 offset="0x0931" name="CP_MRB_BASE"/>
+	<reg32 offset="0x0933" name="CP_MRB_REM_SIZE"/>
+	<!--
+	VSD == Visibility Stream Decode
+	This is used by CP to read the draw stream and skip empty draws
+	-->
+	<reg64 offset="0x0934" name="CP_VSD_BASE"/>
+
+	<bitset name="a6xx_roq_stat" inline="yes">
+		<bitfield name="RPTR" low="0" high="9"/>
+		<bitfield name="WPTR" low="16" high="25"/>
+	</bitset>
+	<reg32 offset="0x0939" name="CP_ROQ_RB_STAT" type="a6xx_roq_stat"/>
+	<reg32 offset="0x093a" name="CP_ROQ_IB1_STAT" type="a6xx_roq_stat"/>
+	<reg32 offset="0x093b" name="CP_ROQ_IB2_STAT" type="a6xx_roq_stat"/>
+	<reg32 offset="0x093c" name="CP_ROQ_SDS_STAT" type="a6xx_roq_stat"/>
+	<reg32 offset="0x093d" name="CP_ROQ_MRB_STAT" type="a6xx_roq_stat"/>
+	<reg32 offset="0x093e" name="CP_ROQ_VSD_STAT" type="a6xx_roq_stat"/>
+
+	<reg32 offset="0x0943" name="CP_IB1_DWORDS"/>
+	<reg32 offset="0x0944" name="CP_IB2_DWORDS"/>
+	<reg32 offset="0x0945" name="CP_SDS_DWORDS"/>
+	<reg32 offset="0x0946" name="CP_MRB_DWORDS"/>
+	<reg32 offset="0x0947" name="CP_VSD_DWORDS"/>
+
+	<reg32 offset="0x0948" name="CP_ROQ_AVAIL_RB">
+		<doc>number of remaining dwords incl current dword being consumed?</doc>
+		<bitfield name="REM" low="16" high="31"/>
+	</reg32>
+	<reg32 offset="0x0949" name="CP_ROQ_AVAIL_IB1">
+		<doc>number of remaining dwords incl current dword being consumed?</doc>
+		<bitfield name="REM" low="16" high="31"/>
+	</reg32>
+	<reg32 offset="0x094a" name="CP_ROQ_AVAIL_IB2">
+		<doc>number of remaining dwords incl current dword being consumed?</doc>
+		<bitfield name="REM" low="16" high="31"/>
+	</reg32>
+	<reg32 offset="0x094b" name="CP_ROQ_AVAIL_SDS">
+		<doc>number of remaining dwords incl current dword being consumed?</doc>
+		<bitfield name="REM" low="16" high="31"/>
+	</reg32>
+	<reg32 offset="0x094c" name="CP_ROQ_AVAIL_MRB">
+		<doc>number of dwords that have already been read but haven't been consumed by $addr</doc>
+		<bitfield name="REM" low="16" high="31"/>
+	</reg32>
+	<reg32 offset="0x094d" name="CP_ROQ_AVAIL_VSD">
+		<doc>number of remaining dwords incl current dword being consumed?</doc>
+		<bitfield name="REM" low="16" high="31"/>
+	</reg32>
+
+	<bitset name="a7xx_aperture_cntl" inline="yes">
+		<bitfield name="PIPE" low="12" high="13" type="a7xx_pipe"/>
+		<bitfield name="CLUSTER" low="8" high="10" type="a7xx_cluster"/>
+		<bitfield name="CONTEXT" low="4" high="5"/>
+	</bitset>
+	<reg64 offset="0x0980" name="CP_ALWAYS_ON_COUNTER"/>
+	<reg32 offset="0x098D" name="CP_AHB_CNTL"/>
+	<reg32 offset="0x0A00" name="CP_APERTURE_CNTL_HOST" variants="A6XX"/>
+	<reg32 offset="0x0A00" name="CP_APERTURE_CNTL_HOST" type="a7xx_aperture_cntl" variants="A7XX-"/>
+	<reg32 offset="0x0A03" name="CP_APERTURE_CNTL_CD" variants="A6XX"/>
+	<reg32 offset="0x0A03" name="CP_APERTURE_CNTL_CD" type="a7xx_aperture_cntl" variants="A7XX-"/>
+
+	<reg32 offset="0x0a61" name="CP_BV_PROTECT_STATUS" variants="A7XX-"/>
+	<reg32 offset="0x0a64" name="CP_BV_HW_FAULT" variants="A7XX-"/>
+	<reg32 offset="0x0a81" name="CP_BV_DRAW_STATE_ADDR" variants="A7XX-"/>
+	<reg32 offset="0x0a82" name="CP_BV_DRAW_STATE_DATA" variants="A7XX-"/>
+	<reg32 offset="0x0a83" name="CP_BV_ROQ_DBG_ADDR" variants="A7XX-"/>
+	<reg32 offset="0x0a84" name="CP_BV_ROQ_DBG_DATA" variants="A7XX-"/>
+	<reg32 offset="0x0a85" name="CP_BV_SQE_UCODE_DBG_ADDR" variants="A7XX-"/>
+	<reg32 offset="0x0a86" name="CP_BV_SQE_UCODE_DBG_DATA" variants="A7XX-"/>
+	<reg32 offset="0x0a87" name="CP_BV_SQE_STAT_ADDR" variants="A7XX-"/>
+	<reg32 offset="0x0a88" name="CP_BV_SQE_STAT_DATA" variants="A7XX-"/>
+	<reg32 offset="0x0a96" name="CP_BV_MEM_POOL_DBG_ADDR" variants="A7XX-"/>
+	<reg32 offset="0x0a97" name="CP_BV_MEM_POOL_DBG_DATA" variants="A7XX-"/>
+	<reg64 offset="0x0a98" name="CP_BV_RB_RPTR_ADDR" variants="A7XX-"/>
+
+	<reg32 offset="0x0a9a" name="CP_RESOURCE_TBL_DBG_ADDR" variants="A7XX-"/>
+	<reg32 offset="0x0a9b" name="CP_RESOURCE_TBL_DBG_DATA" variants="A7XX-"/>
+	<reg32 offset="0x0ad0" name="CP_BV_APRIV_CNTL" variants="A7XX-"/>
+	<reg32 offset="0x0ada" name="CP_BV_CHICKEN_DBG" variants="A7XX-"/>
+
+	<reg32 offset="0x0b0a" name="CP_LPAC_DRAW_STATE_ADDR" variants="A7XX-"/>
+	<reg32 offset="0x0b0b" name="CP_LPAC_DRAW_STATE_DATA" variants="A7XX-"/>
+	<reg32 offset="0x0b0c" name="CP_LPAC_ROQ_DBG_ADDR" variants="A7XX-"/>
+	<reg32 offset="0x0b27" name="CP_SQE_AC_UCODE_DBG_ADDR" variants="A7XX-"/>
+	<reg32 offset="0x0b28" name="CP_SQE_AC_UCODE_DBG_DATA" variants="A7XX-"/>
+	<reg32 offset="0x0b29" name="CP_SQE_AC_STAT_ADDR" variants="A7XX-"/>
+	<reg32 offset="0x0b2a" name="CP_SQE_AC_STAT_DATA" variants="A7XX-"/>
+
+	<reg32 offset="0x0b31" name="CP_LPAC_APRIV_CNTL" variants="A7XX-"/>
+	<reg32 offset="0x0B34" name="CP_LPAC_PROG_FIFO_SIZE"/>
+	<reg32 offset="0x0b35" name="CP_LPAC_ROQ_DBG_DATA" variants="A7XX-"/>
+	<reg32 offset="0x0b36" name="CP_LPAC_FIFO_DBG_DATA" variants="A7XX-"/>
+	<reg32 offset="0x0b40" name="CP_LPAC_FIFO_DBG_ADDR" variants="A7XX-"/>
+	<reg32 offset="0x0b81" name="CP_LPAC_SQE_CNTL"/>
+	<reg64 offset="0x0b82" name="CP_LPAC_SQE_INSTR_BASE"/>
+
+	<reg64 offset="0x0b70" name="CP_AQE_INSTR_BASE_0" variants="A7XX-"/>
+	<reg64 offset="0x0b72" name="CP_AQE_INSTR_BASE_1" variants="A7XX-"/>
+	<reg32 offset="0x0b78" name="CP_AQE_APRIV_CNTL" variants="A7XX-"/>
+
+	<reg32 offset="0x0ba8" name="CP_AQE_ROQ_DBG_ADDR_0" variants="A7XX-"/>
+	<reg32 offset="0x0ba9" name="CP_AQE_ROQ_DBG_ADDR_1" variants="A7XX-"/>
+	<reg32 offset="0x0bac" name="CP_AQE_ROQ_DBG_DATA_0" variants="A7XX-"/>
+	<reg32 offset="0x0bad" name="CP_AQE_ROQ_DBG_DATA_1" variants="A7XX-"/>
+	<reg32 offset="0x0bb0" name="CP_AQE_UCODE_DBG_ADDR_0" variants="A7XX-"/>
+	<reg32 offset="0x0bb1" name="CP_AQE_UCODE_DBG_ADDR_1" variants="A7XX-"/>
+	<reg32 offset="0x0bb4" name="CP_AQE_UCODE_DBG_DATA_0" variants="A7XX-"/>
+	<reg32 offset="0x0bb5" name="CP_AQE_UCODE_DBG_DATA_1" variants="A7XX-"/>
+	<reg32 offset="0x0bb8" name="CP_AQE_STAT_ADDR_0" variants="A7XX-"/>
+	<reg32 offset="0x0bb9" name="CP_AQE_STAT_ADDR_1" variants="A7XX-"/>
+	<reg32 offset="0x0bbc" name="CP_AQE_STAT_DATA_0" variants="A7XX-"/>
+	<reg32 offset="0x0bbd" name="CP_AQE_STAT_DATA_1" variants="A7XX-"/>
+
+	<reg32 offset="0x0C01" name="VSC_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0018" name="RBBM_GPR0_CNTL"/>
+	<reg32 offset="0x0201" name="RBBM_INT_0_STATUS" type="A6XX_RBBM_INT_0_MASK"/>
+	<reg32 offset="0x0210" name="RBBM_STATUS">
+		<bitfield pos="23" name="GPU_BUSY_IGN_AHB" type="boolean"/>
+		<bitfield pos="22" name="GPU_BUSY_IGN_AHB_CP" type="boolean"/>
+		<bitfield pos="21" name="HLSQ_BUSY" type="boolean"/>
+		<bitfield pos="20" name="VSC_BUSY" type="boolean"/>
+		<bitfield pos="19" name="TPL1_BUSY" type="boolean"/>
+		<bitfield pos="18" name="SP_BUSY" type="boolean"/>
+		<bitfield pos="17" name="UCHE_BUSY" type="boolean"/>
+		<bitfield pos="16" name="VPC_BUSY" type="boolean"/>
+		<bitfield pos="15" name="VFD_BUSY" type="boolean"/>
+		<bitfield pos="14" name="TESS_BUSY" type="boolean"/>
+		<bitfield pos="13" name="PC_VSD_BUSY" type="boolean"/>
+		<bitfield pos="12" name="PC_DCALL_BUSY" type="boolean"/>
+		<bitfield pos="11" name="COM_DCOM_BUSY" type="boolean"/>
+		<bitfield pos="10" name="LRZ_BUSY" type="boolean"/>
+		<bitfield pos="9"  name="A2D_BUSY" type="boolean"/>
+		<bitfield pos="8"  name="CCU_BUSY" type="boolean"/>
+		<bitfield pos="7"  name="RB_BUSY" type="boolean"/>
+		<bitfield pos="6"  name="RAS_BUSY" type="boolean"/>
+		<bitfield pos="5"  name="TSE_BUSY" type="boolean"/>
+		<bitfield pos="4"  name="VBIF_BUSY" type="boolean"/>
+		<bitfield pos="3"  name="GFX_DBGC_BUSY" type="boolean"/>
+		<bitfield pos="2"  name="CP_BUSY" type="boolean"/>
+		<bitfield pos="1"  name="CP_AHB_BUSY_CP_MASTER" type="boolean"/>
+		<bitfield pos="0"  name="CP_AHB_BUSY_CX_MASTER" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0211" name="RBBM_STATUS1"/>
+	<reg32 offset="0x0212" name="RBBM_STATUS2"/>
+	<reg32 offset="0x0213" name="RBBM_STATUS3">
+		<bitfield pos="24" name="SMMU_STALLED_ON_FAULT" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0215" name="RBBM_VBIF_GX_RESET_STATUS"/>
+
+	<reg32 offset="0x0260" name="RBBM_CLOCK_MODE_CP" variants="A7XX-"/>
+	<reg32 offset="0x0284" name="RBBM_CLOCK_MODE_BV_LRZ" variants="A7XX-"/>
+	<reg32 offset="0x0285" name="RBBM_CLOCK_MODE_BV_GRAS" variants="A7XX-"/>
+	<reg32 offset="0x0286" name="RBBM_CLOCK_MODE2_GRAS" variants="A7XX-"/>
+	<reg32 offset="0x0287" name="RBBM_CLOCK_MODE_BV_VFD" variants="A7XX-"/>
+	<reg32 offset="0x0288" name="RBBM_CLOCK_MODE_BV_GPC" variants="A7XX-"/>
+
+	<reg32 offset="0x02c0" name="RBBM_SW_FUSE_INT_STATUS" variants="A7XX-"/>
+	<reg32 offset="0x02c1" name="RBBM_SW_FUSE_INT_MASK" variants="A7XX-"/>
+
+	<array offset="0x0400" name="RBBM_PERFCTR_CP" stride="2" length="14" variants="A6XX"/>
+	<array offset="0x041c" name="RBBM_PERFCTR_RBBM" stride="2" length="4" variants="A6XX"/>
+	<array offset="0x0424" name="RBBM_PERFCTR_PC" stride="2" length="8" variants="A6XX"/>
+	<array offset="0x0434" name="RBBM_PERFCTR_VFD" stride="2" length="8" variants="A6XX"/>
+	<array offset="0x0444" name="RBBM_PERFCTR_HLSQ" stride="2" length="6" variants="A6XX"/>
+	<array offset="0x0450" name="RBBM_PERFCTR_VPC" stride="2" length="6" variants="A6XX"/>
+	<array offset="0x045c" name="RBBM_PERFCTR_CCU" stride="2" length="5" variants="A6XX"/>
+	<array offset="0x0466" name="RBBM_PERFCTR_TSE" stride="2" length="4" variants="A6XX"/>
+	<array offset="0x046e" name="RBBM_PERFCTR_RAS" stride="2" length="4" variants="A6XX"/>
+	<array offset="0x0476" name="RBBM_PERFCTR_UCHE" stride="2" length="12" variants="A6XX"/>
+	<array offset="0x048e" name="RBBM_PERFCTR_TP" stride="2" length="12" variants="A6XX"/>
+	<array offset="0x04a6" name="RBBM_PERFCTR_SP" stride="2" length="24" variants="A6XX"/>
+	<array offset="0x04d6" name="RBBM_PERFCTR_RB" stride="2" length="8" variants="A6XX"/>
+	<array offset="0x04e6" name="RBBM_PERFCTR_VSC" stride="2" length="2" variants="A6XX"/>
+	<array offset="0x04ea" name="RBBM_PERFCTR_LRZ" stride="2" length="4" variants="A6XX"/>
+	<array offset="0x04f2" name="RBBM_PERFCTR_CMP" stride="2" length="4" variants="A6XX"/>
+
+	<array offset="0x0300" name="RBBM_PERFCTR_CP" stride="2" length="14" variants="A7XX-"/>
+	<array offset="0x031c" name="RBBM_PERFCTR_RBBM" stride="2" length="4" variants="A7XX-"/>
+	<array offset="0x0324" name="RBBM_PERFCTR_PC" stride="2" length="8" variants="A7XX-"/>
+	<array offset="0x0334" name="RBBM_PERFCTR_VFD" stride="2" length="8" variants="A7XX-"/>
+	<array offset="0x0344" name="RBBM_PERFCTR_HLSQ" stride="2" length="6" variants="A7XX-"/>
+	<array offset="0x0350" name="RBBM_PERFCTR_VPC" stride="2" length="6" variants="A7XX-"/>
+	<array offset="0x035c" name="RBBM_PERFCTR_CCU" stride="2" length="5" variants="A7XX-"/>
+	<array offset="0x0366" name="RBBM_PERFCTR_TSE" stride="2" length="4" variants="A7XX-"/>
+	<array offset="0x036e" name="RBBM_PERFCTR_RAS" stride="2" length="4" variants="A7XX-"/>
+	<array offset="0x0376" name="RBBM_PERFCTR_UCHE" stride="2" length="12" variants="A7XX-"/>
+	<array offset="0x038e" name="RBBM_PERFCTR_TP" stride="2" length="12" variants="A7XX-"/>
+	<array offset="0x03a6" name="RBBM_PERFCTR_SP" stride="2" length="24" variants="A7XX-"/>
+	<array offset="0x03d6" name="RBBM_PERFCTR_RB" stride="2" length="8" variants="A7XX-"/>
+	<array offset="0x03e6" name="RBBM_PERFCTR_VSC" stride="2" length="2" variants="A7XX-"/>
+	<array offset="0x03ea" name="RBBM_PERFCTR_LRZ" stride="2" length="4" variants="A7XX-"/>
+	<array offset="0x03f2" name="RBBM_PERFCTR_CMP" stride="2" length="4" variants="A7XX-"/>
+	<array offset="0x03fa" name="RBBM_PERFCTR_UFC" stride="2" length="4" variants="A7XX-"/>
+	<array offset="0x0410" name="RBBM_PERFCTR2_HLSQ" stride="2" length="6" variants="A7XX-"/>
+	<array offset="0x041c" name="RBBM_PERFCTR2_CP" stride="2" length="7" variants="A7XX-"/>
+	<array offset="0x042a" name="RBBM_PERFCTR2_SP" stride="2" length="12" variants="A7XX-"/>
+	<array offset="0x0442" name="RBBM_PERFCTR2_TP" stride="2" length="6" variants="A7XX-"/>
+	<array offset="0x044e" name="RBBM_PERFCTR2_UFC" stride="2" length="2" variants="A7XX-"/>
+	<array offset="0x0460" name="RBBM_PERFCTR_BV_PC" stride="2" length="8" variants="A7XX-"/>
+	<array offset="0x0470" name="RBBM_PERFCTR_BV_VFD" stride="2" length="8" variants="A7XX-"/>
+	<array offset="0x0480" name="RBBM_PERFCTR_BV_VPC" stride="2" length="6" variants="A7XX-"/>
+	<array offset="0x048c" name="RBBM_PERFCTR_BV_TSE" stride="2" length="4" variants="A7XX-"/>
+	<array offset="0x0494" name="RBBM_PERFCTR_BV_RAS" stride="2" length="4" variants="A7XX-"/>
+	<array offset="0x049c" name="RBBM_PERFCTR_BV_LRZ" stride="2" length="4" variants="A7XX-"/>
+
+	<reg32 offset="0x0500" name="RBBM_PERFCTR_CNTL"/>
+	<reg32 offset="0x0501" name="RBBM_PERFCTR_LOAD_CMD0"/>
+	<reg32 offset="0x0502" name="RBBM_PERFCTR_LOAD_CMD1"/>
+	<reg32 offset="0x0503" name="RBBM_PERFCTR_LOAD_CMD2"/>
+	<reg32 offset="0x0504" name="RBBM_PERFCTR_LOAD_CMD3"/>
+	<reg32 offset="0x0505" name="RBBM_PERFCTR_LOAD_VALUE_LO"/>
+	<reg32 offset="0x0506" name="RBBM_PERFCTR_LOAD_VALUE_HI"/>
+	<array offset="0x0507" name="RBBM_PERFCTR_RBBM_SEL" stride="1" length="4"/>
+	<reg32 offset="0x050B" name="RBBM_PERFCTR_GPU_BUSY_MASKED"/>
+	<reg32 offset="0x050e" name="RBBM_PERFCTR_SRAM_INIT_CMD"/>
+	<reg32 offset="0x050f" name="RBBM_PERFCTR_SRAM_INIT_STATUS"/>
+	<reg32 offset="0x0533" name="RBBM_ISDB_CNT"/>
+	<reg32 offset="0x0534" name="RBBM_NC_MODE_CNTL" variants="A7XX-"/>
+	<reg32 offset="0x0535" name="RBBM_SNAPSHOT_STATUS" variants="A7XX-"/>
+
+	<!---
+	    This block of registers aren't tied to perf counters. They
+	    count various geometry stats, for example number of
+	    vertices in, number of primnitives assembled etc.
+	-->
+
+	<reg32 offset="0x0540" name="RBBM_PRIMCTR_0_LO"/>  <!-- vs vertices in -->
+	<reg32 offset="0x0541" name="RBBM_PRIMCTR_0_HI"/>
+	<reg32 offset="0x0542" name="RBBM_PRIMCTR_1_LO"/>  <!-- vs primitives out -->
+	<reg32 offset="0x0543" name="RBBM_PRIMCTR_1_HI"/>
+	<reg32 offset="0x0544" name="RBBM_PRIMCTR_2_LO"/>  <!-- hs vertices in -->
+	<reg32 offset="0x0545" name="RBBM_PRIMCTR_2_HI"/>
+	<reg32 offset="0x0546" name="RBBM_PRIMCTR_3_LO"/>  <!-- hs patches out -->
+	<reg32 offset="0x0547" name="RBBM_PRIMCTR_3_HI"/>
+	<reg32 offset="0x0548" name="RBBM_PRIMCTR_4_LO"/>  <!-- dss vertices in -->
+	<reg32 offset="0x0549" name="RBBM_PRIMCTR_4_HI"/>
+	<reg32 offset="0x054a" name="RBBM_PRIMCTR_5_LO"/>  <!-- ds primitives out -->
+	<reg32 offset="0x054b" name="RBBM_PRIMCTR_5_HI"/>
+	<reg32 offset="0x054c" name="RBBM_PRIMCTR_6_LO"/>  <!-- gs primitives in -->
+	<reg32 offset="0x054d" name="RBBM_PRIMCTR_6_HI"/>
+	<reg32 offset="0x054e" name="RBBM_PRIMCTR_7_LO"/>  <!-- gs primitives out -->
+	<reg32 offset="0x054f" name="RBBM_PRIMCTR_7_HI"/>
+	<reg32 offset="0x0550" name="RBBM_PRIMCTR_8_LO"/>  <!-- gs primitives out -->
+	<reg32 offset="0x0551" name="RBBM_PRIMCTR_8_HI"/>
+	<reg32 offset="0x0552" name="RBBM_PRIMCTR_9_LO"/>  <!-- raster primitives in -->
+	<reg32 offset="0x0553" name="RBBM_PRIMCTR_9_HI"/>
+	<reg32 offset="0x0554" name="RBBM_PRIMCTR_10_LO"/>
+	<reg32 offset="0x0555" name="RBBM_PRIMCTR_10_HI"/>
+
+	<reg32 offset="0xF400" name="RBBM_SECVID_TRUST_CNTL"/>
+	<reg64 offset="0xF800" name="RBBM_SECVID_TSB_TRUSTED_BASE"/>
+	<reg32 offset="0xF802" name="RBBM_SECVID_TSB_TRUSTED_SIZE"/>
+	<reg32 offset="0xF803" name="RBBM_SECVID_TSB_CNTL"/>
+	<reg32 offset="0xF810" name="RBBM_SECVID_TSB_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg64 offset="0xfc00" name="RBBM_SECVID_TSB_STATUS" variants="A7XX-"/>
+	<reg32 offset="0x00010" name="RBBM_VBIF_CLIENT_QOS_CNTL"/>
+	<reg32 offset="0x00011" name="RBBM_GBIF_CLIENT_QOS_CNTL"/>
+	<reg32 offset="0x00016" name="RBBM_GBIF_HALT"/>
+	<reg32 offset="0x00017" name="RBBM_GBIF_HALT_ACK"/>
+	<reg32 offset="0x0001c" name="RBBM_WAIT_FOR_GPU_IDLE_CMD">
+		<bitfield pos="0" name="WAIT_GPU_IDLE" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x00016" name="RBBM_GBIF_HALT" variants="A7XX-"/>
+	<reg32 offset="0x00017" name="RBBM_GBIF_HALT_ACK" variants="A7XX-"/>
+	<reg32 offset="0x0001f" name="RBBM_INTERFACE_HANG_INT_CNTL"/>
+	<reg32 offset="0x00037" name="RBBM_INT_CLEAR_CMD" type="A6XX_RBBM_INT_0_MASK"/>
+	<reg32 offset="0x00038" name="RBBM_INT_0_MASK" type="A6XX_RBBM_INT_0_MASK"/>
+	<reg32 offset="0x0003a" name="RBBM_INT_2_MASK" variants="A7XX-"/>
+	<reg32 offset="0x00042" name="RBBM_SP_HYST_CNT"/>
+	<reg32 offset="0x00043" name="RBBM_SW_RESET_CMD"/>
+	<reg32 offset="0x00044" name="RBBM_RAC_THRESHOLD_CNT"/>
+	<reg32 offset="0x00045" name="RBBM_BLOCK_SW_RESET_CMD"/>
+	<reg32 offset="0x00046" name="RBBM_BLOCK_SW_RESET_CMD2"/>
+	<reg32 offset="0x000ad" name="RBBM_CLOCK_CNTL_GLOBAL" variants="A7XX-"/>
+	<reg32 offset="0x000ae" name="RBBM_CLOCK_CNTL"/>
+	<reg32 offset="0x000b0" name="RBBM_CLOCK_CNTL_SP0"/>
+	<reg32 offset="0x000b1" name="RBBM_CLOCK_CNTL_SP1"/>
+	<reg32 offset="0x000b2" name="RBBM_CLOCK_CNTL_SP2"/>
+	<reg32 offset="0x000b3" name="RBBM_CLOCK_CNTL_SP3"/>
+	<reg32 offset="0x000b4" name="RBBM_CLOCK_CNTL2_SP0"/>
+	<reg32 offset="0x000b5" name="RBBM_CLOCK_CNTL2_SP1"/>
+	<reg32 offset="0x000b6" name="RBBM_CLOCK_CNTL2_SP2"/>
+	<reg32 offset="0x000b7" name="RBBM_CLOCK_CNTL2_SP3"/>
+	<reg32 offset="0x000b8" name="RBBM_CLOCK_DELAY_SP0"/>
+	<reg32 offset="0x000b9" name="RBBM_CLOCK_DELAY_SP1"/>
+	<reg32 offset="0x000ba" name="RBBM_CLOCK_DELAY_SP2"/>
+	<reg32 offset="0x000bb" name="RBBM_CLOCK_DELAY_SP3"/>
+	<reg32 offset="0x000bc" name="RBBM_CLOCK_HYST_SP0"/>
+	<reg32 offset="0x000bd" name="RBBM_CLOCK_HYST_SP1"/>
+	<reg32 offset="0x000be" name="RBBM_CLOCK_HYST_SP2"/>
+	<reg32 offset="0x000bf" name="RBBM_CLOCK_HYST_SP3"/>
+	<reg32 offset="0x000c0" name="RBBM_CLOCK_CNTL_TP0"/>
+	<reg32 offset="0x000c1" name="RBBM_CLOCK_CNTL_TP1"/>
+	<reg32 offset="0x000c2" name="RBBM_CLOCK_CNTL_TP2"/>
+	<reg32 offset="0x000c3" name="RBBM_CLOCK_CNTL_TP3"/>
+	<reg32 offset="0x000c4" name="RBBM_CLOCK_CNTL2_TP0"/>
+	<reg32 offset="0x000c5" name="RBBM_CLOCK_CNTL2_TP1"/>
+	<reg32 offset="0x000c6" name="RBBM_CLOCK_CNTL2_TP2"/>
+	<reg32 offset="0x000c7" name="RBBM_CLOCK_CNTL2_TP3"/>
+	<reg32 offset="0x000c8" name="RBBM_CLOCK_CNTL3_TP0"/>
+	<reg32 offset="0x000c9" name="RBBM_CLOCK_CNTL3_TP1"/>
+	<reg32 offset="0x000ca" name="RBBM_CLOCK_CNTL3_TP2"/>
+	<reg32 offset="0x000cb" name="RBBM_CLOCK_CNTL3_TP3"/>
+	<reg32 offset="0x000cc" name="RBBM_CLOCK_CNTL4_TP0"/>
+	<reg32 offset="0x000cd" name="RBBM_CLOCK_CNTL4_TP1"/>
+	<reg32 offset="0x000ce" name="RBBM_CLOCK_CNTL4_TP2"/>
+	<reg32 offset="0x000cf" name="RBBM_CLOCK_CNTL4_TP3"/>
+	<reg32 offset="0x000d0" name="RBBM_CLOCK_DELAY_TP0"/>
+	<reg32 offset="0x000d1" name="RBBM_CLOCK_DELAY_TP1"/>
+	<reg32 offset="0x000d2" name="RBBM_CLOCK_DELAY_TP2"/>
+	<reg32 offset="0x000d3" name="RBBM_CLOCK_DELAY_TP3"/>
+	<reg32 offset="0x000d4" name="RBBM_CLOCK_DELAY2_TP0"/>
+	<reg32 offset="0x000d5" name="RBBM_CLOCK_DELAY2_TP1"/>
+	<reg32 offset="0x000d6" name="RBBM_CLOCK_DELAY2_TP2"/>
+	<reg32 offset="0x000d7" name="RBBM_CLOCK_DELAY2_TP3"/>
+	<reg32 offset="0x000d8" name="RBBM_CLOCK_DELAY3_TP0"/>
+	<reg32 offset="0x000d9" name="RBBM_CLOCK_DELAY3_TP1"/>
+	<reg32 offset="0x000da" name="RBBM_CLOCK_DELAY3_TP2"/>
+	<reg32 offset="0x000db" name="RBBM_CLOCK_DELAY3_TP3"/>
+	<reg32 offset="0x000dc" name="RBBM_CLOCK_DELAY4_TP0"/>
+	<reg32 offset="0x000dd" name="RBBM_CLOCK_DELAY4_TP1"/>
+	<reg32 offset="0x000de" name="RBBM_CLOCK_DELAY4_TP2"/>
+	<reg32 offset="0x000df" name="RBBM_CLOCK_DELAY4_TP3"/>
+	<reg32 offset="0x000e0" name="RBBM_CLOCK_HYST_TP0"/>
+	<reg32 offset="0x000e1" name="RBBM_CLOCK_HYST_TP1"/>
+	<reg32 offset="0x000e2" name="RBBM_CLOCK_HYST_TP2"/>
+	<reg32 offset="0x000e3" name="RBBM_CLOCK_HYST_TP3"/>
+	<reg32 offset="0x000e4" name="RBBM_CLOCK_HYST2_TP0"/>
+	<reg32 offset="0x000e5" name="RBBM_CLOCK_HYST2_TP1"/>
+	<reg32 offset="0x000e6" name="RBBM_CLOCK_HYST2_TP2"/>
+	<reg32 offset="0x000e7" name="RBBM_CLOCK_HYST2_TP3"/>
+	<reg32 offset="0x000e8" name="RBBM_CLOCK_HYST3_TP0"/>
+	<reg32 offset="0x000e9" name="RBBM_CLOCK_HYST3_TP1"/>
+	<reg32 offset="0x000ea" name="RBBM_CLOCK_HYST3_TP2"/>
+	<reg32 offset="0x000eb" name="RBBM_CLOCK_HYST3_TP3"/>
+	<reg32 offset="0x000ec" name="RBBM_CLOCK_HYST4_TP0"/>
+	<reg32 offset="0x000ed" name="RBBM_CLOCK_HYST4_TP1"/>
+	<reg32 offset="0x000ee" name="RBBM_CLOCK_HYST4_TP2"/>
+	<reg32 offset="0x000ef" name="RBBM_CLOCK_HYST4_TP3"/>
+	<reg32 offset="0x000f0" name="RBBM_CLOCK_CNTL_RB0"/>
+	<reg32 offset="0x000f1" name="RBBM_CLOCK_CNTL_RB1"/>
+	<reg32 offset="0x000f2" name="RBBM_CLOCK_CNTL_RB2"/>
+	<reg32 offset="0x000f3" name="RBBM_CLOCK_CNTL_RB3"/>
+	<reg32 offset="0x000f4" name="RBBM_CLOCK_CNTL2_RB0"/>
+	<reg32 offset="0x000f5" name="RBBM_CLOCK_CNTL2_RB1"/>
+	<reg32 offset="0x000f6" name="RBBM_CLOCK_CNTL2_RB2"/>
+	<reg32 offset="0x000f7" name="RBBM_CLOCK_CNTL2_RB3"/>
+	<reg32 offset="0x000f8" name="RBBM_CLOCK_CNTL_CCU0"/>
+	<reg32 offset="0x000f9" name="RBBM_CLOCK_CNTL_CCU1"/>
+	<reg32 offset="0x000fa" name="RBBM_CLOCK_CNTL_CCU2"/>
+	<reg32 offset="0x000fb" name="RBBM_CLOCK_CNTL_CCU3"/>
+	<reg32 offset="0x00100" name="RBBM_CLOCK_HYST_RB_CCU0"/>
+	<reg32 offset="0x00101" name="RBBM_CLOCK_HYST_RB_CCU1"/>
+	<reg32 offset="0x00102" name="RBBM_CLOCK_HYST_RB_CCU2"/>
+	<reg32 offset="0x00103" name="RBBM_CLOCK_HYST_RB_CCU3"/>
+	<reg32 offset="0x00104" name="RBBM_CLOCK_CNTL_RAC"/>
+	<reg32 offset="0x00105" name="RBBM_CLOCK_CNTL2_RAC"/>
+	<reg32 offset="0x00106" name="RBBM_CLOCK_DELAY_RAC"/>
+	<reg32 offset="0x00107" name="RBBM_CLOCK_HYST_RAC"/>
+	<reg32 offset="0x00108" name="RBBM_CLOCK_CNTL_TSE_RAS_RBBM"/>
+	<reg32 offset="0x00109" name="RBBM_CLOCK_DELAY_TSE_RAS_RBBM"/>
+	<reg32 offset="0x0010a" name="RBBM_CLOCK_HYST_TSE_RAS_RBBM"/>
+	<reg32 offset="0x0010b" name="RBBM_CLOCK_CNTL_UCHE"/>
+	<reg32 offset="0x0010c" name="RBBM_CLOCK_CNTL2_UCHE"/>
+	<reg32 offset="0x0010d" name="RBBM_CLOCK_CNTL3_UCHE"/>
+	<reg32 offset="0x0010e" name="RBBM_CLOCK_CNTL4_UCHE"/>
+	<reg32 offset="0x0010f" name="RBBM_CLOCK_DELAY_UCHE"/>
+	<reg32 offset="0x00110" name="RBBM_CLOCK_HYST_UCHE"/>
+	<reg32 offset="0x00111" name="RBBM_CLOCK_MODE_VFD"/>
+	<reg32 offset="0x00112" name="RBBM_CLOCK_DELAY_VFD"/>
+	<reg32 offset="0x00113" name="RBBM_CLOCK_HYST_VFD"/>
+	<reg32 offset="0x00114" name="RBBM_CLOCK_MODE_GPC"/>
+	<reg32 offset="0x00115" name="RBBM_CLOCK_DELAY_GPC"/>
+	<reg32 offset="0x00116" name="RBBM_CLOCK_HYST_GPC"/>
+	<reg32 offset="0x00117" name="RBBM_CLOCK_DELAY_HLSQ_2"/>
+	<reg32 offset="0x00118" name="RBBM_CLOCK_CNTL_GMU_GX"/>
+	<reg32 offset="0x00119" name="RBBM_CLOCK_DELAY_GMU_GX"/>
+	<reg32 offset="0x0011a" name="RBBM_CLOCK_HYST_GMU_GX"/>
+	<reg32 offset="0x0011b" name="RBBM_CLOCK_MODE_HLSQ"/>
+	<reg32 offset="0x0011c" name="RBBM_CLOCK_DELAY_HLSQ"/>
+	<reg32 offset="0x0011d" name="RBBM_CLOCK_HYST_HLSQ"/>
+	<reg32 offset="0x0011e" name="RBBM_CGC_GLOBAL_LOAD_CMD" variants="A7XX-"/>
+	<reg32 offset="0x0011f" name="RBBM_CGC_P2S_TRIG_CMD" variants="A7XX-"/>
+	<reg32 offset="0x00120" name="RBBM_CLOCK_CNTL_TEX_FCHE"/>
+	<reg32 offset="0x00121" name="RBBM_CLOCK_DELAY_TEX_FCHE"/>
+	<reg32 offset="0x00122" name="RBBM_CLOCK_HYST_TEX_FCHE"/>
+	<reg32 offset="0x00122" name="RBBM_CGC_P2S_STATUS" variants="A7XX-">
+		<bitfield name="TXDONE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00123" name="RBBM_CLOCK_CNTL_FCHE"/>
+	<reg32 offset="0x00124" name="RBBM_CLOCK_DELAY_FCHE"/>
+	<reg32 offset="0x00125" name="RBBM_CLOCK_HYST_FCHE"/>
+	<reg32 offset="0x00126" name="RBBM_CLOCK_CNTL_MHUB"/>
+	<reg32 offset="0x00127" name="RBBM_CLOCK_DELAY_MHUB"/>
+	<reg32 offset="0x00128" name="RBBM_CLOCK_HYST_MHUB"/>
+	<reg32 offset="0x00129" name="RBBM_CLOCK_DELAY_GLC"/>
+	<reg32 offset="0x0012a" name="RBBM_CLOCK_HYST_GLC"/>
+	<reg32 offset="0x0012b" name="RBBM_CLOCK_CNTL_GLC"/>
+	<reg32 offset="0x0012f" name="RBBM_CLOCK_HYST2_VFD" variants="A7XX-"/>
+	<reg32 offset="0x005ff" name="RBBM_LPAC_GBIF_CLIENT_QOS_CNTL"/>
+
+	<reg32 offset="0x0600" name="DBGC_CFG_DBGBUS_SEL_A"/>
+	<reg32 offset="0x0601" name="DBGC_CFG_DBGBUS_SEL_B"/>
+	<reg32 offset="0x0602" name="DBGC_CFG_DBGBUS_SEL_C"/>
+	<reg32 offset="0x0603" name="DBGC_CFG_DBGBUS_SEL_D">
+		<bitfield high="7" low="0" name="PING_INDEX"/>
+		<bitfield high="15" low="8" name="PING_BLK_SEL"/>
+	</reg32>
+	<reg32 offset="0x0604" name="DBGC_CFG_DBGBUS_CNTLT">
+		<bitfield high="5" low="0" name="TRACEEN"/>
+		<bitfield high="14" low="12" name="GRANU"/>
+		<bitfield high="31" low="28" name="SEGT"/>
+	</reg32>
+	<reg32 offset="0x0605" name="DBGC_CFG_DBGBUS_CNTLM">
+		<bitfield high="27" low="24" name="ENABLE"/>
+	</reg32>
+	<reg32 offset="0x0608" name="DBGC_CFG_DBGBUS_IVTL_0"/>
+	<reg32 offset="0x0609" name="DBGC_CFG_DBGBUS_IVTL_1"/>
+	<reg32 offset="0x060a" name="DBGC_CFG_DBGBUS_IVTL_2"/>
+	<reg32 offset="0x060b" name="DBGC_CFG_DBGBUS_IVTL_3"/>
+	<reg32 offset="0x060c" name="DBGC_CFG_DBGBUS_MASKL_0"/>
+	<reg32 offset="0x060d" name="DBGC_CFG_DBGBUS_MASKL_1"/>
+	<reg32 offset="0x060e" name="DBGC_CFG_DBGBUS_MASKL_2"/>
+	<reg32 offset="0x060f" name="DBGC_CFG_DBGBUS_MASKL_3"/>
+	<reg32 offset="0x0610" name="DBGC_CFG_DBGBUS_BYTEL_0">
+		<bitfield high="3" low="0" name="BYTEL0"/>
+		<bitfield high="7" low="4" name="BYTEL1"/>
+		<bitfield high="11" low="8" name="BYTEL2"/>
+		<bitfield high="15" low="12" name="BYTEL3"/>
+		<bitfield high="19" low="16" name="BYTEL4"/>
+		<bitfield high="23" low="20" name="BYTEL5"/>
+		<bitfield high="27" low="24" name="BYTEL6"/>
+		<bitfield high="31" low="28" name="BYTEL7"/>
+	</reg32>
+	<reg32 offset="0x0611" name="DBGC_CFG_DBGBUS_BYTEL_1">
+		<bitfield high="3" low="0" name="BYTEL8"/>
+		<bitfield high="7" low="4" name="BYTEL9"/>
+		<bitfield high="11" low="8" name="BYTEL10"/>
+		<bitfield high="15" low="12" name="BYTEL11"/>
+		<bitfield high="19" low="16" name="BYTEL12"/>
+		<bitfield high="23" low="20" name="BYTEL13"/>
+		<bitfield high="27" low="24" name="BYTEL14"/>
+		<bitfield high="31" low="28" name="BYTEL15"/>
+	</reg32>
+	<reg32 offset="0x062f" name="DBGC_CFG_DBGBUS_TRACE_BUF1"/>
+	<reg32 offset="0x0630" name="DBGC_CFG_DBGBUS_TRACE_BUF2"/>
+	<array offset="0x0CD8" name="VSC_PERFCTR_VSC_SEL" stride="1" length="2"/>
+	<reg32 offset="0x0CD8" name="VSC_UNKNOWN_0CD8" variants="A7XX">
+		<doc>
+			Set to true when binning, isn't changed afterwards
+		</doc>
+		<bitfield name="BINNING" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xC800" name="HLSQ_DBG_AHB_READ_APERTURE"/>
+	<reg32 offset="0xD000" name="HLSQ_DBG_READ_SEL"/>
+	<reg32 offset="0x0E00" name="UCHE_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0x0E01" name="UCHE_MODE_CNTL"/>
+	<reg64 offset="0x0E05" name="UCHE_WRITE_RANGE_MAX"/>
+	<reg64 offset="0x0E07" name="UCHE_WRITE_THRU_BASE"/>
+	<reg64 offset="0x0E09" name="UCHE_TRAP_BASE"/>
+	<reg64 offset="0x0E0B" name="UCHE_GMEM_RANGE_MIN"/>
+	<reg64 offset="0x0E0D" name="UCHE_GMEM_RANGE_MAX"/>
+	<reg32 offset="0x0E17" name="UCHE_CACHE_WAYS" usage="cmd"/>
+	<reg32 offset="0x0E18" name="UCHE_FILTER_CNTL"/>
+	<reg32 offset="0x0E19" name="UCHE_CLIENT_PF" usage="cmd">
+		<bitfield high="7" low="0" name="PERFSEL"/>
+	</reg32>
+	<array offset="0x0E1C" name="UCHE_PERFCTR_UCHE_SEL" stride="1" length="12"/>
+	<reg32 offset="0x0e3a" name="UCHE_GBIF_GX_CONFIG"/>
+	<reg32 offset="0x0e3c" name="UCHE_CMDQ_CONFIG"/>
+
+	<reg32 offset="0x3000" name="VBIF_VERSION"/>
+	<reg32 offset="0x3001" name="VBIF_CLKON">
+		<bitfield pos="1" name="FORCE_ON_TESTBUS" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x302A" name="VBIF_GATE_OFF_WRREQ_EN"/>
+	<reg32 offset="0x3080" name="VBIF_XIN_HALT_CTRL0"/>
+	<reg32 offset="0x3081" name="VBIF_XIN_HALT_CTRL1"/>
+	<reg32 offset="0x3084" name="VBIF_TEST_BUS_OUT_CTRL"/>
+	<reg32 offset="0x3085" name="VBIF_TEST_BUS1_CTRL0"/>
+	<reg32 offset="0x3086" name="VBIF_TEST_BUS1_CTRL1">
+		<bitfield low="0" high="3" name="DATA_SEL"/>
+	</reg32>
+	<reg32 offset="0x3087" name="VBIF_TEST_BUS2_CTRL0"/>
+	<reg32 offset="0x3088" name="VBIF_TEST_BUS2_CTRL1">
+		<bitfield low="0" high="8" name="DATA_SEL"/>
+	</reg32>
+	<reg32 offset="0x308c" name="VBIF_TEST_BUS_OUT"/>
+	<reg32 offset="0x30d0" name="VBIF_PERF_CNT_SEL0"/>
+	<reg32 offset="0x30d1" name="VBIF_PERF_CNT_SEL1"/>
+	<reg32 offset="0x30d2" name="VBIF_PERF_CNT_SEL2"/>
+	<reg32 offset="0x30d3" name="VBIF_PERF_CNT_SEL3"/>
+	<reg32 offset="0x30d8" name="VBIF_PERF_CNT_LOW0"/>
+	<reg32 offset="0x30d9" name="VBIF_PERF_CNT_LOW1"/>
+	<reg32 offset="0x30da" name="VBIF_PERF_CNT_LOW2"/>
+	<reg32 offset="0x30db" name="VBIF_PERF_CNT_LOW3"/>
+	<reg32 offset="0x30e0" name="VBIF_PERF_CNT_HIGH0"/>
+	<reg32 offset="0x30e1" name="VBIF_PERF_CNT_HIGH1"/>
+	<reg32 offset="0x30e2" name="VBIF_PERF_CNT_HIGH2"/>
+	<reg32 offset="0x30e3" name="VBIF_PERF_CNT_HIGH3"/>
+	<reg32 offset="0x3100" name="VBIF_PERF_PWR_CNT_EN0"/>
+	<reg32 offset="0x3101" name="VBIF_PERF_PWR_CNT_EN1"/>
+	<reg32 offset="0x3102" name="VBIF_PERF_PWR_CNT_EN2"/>
+	<reg32 offset="0x3110" name="VBIF_PERF_PWR_CNT_LOW0"/>
+	<reg32 offset="0x3111" name="VBIF_PERF_PWR_CNT_LOW1"/>
+	<reg32 offset="0x3112" name="VBIF_PERF_PWR_CNT_LOW2"/>
+	<reg32 offset="0x3118" name="VBIF_PERF_PWR_CNT_HIGH0"/>
+	<reg32 offset="0x3119" name="VBIF_PERF_PWR_CNT_HIGH1"/>
+	<reg32 offset="0x311a" name="VBIF_PERF_PWR_CNT_HIGH2"/>
+
+	<reg32 offset="0x3c01" name="GBIF_SCACHE_CNTL0"/>
+	<reg32 offset="0x3c02" name="GBIF_SCACHE_CNTL1"/>
+	<reg32 offset="0x3c03" name="GBIF_QSB_SIDE0"/>
+	<reg32 offset="0x3c04" name="GBIF_QSB_SIDE1"/>
+	<reg32 offset="0x3c05" name="GBIF_QSB_SIDE2"/>
+	<reg32 offset="0x3c06" name="GBIF_QSB_SIDE3"/>
+	<reg32 offset="0x3c45" name="GBIF_HALT"/>
+	<reg32 offset="0x3c46" name="GBIF_HALT_ACK"/>
+	<reg32 offset="0x3cc0" name="GBIF_PERF_PWR_CNT_EN"/>
+	<reg32 offset="0x3cc1" name="GBIF_PERF_PWR_CNT_CLR"/>
+	<reg32 offset="0x3cc2" name="GBIF_PERF_CNT_SEL"/>
+	<reg32 offset="0x3cc3" name="GBIF_PERF_PWR_CNT_SEL"/>
+	<reg32 offset="0x3cc4" name="GBIF_PERF_CNT_LOW0"/>
+	<reg32 offset="0x3cc5" name="GBIF_PERF_CNT_LOW1"/>
+	<reg32 offset="0x3cc6" name="GBIF_PERF_CNT_LOW2"/>
+	<reg32 offset="0x3cc7" name="GBIF_PERF_CNT_LOW3"/>
+	<reg32 offset="0x3cc8" name="GBIF_PERF_CNT_HIGH0"/>
+	<reg32 offset="0x3cc9" name="GBIF_PERF_CNT_HIGH1"/>
+	<reg32 offset="0x3cca" name="GBIF_PERF_CNT_HIGH2"/>
+	<reg32 offset="0x3ccb" name="GBIF_PERF_CNT_HIGH3"/>
+	<reg32 offset="0x3ccc" name="GBIF_PWR_CNT_LOW0"/>
+	<reg32 offset="0x3ccd" name="GBIF_PWR_CNT_LOW1"/>
+	<reg32 offset="0x3cce" name="GBIF_PWR_CNT_LOW2"/>
+	<reg32 offset="0x3ccf" name="GBIF_PWR_CNT_HIGH0"/>
+	<reg32 offset="0x3cd0" name="GBIF_PWR_CNT_HIGH1"/>
+	<reg32 offset="0x3cd1" name="GBIF_PWR_CNT_HIGH2"/>
+
+	<reg32 offset="0x0c00" name="VSC_DBG_ECO_CNTL"/>
+	<reg32 offset="0x0c02" name="VSC_BIN_SIZE" usage="rp_blit">
+		<bitfield name="WIDTH" low="0" high="7" shr="5" type="uint"/>
+		<bitfield name="HEIGHT" low="8" high="16" shr="4" type="uint"/>
+	</reg32>
+	<reg64 offset="0x0c03" name="VSC_DRAW_STRM_SIZE_ADDRESS" type="waddress" usage="cmd"/>
+	<reg32 offset="0x0c06" name="VSC_BIN_COUNT" usage="rp_blit">
+		<bitfield name="NX" low="1" high="10" type="uint"/>
+		<bitfield name="NY" low="11" high="20" type="uint"/>
+	</reg32>
+	<array offset="0x0c10" name="VSC_PIPE_CONFIG" stride="1" length="32" usage="rp_blit">
+		<reg32 offset="0x0" name="REG">
+			<doc>
+				Configures the mapping between VSC_PIPE buffer and
+				bin, X/Y specify the bin index in the horiz/vert
+				direction (0,0 is upper left, 0,1 is leftmost bin
+				on second row, and so on).  W/H specify the number
+				of bins assigned to this VSC_PIPE in the horiz/vert
+				dimension.
+			</doc>
+			<bitfield name="X" low="0" high="9" type="uint"/>
+			<bitfield name="Y" low="10" high="19" type="uint"/>
+			<bitfield name="W" low="20" high="25" type="uint"/>
+			<bitfield name="H" low="26" high="31" type="uint"/>
+		</reg32>
+	</array>
+	<!--
+	HW binning primitive & draw streams, which enable draws and primitives
+	within a draw to be skipped in the main tile pass.  See:
+	https://github.com/freedreno/freedreno/wiki/Visibility-Stream-Format
+
+	Compared to a5xx and earlier, we just program the address of the first
+	stream and hw adds (pipe_num * VSC_*_STRM_PITCH)
+
+	LIMIT is set to PITCH - 64, to make room for a bit of overflow
+	 -->
+	<reg64 offset="0x0c30" name="VSC_PRIM_STRM_ADDRESS" type="waddress" usage="cmd"/>
+	<reg32 offset="0x0c32" name="VSC_PRIM_STRM_PITCH" usage="cmd"/>
+	<reg32 offset="0x0c33" name="VSC_PRIM_STRM_LIMIT" usage="cmd"/>
+	<reg64 offset="0x0c34" name="VSC_DRAW_STRM_ADDRESS" type="waddress" usage="cmd"/>
+	<reg32 offset="0x0c36" name="VSC_DRAW_STRM_PITCH" usage="cmd"/>
+	<reg32 offset="0x0c37" name="VSC_DRAW_STRM_LIMIT" usage="cmd"/>
+
+	<array offset="0x0c38" name="VSC_STATE" stride="1" length="32" usage="rp_blit">
+		<doc>
+			Seems to be a bitmap of which tiles mapped to the VSC
+			pipe contain geometry.
+
+			I suppose we can connect a maximum of 32 tiles to a
+			single VSC pipe.
+		</doc>
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+
+	<array offset="0x0c58" name="VSC_PRIM_STRM_SIZE" stride="1" length="32" variants="A6XX" usage="rp_blit">
+		<doc>
+			Has the size of data written to corresponding VSC_PRIM_STRM
+			buffer.
+		</doc>
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+
+	<array offset="0x0c78" name="VSC_DRAW_STRM_SIZE" stride="1" length="32" variants="A6XX" usage="rp_blit">
+		<doc>
+			Has the size of data written to corresponding VSC pipe, ie.
+			same thing that is written out to VSC_DRAW_STRM_SIZE_ADDRESS_LO/HI
+		</doc>
+		<reg32 offset="0x0" name="REG"/>
+	</array>
+
+	<reg32 offset="0x0d08" name="VSC_UNKNOWN_0D08" variants="A7XX-" usage="rp_blit"/>
+
+	<reg32 offset="0x0E10" name="UCHE_UNKNOWN_0E10" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0x0E11" name="UCHE_UNKNOWN_0E11" variants="A7XX-" usage="cmd"/>
+	<!-- always 0x03200000 ? -->
+	<reg32 offset="0x0e12" name="UCHE_UNKNOWN_0E12"  usage="cmd"/>
+
+	<!-- adreno_reg_xy has 15 bits per coordinate, but a6xx registers only have 14 -->
+	<bitset name="a6xx_reg_xy" inline="yes">
+		<bitfield name="X" low="0" high="13" type="uint"/>
+		<bitfield name="Y" low="16" high="29" type="uint"/>
+	</bitset>
+
+	<reg32 offset="0x8000" name="GRAS_CL_CNTL" usage="rp_blit">
+		<bitfield name="CLIP_DISABLE" pos="0" type="boolean"/>
+		<bitfield name="ZNEAR_CLIP_DISABLE" pos="1" type="boolean"/>
+		<bitfield name="ZFAR_CLIP_DISABLE" pos="2" type="boolean"/>
+		<bitfield name="Z_CLAMP_ENABLE" pos="5" type="boolean"/>
+		<!-- controls near z clip behavior (set for vulkan) -->
+		<bitfield name="ZERO_GB_SCALE_Z" pos="6" type="boolean"/>
+		<!-- guess based on a3xx and meaning of bits 8 and 9
+		     if the guess is right then this is related to point sprite clipping -->
+		<bitfield name="VP_CLIP_CODE_IGNORE" pos="7" type="boolean"/>
+		<bitfield name="VP_XFORM_DISABLE" pos="8" type="boolean"/>
+		<bitfield name="PERSP_DIVISION_DISABLE" pos="9" type="boolean"/>
+	</reg32>
+
+	<bitset name="a6xx_gras_xs_cl_cntl" inline="yes">
+		<bitfield name="CLIP_MASK" low="0" high="7"/>
+		<bitfield name="CULL_MASK" low="8" high="15"/>
+	</bitset>
+	<reg32 offset="0x8001" name="GRAS_VS_CL_CNTL" type="a6xx_gras_xs_cl_cntl" usage="rp_blit"/>
+	<reg32 offset="0x8002" name="GRAS_DS_CL_CNTL" type="a6xx_gras_xs_cl_cntl" usage="rp_blit"/>
+	<reg32 offset="0x8003" name="GRAS_GS_CL_CNTL" type="a6xx_gras_xs_cl_cntl" usage="rp_blit"/>
+	<reg32 offset="0x8004" name="GRAS_MAX_LAYER_INDEX" low="0" high="10" type="uint" usage="rp_blit"/>
+
+	<reg32 offset="0x8005" name="GRAS_CNTL" usage="rp_blit">
+		<!-- see also RB_RENDER_CONTROL0 -->
+		<bitfield name="IJ_PERSP_PIXEL" pos="0" type="boolean"/>
+		<bitfield name="IJ_PERSP_CENTROID" pos="1" type="boolean"/>
+		<bitfield name="IJ_PERSP_SAMPLE" pos="2" type="boolean"/>
+		<bitfield name="IJ_LINEAR_PIXEL" pos="3" type="boolean"/>
+		<bitfield name="IJ_LINEAR_CENTROID" pos="4" type="boolean"/>
+		<bitfield name="IJ_LINEAR_SAMPLE" pos="5" type="boolean"/>
+		<bitfield name="COORD_MASK" low="6" high="9" type="hex"/>
+		<bitfield name="UNK10" pos="10" type="boolean" variants="A7XX-"/>
+		<bitfield name="UNK11" pos="11" type="boolean" variants="A7XX-"/>
+	</reg32>
+	<reg32 offset="0x8006" name="GRAS_CL_GUARDBAND_CLIP_ADJ" usage="rp_blit">
+		<bitfield name="HORZ" low="0" high="8" type="uint"/>
+		<bitfield name="VERT" low="10" high="18" type="uint"/>
+	</reg32>
+
+	<!-- Something connected to depth-stencil attachment size -->
+	<reg32 offset="0x8007" name="GRAS_UNKNOWN_8007" variants="A7XX-" usage="rp_blit"/>
+
+	<reg32 offset="0x8008" name="GRAS_UNKNOWN_8008" variants="A7XX-" usage="cmd"/>
+
+	<reg32 offset="0x8009" name="GRAS_UNKNOWN_8009" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0x800a" name="GRAS_UNKNOWN_800A" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0x800b" name="GRAS_UNKNOWN_800B" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0x800c" name="GRAS_UNKNOWN_800C" variants="A7XX-" usage="cmd"/>
+
+	<!-- <reg32 offset="0x80f0" name="GRAS_UNKNOWN_80F0" type="a6xx_reg_xy"/> -->
+
+	<!-- 0x8006-0x800f invalid -->
+	<array offset="0x8010" name="GRAS_CL_VPORT" stride="6" length="16" usage="rp_blit">
+		<reg32 offset="0" name="XOFFSET" type="float"/>
+		<reg32 offset="1" name="XSCALE" type="float"/>
+		<reg32 offset="2" name="YOFFSET" type="float"/>
+		<reg32 offset="3" name="YSCALE" type="float"/>
+		<reg32 offset="4" name="ZOFFSET" type="float"/>
+		<reg32 offset="5" name="ZSCALE" type="float"/>
+	</array>
+	<array offset="0x8070" name="GRAS_CL_Z_CLAMP" stride="2" length="16" usage="rp_blit">
+		<reg32 offset="0" name="MIN" type="float"/>
+		<reg32 offset="1" name="MAX" type="float"/>
+	</array>
+
+	<reg32 offset="0x8090" name="GRAS_SU_CNTL" usage="rp_blit">
+		<bitfield name="CULL_FRONT" pos="0" type="boolean"/>
+		<bitfield name="CULL_BACK" pos="1" type="boolean"/>
+		<bitfield name="FRONT_CW" pos="2" type="boolean"/>
+		<bitfield name="LINEHALFWIDTH" low="3" high="10" radix="2" type="fixed"/>
+		<bitfield name="POLY_OFFSET" pos="11" type="boolean"/>
+		<bitfield name="UNK12" pos="12"/>
+		<bitfield name="LINE_MODE" pos="13" type="a5xx_line_mode"/>
+		<bitfield name="UNK15" low="15" high="16"/>
+		<!--
+                        On gen1 only MULTIVIEW_ENABLE exists. On gen3 we have
+                        the ability to add the view index to either the RT array
+                        index or the viewport index, and it seems that
+                        MULTIVIEW_ENABLE doesn't do anything, instead we need to
+                        set at least one of RENDERTARGETINDEXINCR or
+                        VIEWPORTINDEXINCR to enable multiview. The blob still
+                        sets MULTIVIEW_ENABLE regardless.
+                        TODO: what about gen2 (a640)?
+		-->
+		<bitfield name="MULTIVIEW_ENABLE" pos="17" type="boolean"/>
+		<bitfield name="RENDERTARGETINDEXINCR" pos="18" type="boolean"/>
+		<bitfield name="VIEWPORTINDEXINCR" pos="19" type="boolean"/>
+		<bitfield name="UNK20" low="20" high="22"/>
+	</reg32>
+	<reg32 offset="0x8091" name="GRAS_SU_POINT_MINMAX" usage="rp_blit">
+		<bitfield name="MIN" low="0" high="15" type="ufixed" radix="4"/>
+		<bitfield name="MAX" low="16" high="31" type="ufixed" radix="4"/>
+	</reg32>
+	<reg32 offset="0x8092" name="GRAS_SU_POINT_SIZE" low="0" high="15" type="fixed" radix="4" usage="rp_blit"/>
+	<!-- 0x8093 invalid -->
+	<reg32 offset="0x8094" name="GRAS_SU_DEPTH_PLANE_CNTL" usage="rp_blit">
+		<bitfield name="Z_MODE" low="0" high="1" type="a6xx_ztest_mode"/>
+	</reg32>
+	<reg32 offset="0x8095" name="GRAS_SU_POLY_OFFSET_SCALE" type="float" usage="rp_blit"/>
+	<reg32 offset="0x8096" name="GRAS_SU_POLY_OFFSET_OFFSET" type="float" usage="rp_blit"/>
+	<reg32 offset="0x8097" name="GRAS_SU_POLY_OFFSET_OFFSET_CLAMP" type="float" usage="rp_blit"/>
+	<!-- duplicates RB_DEPTH_BUFFER_INFO: -->
+	<reg32 offset="0x8098" name="GRAS_SU_DEPTH_BUFFER_INFO" usage="rp_blit">
+		<bitfield name="DEPTH_FORMAT" low="0" high="2" type="a6xx_depth_format"/>
+		<bitfield name="UNK3" pos="3"/>
+	</reg32>
+
+	<reg32 offset="0x8099" name="GRAS_SU_CONSERVATIVE_RAS_CNTL" usage="cmd">
+		<bitfield name="CONSERVATIVERASEN" pos="0" type="boolean"/>
+		<bitfield name="SHIFTAMOUNT" low="1" high="2"/>
+		<bitfield name="INNERCONSERVATIVERASEN" pos="3" type="boolean"/>
+		<bitfield name="UNK4" low="4" high="5"/>
+	</reg32>
+	<reg32 offset="0x809a" name="GRAS_SU_PATH_RENDERING_CNTL">
+		<bitfield name="UNK0" pos="0" type="boolean"/>
+		<bitfield name="LINELENGTHEN" pos="1" type="boolean"/>
+	</reg32>
+
+	<bitset name="a6xx_gras_layer_cntl" inline="yes">
+		<bitfield name="WRITES_LAYER" pos="0" type="boolean"/>
+		<bitfield name="WRITES_VIEW" pos="1" type="boolean"/>
+	</bitset>
+	<reg32 offset="0x809b" name="GRAS_VS_LAYER_CNTL" type="a6xx_gras_layer_cntl" usage="rp_blit"/>
+	<reg32 offset="0x809c" name="GRAS_GS_LAYER_CNTL" type="a6xx_gras_layer_cntl" usage="rp_blit"/>
+	<reg32 offset="0x809d" name="GRAS_DS_LAYER_CNTL" type="a6xx_gras_layer_cntl" usage="rp_blit"/>
+	<!-- 0x809e/0x809f invalid -->
+
+	<enum name="a6xx_sequenced_thread_dist">
+		<value value="0x0" name="DIST_SCREEN_COORD"/>
+		<value value="0x1" name="DIST_ALL_TO_RB0"/>
+	</enum>
+
+	<enum name="a6xx_single_prim_mode">
+		<value value="0x0" name="NO_FLUSH"/>
+		<doc>
+			In addition to FLUSH_PER_OVERLAP, guarantee that UCHE
+			and CCU don't get out of sync when fetching the previous
+			value for the current pixel. With NO_FLUSH, there's the
+			possibility that the flags for the current pixel are
+			flushed before the data or vice-versa, leading to
+			texture fetches via UCHE getting out of sync values.
+			This mode should eliminate that. It's used in bypass
+			mode for coherent blending
+			(GL_KHR_blend_equation_advanced_coherent) as well as
+			non-coherent blending.
+		</doc>
+		<value value="0x1" name="FLUSH_PER_OVERLAP_AND_OVERWRITE"/>
+		<doc>
+			Invalidate UCHE and wait for any pending work to finish
+			if there was possibly an overlapping primitive prior to
+			the current one. This is similar to a combination of
+			GRAS_SC_CONTROL::INJECT_L2_INVALIDATE_EVENT and
+			WAIT_RB_IDLE_ALL_TRI on a3xx. It's used in GMEM mode for
+			coherent blending
+			(GL_KHR_blend_equation_advanced_coherent).
+		</doc>
+		<value value="0x3" name="FLUSH_PER_OVERLAP"/>
+	</enum>
+
+	<!-- this probably has the same meaning as a3xx GRAS_SC_CONTROL::RASTER_MODE -->
+	<enum name="a6xx_raster_mode">
+		<value value="0x0" name="TYPE_TILED"/>
+		<value value="0x1" name="TYPE_WRITER"/>
+	</enum>
+
+	<!-- I'm guessing this is the same as a3xx -->
+	<enum name="a6xx_raster_direction">
+		<value value="0x0" name="LR_TB"/>
+		<value value="0x1" name="RL_TB"/>
+		<value value="0x2" name="LR_BT"/>
+		<value value="0x3" name="RB_BT"/>
+	</enum>
+
+	<reg32 offset="0x80a0" name="GRAS_SC_CNTL" usage="rp_blit">
+		<bitfield name="CCUSINGLECACHELINESIZE" low="0" high="2"/>
+		<bitfield name="SINGLE_PRIM_MODE" low="3" high="4" type="a6xx_single_prim_mode"/>
+		<bitfield name="RASTER_MODE" pos="5" type="a6xx_raster_mode"/>
+		<bitfield name="RASTER_DIRECTION" low="6" high="7" type="a6xx_raster_direction"/>
+		<bitfield name="SEQUENCED_THREAD_DISTRIBUTION" pos="8" type="a6xx_sequenced_thread_dist"/>
+		<!-- CCUSINGLECACHELINESIZE is ignored unless bit 9 is set -->
+		<bitfield name="UNK9" pos="9" type="boolean"/>
+		<bitfield name="ROTATION" low="10" high="11" type="uint"/>
+		<bitfield name="EARLYVIZOUTEN" pos="12" type="boolean"/>
+	</reg32>
+
+	<enum name="a6xx_render_mode">
+		<value value="0x0" name="RENDERING_PASS"/>
+		<value value="0x1" name="BINNING_PASS"/>
+	</enum>
+
+	<enum name="a6xx_buffers_location">
+		<value value="0" name="BUFFERS_IN_GMEM"/>
+		<value value="3" name="BUFFERS_IN_SYSMEM"/>
+	</enum>
+
+	<reg32 offset="0x80a1" name="GRAS_BIN_CONTROL" usage="rp_blit">
+		<bitfield name="BINW" low="0" high="5" shr="5" type="uint"/>
+		<bitfield name="BINH" low="8" high="14" shr="4" type="uint"/>
+		<bitfield name="RENDER_MODE" low="18" high="20" type="a6xx_render_mode"/>
+		<bitfield name="FORCE_LRZ_WRITE_DIS" pos="21" type="boolean"/>
+		<bitfield name="BUFFERS_LOCATION" low="22" high="23" type="a6xx_buffers_location" variants="A6XX"/>
+		<bitfield name="LRZ_FEEDBACK_ZMODE_MASK" low="24" high="26"/>
+		<bitfield name="UNK27" pos="27"/>
+	</reg32>
+
+	<reg32 offset="0x80a2" name="GRAS_RAS_MSAA_CNTL" usage="rp_blit">
+		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>
+		<bitfield name="UNK2" pos="2"/>
+		<bitfield name="UNK3" pos="3"/>
+	</reg32>
+	<reg32 offset="0x80a3" name="GRAS_DEST_MSAA_CNTL" usage="rp_blit">
+		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>
+		<bitfield name="MSAA_DISABLE" pos="2" type="boolean"/>
+	</reg32>
+
+	<bitset name="a6xx_sample_config" inline="yes">
+		<bitfield name="UNK0" pos="0"/>
+		<bitfield name="LOCATION_ENABLE" pos="1" type="boolean"/>
+	</bitset>
+
+	<bitset name="a6xx_sample_locations" inline="yes">
+		<bitfield name="SAMPLE_0_X" low="0" high="3" radix="4" type="fixed"/>
+		<bitfield name="SAMPLE_0_Y" low="4" high="7" radix="4" type="fixed"/>
+		<bitfield name="SAMPLE_1_X" low="8" high="11" radix="4" type="fixed"/>
+		<bitfield name="SAMPLE_1_Y" low="12" high="15" radix="4" type="fixed"/>
+		<bitfield name="SAMPLE_2_X" low="16" high="19" radix="4" type="fixed"/>
+		<bitfield name="SAMPLE_2_Y" low="20" high="23" radix="4" type="fixed"/>
+		<bitfield name="SAMPLE_3_X" low="24" high="27" radix="4" type="fixed"/>
+		<bitfield name="SAMPLE_3_Y" low="28" high="31" radix="4" type="fixed"/>
+	</bitset>
+
+	<reg32 offset="0x80a4" name="GRAS_SAMPLE_CONFIG" type="a6xx_sample_config" usage="rp_blit"/>
+	<reg32 offset="0x80a5" name="GRAS_SAMPLE_LOCATION_0" type="a6xx_sample_locations" usage="rp_blit"/>
+	<reg32 offset="0x80a6" name="GRAS_SAMPLE_LOCATION_1" type="a6xx_sample_locations" usage="rp_blit"/>
+
+	<reg32 offset="0x80a7" name="GRAS_UNKNOWN_80A7" variants="A7XX-" usage="cmd"/>
+
+	<!-- 0x80a7-0x80ae invalid -->
+	<reg32 offset="0x80af" name="GRAS_UNKNOWN_80AF" pos="0" usage="cmd"/>
+
+	<bitset name="a6xx_scissor_xy" inline="yes">
+		<bitfield name="X" low="0" high="15" type="uint"/>
+		<bitfield name="Y" low="16" high="31" type="uint"/>
+	</bitset>
+	<array offset="0x80b0" name="GRAS_SC_SCREEN_SCISSOR" stride="2" length="16" usage="rp_blit">
+		<reg32 offset="0" name="TL" type="a6xx_scissor_xy"/>
+		<reg32 offset="1" name="BR" type="a6xx_scissor_xy"/>
+	</array>
+	<array offset="0x80d0" name="GRAS_SC_VIEWPORT_SCISSOR" stride="2" length="16" usage="rp_blit">
+		<reg32 offset="0" name="TL" type="a6xx_scissor_xy"/>
+		<reg32 offset="1" name="BR" type="a6xx_scissor_xy"/>
+	</array>
+
+	<reg32 offset="0x80f0" name="GRAS_SC_WINDOW_SCISSOR_TL" type="a6xx_reg_xy" usage="rp_blit"/>
+	<reg32 offset="0x80f1" name="GRAS_SC_WINDOW_SCISSOR_BR" type="a6xx_reg_xy" usage="rp_blit"/>
+
+	<!-- 0x80f4 - 0x80fa are used for VK_KHR_fragment_shading_rate -->
+	<reg64 offset="0x80f4" name="GRAS_UNKNOWN_80F4" variants="A7XX-" usage="cmd"/>
+	<reg64 offset="0x80f5" name="GRAS_UNKNOWN_80F5" variants="A7XX-" usage="cmd"/>
+	<reg64 offset="0x80f6" name="GRAS_UNKNOWN_80F6" variants="A7XX-" usage="cmd"/>
+	<reg64 offset="0x80f8" name="GRAS_UNKNOWN_80F8" variants="A7XX-" usage="cmd"/>
+	<reg64 offset="0x80f9" name="GRAS_UNKNOWN_80F9" variants="A7XX-" usage="cmd"/>
+	<reg64 offset="0x80fa" name="GRAS_UNKNOWN_80FA" variants="A7XX-" usage="cmd"/>
+
+	<enum name="a6xx_lrz_dir_status">
+		<value value="0x1" name="LRZ_DIR_LE"/>
+		<value value="0x2" name="LRZ_DIR_GE"/>
+		<value value="0x3" name="LRZ_DIR_INVALID"/>
+	</enum>
+
+	<reg32 offset="0x8100" name="GRAS_LRZ_CNTL" usage="rp_blit">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+		<doc>LRZ write also disabled for blend/etc.</doc>
+		<bitfield name="LRZ_WRITE" pos="1" type="boolean"/>
+		<doc>update MAX instead of MIN value, ie. GL_GREATER/GL_GEQUAL</doc>
+		<bitfield name="GREATER" pos="2" type="boolean"/>
+		<doc>
+			Clears the LRZ block being touched to:
+			- 0.0 if GREATER
+			- 1.0 if LESS
+		</doc>
+		<bitfield name="FC_ENABLE" pos="3" type="boolean"/>
+		<!-- set when depth-test + depth-write enabled -->
+		<bitfield name="Z_TEST_ENABLE" pos="4" type="boolean"/>
+		<bitfield name="Z_BOUNDS_ENABLE" pos="5" type="boolean"/>
+		<bitfield name="DIR" low="6" high="7" type="a6xx_lrz_dir_status"/>
+		<doc>
+			If DISABLE_ON_WRONG_DIR enabled - write new LRZ direction into
+			buffer, in case of mismatched direction writes 0 (disables LRZ).
+		</doc>
+		<bitfield name="DIR_WRITE" pos="8" type="boolean"/>
+		<doc>
+			Disable LRZ based on previous direction and the current one.
+			If DIR_WRITE is not enabled - there is no write to direction buffer.
+		</doc>
+		<bitfield name="DISABLE_ON_WRONG_DIR" pos="9" type="boolean"/>
+		<bitfield name="Z_FUNC" low="11" high="13" type="adreno_compare_func" variants="A7XX-"/>
+	</reg32>
+
+	<enum name="a6xx_fragcoord_sample_mode">
+		<value value="0" name="FRAGCOORD_CENTER"/>
+		<value value="3" name="FRAGCOORD_SAMPLE"/>
+	</enum>
+
+	<reg32 offset="0x8101" name="GRAS_LRZ_PS_INPUT_CNTL" low="0" high="2" usage="rp_blit">
+		<bitfield name="SAMPLEID" pos="0" type="boolean"/>
+		<bitfield name="FRAGCOORDSAMPLEMODE" low="1" high="2" type="a6xx_fragcoord_sample_mode"/>
+	</reg32>
+
+	<reg32 offset="0x8102" name="GRAS_LRZ_MRT_BUF_INFO_0" usage="rp_blit">
+		<bitfield name="COLOR_FORMAT" low="0" high="7" type="a6xx_format"/>
+	</reg32>
+	<reg64 offset="0x8103" name="GRAS_LRZ_BUFFER_BASE" align="256" type="waddress" usage="rp_blit"/>
+	<reg32 offset="0x8105" name="GRAS_LRZ_BUFFER_PITCH" usage="rp_blit">
+		<!-- TODO: fix the shr fields -->
+		<bitfield name="PITCH" low="0" high="7" shr="5" type="uint"/>
+		<bitfield name="ARRAY_PITCH" low="10" high="28" shr="4" type="uint"/>
+	</reg32>
+
+	<!--
+	The LRZ "fast clear" buffer is initialized to zero's by blob, and
+	read/written when GRAS_LRZ_CNTL.FC_ENABLE (b3) is set.  It appears
+	to store 1b/block.  It appears that '0' means block has original
+	depth clear value, and '1' means that the corresponding block in
+	LRZ has been modified.  Ignoring alignment/padding, the size is
+	given by the formula:
+
+		// calculate LRZ size from depth size:
+		if (nr_samples == 4) {
+			width *= 2;
+			height *= 2;
+		} else if (nr_samples == 2) {
+			height *= 2;
+		}
+
+		lrz_width = div_round_up(width, 8);
+		lrz_heigh = div_round_up(height, 8);
+
+		// calculate # of blocks:
+		nblocksx = div_round_up(lrz_width, 16);
+		nblocksy = div_round_up(lrz_height, 4);
+
+		// fast-clear buffer is 1bit/block:
+		fc_sz = div_round_up(nblocksx * nblocksy, 8);
+
+	In practice the blob seems to switch off FC_ENABLE once the size
+	increases beyond 1 page.  Not sure if that is an actual limit or
+	not.
+	 -->
+	<reg64 offset="0x8106" name="GRAS_LRZ_FAST_CLEAR_BUFFER_BASE" align="64" type="waddress" usage="rp_blit"/>
+	<!-- 0x8108 invalid -->
+	<reg32 offset="0x8109" name="GRAS_SAMPLE_CNTL" usage="rp_blit">
+		<bitfield name="PER_SAMP_MODE" pos="0" type="boolean"/>
+	</reg32>
+	<!--
+	LRZ buffer represents a single array layer + mip level, and there is
+	a single buffer per depth image. Thus to reuse LRZ between renderpasses
+	it is necessary to track the depth view used in the past renderpass, which
+	GRAS_LRZ_DEPTH_VIEW is for.
+	GRAS_LRZ_CNTL checks if current value of GRAS_LRZ_DEPTH_VIEW is equal to
+	the value stored in the LRZ buffer, if not - LRZ is disabled.
+	-->
+	<reg32 offset="0x810a" name="GRAS_LRZ_DEPTH_VIEW" usage="cmd">
+		<bitfield name="BASE_LAYER" low="0" high="10" type="uint"/>
+		<bitfield name="LAYER_COUNT" low="16" high="26" type="uint"/>
+		<bitfield name="BASE_MIP_LEVEL" low="28" high="31" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x810b" name="GRAS_UNKNOWN_810B" variants="A7XX-" usage="cmd"/>
+
+	<!-- 0x810c-0x810f invalid -->
+
+	<reg32 offset="0x8110" name="GRAS_UNKNOWN_8110" low="0" high="1" usage="cmd"/>
+
+	<!-- A bit tentative but it's a color and it is followed by LRZ_CLEAR -->
+	<reg32 offset="0x8111" name="GRAS_LRZ_CLEAR_DEPTH_F32" type="float" variants="A7XX-"/>
+
+	<reg32 offset="0x8113" name="GRAS_LRZ_DEPTH_BUFFER_INFO" variants="A7XX-" usage="rp_blit"/>
+
+	<!-- Always written together and always equal 09510840 00000a62 -->
+	<reg32 offset="0x8120" name="GRAS_UNKNOWN_8120" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0x8121" name="GRAS_UNKNOWN_8121" variants="A7XX-" usage="cmd"/>
+
+	<!-- 0x8112-0x83ff invalid -->
+
+	<enum name="a6xx_rotation">
+		<value value="0x0" name="ROTATE_0"/>
+		<value value="0x1" name="ROTATE_90"/>
+		<value value="0x2" name="ROTATE_180"/>
+		<value value="0x3" name="ROTATE_270"/>
+		<value value="0x4" name="ROTATE_HFLIP"/>
+		<value value="0x5" name="ROTATE_VFLIP"/>
+	</enum>
+
+	<bitset name="a6xx_2d_blit_cntl" inline="yes">
+		<bitfield name="ROTATE" low="0" high="2" type="a6xx_rotation"/>
+		<bitfield name="OVERWRITEEN" pos="3" type="boolean"/>
+		<bitfield name="UNK4" low="4" high="6"/>
+		<bitfield name="SOLID_COLOR" pos="7" type="boolean"/>
+		<bitfield name="COLOR_FORMAT" low="8" high="15" type="a6xx_format"/>
+		<bitfield name="SCISSOR" pos="16" type="boolean"/>
+		<bitfield name="UNK17" low="17" high="18"/>
+		<!-- required when blitting D24S8/D24X8 -->
+		<bitfield name="D24S8" pos="19" type="boolean"/>
+		<!-- some sort of channel mask, disabled channels are set to zero ? -->
+		<bitfield name="MASK" low="20" high="23"/>
+		<bitfield name="IFMT" low="24" high="28" type="a6xx_2d_ifmt"/>
+		<bitfield name="RASTER_MODE" pos="29" type="a6xx_raster_mode"/>
+		<bitfield name="UNK30" pos="30" type="boolean" variants="A7XX-"/>
+	</bitset>
+
+	<reg32 offset="0x8400" name="GRAS_2D_BLIT_CNTL" type="a6xx_2d_blit_cntl" usage="rp_blit"/>
+	<!-- note: the low 8 bits for src coords are valid, probably fixed point
+	     it would be a bit weird though, since we subtract 1 from BR coords
+	     apparently signed, gallium driver uses negative coords and it works?
+	 -->
+	<reg32 offset="0x8401" name="GRAS_2D_SRC_TL_X" low="8" high="24" type="int" usage="rp_blit"/>
+	<reg32 offset="0x8402" name="GRAS_2D_SRC_BR_X" low="8" high="24" type="int" usage="rp_blit"/>
+	<reg32 offset="0x8403" name="GRAS_2D_SRC_TL_Y" low="8" high="24" type="int" usage="rp_blit"/>
+	<reg32 offset="0x8404" name="GRAS_2D_SRC_BR_Y" low="8" high="24" type="int" usage="rp_blit"/>
+	<reg32 offset="0x8405" name="GRAS_2D_DST_TL" type="a6xx_reg_xy" usage="rp_blit"/>
+	<reg32 offset="0x8406" name="GRAS_2D_DST_BR" type="a6xx_reg_xy" usage="rp_blit"/>
+	<reg32 offset="0x8407" name="GRAS_2D_UNKNOWN_8407" low="0" high="31"/>
+	<reg32 offset="0x8408" name="GRAS_2D_UNKNOWN_8408" low="0" high="31"/>
+	<reg32 offset="0x8409" name="GRAS_2D_UNKNOWN_8409" low="0" high="31"/>
+	<reg32 offset="0x840a" name="GRAS_2D_RESOLVE_CNTL_1" type="a6xx_reg_xy" usage="rp_blit"/>
+	<reg32 offset="0x840b" name="GRAS_2D_RESOLVE_CNTL_2" type="a6xx_reg_xy" usage="rp_blit"/>
+	<!-- 0x840c-0x85ff invalid -->
+
+	<!-- always 0x880 ? (and 0 in a640/a650 traces?) -->
+	<reg32 offset="0x8600" name="GRAS_DBG_ECO_CNTL" usage="cmd">
+		<bitfield name="UNK7" pos="7" type="boolean"/>
+		<bitfield name="LRZCACHELOCKDIS" pos="11" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x8601" name="GRAS_ADDR_MODE_CNTL" pos="0" type="a5xx_address_mode"/>
+	<reg32 offset="0x8602" name="GRAS_NC_MODE_CNTL" variants="A7XX-"/>
+	<array offset="0x8610" name="GRAS_PERFCTR_TSE_SEL" stride="1" length="4"/>
+	<array offset="0x8614" name="GRAS_PERFCTR_RAS_SEL" stride="1" length="4"/>
+	<array offset="0x8618" name="GRAS_PERFCTR_LRZ_SEL" stride="1" length="4"/>
+
+	<!-- note 0x8620-0x87ff are not all invalid
+	(in particular, 0x8631/0x8632 have 0x3fff3fff mask and would be xy coords)
+	-->
+
+	<!-- same as GRAS_BIN_CONTROL, but without bit 27: -->
+	<reg32 offset="0x8800" name="RB_BIN_CONTROL" variants="A6XX" usage="rp_blit">
+		<bitfield name="BINW" low="0" high="5" shr="5" type="uint"/>
+		<bitfield name="BINH" low="8" high="14" shr="4" type="uint"/>
+		<bitfield name="RENDER_MODE" low="18" high="20" type="a6xx_render_mode"/>
+		<bitfield name="FORCE_LRZ_WRITE_DIS" pos="21" type="boolean"/>
+		<bitfield name="BUFFERS_LOCATION" low="22" high="23" type="a6xx_buffers_location"/>
+		<bitfield name="LRZ_FEEDBACK_ZMODE_MASK" low="24" high="26"/>
+	</reg32>
+
+	<reg32 offset="0x8800" name="RB_BIN_CONTROL" variants="A7XX-" usage="rp_blit">
+		<bitfield name="BINW" low="0" high="5" shr="5" type="uint"/>
+		<bitfield name="BINH" low="8" high="14" shr="4" type="uint"/>
+		<bitfield name="RENDER_MODE" low="18" high="20" type="a6xx_render_mode"/>
+		<bitfield name="FORCE_LRZ_WRITE_DIS" pos="21" type="boolean"/>
+		<bitfield name="LRZ_FEEDBACK_ZMODE_MASK" low="24" high="26"/>
+	</reg32>
+
+	<reg32 offset="0x8801" name="RB_RENDER_CNTL" variants="A6XX" usage="rp_blit">
+		<bitfield name="CCUSINGLECACHELINESIZE" low="3" high="5"/>
+		<bitfield name="EARLYVIZOUTEN" pos="6" type="boolean"/>
+		<!-- set during binning pass: -->
+		<bitfield name="BINNING" pos="7" type="boolean"/>
+		<bitfield name="UNK8" low="8" high="10"/>
+		<bitfield name="RASTER_MODE" pos="8" type="a6xx_raster_mode"/>
+		<bitfield name="RASTER_DIRECTION" low="9" high="10" type="a6xx_raster_direction"/>
+		<bitfield name="CONSERVATIVERASEN" pos="11" type="boolean"/>
+		<bitfield name="INNERCONSERVATIVERASEN" pos="12" type="boolean"/>
+		<!-- bit seems to be set whenever depth buffer enabled: -->
+		<bitfield name="FLAG_DEPTH" pos="14" type="boolean"/>
+		<!-- bitmask of MRTs using UBWC flag buffer: -->
+		<bitfield name="FLAG_MRTS" low="16" high="23"/>
+	</reg32>
+	<reg32 offset="0x8801" name="RB_RENDER_CNTL" variants="A7XX-" usage="rp_blit">
+		<bitfield name="EARLYVIZOUTEN" pos="6" type="boolean"/>
+		<!-- set during binning pass: -->
+		<bitfield name="BINNING" pos="7" type="boolean"/>
+		<bitfield name="RASTER_MODE" pos="8" type="a6xx_raster_mode"/>
+		<bitfield name="RASTER_DIRECTION" low="9" high="10" type="a6xx_raster_direction"/>
+		<bitfield name="CONSERVATIVERASEN" pos="11" type="boolean"/>
+		<bitfield name="INNERCONSERVATIVERASEN" pos="12" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x8116" name="GRAS_SU_RENDER_CNTL" variants="A7XX-" usage="rp_blit">
+		<bitfield name="BINNING" pos="7" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x8802" name="RB_RAS_MSAA_CNTL" usage="rp_blit">
+		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>
+		<bitfield name="UNK2" pos="2"/>
+		<bitfield name="UNK3" pos="3"/>
+	</reg32>
+	<reg32 offset="0x8803" name="RB_DEST_MSAA_CNTL" usage="rp_blit">
+		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>
+		<bitfield name="MSAA_DISABLE" pos="2" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x8804" name="RB_SAMPLE_CONFIG" type="a6xx_sample_config" usage="rp_blit"/>
+	<reg32 offset="0x8805" name="RB_SAMPLE_LOCATION_0" type="a6xx_sample_locations" usage="rp_blit"/>
+	<reg32 offset="0x8806" name="RB_SAMPLE_LOCATION_1" type="a6xx_sample_locations" usage="rp_blit"/>
+	<!-- 0x8807-0x8808 invalid -->
+	<!--
+	note: maybe not actually called RB_RENDER_CONTROLn (since RB_RENDER_CNTL
+	name comes from kernel and is probably right)
+	 -->
+	<reg32 offset="0x8809" name="RB_RENDER_CONTROL0" usage="rp_blit">
+		<!-- see also GRAS_CNTL -->
+		<bitfield name="IJ_PERSP_PIXEL" pos="0" type="boolean"/>
+		<bitfield name="IJ_PERSP_CENTROID" pos="1" type="boolean"/>
+		<bitfield name="IJ_PERSP_SAMPLE" pos="2" type="boolean"/>
+		<bitfield name="IJ_LINEAR_PIXEL" pos="3" type="boolean"/>
+		<bitfield name="IJ_LINEAR_CENTROID" pos="4" type="boolean"/>
+		<bitfield name="IJ_LINEAR_SAMPLE" pos="5" type="boolean"/>
+		<bitfield name="COORD_MASK" low="6" high="9" type="hex"/>
+		<bitfield name="UNK10" pos="10" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x880a" name="RB_RENDER_CONTROL1" usage="rp_blit">
+		<!-- enable bits for various FS sysvalue regs: -->
+		<bitfield name="SAMPLEMASK" pos="0" type="boolean"/>
+		<bitfield name="POSTDEPTHCOVERAGE" pos="1" type="boolean"/>
+		<bitfield name="FACENESS" pos="2" type="boolean"/>
+		<bitfield name="SAMPLEID" pos="3" type="boolean"/>
+		<bitfield name="FRAGCOORDSAMPLEMODE" low="4" high="5" type="a6xx_fragcoord_sample_mode"/>
+		<bitfield name="CENTERRHW" pos="6" type="boolean"/>
+		<bitfield name="LINELENGTHEN" pos="7" type="boolean"/>
+		<bitfield name="FOVEATION" pos="8" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x880b" name="RB_FS_OUTPUT_CNTL0" usage="rp_blit">
+		<bitfield name="DUAL_COLOR_IN_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="FRAG_WRITES_Z" pos="1" type="boolean"/>
+		<bitfield name="FRAG_WRITES_SAMPMASK" pos="2" type="boolean"/>
+		<bitfield name="FRAG_WRITES_STENCILREF" pos="3" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x880c" name="RB_FS_OUTPUT_CNTL1" usage="rp_blit">
+		<bitfield name="MRT" low="0" high="3" type="uint"/>
+	</reg32>
+	<reg32 offset="0x880d" name="RB_RENDER_COMPONENTS" usage="rp_blit">
+		<bitfield name="RT0" low="0" high="3"/>
+		<bitfield name="RT1" low="4" high="7"/>
+		<bitfield name="RT2" low="8" high="11"/>
+		<bitfield name="RT3" low="12" high="15"/>
+		<bitfield name="RT4" low="16" high="19"/>
+		<bitfield name="RT5" low="20" high="23"/>
+		<bitfield name="RT6" low="24" high="27"/>
+		<bitfield name="RT7" low="28" high="31"/>
+	</reg32>
+	<reg32 offset="0x880e" name="RB_DITHER_CNTL" usage="cmd">
+		<bitfield name="DITHER_MODE_MRT0" low="0"  high="1"  type="adreno_rb_dither_mode"/>
+		<bitfield name="DITHER_MODE_MRT1" low="2"  high="3"  type="adreno_rb_dither_mode"/>
+		<bitfield name="DITHER_MODE_MRT2" low="4"  high="5"  type="adreno_rb_dither_mode"/>
+		<bitfield name="DITHER_MODE_MRT3" low="6"  high="7"  type="adreno_rb_dither_mode"/>
+		<bitfield name="DITHER_MODE_MRT4" low="8"  high="9"  type="adreno_rb_dither_mode"/>
+		<bitfield name="DITHER_MODE_MRT5" low="10" high="11" type="adreno_rb_dither_mode"/>
+		<bitfield name="DITHER_MODE_MRT6" low="12" high="13" type="adreno_rb_dither_mode"/>
+		<bitfield name="DITHER_MODE_MRT7" low="14" high="15" type="adreno_rb_dither_mode"/>
+	</reg32>
+	<reg32 offset="0x880f" name="RB_SRGB_CNTL" usage="rp_blit">
+		<!-- Same as SP_SRGB_CNTL -->
+		<bitfield name="SRGB_MRT0" pos="0" type="boolean"/>
+		<bitfield name="SRGB_MRT1" pos="1" type="boolean"/>
+		<bitfield name="SRGB_MRT2" pos="2" type="boolean"/>
+		<bitfield name="SRGB_MRT3" pos="3" type="boolean"/>
+		<bitfield name="SRGB_MRT4" pos="4" type="boolean"/>
+		<bitfield name="SRGB_MRT5" pos="5" type="boolean"/>
+		<bitfield name="SRGB_MRT6" pos="6" type="boolean"/>
+		<bitfield name="SRGB_MRT7" pos="7" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x8810" name="RB_SAMPLE_CNTL" usage="rp_blit">
+		<bitfield name="PER_SAMP_MODE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x8811" name="RB_UNKNOWN_8811" low="4" high="6" usage="cmd"/>
+	<reg32 offset="0x8812" name="RB_UNKNOWN_8812" variants="A7XX-" usage="rp_blit"/>
+	<!-- 0x8813-0x8817 invalid -->
+	<!-- always 0x0 ? -->
+	<reg32 offset="0x8818" name="RB_UNKNOWN_8818" low="0" high="6" usage="cmd"/>
+	<!-- 0x8819-0x881e all 32 bits -->
+	<reg32 offset="0x8819" name="RB_UNKNOWN_8819" usage="cmd"/>
+	<reg32 offset="0x881a" name="RB_UNKNOWN_881A" usage="cmd"/>
+	<reg32 offset="0x881b" name="RB_UNKNOWN_881B" usage="cmd"/>
+	<reg32 offset="0x881c" name="RB_UNKNOWN_881C" usage="cmd"/>
+	<reg32 offset="0x881d" name="RB_UNKNOWN_881D" usage="cmd"/>
+	<reg32 offset="0x881e" name="RB_UNKNOWN_881E" usage="cmd"/>
+	<!-- 0x881f invalid -->
+	<array offset="0x8820" name="RB_MRT" stride="8" length="8" usage="rp_blit">
+		<reg32 offset="0x0" name="CONTROL">
+			<bitfield name="BLEND" pos="0" type="boolean"/>
+			<bitfield name="BLEND2" pos="1" type="boolean"/>
+			<bitfield name="ROP_ENABLE" pos="2" type="boolean"/>
+			<bitfield name="ROP_CODE" low="3" high="6" type="a3xx_rop_code"/>
+			<bitfield name="COMPONENT_ENABLE" low="7" high="10" type="hex"/>
+		</reg32>
+		<reg32 offset="0x1" name="BLEND_CONTROL">
+			<bitfield name="RGB_SRC_FACTOR" low="0" high="4" type="adreno_rb_blend_factor"/>
+			<bitfield name="RGB_BLEND_OPCODE" low="5" high="7" type="a3xx_rb_blend_opcode"/>
+			<bitfield name="RGB_DEST_FACTOR" low="8" high="12" type="adreno_rb_blend_factor"/>
+			<bitfield name="ALPHA_SRC_FACTOR" low="16" high="20" type="adreno_rb_blend_factor"/>
+			<bitfield name="ALPHA_BLEND_OPCODE" low="21" high="23" type="a3xx_rb_blend_opcode"/>
+			<bitfield name="ALPHA_DEST_FACTOR" low="24" high="28" type="adreno_rb_blend_factor"/>
+		</reg32>
+		<reg32 offset="0x2" name="BUF_INFO" variants="A6XX">
+			<bitfield name="COLOR_FORMAT" low="0" high="7" type="a6xx_format"/>
+			<bitfield name="COLOR_TILE_MODE" low="8" high="9" type="a6xx_tile_mode"/>
+			<bitfield name="UNK10" pos="10"/>
+			<bitfield name="COLOR_SWAP" low="13" high="14" type="a3xx_color_swap"/>
+		</reg32>
+		<reg32 offset="0x2" name="BUF_INFO" variants="A7XX-">
+			<bitfield name="COLOR_FORMAT" low="0" high="7" type="a6xx_format"/>
+			<bitfield name="COLOR_TILE_MODE" low="8" high="9" type="a6xx_tile_mode"/>
+			<bitfield name="UNK10" pos="10"/>
+			<bitfield name="LOSSLESSCOMPEN" pos="11" type="boolean"/>
+			<bitfield name="COLOR_SWAP" low="13" high="14" type="a3xx_color_swap"/>
+		</reg32>
+		<!--
+		at least in gmem, things seem to be aligned to pitch of 64..
+		maybe an artifact of tiled format used in gmem?
+		 -->
+		<reg32 offset="0x3" name="PITCH" shr="6" high="15" type="uint"/>
+		<reg32 offset="0x4" name="ARRAY_PITCH" shr="6" high="28" type="uint"/>
+		<!--
+		Compared to a5xx and before, we configure both a GMEM base and
+		external base.  Not sure if this is to facilitate GMEM save/
+		restore for context switch, or just to simplify state setup to
+		not have to care about GMEM vs BYPASS mode.
+		 -->
+		<!-- maybe something in low bits since alignment of 1 doesn't make sense? -->
+		<reg64 offset="0x5" name="BASE" type="waddress" align="1"/>
+
+		<reg32 offset="0x7" name="BASE_GMEM" low="12" high="31" shr="12"/>
+	</array>
+
+	<reg32 offset="0x8860" name="RB_BLEND_RED_F32" type="float" usage="rp_blit"/>
+	<reg32 offset="0x8861" name="RB_BLEND_GREEN_F32" type="float" usage="rp_blit"/>
+	<reg32 offset="0x8862" name="RB_BLEND_BLUE_F32" type="float" usage="rp_blit"/>
+	<reg32 offset="0x8863" name="RB_BLEND_ALPHA_F32" type="float" usage="rp_blit"/>
+	<reg32 offset="0x8864" name="RB_ALPHA_CONTROL" usage="cmd">
+		<bitfield name="ALPHA_REF" low="0" high="7" type="hex"/>
+		<bitfield name="ALPHA_TEST" pos="8" type="boolean"/>
+		<bitfield name="ALPHA_TEST_FUNC" low="9" high="11" type="adreno_compare_func"/>
+	</reg32>
+	<reg32 offset="0x8865" name="RB_BLEND_CNTL" usage="rp_blit">
+		<!-- per-mrt enable bit -->
+		<bitfield name="ENABLE_BLEND" low="0" high="7"/>
+		<bitfield name="INDEPENDENT_BLEND" pos="8" type="boolean"/>
+		<bitfield name="DUAL_COLOR_IN_ENABLE" pos="9" type="boolean"/>
+		<bitfield name="ALPHA_TO_COVERAGE" pos="10" type="boolean"/>
+		<bitfield name="ALPHA_TO_ONE" pos="11" type="boolean"/>
+		<bitfield name="SAMPLE_MASK" low="16" high="31"/>
+	</reg32>
+	<!-- 0x8866-0x886f invalid -->
+	<reg32 offset="0x8870" name="RB_DEPTH_PLANE_CNTL" usage="rp_blit">
+		<bitfield name="Z_MODE" low="0" high="1" type="a6xx_ztest_mode"/>
+	</reg32>
+
+	<reg32 offset="0x8871" name="RB_DEPTH_CNTL" usage="rp_blit">
+		<bitfield name="Z_TEST_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="Z_WRITE_ENABLE" pos="1" type="boolean"/>
+		<bitfield name="ZFUNC" low="2" high="4" type="adreno_compare_func"/>
+		<bitfield name="Z_CLAMP_ENABLE" pos="5" type="boolean"/>
+		<doc>
+		Z_READ_ENABLE bit is set for zfunc other than GL_ALWAYS or GL_NEVER
+		also set when Z_BOUNDS_ENABLE is set
+		</doc>
+		<bitfield name="Z_READ_ENABLE" pos="6" type="boolean"/>
+		<bitfield name="Z_BOUNDS_ENABLE" pos="7" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x8114" name="GRAS_SU_DEPTH_CNTL" usage="rp_blit">
+		<bitfield name="Z_TEST_ENABLE" pos="0" type="boolean"/>
+	</reg32>
+	<!-- duplicates GRAS_SU_DEPTH_BUFFER_INFO: -->
+	<reg32 offset="0x8872" name="RB_DEPTH_BUFFER_INFO" variants="A6XX" usage="rp_blit">
+		<bitfield name="DEPTH_FORMAT" low="0" high="2" type="a6xx_depth_format"/>
+		<bitfield name="UNK3" low="3" high="4"/>
+	</reg32>
+	<!-- first 4 bits duplicates GRAS_SU_DEPTH_BUFFER_INFO -->
+	<reg32 offset="0x8872" name="RB_DEPTH_BUFFER_INFO" variants="A7XX-" usage="rp_blit">
+		<bitfield name="DEPTH_FORMAT" low="0" high="2" type="a6xx_depth_format"/>
+		<bitfield name="UNK3" low="3" high="4"/>
+		<bitfield name="TILEMODE" low="5" high="6" type="a6xx_tile_mode"/>
+		<bitfield name="LOSSLESSCOMPEN" pos="7" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x8873" name="RB_DEPTH_BUFFER_PITCH" low="0" high="13" shr="6" type="uint" usage="rp_blit"/>
+	<reg32 offset="0x8874" name="RB_DEPTH_BUFFER_ARRAY_PITCH" low="0" high="27" shr="6" type="uint" usage="rp_blit"/>
+	<reg64 offset="0x8875" name="RB_DEPTH_BUFFER_BASE" type="waddress" align="64" usage="rp_blit"/>
+	<reg32 offset="0x8877" name="RB_DEPTH_BUFFER_BASE_GMEM" low="12" high="31" shr="12" usage="rp_blit"/>
+
+	<reg32 offset="0x8878" name="RB_Z_BOUNDS_MIN" type="float" usage="rp_blit"/>
+	<reg32 offset="0x8879" name="RB_Z_BOUNDS_MAX" type="float" usage="rp_blit"/>
+	<!-- 0x887a-0x887f invalid -->
+	<reg32 offset="0x8880" name="RB_STENCIL_CONTROL" usage="rp_blit">
+		<bitfield name="STENCIL_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="STENCIL_ENABLE_BF" pos="1" type="boolean"/>
+		<!--
+			set for stencil operations that require read from stencil
+			buffer, but not for example for stencil clear (which does
+			not require read).. so guessing this is analogous to
+			READ_DEST_ENABLE for color buffer..
+		 -->
+		<bitfield name="STENCIL_READ" pos="2" type="boolean"/>
+		<bitfield name="FUNC" low="8" high="10" type="adreno_compare_func"/>
+		<bitfield name="FAIL" low="11" high="13" type="adreno_stencil_op"/>
+		<bitfield name="ZPASS" low="14" high="16" type="adreno_stencil_op"/>
+		<bitfield name="ZFAIL" low="17" high="19" type="adreno_stencil_op"/>
+		<bitfield name="FUNC_BF" low="20" high="22" type="adreno_compare_func"/>
+		<bitfield name="FAIL_BF" low="23" high="25" type="adreno_stencil_op"/>
+		<bitfield name="ZPASS_BF" low="26" high="28" type="adreno_stencil_op"/>
+		<bitfield name="ZFAIL_BF" low="29" high="31" type="adreno_stencil_op"/>
+	</reg32>
+	<reg32 offset="0x8115" name="GRAS_SU_STENCIL_CNTL" usage="rp_blit">
+		<bitfield name="STENCIL_ENABLE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x8881" name="RB_STENCIL_INFO" variants="A6XX" usage="rp_blit">
+		<bitfield name="SEPARATE_STENCIL" pos="0" type="boolean"/>
+		<bitfield name="UNK1" pos="1" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x8881" name="RB_STENCIL_INFO" variants="A7XX-" usage="rp_blit">
+		<bitfield name="SEPARATE_STENCIL" pos="0" type="boolean"/>
+		<bitfield name="UNK1" pos="1" type="boolean"/>
+		<bitfield name="TILEMODE" low="2" high="3" type="a6xx_tile_mode"/>
+	</reg32>
+	<reg32 offset="0x8882" name="RB_STENCIL_BUFFER_PITCH" low="0" high="11" shr="6" type="uint" usage="rp_blit"/>
+	<reg32 offset="0x8883" name="RB_STENCIL_BUFFER_ARRAY_PITCH" low="0" high="23" shr="6" type="uint" usage="rp_blit"/>
+	<reg64 offset="0x8884" name="RB_STENCIL_BUFFER_BASE" type="waddress" align="64" usage="rp_blit"/>
+	<reg32 offset="0x8886" name="RB_STENCIL_BUFFER_BASE_GMEM" low="12" high="31" shr="12" usage="rp_blit"/>
+	<reg32 offset="0x8887" name="RB_STENCILREF" usage="rp_blit">
+		<bitfield name="REF" low="0" high="7"/>
+		<bitfield name="BFREF" low="8" high="15"/>
+	</reg32>
+	<reg32 offset="0x8888" name="RB_STENCILMASK" usage="rp_blit">
+		<bitfield name="MASK" low="0" high="7"/>
+		<bitfield name="BFMASK" low="8" high="15"/>
+	</reg32>
+	<reg32 offset="0x8889" name="RB_STENCILWRMASK" usage="rp_blit">
+		<bitfield name="WRMASK" low="0" high="7"/>
+		<bitfield name="BFWRMASK" low="8" high="15"/>
+	</reg32>
+	<!-- 0x888a-0x888f invalid -->
+	<reg32 offset="0x8890" name="RB_WINDOW_OFFSET" type="a6xx_reg_xy" usage="rp_blit"/>
+	<reg32 offset="0x8891" name="RB_SAMPLE_COUNT_CONTROL" usage="cmd">
+		<bitfield name="DISABLE" pos="0" type="boolean"/>
+		<bitfield name="COPY" pos="1" type="boolean"/>
+	</reg32>
+	<!-- 0x8892-0x8897 invalid -->
+	<reg32 offset="0x8898" name="RB_LRZ_CNTL" usage="rp_blit">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x8899" name="RB_UNKNOWN_8899" variants="A7XX-" usage="cmd"/>
+	<!-- 0x8899-0x88bf invalid -->
+	<!-- clamps depth value for depth test/write -->
+	<reg32 offset="0x88c0" name="RB_Z_CLAMP_MIN" type="float" usage="rp_blit"/>
+	<reg32 offset="0x88c1" name="RB_Z_CLAMP_MAX" type="float" usage="rp_blit"/>
+	<!-- 0x88c2-0x88cf invalid-->
+	<reg32 offset="0x88d0" name="RB_UNKNOWN_88D0" usage="rp_blit">
+		<bitfield name="UNK0" low="0" high="12"/>
+		<bitfield name="UNK16" low="16" high="26"/>
+	</reg32>
+	<reg32 offset="0x88d1" name="RB_BLIT_SCISSOR_TL" type="a6xx_reg_xy" usage="rp_blit"/>
+	<reg32 offset="0x88d2" name="RB_BLIT_SCISSOR_BR" type="a6xx_reg_xy" usage="rp_blit"/>
+	<!-- weird to duplicate other regs from same block?? -->
+	<reg32 offset="0x88d3" name="RB_BIN_CONTROL2" usage="rp_blit">
+		<bitfield name="BINW" low="0" high="5" shr="5" type="uint"/>
+		<bitfield name="BINH" low="8" high="14" shr="4" type="uint"/>
+	</reg32>
+	<reg32 offset="0x88d4" name="RB_WINDOW_OFFSET2" type="a6xx_reg_xy" usage="rp_blit"/>
+	<reg32 offset="0x88d5" name="RB_BLIT_GMEM_MSAA_CNTL" usage="rp_blit">
+		<bitfield name="SAMPLES" low="3" high="4" type="a3xx_msaa_samples"/>
+	</reg32>
+	<reg32 offset="0x88d6" name="RB_BLIT_BASE_GMEM" low="12" high="31" shr="12" usage="rp_blit"/>
+	<!-- s/DST_FORMAT/DST_INFO/ probably: -->
+	<reg32 offset="0x88d7" name="RB_BLIT_DST_INFO" usage="rp_blit">
+		<bitfield name="TILE_MODE" low="0" high="1" type="a6xx_tile_mode"/>
+		<bitfield name="FLAGS" pos="2" type="boolean"/>
+		<bitfield name="SAMPLES" low="3" high="4" type="a3xx_msaa_samples"/>
+		<bitfield name="COLOR_SWAP" low="5" high="6" type="a3xx_color_swap"/>
+		<bitfield name="COLOR_FORMAT" low="7" high="14" type="a6xx_format"/>
+		<bitfield name="UNK15" pos="15" type="boolean"/>
+	</reg32>
+	<reg64 offset="0x88d8" name="RB_BLIT_DST" type="waddress" align="64" usage="rp_blit"/>
+	<reg32 offset="0x88da" name="RB_BLIT_DST_PITCH" low="0" high="15" shr="6" type="uint" usage="rp_blit"/>
+	<!-- array-pitch is size of layer -->
+	<reg32 offset="0x88db" name="RB_BLIT_DST_ARRAY_PITCH" low="0" high="28" shr="6" type="uint" usage="rp_blit"/>
+	<reg64 offset="0x88dc" name="RB_BLIT_FLAG_DST" type="waddress" align="64" usage="rp_blit"/>
+	<reg32 offset="0x88de" name="RB_BLIT_FLAG_DST_PITCH" usage="rp_blit">
+		<bitfield name="PITCH" low="0" high="10" shr="6" type="uint"/>
+		<bitfield name="ARRAY_PITCH" low="11" high="27" shr="7" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x88df" name="RB_BLIT_CLEAR_COLOR_DW0" usage="rp_blit"/>
+	<reg32 offset="0x88e0" name="RB_BLIT_CLEAR_COLOR_DW1" usage="rp_blit"/>
+	<reg32 offset="0x88e1" name="RB_BLIT_CLEAR_COLOR_DW2" usage="rp_blit"/>
+	<reg32 offset="0x88e2" name="RB_BLIT_CLEAR_COLOR_DW3" usage="rp_blit"/>
+
+	<!-- seems somewhat similar to what we called RB_CLEAR_CNTL on a5xx: -->
+	<reg32 offset="0x88e3" name="RB_BLIT_INFO" usage="rp_blit">
+		<bitfield name="UNK0" pos="0" type="boolean"/> <!-- s8 stencil restore/clear?  But also color restore? -->
+		<bitfield name="GMEM" pos="1" type="boolean"/> <!-- set for restore and clear to gmem? -->
+		<bitfield name="SAMPLE_0" pos="2" type="boolean"/> <!-- takes sample 0 instead of averaging -->
+		<bitfield name="DEPTH" pos="3" type="boolean"/> <!-- z16/z32/z24s8/x24x8 clear or resolve? -->
+		<doc>
+			For clearing depth/stencil
+				1 - depth
+				2 - stencil
+				3 - depth+stencil
+			For clearing color buffer:
+				then probably a component mask, I always see 0xf
+		</doc>
+		<bitfield name="CLEAR_MASK" low="4" high="7"/>
+		<!-- set when this is the last resolve on a650+ -->
+		<bitfield name="LAST" low="8" high="9"/>
+		<!--
+			a618 GLES: color render target number being resolved for RM6_RESOLVE, 0x8 for depth, 0x9 for separate stencil.
+			a618 VK: 0x8 for depth RM6_RESOLVE, 0x9 for separate stencil, 0 otherwise.
+
+			We believe this is related to concurrent resolves
+		 -->
+		<bitfield name="BUFFER_ID" low="12" high="15"/>
+	</reg32>
+	<reg32 offset="0x88e4" name="RB_UNKNOWN_88E4" variants="A7XX-" usage="rp_blit">
+		<!-- Value conditioned based on predicate, changed before blits -->
+		<bitfield name="UNK0" pos="0" type="boolean"/>
+	</reg32>
+
+	<enum name="a6xx_ccu_cache_size">
+		<value value="0x0" name="CCU_CACHE_SIZE_FULL"/>
+		<value value="0x1" name="CCU_CACHE_SIZE_HALF"/>
+		<value value="0x2" name="CCU_CACHE_SIZE_QUARTER"/>
+		<value value="0x3" name="CCU_CACHE_SIZE_EIGHTH"/>
+	</enum>
+	<reg32 offset="0x88e5" name="RB_CCU_CNTL2" variants="A7XX-" usage="cmd">
+		<bitfield name="DEPTH_OFFSET_HI" pos="0" type="hex"/>
+		<bitfield name="COLOR_OFFSET_HI" pos="2" type="hex"/>
+		<bitfield name="DEPTH_CACHE_SIZE" low="10" high="11" type="a6xx_ccu_cache_size"/>
+		<!-- GMEM offset of CCU depth cache -->
+		<bitfield name="DEPTH_OFFSET" low="12" high="20" shr="12" type="hex"/>
+		<bitfield name="COLOR_CACHE_SIZE" low="21" high="22" type="a6xx_ccu_cache_size"/>
+		<!-- GMEM offset of CCU color cache
+			for GMEM rendering, we set it to GMEM size minus the minimum
+			CCU color cache size. CCU color cache will be needed in some
+			resolve cases, and in those cases we need to reserve the end
+			of GMEM for color cache.
+		-->
+		<bitfield name="COLOR_OFFSET" low="23" high="31" shr="12" type="hex"/>
+	</reg32>
+	<!-- 0x88e6-0x88ef invalid -->
+	<!-- always 0x0 ? -->
+	<reg32 offset="0x88f0" name="RB_UNKNOWN_88F0" low="0" high="11" usage="cmd"/>
+	<!-- could be for separate stencil? (or may not be a flag buffer at all) -->
+	<reg64 offset="0x88f1" name="RB_UNK_FLAG_BUFFER_BASE" type="waddress" align="64"/>
+	<reg32 offset="0x88f3" name="RB_UNK_FLAG_BUFFER_PITCH">
+		<bitfield name="PITCH" low="0" high="10" shr="6" type="uint"/>
+		<bitfield name="ARRAY_PITCH" low="11" high="23" shr="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x88f4" name="RB_UNKNOWN_88F4" low="0" high="2"/>
+	<!-- Connected to VK_EXT_fragment_density_map? -->
+	<reg32 offset="0x88f5" name="RB_UNKNOWN_88F5" variants="A7XX-"/>
+	<!-- 0x88f6-0x88ff invalid -->
+	<reg64 offset="0x8900" name="RB_DEPTH_FLAG_BUFFER_BASE" type="waddress" align="64" usage="rp_blit"/>
+	<reg32 offset="0x8902" name="RB_DEPTH_FLAG_BUFFER_PITCH" usage="rp_blit">
+		<bitfield name="PITCH" low="0" high="6" shr="6" type="uint"/>
+		<!-- TODO: actually part of array pitch -->
+		<bitfield name="UNK8" low="8" high="10"/>
+		<bitfield name="ARRAY_PITCH" low="11" high="27" shr="7" type="uint"/>
+	</reg32>
+	<array offset="0x8903" name="RB_MRT_FLAG_BUFFER" stride="3" length="8" usage="rp_blit">
+		<reg64 offset="0" name="ADDR" type="waddress" align="64"/>
+		<reg32 offset="2" name="PITCH">
+			<bitfield name="PITCH" low="0" high="10" shr="6" type="uint"/>
+			<bitfield name="ARRAY_PITCH" low="11" high="28" shr="7" type="uint"/>
+		</reg32>
+	</array>
+	<!-- 0x891b-0x8926 invalid -->
+	<doc>
+		RB_SAMPLE_COUNT_ADDR register is used up to (and including) a730. After that
+		the address is specified through CP_EVENT_WRITE7::WRITE_SAMPLE_COUNT.
+	</doc>
+	<reg64 offset="0x8927" name="RB_SAMPLE_COUNT_ADDR" type="waddress" align="16" usage="cmd"/>
+	<!-- 0x8929-0x89ff invalid -->
+
+	<!-- TODO: there are some registers in the 0x8a00-0x8bff range -->
+
+	<!--
+		These show up in a6xx gen3+ but so far haven't found an example of
+		blob writing non-zero:
+	 -->
+	<reg32 offset="0x8a00" name="RB_UNKNOWN_8A00" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0x8a10" name="RB_UNKNOWN_8A10" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0x8a20" name="RB_UNKNOWN_8A20" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0x8a30" name="RB_UNKNOWN_8A30" variants="A6XX" usage="rp_blit"/>
+
+	<reg32 offset="0x8c00" name="RB_2D_BLIT_CNTL" type="a6xx_2d_blit_cntl" usage="rp_blit"/>
+	<reg32 offset="0x8c01" name="RB_2D_UNKNOWN_8C01" low="0" high="31" usage="rp_blit"/>
+
+	<bitset name="a6xx_2d_surf_info" inline="yes">
+		<bitfield name="COLOR_FORMAT" low="0" high="7" type="a6xx_format"/>
+		<bitfield name="TILE_MODE" low="8" high="9" type="a6xx_tile_mode"/>
+		<bitfield name="COLOR_SWAP" low="10" high="11" type="a3xx_color_swap"/>
+		<bitfield name="FLAGS" pos="12" type="boolean"/>
+		<bitfield name="SRGB" pos="13" type="boolean"/>
+		<!-- the rest is only for src -->
+		<bitfield name="SAMPLES" low="14" high="15" type="a3xx_msaa_samples"/>
+		<bitfield name="FILTER" pos="16" type="boolean"/>
+		<bitfield name="UNK17" pos="17" type="boolean"/>
+		<bitfield name="SAMPLES_AVERAGE" pos="18" type="boolean"/>
+		<bitfield name="UNK19" pos="19" type="boolean"/>
+		<bitfield name="UNK20" pos="20" type="boolean"/>
+		<bitfield name="UNK21" pos="21" type="boolean"/>
+		<bitfield name="UNK22" pos="22" type="boolean"/>
+		<bitfield name="UNK23" low="23" high="26"/>
+		<bitfield name="UNK28" pos="28" type="boolean"/>
+	</bitset>
+
+	<!-- 0x8c02-0x8c16 invalid -->
+	<!-- TODO: RB_2D_DST_INFO has 17 valid bits (doesn't match a6xx_2d_surf_info) -->
+	<reg32 offset="0x8c17" name="RB_2D_DST_INFO" type="a6xx_2d_surf_info" usage="rp_blit"/>
+	<reg64 offset="0x8c18" name="RB_2D_DST" type="waddress" align="64" usage="rp_blit"/>
+	<reg32 offset="0x8c1a" name="RB_2D_DST_PITCH" low="0" high="15" shr="6" type="uint" usage="rp_blit"/>
+	<!-- this is a guess but seems likely (for NV12/IYUV): -->
+	<reg64 offset="0x8c1b" name="RB_2D_DST_PLANE1" type="waddress" align="64" usage="rp_blit"/>
+	<reg32 offset="0x8c1d" name="RB_2D_DST_PLANE_PITCH" low="0" high="15" shr="6" type="uint" usage="rp_blit"/>
+	<reg64 offset="0x8c1e" name="RB_2D_DST_PLANE2" type="waddress" align="64" usage="rp_blit"/>
+
+	<reg64 offset="0x8c20" name="RB_2D_DST_FLAGS" type="waddress" align="64" usage="rp_blit"/>
+	<reg32 offset="0x8c22" name="RB_2D_DST_FLAGS_PITCH" low="0" high="7" shr="6" type="uint" usage="rp_blit"/>
+	<!-- this is a guess but seems likely (for NV12 with UBWC): -->
+	<reg64 offset="0x8c23" name="RB_2D_DST_FLAGS_PLANE" type="waddress" align="64" usage="rp_blit"/>
+	<reg32 offset="0x8c25" name="RB_2D_DST_FLAGS_PLANE_PITCH" low="0" high="7" shr="6" type="uint" usage="rp_blit"/>
+
+	<!-- TODO: 0x8c26-0x8c33 are all full 32-bit registers -->
+	<!-- unlike a5xx, these are per channel values rather than packed -->
+	<reg32 offset="0x8c2c" name="RB_2D_SRC_SOLID_C0" usage="rp_blit"/>
+	<reg32 offset="0x8c2d" name="RB_2D_SRC_SOLID_C1" usage="rp_blit"/>
+	<reg32 offset="0x8c2e" name="RB_2D_SRC_SOLID_C2" usage="rp_blit"/>
+	<reg32 offset="0x8c2f" name="RB_2D_SRC_SOLID_C3" usage="rp_blit"/>
+	<!-- 0x8c34-0x8dff invalid -->
+
+	<!-- always 0x1 ? either doesn't exist for a650 or write-only: -->
+	<reg32 offset="0x8e01" name="RB_UNKNOWN_8E01" usage="cmd"/>
+	<!-- 0x8e00-0x8e03 invalid -->
+	<reg32 offset="0x8e04" name="RB_DBG_ECO_CNTL" usage="cmd"/> <!-- TODO: valid mask 0xfffffeff -->
+	<reg32 offset="0x8e05" name="RB_ADDR_MODE_CNTL" pos="0" type="a5xx_address_mode"/>
+	<!-- 0x02080000 in GMEM, zero otherwise?  -->
+	<reg32 offset="0x8e06" name="RB_UNKNOWN_8E06" variants="A7XX-" usage="cmd"/>
+
+	<reg32 offset="0x8e07" name="RB_CCU_CNTL" usage="cmd" variants="A6XX">
+		<bitfield name="GMEM_FAST_CLEAR_DISABLE" pos="0" type="boolean"/>
+		<!-- concurrent resolves are apparently a 2-bit enum on a650+ -->
+		<bitfield name="CONCURRENT_RESOLVE" pos="2" type="boolean"/>
+		<bitfield name="DEPTH_OFFSET_HI" pos="7" type="hex"/>
+		<bitfield name="COLOR_OFFSET_HI" pos="9" type="hex"/>
+		<bitfield name="DEPTH_CACHE_SIZE" low="10" high="11" type="a6xx_ccu_cache_size"/>
+		<!-- GMEM offset of CCU depth cache -->
+		<bitfield name="DEPTH_OFFSET" low="12" high="20" shr="12" type="hex"/>
+		<bitfield name="COLOR_CACHE_SIZE" low="21" high="22" type="a6xx_ccu_cache_size"/>
+		<!-- GMEM offset of CCU color cache
+			for GMEM rendering, we set it to GMEM size minus the minimum
+			CCU color cache size. CCU color cache will be needed in some
+			resolve cases, and in those cases we need to reserve the end
+			of GMEM for color cache.
+		-->
+		<bitfield name="COLOR_OFFSET" low="23" high="31" shr="12" type="hex"/>
+		<!--TODO: valid mask 0xfffffc1f -->
+	</reg32>
+	<reg32 offset="0x8e07" name="RB_CCU_CNTL" usage="cmd" variants="A7XX-">
+		<bitfield name="GMEM_FAST_CLEAR_DISABLE" pos="0" type="boolean"/>
+		<bitfield name="CONCURRENT_RESOLVE" pos="2" type="boolean"/>
+		<!-- rest of the bits were moved to RB_CCU_CNTL2 -->
+	</reg32>
+	<reg32 offset="0x8e08" name="RB_NC_MODE_CNTL">
+		<bitfield name="MODE" pos="0" type="boolean"/>
+		<bitfield name="LOWER_BIT" low="1" high="2" type="uint"/>
+		<bitfield name="MIN_ACCESS_LENGTH" pos="3" type="boolean"/> <!-- true=64b false=32b -->
+		<bitfield name="AMSBC" pos="4" type="boolean"/>
+		<bitfield name="UPPER_BIT" pos="10" type="uint"/>
+		<bitfield name="RGB565_PREDICATOR" pos="11" type="boolean"/>
+		<bitfield name="UNK12" low="12" high="13"/>
+	</reg32>
+	<reg32 offset="0x8e09" name="RB_UNKNOWN_8E09" variants="A7XX-" usage="cmd"/>
+	<!-- 0x8e09-0x8e0f invalid -->
+	<array offset="0x8e10" name="RB_PERFCTR_RB_SEL" stride="1" length="8"/>
+	<array offset="0x8e18" name="RB_PERFCTR_CCU_SEL" stride="1" length="5"/>
+	<!-- 0x8e1d-0x8e1f invalid -->
+	<!-- 0x8e20-0x8e25 more perfcntr sel? -->
+	<!-- 0x8e26-0x8e27 invalid -->
+	<reg32 offset="0x8e28" name="RB_CMP_DBG_ECO_CNTL"/>
+	<!-- 0x8e29-0x8e2b invalid -->
+	<array offset="0x8e2c" name="RB_PERFCTR_CMP_SEL" stride="1" length="4"/>
+	<array offset="0x8e30" name="RB_PERFCTR_UFC_SEL" stride="1" length="6" variants="A7XX-"/>
+	<reg32 offset="0x8e3b" name="RB_RB_SUB_BLOCK_SEL_CNTL_HOST"/>
+	<reg32 offset="0x8e3d" name="RB_RB_SUB_BLOCK_SEL_CNTL_CD"/>
+	<!-- 0x8e3e-0x8e4f invalid -->
+	<!-- GMEM save/restore for preemption: -->
+	<reg32 offset="0x8e50" name="RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE" pos="0" type="boolean"/>
+	<!-- address for GMEM save/restore? -->
+	<reg32 offset="0x8e51" name="RB_UNKNOWN_8E51" type="waddress" align="1"/>
+	<!-- 0x8e53-0x8e7f invalid -->
+	<reg32 offset="0x8e79" name="RB_UNKNOWN_8E79" variants="A7XX-" usage="cmd"/>
+	<!-- 0x8e80-0x8e83 are valid -->
+	<!-- 0x8e84-0x90ff invalid -->
+
+	<!-- 0x9000-0x90ff invalid -->
+
+	<reg32 offset="0x9100" name="VPC_GS_PARAM" variants="A6XX" usage="rp_blit">
+		<bitfield name="LINELENGTHLOC" low="0" high="7" type="uint"/>
+	</reg32>
+
+	<bitset name="a6xx_vpc_xs_clip_cntl" inline="yes">
+		<bitfield name="CLIP_MASK" low="0" high="7" type="uint"/>
+		<!-- there can be up to 8 total clip/cull distance outputs,
+		     but apparenly VPC can only deal with vec4, so when there are
+		     more than 4 outputs a second location needs to be programmed
+		-->
+		<bitfield name="CLIP_DIST_03_LOC" low="8" high="15" type="uint"/>
+		<bitfield name="CLIP_DIST_47_LOC" low="16" high="23" type="uint"/>
+	</bitset>
+	<reg32 offset="0x9101" name="VPC_VS_CLIP_CNTL" type="a6xx_vpc_xs_clip_cntl" usage="rp_blit"/>
+	<reg32 offset="0x9102" name="VPC_GS_CLIP_CNTL" type="a6xx_vpc_xs_clip_cntl" usage="rp_blit"/>
+	<reg32 offset="0x9103" name="VPC_DS_CLIP_CNTL" type="a6xx_vpc_xs_clip_cntl" usage="rp_blit"/>
+
+	<reg32 offset="0x9311" name="VPC_VS_CLIP_CNTL_V2" type="a6xx_vpc_xs_clip_cntl" usage="rp_blit"/>
+	<reg32 offset="0x9312" name="VPC_GS_CLIP_CNTL_V2" type="a6xx_vpc_xs_clip_cntl" usage="rp_blit"/>
+	<reg32 offset="0x9313" name="VPC_DS_CLIP_CNTL_V2" type="a6xx_vpc_xs_clip_cntl" usage="rp_blit"/>
+
+	<bitset name="a6xx_vpc_xs_layer_cntl" inline="yes">
+		<bitfield name="LAYERLOC" low="0" high="7" type="uint"/>
+		<bitfield name="VIEWLOC" low="8" high="15" type="uint"/>
+		<bitfield name="SHADINGRATELOC" low="16" high="23" type="uint" variants="A7XX-"/>
+	</bitset>
+
+	<reg32 offset="0x9104" name="VPC_VS_LAYER_CNTL" type="a6xx_vpc_xs_layer_cntl" usage="rp_blit"/>
+	<reg32 offset="0x9105" name="VPC_GS_LAYER_CNTL" type="a6xx_vpc_xs_layer_cntl" usage="rp_blit"/>
+	<reg32 offset="0x9106" name="VPC_DS_LAYER_CNTL" type="a6xx_vpc_xs_layer_cntl" usage="rp_blit"/>
+
+	<reg32 offset="0x9314" name="VPC_VS_LAYER_CNTL_V2" type="a6xx_vpc_xs_layer_cntl" usage="rp_blit"/>
+	<reg32 offset="0x9315" name="VPC_GS_LAYER_CNTL_V2" type="a6xx_vpc_xs_layer_cntl" usage="rp_blit"/>
+	<reg32 offset="0x9316" name="VPC_DS_LAYER_CNTL_V2" type="a6xx_vpc_xs_layer_cntl" usage="rp_blit"/>
+
+	<reg32 offset="0x9107" name="VPC_UNKNOWN_9107" variants="A6XX" usage="rp_blit">
+		<!-- this mirrors PC_RASTER_CNTL::DISCARD, although it seems it's unused -->
+		<bitfield name="RASTER_DISCARD" pos="0" type="boolean"/>
+		<bitfield name="UNK2" pos="2" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x9108" name="VPC_POLYGON_MODE" usage="rp_blit">
+		<bitfield name="MODE" low="0" high="1" type="a6xx_polygon_mode"/>
+	</reg32>
+
+	<bitset name="a6xx_primitive_cntl_0" inline="yes">
+		<bitfield name="PRIMITIVE_RESTART" pos="0" type="boolean"/>
+		<bitfield name="PROVOKING_VTX_LAST" pos="1" type="boolean"/>
+		<bitfield name="D3D_VERTEX_ORDERING" pos="2" type="boolean">
+			<doc>
+				Swaps TESS_CW_TRIS/TESS_CCW_TRIS, and also makes
+				triangle fans and triangle strips use the D3D
+				order instead of the OpenGL order.
+			</doc>
+		</bitfield>
+		<bitfield name="UNK3" pos="3" type="boolean"/>
+	</bitset>
+
+	<bitset name="a6xx_primitive_cntl_5" inline="yes">
+		<doc>
+		  geometry shader
+		</doc>
+		<!-- TODO: first 16 bits are valid so something is wrong or missing here -->
+		<bitfield name="GS_VERTICES_OUT" low="0" high="7" type="uint"/>
+		<bitfield name="GS_INVOCATIONS" low="10" high="14" type="uint"/>
+		<bitfield name="LINELENGTHEN" pos="15" type="boolean"/>
+		<bitfield name="GS_OUTPUT" low="16" high="17" type="a6xx_tess_output"/>
+		<bitfield name="UNK18" pos="18"/>
+	</bitset>
+
+	<bitset name="a6xx_multiview_cntl" inline="yes">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+		<bitfield name="DISABLEMULTIPOS" pos="1" type="boolean">
+			<doc>
+				Multi-position output lets the last geometry
+				stage shader write multiple copies of
+				gl_Position. If disabled then the VS is run once
+				for each view, and ViewID is passed as a
+				register to the VS.
+			</doc>
+		</bitfield>
+		<bitfield name="VIEWS" low="2" high="6" type="uint"/>
+	</bitset>
+
+	<reg32 offset="0x9109" name="VPC_PRIMITIVE_CNTL_0" type="a6xx_primitive_cntl_0" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0x910a" name="VPC_PRIMITIVE_CNTL_5" type="a6xx_primitive_cntl_5" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0x910b" name="VPC_MULTIVIEW_MASK" type="hex" low="0" high="15" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0x910c" name="VPC_MULTIVIEW_CNTL" type="a6xx_multiview_cntl" variants="A7XX-" usage="rp_blit"/>
+
+	<enum name="a6xx_varying_interp_mode">
+		<value value="0" name="INTERP_SMOOTH"/>
+		<value value="1" name="INTERP_FLAT"/>
+		<value value="2" name="INTERP_ZERO"/>
+		<value value="3" name="INTERP_ONE"/>
+	</enum>
+
+	<enum name="a6xx_varying_ps_repl_mode">
+		<value value="0" name="PS_REPL_NONE"/>
+		<value value="1" name="PS_REPL_S"/>
+		<value value="2" name="PS_REPL_T"/>
+		<value value="3" name="PS_REPL_ONE_MINUS_T"/>
+	</enum>
+
+	<!-- 0x9109-0x91ff invalid -->
+	<array offset="0x9200" name="VPC_VARYING_INTERP" stride="1" length="8" usage="rp_blit">
+		<doc>Packed array of a6xx_varying_interp_mode</doc>
+		<reg32 offset="0x0" name="MODE"/>
+	</array>
+	<array offset="0x9208" name="VPC_VARYING_PS_REPL" stride="1" length="8" usage="rp_blit">
+		<doc>Packed array of a6xx_varying_ps_repl_mode</doc>
+		<reg32 offset="0x0" name="MODE"/>
+	</array>
+
+	<!-- always 0x0 -->
+	<reg32 offset="0x9210" name="VPC_UNKNOWN_9210" low="0" high="31" variants="A6XX" usage="cmd"/>
+	<reg32 offset="0x9211" name="VPC_UNKNOWN_9211" low="0" high="31" variants="A6XX" usage="cmd"/>
+
+	<array offset="0x9212" name="VPC_VAR" stride="1" length="4" usage="rp_blit">
+		<!-- one bit per varying component: -->
+		<reg32 offset="0" name="DISABLE"/>
+	</array>
+
+	<reg32 offset="0x9216" name="VPC_SO_CNTL" usage="rp_blit">
+		<!--
+			Choose which DWORD to write to. There is an array of
+			(4 * 64) DWORD's, dumped in the devcoredump at
+			HLSQ_INST_RAM dword 0x400. Each DWORD corresponds to a
+			(VPC location, stream) pair like so:
+
+			location 0, stream 0
+			location 2, stream 0
+			...
+			location 126, stream 0
+			location 0, stream 1
+			location 2, stream 1
+			...
+			location 126, stream 1
+			location 0, stream 2
+			...
+
+			When EmitStreamVertex(N) happens, the HW goes to DWORD
+			64 * N and then "executes" the next 64 DWORD's.
+
+			This field is auto-incremented when VPC_SO_PROG is
+			written to.
+		-->
+		<bitfield name="ADDR" low="0" high="7" type="hex"/>
+		<!-- clear all A_EN and B_EN bits for all DWORD's -->
+		<bitfield name="RESET" pos="16" type="boolean"/>
+	</reg32>
+	<!-- special register, write multiple times to load SO program (not readable) -->
+	<reg32 offset="0x9217" name="VPC_SO_PROG" usage="rp_blit">
+		<bitfield name="A_BUF" low="0" high="1" type="uint"/>
+		<bitfield name="A_OFF" low="2" high="10" shr="2" type="uint"/>
+		<bitfield name="A_EN" pos="11" type="boolean"/>
+		<bitfield name="B_BUF" low="12" high="13" type="uint"/>
+		<bitfield name="B_OFF" low="14" high="22" shr="2" type="uint"/>
+		<bitfield name="B_EN" pos="23" type="boolean"/>
+	</reg32>
+
+	<reg64 offset="0x9218" name="VPC_SO_STREAM_COUNTS" type="waddress" align="32" usage="cmd"/>
+
+	<array offset="0x921a" name="VPC_SO" stride="7" length="4" usage="cmd">
+		<reg64 offset="0" name="BUFFER_BASE" type="waddress" align="32"/>
+		<reg32 offset="2" name="BUFFER_SIZE" low="2" high="31" shr="2"/>
+		<reg32 offset="3" name="BUFFER_STRIDE" low="0" high="9" shr="2"/>
+		<reg32 offset="4" name="BUFFER_OFFSET" low="2" high="31" shr="2"/>
+		<reg64 offset="5" name="FLUSH_BASE" type="waddress" align="32"/>
+	</array>
+
+	<reg32 offset="0x9236" name="VPC_POINT_COORD_INVERT" usage="cmd">
+		<bitfield name="INVERT" pos="0" type="boolean"/>
+	</reg32>
+	<!-- 0x9237-0x92ff invalid -->
+	<!-- always 0x0 ? -->
+	<reg32 offset="0x9300" name="VPC_UNKNOWN_9300" low="0" high="2" usage="cmd"/>
+
+	<bitset name="a6xx_vpc_xs_pack" inline="yes">
+		<doc>
+			num of varyings plus four for gl_Position (plus one if gl_PointSize)
+			plus # of transform-feedback (streamout) varyings if using the
+			hw streamout (rather than stg instructions in shader)
+		</doc>
+		<bitfield name="STRIDE_IN_VPC" low="0" high="7" type="uint"/>
+		<bitfield name="POSITIONLOC" low="8" high="15" type="uint"/>
+		<bitfield name="PSIZELOC" low="16" high="23" type="uint"/>
+		<bitfield name="EXTRAPOS" low="24" high="27" type="uint">
+			<doc>
+				The number of extra copies of POSITION, i.e.
+				number of views minus one when multi-position
+				output is enabled, otherwise 0.
+			</doc>
+		</bitfield>
+	</bitset>
+	<reg32 offset="0x9301" name="VPC_VS_PACK" type="a6xx_vpc_xs_pack" usage="rp_blit"/>
+	<reg32 offset="0x9302" name="VPC_GS_PACK" type="a6xx_vpc_xs_pack" usage="rp_blit"/>
+	<reg32 offset="0x9303" name="VPC_DS_PACK" type="a6xx_vpc_xs_pack" usage="rp_blit"/>
+
+	<reg32 offset="0x9304" name="VPC_CNTL_0" usage="rp_blit">
+		<bitfield name="NUMNONPOSVAR" low="0" high="7" type="uint"/>
+		<!-- for fixed-function (i.e. no GS) gl_PrimitiveID in FS -->
+		<bitfield name="PRIMIDLOC" low="8" high="15" type="uint"/>
+		<bitfield name="VARYING" pos="16" type="boolean"/>
+		<bitfield name="VIEWIDLOC" low="24" high="31" type="uint">
+			<doc>
+				This VPC location will be overwritten with
+				ViewID when multiview is enabled. It's used when
+				fragment shaders read ViewID. It's only
+				strictly required for multi-position output,
+				where the same VS invocation is used for all the
+				views at once, but it can be used when multi-pos
+				output is disabled too, to avoid having to pass
+				ViewID through the VS.
+			</doc>
+		</bitfield>
+	</reg32>
+
+	<reg32 offset="0x9305" name="VPC_SO_STREAM_CNTL" usage="rp_blit">
+		<!--
+		It's offset by 1, and 0 means "disabled"
+		-->
+		<bitfield name="BUF0_STREAM" low="0" high="2" type="uint"/>
+		<bitfield name="BUF1_STREAM" low="3" high="5" type="uint"/>
+		<bitfield name="BUF2_STREAM" low="6" high="8" type="uint"/>
+		<bitfield name="BUF3_STREAM" low="9" high="11" type="uint"/>
+		<bitfield name="STREAM_ENABLE" low="15" high="18" type="hex"/>
+	</reg32>
+	<reg32 offset="0x9306" name="VPC_SO_DISABLE" usage="rp_blit">
+		<bitfield name="DISABLE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x9307" name="VPC_POLYGON_MODE2" variants="A7XX-" usage="rp_blit">
+		<bitfield name="MODE" low="0" high="1" type="a6xx_polygon_mode"/>
+	</reg32>
+	<reg32 offset="0x9308" name="VPC_ATTR_BUF_SIZE_GMEM" variants="A7XX-" usage="rp_blit">
+		<bitfield name="SIZE_GMEM" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="0x9309" name="VPC_ATTR_BUF_BASE_GMEM" variants="A7XX-" usage="rp_blit">
+		<bitfield name="BASE_GMEM" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="0x9b09" name="PC_ATTR_BUF_SIZE_GMEM" variants="A7XX-" usage="rp_blit">
+		<bitfield name="SIZE_GMEM" low="0" high="31"/>
+	</reg32>
+
+	<!-- 0x9307-0x95ff invalid -->
+
+	<!-- TODO: 0x9600-0x97ff range -->
+	<reg32 offset="0x9600" name="VPC_DBG_ECO_CNTL" usage="cmd"/> <!-- always 0x0 ? TODO: 0x1fbf37ff valid mask -->
+	<reg32 offset="0x9601" name="VPC_ADDR_MODE_CNTL" pos="0" type="a5xx_address_mode" usage="cmd"/>
+	<reg32 offset="0x9602" name="VPC_UNKNOWN_9602" pos="0" usage="cmd"/> <!-- always 0x0 ? -->
+	<reg32 offset="0x9603" name="VPC_UNKNOWN_9603" low="0" high="26"/>
+	<array offset="0x9604" name="VPC_PERFCTR_VPC_SEL" stride="1" length="6" variants="A6XX"/>
+	<array offset="0x960b" name="VPC_PERFCTR_VPC_SEL" stride="1" length="12" variants="A7XX-"/>
+	<!-- 0x960a-0x9623 invalid -->
+	<!-- TODO: regs from 0x9624-0x963a -->
+	<!-- 0x963b-0x97ff invalid -->
+
+	<reg32 offset="0x9800" name="PC_TESS_NUM_VERTEX" low="0" high="5" type="uint" usage="rp_blit"/>
+
+	<!-- always 0x0 ? -->
+	<reg32 offset="0x9801" name="PC_HS_INPUT_SIZE" usage="rp_blit">
+		<bitfield name="SIZE" low="0" high="10" type="uint"/>
+		<bitfield name="UNK13" pos="13"/>
+	</reg32>
+
+	<reg32 offset="0x9802" name="PC_TESS_CNTL" usage="rp_blit">
+		<bitfield name="SPACING" low="0" high="1" type="a6xx_tess_spacing"/>
+		<bitfield name="OUTPUT" low="2" high="3" type="a6xx_tess_output"/>
+	</reg32>
+
+	<reg32 offset="0x9803" name="PC_RESTART_INDEX" low="0" high="31" type="uint" usage="rp_blit"/>
+	<reg32 offset="0x9804" name="PC_MODE_CNTL" low="0" high="7" usage="rp_blit"/>
+
+	<reg32 offset="0x9805" name="PC_POWER_CNTL" low="0" high="2" usage="rp_blit"/>
+
+	<reg32 offset="0x9806" name="PC_PS_CNTL" usage="rp_blit">
+		<bitfield name="PRIMITIVEIDEN" pos="0" type="boolean"/>
+	</reg32>
+
+	<!-- New in a6xx gen3+ -->
+	<reg32 offset="0x9808" name="PC_SO_STREAM_CNTL" usage="rp_blit">
+		<bitfield name="STREAM_ENABLE" low="15" high="18" type="hex"/>
+	</reg32>
+
+	<reg32 offset="0x980a" name="PC_DGEN_SU_CONSERVATIVE_RAS_CNTL">
+		<bitfield name="CONSERVATIVERASEN" pos="0" type="boolean"/>
+	</reg32>
+	<!-- 0x980b-0x983f invalid -->
+
+	<!-- 0x9840 - 0x9842 are not readable -->
+	<reg32 offset="0x9840" name="PC_DRAW_CMD">
+		<bitfield name="STATE_ID" low="0" high="7"/>
+	</reg32>
+
+	<reg32 offset="0x9841" name="PC_DISPATCH_CMD">
+		<bitfield name="STATE_ID" low="0" high="7"/>
+	</reg32>
+
+	<reg32 offset="0x9842" name="PC_EVENT_CMD">
+		<!-- I think only the low bit is actually used? -->
+		<bitfield name="STATE_ID" low="16" high="23"/>
+		<bitfield name="EVENT" low="0" high="6" type="vgt_event_type"/>
+	</reg32>
+
+	<!--
+		0x9880 written in a lot of places by SQE, same value gets written
+		to control reg 0x12a.  Set by CP_SET_MARKER, so lets name it after
+		that
+	 -->
+	<reg32 offset="0x9880" name="PC_MARKER"/>
+
+	<!-- 0x9843-0x997f invalid -->
+
+	<reg32 offset="0x9981" name="PC_POLYGON_MODE" variants="A6XX" usage="rp_blit">
+		<bitfield name="MODE" low="0" high="1" type="a6xx_polygon_mode"/>
+	</reg32>
+	<reg32 offset="0x9809" name="PC_POLYGON_MODE" variants="A7XX-" usage="rp_blit">
+		<bitfield name="MODE" low="0" high="1" type="a6xx_polygon_mode"/>
+	</reg32>
+
+	<reg32 offset="0x9980" name="PC_RASTER_CNTL" variants="A6XX" usage="rp_blit">
+		<!-- which stream to send to GRAS -->
+		<bitfield name="STREAM" low="0" high="1" type="uint"/>
+		<!-- discard primitives before rasterization -->
+		<bitfield name="DISCARD" pos="2" type="boolean"/>
+	</reg32>
+	<!-- VPC_RASTER_CNTL -->
+	<reg32 offset="0x9107" name="PC_RASTER_CNTL" variants="A7XX-" usage="rp_blit">
+		<!-- which stream to send to GRAS -->
+		<bitfield name="STREAM" low="0" high="1" type="uint"/>
+		<!-- discard primitives before rasterization -->
+		<bitfield name="DISCARD" pos="2" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x9317" name="PC_RASTER_CNTL_V2" variants="A7XX-" usage="rp_blit">
+		<!-- which stream to send to GRAS -->
+		<bitfield name="STREAM" low="0" high="1" type="uint"/>
+		<!-- discard primitives before rasterization -->
+		<bitfield name="DISCARD" pos="2" type="boolean"/>
+	</reg32>
+
+	<!-- Both are a750+.
+	     Probably needed to correctly overlap execution of several draws.
+	-->
+	<reg32 offset="0x9885" name="PC_TESS_PARAM_SIZE" variants="A7XX-" usage="cmd"/>
+	<!-- Blob adds a bit more space {0x10, 0x20, 0x30, 0x40} bytes, but the meaning of
+	     this additional space is not known.
+	-->
+	<reg32 offset="0x9886" name="PC_TESS_FACTOR_SIZE" variants="A7XX-" usage="cmd"/>
+
+	<!-- 0x9982-0x9aff invalid -->
+
+	<reg32 offset="0x9b00" name="PC_PRIMITIVE_CNTL_0" type="a6xx_primitive_cntl_0" usage="rp_blit"/>
+
+	<bitset name="a6xx_xs_out_cntl" inline="yes">
+		<doc>
+			num of varyings plus four for gl_Position (plus one if gl_PointSize)
+			plus # of transform-feedback (streamout) varyings if using the
+			hw streamout (rather than stg instructions in shader)
+		</doc>
+		<bitfield name="STRIDE_IN_VPC" low="0" high="7" type="uint"/>
+		<bitfield name="PSIZE" pos="8" type="boolean"/>
+		<bitfield name="LAYER" pos="9" type="boolean"/>
+		<bitfield name="VIEW" pos="10" type="boolean"/>
+		<!-- note: PC_VS_OUT_CNTL doesn't have the PRIMITIVE_ID bit -->
+		<bitfield name="PRIMITIVE_ID" pos="11" type="boolean"/>
+		<bitfield name="CLIP_MASK" low="16" high="23" type="uint"/>
+		<bitfield name="SHADINGRATE" pos="24" type="boolean" variants="A7XX-"/>
+	</bitset>
+
+	<reg32 offset="0x9b01" name="PC_VS_OUT_CNTL" type="a6xx_xs_out_cntl" usage="rp_blit"/>
+	<reg32 offset="0x9b02" name="PC_GS_OUT_CNTL" type="a6xx_xs_out_cntl" usage="rp_blit"/>
+	<!-- since HS can't output anything, only PRIMITIVE_ID is valid -->
+	<reg32 offset="0x9b03" name="PC_HS_OUT_CNTL" type="a6xx_xs_out_cntl" usage="rp_blit"/>
+	<reg32 offset="0x9b04" name="PC_DS_OUT_CNTL" type="a6xx_xs_out_cntl" usage="rp_blit"/>
+
+	<reg32 offset="0x9b05" name="PC_PRIMITIVE_CNTL_5" type="a6xx_primitive_cntl_5" usage="rp_blit"/>
+
+	<reg32 offset="0x9b06" name="PC_PRIMITIVE_CNTL_6" variants="A6XX" usage="rp_blit">
+		<doc>
+		  size in vec4s of per-primitive storage for gs. TODO: not actually in VPC
+		</doc>
+		<bitfield name="STRIDE_IN_VPC" low="0" high="10" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x9b07" name="PC_MULTIVIEW_CNTL" type="a6xx_multiview_cntl" usage="rp_blit"/>
+	<!-- mask of enabled views, doesn't exist on A630 -->
+	<reg32 offset="0x9b08" name="PC_MULTIVIEW_MASK" type="hex" low="0" high="15" usage="rp_blit"/>
+	<!-- 0x9b09-0x9bff invalid -->
+	<reg32 offset="0x9c00" name="PC_2D_EVENT_CMD">
+		<!-- special register (but note first 8 bits can be written/read) -->
+		<bitfield name="EVENT" low="0" high="6" type="vgt_event_type"/>
+		<bitfield name="STATE_ID" low="8" high="15"/>
+	</reg32>
+	<!-- 0x9c01-0x9dff invalid -->
+	<!-- TODO: 0x9e00-0xa000 range incomplete -->
+	<reg32 offset="0x9e00" name="PC_DBG_ECO_CNTL"/>
+	<reg32 offset="0x9e01" name="PC_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg64 offset="0x9e04" name="PC_DRAW_INDX_BASE"/>
+	<reg32 offset="0x9e06" name="PC_DRAW_FIRST_INDX" type="uint"/>
+	<reg32 offset="0x9e07" name="PC_DRAW_MAX_INDICES" type="uint"/>
+	<reg64 offset="0x9e08" name="PC_TESSFACTOR_ADDR" variants="A6XX" type="waddress" align="32" usage="cmd"/>
+	<reg64 offset="0x9810" name="PC_TESSFACTOR_ADDR" variants="A7XX-" type="waddress" align="32" usage="cmd"/>
+
+	<reg32 offset="0x9e0b" name="PC_DRAW_INITIATOR" type="vgt_draw_initiator_a4xx">
+		<doc>
+			Possibly not really "initiating" the draw but the layout is similar
+			to VGT_DRAW_INITIATOR on older gens
+		</doc>
+	</reg32>
+	<reg32 offset="0x9e0c" name="PC_DRAW_NUM_INSTANCES" type="uint"/>
+	<reg32 offset="0x9e0d" name="PC_DRAW_NUM_INDICES" type="uint"/>
+
+	<!-- These match the contents of CP_SET_BIN_DATA (not written directly) -->
+	<reg32 offset="0x9e11" name="PC_VSTREAM_CONTROL">
+		<bitfield name="UNK0" low="0" high="15"/>
+		<bitfield name="VSC_SIZE" low="16" high="21" type="uint"/>
+		<bitfield name="VSC_N" low="22" high="26" type="uint"/>
+	</reg32>
+	<reg64 offset="0x9e12" name="PC_BIN_PRIM_STRM" type="waddress" align="32"/>
+	<reg64 offset="0x9e14" name="PC_BIN_DRAW_STRM" type="waddress" align="32"/>
+
+	<reg32 offset="0x9e1c" name="PC_VISIBILITY_OVERRIDE">
+		<doc>Written by CP_SET_VISIBILITY_OVERRIDE handler</doc>
+		<bitfield name="OVERRIDE" pos="0" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x9e24" name="PC_UNKNOWN_9E24" variants="A7XX-" usage="cmd"/>
+
+	<array offset="0x9e34" name="PC_PERFCTR_PC_SEL" stride="1" length="8" variants="A6XX"/>
+	<array offset="0x9e42" name="PC_PERFCTR_PC_SEL" stride="1" length="16" variants="A7XX-"/>
+
+	<!-- always 0x0 -->
+	<reg32 offset="0x9e72" name="PC_UNKNOWN_9E72" usage="cmd"/>
+
+	<reg32 offset="0xa000" name="VFD_CONTROL_0" usage="rp_blit">
+		<bitfield name="FETCH_CNT" low="0" high="5" type="uint"/>
+		<bitfield name="DECODE_CNT" low="8" high="13" type="uint"/>
+	</reg32>
+	<reg32 offset="0xa001" name="VFD_CONTROL_1" usage="rp_blit">
+		<bitfield name="REGID4VTX" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="REGID4INST" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="REGID4PRIMID" low="16" high="23" type="a3xx_regid"/>
+		<!-- only used for VS in non-multi-position-output case -->
+		<bitfield name="REGID4VIEWID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xa002" name="VFD_CONTROL_2" usage="rp_blit">
+		<bitfield name="REGID_HSRELPATCHID" low="0" high="7" type="a3xx_regid">
+			<doc>
+				This is the ID of the current patch within the
+				subdraw, used to calculate the offset of the
+				patch within the HS->DS buffers. When a draw is
+				split into multiple subdraws then this differs
+				from gl_PrimitiveID on the second, third, etc.
+				subdraws.
+			</doc>
+		</bitfield>
+		<bitfield name="REGID_INVOCATIONID" low="8" high="15" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xa003" name="VFD_CONTROL_3" usage="rp_blit">
+		<bitfield name="REGID_DSPRIMID" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="REGID_DSRELPATCHID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="REGID_TESSX" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="REGID_TESSY" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xa004" name="VFD_CONTROL_4" usage="rp_blit">
+		<bitfield name="UNK0" low="0" high="7" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xa005" name="VFD_CONTROL_5" usage="rp_blit">
+		<bitfield name="REGID_GSHEADER" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="UNK8" low="8" high="15" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xa006" name="VFD_CONTROL_6" usage="rp_blit">
+		<!--
+			True if gl_PrimitiveID is read via the FS
+		-->
+		<bitfield name="PRIMID4PSEN" pos="0" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0xa007" name="VFD_MODE_CNTL" usage="cmd">
+		<bitfield name="RENDER_MODE" low="0" high="2" type="a6xx_render_mode"/>
+	</reg32>
+
+	<reg32 offset="0xa008" name="VFD_MULTIVIEW_CNTL" type="a6xx_multiview_cntl" usage="rp_blit"/>
+	<reg32 offset="0xa009" name="VFD_ADD_OFFSET" usage="cmd">
+		<!-- add VFD_INDEX_OFFSET to REGID4VTX -->
+		<bitfield name="VERTEX" pos="0" type="boolean"/>
+		<!-- add VFD_INSTANCE_START_OFFSET to REGID4INST -->
+		<bitfield name="INSTANCE" pos="1" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0xa00e" name="VFD_INDEX_OFFSET" usage="rp_blit"/>
+	<reg32 offset="0xa00f" name="VFD_INSTANCE_START_OFFSET" usage="rp_blit"/>
+	<array offset="0xa010" name="VFD_FETCH" stride="4" length="32" usage="rp_blit">
+		<reg64 offset="0x0" name="BASE" type="address" align="1"/>
+		<reg32 offset="0x2" name="SIZE" type="uint"/>
+		<reg32 offset="0x3" name="STRIDE" low="0" high="11" type="uint"/>
+	</array>
+	<array offset="0xa090" name="VFD_DECODE" stride="2" length="32" usage="rp_blit">
+		<reg32 offset="0x0" name="INSTR">
+			<!-- IDX and byte OFFSET into VFD_FETCH -->
+			<bitfield name="IDX" low="0" high="4" type="uint"/>
+			<bitfield name="OFFSET" low="5" high="16"/>
+			<bitfield name="INSTANCED" pos="17" type="boolean"/>
+			<bitfield name="FORMAT" low="20" high="27" type="a6xx_format"/>
+			<bitfield name="SWAP" low="28" high="29" type="a3xx_color_swap"/>
+			<bitfield name="UNK30" pos="30" type="boolean"/>
+			<bitfield name="FLOAT" pos="31" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x1" name="STEP_RATE" type="uint"/>
+	</array>
+	<array offset="0xa0d0" name="VFD_DEST_CNTL" stride="1" length="32" usage="rp_blit">
+		<reg32 offset="0x0" name="INSTR">
+			<bitfield name="WRITEMASK" low="0" high="3" type="hex"/>
+			<bitfield name="REGID" low="4" high="11" type="a3xx_regid"/>
+		</reg32>
+	</array>
+
+	<reg32 offset="0xa0f8" name="VFD_POWER_CNTL" low="0" high="2" usage="rp_blit"/>
+
+	<reg32 offset="0xa600" name="VFD_UNKNOWN_A600" variants="A7XX-" usage="cmd"/>
+
+	<reg32 offset="0xa601" name="VFD_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<array offset="0xa610" name="VFD_PERFCTR_VFD_SEL" stride="1" length="8" variants="A6XX"/>
+	<array offset="0xa610" name="VFD_PERFCTR_VFD_SEL" stride="1" length="16" variants="A7XX-"/>
+
+	<!--
+	Note: this seems to always be paired with another bit in another
+	block.
+	-->
+	<enum name="a6xx_threadsize">
+		<value value="0" name="THREAD64"/>
+		<value value="1" name="THREAD128"/>
+	</enum>
+
+	<bitset name="a6xx_sp_xs_ctrl_reg0" inline="yes">
+		<!-- if set to SINGLE, only use 1 concurrent wave on each SP -->
+		<bitfield name="THREADMODE" pos="0" type="a3xx_threadmode"/>
+		<!--
+		When b31 set we just see FULLREGFOOTPRINT set.  The pattern of
+		used registers is a bit odd too:
+			- used (half): 0-15 68-179 (cnt=128, max=179)
+			- used (full): 0-33 50-69 71 73 75 77 79 81 83 85 87 89-105 107 109 111 113 115 117 119 121 123 125 127>
+		whereas we usually see a (mostly) contiguous range of regs used.  But if
+		I merge the full and half ranges (ie. rN counts as hr(N*2) and hr(N*2+1)),
+		then:
+			- used (merged): 0-191 (cnt=192, max=191)
+		So I think if b31 is set, then the half precision registers overlap
+		the full precision registers.  (Which seems like a pretty sensible
+		feature, actually I'm not sure when you *wouldn't* want to use that,
+		since it gives register allocation more flexibility)
+		 -->
+		<bitfield name="HALFREGFOOTPRINT" low="1" high="6" type="uint"/>
+		<bitfield name="FULLREGFOOTPRINT" low="7" high="12" type="uint"/>
+		<!-- could it be a low bit of branchstack? -->
+		<bitfield name="UNK13" pos="13" type="boolean"/>
+		<!-- seems to be nesting level for flow control:.. -->
+		<bitfield name="BRANCHSTACK" low="14" high="19" type="uint"/>
+	</bitset>
+
+	<bitset name="a6xx_sp_xs_config" inline="yes">
+		<!--
+		Each of these are set if the given resource type is used
+		with the Vulkan/bindless binding model.
+		-->
+		<bitfield name="BINDLESS_TEX" pos="0" type="boolean"/>
+		<bitfield name="BINDLESS_SAMP" pos="1" type="boolean"/>
+		<bitfield name="BINDLESS_IBO" pos="2" type="boolean"/>
+		<bitfield name="BINDLESS_UBO" pos="3" type="boolean"/>
+
+		<bitfield name="ENABLED" pos="8" type="boolean"/>
+		<!--
+		number of textures and samplers.. these might be swapped, with GL I
+		always see the same value for both.
+		 -->
+		<bitfield name="NTEX" low="9" high="16" type="uint"/>
+		<bitfield name="NSAMP" low="17" high="21" type="uint"/>
+		<bitfield name="NIBO" low="22" high="28" type="uint"/>
+	</bitset>
+
+	<bitset name="a6xx_sp_xs_prim_cntl" inline="yes">
+		<!-- # of VS outputs including pos/psize -->
+		<bitfield name="OUT" low="0" high="5" type="uint"/>
+		<!-- FLAGS_REGID only for GS -->
+		<bitfield name="FLAGS_REGID" low="6" high="13" type="a3xx_regid"/>
+	</bitset>
+
+	<reg32 offset="0xa800" name="SP_VS_CTRL_REG0" type="a6xx_sp_xs_ctrl_reg0" usage="rp_blit">
+		<!--
+		This field actually controls all geometry stages. TCS, TES, and
+		GS must have the same mergedregs setting as VS.
+		-->
+		<bitfield name="MERGEDREGS" pos="20" type="boolean"/>
+		<!--
+		Creates a separate preamble-only thread?
+
+		Early preamble has the following limitations:
+		- Only shared, a1, and consts regs could be used
+		  (accessing other regs would result in GPU fault);
+		- No cat5/cat6, only stc/ldc variants are working;
+		- Values writen to shared regs are not accessible by the rest
+		  of the shader;
+		- Instructions before shps are also considered to be a part of
+		  early preamble;
+
+		Note, for all shaders from d3d11 games blob produced preambles
+		compatible with early preamble mode.
+		-->
+		<bitfield name="EARLYPREAMBLE" pos="21" type="boolean"/>
+	</reg32>
+	<!-- bitmask of true/false conditions for VS brac.N instructions,
+	     bit N corresponds to brac.N -->
+	<reg32 offset="0xa801" name="SP_VS_BRANCH_COND" type="hex"/>
+	<!-- # of VS outputs including pos/psize -->
+	<reg32 offset="0xa802" name="SP_VS_PRIMITIVE_CNTL" type="a6xx_sp_xs_prim_cntl" usage="rp_blit"/>
+	<array offset="0xa803" name="SP_VS_OUT" stride="1" length="16" usage="rp_blit">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="A_REGID" low="0" high="7" type="a3xx_regid"/>
+			<bitfield name="A_COMPMASK" low="8" high="11" type="hex"/>
+			<bitfield name="B_REGID" low="16" high="23" type="a3xx_regid"/>
+			<bitfield name="B_COMPMASK" low="24" high="27" type="hex"/>
+		</reg32>
+	</array>
+	<!--
+	Starting with a5xx, position/psize outputs from shader end up in the
+	SP_VS_OUT map, with highest OUTLOCn position.  (Generally they are
+	the last entries too, except when gl_PointCoord is used, blob inserts
+	an extra varying after, but with a lower OUTLOC position.  If present,
+	psize is last, preceded by position.
+	 -->
+	<array offset="0xa813" name="SP_VS_VPC_DST" stride="1" length="8" usage="rp_blit">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="OUTLOC0" low="0" high="7" type="uint"/>
+			<bitfield name="OUTLOC1" low="8" high="15" type="uint"/>
+			<bitfield name="OUTLOC2" low="16" high="23" type="uint"/>
+			<bitfield name="OUTLOC3" low="24" high="31" type="uint"/>
+		</reg32>
+	</array>
+
+	<bitset name="a6xx_sp_xs_pvt_mem_param" inline="yes">
+		<bitfield name="MEMSIZEPERITEM" low="0" high="7" shr="9">
+			<doc>The size of memory that ldp/stp can address.</doc>
+		</bitfield>
+		<bitfield name="HWSTACKSIZEPERTHREAD" low="24" high="31">
+                        <doc>
+				Seems to be the same as a3xx. The maximum stack
+				size in units of 4 calls, so a call depth of 7
+				would result in a value of 2.
+				TODO: What's the actual size per call, i.e. the
+				size of the PC? a3xx docs say it's 16 bits
+				there, but the length register now takes 28 bits
+				so it's probably been bumped to 32 bits.
+                        </doc>
+		</bitfield>
+	</bitset>
+
+	<bitset name="a6xx_sp_xs_pvt_mem_size" inline="yes">
+		<bitfield name="TOTALPVTMEMSIZE" low="0" high="17" shr="12"/>
+		<bitfield name="PERWAVEMEMLAYOUT" pos="31" type="boolean">
+			<doc>
+				There are four indices used to compute the
+				private memory location for an access:
+
+				- stp/ldp offset
+				- fiber id
+				- wavefront id (a swizzled version of what "getwid" returns)
+				- SP ID (the same as what "getspid" returns)
+
+				The stride for the SP ID is always set by
+				TOTALPVTMEMSIZE. In the per-wave layout, the
+				indices are used in this order:
+
+				- offset % 4 (offset within dword)
+				- fiber id
+				- offset / 4
+				- wavefront id
+				- SP ID
+
+				and the stride for the wavefront ID is
+				MEMSIZEPERITEM, multiplied by 128 (fibers per
+				wavefront). In the per-fiber layout, the indices
+				are used in this order:
+
+				- offset
+				- fiber id % 4
+				- wavefront id
+				- fiber id / 4
+				- SP ID
+
+				and the stride for the fiber id/wavefront id
+				combo is MEMSIZEPERITEM.
+
+				Note: Accesses of more than 1 dword do not work
+				with per-fiber layout. The blob will fall back
+				to per-wave instead.
+			</doc>
+		</bitfield>
+	</bitset>
+
+	<bitset name="a6xx_sp_xs_pvt_mem_hw_stack_offset" inline="yes">
+		<doc>
+			This seems to be be the equivalent of HWSTACKOFFSET in
+			a3xx. The ldp/stp offset formula above isn't affected by
+			HWSTACKSIZEPERTHREAD at all, so the HW return address
+			stack seems to be after all the normal per-SP private
+			memory.
+		</doc>
+		<bitfield name="OFFSET" low="0" high="18" shr="11"/>
+	</bitset>
+
+	<reg32 offset="0xa81b" name="SP_VS_OBJ_FIRST_EXEC_OFFSET" type="uint" usage="rp_blit"/>
+	<reg64 offset="0xa81c" name="SP_VS_OBJ_START" type="address" align="32" usage="rp_blit"/>
+	<reg32 offset="0xa81e" name="SP_VS_PVT_MEM_PARAM" type="a6xx_sp_xs_pvt_mem_param" usage="rp_blit"/>
+	<reg64 offset="0xa81f" name="SP_VS_PVT_MEM_ADDR" type="waddress" align="32" usage="rp_blit"/>
+	<reg32 offset="0xa821" name="SP_VS_PVT_MEM_SIZE" type="a6xx_sp_xs_pvt_mem_size" usage="rp_blit"/>
+	<reg32 offset="0xa822" name="SP_VS_TEX_COUNT" low="0" high="7" type="uint" usage="rp_blit"/>
+	<reg32 offset="0xa823" name="SP_VS_CONFIG" type="a6xx_sp_xs_config" usage="rp_blit"/>
+	<reg32 offset="0xa824" name="SP_VS_INSTRLEN" low="0" high="27" type="uint" usage="rp_blit"/>
+	<reg32 offset="0xa825" name="SP_VS_PVT_MEM_HW_STACK_OFFSET" type="a6xx_sp_xs_pvt_mem_hw_stack_offset" usage="rp_blit"/>
+	<reg32 offset="0xa82d" name="SP_VS_VGPR_CONFIG" variants="A7XX-" usage="cmd"/>
+
+	<reg32 offset="0xa830" name="SP_HS_CTRL_REG0" type="a6xx_sp_xs_ctrl_reg0" usage="rp_blit">
+		<!-- There is no mergedregs bit, that comes from the VS. -->
+		<bitfield name="EARLYPREAMBLE" pos="20" type="boolean"/>
+	</reg32>
+	<!--
+	Total size of local storage in dwords divided by the wave size.
+	The maximum value is 64. With the wave size being always 64 for HS,
+	the maximum size of local storage should be:
+	 64 (wavesize) * 64 (SP_HS_WAVE_INPUT_SIZE) * 4 = 16k
+	-->
+	<reg32 offset="0xa831" name="SP_HS_WAVE_INPUT_SIZE" low="0" high="7" type="uint" usage="rp_blit"/>
+	<reg32 offset="0xa832" name="SP_HS_BRANCH_COND" type="hex" usage="rp_blit"/>
+
+	<!-- TODO: exact same layout as 0xa81b-0xa825 -->
+	<reg32 offset="0xa833" name="SP_HS_OBJ_FIRST_EXEC_OFFSET" type="uint" usage="rp_blit"/>
+	<reg64 offset="0xa834" name="SP_HS_OBJ_START" type="address" align="32" usage="rp_blit"/>
+	<reg32 offset="0xa836" name="SP_HS_PVT_MEM_PARAM" type="a6xx_sp_xs_pvt_mem_param" usage="rp_blit"/>
+	<reg64 offset="0xa837" name="SP_HS_PVT_MEM_ADDR" type="waddress" align="32" usage="rp_blit"/>
+	<reg32 offset="0xa839" name="SP_HS_PVT_MEM_SIZE" type="a6xx_sp_xs_pvt_mem_size" usage="rp_blit"/>
+	<reg32 offset="0xa83a" name="SP_HS_TEX_COUNT" low="0" high="7" type="uint" usage="rp_blit"/>
+	<reg32 offset="0xa83b" name="SP_HS_CONFIG" type="a6xx_sp_xs_config" usage="rp_blit"/>
+	<reg32 offset="0xa83c" name="SP_HS_INSTRLEN" low="0" high="27" type="uint" usage="rp_blit"/>
+	<reg32 offset="0xa83d" name="SP_HS_PVT_MEM_HW_STACK_OFFSET" type="a6xx_sp_xs_pvt_mem_hw_stack_offset" usage="rp_blit"/>
+	<reg32 offset="0xa82f" name="SP_HS_VGPR_CONFIG" variants="A7XX-" usage="cmd"/>
+
+	<reg32 offset="0xa840" name="SP_DS_CTRL_REG0" type="a6xx_sp_xs_ctrl_reg0" usage="rp_blit">
+		<!-- There is no mergedregs bit, that comes from the VS. -->
+		<bitfield name="EARLYPREAMBLE" pos="20" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xa841" name="SP_DS_BRANCH_COND" type="hex"/>
+
+	<!-- TODO: exact same layout as 0xa802-0xa81a -->
+	<reg32 offset="0xa842" name="SP_DS_PRIMITIVE_CNTL" type="a6xx_sp_xs_prim_cntl" usage="rp_blit"/>
+	<array offset="0xa843" name="SP_DS_OUT" stride="1" length="16" usage="rp_blit">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="A_REGID" low="0" high="7" type="a3xx_regid"/>
+			<bitfield name="A_COMPMASK" low="8" high="11" type="hex"/>
+			<bitfield name="B_REGID" low="16" high="23" type="a3xx_regid"/>
+			<bitfield name="B_COMPMASK" low="24" high="27" type="hex"/>
+		</reg32>
+	</array>
+	<array offset="0xa853" name="SP_DS_VPC_DST" stride="1" length="8" usage="rp_blit">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="OUTLOC0" low="0" high="7" type="uint"/>
+			<bitfield name="OUTLOC1" low="8" high="15" type="uint"/>
+			<bitfield name="OUTLOC2" low="16" high="23" type="uint"/>
+			<bitfield name="OUTLOC3" low="24" high="31" type="uint"/>
+		</reg32>
+	</array>
+
+	<!-- TODO: exact same layout as 0xa81b-0xa825 -->
+	<reg32 offset="0xa85b" name="SP_DS_OBJ_FIRST_EXEC_OFFSET" type="uint" usage="rp_blit"/>
+	<reg64 offset="0xa85c" name="SP_DS_OBJ_START" type="address" align="32" usage="rp_blit"/>
+	<reg32 offset="0xa85e" name="SP_DS_PVT_MEM_PARAM" type="a6xx_sp_xs_pvt_mem_param" usage="rp_blit"/>
+	<reg64 offset="0xa85f" name="SP_DS_PVT_MEM_ADDR" type="waddress" align="32" usage="rp_blit"/>
+	<reg32 offset="0xa861" name="SP_DS_PVT_MEM_SIZE" type="a6xx_sp_xs_pvt_mem_size" usage="rp_blit"/>
+	<reg32 offset="0xa862" name="SP_DS_TEX_COUNT" low="0" high="7" type="uint" usage="rp_blit"/>
+	<reg32 offset="0xa863" name="SP_DS_CONFIG" type="a6xx_sp_xs_config" usage="rp_blit"/>
+	<reg32 offset="0xa864" name="SP_DS_INSTRLEN" low="0" high="27" type="uint" usage="rp_blit"/>
+	<reg32 offset="0xa865" name="SP_DS_PVT_MEM_HW_STACK_OFFSET" type="a6xx_sp_xs_pvt_mem_hw_stack_offset" usage="rp_blit"/>
+	<reg32 offset="0xa868" name="SP_DS_VGPR_CONFIG" variants="A7XX-" usage="cmd"/>
+
+	<reg32 offset="0xa870" name="SP_GS_CTRL_REG0" type="a6xx_sp_xs_ctrl_reg0" usage="rp_blit">
+		<!-- There is no mergedregs bit, that comes from the VS. -->
+		<bitfield name="EARLYPREAMBLE" pos="20" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xa871" name="SP_GS_PRIM_SIZE" low="0" high="7" type="uint" usage="rp_blit">
+		<doc>
+			Normally the size of the output of the last stage in
+			dwords. It should be programmed as follows:
+
+			size less than 63    - size
+			size of 63 (?) or 64 - 63
+			size greater than 64 - 64
+
+			What to program when the size is 61-63 is a guess, but
+			both the blob and ir3 align the size to 4 dword's so it
+			doesn't matter in practice.
+		</doc>
+	</reg32>
+	<reg32 offset="0xa872" name="SP_GS_BRANCH_COND" type="hex" usage="rp_blit"/>
+
+	<!-- TODO: exact same layout as 0xa802-0xa81a -->
+	<reg32 offset="0xa873" name="SP_GS_PRIMITIVE_CNTL" type="a6xx_sp_xs_prim_cntl" usage="rp_blit"/>
+	<array offset="0xa874" name="SP_GS_OUT" stride="1" length="16" usage="rp_blit">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="A_REGID" low="0" high="7" type="a3xx_regid"/>
+			<bitfield name="A_COMPMASK" low="8" high="11" type="hex"/>
+			<bitfield name="B_REGID" low="16" high="23" type="a3xx_regid"/>
+			<bitfield name="B_COMPMASK" low="24" high="27" type="hex"/>
+		</reg32>
+	</array>
+
+	<array offset="0xa884" name="SP_GS_VPC_DST" stride="1" length="8" usage="rp_blit">
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="OUTLOC0" low="0" high="7" type="uint"/>
+			<bitfield name="OUTLOC1" low="8" high="15" type="uint"/>
+			<bitfield name="OUTLOC2" low="16" high="23" type="uint"/>
+			<bitfield name="OUTLOC3" low="24" high="31" type="uint"/>
+		</reg32>
+	</array>
+
+	<!-- TODO: exact same layout as 0xa81b-0xa825 -->
+	<reg32 offset="0xa88c" name="SP_GS_OBJ_FIRST_EXEC_OFFSET" type="uint" usage="rp_blit"/>
+	<reg64 offset="0xa88d" name="SP_GS_OBJ_START" type="address" align="32" usage="rp_blit"/>
+	<reg32 offset="0xa88f" name="SP_GS_PVT_MEM_PARAM" type="a6xx_sp_xs_pvt_mem_param" usage="rp_blit"/>
+	<reg64 offset="0xa890" name="SP_GS_PVT_MEM_ADDR" type="waddress" align="32" usage="rp_blit"/>
+	<reg32 offset="0xa892" name="SP_GS_PVT_MEM_SIZE" type="a6xx_sp_xs_pvt_mem_size" usage="rp_blit"/>
+	<reg32 offset="0xa893" name="SP_GS_TEX_COUNT" low="0" high="7" type="uint" usage="rp_blit"/>
+	<reg32 offset="0xa894" name="SP_GS_CONFIG" type="a6xx_sp_xs_config" usage="rp_blit"/>
+	<reg32 offset="0xa895" name="SP_GS_INSTRLEN" low="0" high="27" type="uint" usage="rp_blit"/>
+	<reg32 offset="0xa896" name="SP_GS_PVT_MEM_HW_STACK_OFFSET" type="a6xx_sp_xs_pvt_mem_hw_stack_offset" usage="rp_blit"/>
+	<reg32 offset="0xa899" name="SP_GS_VGPR_CONFIG" variants="A7XX-" usage="cmd"/>
+
+	<reg64 offset="0xa8a0" name="SP_VS_TEX_SAMP" type="address" align="16" usage="cmd"/>
+	<reg64 offset="0xa8a2" name="SP_HS_TEX_SAMP" type="address" align="16" usage="cmd"/>
+	<reg64 offset="0xa8a4" name="SP_DS_TEX_SAMP" type="address" align="16" usage="cmd"/>
+	<reg64 offset="0xa8a6" name="SP_GS_TEX_SAMP" type="address" align="16" usage="cmd"/>
+	<reg64 offset="0xa8a8" name="SP_VS_TEX_CONST" type="address" align="64" usage="cmd"/>
+	<reg64 offset="0xa8aa" name="SP_HS_TEX_CONST" type="address" align="64" usage="cmd"/>
+	<reg64 offset="0xa8ac" name="SP_DS_TEX_CONST" type="address" align="64" usage="cmd"/>
+	<reg64 offset="0xa8ae" name="SP_GS_TEX_CONST" type="address" align="64" usage="cmd"/>
+
+	<!-- TODO: 4 unknown bool registers 0xa8c0-0xa8c3 -->
+
+	<reg32 offset="0xa980" name="SP_FS_CTRL_REG0" type="a6xx_sp_xs_ctrl_reg0" usage="rp_blit">
+		<bitfield name="THREADSIZE" pos="20" type="a6xx_threadsize"/>
+		<bitfield name="UNK21" pos="21" type="boolean"/>
+		<bitfield name="VARYING" pos="22" type="boolean"/>
+		<bitfield name="LODPIXMASK" pos="23" type="boolean">
+			<doc>
+				Enable ALL helper invocations in a quad. Necessary for
+				fine derivatives and quad subgroup ops.
+			</doc>
+		</bitfield>
+		<!-- note: vk blob uses bit24 -->
+		<bitfield name="UNK24" pos="24" type="boolean"/>
+		<bitfield name="UNK25" pos="25" type="boolean"/>
+		<bitfield name="PIXLODENABLE" pos="26" type="boolean">
+			<doc>
+				Enable helper invocations. Enables 3 out of 4 fragments,
+				because the coarse derivatives only use half of the quad
+				and so one pixel's value is always unused.
+			</doc>
+		</bitfield>
+		<bitfield name="UNK27" pos="27" type="boolean"/>
+		<bitfield name="EARLYPREAMBLE" pos="28" type="boolean"/>
+		<bitfield name="MERGEDREGS" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xa981" name="SP_FS_BRANCH_COND" type="hex"/>
+	<reg32 offset="0xa982" name="SP_FS_OBJ_FIRST_EXEC_OFFSET" type="uint" usage="rp_blit"/>
+	<reg64 offset="0xa983" name="SP_FS_OBJ_START" type="address" align="32" usage="rp_blit"/>
+	<reg32 offset="0xa985" name="SP_FS_PVT_MEM_PARAM" type="a6xx_sp_xs_pvt_mem_param" usage="rp_blit"/>
+	<reg64 offset="0xa986" name="SP_FS_PVT_MEM_ADDR" type="waddress" align="32" usage="rp_blit"/>
+	<reg32 offset="0xa988" name="SP_FS_PVT_MEM_SIZE" type="a6xx_sp_xs_pvt_mem_size" usage="rp_blit"/>
+
+	<reg32 offset="0xa989" name="SP_BLEND_CNTL" usage="rp_blit">
+		<!-- per-mrt enable bit -->
+		<bitfield name="ENABLE_BLEND" low="0" high="7"/>
+		<bitfield name="UNK8" pos="8" type="boolean"/>
+		<bitfield name="DUAL_COLOR_IN_ENABLE" pos="9" type="boolean"/>
+		<bitfield name="ALPHA_TO_COVERAGE" pos="10" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xa98a" name="SP_SRGB_CNTL" usage="rp_blit">
+		<!-- Same as RB_SRGB_CNTL -->
+		<bitfield name="SRGB_MRT0" pos="0" type="boolean"/>
+		<bitfield name="SRGB_MRT1" pos="1" type="boolean"/>
+		<bitfield name="SRGB_MRT2" pos="2" type="boolean"/>
+		<bitfield name="SRGB_MRT3" pos="3" type="boolean"/>
+		<bitfield name="SRGB_MRT4" pos="4" type="boolean"/>
+		<bitfield name="SRGB_MRT5" pos="5" type="boolean"/>
+		<bitfield name="SRGB_MRT6" pos="6" type="boolean"/>
+		<bitfield name="SRGB_MRT7" pos="7" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xa98b" name="SP_FS_RENDER_COMPONENTS" usage="rp_blit">
+		<bitfield name="RT0" low="0" high="3"/>
+		<bitfield name="RT1" low="4" high="7"/>
+		<bitfield name="RT2" low="8" high="11"/>
+		<bitfield name="RT3" low="12" high="15"/>
+		<bitfield name="RT4" low="16" high="19"/>
+		<bitfield name="RT5" low="20" high="23"/>
+		<bitfield name="RT6" low="24" high="27"/>
+		<bitfield name="RT7" low="28" high="31"/>
+	</reg32>
+	<reg32 offset="0xa98c" name="SP_FS_OUTPUT_CNTL0" usage="rp_blit">
+		<bitfield name="DUAL_COLOR_IN_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="DEPTH_REGID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="SAMPMASK_REGID" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="STENCILREF_REGID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xa98d" name="SP_FS_OUTPUT_CNTL1" usage="rp_blit">
+		<bitfield name="MRT" low="0" high="3" type="uint"/>
+	</reg32>
+
+	<array offset="0xa98e" name="SP_FS_OUTPUT" stride="1" length="8" usage="rp_blit">
+		<doc>per MRT</doc>
+		<reg32 offset="0x0" name="REG">
+			<bitfield name="REGID" low="0" high="7" type="a3xx_regid"/>
+			<bitfield name="HALF_PRECISION" pos="8" type="boolean"/>
+		</reg32>
+	</array>
+
+	<array offset="0xa996" name="SP_FS_MRT" stride="1" length="8" usage="rp_blit">
+		<reg32 offset="0" name="REG">
+			<bitfield name="COLOR_FORMAT" low="0" high="7" type="a6xx_format"/>
+			<bitfield name="COLOR_SINT" pos="8" type="boolean"/>
+			<bitfield name="COLOR_UINT" pos="9" type="boolean"/>
+			<bitfield name="UNK10" pos="10" type="boolean"/>
+		</reg32>
+	</array>
+
+	<reg32 offset="0xa99e" name="SP_FS_PREFETCH_CNTL" usage="rp_blit">
+		<bitfield name="COUNT" low="0" high="2" type="uint"/>
+		<bitfield name="IJ_WRITE_DISABLE" pos="3" type="boolean"/>
+		<doc>
+			Similar to "(eq)" flag but disables helper invocations
+			after the texture prefetch.
+		</doc>
+		<bitfield name="ENDOFQUAD" pos="4" type="boolean" />
+		<doc>
+			Bypass writing to regs and overwrite output with color from
+			CONSTSLOTID const regs.
+		</doc>
+		<bitfield name="WRITE_COLOR_TO_OUTPUT" pos="5" type="boolean"/>
+		<bitfield name="CONSTSLOTID" low="6" high="14" type="uint"/>
+		<!-- Blob never uses it -->
+		<bitfield name="CONSTSLOTID4COORD" low="16" high="24" type="uint" variants="A7XX-"/>
+	</reg32>
+	<array offset="0xa99f" name="SP_FS_PREFETCH" stride="1" length="4" variants="A6XX" usage="rp_blit">
+		<reg32 offset="0" name="CMD" variants="A6XX">
+			<bitfield name="SRC" low="0" high="6" type="uint"/>
+			<bitfield name="SAMP_ID" low="7" high="10" type="uint"/>
+			<bitfield name="TEX_ID" low="11" high="15" type="uint"/>
+			<bitfield name="DST" low="16" high="21" type="a3xx_regid"/>
+			<bitfield name="WRMASK" low="22" high="25" type="hex"/>
+			<bitfield name="HALF" pos="26" type="boolean"/>
+			<doc>Results in color being zero</doc>
+			<bitfield name="UNK27" pos="27" type="boolean"/>
+			<bitfield name="BINDLESS" pos="28" type="boolean"/>
+			<bitfield name="CMD" low="29" high="31" type="a6xx_tex_prefetch_cmd"/>
+		</reg32>
+	</array>
+	<array offset="0xa99f" name="SP_FS_PREFETCH" stride="1" length="4" variants="A7XX-" usage="rp_blit">
+		<reg32 offset="0" name="CMD" variants="A7XX-">
+			<bitfield name="SRC" low="0" high="6" type="uint"/>
+			<bitfield name="SAMP_ID" low="7" high="9" type="uint"/>
+			<bitfield name="TEX_ID" low="10" high="12" type="uint"/>
+			<bitfield name="DST" low="13" high="18" type="a3xx_regid"/>
+			<bitfield name="WRMASK" low="19" high="22" type="hex"/>
+			<bitfield name="HALF" pos="23" type="boolean"/>
+			<bitfield name="BINDLESS" pos="25" type="boolean"/>
+			<bitfield name="CMD" low="26" high="29" type="a6xx_tex_prefetch_cmd"/>
+		</reg32>
+	</array>
+	<array offset="0xa9a3" name="SP_FS_BINDLESS_PREFETCH" stride="1" length="4" usage="rp_blit">
+		<reg32 offset="0" name="CMD">
+			<bitfield name="SAMP_ID" low="0" high="15" type="uint"/>
+			<bitfield name="TEX_ID" low="16" high="31" type="uint"/>
+		</reg32>
+	</array>
+	<reg32 offset="0xa9a7" name="SP_FS_TEX_COUNT" low="0" high="7" type="uint" usage="rp_blit"/>
+	<reg32 offset="0xa9a8" name="SP_UNKNOWN_A9A8" low="0" high="16" usage="cmd"/> <!-- always 0x0 ? -->
+	<reg32 offset="0xa9a9" name="SP_FS_PVT_MEM_HW_STACK_OFFSET" type="a6xx_sp_xs_pvt_mem_hw_stack_offset" usage="rp_blit"/>
+
+	<!-- TODO: unknown bool register at 0xa9aa, likely same as 0xa8c0-0xa8c3 but for FS -->
+
+
+
+
+	<reg32 offset="0xa9b0" name="SP_CS_CTRL_REG0" type="a6xx_sp_xs_ctrl_reg0" usage="cmd">
+		<bitfield name="THREADSIZE" pos="20" type="a6xx_threadsize"/>
+		<!-- seems to make SP use less concurrent threads when possible? -->
+		<bitfield name="UNK21" pos="21" type="boolean"/>
+		<!-- has a small impact on performance, not clear what it does -->
+		<bitfield name="UNK22" pos="22" type="boolean"/>
+		<bitfield name="EARLYPREAMBLE" pos="23" type="boolean"/>
+		<bitfield name="MERGEDREGS" pos="31" type="boolean"/>
+	</reg32>
+
+	<!-- set for compute shaders -->
+	<reg32 offset="0xa9b1" name="SP_CS_UNKNOWN_A9B1" usage="cmd">
+		<bitfield name="SHARED_SIZE" low="0" high="4" type="uint">
+			<doc>
+				If 0 - all 32k of shared storage is enabled, otherwise
+				(SHARED_SIZE + 1) * 1k is enabled.
+				The ldl/stl offset seems to be rewritten to 0 when it is beyond
+				this limit. This is different from ldlw/stlw, which wraps at
+				64k (and has 36k of storage on A640 - reads between 36k-64k
+				always return 0)
+			</doc>
+		</bitfield>
+		<bitfield name="UNK5" pos="5" type="boolean"/>
+		<!-- always 1 ? -->
+		<bitfield name="UNK6" pos="6" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xa9b2" name="SP_CS_BRANCH_COND" type="hex" usage="cmd"/>
+	<reg32 offset="0xa9b3" name="SP_CS_OBJ_FIRST_EXEC_OFFSET" type="uint" usage="cmd"/>
+	<reg64 offset="0xa9b4" name="SP_CS_OBJ_START" type="address" align="32" usage="cmd"/>
+	<reg32 offset="0xa9b6" name="SP_CS_PVT_MEM_PARAM" type="a6xx_sp_xs_pvt_mem_param" usage="cmd"/>
+	<reg64 offset="0xa9b7" name="SP_CS_PVT_MEM_ADDR" align="32" usage="cmd"/>
+	<reg32 offset="0xa9b9" name="SP_CS_PVT_MEM_SIZE" type="a6xx_sp_xs_pvt_mem_size" usage="cmd"/>
+	<reg32 offset="0xa9ba" name="SP_CS_TEX_COUNT" low="0" high="7" type="uint" usage="cmd"/>
+	<reg32 offset="0xa9bb" name="SP_CS_CONFIG" type="a6xx_sp_xs_config" usage="cmd"/>
+	<reg32 offset="0xa9bc" name="SP_CS_INSTRLEN" low="0" high="27" type="uint" usage="cmd"/>
+	<reg32 offset="0xa9bd" name="SP_CS_PVT_MEM_HW_STACK_OFFSET" type="a6xx_sp_xs_pvt_mem_hw_stack_offset" usage="cmd"/>
+	<reg32 offset="0xa9be" name="SP_CS_UNKNOWN_A9BE" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0xa9c5" name="SP_CS_VGPR_CONFIG" variants="A7XX-" usage="cmd"/>
+
+	<!-- new in a6xx gen4, matches HLSQ_CS_CNTL_0 -->
+	<reg32 offset="0xa9c2" name="SP_CS_CNTL_0" usage="cmd">
+		<bitfield name="WGIDCONSTID" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="WGSIZECONSTID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="WGOFFSETCONSTID" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="LOCALIDREGID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<!-- new in a6xx gen4, matches HLSQ_CS_CNTL_1 -->
+	<reg32 offset="0xa9c3" name="SP_CS_CNTL_1" variants="A6XX" usage="cmd">
+		<!-- gl_LocalInvocationIndex -->
+		<bitfield name="LINEARLOCALIDREGID" low="0" high="7" type="a3xx_regid"/>
+		<!-- a650 has 6 "SP cores" (but 3 "SP"). this makes it use only
+		     one of those 6 "SP cores" -->
+		<bitfield name="SINGLE_SP_CORE" pos="8" type="boolean"/>
+		<!-- Must match SP_CS_CTRL -->
+		<bitfield name="THREADSIZE" pos="9" type="a6xx_threadsize"/>
+		<!-- 1 thread per wave (ignored if bit9 set) -->
+		<bitfield name="THREADSIZE_SCALAR" pos="10" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0xa9c3" name="SP_CS_CNTL_1" variants="A7XX-" usage="cmd">
+		<!-- gl_LocalInvocationIndex -->
+		<bitfield name="LINEARLOCALIDREGID" low="0" high="7" type="a3xx_regid"/>
+		<!-- Must match SP_CS_CTRL -->
+		<bitfield name="THREADSIZE" pos="8" type="a6xx_threadsize"/>
+		<!-- 1 thread per wave (would hang if THREAD128 is also set) -->
+		<bitfield name="THREADSIZE_SCALAR" pos="9" type="boolean"/>
+
+		<!-- Affects getone. If enabled, getone sometimes executed 1? less times
+		     than there are subgroups.
+		 -->
+		<bitfield name="UNK15" pos="15" type="boolean"/>
+	</reg32>
+
+	<!-- TODO: two 64kb aligned addresses at a9d0/a9d2 -->
+
+	<reg64 offset="0xa9e0" name="SP_FS_TEX_SAMP" type="address" align="16" usage="rp_blit"/>
+	<reg64 offset="0xa9e2" name="SP_CS_TEX_SAMP" type="address" align="16" usage="cmd"/>
+	<reg64 offset="0xa9e4" name="SP_FS_TEX_CONST" type="address" align="64" usage="rp_blit"/>
+	<reg64 offset="0xa9e6" name="SP_CS_TEX_CONST" type="address" align="64" usage="cmd"/>
+
+	<enum name="a6xx_bindless_descriptor_size">
+		<doc>
+			This can alternatively be interpreted as a pitch shift, ie, the
+			descriptor size is 2 &lt;&lt; N dwords
+		</doc>
+		<value value="1" name="BINDLESS_DESCRIPTOR_16B"/>
+		<value value="3" name="BINDLESS_DESCRIPTOR_64B"/>
+	</enum>
+
+	<array offset="0xa9e8" name="SP_CS_BINDLESS_BASE" stride="2" length="5" variants="A6XX" usage="cmd">
+		<reg64 offset="0" name="DESCRIPTOR" variants="A6XX">
+			<bitfield name="DESC_SIZE" low="0" high="1" type="a6xx_bindless_descriptor_size"/>
+			<bitfield name="ADDR" low="2" high="63" shr="2" type="address"/>
+		</reg64>
+	</array>
+	<array offset="0xa9e8" name="SP_CS_BINDLESS_BASE" stride="2" length="8" variants="A7XX-" usage="cmd">
+		<reg64 offset="0" name="DESCRIPTOR" variants="A7XX-">
+			<bitfield name="DESC_SIZE" low="0" high="1" type="a6xx_bindless_descriptor_size"/>
+			<bitfield name="ADDR" low="2" high="63" shr="2" type="address"/>
+		</reg64>
+	</array>
+
+	<!--
+	IBO state for compute shader:
+	 -->
+	<reg64 offset="0xa9f2" name="SP_CS_IBO" type="address" align="16"/>
+	<reg32 offset="0xaa00" name="SP_CS_IBO_COUNT" low="0" high="6" type="uint"/>
+
+	<!-- Correlated with avgs/uvgs usage in FS -->
+	<reg32 offset="0xaa01" name="SP_FS_VGPR_CONFIG" type="uint" variants="A7XX-" usage="cmd"/>
+
+	<reg32 offset="0xaa02" name="SP_PS_ALIASED_COMPONENTS_CONTROL" variants="A7XX-" usage="cmd">
+		<bitfield name="ENABLED" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0xaa03" name="SP_PS_ALIASED_COMPONENTS" variants="A7XX-" usage="cmd">
+		<doc>
+			Specify for which components the output color should be read
+			from alias, e.g. for:
+
+				alias.1.b32.0 r3.x, c8.x
+				alias.1.b32.0 r2.x, c4.x
+				alias.1.b32.0 r1.x, c4.x
+				alias.1.b32.0 r0.x, c0.x
+
+			the SP_PS_ALIASED_COMPONENTS would be 0x00001111
+		</doc>
+
+		<bitfield name="RT0" low="0" high="3"/>
+		<bitfield name="RT1" low="4" high="7"/>
+		<bitfield name="RT2" low="8" high="11"/>
+		<bitfield name="RT3" low="12" high="15"/>
+		<bitfield name="RT4" low="16" high="19"/>
+		<bitfield name="RT5" low="20" high="23"/>
+		<bitfield name="RT6" low="24" high="27"/>
+		<bitfield name="RT7" low="28" high="31"/>
+	</reg32>
+
+	<reg32 offset="0xaaf2" name="SP_UNKNOWN_AAF2" type="uint" usage="cmd"/>
+
+	<!--
+                This enum is probably similar in purpose to SNORMMODE on a3xx,
+                minus the snorm stuff, i.e. it controls what happens with an
+                out-of-bounds isam/isamm. GL and Vulkan robustness require us to
+                return 0 on out-of-bound textureFetch().
+	-->
+	<enum name="a6xx_isam_mode">
+		<value value="0x1" name="ISAMMODE_CL"/>
+		<value value="0x2" name="ISAMMODE_GL"/>
+	</enum>
+
+	<reg32 offset="0xab00" name="SP_MODE_CONTROL" usage="rp_blit">
+	  <!--
+	  When set, half register loads from the constant file will
+	  load a 32-bit value (so hc0.y loads the same value as c0.y)
+	  and implicitly convert it to 16b (f2f16, or u2u16, based on
+	  operand type).  When unset, half register loads from the
+	  constant file will load 16 bits from the packed constant
+	  file (so hc0.y loads the top 16 bits of the value of c0.x)
+	  -->
+		<bitfield name="CONSTANT_DEMOTION_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="ISAMMODE" low="1" high="2" type="a6xx_isam_mode"/>
+		<bitfield name="SHARED_CONSTS_ENABLE" pos="3" type="boolean"/> <!-- see HLSQ_SHARED_CONSTS -->
+	</reg32>
+
+	<reg32 offset="0xab01" name="SP_UNKNOWN_AB01" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0xab02" name="SP_UNKNOWN_AB02" variants="A7XX-" usage="cmd"/>
+
+	<reg32 offset="0xab04" name="SP_FS_CONFIG" type="a6xx_sp_xs_config" usage="rp_blit"/>
+	<reg32 offset="0xab05" name="SP_FS_INSTRLEN" low="0" high="27" type="uint" usage="rp_blit"/>
+
+	<array offset="0xab10" name="SP_BINDLESS_BASE" stride="2" length="5" variants="A6XX" usage="rp_blit">
+		<reg64 offset="0" name="DESCRIPTOR" variants="A6XX">
+			<bitfield name="DESC_SIZE" low="0" high="1" type="a6xx_bindless_descriptor_size"/>
+			<bitfield name="ADDR" low="2" high="63" shr="2" type="address"/>
+		</reg64>
+	</array>
+	<array offset="0xab0a" name="SP_BINDLESS_BASE" stride="2" length="8" variants="A7XX-" usage="rp_blit">
+		<reg64 offset="0" name="DESCRIPTOR" variants="A7XX-">
+			<bitfield name="DESC_SIZE" low="0" high="1" type="a6xx_bindless_descriptor_size"/>
+			<bitfield name="ADDR" low="2" high="63" shr="2" type="address"/>
+		</reg64>
+	</array>
+
+	<!--
+	Combined IBO state for 3d pipe, used for Image and SSBO write/atomic
+	instructions VS/HS/DS/GS/FS.  See SP_CS_IBO_* for compute shaders.
+	 -->
+	<reg64 offset="0xab1a" name="SP_IBO" type="address" align="16" usage="cmd"/>
+	<reg32 offset="0xab20" name="SP_IBO_COUNT" low="0" high="6" type="uint" usage="cmd"/>
+
+	<reg32 offset="0xab22" name="SP_UNKNOWN_AB22" variants="A7XX-" usage="cmd"/>
+
+	<bitset name="a6xx_sp_2d_dst_format" inline="yes">
+		<bitfield name="NORM" pos="0" type="boolean"/>
+		<bitfield name="SINT" pos="1" type="boolean"/>
+		<bitfield name="UINT" pos="2" type="boolean"/>
+		<!-- looks like HW only cares about the base type of this format,
+		     which matches the ifmt? -->
+		<bitfield name="COLOR_FORMAT" low="3" high="10" type="a6xx_format"/>
+		<!-- set when ifmt is R2D_UNORM8_SRGB -->
+		<bitfield name="SRGB" pos="11" type="boolean"/>
+		<!-- some sort of channel mask, not sure what it is for -->
+		<bitfield name="MASK" low="12" high="15"/>
+	</bitset>
+
+	<reg32 offset="0xacc0" name="SP_2D_DST_FORMAT" type="a6xx_sp_2d_dst_format" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xa9bf" name="SP_2D_DST_FORMAT" type="a6xx_sp_2d_dst_format" variants="A7XX-" usage="rp_blit"/>
+
+	<reg32 offset="0xae00" name="SP_DBG_ECO_CNTL" usage="cmd"/>
+	<reg32 offset="0xae01" name="SP_ADDR_MODE_CNTL" pos="0" type="a5xx_address_mode"/>
+	<reg32 offset="0xae02" name="SP_NC_MODE_CNTL">
+		<!-- TODO: valid bits 0x3c3f, see kernel -->
+	</reg32>
+	<reg32 offset="0xae03" name="SP_CHICKEN_BITS" usage="cmd"/>
+	<reg32 offset="0xae04" name="SP_FLOAT_CNTL" usage="cmd">
+		<bitfield name="F16_NO_INF" pos="3" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0xae06" name="SP_UNKNOWN_AE06" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0xae08" name="SP_UNKNOWN_AE08" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0xae09" name="SP_UNKNOWN_AE09" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0xae0a" name="SP_UNKNOWN_AE0A" variants="A7XX-" usage="cmd"/>
+
+	<reg32 offset="0xae0f" name="SP_PERFCTR_ENABLE" usage="cmd">
+		<!-- some perfcntrs are affected by a per-stage enable bit
+		     (PERF_SP_ALU_WORKING_CYCLES for example)
+		     TODO: verify position of HS/DS/GS bits -->
+		<bitfield name="VS" pos="0" type="boolean"/>
+		<bitfield name="HS" pos="1" type="boolean"/>
+		<bitfield name="DS" pos="2" type="boolean"/>
+		<bitfield name="GS" pos="3" type="boolean"/>
+		<bitfield name="FS" pos="4" type="boolean"/>
+		<bitfield name="CS" pos="5" type="boolean"/>
+	</reg32>
+	<array offset="0xae10" name="SP_PERFCTR_SP_SEL" stride="1" length="24"/>
+	<array offset="0xae60" name="SP_PERFCTR_HLSQ_SEL" stride="1" length="6" variants="A7XX-"/>
+	<reg32 offset="0xae6a" name="SP_UNKNOWN_AE6A" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0xae6b" name="SP_UNKNOWN_AE6B" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0xae6c" name="SP_UNKNOWN_AE6C" variants="A7XX-" usage="cmd"/>
+	<reg32 offset="0xae6d" name="SP_READ_SEL" variants="A7XX-">
+		<bitfield name="LOCATION" low="18" high="19" type="a7xx_state_location"/>
+		<bitfield name="PIPE" low="16" high="17" type="a7xx_pipe"/>
+		<bitfield name="STATETYPE" low="8" high="15" type="a7xx_statetype_id"/>
+		<bitfield name="USPTP" low="4" high="7"/>
+		<bitfield name="SPTP" low="0" high="3"/>
+	</reg32>
+	<reg32 offset="0xae71" name="SP_DBG_CNTL" variants="A7XX-"/>
+	<reg32 offset="0xae73" name="SP_UNKNOWN_AE73" variants="A7XX-" usage="cmd"/>
+	<array offset="0xae80" name="SP_PERFCTR_SP_SEL" stride="1" length="36" variants="A7XX-"/>
+	<!-- TODO: there are 4 more percntr select registers (0xae28-0xae2b) -->
+	<!-- TODO: there are a few unknown registers in the 0xae30-0xae52 range -->
+	<reg32 offset="0xbe22" name="SP_CONTEXT_SWITCH_GFX_PREEMPTION_SAFE_MODE"/>
+
+	<!--
+	The downstream kernel calls the debug cluster of registers
+	"a6xx_sp_ps_tp_cluster" but this actually specifies the border
+	color base for compute shaders.
+	-->
+	<reg64 offset="0xb180" name="SP_PS_TP_BORDER_COLOR_BASE_ADDR" type="address" align="128" usage="cmd"/>
+	<reg32 offset="0xb182" name="SP_UNKNOWN_B182" low="0" high="2" usage="cmd"/>
+	<reg32 offset="0xb183" name="SP_UNKNOWN_B183" low="0" high="23" usage="cmd"/>
+
+	<reg32 offset="0xb190" name="SP_UNKNOWN_B190"/>
+	<reg32 offset="0xb191" name="SP_UNKNOWN_B191"/>
+
+	<!-- could be all the stuff below here is actually TPL1?? -->
+
+	<reg32 offset="0xb300" name="SP_TP_RAS_MSAA_CNTL" usage="rp_blit">
+		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>
+		<bitfield name="UNK2" low="2" high="3"/>
+	</reg32>
+	<reg32 offset="0xb301" name="SP_TP_DEST_MSAA_CNTL" usage="rp_blit">
+		<bitfield name="SAMPLES" low="0" high="1" type="a3xx_msaa_samples"/>
+		<bitfield name="MSAA_DISABLE" pos="2" type="boolean"/>
+	</reg32>
+
+	<!-- looks to work in the same way as a5xx: -->
+	<reg64 offset="0xb302" name="SP_TP_BORDER_COLOR_BASE_ADDR" type="address" align="128" usage="cmd"/>
+	<reg32 offset="0xb304" name="SP_TP_SAMPLE_CONFIG" type="a6xx_sample_config" usage="rp_blit"/>
+	<reg32 offset="0xb305" name="SP_TP_SAMPLE_LOCATION_0" type="a6xx_sample_locations" usage="rp_blit"/>
+	<reg32 offset="0xb306" name="SP_TP_SAMPLE_LOCATION_1" type="a6xx_sample_locations" usage="rp_blit"/>
+	<reg32 offset="0xb307" name="SP_TP_WINDOW_OFFSET" type="a6xx_reg_xy" usage="rp_blit"/>
+	<reg32 offset="0xb309" name="SP_TP_MODE_CNTL" usage="cmd">
+		<bitfield name="ISAMMODE" low="0" high="1" type="a6xx_isam_mode"/>
+		<bitfield name="UNK3" low="2" high="7"/>
+	</reg32>
+	<reg32 offset="0xb310" name="SP_UNKNOWN_B310" variants="A7XX-" usage="cmd"/>
+
+	<!--
+	Equiv to corresponding RB_2D_SRC_* regs on a5xx.. which were either
+	badly named or the functionality moved in a6xx.  But downstream kernel
+	calls this "a6xx_sp_ps_tp_2d_cluster"
+	 -->
+	<reg32 offset="0xb4c0" name="SP_PS_2D_SRC_INFO" type="a6xx_2d_surf_info" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb4c1" name="SP_PS_2D_SRC_SIZE" variants="A6XX" usage="rp_blit">
+		<bitfield name="WIDTH" low="0" high="14" type="uint"/>
+		<bitfield name="HEIGHT" low="15" high="29" type="uint"/>
+	</reg32>
+	<reg64 offset="0xb4c2" name="SP_PS_2D_SRC" type="address" align="16" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb4c4" name="SP_PS_2D_SRC_PITCH" variants="A6XX" usage="rp_blit">
+		<bitfield name="UNK0" low="0" high="8"/>
+		<bitfield name="PITCH" low="9" high="23" shr="6" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0xb2c0" name="SP_PS_2D_SRC_INFO" type="a6xx_2d_surf_info" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xb2c1" name="SP_PS_2D_SRC_SIZE" variants="A7XX">
+		<bitfield name="WIDTH" low="0" high="14" type="uint"/>
+		<bitfield name="HEIGHT" low="15" high="29" type="uint"/>
+	</reg32>
+	<reg64 offset="0xb2c2" name="SP_PS_2D_SRC" type="address" align="16" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xb2c4" name="SP_PS_2D_SRC_PITCH" variants="A7XX">
+		<bitfield name="UNK0" low="0" high="8"/>
+		<bitfield name="PITCH" low="9" high="23" shr="6" type="uint"/>
+	</reg32>
+
+	<!-- planes for NV12, etc. (TODO: not tested) -->
+	<reg64 offset="0xb4c5" name="SP_PS_2D_SRC_PLANE1" type="address" align="16" variants="A6XX"/>
+	<reg32 offset="0xb4c7" name="SP_PS_2D_SRC_PLANE_PITCH" low="0" high="11" shr="6" type="uint" variants="A6XX"/>
+	<reg64 offset="0xb4c8" name="SP_PS_2D_SRC_PLANE2" type="address" align="16" variants="A6XX"/>
+
+	<reg64 offset="0xb2c5" name="SP_PS_2D_SRC_PLANE1" type="address" align="16" variants="A7XX-"/>
+	<reg32 offset="0xb2c7" name="SP_PS_2D_SRC_PLANE_PITCH" low="0" high="11" shr="6" type="uint" variants="A7XX-"/>
+	<reg64 offset="0xb2c8" name="SP_PS_2D_SRC_PLANE2" type="address" align="16" variants="A7XX-"/>
+
+	<reg64 offset="0xb4ca" name="SP_PS_2D_SRC_FLAGS" type="address" align="16" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb4cc" name="SP_PS_2D_SRC_FLAGS_PITCH" low="0" high="7" shr="6" type="uint" variants="A6XX" usage="rp_blit"/>
+
+	<reg64 offset="0xb2ca" name="SP_PS_2D_SRC_FLAGS" type="address" align="16" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xb2cc" name="SP_PS_2D_SRC_FLAGS_PITCH" low="0" high="7" shr="6" type="uint" variants="A7XX-" usage="rp_blit"/>
+
+	<reg32 offset="0xb4cd" name="SP_PS_UNKNOWN_B4CD" low="6" high="31" variants="A6XX"/>
+	<reg32 offset="0xb4ce" name="SP_PS_UNKNOWN_B4CE" low="0" high="31" variants="A6XX"/>
+	<reg32 offset="0xb4cf" name="SP_PS_UNKNOWN_B4CF" low="0" high="30" variants="A6XX"/>
+	<reg32 offset="0xb4d0" name="SP_PS_UNKNOWN_B4D0" low="0" high="29" variants="A6XX"/>
+	<reg32 offset="0xb4d1" name="SP_WINDOW_OFFSET" type="a6xx_reg_xy" variants="A6XX" usage="rp_blit"/>
+
+	<reg32 offset="0xb2cd" name="SP_PS_UNKNOWN_B4CD" low="6" high="31" variants="A7XX"/>
+	<reg32 offset="0xb2ce" name="SP_PS_UNKNOWN_B4CE" low="0" high="31" variants="A7XX"/>
+	<reg32 offset="0xb2cf" name="SP_PS_UNKNOWN_B4CF" low="0" high="30" variants="A7XX"/>
+	<reg32 offset="0xb2d0" name="SP_PS_UNKNOWN_B4D0" low="0" high="29" variants="A7XX"/>
+	<reg32 offset="0xb2d1" name="SP_PS_2D_WINDOW_OFFSET" type="a6xx_reg_xy" variants="A7XX"/>
+	<reg32 offset="0xb2d2" name="SP_PS_UNKNOWN_B2D2" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xab21" name="SP_WINDOW_OFFSET" type="a6xx_reg_xy" variants="A7XX-" usage="rp_blit"/>
+
+	<!-- always 0x100000 or 0x1000000? -->
+	<reg32 offset="0xb600" name="TPL1_DBG_ECO_CNTL" low="0" high="25" usage="cmd"/>
+	<reg32 offset="0xb601" name="TPL1_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0xb602" name="TPL1_DBG_ECO_CNTL1" usage="cmd"/>
+	<reg32 offset="0xb604" name="TPL1_NC_MODE_CNTL">
+		<bitfield name="MODE" pos="0" type="boolean"/>
+		<bitfield name="LOWER_BIT" low="1" high="2" type="uint"/>
+		<bitfield name="MIN_ACCESS_LENGTH" pos="3" type="boolean"/> <!-- true=64b false=32b -->
+		<bitfield name="UPPER_BIT" pos="4" type="uint"/>
+		<bitfield name="UNK6" low="6" high="7"/>
+	</reg32>
+	<reg32 offset="0xb605" name="TPL1_UNKNOWN_B605" low="0" high="7" type="uint" variants="A6XX" usage="cmd"/> <!-- always 0x0 or 0x44 ? -->
+
+	<reg32 offset="0xb608" name="TPL1_BICUBIC_WEIGHTS_TABLE_0" low="0" high="29" variants="A6XX"/>
+	<reg32 offset="0xb609" name="TPL1_BICUBIC_WEIGHTS_TABLE_1" low="0" high="29" variants="A6XX"/>
+	<reg32 offset="0xb60a" name="TPL1_BICUBIC_WEIGHTS_TABLE_2" low="0" high="29" variants="A6XX"/>
+	<reg32 offset="0xb60b" name="TPL1_BICUBIC_WEIGHTS_TABLE_3" low="0" high="29" variants="A6XX"/>
+	<reg32 offset="0xb60c" name="TPL1_BICUBIC_WEIGHTS_TABLE_4" low="0" high="29" variants="A6XX"/>
+
+	<reg32 offset="0xb608" name="TPL1_BICUBIC_WEIGHTS_TABLE_0" low="0" high="29" variants="A7XX" usage="cmd"/>
+	<reg32 offset="0xb609" name="TPL1_BICUBIC_WEIGHTS_TABLE_1" low="0" high="29" variants="A7XX" usage="cmd"/>
+	<reg32 offset="0xb60a" name="TPL1_BICUBIC_WEIGHTS_TABLE_2" low="0" high="29" variants="A7XX" usage="cmd"/>
+	<reg32 offset="0xb60b" name="TPL1_BICUBIC_WEIGHTS_TABLE_3" low="0" high="29" variants="A7XX" usage="cmd"/>
+	<reg32 offset="0xb60c" name="TPL1_BICUBIC_WEIGHTS_TABLE_4" low="0" high="29" variants="A7XX" usage="cmd"/>
+
+	<array offset="0xb610" name="TPL1_PERFCTR_TP_SEL" stride="1" length="12"/>
+
+	<!-- TODO: 4 more perfcntr sel at 0xb620 ? -->
+
+	<bitset name="a6xx_hlsq_xs_cntl" inline="yes">
+		<bitfield name="CONSTLEN" low="0" high="7" shr="2" type="uint"/>
+		<bitfield name="ENABLED" pos="8" type="boolean"/>
+		<bitfield name="READ_IMM_SHARED_CONSTS" pos="9" type="boolean" variants="A7XX-"/>
+	</bitset>
+
+	<reg32 offset="0xb800" name="HLSQ_VS_CNTL" type="a6xx_hlsq_xs_cntl" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb801" name="HLSQ_HS_CNTL" type="a6xx_hlsq_xs_cntl" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb802" name="HLSQ_DS_CNTL" type="a6xx_hlsq_xs_cntl" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb803" name="HLSQ_GS_CNTL" type="a6xx_hlsq_xs_cntl" variants="A6XX" usage="rp_blit"/>
+
+	<reg32 offset="0xa827" name="HLSQ_VS_CNTL" type="a6xx_hlsq_xs_cntl" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xa83f" name="HLSQ_HS_CNTL" type="a6xx_hlsq_xs_cntl" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xa867" name="HLSQ_DS_CNTL" type="a6xx_hlsq_xs_cntl" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xa898" name="HLSQ_GS_CNTL" type="a6xx_hlsq_xs_cntl" variants="A7XX-" usage="rp_blit"/>
+
+	<reg32 offset="0xa9aa" name="HLSQ_FS_UNKNOWN_A9AA" variants="A7XX-" usage="rp_blit">
+		<!-- Tentatively named, appears to disable consts being loaded via CP_LOAD_STATE6_FRAG -->
+		<bitfield name="CONSTS_LOAD_DISABLE" pos="0" type="boolean"/>
+	</reg32>
+
+	<!-- Always 0 -->
+	<reg32 offset="0xa9ac" name="HLSQ_UNKNOWN_A9AC" variants="A7XX-" usage="cmd"/>
+
+	<!-- Used in VK_KHR_fragment_shading_rate -->
+	<reg32 offset="0xa9ad" name="HLSQ_UNKNOWN_A9AD" variants="A7XX-" usage="cmd"/>
+
+	<reg32 offset="0xa9ae" name="HLSQ_UNKNOWN_A9AE" variants="A7XX-" usage="rp_blit">
+		<bitfield name="SYSVAL_REGS_COUNT" low="0" high="7" type="uint"/>
+		<!-- UNK8 is set on a730/a740 -->
+		<bitfield name="UNK8" pos="8" type="boolean"/>
+		<!-- UNK9 is set on a750 -->
+		<bitfield name="UNK9" pos="9" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0xb820" name="HLSQ_LOAD_STATE_GEOM_CMD"/>
+	<reg64 offset="0xb821" name="HLSQ_LOAD_STATE_GEOM_EXT_SRC_ADDR" align="16" type="address"/>
+	<reg32 offset="0xb823" name="HLSQ_LOAD_STATE_GEOM_DATA"/>
+
+
+	<bitset name="a6xx_hlsq_fs_cntl_0" inline="yes">
+		<!-- must match SP_FS_CTRL -->
+		<bitfield name="THREADSIZE" pos="0" type="a6xx_threadsize"/>
+		<bitfield name="VARYINGS" pos="1" type="boolean"/>
+		<bitfield name="UNK2" low="2" high="11"/>
+	</bitset>
+	<bitset name="a6xx_hlsq_control_3_reg" inline="yes">
+		<!-- register loaded with position (bary.f) -->
+		<bitfield name="IJ_PERSP_PIXEL" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="IJ_LINEAR_PIXEL" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="IJ_PERSP_CENTROID" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="IJ_LINEAR_CENTROID" low="24" high="31" type="a3xx_regid"/>
+	</bitset>
+	<bitset name="a6xx_hlsq_control_4_reg" inline="yes">
+		<bitfield name="IJ_PERSP_SAMPLE" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="IJ_LINEAR_SAMPLE" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="XYCOORDREGID" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="ZWCOORDREGID" low="24" high="31" type="a3xx_regid"/>
+	</bitset>
+	<bitset name="a6xx_hlsq_control_5_reg" inline="yes">
+		<bitfield name="LINELENGTHREGID" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="FOVEATIONQUALITYREGID" low="8" high="15" type="a3xx_regid"/>
+	</bitset>
+
+	<reg32 offset="0xb980" type="a6xx_hlsq_fs_cntl_0" name="HLSQ_FS_CNTL_0" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb981" name="HLSQ_UNKNOWN_B981" pos="0" type="boolean" variants="A6XX"/> <!-- never used by blob -->
+	<reg32 offset="0xb982" name="HLSQ_CONTROL_1_REG" low="0" high="2" variants="A6XX" usage="rp_blit">
+		<!-- Sets the maximum number of primitives allowed in one FS wave minus one, similarly to the
+				 A3xx field, except that it's not necessary to set it to anything but the maximum, since
+				 the hardware will simply emit smaller waves when it runs out of space.	-->
+		<bitfield name="PRIMALLOCTHRESHOLD" low="0" high="2" type="uint"/>
+	</reg32>
+	<reg32 offset="0xb983" name="HLSQ_CONTROL_2_REG" variants="A6XX" usage="rp_blit">
+		<bitfield name="FACEREGID" low="0" high="7" type="a3xx_regid"/>
+		<!-- SAMPLEID is loaded into a half-precision register: -->
+		<bitfield name="SAMPLEID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="SAMPLEMASK" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="CENTERRHW" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xb984" type="a6xx_hlsq_control_3_reg" name="HLSQ_CONTROL_3_REG" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb985" type="a6xx_hlsq_control_4_reg" name="HLSQ_CONTROL_4_REG" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb986" type="a6xx_hlsq_control_5_reg" name="HLSQ_CONTROL_5_REG" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb987" name="HLSQ_CS_CNTL" type="a6xx_hlsq_xs_cntl" variants="A6XX" usage="cmd"/>
+	<reg32 offset="0xa9c6" type="a6xx_hlsq_fs_cntl_0" name="HLSQ_FS_CNTL_0" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xa9c7" name="HLSQ_CONTROL_1_REG" low="0" high="2" variants="A7XX-" usage="rp_blit">
+			<bitfield name="PRIMALLOCTHRESHOLD" low="0" high="2" type="uint"/>
+	</reg32>
+	<reg32 offset="0xa9c8" name="HLSQ_CONTROL_2_REG" variants="A7XX-" usage="rp_blit">
+		<bitfield name="FACEREGID" low="0" high="7" type="a3xx_regid"/>
+		<!-- SAMPLEID is loaded into a half-precision register: -->
+		<bitfield name="SAMPLEID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="SAMPLEMASK" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="CENTERRHW" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xa9c9" type="a6xx_hlsq_control_3_reg" name="HLSQ_CONTROL_3_REG" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xa9ca" type="a6xx_hlsq_control_4_reg" name="HLSQ_CONTROL_4_REG" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xa9cb" type="a6xx_hlsq_control_5_reg" name="HLSQ_CONTROL_5_REG" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xa9cd" name="HLSQ_CS_CNTL" type="a6xx_hlsq_xs_cntl" variants="A7XX-" usage="cmd"/>
+
+	<!-- TODO: what does KERNELDIM do exactly (blob sets it differently from turnip) -->
+	<reg32 offset="0xb990" name="HLSQ_CS_NDRANGE_0" variants="A6XX" usage="rp_blit">
+		<bitfield name="KERNELDIM" low="0" high="1" type="uint"/>
+		<!-- localsize is value minus one: -->
+		<bitfield name="LOCALSIZEX" low="2" high="11" type="uint"/>
+		<bitfield name="LOCALSIZEY" low="12" high="21" type="uint"/>
+		<bitfield name="LOCALSIZEZ" low="22" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xb991" name="HLSQ_CS_NDRANGE_1" variants="A6XX" usage="rp_blit">
+		<bitfield name="GLOBALSIZE_X" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xb992" name="HLSQ_CS_NDRANGE_2" variants="A6XX" usage="rp_blit">
+		<bitfield name="GLOBALOFF_X" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xb993" name="HLSQ_CS_NDRANGE_3" variants="A6XX" usage="rp_blit">
+		<bitfield name="GLOBALSIZE_Y" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xb994" name="HLSQ_CS_NDRANGE_4" variants="A6XX" usage="rp_blit">
+		<bitfield name="GLOBALOFF_Y" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xb995" name="HLSQ_CS_NDRANGE_5" variants="A6XX" usage="rp_blit">
+		<bitfield name="GLOBALSIZE_Z" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xb996" name="HLSQ_CS_NDRANGE_6" variants="A6XX" usage="rp_blit">
+		<bitfield name="GLOBALOFF_Z" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xb997" name="HLSQ_CS_CNTL_0" variants="A6XX" usage="rp_blit">
+		<!-- these are all vec3. first 3 need to be high regs
+		     WGSIZECONSTID is the local size (from HLSQ_CS_NDRANGE_0)
+		     WGOFFSETCONSTID is WGIDCONSTID*WGSIZECONSTID
+		-->
+		<bitfield name="WGIDCONSTID" low="0" high="7" type="a3xx_regid"/>
+		<bitfield name="WGSIZECONSTID" low="8" high="15" type="a3xx_regid"/>
+		<bitfield name="WGOFFSETCONSTID" low="16" high="23" type="a3xx_regid"/>
+		<bitfield name="LOCALIDREGID" low="24" high="31" type="a3xx_regid"/>
+	</reg32>
+	<reg32 offset="0xb998" name="HLSQ_CS_CNTL_1" variants="A6XX" usage="rp_blit">
+		<!-- gl_LocalInvocationIndex -->
+		<bitfield name="LINEARLOCALIDREGID" low="0" high="7" type="a3xx_regid"/>
+		<!-- a650 has 6 "SP cores" (but 3 "SP"). this makes it use only
+		     one of those 6 "SP cores" -->
+		<bitfield name="SINGLE_SP_CORE" pos="8" type="boolean"/>
+		<!-- Must match SP_CS_CTRL -->
+		<bitfield name="THREADSIZE" pos="9" type="a6xx_threadsize"/>
+		<!-- 1 thread per wave (ignored if bit9 set) -->
+		<bitfield name="THREADSIZE_SCALAR" pos="10" type="boolean"/>
+	</reg32>
+	<!--note: vulkan blob doesn't use these -->
+	<reg32 offset="0xb999" name="HLSQ_CS_KERNEL_GROUP_X" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb99a" name="HLSQ_CS_KERNEL_GROUP_Y" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xb99b" name="HLSQ_CS_KERNEL_GROUP_Z" variants="A6XX" usage="rp_blit"/>
+
+	<!-- TODO: what does KERNELDIM do exactly (blob sets it differently from turnip) -->
+	<reg32 offset="0xa9d4" name="HLSQ_CS_NDRANGE_0" variants="A7XX-" usage="rp_blit">
+		<bitfield name="KERNELDIM" low="0" high="1" type="uint"/>
+		<!-- localsize is value minus one: -->
+		<bitfield name="LOCALSIZEX" low="2" high="11" type="uint"/>
+		<bitfield name="LOCALSIZEY" low="12" high="21" type="uint"/>
+		<bitfield name="LOCALSIZEZ" low="22" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xa9d5" name="HLSQ_CS_NDRANGE_1" variants="A7XX-" usage="rp_blit">
+		<bitfield name="GLOBALSIZE_X" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xa9d6" name="HLSQ_CS_NDRANGE_2" variants="A7XX-" usage="rp_blit">
+		<bitfield name="GLOBALOFF_X" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xa9d7" name="HLSQ_CS_NDRANGE_3" variants="A7XX-" usage="rp_blit">
+		<bitfield name="GLOBALSIZE_Y" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xa9d8" name="HLSQ_CS_NDRANGE_4" variants="A7XX-" usage="rp_blit">
+		<bitfield name="GLOBALOFF_Y" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xa9d9" name="HLSQ_CS_NDRANGE_5" variants="A7XX-" usage="rp_blit">
+		<bitfield name="GLOBALSIZE_Z" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0xa9da" name="HLSQ_CS_NDRANGE_6" variants="A7XX-" usage="rp_blit">
+		<bitfield name="GLOBALOFF_Z" low="0" high="31" type="uint"/>
+	</reg32>
+	<!--note: vulkan blob doesn't use these -->
+	<reg32 offset="0xa9dc" name="HLSQ_CS_KERNEL_GROUP_X" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xa9dd" name="HLSQ_CS_KERNEL_GROUP_Y" variants="A7XX-" usage="rp_blit"/>
+	<reg32 offset="0xa9de" name="HLSQ_CS_KERNEL_GROUP_Z" variants="A7XX-" usage="rp_blit"/>
+
+	<enum name="a7xx_cs_yalign">
+		<value name="CS_YALIGN_1" value="8"/>
+		<value name="CS_YALIGN_2" value="4"/>
+		<value name="CS_YALIGN_4" value="2"/>
+		<value name="CS_YALIGN_8" value="1"/>
+	</enum>
+
+	<reg32 offset="0xa9db" name="HLSQ_CS_CNTL_1" variants="A7XX-" usage="rp_blit">
+		<!-- gl_LocalInvocationIndex -->
+		<bitfield name="LINEARLOCALIDREGID" low="0" high="7" type="a3xx_regid"/>
+		<!-- Must match SP_CS_CTRL -->
+		<bitfield name="THREADSIZE" pos="9" type="a6xx_threadsize"/>
+		<bitfield name="UNK11" pos="11" type="boolean"/>
+		<bitfield name="UNK22" pos="22" type="boolean"/>
+		<bitfield name="UNK26" pos="26" type="boolean"/>
+		<bitfield name="YALIGN" low="27" high="30" type="a7xx_cs_yalign"/>
+	</reg32>
+
+	<reg32 offset="0xa9df" name="HLSQ_CS_LOCAL_SIZE" variants="A7XX-" usage="cmd">
+		<!-- localsize is value minus one: -->
+		<bitfield name="LOCALSIZEX" low="2" high="11" type="uint"/>
+		<bitfield name="LOCALSIZEY" low="12" high="21" type="uint"/>
+		<bitfield name="LOCALSIZEZ" low="22" high="31" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0xb9a0" name="HLSQ_LOAD_STATE_FRAG_CMD"/>
+	<reg64 offset="0xb9a1" name="HLSQ_LOAD_STATE_FRAG_EXT_SRC_ADDR" align="16" type="address"/>
+	<reg32 offset="0xb9a3" name="HLSQ_LOAD_STATE_FRAG_DATA"/>
+
+	<!-- mirror of SP_CS_BINDLESS_BASE -->
+	<array offset="0xb9c0" name="HLSQ_CS_BINDLESS_BASE" stride="2" length="5" variants="A6XX" usage="rp_blit">
+		<reg64 offset="0" name="DESCRIPTOR">
+			<bitfield name="DESC_SIZE" low="0" high="1" type="a6xx_bindless_descriptor_size"/>
+			<bitfield name="ADDR" low="2" high="63" shr="2" type="address"/>
+		</reg64>
+	</array>
+
+	<!-- new in a6xx gen4, mirror of SP_CS_UNKNOWN_A9B1? -->
+	<reg32 offset="0xb9d0" name="HLSQ_CS_UNKNOWN_B9D0" variants="A6XX" usage="cmd">
+		<bitfield name="SHARED_SIZE" low="0" high="4" type="uint"/>
+		<bitfield name="UNK5" pos="5" type="boolean"/>
+		<!-- always 1 ? -->
+		<bitfield name="UNK6" pos="6" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0xbb00" name="HLSQ_DRAW_CMD">
+		<bitfield name="STATE_ID" low="0" high="7"/>
+	</reg32>
+
+	<reg32 offset="0xbb01" name="HLSQ_DISPATCH_CMD">
+		<bitfield name="STATE_ID" low="0" high="7"/>
+	</reg32>
+
+	<reg32 offset="0xbb02" name="HLSQ_EVENT_CMD">
+		<!-- I think only the low bit is actually used? -->
+		<bitfield name="STATE_ID" low="16" high="23"/>
+		<bitfield name="EVENT" low="0" high="6" type="vgt_event_type"/>
+	</reg32>
+
+	<reg32 offset="0xbb08" name="HLSQ_INVALIDATE_CMD" variants="A6XX" usage="cmd">
+		<doc>
+			This register clears pending loads queued up by
+			CP_LOAD_STATE6. Each bit resets a particular kind(s) of
+			CP_LOAD_STATE6.
+		</doc>
+
+		<!-- per-stage state: shader, non-bindless UBO, textures, and samplers -->
+		<bitfield name="VS_STATE" pos="0" type="boolean"/>
+		<bitfield name="HS_STATE" pos="1" type="boolean"/>
+		<bitfield name="DS_STATE" pos="2" type="boolean"/>
+		<bitfield name="GS_STATE" pos="3" type="boolean"/>
+		<bitfield name="FS_STATE" pos="4" type="boolean"/>
+		<bitfield name="CS_STATE" pos="5" type="boolean"/>
+
+		<bitfield name="CS_IBO" pos="6" type="boolean"/>
+		<bitfield name="GFX_IBO" pos="7" type="boolean"/>
+
+		<!-- Note: these only do something when HLSQ_SHARED_CONSTS is set to 1 -->
+		<bitfield name="CS_SHARED_CONST" pos="19" type="boolean"/>
+		<bitfield name="GFX_SHARED_CONST" pos="8" type="boolean"/>
+
+		<!-- SS6_BINDLESS: one bit per bindless base -->
+		<bitfield name="CS_BINDLESS" low="9" high="13" type="hex"/>
+		<bitfield name="GFX_BINDLESS" low="14" high="18" type="hex"/>
+	</reg32>
+
+	<reg32 offset="0xab1f" name="HLSQ_INVALIDATE_CMD" variants="A7XX-" usage="cmd">
+		<doc>
+			This register clears pending loads queued up by
+			CP_LOAD_STATE6. Each bit resets a particular kind(s) of
+			CP_LOAD_STATE6.
+		</doc>
+
+		<!-- per-stage state: shader, non-bindless UBO, textures, and samplers -->
+		<bitfield name="VS_STATE" pos="0" type="boolean"/>
+		<bitfield name="HS_STATE" pos="1" type="boolean"/>
+		<bitfield name="DS_STATE" pos="2" type="boolean"/>
+		<bitfield name="GS_STATE" pos="3" type="boolean"/>
+		<bitfield name="FS_STATE" pos="4" type="boolean"/>
+		<bitfield name="CS_STATE" pos="5" type="boolean"/>
+
+		<bitfield name="CS_IBO" pos="6" type="boolean"/>
+		<bitfield name="GFX_IBO" pos="7" type="boolean"/>
+
+		<!-- SS6_BINDLESS: one bit per bindless base -->
+		<bitfield name="CS_BINDLESS" low="9" high="16" type="hex"/>
+		<bitfield name="GFX_BINDLESS" low="17" high="24" type="hex"/>
+	</reg32>
+
+	<reg32 offset="0xbb10" name="HLSQ_FS_CNTL" type="a6xx_hlsq_xs_cntl" variants="A6XX" usage="rp_blit"/>
+	<reg32 offset="0xab03" name="HLSQ_FS_CNTL" type="a6xx_hlsq_xs_cntl" variants="A7XX-" usage="rp_blit"/>
+
+	<array offset="0xab40" name="HLSQ_SHARED_CONSTS_IMM" stride="1" length="64" variants="A7XX-"/>
+
+	<reg32 offset="0xbb11" name="HLSQ_SHARED_CONSTS" variants="A6XX" usage="cmd">
+		<doc>
+			Shared constants are intended to be used for Vulkan push
+			constants. When enabled, 8 vec4's are reserved in the FS
+			const pool and 16 in the geometry const pool although
+			only 8 are actually used (why?) and they are mapped to
+			c504-c511 in each stage. Both VS and FS shared consts
+			are written using ST6_CONSTANTS/SB6_IBO, so that both
+			the geometry and FS shared consts can be written at once
+			by using CP_LOAD_STATE6 rather than
+			CP_LOAD_STATE6_FRAG/CP_LOAD_STATE6_GEOM. In addition
+			DST_OFF and NUM_UNIT are in units of dwords instead of
+			vec4's.
+
+			There is also a separate shared constant pool for CS,
+			which is loaded through CP_LOAD_STATE6_FRAG with
+			ST6_UBO/ST6_IBO. However the only real difference for CS
+			is the dword units.
+		</doc>
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+	</reg32>
+
+	<!-- mirror of SP_BINDLESS_BASE -->
+	<array offset="0xbb20" name="HLSQ_BINDLESS_BASE" stride="2" length="5" variants="A6XX" usage="cmd">
+		<reg64 offset="0" name="DESCRIPTOR">
+			<bitfield name="DESC_SIZE" low="0" high="1" type="a6xx_bindless_descriptor_size"/>
+			<bitfield name="ADDR" low="2" high="63" shr="2" type="address"/>
+		</reg64>
+	</array>
+
+	<reg32 offset="0xbd80" name="HLSQ_2D_EVENT_CMD">
+		<bitfield name="STATE_ID" low="8" high="15"/>
+		<bitfield name="EVENT" low="0" high="6" type="vgt_event_type"/>
+	</reg32>
+
+	<reg32 offset="0xbe00" name="HLSQ_UNKNOWN_BE00" variants="A6XX" usage="cmd"/> <!-- all bits valid except bit 29 -->
+	<reg32 offset="0xbe01" name="HLSQ_UNKNOWN_BE01" low="4" high="6" variants="A6XX" usage="cmd"/>
+	<reg32 offset="0xbe04" name="HLSQ_DBG_ECO_CNTL" variants="A6XX" usage="cmd"/>
+	<reg32 offset="0xbe05" name="HLSQ_ADDR_MODE_CNTL" type="a5xx_address_mode"/>
+	<reg32 offset="0xbe08" name="HLSQ_UNKNOWN_BE08" low="0" high="15"/>
+	<array offset="0xbe10" name="HLSQ_PERFCTR_HLSQ_SEL" stride="1" length="6"/>
+
+	<!-- TODO: some valid registers between 0xbe20 and 0xbe33 -->
+	<reg32 offset="0xbe22" name="HLSQ_CONTEXT_SWITCH_GFX_PREEMPTION_SAFE_MODE"/>
+
+	<reg32 offset="0xc000" name="SP_AHB_READ_APERTURE" variants="A7XX-"/>
+
+	<!-- Don't know if these are SP, always 0 -->
+	<reg64 offset="0x0ce2" name="SP_UNKNOWN_0CE2" variants="A7XX-" usage="cmd"/>
+	<reg64 offset="0x0ce4" name="SP_UNKNOWN_0CE4" variants="A7XX-" usage="cmd"/>
+	<reg64 offset="0x0ce6" name="SP_UNKNOWN_0CE6" variants="A7XX-" usage="cmd"/>
+
+	<!--
+		These special registers signal the beginning/end of an event
+		sequence. The sequence used internally for an event looks like:
+		- write EVENT_CMD pipe register
+		- write CP_EVENT_START
+		- write HLSQ_EVENT_CMD with event or HLSQ_DRAW_CMD
+		- write PC_EVENT_CMD with event or PC_DRAW_CMD
+		- write HLSQ_EVENT_CMD(CONTEXT_DONE)
+		- write PC_EVENT_CMD(CONTEXT_DONE)
+		- write CP_EVENT_END
+		Writing to CP_EVENT_END seems to actually trigger the context roll
+	-->
+	<reg32 offset="0xd600" name="CP_EVENT_START">
+		<bitfield name="STATE_ID" low="0" high="7"/>
+	</reg32>
+	<reg32 offset="0xd601" name="CP_EVENT_END">
+		<bitfield name="STATE_ID" low="0" high="7"/>
+	</reg32>
+	<reg32 offset="0xd700" name="CP_2D_EVENT_START">
+		<bitfield name="STATE_ID" low="0" high="7"/>
+	</reg32>
+	<reg32 offset="0xd701" name="CP_2D_EVENT_END">
+		<bitfield name="STATE_ID" low="0" high="7"/>
+	</reg32>
+</domain>
+
+<!-- Seems basically the same as a5xx, maybe move to common.xml.. -->
+<domain name="A6XX_TEX_SAMP" width="32">
+	<doc>Texture sampler dwords</doc>
+	<enum name="a6xx_tex_filter"> <!-- same as a4xx? -->
+		<value name="A6XX_TEX_NEAREST" value="0"/>
+		<value name="A6XX_TEX_LINEAR" value="1"/>
+		<value name="A6XX_TEX_ANISO" value="2"/>
+		<value name="A6XX_TEX_CUBIC" value="3"/> <!-- a650 only -->
+	</enum>
+	<enum name="a6xx_tex_clamp"> <!-- same as a4xx? -->
+		<value name="A6XX_TEX_REPEAT" value="0"/>
+		<value name="A6XX_TEX_CLAMP_TO_EDGE" value="1"/>
+		<value name="A6XX_TEX_MIRROR_REPEAT" value="2"/>
+		<value name="A6XX_TEX_CLAMP_TO_BORDER" value="3"/>
+		<value name="A6XX_TEX_MIRROR_CLAMP" value="4"/>
+	</enum>
+	<enum name="a6xx_tex_aniso"> <!-- same as a4xx? -->
+		<value name="A6XX_TEX_ANISO_1" value="0"/>
+		<value name="A6XX_TEX_ANISO_2" value="1"/>
+		<value name="A6XX_TEX_ANISO_4" value="2"/>
+		<value name="A6XX_TEX_ANISO_8" value="3"/>
+		<value name="A6XX_TEX_ANISO_16" value="4"/>
+	</enum>
+	<enum name="a6xx_reduction_mode">
+		<value name="A6XX_REDUCTION_MODE_AVERAGE" value="0"/>
+		<value name="A6XX_REDUCTION_MODE_MIN" value="1"/>
+		<value name="A6XX_REDUCTION_MODE_MAX" value="2"/>
+	</enum>
+
+	<reg32 offset="0" name="0">
+		<bitfield name="MIPFILTER_LINEAR_NEAR" pos="0" type="boolean"/>
+		<bitfield name="XY_MAG" low="1" high="2" type="a6xx_tex_filter"/>
+		<bitfield name="XY_MIN" low="3" high="4" type="a6xx_tex_filter"/>
+		<bitfield name="WRAP_S" low="5" high="7" type="a6xx_tex_clamp"/>
+		<bitfield name="WRAP_T" low="8" high="10" type="a6xx_tex_clamp"/>
+		<bitfield name="WRAP_R" low="11" high="13" type="a6xx_tex_clamp"/>
+		<bitfield name="ANISO" low="14" high="16" type="a6xx_tex_aniso"/>
+		<bitfield name="LOD_BIAS" low="19" high="31" type="fixed" radix="8"/><!-- no idea how many bits for real -->
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="CLAMPENABLE" pos="0" type="boolean">
+			<doc>
+				clamp result to [0, 1] if the format is unorm or
+				[-1, 1] if the format is snorm, *after*
+				filtering. Has no effect for other formats.
+			</doc>
+		</bitfield>
+		<bitfield name="COMPARE_FUNC" low="1" high="3" type="adreno_compare_func"/>
+		<bitfield name="CUBEMAPSEAMLESSFILTOFF" pos="4" type="boolean"/>
+		<bitfield name="UNNORM_COORDS" pos="5" type="boolean"/>
+		<bitfield name="MIPFILTER_LINEAR_FAR" pos="6" type="boolean"/>
+		<bitfield name="MAX_LOD" low="8" high="19" type="ufixed" radix="8"/>
+		<bitfield name="MIN_LOD" low="20" high="31" type="ufixed" radix="8"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="REDUCTION_MODE" low="0" high="1" type="a6xx_reduction_mode"/>
+		<bitfield name="CHROMA_LINEAR" pos="5" type="boolean"/>
+		<bitfield name="BCOLOR" low="7" high="31"/>
+	</reg32>
+	<reg32 offset="3" name="3"/>
+</domain>
+
+<domain name="A6XX_TEX_CONST" width="32">
+	<doc>Texture constant dwords</doc>
+	<enum name="a6xx_tex_swiz"> <!-- same as a4xx? -->
+		<value name="A6XX_TEX_X" value="0"/>
+		<value name="A6XX_TEX_Y" value="1"/>
+		<value name="A6XX_TEX_Z" value="2"/>
+		<value name="A6XX_TEX_W" value="3"/>
+		<value name="A6XX_TEX_ZERO" value="4"/>
+		<value name="A6XX_TEX_ONE" value="5"/>
+	</enum>
+	<enum name="a6xx_tex_type"> <!-- same as a4xx? -->
+		<value name="A6XX_TEX_1D" value="0"/>
+		<value name="A6XX_TEX_2D" value="1"/>
+		<value name="A6XX_TEX_CUBE" value="2"/>
+		<value name="A6XX_TEX_3D" value="3"/>
+		<value name="A6XX_TEX_BUFFER" value="4"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="TILE_MODE" low="0" high="1" type="a6xx_tile_mode"/>
+		<bitfield name="SRGB" pos="2" type="boolean"/>
+		<bitfield name="SWIZ_X" low="4" high="6" type="a6xx_tex_swiz"/>
+		<bitfield name="SWIZ_Y" low="7" high="9" type="a6xx_tex_swiz"/>
+		<bitfield name="SWIZ_Z" low="10" high="12" type="a6xx_tex_swiz"/>
+		<bitfield name="SWIZ_W" low="13" high="15" type="a6xx_tex_swiz"/>
+		<bitfield name="MIPLVLS" low="16" high="19" type="uint"/>
+		<!-- overlaps with MIPLVLS -->
+		<bitfield name="CHROMA_MIDPOINT_X" pos="16" type="boolean"/>
+		<bitfield name="CHROMA_MIDPOINT_Y" pos="18" type="boolean"/>
+		<bitfield name="SAMPLES" low="20" high="21" type="a3xx_msaa_samples"/>
+		<bitfield name="FMT" low="22" high="29" type="a6xx_format"/>
+		<!--
+			Why is the swap needed in addition to SWIZ_*? The swap
+			is performed before border color replacement, while the
+			swizzle is applied after after it.
+		-->
+		<bitfield name="SWAP" low="30" high="31" type="a3xx_color_swap"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="WIDTH" low="0" high="14" type="uint"/>
+		<bitfield name="HEIGHT" low="15" high="29" type="uint"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<!--
+			These fields overlap PITCH, and are used instead of
+			PITCH/PITCHALIGN when TYPE is A6XX_TEX_BUFFER.
+		 -->
+		<doc> probably for D3D structured UAVs, normally set to 1 </doc>
+		<bitfield name="STRUCTSIZETEXELS" low="4" high="15" type="uint"/>
+		<bitfield name="STARTOFFSETTEXELS" low="16" high="21" type="uint"/>
+
+		<!-- minimum pitch (for mipmap levels): log2(pitchalign / 64) -->
+		<bitfield name="PITCHALIGN" low="0" high="3" type="uint"/>
+		<doc>Pitch in bytes (so actually stride)</doc>
+		<bitfield name="PITCH" low="7" high="28" type="uint"/>
+		<bitfield name="TYPE" low="29" high="31" type="a6xx_tex_type"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<!--
+		ARRAY_PITCH is basically LAYERSZ for the first mipmap level, and
+		for 3d textures (laid out mipmap level first) MIN_LAYERSZ is the
+		layer size at the point that it stops being reduced moving to
+		higher (smaller) mipmap levels
+		 -->
+		<bitfield name="ARRAY_PITCH" low="0" high="22" shr="12" type="uint"/>
+		<bitfield name="MIN_LAYERSZ" low="23" high="26" shr="12"/>
+		<!--
+		by default levels with w < 16 are linear
+		TILE_ALL makes all levels have tiling
+		seems required when using UBWC, since all levels have UBWC (can possibly be disabled?)
+		 -->
+		<bitfield name="TILE_ALL" pos="27" type="boolean"/>
+		<bitfield name="FLAG" pos="28" type="boolean"/>
+	</reg32>
+	<!-- for 2-3 plane format, BASE is flag buffer address (if enabled)
+	     the address of the non-flag base buffer is determined automatically,
+	     and must follow the flag buffer
+	 -->
+	<reg32 offset="4" name="4">
+		<bitfield name="BASE_LO" low="5" high="31" shr="5"/>
+	</reg32>
+	<reg32 offset="5" name="5">
+		<bitfield name="BASE_HI" low="0" high="16"/>
+		<bitfield name="DEPTH" low="17" high="29" type="uint"/>
+	</reg32>
+	<reg32 offset="6" name="6">
+		<!-- overlaps with PLANE_PITCH -->
+		<bitfield name="MIN_LOD_CLAMP" low="0" high="11" type="ufixed" radix="8"/>
+		<!-- pitch for plane 2 / plane 3 -->
+		<bitfield name="PLANE_PITCH" low="8" high="31" type="uint"/>
+	</reg32>
+	<!-- 7/8 is plane 2 address for planar formats -->
+	<reg32 offset="7" name="7">
+		<bitfield name="FLAG_LO" low="5" high="31" shr="5"/>
+	</reg32>
+	<reg32 offset="8" name="8">
+		<bitfield name="FLAG_HI" low="0" high="16"/>
+	</reg32>
+	<!-- 9/10 is plane 3 address for planar formats -->
+	<reg32 offset="9" name="9">
+		<bitfield name="FLAG_BUFFER_ARRAY_PITCH" low="0" high="16" shr="4" type="uint"/>
+	</reg32>
+	<reg32 offset="10" name="10">
+		<bitfield name="FLAG_BUFFER_PITCH" low="0" high="6" shr="6" type="uint"/>
+		<!-- log2 size of the first level, required for mipmapping -->
+		<bitfield name="FLAG_BUFFER_LOGW" low="8" high="11" type="uint"/>
+		<bitfield name="FLAG_BUFFER_LOGH" low="12" high="15" type="uint"/>
+	</reg32>
+	<reg32 offset="11" name="11"/>
+	<reg32 offset="12" name="12"/>
+	<reg32 offset="13" name="13"/>
+	<reg32 offset="14" name="14"/>
+	<reg32 offset="15" name="15"/>
+</domain>
+
+<domain name="A6XX_UBO" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="BASE_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="BASE_HI" low="0" high="16"/>
+		<bitfield name="SIZE" low="17" high="31"/> <!-- size in vec4 (4xDWORD) units -->
+	</reg32>
+</domain>
+
+<domain name="A6XX_PDC" width="32">
+	<reg32 offset="0x1140" name="GPU_ENABLE_PDC"/>
+	<reg32 offset="0x1148" name="GPU_SEQ_START_ADDR"/>
+	<reg32 offset="0x1540" name="GPU_TCS0_CONTROL"/>
+	<reg32 offset="0x1541" name="GPU_TCS0_CMD_ENABLE_BANK"/>
+	<reg32 offset="0x1542" name="GPU_TCS0_CMD_WAIT_FOR_CMPL_BANK"/>
+	<reg32 offset="0x1543" name="GPU_TCS0_CMD0_MSGID"/>
+	<reg32 offset="0x1544" name="GPU_TCS0_CMD0_ADDR"/>
+	<reg32 offset="0x1545" name="GPU_TCS0_CMD0_DATA"/>
+	<reg32 offset="0x1572" name="GPU_TCS1_CONTROL"/>
+	<reg32 offset="0x1573" name="GPU_TCS1_CMD_ENABLE_BANK"/>
+	<reg32 offset="0x1574" name="GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK"/>
+	<reg32 offset="0x1575" name="GPU_TCS1_CMD0_MSGID"/>
+	<reg32 offset="0x1576" name="GPU_TCS1_CMD0_ADDR"/>
+	<reg32 offset="0x1577" name="GPU_TCS1_CMD0_DATA"/>
+	<reg32 offset="0x15A4" name="GPU_TCS2_CONTROL"/>
+	<reg32 offset="0x15A5" name="GPU_TCS2_CMD_ENABLE_BANK"/>
+	<reg32 offset="0x15A6" name="GPU_TCS2_CMD_WAIT_FOR_CMPL_BANK"/>
+	<reg32 offset="0x15A7" name="GPU_TCS2_CMD0_MSGID"/>
+	<reg32 offset="0x15A8" name="GPU_TCS2_CMD0_ADDR"/>
+	<reg32 offset="0x15A9" name="GPU_TCS2_CMD0_DATA"/>
+	<reg32 offset="0x15D6" name="GPU_TCS3_CONTROL"/>
+	<reg32 offset="0x15D7" name="GPU_TCS3_CMD_ENABLE_BANK"/>
+	<reg32 offset="0x15D8" name="GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK"/>
+	<reg32 offset="0x15D9" name="GPU_TCS3_CMD0_MSGID"/>
+	<reg32 offset="0x15DA" name="GPU_TCS3_CMD0_ADDR"/>
+	<reg32 offset="0x15DB" name="GPU_TCS3_CMD0_DATA"/>
+</domain>
+
+<domain name="A6XX_PDC_GPU_SEQ" width="32">
+	<reg32 offset="0x0" name="MEM_0"/>
+</domain>
+
+<domain name="A6XX_CX_DBGC" width="32">
+	<reg32 offset="0x0000" name="CFG_DBGBUS_SEL_A">
+		<bitfield high="7" low="0" name="PING_INDEX"/>
+		<bitfield high="15" low="8" name="PING_BLK_SEL"/>
+	</reg32>
+	<reg32 offset="0x0001" name="CFG_DBGBUS_SEL_B"/>
+	<reg32 offset="0x0002" name="CFG_DBGBUS_SEL_C"/>
+	<reg32 offset="0x0003" name="CFG_DBGBUS_SEL_D"/>
+	<reg32 offset="0x0004" name="CFG_DBGBUS_CNTLT">
+		<bitfield high="5" low="0" name="TRACEEN"/>
+		<bitfield high="14" low="12" name="GRANU"/>
+		<bitfield high="31" low="28" name="SEGT"/>
+	</reg32>
+	<reg32 offset="0x0005" name="CFG_DBGBUS_CNTLM">
+		<bitfield high="27" low="24" name="ENABLE"/>
+	</reg32>
+	<reg32 offset="0x0008" name="CFG_DBGBUS_IVTL_0"/>
+	<reg32 offset="0x0009" name="CFG_DBGBUS_IVTL_1"/>
+	<reg32 offset="0x000a" name="CFG_DBGBUS_IVTL_2"/>
+	<reg32 offset="0x000b" name="CFG_DBGBUS_IVTL_3"/>
+	<reg32 offset="0x000c" name="CFG_DBGBUS_MASKL_0"/>
+	<reg32 offset="0x000d" name="CFG_DBGBUS_MASKL_1"/>
+	<reg32 offset="0x000e" name="CFG_DBGBUS_MASKL_2"/>
+	<reg32 offset="0x000f" name="CFG_DBGBUS_MASKL_3"/>
+	<reg32 offset="0x0010" name="CFG_DBGBUS_BYTEL_0">
+		<bitfield high="3" low="0" name="BYTEL0"/>
+		<bitfield high="7" low="4" name="BYTEL1"/>
+		<bitfield high="11" low="8" name="BYTEL2"/>
+		<bitfield high="15" low="12" name="BYTEL3"/>
+		<bitfield high="19" low="16" name="BYTEL4"/>
+		<bitfield high="23" low="20" name="BYTEL5"/>
+		<bitfield high="27" low="24" name="BYTEL6"/>
+		<bitfield high="31" low="28" name="BYTEL7"/>
+	</reg32>
+	<reg32 offset="0x0011" name="CFG_DBGBUS_BYTEL_1">
+		<bitfield high="3" low="0" name="BYTEL8"/>
+		<bitfield high="7" low="4" name="BYTEL9"/>
+		<bitfield high="11" low="8" name="BYTEL10"/>
+		<bitfield high="15" low="12" name="BYTEL11"/>
+		<bitfield high="19" low="16" name="BYTEL12"/>
+		<bitfield high="23" low="20" name="BYTEL13"/>
+		<bitfield high="27" low="24" name="BYTEL14"/>
+		<bitfield high="31" low="28" name="BYTEL15"/>
+	</reg32>
+
+	<reg32 offset="0x002f" name="CFG_DBGBUS_TRACE_BUF1"/>
+	<reg32 offset="0x0030" name="CFG_DBGBUS_TRACE_BUF2"/>
+</domain>
+
+<domain name="A6XX_CX_MISC" width="32" prefix="variant" varset="chip">
+	<reg32 offset="0x0001" name="SYSTEM_CACHE_CNTL_0"/>
+	<reg32 offset="0x0002" name="SYSTEM_CACHE_CNTL_1"/>
+	<reg32 offset="0x0039" name="CX_MISC_TCM_RET_CNTL" variants="A7XX-"/>
+	<reg32 offset="0x0400" name="CX_MISC_SW_FUSE_VALUE" variants="A7XX-">
+		<bitfield pos="0" name="FASTBLEND" type="boolean"/>
+		<bitfield pos="1" name="LPAC" type="boolean"/>
+		<bitfield pos="2" name="RAYTRACING" type="boolean"/>
+	</reg32>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml b/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml
new file mode 100644
index 000000000000..6531749d30f4
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml
@@ -0,0 +1,228 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+<import file="adreno/adreno_common.xml"/>
+
+<domain name="A6XX" width="32" prefix="variant" varset="chip">
+
+	<bitset name="A6XX_GMU_GPU_IDLE_STATUS">
+		<bitfield name="BUSY_IGN_AHB" pos="23"/>
+		<bitfield name="CX_GX_CPU_BUSY_IGN_AHB" pos="30"/>
+	</bitset>
+
+	<bitset name="A6XX_GMU_OOB">
+		<bitfield name="BOOT_SLUMBER_SET_MASK" pos="22"/>
+		<bitfield name="BOOT_SLUMBER_CHECK_MASK" pos="30"/>
+		<bitfield name="BOOT_SLUMBER_CLEAR_MASK" pos="30"/>
+		<bitfield name="DCVS_SET_MASK" pos="23"/>
+		<bitfield name="DCVS_CHECK_MASK" pos="31"/>
+		<bitfield name="DCVS_CLEAR_MASK" pos="31"/>
+		<bitfield name="GPU_SET_MASK" pos="18"/>
+		<bitfield name="GPU_CHECK_MASK" pos="26"/>
+		<bitfield name="GPU_CLEAR_MASK" pos="26"/>
+		<bitfield name="PERFCNTR_SET_MASK" pos="17"/>
+		<bitfield name="PERFCNTR_CHECK_MASK" pos="25"/>
+		<bitfield name="PERFCNTR_CLEAR_MASK" pos="25"/>
+	</bitset>
+
+	<bitset name="A6XX_HFI_IRQ">
+		<bitfield name="MSGQ_MASK" pos="0" />
+		<bitfield name="DSGQ_MASK" pos="1"/>
+		<bitfield name="BLOCKED_MSG_MASK" pos="2"/>
+		<bitfield name="CM3_FAULT_MASK" pos="23"/>
+		<bitfield name="GMU_ERR_MASK" low="16" high="22"/>
+		<bitfield name="OOB_MASK" low="24" high="31"/>
+	</bitset>
+
+	<bitset name="A6XX_HFI_H2F">
+		<bitfield name="IRQ_MASK_BIT" pos="0" />
+	</bitset>
+
+	<reg32 offset="0x80" name="GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL"/>
+	<reg32 offset="0x81" name="GMU_GX_SPTPRAC_POWER_CONTROL"/>
+	<reg32 offset="0xc00" name="GMU_CM3_ITCM_START"/>
+	<reg32 offset="0x1c00" name="GMU_CM3_DTCM_START"/>
+	<reg32 offset="0x23f0" name="GMU_NMI_CONTROL_STATUS"/>
+	<reg32 offset="0x23f8" name="GMU_BOOT_SLUMBER_OPTION"/>
+	<reg32 offset="0x23f9" name="GMU_GX_VOTE_IDX"/>
+	<reg32 offset="0x23fa" name="GMU_MX_VOTE_IDX"/>
+	<reg32 offset="0x23fc" name="GMU_DCVS_ACK_OPTION"/>
+	<reg32 offset="0x23fd" name="GMU_DCVS_PERF_SETTING"/>
+	<reg32 offset="0x23fe" name="GMU_DCVS_BW_SETTING"/>
+	<reg32 offset="0x23ff" name="GMU_DCVS_RETURN"/>
+	<reg32 offset="0x4c00" name="GMU_ICACHE_CONFIG"/>
+	<reg32 offset="0x4c01" name="GMU_DCACHE_CONFIG"/>
+	<reg32 offset="0x4c0f" name="GMU_SYS_BUS_CONFIG"/>
+	<reg32 offset="0x5000" name="GMU_CM3_SYSRESET"/>
+	<reg32 offset="0x5001" name="GMU_CM3_BOOT_CONFIG"/>
+	<reg32 offset="0x501a" name="GMU_CM3_FW_BUSY"/>
+	<reg32 offset="0x501c" name="GMU_CM3_FW_INIT_RESULT"/>
+	<reg32 offset="0x502d" name="GMU_CM3_CFG"/>
+	<reg32 offset="0x5040" name="GMU_CX_GMU_POWER_COUNTER_ENABLE"/>
+	<reg32 offset="0x5041" name="GMU_CX_GMU_POWER_COUNTER_SELECT_0"/>
+	<reg32 offset="0x5042" name="GMU_CX_GMU_POWER_COUNTER_SELECT_1"/>
+	<reg32 offset="0x5044" name="GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L"/>
+	<reg32 offset="0x5045" name="GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H"/>
+	<reg32 offset="0x5046" name="GMU_CX_GMU_POWER_COUNTER_XOCLK_1_L"/>
+	<reg32 offset="0x5047" name="GMU_CX_GMU_POWER_COUNTER_XOCLK_1_H"/>
+	<reg32 offset="0x5048" name="GMU_CX_GMU_POWER_COUNTER_XOCLK_2_L"/>
+	<reg32 offset="0x5049" name="GMU_CX_GMU_POWER_COUNTER_XOCLK_2_H"/>
+	<reg32 offset="0x504a" name="GMU_CX_GMU_POWER_COUNTER_XOCLK_3_L"/>
+	<reg32 offset="0x504b" name="GMU_CX_GMU_POWER_COUNTER_XOCLK_3_H"/>
+	<reg32 offset="0x504c" name="GMU_CX_GMU_POWER_COUNTER_XOCLK_4_L"/>
+	<reg32 offset="0x504d" name="GMU_CX_GMU_POWER_COUNTER_XOCLK_4_H"/>
+	<reg32 offset="0x504e" name="GMU_CX_GMU_POWER_COUNTER_XOCLK_5_L"/>
+	<reg32 offset="0x504f" name="GMU_CX_GMU_POWER_COUNTER_XOCLK_5_H"/>
+	<reg32 offset="0x50c0" name="GMU_PWR_COL_INTER_FRAME_CTRL">
+		<bitfield name="IFPC_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="HM_POWER_COLLAPSE_ENABLE" pos="1" type="boolean"/>
+		<bitfield name="SPTPRAC_POWER_CONTROL_ENABLE" pos="2" type="boolean"/>
+		<bitfield name="NUM_PASS_SKIPS" low="10" high="13"/>
+		<bitfield name="MIN_PASS_LENGTH" low="14" high="31"/>
+	</reg32>
+	<reg32 offset="0x50c1" name="GMU_PWR_COL_INTER_FRAME_HYST"/>
+	<reg32 offset="0x50c2" name="GMU_PWR_COL_SPTPRAC_HYST"/>
+	<reg32 offset="0x50d0" name="GMU_SPTPRAC_PWR_CLK_STATUS">
+		<bitfield name="SPTPRAC_GDSC_POWERING_OFF" pos="0" type="boolean"/>
+		<bitfield name="SPTPRAC_GDSC_POWERING_ON" pos="1" type="boolean"/>
+		<bitfield name="SPTPRAC_GDSC_POWER_OFF" pos="2" type="boolean"/>
+		<bitfield name="SPTPRAC_GDSC_POWER_ON" pos="3" type="boolean"/>
+		<bitfield name="SP_CLOCK_OFF" pos="4" type="boolean"/>
+		<bitfield name="GMU_UP_POWER_STATE" pos="5" type="boolean"/>
+		<bitfield name="GX_HM_GDSC_POWER_OFF" pos="6" type="boolean"/>
+		<bitfield name="GX_HM_CLK_OFF" pos="7" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x50e4" name="GMU_GPU_NAP_CTRL">
+		<bitfield name="HW_NAP_ENABLE" pos="0"/>
+		<bitfield name="SID" low="4" high="8"/>
+	</reg32>
+	<reg32 offset="0x50e8" name="GMU_RPMH_CTRL">
+		<bitfield name="RPMH_INTERFACE_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="LLC_VOTE_ENABLE" pos="4" type="boolean"/>
+		<bitfield name="DDR_VOTE_ENABLE" pos="8" type="boolean"/>
+		<bitfield name="MX_VOTE_ENABLE" pos="9" type="boolean"/>
+		<bitfield name="CX_VOTE_ENABLE" pos="10" type="boolean"/>
+		<bitfield name="GFX_VOTE_ENABLE" pos="11" type="boolean"/>
+		<bitfield name="DDR_MIN_VOTE_ENABLE" pos="12" type="boolean"/>
+		<bitfield name="MX_MIN_VOTE_ENABLE" pos="13" type="boolean"/>
+		<bitfield name="CX_MIN_VOTE_ENABLE" pos="14" type="boolean"/>
+		<bitfield name="GFX_MIN_VOTE_ENABLE" pos="15" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x50e9" name="GMU_RPMH_HYST_CTRL"/>
+	<reg32 offset="0x50ec" name="GPU_GMU_CX_GMU_RPMH_POWER_STATE"/>
+	<reg32 offset="0x50f0" name="GPU_GMU_CX_GMU_CX_FAL_INTF"/>
+	<reg32 offset="0x50f1" name="GPU_GMU_CX_GMU_CX_FALNEXT_INTF"/>
+	<reg32 offset="0x5100" name="GPU_GMU_CX_GMU_PWR_COL_CP_MSG"/>
+	<reg32 offset="0x5101" name="GPU_GMU_CX_GMU_PWR_COL_CP_RESP"/>
+	<reg32 offset="0x51f0" name="GMU_BOOT_KMD_LM_HANDSHAKE"/>
+	<reg32 offset="0x5157" name="GMU_LLM_GLM_SLEEP_CTRL"/>
+	<reg32 offset="0x5158" name="GMU_LLM_GLM_SLEEP_STATUS"/>
+	<reg32 offset="0x5088" name="GMU_ALWAYS_ON_COUNTER_L"/>
+	<reg32 offset="0x5089" name="GMU_ALWAYS_ON_COUNTER_H"/>
+	<reg32 offset="0x50c3" name="GMU_GMU_PWR_COL_KEEPALIVE"/>
+	<reg32 offset="0x5180" name="GMU_HFI_CTRL_STATUS"/>
+	<reg32 offset="0x5181" name="GMU_HFI_VERSION_INFO"/>
+	<reg32 offset="0x5182" name="GMU_HFI_SFR_ADDR"/>
+	<reg32 offset="0x5183" name="GMU_HFI_MMAP_ADDR"/>
+	<reg32 offset="0x5184" name="GMU_HFI_QTBL_INFO"/>
+	<reg32 offset="0x5185" name="GMU_HFI_QTBL_ADDR"/>
+	<reg32 offset="0x5186" name="GMU_HFI_CTRL_INIT"/>
+	<reg32 offset="0x5190" name="GMU_GMU2HOST_INTR_SET"/>
+	<reg32 offset="0x5191" name="GMU_GMU2HOST_INTR_CLR"/>
+	<reg32 offset="0x5192" name="GMU_GMU2HOST_INTR_INFO">
+		<bitfield name="MSGQ" pos="0" type="boolean"/>
+		<bitfield name="CM3_FAULT" pos="23" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x5193" name="GMU_GMU2HOST_INTR_MASK"/>
+	<reg32 offset="0x5194" name="GMU_HOST2GMU_INTR_SET"/>
+	<reg32 offset="0x5195" name="GMU_HOST2GMU_INTR_CLR"/>
+	<reg32 offset="0x5196" name="GMU_HOST2GMU_INTR_RAW_INFO"/>
+	<reg32 offset="0x5197" name="GMU_HOST2GMU_INTR_EN_0"/>
+	<reg32 offset="0x5198" name="GMU_HOST2GMU_INTR_EN_1"/>
+	<reg32 offset="0x5199" name="GMU_HOST2GMU_INTR_EN_2"/>
+	<reg32 offset="0x519a" name="GMU_HOST2GMU_INTR_EN_3"/>
+	<reg32 offset="0x519b" name="GMU_HOST2GMU_INTR_INFO_0"/>
+	<reg32 offset="0x519c" name="GMU_HOST2GMU_INTR_INFO_1"/>
+	<reg32 offset="0x519d" name="GMU_HOST2GMU_INTR_INFO_2"/>
+	<reg32 offset="0x519e" name="GMU_HOST2GMU_INTR_INFO_3"/>
+	<reg32 offset="0x51c5" name="GMU_GENERAL_0"/>
+	<reg32 offset="0x51c6" name="GMU_GENERAL_1"/>
+	<reg32 offset="0x51cb" name="GMU_GENERAL_6"/>
+	<reg32 offset="0x51cc" name="GMU_GENERAL_7"/>
+	<reg32 offset="0x51cd" name="GMU_GENERAL_8" variants="A7XX"/>
+	<reg32 offset="0x51ce" name="GMU_GENERAL_9" variants="A7XX"/>
+	<reg32 offset="0x51cf" name="GMU_GENERAL_10" variants="A7XX"/>
+	<reg32 offset="0x515d" name="GMU_ISENSE_CTRL"/>
+	<reg32 offset="0x8920" name="GPU_CS_ENABLE_REG"/>
+	<reg32 offset="0x515d" name="GPU_GMU_CX_GMU_ISENSE_CTRL"/>
+	<reg32 offset="0x8578" name="GPU_CS_AMP_CALIBRATION_CONTROL3"/>
+	<reg32 offset="0x8558" name="GPU_CS_AMP_CALIBRATION_CONTROL2"/>
+	<reg32 offset="0x8580" name="GPU_CS_A_SENSOR_CTRL_0"/>
+	<reg32 offset="0x27ada" name="GPU_CS_A_SENSOR_CTRL_2"/>
+	<reg32 offset="0x881a" name="GPU_CS_SENSOR_GENERAL_STATUS"/>
+	<reg32 offset="0x8957" name="GPU_CS_AMP_CALIBRATION_CONTROL1"/>
+	<reg32 offset="0x881a" name="GPU_CS_SENSOR_GENERAL_STATUS"/>
+	<reg32 offset="0x881d" name="GPU_CS_AMP_CALIBRATION_STATUS1_0"/>
+	<reg32 offset="0x881f" name="GPU_CS_AMP_CALIBRATION_STATUS1_2"/>
+	<reg32 offset="0x8821" name="GPU_CS_AMP_CALIBRATION_STATUS1_4"/>
+	<reg32 offset="0x8965" name="GPU_CS_AMP_CALIBRATION_DONE"/>
+	<reg32 offset="0x896d" name="GPU_CS_AMP_PERIOD_CTRL"/>
+	<reg32 offset="0x8965" name="GPU_CS_AMP_CALIBRATION_DONE"/>
+	<reg32 offset="0x514d" name="GPU_GMU_CX_GMU_PWR_THRESHOLD"/>
+	<reg32 offset="0x9303" name="GMU_AO_INTERRUPT_EN"/>
+	<reg32 offset="0x9304" name="GMU_AO_HOST_INTERRUPT_CLR"/>
+	<reg32 offset="0x9305" name="GMU_AO_HOST_INTERRUPT_STATUS">
+		<bitfield name="WDOG_BITE" pos="0" type="boolean"/>
+		<bitfield name="RSCC_COMP" pos="1" type="boolean"/>
+		<bitfield name="VDROOP" pos="2" type="boolean"/>
+		<bitfield name="FENCE_ERR" pos="3" type="boolean"/>
+		<bitfield name="DBD_WAKEUP" pos="4" type="boolean"/>
+		<bitfield name="HOST_AHB_BUS_ERROR" pos="5" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x9306" name="GMU_AO_HOST_INTERRUPT_MASK"/>
+	<reg32 offset="0x9309" name="GPU_GMU_AO_GMU_CGC_MODE_CNTL"/>
+	<reg32 offset="0x930a" name="GPU_GMU_AO_GMU_CGC_DELAY_CNTL"/>
+	<reg32 offset="0x930b" name="GPU_GMU_AO_GMU_CGC_HYST_CNTL"/>
+	<reg32 offset="0x930c" name="GPU_GMU_AO_GPU_CX_BUSY_STATUS">
+		<bitfield name = "GPUBUSYIGNAHB" pos="23" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x930d" name="GPU_GMU_AO_GPU_CX_BUSY_STATUS2"/>
+	<reg32 offset="0x930e" name="GPU_GMU_AO_GPU_CX_BUSY_MASK"/>
+	<reg32 offset="0x9310" name="GMU_AO_AHB_FENCE_CTRL"/>
+	<reg32 offset="0x9313" name="GMU_AHB_FENCE_STATUS"/>
+	<reg32 offset="0x9314" name="GMU_AHB_FENCE_STATUS_CLR"/>
+	<reg32 offset="0x9315" name="GMU_RBBM_INT_UNMASKED_STATUS"/>
+	<reg32 offset="0x9316" name="GMU_AO_SPARE_CNTL"/>
+	<reg32 offset="0x9307" name="GMU_RSCC_CONTROL_REQ"/>
+	<reg32 offset="0x9308" name="GMU_RSCC_CONTROL_ACK"/>
+	<reg32 offset="0x9311" name="GMU_AHB_FENCE_RANGE_0"/>
+	<reg32 offset="0x9312" name="GMU_AHB_FENCE_RANGE_1"/>
+	<reg32 offset="0x9c03" name="GPU_CC_GX_GDSCR"/>
+	<reg32 offset="0x9d42" name="GPU_CC_GX_DOMAIN_MISC"/>
+	<reg32 offset="0xc001" name="GPU_CPR_FSM_CTL"/>
+
+	<!-- starts at offset 0x8c00 on most gpus -->
+	<reg32 offset="0x0004" name="GPU_RSCC_RSC_STATUS0_DRV0"/>
+	<reg32 offset="0x0008" name="RSCC_PDC_SEQ_START_ADDR"/>
+	<reg32 offset="0x0009" name="RSCC_PDC_MATCH_VALUE_LO"/>
+	<reg32 offset="0x000a" name="RSCC_PDC_MATCH_VALUE_HI"/>
+	<reg32 offset="0x000b" name="RSCC_PDC_SLAVE_ID_DRV0"/>
+	<reg32 offset="0x000d" name="RSCC_HIDDEN_TCS_CMD0_ADDR"/>
+	<reg32 offset="0x000e" name="RSCC_HIDDEN_TCS_CMD0_DATA"/>
+	<reg32 offset="0x0082" name="RSCC_TIMESTAMP_UNIT0_TIMESTAMP_L_DRV0"/>
+	<reg32 offset="0x0083" name="RSCC_TIMESTAMP_UNIT0_TIMESTAMP_H_DRV0"/>
+	<reg32 offset="0x0089" name="RSCC_TIMESTAMP_UNIT1_EN_DRV0"/>
+	<reg32 offset="0x008c" name="RSCC_TIMESTAMP_UNIT1_OUTPUT_DRV0"/>
+	<reg32 offset="0x0100" name="RSCC_OVERRIDE_START_ADDR"/>
+	<reg32 offset="0x0101" name="RSCC_SEQ_BUSY_DRV0"/>
+	<reg32 offset="0x0154" name="RSCC_SEQ_MEM_0_DRV0_A740" variants="A7XX"/>
+	<reg32 offset="0x0180" name="RSCC_SEQ_MEM_0_DRV0"/>
+	<reg32 offset="0x0346" name="RSCC_TCS0_DRV0_STATUS"/>
+	<reg32 offset="0x03ee" name="RSCC_TCS1_DRV0_STATUS"/>
+	<reg32 offset="0x0496" name="RSCC_TCS2_DRV0_STATUS"/>
+	<reg32 offset="0x053e" name="RSCC_TCS3_DRV0_STATUS"/>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/adreno/adreno_common.xml b/drivers/gpu/drm/msm/registers/adreno/adreno_common.xml
new file mode 100644
index 000000000000..218ec8bb966e
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/adreno/adreno_common.xml
@@ -0,0 +1,400 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+
+<enum name="chip" bare="yes">
+	<value name="A2XX" value="2"/>
+	<value name="A3XX" value="3"/>
+	<value name="A4XX" value="4"/>
+	<value name="A5XX" value="5"/>
+	<value name="A6XX" value="6"/>
+	<value name="A7XX" value="7"/>
+</enum>
+
+<enum name="adreno_pa_su_sc_draw">
+	<value name="PC_DRAW_POINTS" value="0"/>
+	<value name="PC_DRAW_LINES" value="1"/>
+	<value name="PC_DRAW_TRIANGLES" value="2"/>
+</enum>
+
+<enum name="adreno_compare_func">
+	<value name="FUNC_NEVER" value="0"/>
+	<value name="FUNC_LESS" value="1"/>
+	<value name="FUNC_EQUAL" value="2"/>
+	<value name="FUNC_LEQUAL" value="3"/>
+	<value name="FUNC_GREATER" value="4"/>
+	<value name="FUNC_NOTEQUAL" value="5"/>
+	<value name="FUNC_GEQUAL" value="6"/>
+	<value name="FUNC_ALWAYS" value="7"/>
+</enum>
+
+<enum name="adreno_stencil_op">
+	<value name="STENCIL_KEEP" value="0"/>
+	<value name="STENCIL_ZERO" value="1"/>
+	<value name="STENCIL_REPLACE" value="2"/>
+	<value name="STENCIL_INCR_CLAMP" value="3"/>
+	<value name="STENCIL_DECR_CLAMP" value="4"/>
+	<value name="STENCIL_INVERT" value="5"/>
+	<value name="STENCIL_INCR_WRAP" value="6"/>
+	<value name="STENCIL_DECR_WRAP" value="7"/>
+</enum>
+
+<enum name="adreno_rb_blend_factor">
+	<value name="FACTOR_ZERO" value="0"/>
+	<value name="FACTOR_ONE" value="1"/>
+	<value name="FACTOR_SRC_COLOR" value="4"/>
+	<value name="FACTOR_ONE_MINUS_SRC_COLOR" value="5"/>
+	<value name="FACTOR_SRC_ALPHA" value="6"/>
+	<value name="FACTOR_ONE_MINUS_SRC_ALPHA" value="7"/>
+	<value name="FACTOR_DST_COLOR" value="8"/>
+	<value name="FACTOR_ONE_MINUS_DST_COLOR" value="9"/>
+	<value name="FACTOR_DST_ALPHA" value="10"/>
+	<value name="FACTOR_ONE_MINUS_DST_ALPHA" value="11"/>
+	<value name="FACTOR_CONSTANT_COLOR" value="12"/>
+	<value name="FACTOR_ONE_MINUS_CONSTANT_COLOR" value="13"/>
+	<value name="FACTOR_CONSTANT_ALPHA" value="14"/>
+	<value name="FACTOR_ONE_MINUS_CONSTANT_ALPHA" value="15"/>
+	<value name="FACTOR_SRC_ALPHA_SATURATE" value="16"/>
+	<value name="FACTOR_SRC1_COLOR" value="20"/>
+	<value name="FACTOR_ONE_MINUS_SRC1_COLOR" value="21"/>
+	<value name="FACTOR_SRC1_ALPHA" value="22"/>
+	<value name="FACTOR_ONE_MINUS_SRC1_ALPHA" value="23"/>
+</enum>
+
+<bitset name="adreno_rb_stencilrefmask" inline="yes">
+	<bitfield name="STENCILREF" low="0" high="7" type="hex"/>
+	<bitfield name="STENCILMASK" low="8" high="15" type="hex"/>
+	<bitfield name="STENCILWRITEMASK" low="16" high="23" type="hex"/>
+</bitset>
+
+<enum name="adreno_rb_surface_endian">
+	<value name="ENDIAN_NONE" value="0"/>
+	<value name="ENDIAN_8IN16" value="1"/>
+	<value name="ENDIAN_8IN32" value="2"/>
+	<value name="ENDIAN_16IN32" value="3"/>
+	<value name="ENDIAN_8IN64" value="4"/>
+	<value name="ENDIAN_8IN128" value="5"/>
+</enum>
+
+<enum name="adreno_rb_dither_mode">
+	<value name="DITHER_DISABLE" value="0"/>
+	<value name="DITHER_ALWAYS" value="1"/>
+	<value name="DITHER_IF_ALPHA_OFF" value="2"/>
+</enum>
+
+<enum name="adreno_rb_depth_format">
+	<value name="DEPTHX_16" value="0"/>
+	<value name="DEPTHX_24_8" value="1"/>
+	<value name="DEPTHX_32" value="2"/>
+</enum>
+
+<enum name="adreno_rb_copy_control_mode">
+	<value name="RB_COPY_RESOLVE" value="1"/>
+	<value name="RB_COPY_CLEAR" value="2"/>
+	<value name="RB_COPY_DEPTH_STENCIL" value="5"/>  <!-- not sure if this is part of MODE or another bitfield?? -->
+</enum>
+
+<bitset name="adreno_reg_xy" inline="yes">
+	<bitfield name="WINDOW_OFFSET_DISABLE" pos="31" type="boolean"/>
+	<bitfield name="X" low="0" high="14" type="uint"/>
+	<bitfield name="Y" low="16" high="30" type="uint"/>
+</bitset>
+
+<bitset name="adreno_cp_protect" inline="yes">
+	<bitfield name="BASE_ADDR" low="0" high="16"/>
+	<bitfield name="MASK_LEN" low="24" high="28"/>
+	<bitfield name="TRAP_WRITE" pos="29"/>
+	<bitfield name="TRAP_READ" pos="30"/>
+</bitset>
+
+<domain name="AXXX" width="32">
+	<brief>Registers in common between a2xx and a3xx</brief>
+
+	<reg32 offset="0x01c0" name="CP_RB_BASE"/>
+	<reg32 offset="0x01c1" name="CP_RB_CNTL">
+		<bitfield name="BUFSZ" low="0" high="5"/>
+		<bitfield name="BLKSZ" low="8" high="13"/>
+		<bitfield name="BUF_SWAP" low="16" high="17"/>
+		<bitfield name="POLL_EN" pos="20" type="boolean"/>
+		<bitfield name="NO_UPDATE" pos="27" type="boolean"/>
+		<bitfield name="RPTR_WR_EN" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x01c3" name="CP_RB_RPTR_ADDR">
+		<bitfield name="SWAP" low="0" high="1" type="uint"/>
+		<bitfield name="ADDR" low="2" high="31" shr="2"/>
+	</reg32>
+	<reg32 offset="0x01c4" name="CP_RB_RPTR" type="uint"/>
+	<reg32 offset="0x01c5" name="CP_RB_WPTR" type="uint"/>
+	<reg32 offset="0x01c6" name="CP_RB_WPTR_DELAY"/>
+	<reg32 offset="0x01c7" name="CP_RB_RPTR_WR"/>
+	<reg32 offset="0x01c8" name="CP_RB_WPTR_BASE"/>
+	<reg32 offset="0x01d5" name="CP_QUEUE_THRESHOLDS">
+		<bitfield name="CSQ_IB1_START" low="0" high="3" type="uint"/>
+		<bitfield name="CSQ_IB2_START" low="8" high="11" type="uint"/>
+		<bitfield name="CSQ_ST_START" low="16" high="19" type="uint"/>
+	</reg32>
+	<reg32 offset="0x01d6" name="CP_MEQ_THRESHOLDS">
+		<bitfield name="MEQ_END" low="16" high="20" type="uint"/>
+		<bitfield name="ROQ_END" low="24" high="28" type="uint"/>
+	</reg32>
+	<reg32 offset="0x01d7" name="CP_CSQ_AVAIL">
+		<bitfield name="RING" low="0" high="6" type="uint"/>
+		<bitfield name="IB1" low="8" high="14" type="uint"/>
+		<bitfield name="IB2" low="16" high="22" type="uint"/>
+	</reg32>
+	<reg32 offset="0x01d8" name="CP_STQ_AVAIL">
+		<bitfield name="ST" low="0" high="6" type="uint"/>
+	</reg32>
+	<reg32 offset="0x01d9" name="CP_MEQ_AVAIL">
+		<bitfield name="MEQ" low="0" high="4" type="uint"/>
+	</reg32>
+	<reg32 offset="0x01dc" name="SCRATCH_UMSK">
+		<bitfield name="UMSK" low="0" high="7" type="uint"/>
+		<bitfield name="SWAP" low="16" high="17" type="uint"/>
+	</reg32>
+	<reg32 offset="0x01dd" name="SCRATCH_ADDR"/>
+	<reg32 offset="0x01ea" name="CP_ME_RDADDR"/>
+
+	<reg32 offset="0x01ec" name="CP_STATE_DEBUG_INDEX"/>
+	<reg32 offset="0x01ed" name="CP_STATE_DEBUG_DATA"/>
+	<reg32 offset="0x01f2" name="CP_INT_CNTL">
+		<bitfield name="SW_INT_MASK" pos="19" type="boolean"/>
+		<bitfield name="T0_PACKET_IN_IB_MASK" pos="23" type="boolean"/>
+		<bitfield name="OPCODE_ERROR_MASK" pos="24" type="boolean"/>
+		<bitfield name="PROTECTED_MODE_ERROR_MASK" pos="25" type="boolean"/>
+		<bitfield name="RESERVED_BIT_ERROR_MASK" pos="26" type="boolean"/>
+		<bitfield name="IB_ERROR_MASK" pos="27" type="boolean"/>
+		<bitfield name="IB2_INT_MASK" pos="29" type="boolean"/>
+		<bitfield name="IB1_INT_MASK" pos="30" type="boolean"/>
+		<bitfield name="RB_INT_MASK" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x01f3" name="CP_INT_STATUS"/>
+	<reg32 offset="0x01f4" name="CP_INT_ACK"/>
+	<reg32 offset="0x01f6" name="CP_ME_CNTL">
+		<bitfield name="BUSY" pos="29" type="boolean"/>
+		<bitfield name="HALT" pos="28" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x01f7" name="CP_ME_STATUS"/>
+	<reg32 offset="0x01f8" name="CP_ME_RAM_WADDR"/>
+	<reg32 offset="0x01f9" name="CP_ME_RAM_RADDR"/>
+	<reg32 offset="0x01fa" name="CP_ME_RAM_DATA"/>
+	<reg32 offset="0x01fc" name="CP_DEBUG">
+		<bitfield name="PREDICATE_DISABLE" pos="23" type="boolean"/>
+		<bitfield name="PROG_END_PTR_ENABLE" pos="24" type="boolean"/>
+		<bitfield name="MIU_128BIT_WRITE_ENABLE" pos="25" type="boolean"/>
+		<bitfield name="PREFETCH_PASS_NOPS" pos="26" type="boolean"/>
+		<bitfield name="DYNAMIC_CLK_DISABLE" pos="27" type="boolean"/>
+		<bitfield name="PREFETCH_MATCH_DISABLE" pos="28" type="boolean"/>
+		<bitfield name="SIMPLE_ME_FLOW_CONTROL" pos="30" type="boolean"/>
+		<bitfield name="MIU_WRITE_PACK_DISABLE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x01fd" name="CP_CSQ_RB_STAT">
+		<bitfield name="RPTR" low="0" high="6" type="uint"/>
+		<bitfield name="WPTR" low="16" high="22" type="uint"/>
+	</reg32>
+	<reg32 offset="0x01fe" name="CP_CSQ_IB1_STAT">
+		<bitfield name="RPTR" low="0" high="6" type="uint"/>
+		<bitfield name="WPTR" low="16" high="22" type="uint"/>
+	</reg32>
+	<reg32 offset="0x01ff" name="CP_CSQ_IB2_STAT">
+		<bitfield name="RPTR" low="0" high="6" type="uint"/>
+		<bitfield name="WPTR" low="16" high="22" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x0440" name="CP_NON_PREFETCH_CNTRS"/>
+	<reg32 offset="0x0443" name="CP_STQ_ST_STAT"/>
+	<reg32 offset="0x044d" name="CP_ST_BASE"/>
+	<reg32 offset="0x044e" name="CP_ST_BUFSZ"/>
+	<reg32 offset="0x044f" name="CP_MEQ_STAT"/>
+	<reg32 offset="0x0452" name="CP_MIU_TAG_STAT"/>
+	<reg32 offset="0x0454" name="CP_BIN_MASK_LO"/>
+	<reg32 offset="0x0455" name="CP_BIN_MASK_HI"/>
+	<reg32 offset="0x0456" name="CP_BIN_SELECT_LO"/>
+	<reg32 offset="0x0457" name="CP_BIN_SELECT_HI"/>
+	<reg32 offset="0x0458" name="CP_IB1_BASE"/>
+	<reg32 offset="0x0459" name="CP_IB1_BUFSZ"/>
+	<reg32 offset="0x045a" name="CP_IB2_BASE"/>
+	<reg32 offset="0x045b" name="CP_IB2_BUFSZ"/>
+	<reg32 offset="0x047f" name="CP_STAT">
+		<bitfield pos="31" name="CP_BUSY"/>
+		<bitfield pos="30" name="VS_EVENT_FIFO_BUSY"/>
+		<bitfield pos="29" name="PS_EVENT_FIFO_BUSY"/>
+		<bitfield pos="28" name="CF_EVENT_FIFO_BUSY"/>
+		<bitfield pos="27" name="RB_EVENT_FIFO_BUSY"/>
+		<bitfield pos="26" name="ME_BUSY"/>
+		<bitfield pos="25" name="MIU_WR_C_BUSY"/>
+		<bitfield pos="23" name="CP_3D_BUSY"/>
+		<bitfield pos="22" name="CP_NRT_BUSY"/>
+		<bitfield pos="21" name="RBIU_SCRATCH_BUSY"/>
+		<bitfield pos="20" name="RCIU_ME_BUSY"/>
+		<bitfield pos="19" name="RCIU_PFP_BUSY"/>
+		<bitfield pos="18" name="MEQ_RING_BUSY"/>
+		<bitfield pos="17" name="PFP_BUSY"/>
+		<bitfield pos="16" name="ST_QUEUE_BUSY"/>
+		<bitfield pos="13" name="INDIRECT2_QUEUE_BUSY"/>
+		<bitfield pos="12" name="INDIRECTS_QUEUE_BUSY"/>
+		<bitfield pos="11" name="RING_QUEUE_BUSY"/>
+		<bitfield pos="10" name="CSF_BUSY"/>
+		<bitfield pos="9"  name="CSF_ST_BUSY"/>
+		<bitfield pos="8"  name="EVENT_BUSY"/>
+		<bitfield pos="7"  name="CSF_INDIRECT2_BUSY"/>
+		<bitfield pos="6"  name="CSF_INDIRECTS_BUSY"/>
+		<bitfield pos="5"  name="CSF_RING_BUSY"/>
+		<bitfield pos="4"  name="RCIU_BUSY"/>
+		<bitfield pos="3"  name="RBIU_BUSY"/>
+		<bitfield pos="2"  name="MIU_RD_RETURN_BUSY"/>
+		<bitfield pos="1"  name="MIU_RD_REQ_BUSY"/>
+		<bitfield pos="0"  name="MIU_WR_BUSY"/>
+	</reg32>
+	<reg32 offset="0x0578" name="CP_SCRATCH_REG0" type="uint"/>
+	<reg32 offset="0x0579" name="CP_SCRATCH_REG1" type="uint"/>
+	<reg32 offset="0x057a" name="CP_SCRATCH_REG2" type="uint"/>
+	<reg32 offset="0x057b" name="CP_SCRATCH_REG3" type="uint"/>
+	<reg32 offset="0x057c" name="CP_SCRATCH_REG4" type="uint"/>
+	<reg32 offset="0x057d" name="CP_SCRATCH_REG5" type="uint"/>
+	<reg32 offset="0x057e" name="CP_SCRATCH_REG6" type="uint"/>
+	<reg32 offset="0x057f" name="CP_SCRATCH_REG7" type="uint"/>
+
+	<reg32 offset="0x0600" name="CP_ME_VS_EVENT_SRC"/>
+	<reg32 offset="0x0601" name="CP_ME_VS_EVENT_ADDR"/>
+	<reg32 offset="0x0602" name="CP_ME_VS_EVENT_DATA"/>
+	<reg32 offset="0x0603" name="CP_ME_VS_EVENT_ADDR_SWM"/>
+	<reg32 offset="0x0604" name="CP_ME_VS_EVENT_DATA_SWM"/>
+	<reg32 offset="0x0605" name="CP_ME_PS_EVENT_SRC"/>
+	<reg32 offset="0x0606" name="CP_ME_PS_EVENT_ADDR"/>
+	<reg32 offset="0x0607" name="CP_ME_PS_EVENT_DATA"/>
+	<reg32 offset="0x0608" name="CP_ME_PS_EVENT_ADDR_SWM"/>
+	<reg32 offset="0x0609" name="CP_ME_PS_EVENT_DATA_SWM"/>
+	<reg32 offset="0x060a" name="CP_ME_CF_EVENT_SRC"/>
+	<reg32 offset="0x060b" name="CP_ME_CF_EVENT_ADDR"/>
+	<reg32 offset="0x060c" name="CP_ME_CF_EVENT_DATA" type="uint"/>
+	<reg32 offset="0x060d" name="CP_ME_NRT_ADDR"/>
+	<reg32 offset="0x060e" name="CP_ME_NRT_DATA"/>
+	<reg32 offset="0x0612" name="CP_ME_VS_FETCH_DONE_SRC"/>
+	<reg32 offset="0x0613" name="CP_ME_VS_FETCH_DONE_ADDR"/>
+	<reg32 offset="0x0614" name="CP_ME_VS_FETCH_DONE_DATA"/>
+
+</domain>
+
+<!--
+	Common between A3xx and A4xx:
+ -->
+
+<enum name="a3xx_rop_code">
+	<value name="ROP_CLEAR"         value="0"/>
+	<value name="ROP_NOR"           value="1"/>
+	<value name="ROP_AND_INVERTED"  value="2"/>
+	<value name="ROP_COPY_INVERTED" value="3"/>
+	<value name="ROP_AND_REVERSE"   value="4"/>
+	<value name="ROP_INVERT"        value="5"/>
+	<value name="ROP_XOR"           value="6"/>
+	<value name="ROP_NAND"          value="7"/>
+	<value name="ROP_AND"           value="8"/>
+	<value name="ROP_EQUIV"         value="9"/>
+	<value name="ROP_NOOP"          value="10"/>
+	<value name="ROP_OR_INVERTED"   value="11"/>
+	<value name="ROP_COPY"          value="12"/>
+	<value name="ROP_OR_REVERSE"    value="13"/>
+	<value name="ROP_OR"            value="14"/>
+	<value name="ROP_SET"           value="15"/>
+</enum>
+
+<enum name="a3xx_render_mode">
+	<value name="RB_RENDERING_PASS" value="0"/>
+	<value name="RB_TILING_PASS" value="1"/>
+	<value name="RB_RESOLVE_PASS" value="2"/>
+	<value name="RB_COMPUTE_PASS" value="3"/>
+</enum>
+
+<enum name="a3xx_msaa_samples">
+	<value name="MSAA_ONE" value="0"/>
+	<value name="MSAA_TWO" value="1"/>
+	<value name="MSAA_FOUR" value="2"/>
+	<value name="MSAA_EIGHT" value="3"/>
+</enum>
+
+<enum name="a3xx_threadmode">
+	<value value="0" name="MULTI"/>
+	<value value="1" name="SINGLE"/>
+</enum>
+
+<enum name="a3xx_instrbuffermode">
+	<!--
+	When shader size goes above ~128 or so, blob switches to '0'
+	and doesn't emit shader in cmdstream.  When either is '0' it
+	doesn't get emitted via CP_LOAD_STATE.  When only one is
+	'0' the other gets size 256-others_size.  So I think that:
+		BUFFER => execute out of state memory
+		CACHE  => use available state memory as local cache
+	NOTE that when CACHE mode, also set CACHEINVALID flag!
+
+	TODO check if that 256 size is same for all a3xx
+	 -->
+	<value value="0" name="CACHE"/>
+	<value value="1" name="BUFFER"/>
+</enum>
+
+<enum name="a3xx_threadsize">
+	<value value="0" name="TWO_QUADS"/>
+	<value value="1" name="FOUR_QUADS"/>
+</enum>
+
+<enum name="a3xx_color_swap">
+	<value name="WZYX" value="0"/>
+	<value name="WXYZ" value="1"/>
+	<value name="ZYXW" value="2"/>
+	<value name="XYZW" value="3"/>
+</enum>
+
+<enum name="a3xx_rb_blend_opcode">
+	<value name="BLEND_DST_PLUS_SRC" value="0"/>
+	<value name="BLEND_SRC_MINUS_DST" value="1"/>
+	<value name="BLEND_DST_MINUS_SRC" value="2"/>
+	<value name="BLEND_MIN_DST_SRC" value="3"/>
+	<value name="BLEND_MAX_DST_SRC" value="4"/>
+</enum>
+
+<enum name="a4xx_tess_spacing">
+	<value name="EQUAL_SPACING" value="0"/>
+	<value name="ODD_SPACING" value="2"/>
+	<value name="EVEN_SPACING" value="3"/>
+</enum>
+
+<doc>Address mode for a5xx+</doc>
+<enum name="a5xx_address_mode">
+	<value name="ADDR_32B" value="0"/>
+	<value name="ADDR_64B" value="1"/>
+</enum>
+
+<doc>
+    Line mode for a5xx+
+	Note that Bresenham lines are only supported with MSAA disabled.
+</doc>
+<enum name="a5xx_line_mode">
+	<value value="0x0"  name="BRESENHAM"/>
+	<value value="0x1"  name="RECTANGULAR"/>
+</enum>
+
+<doc>
+	Blob (v615) seem to only use SAM and I wasn't able to coerce
+	it to produce any other command.
+	Probably valid for a4xx+ but not enabled or tested on anything
+	but a6xx.
+</doc>
+<enum name="a6xx_tex_prefetch_cmd">
+	<doc> Produces garbage </doc>
+	<value value="0x0" name="TEX_PREFETCH_UNK0"/>
+	<value value="0x1" name="TEX_PREFETCH_SAM"/>
+	<value value="0x2" name="TEX_PREFETCH_GATHER4R"/>
+	<value value="0x3" name="TEX_PREFETCH_GATHER4G"/>
+	<value value="0x4" name="TEX_PREFETCH_GATHER4B"/>
+	<value value="0x5" name="TEX_PREFETCH_GATHER4A"/>
+	<doc> Causes reads from an invalid address </doc>
+	<value value="0x6" name="TEX_PREFETCH_UNK6"/>
+	<doc> Results in color being zero </doc>
+	<value value="0x7" name="TEX_PREFETCH_UNK7"/>
+</enum>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/adreno/adreno_pm4.xml b/drivers/gpu/drm/msm/registers/adreno/adreno_pm4.xml
new file mode 100644
index 000000000000..cab01af55d22
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/adreno/adreno_pm4.xml
@@ -0,0 +1,2268 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+<import file="adreno/adreno_common.xml"/>
+
+<enum name="vgt_event_type" varset="chip">
+	<value name="VS_DEALLOC" value="0"/>
+	<value name="PS_DEALLOC" value="1" variants="A2XX-A6XX"/>
+	<value name="VS_DONE_TS" value="2"/>
+	<value name="PS_DONE_TS" value="3"/>
+	<doc>
+		Flushes dirty data from UCHE, and also writes a GPU timestamp to
+		the address if one is provided.
+	</doc>
+	<value name="CACHE_FLUSH_TS" value="4"/>
+	<value name="CONTEXT_DONE" value="5"/>
+	<value name="CACHE_FLUSH" value="6" variants="A2XX-A4XX"/>
+	<value name="VIZQUERY_START" value="7" variants="A2XX"/>
+	<value name="HLSQ_FLUSH" value="7" variants="A3XX-A4XX"/>
+	<value name="VIZQUERY_END" value="8" variants="A2XX"/>
+	<value name="SC_WAIT_WC" value="9" variants="A2XX"/>
+	<value name="WRITE_PRIMITIVE_COUNTS" value="9" variants="A6XX"/>
+	<value name="START_PRIMITIVE_CTRS" value="11" variants="A6XX"/>
+	<value name="STOP_PRIMITIVE_CTRS" value="12" variants="A6XX"/>
+	<!-- Not sure that these 4 events don't have the same meaning as on A5XX+ -->
+	<value name="RST_PIX_CNT" value="13" variants="A2XX-A4XX"/>
+	<value name="RST_VTX_CNT" value="14" variants="A2XX-A4XX"/>
+	<value name="TILE_FLUSH" value="15" variants="A2XX-A4XX"/>
+	<value name="STAT_EVENT" value="16" variants="A2XX-A4XX"/>
+	<value name="CACHE_FLUSH_AND_INV_TS_EVENT" value="20" variants="A2XX-A4XX"/>
+	<doc>
+		If A6XX_RB_SAMPLE_COUNT_CONTROL.copy is true, writes OQ Z passed
+		sample counts to RB_SAMPLE_COUNT_ADDR.  This writes to main
+		memory, skipping UCHE.
+	</doc>
+	<value name="ZPASS_DONE" value="21"/>
+	<value name="CACHE_FLUSH_AND_INV_EVENT" value="22" variants="A2XX"/>
+
+	<doc>
+		Writes the GPU timestamp to the address that follows, once RB
+		access and flushes are complete.
+	</doc>
+	<value name="RB_DONE_TS" value="22" variants="A3XX-"/>
+
+	<value name="PERFCOUNTER_START" value="23" variants="A2XX-A4XX"/>
+	<value name="PERFCOUNTER_STOP" value="24" variants="A2XX-A4XX"/>
+	<value name="VS_FETCH_DONE" value="27"/>
+	<value name="FACENESS_FLUSH" value="28" variants="A2XX-A4XX"/>
+
+	<!-- a5xx events -->
+	<value name="WT_DONE_TS" value="8" variants="A5XX-"/>
+	<value name="START_FRAGMENT_CTRS" value="13" variants="A5XX-"/>
+	<value name="STOP_FRAGMENT_CTRS" value="14" variants="A5XX-"/>
+	<value name="START_COMPUTE_CTRS" value="15" variants="A5XX-"/>
+	<value name="STOP_COMPUTE_CTRS" value="16" variants="A5XX-"/>
+	<value name="FLUSH_SO_0" value="17" variants="A5XX-"/>
+	<value name="FLUSH_SO_1" value="18" variants="A5XX-"/>
+	<value name="FLUSH_SO_2" value="19" variants="A5XX-"/>
+	<value name="FLUSH_SO_3" value="20" variants="A5XX-"/>
+
+	<doc>
+		Invalidates depth attachment data from the CCU.  We assume this
+		happens in the last stage.
+	</doc>
+	<value name="PC_CCU_INVALIDATE_DEPTH" value="24" variants="A5XX-"/>
+
+	<doc>
+		Invalidates color attachment data from the CCU.  We assume this
+		happens in the last stage.
+	</doc>
+	<value name="PC_CCU_INVALIDATE_COLOR" value="25" variants="A5XX-"/>
+
+	<doc>
+		Flushes the small cache used by CP_EVENT_WRITE::BLIT (which,
+		along with its registers, would be better named RESOLVE).
+	</doc>
+	<value name="PC_CCU_RESOLVE_TS" value="26" variants="A6XX"/>
+
+	<doc>
+		Flushes depth attachment data from the CCU.  We assume this
+		happens in the last stage.
+	</doc>
+	<value name="PC_CCU_FLUSH_DEPTH_TS" value="28" variants="A5XX-"/>
+
+	<doc>
+		Flushes color attachment data from the CCU.  We assume this
+		happens in the last stage.
+	</doc>
+	<value name="PC_CCU_FLUSH_COLOR_TS" value="29" variants="A5XX-"/>
+
+	<doc>
+		2D blit to resolve GMEM to system memory (skipping CCU) at the
+		end of a render pass.  Compare to CP_BLIT's BLIT_OP_SCALE for
+		more general blitting.
+	</doc>
+	<value name="BLIT" value="30" variants="A5XX-"/>
+
+	<doc>
+		Clears based on GRAS_LRZ_CNTL configuration, could clear
+		fast-clear buffer or LRZ direction.
+		LRZ direction is stored at lrz_fc_offset + 0x200, has 1 byte which
+		could be expressed by enum:
+			CUR_DIR_DISABLED = 0x0
+			CUR_DIR_GE = 0x1
+			CUR_DIR_LE = 0x2
+			CUR_DIR_UNSET = 0x3
+		Clear of direction means setting the direction to CUR_DIR_UNSET.
+	</doc>
+	<value name="LRZ_CLEAR" value="37" variants="A5XX-"/>
+
+	<value name="LRZ_FLUSH" value="38" variants="A5XX-"/>
+	<value name="BLIT_OP_FILL_2D" value="39" variants="A5XX-"/>
+	<value name="BLIT_OP_COPY_2D" value="40" variants="A5XX-A6XX"/>
+	<value name="UNK_40" value="40" variants="A7XX"/>
+	<value name="BLIT_OP_SCALE_2D" value="42" variants="A5XX-"/>
+	<value name="CONTEXT_DONE_2D" value="43" variants="A5XX-"/>
+	<value name="UNK_2C" value="44" variants="A5XX-"/>
+	<value name="UNK_2D" value="45" variants="A5XX-"/>
+
+	<!-- a6xx events -->
+	<doc>
+		Invalidates UCHE.
+	</doc>
+	<value name="CACHE_INVALIDATE" value="49" variants="A6XX"/>
+
+	<value name="LABEL" value="63" variants="A6XX-"/>
+
+	<!-- note, some of these are the same as a6xx, just named differently -->
+
+	<doc> Doesn't seem to do anything </doc>
+	<value name="DUMMY_EVENT" value="1" variants="A7XX"/>
+	<value name="CCU_INVALIDATE_DEPTH" value="24" variants="A7XX"/>
+	<value name="CCU_INVALIDATE_COLOR" value="25" variants="A7XX"/>
+	<value name="CCU_RESOLVE_CLEAN" value="26" variants="A7XX"/>
+	<value name="CCU_FLUSH_DEPTH" value="28" variants="A7XX"/>
+	<value name="CCU_FLUSH_COLOR" value="29" variants="A7XX"/>
+	<value name="CCU_RESOLVE" value="30" variants="A7XX"/>
+	<value name="CCU_END_RESOLVE_GROUP" value="31" variants="A7XX"/>
+	<value name="CCU_CLEAN_DEPTH" value="32" variants="A7XX"/>
+	<value name="CCU_CLEAN_COLOR" value="33" variants="A7XX"/>
+	<value name="CACHE_RESET" value="48" variants="A7XX"/>
+	<value name="CACHE_CLEAN" value="49" variants="A7XX"/>
+	<!-- TODO: deal with name conflicts with other gens -->
+	<value name="CACHE_FLUSH7" value="50" variants="A7XX"/>
+	<value name="CACHE_INVALIDATE7" value="51" variants="A7XX"/>
+</enum>
+
+<enum name="pc_di_primtype">
+	<value name="DI_PT_NONE" value="0"/>
+	<!-- POINTLIST_PSIZE is used on a3xx/a4xx when gl_PointSize is written: -->
+	<value name="DI_PT_POINTLIST_PSIZE" value="1"/>
+	<value name="DI_PT_LINELIST" value="2"/>
+	<value name="DI_PT_LINESTRIP" value="3"/>
+	<value name="DI_PT_TRILIST" value="4"/>
+	<value name="DI_PT_TRIFAN" value="5"/>
+	<value name="DI_PT_TRISTRIP" value="6"/>
+	<value name="DI_PT_LINELOOP" value="7"/>  <!-- a22x, a3xx -->
+	<value name="DI_PT_RECTLIST" value="8"/>
+	<value name="DI_PT_POINTLIST" value="9"/>
+	<value name="DI_PT_LINE_ADJ" value="0xa"/>
+	<value name="DI_PT_LINESTRIP_ADJ" value="0xb"/>
+	<value name="DI_PT_TRI_ADJ" value="0xc"/>
+	<value name="DI_PT_TRISTRIP_ADJ" value="0xd"/>
+
+	<value name="DI_PT_PATCHES0" value="0x1f"/>
+	<value name="DI_PT_PATCHES1" value="0x20"/>
+	<value name="DI_PT_PATCHES2" value="0x21"/>
+	<value name="DI_PT_PATCHES3" value="0x22"/>
+	<value name="DI_PT_PATCHES4" value="0x23"/>
+	<value name="DI_PT_PATCHES5" value="0x24"/>
+	<value name="DI_PT_PATCHES6" value="0x25"/>
+	<value name="DI_PT_PATCHES7" value="0x26"/>
+	<value name="DI_PT_PATCHES8" value="0x27"/>
+	<value name="DI_PT_PATCHES9" value="0x28"/>
+	<value name="DI_PT_PATCHES10" value="0x29"/>
+	<value name="DI_PT_PATCHES11" value="0x2a"/>
+	<value name="DI_PT_PATCHES12" value="0x2b"/>
+	<value name="DI_PT_PATCHES13" value="0x2c"/>
+	<value name="DI_PT_PATCHES14" value="0x2d"/>
+	<value name="DI_PT_PATCHES15" value="0x2e"/>
+	<value name="DI_PT_PATCHES16" value="0x2f"/>
+	<value name="DI_PT_PATCHES17" value="0x30"/>
+	<value name="DI_PT_PATCHES18" value="0x31"/>
+	<value name="DI_PT_PATCHES19" value="0x32"/>
+	<value name="DI_PT_PATCHES20" value="0x33"/>
+	<value name="DI_PT_PATCHES21" value="0x34"/>
+	<value name="DI_PT_PATCHES22" value="0x35"/>
+	<value name="DI_PT_PATCHES23" value="0x36"/>
+	<value name="DI_PT_PATCHES24" value="0x37"/>
+	<value name="DI_PT_PATCHES25" value="0x38"/>
+	<value name="DI_PT_PATCHES26" value="0x39"/>
+	<value name="DI_PT_PATCHES27" value="0x3a"/>
+	<value name="DI_PT_PATCHES28" value="0x3b"/>
+	<value name="DI_PT_PATCHES29" value="0x3c"/>
+	<value name="DI_PT_PATCHES30" value="0x3d"/>
+	<value name="DI_PT_PATCHES31" value="0x3e"/>
+</enum>
+
+<enum name="pc_di_src_sel">
+	<value name="DI_SRC_SEL_DMA" value="0"/>
+	<value name="DI_SRC_SEL_IMMEDIATE" value="1"/>
+	<value name="DI_SRC_SEL_AUTO_INDEX" value="2"/>
+	<value name="DI_SRC_SEL_AUTO_XFB" value="3"/>
+</enum>
+
+<enum name="pc_di_face_cull_sel">
+	<value name="DI_FACE_CULL_NONE" value="0"/>
+	<value name="DI_FACE_CULL_FETCH" value="1"/>
+	<value name="DI_FACE_BACKFACE_CULL" value="2"/>
+	<value name="DI_FACE_FRONTFACE_CULL" value="3"/>
+</enum>
+
+<enum name="pc_di_index_size">
+	<value name="INDEX_SIZE_IGN" value="0"/>
+	<value name="INDEX_SIZE_16_BIT" value="0"/>
+	<value name="INDEX_SIZE_32_BIT" value="1"/>
+	<value name="INDEX_SIZE_8_BIT" value="2"/>
+	<value name="INDEX_SIZE_INVALID"/>
+</enum>
+
+<enum name="pc_di_vis_cull_mode">
+	<value name="IGNORE_VISIBILITY" value="0"/>
+	<value name="USE_VISIBILITY" value="1"/>
+</enum>
+
+<enum name="adreno_pm4_packet_type">
+	<value name="CP_TYPE0_PKT" value="0x00000000"/>
+	<value name="CP_TYPE1_PKT" value="0x40000000"/>
+	<value name="CP_TYPE2_PKT" value="0x80000000"/>
+	<value name="CP_TYPE3_PKT" value="0xc0000000"/>
+	<value name="CP_TYPE4_PKT" value="0x40000000"/>
+	<value name="CP_TYPE7_PKT" value="0x70000000"/>
+</enum>
+
+<!--
+   Note that in some cases, the same packet id is recycled on a later
+   generation, so variants attribute is used to distinguish.   They
+   may not be completely accurate, we would probably have to analyze
+   the pfp and me/pm4 firmware to verify the packet is actually
+   handled on a particular generation.  But it is at least enough to
+   disambiguate the packet-id's that were re-used for different
+   packets starting with a5xx.
+ -->
+<enum name="adreno_pm4_type3_packets" varset="chip">
+	<doc>initialize CP's micro-engine</doc>
+	<value name="CP_ME_INIT" value="0x48"/>
+	<doc>skip N 32-bit words to get to the next packet</doc>
+	<value name="CP_NOP" value="0x10"/>
+	<doc>
+		indirect buffer dispatch.  prefetch parser uses this packet
+		type to determine whether to pre-fetch the IB
+	</doc>
+	<value name="CP_PREEMPT_ENABLE" value="0x1c" variants="A5XX"/>
+	<value name="CP_PREEMPT_TOKEN" value="0x1e" variants="A5XX"/>
+	<value name="CP_INDIRECT_BUFFER" value="0x3f"/>
+	<doc>
+		Takes the same arguments as CP_INDIRECT_BUFFER, but jumps to
+		another buffer at the same level. Must be at the end of IB, and
+		doesn't work with draw state IB's.
+	</doc>
+	<value name="CP_INDIRECT_BUFFER_CHAIN" value="0x57" variants="A5XX-"/>
+	<doc>indirect buffer dispatch.  same as IB, but init is pipelined</doc>
+	<value name="CP_INDIRECT_BUFFER_PFD" value="0x37"/>
+	<doc>
+		Waits for the IDLE state of the engine before further drawing.
+		This is pipelined, so the CP may continue.
+	</doc>
+	<value name="CP_WAIT_FOR_IDLE" value="0x26"/>
+	<doc>wait until a register or memory location is a specific value</doc>
+	<value name="CP_WAIT_REG_MEM" value="0x3c"/>
+	<doc>wait until a register location is equal to a specific value</doc>
+	<value name="CP_WAIT_REG_EQ" value="0x52"/>
+	<doc>wait until a register location is >= a specific value</doc>
+	<value name="CP_WAIT_REG_GTE" value="0x53" variants="A2XX-A4XX"/>
+	<doc>wait until a read completes</doc>
+	<value name="CP_WAIT_UNTIL_READ" value="0x5c" variants="A2XX-A4XX"/>
+	<doc>wait until all base/size writes from an IB_PFD packet have completed</doc>
+	<!--
+		NOTE: CP_WAIT_IB_PFD_COMPLETE unimplemented at least since a5xx fw, and
+		recycled for something new on a7xx
+	 -->
+	<value name="CP_WAIT_IB_PFD_COMPLETE" value="0x5d" varset="chip" variants="A2XX-A4XX"/>
+	<doc>register read/modify/write</doc>
+	<value name="CP_REG_RMW" value="0x21"/>
+	<doc>Set binning configuration registers</doc>
+	<value name="CP_SET_BIN_DATA" value="0x2f" variants="A2XX-A4XX"/>
+	<value name="CP_SET_BIN_DATA5" value="0x2f" variants="A5XX-"/>
+	<doc>reads register in chip and writes to memory</doc>
+	<value name="CP_REG_TO_MEM" value="0x3e"/>
+	<doc>write N 32-bit words to memory</doc>
+	<value name="CP_MEM_WRITE" value="0x3d"/>
+	<doc>write CP_PROG_COUNTER value to memory</doc>
+	<value name="CP_MEM_WRITE_CNTR" value="0x4f"/>
+	<doc>conditional execution of a sequence of packets</doc>
+	<value name="CP_COND_EXEC" value="0x44"/>
+	<doc>conditional write to memory or register</doc>
+	<value name="CP_COND_WRITE" value="0x45" variants="A2XX-A4XX"/>
+	<value name="CP_COND_WRITE5" value="0x45" variants="A5XX-"/>
+	<doc>generate an event that creates a write to memory when completed</doc>
+	<value name="CP_EVENT_WRITE" value="0x46" variants="A2XX-A6XX"/>
+	<value name="CP_EVENT_WRITE7" value="0x46" variants="A7XX-"/>
+	<doc>generate a VS|PS_done event</doc>
+	<value name="CP_EVENT_WRITE_SHD" value="0x58"/>
+	<doc>generate a cache flush done event</doc>
+	<value name="CP_EVENT_WRITE_CFL" value="0x59"/>
+	<doc>generate a z_pass done event</doc>
+	<value name="CP_EVENT_WRITE_ZPD" value="0x5b"/>
+	<doc>
+		not sure the real name, but this seems to be what is used for
+		opencl, instead of CP_DRAW_INDX..
+	</doc>
+	<value name="CP_RUN_OPENCL" value="0x31"/>
+	<doc>initiate fetch of index buffer and draw</doc>
+	<value name="CP_DRAW_INDX" value="0x22"/>
+	<doc>draw using supplied indices in packet</doc>
+	<value name="CP_DRAW_INDX_2" value="0x36" variants="A2XX-A4XX"/>  <!-- this is something different on a6xx and unused on a5xx -->
+	<doc>initiate fetch of index buffer and binIDs and draw</doc>
+	<value name="CP_DRAW_INDX_BIN" value="0x34" variants="A2XX-A4XX"/>
+	<doc>initiate fetch of bin IDs and draw using supplied indices</doc>
+	<value name="CP_DRAW_INDX_2_BIN" value="0x35" variants="A2XX-A4XX"/>
+	<doc>begin/end initiator for viz query extent processing</doc>
+	<value name="CP_VIZ_QUERY" value="0x23" variants="A2XX-A4XX"/>
+	<doc>fetch state sub-blocks and initiate shader code DMAs</doc>
+	<value name="CP_SET_STATE" value="0x25"/>
+	<doc>load constant into chip and to memory</doc>
+	<value name="CP_SET_CONSTANT" value="0x2d" variants="A2XX"/>
+	<doc>load sequencer instruction memory (pointer-based)</doc>
+	<value name="CP_IM_LOAD" value="0x27"/>
+	<doc>load sequencer instruction memory (code embedded in packet)</doc>
+	<value name="CP_IM_LOAD_IMMEDIATE" value="0x2b"/>
+	<doc>load constants from a location in memory</doc>
+	<value name="CP_LOAD_CONSTANT_CONTEXT" value="0x2e" variants="A2XX"/>
+	<doc>selective invalidation of state pointers</doc>
+	<value name="CP_INVALIDATE_STATE" value="0x3b"/>
+	<doc>dynamically changes shader instruction memory partition</doc>
+	<value name="CP_SET_SHADER_BASES" value="0x4a" variants="A2XX-A4XX"/>
+	<doc>sets the 64-bit BIN_MASK register in the PFP</doc>
+	<value name="CP_SET_BIN_MASK" value="0x50" variants="A2XX-A4XX"/>
+	<doc>sets the 64-bit BIN_SELECT register in the PFP</doc>
+	<value name="CP_SET_BIN_SELECT" value="0x51" variants="A2XX-A4XX"/>
+	<doc>updates the current context, if needed</doc>
+	<value name="CP_CONTEXT_UPDATE" value="0x5e"/>
+	<doc>generate interrupt from the command stream</doc>
+	<value name="CP_INTERRUPT" value="0x40"/>
+	<doc>copy sequencer instruction memory to system memory</doc>
+	<value name="CP_IM_STORE" value="0x2c" variants="A2XX"/>
+
+	<!-- For a20x -->
+<!-- TODO handle variants..
+	<doc>
+		Program an offset that will added to the BIN_BASE value of
+		the 3D_DRAW_INDX_BIN packet
+	</doc>
+	<value name="CP_SET_BIN_BASE_OFFSET" value="0x4b"/>
+ -->
+
+	<!-- for a22x -->
+	<doc>
+		sets draw initiator flags register in PFP, gets bitwise-ORed into
+		every draw initiator
+	</doc>
+	<value name="CP_SET_DRAW_INIT_FLAGS" value="0x4b"/>
+	<doc>sets the register protection mode</doc>
+	<value name="CP_SET_PROTECTED_MODE" value="0x5f"/>
+
+	<value name="CP_BOOTSTRAP_UCODE" value="0x6f"/>
+
+	<!-- for a3xx -->
+	<doc>load high level sequencer command</doc>
+	<value name="CP_LOAD_STATE" value="0x30" variants="A3XX"/>
+	<value name="CP_LOAD_STATE4" value="0x30" variants="A4XX-A5XX"/>
+	<doc>Conditionally load a IB based on a flag, prefetch enabled</doc>
+	<value name="CP_COND_INDIRECT_BUFFER_PFE" value="0x3a"/>
+	<doc>Conditionally load a IB based on a flag, prefetch disabled</doc>
+	<value name="CP_COND_INDIRECT_BUFFER_PFD" value="0x32" variants="A3XX"/>
+	<doc>Load a buffer with pre-fetch enabled</doc>
+	<value name="CP_INDIRECT_BUFFER_PFE" value="0x3f" variants="A5XX"/>
+	<doc>Set bin (?)</doc>
+	<value name="CP_SET_BIN" value="0x4c" variants="A2XX"/>
+
+	<doc>test 2 memory locations to dword values specified</doc>
+	<value name="CP_TEST_TWO_MEMS" value="0x71"/>
+
+	<doc>Write register, ignoring context state for context sensitive registers</doc>
+	<value name="CP_REG_WR_NO_CTXT" value="0x78"/>
+
+	<doc>Record the real-time when this packet is processed by PFP</doc>
+	<value name="CP_RECORD_PFP_TIMESTAMP" value="0x11"/>
+
+	<!-- Used to switch GPU between secure and non-secure modes -->
+	<value name="CP_SET_SECURE_MODE" value="0x66"/>
+
+	<doc>PFP waits until the FIFO between the PFP and the ME is empty</doc>
+	<value name="CP_WAIT_FOR_ME" value="0x13"/>
+
+	<!-- for a4xx -->
+	<doc>
+		Used a bit like CP_SET_CONSTANT on a2xx, but can write multiple
+		groups of registers.  Looks like it can be used to create state
+		objects in GPU memory, and on state change only emit pointer
+		(via CP_SET_DRAW_STATE), which should be nice for reducing CPU
+		overhead:
+
+		(A4x) save PM4 stream pointers to execute upon a visible draw
+	</doc>
+	<value name="CP_SET_DRAW_STATE" value="0x43" variants="A4XX-"/>
+	<value name="CP_DRAW_INDX_OFFSET" value="0x38"/>
+	<value name="CP_DRAW_INDIRECT" value="0x28" variants="A4XX-"/>
+	<value name="CP_DRAW_INDX_INDIRECT" value="0x29" variants="A4XX-"/>
+	<value name="CP_DRAW_INDIRECT_MULTI" value="0x2a" variants="A6XX-"/>
+	<value name="CP_DRAW_AUTO" value="0x24"/>
+
+	<doc>
+		Enable or disable predication globally. Also resets the
+		predicate to "passing" and the local bit to enabled when
+		enabling global predication.
+	</doc>
+	<value name="CP_DRAW_PRED_ENABLE_GLOBAL" value="0x19"/>
+
+	<doc>
+		Enable or disable predication locally. Unlike globally enabling
+		predication, this packet doesn't touch any other state.
+		Predication only happens when enabled globally and locally and a
+		predicate has been set. This should be used for internal draws
+		which aren't supposed to use the predication state:
+
+		CP_DRAW_PRED_ENABLE_LOCAL(0)
+		... do draw...
+		CP_DRAW_PRED_ENABLE_LOCAL(1)
+	</doc>
+	<value name="CP_DRAW_PRED_ENABLE_LOCAL" value="0x1a"/>
+
+	<doc>
+		Latch a draw predicate into the internal register.
+	</doc>
+	<value name="CP_DRAW_PRED_SET" value="0x4e"/>
+
+	<doc>
+		for A4xx
+		Write to register with address that does not fit into type-0 pkt
+	</doc>
+	<value name="CP_WIDE_REG_WRITE" value="0x74" variants="A4XX"/>
+
+	<doc>copy from ME scratch RAM to a register</doc>
+	<value name="CP_SCRATCH_TO_REG" value="0x4d"/>
+
+	<doc>Copy from REG to ME scratch RAM</doc>
+	<value name="CP_REG_TO_SCRATCH" value="0x4a"/>
+
+	<doc>Wait for memory writes to complete</doc>
+	<value name="CP_WAIT_MEM_WRITES" value="0x12"/>
+
+	<doc>Conditional execution based on register comparison</doc>
+	<value name="CP_COND_REG_EXEC" value="0x47"/>
+
+	<doc>Memory to REG copy</doc>
+	<value name="CP_MEM_TO_REG" value="0x42"/>
+
+	<value name="CP_EXEC_CS_INDIRECT" value="0x41" variants="A4XX-"/>
+	<value name="CP_EXEC_CS" value="0x33"/>
+
+	<doc>
+		for a5xx
+	</doc>
+	<value name="CP_PERFCOUNTER_ACTION" value="0x50" variants="A5XX"/>
+	<!-- switches SMMU pagetable, used on a5xx+ only -->
+	<value name="CP_SMMU_TABLE_UPDATE" value="0x53" variants="A5XX-"/>
+	<!-- for a6xx -->
+	<doc>Tells CP the current mode of GPU operation</doc>
+	<value name="CP_SET_MARKER" value="0x65" variants="A6XX-"/>
+	<doc>Instruct CP to set a few internal CP registers</doc>
+	<value name="CP_SET_PSEUDO_REG" value="0x56" variants="A6XX-"/>
+	<!--
+	pairs of regid and value.. seems to be used to program some TF
+	related regs:
+	 -->
+	<value name="CP_CONTEXT_REG_BUNCH" value="0x5c" variants="A5XX-"/>
+	<!-- A5XX Enable yield in RB only -->
+	<value name="CP_YIELD_ENABLE" value="0x1c" variants="A5XX"/>
+	<doc>
+		Enables IB2 skipping.  If both GLOBAL and LOCAL are 1 and
+		nothing is left in the visibility stream, then
+		CP_INDIRECT_BUFFER will be skipped, and draws will early return
+		from their IB.
+	</doc>
+	<value name="CP_SKIP_IB2_ENABLE_GLOBAL" value="0x1d" variants="A5XX-"/>
+	<value name="CP_SKIP_IB2_ENABLE_LOCAL" value="0x23" variants="A5XX-"/>
+	<value name="CP_SET_SUBDRAW_SIZE" value="0x35" variants="A5XX-"/>
+	<value name="CP_WHERE_AM_I" value="0x62" variants="A5XX-"/>
+	<value name="CP_SET_VISIBILITY_OVERRIDE" value="0x64" variants="A5XX-"/>
+	<!-- Enable/Disable/Defer A5x global preemption model -->
+	<value name="CP_PREEMPT_ENABLE_GLOBAL" value="0x69" variants="A5XX"/>
+	<!-- Enable/Disable A5x local preemption model -->
+	<value name="CP_PREEMPT_ENABLE_LOCAL" value="0x6a" variants="A5XX"/>
+	<!-- Yield token on a5xx similar to CP_PREEMPT on a4xx -->
+	<value name="CP_CONTEXT_SWITCH_YIELD" value="0x6b" variants="A5XX-"/>
+	<!-- Inform CP about current render mode (needed for a5xx preemption) -->
+	<value name="CP_SET_RENDER_MODE" value="0x6c" variants="A5XX"/>
+	<value name="CP_COMPUTE_CHECKPOINT" value="0x6e" variants="A5XX"/>
+	<!-- check if this works on earlier.. -->
+	<value name="CP_MEM_TO_MEM" value="0x73" variants="A5XX-"/>
+
+	<doc>
+		General purpose 2D blit engine for image transfers and mipmap
+		generation.  Reads through UCHE, writes through the CCU cache in
+		the PS stage.
+	</doc>
+	<value name="CP_BLIT" value="0x2c" variants="A5XX-"/>
+
+	<!-- Test specified bit in specified register and set predicate -->
+	<value name="CP_REG_TEST" value="0x39" variants="A5XX-"/>
+
+	<!--
+	Seems to set the mode flags which control which CP_SET_DRAW_STATE
+	packets are executed, based on their ENABLE_MASK values
+	
+	CP_SET_MODE w/ payload of 0x1 seems to cause CP_SET_DRAW_STATE
+	packets w/ ENABLE_MASK & 0x6 to execute immediately
+	 -->
+	<value name="CP_SET_MODE" value="0x63" variants="A6XX-"/>
+
+	<!--
+	Seems like there are now separate blocks of state for VS vs FS/CS
+	(probably these amounts to geometry vs fragments so that geometry
+	stage of the pipeline for next draw can start while fragment stage
+	of current draw is still running.  The format of the payload of the
+	packets is the same, the only difference is the offsets of the regs
+	the firmware code that handles the packet writes.
+
+	Note that for CL, starting with a6xx, the preferred # of local
+	threads is no longer the same as the max, implying that the shader
+	core can now run warps from unrelated shaders (ie.
+	CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE vs
+	CL_KERNEL_WORK_GROUP_SIZE)
+	 -->
+	<value name="CP_LOAD_STATE6_GEOM" value="0x32" variants="A6XX-"/>
+	<value name="CP_LOAD_STATE6_FRAG" value="0x34" variants="A6XX-"/>
+	<!--
+	Note: For IBO state (Image/SSBOs) which have shared state across
+	shader stages, for 3d pipeline CP_LOAD_STATE6 is used.  But for
+	compute shaders, CP_LOAD_STATE6_FRAG is used.  Possibly they are
+	interchangable.
+	 -->
+	<value name="CP_LOAD_STATE6" value="0x36" variants="A6XX-"/>
+
+	<!-- internal packets: -->
+	<value name="IN_IB_PREFETCH_END" value="0x17" variants="A2XX"/>
+	<value name="IN_SUBBLK_PREFETCH" value="0x1f" variants="A2XX"/>
+	<value name="IN_INSTR_PREFETCH" value="0x20" variants="A2XX"/>
+	<value name="IN_INSTR_MATCH" value="0x47" variants="A2XX"/>
+	<value name="IN_CONST_PREFETCH" value="0x49" variants="A2XX"/>
+	<value name="IN_INCR_UPDT_STATE" value="0x55" variants="A2XX"/>
+	<value name="IN_INCR_UPDT_CONST" value="0x56" variants="A2XX"/>
+	<value name="IN_INCR_UPDT_INSTR" value="0x57" variants="A2XX"/>
+
+	<!-- internal jumptable entries on a6xx+, possibly a5xx: -->
+
+	<!-- jmptable entry used to handle type4 packet on a5xx+: -->
+	<value name="PKT4" value="0x04" variants="A5XX-"/>
+	<!-- called when ROQ is empty, "returns" from an IB or merged sequence of IBs -->
+	<value name="IN_IB_END" value="0x0a" variants="A6XX-"/>
+	<!-- handles IFPC save/restore -->
+	<value name="IN_GMU_INTERRUPT" value="0x0b" variants="A6XX-"/>
+	<!-- preemption/context-swtich routine -->
+	<value name="IN_PREEMPT" value="0x0f" variants="A6XX-"/>
+
+	<!-- TODO do these exist on A5xx? -->
+	<value name="CP_SCRATCH_WRITE" value="0x4c" variants="A6XX"/>
+	<value name="CP_REG_TO_MEM_OFFSET_MEM" value="0x74" variants="A6XX-"/>
+	<value name="CP_REG_TO_MEM_OFFSET_REG" value="0x72" variants="A6XX-"/>
+	<value name="CP_WAIT_MEM_GTE" value="0x14" variants="A6XX"/>
+	<value name="CP_WAIT_TWO_REGS" value="0x70" variants="A6XX"/>
+	<value name="CP_MEMCPY" value="0x75" variants="A6XX-"/>
+	<value name="CP_SET_BIN_DATA5_OFFSET" value="0x2e" variants="A6XX-"/>
+	<!-- A750+, set in place of CP_SET_BIN_DATA5_OFFSET but has different values -->
+	<value name="CP_SET_UNK_BIN_DATA" value="0x2d" variants="A7XX-"/>
+	<doc>
+                Write CP_CONTEXT_SWITCH_*_INFO from CP to the following dwords,
+                and forcibly switch to the indicated context.
+	</doc>
+	<value name="CP_CONTEXT_SWITCH" value="0x54" variants="A6XX"/>
+	<!-- Note, kgsl calls this CP_SET_AMBLE: -->
+	<value name="CP_SET_CTXSWITCH_IB" value="0x55" variants="A6XX-"/>
+
+	<!--
+	Seems to always have the payload:
+	  00000002 00008801 00004010
+	or:
+	  00000002 00008801 00004090
+	or:
+	  00000002 00008801 00000010
+	  00000002 00008801 00010010
+	  00000002 00008801 00d64010
+	  ...
+	Note set for compute shaders..
+	Is 0x8801 a register offset?
+	This appears to be a special sort of register write packet
+	more or less, but the firmware has some special handling..
+	Seems like it intercepts/modifies certain register offsets,
+	but others are treated like a normal PKT4 reg write.  I
+	guess there are some registers that the fw controls certain
+	bits.
+	 -->
+	<value name="CP_REG_WRITE" value="0x6d" variants="A6XX"/>
+
+	<doc>
+		These first appear in a650_sqe.bin. They can in theory be used
+		to loop any sequence of IB1 commands, but in practice they are
+		used to loop over bins. There is a fixed-size per-iteration
+		prefix, used to set per-bin state, and then the following IB1
+		commands are executed until CP_END_BIN which are always the same
+		for each iteration and usually contain a list of
+		CP_INDIRECT_BUFFER calls to IB2 commands which setup state and
+		execute restore/draw/save commands. This replaces the previous
+		technique of just repeating the CP_INDIRECT_BUFFER calls and
+		"unrolling" the loop.
+	</doc>
+	<value name="CP_START_BIN" value="0x50" variants="A6XX-"/>
+	<value name="CP_END_BIN" value="0x51" variants="A6XX-"/>
+
+	<doc> Make next dword 1 to disable preemption, 0 to re-enable it. </doc>
+	<value name="CP_PREEMPT_DISABLE" value="0x6c" variants="A6XX"/>
+
+	<value name="CP_WAIT_TIMESTAMP" value="0x14" variants="A7XX-"/>
+	<value name="CP_GLOBAL_TIMESTAMP" value="0x15" variants="A7XX-"/>  <!-- payload 1 dword -->
+	<value name="CP_LOCAL_TIMESTAMP" value="0x16" variants="A7XX-"/>  <!-- payload 1 dword, follows 0x15 -->
+	<value name="CP_THREAD_CONTROL" value="0x17" variants="A7XX-"/>
+	<!-- payload 4 dwords, last two could be render target addr (one pkt per MRT), possibly used for GMEM save/restore?-->
+	<value name="CP_RESOURCE_LIST" value="0x18" variants="A7XX-"/>
+	<doc> Can clear BV/BR counters, or wait until one catches up to another </doc>
+	<value name="CP_BV_BR_COUNT_OPS" value="0x1b" variants="A7XX-"/>
+	<doc> Clears, adds to local, or adds to global timestamp </doc>
+	<value name="CP_MODIFY_TIMESTAMP" value="0x1c" variants="A7XX-"/>
+	<!-- similar to CP_CONTEXT_REG_BUNCH, but discards first two dwords?? -->
+	<value name="CP_CONTEXT_REG_BUNCH2" value="0x5d" variants="A7XX-"/>
+	<doc>
+		Write to a scratch memory that is read by CP_REG_TEST with
+		SOURCE_SCRATCH_MEM set. It's not the same scratch as scratch registers.
+		However it uses the same memory space.
+	</doc>
+	<value name="CP_MEM_TO_SCRATCH_MEM" value="0x49" variants="A7XX-"/>
+
+	<doc>
+		Executes an array of fixed-size command buffers where each
+		buffer is assumed to have one draw call, skipping buffers with
+		non-visible draw calls.
+	</doc>
+	<value name="CP_FIXED_STRIDE_DRAW_TABLE" value="0x7f" variants="A7XX-"/>
+
+	<doc>Reset various on-chip state used for synchronization</doc>
+	<value name="CP_RESET_CONTEXT_STATE" value="0x1f" variants="A7XX-"/>
+</enum>
+
+
+<domain name="CP_LOAD_STATE" width="32">
+	<doc>Load state, a3xx (and later?)</doc>
+	<enum name="adreno_state_block">
+		<value name="SB_VERT_TEX" value="0"/>
+		<value name="SB_VERT_MIPADDR" value="1"/>
+		<value name="SB_FRAG_TEX" value="2"/>
+		<value name="SB_FRAG_MIPADDR" value="3"/>
+		<value name="SB_VERT_SHADER" value="4"/>
+		<value name="SB_GEOM_SHADER" value="5"/>
+		<value name="SB_FRAG_SHADER" value="6"/>
+		<value name="SB_COMPUTE_SHADER" value="7"/>
+	</enum>
+	<enum name="adreno_state_type">
+		<value name="ST_SHADER" value="0"/>
+		<value name="ST_CONSTANTS" value="1"/>
+	</enum>
+	<enum name="adreno_state_src">
+		<value name="SS_DIRECT" value="0">
+			<doc>inline with the CP_LOAD_STATE packet</doc>
+		</value>
+		<value name="SS_INVALID_ALL_IC" value="2"/>
+		<value name="SS_INVALID_PART_IC" value="3"/>
+		<value name="SS_INDIRECT" value="4">
+			<doc>in buffer pointed to by EXT_SRC_ADDR</doc>
+		</value>
+		<value name="SS_INDIRECT_TCM" value="5"/>
+		<value name="SS_INDIRECT_STM" value="6"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="DST_OFF" low="0" high="15" type="uint"/>
+		<bitfield name="STATE_SRC" low="16" high="18" type="adreno_state_src"/>
+		<bitfield name="STATE_BLOCK" low="19" high="21" type="adreno_state_block"/>
+		<bitfield name="NUM_UNIT" low="22" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="STATE_TYPE" low="0" high="1" type="adreno_state_type"/>
+		<bitfield name="EXT_SRC_ADDR" low="2" high="31" shr="2"/>
+	</reg32>
+</domain>
+
+<domain name="CP_LOAD_STATE4" width="32" varset="chip">
+	<doc>Load state, a4xx+</doc>
+	<enum name="a4xx_state_block">
+		<!--
+		unknown: 0x7 and 0xf <- seen in compute shader
+
+		STATE_BLOCK = 0x6, STATE_TYPE = 0x2 possibly used for preemption?
+		Seen in some GL shaders.  Payload is NUM_UNIT dwords, and it contains
+		the gpuaddr of the following shader constants block.  DST_OFF seems
+		to specify which shader stage:
+
+		    16 -> vert
+		    36 -> tcs
+		    56 -> tes
+		    76 -> geom
+		    96 -> frag
+
+		Example:
+
+opcode: CP_LOAD_STATE4 (30) (12 dwords)
+        { DST_OFF = 16 | STATE_SRC = SS4_DIRECT | STATE_BLOCK = 0x6 | NUM_UNIT = 4 }
+        { STATE_TYPE = 0x2 | EXT_SRC_ADDR = 0 }
+        { EXT_SRC_ADDR_HI = 0 }
+                        0000: c0264100 00000000 00000000 00000000
+                0000: 70b0000b 01180010 00000002 00000000 c0264100 00000000 00000000 00000000
+
+opcode: CP_LOAD_STATE4 (30) (4 dwords)
+        { DST_OFF = 16 | STATE_SRC = SS4_INDIRECT | STATE_BLOCK = SB4_VS_SHADER | NUM_UNIT = 4 }
+        { STATE_TYPE = ST4_CONSTANTS | EXT_SRC_ADDR = 0xc0264100 }
+        { EXT_SRC_ADDR_HI = 0 }
+                        0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
+                        0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
+                        0000: 00000040 0000000c 00000000 00000000 00000000 00000000 00000000 00000000
+
+		STATE_BLOCK = 0x6, STATE_TYPE = 0x1, seen in compute shader.  NUM_UNITS * 2 dwords.
+
+		 -->
+		<value name="SB4_VS_TEX"    value="0x0"/>
+		<value name="SB4_HS_TEX"    value="0x1"/>    <!-- aka. TCS -->
+		<value name="SB4_DS_TEX"    value="0x2"/>    <!-- aka. TES -->
+		<value name="SB4_GS_TEX"    value="0x3"/>
+		<value name="SB4_FS_TEX"    value="0x4"/>
+		<value name="SB4_CS_TEX"    value="0x5"/>
+		<value name="SB4_VS_SHADER" value="0x8"/>
+		<value name="SB4_HS_SHADER" value="0x9"/>
+		<value name="SB4_DS_SHADER" value="0xa"/>
+		<value name="SB4_GS_SHADER" value="0xb"/>
+		<value name="SB4_FS_SHADER" value="0xc"/>
+		<value name="SB4_CS_SHADER" value="0xd"/>
+		<!--
+		for SSBO, STATE_TYPE=0 appears to be addresses (four dwords each),
+		STATE_TYPE=1 sizes, STATE_TYPE=2 addresses again (two dwords each)
+
+		Compute has it's own dedicated SSBO state, it seems, but the rest
+		of the stages share state
+		 -->
+		<value name="SB4_SSBO"   value="0xe"/>
+		<value name="SB4_CS_SSBO"   value="0xf"/>
+	</enum>
+	<enum name="a4xx_state_type">
+		<value name="ST4_SHADER" value="0"/>
+		<value name="ST4_CONSTANTS" value="1"/>
+		<value name="ST4_UBO" value="2"/>
+	</enum>
+	<enum name="a4xx_state_src">
+		<value name="SS4_DIRECT" value="0"/>
+		<value name="SS4_INDIRECT" value="2"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="DST_OFF" low="0" high="13" type="uint"/>
+		<bitfield name="STATE_SRC" low="16" high="17" type="a4xx_state_src"/>
+		<bitfield name="STATE_BLOCK" low="18" high="21" type="a4xx_state_block"/>
+		<bitfield name="NUM_UNIT" low="22" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="STATE_TYPE" low="0" high="1" type="a4xx_state_type"/>
+		<bitfield name="EXT_SRC_ADDR" low="2" high="31" shr="2"/>
+	</reg32>
+	<reg32 offset="2" name="2" varset="chip" variants="A5XX-">
+		<bitfield name="EXT_SRC_ADDR_HI" low="0" high="31" shr="0"/>
+	</reg32>
+</domain>
+
+<!-- looks basically same CP_LOAD_STATE4 -->
+<domain name="CP_LOAD_STATE6" width="32" varset="chip">
+	<doc>Load state, a6xx+</doc>
+	<enum name="a6xx_state_block">
+		<value name="SB6_VS_TEX"    value="0x0"/>
+		<value name="SB6_HS_TEX"    value="0x1"/>    <!-- aka. TCS -->
+		<value name="SB6_DS_TEX"    value="0x2"/>    <!-- aka. TES -->
+		<value name="SB6_GS_TEX"    value="0x3"/>
+		<value name="SB6_FS_TEX"    value="0x4"/>
+		<value name="SB6_CS_TEX"    value="0x5"/>
+		<value name="SB6_VS_SHADER" value="0x8"/>
+		<value name="SB6_HS_SHADER" value="0x9"/>
+		<value name="SB6_DS_SHADER" value="0xa"/>
+		<value name="SB6_GS_SHADER" value="0xb"/>
+		<value name="SB6_FS_SHADER" value="0xc"/>
+		<value name="SB6_CS_SHADER" value="0xd"/>
+		<value name="SB6_IBO"       value="0xe"/>
+		<value name="SB6_CS_IBO"    value="0xf"/>
+	</enum>
+	<enum name="a6xx_state_type">
+		<value name="ST6_SHADER" value="0"/>
+		<value name="ST6_CONSTANTS" value="1"/>
+		<value name="ST6_UBO" value="2"/>
+		<value name="ST6_IBO" value="3"/>
+	</enum>
+	<enum name="a6xx_state_src">
+		<value name="SS6_DIRECT" value="0"/>
+		<value name="SS6_BINDLESS" value="1"/> <!-- TODO does this exist on a4xx/a5xx? -->
+		<value name="SS6_INDIRECT" value="2"/>
+		<doc>
+		SS6_UBO used by the a6xx vulkan blob with tesselation constants
+		in this case, EXT_SRC_ADDR is (ubo_id shl 16 | offset)
+		to load constants from a UBO loaded with DST_OFF = 14 and offset 0,
+		EXT_SRC_ADDR = 0xe0000
+		(offset is a guess, should be in bytes given that maxUniformBufferRange=64k)
+		</doc>
+		<value name="SS6_UBO" value="3"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="DST_OFF" low="0" high="13" type="uint"/>
+		<bitfield name="STATE_TYPE" low="14" high="15" type="a6xx_state_type"/>
+		<bitfield name="STATE_SRC" low="16" high="17" type="a6xx_state_src"/>
+		<bitfield name="STATE_BLOCK" low="18" high="21" type="a6xx_state_block"/>
+		<bitfield name="NUM_UNIT" low="22" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="EXT_SRC_ADDR" low="2" high="31" shr="2"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="EXT_SRC_ADDR_HI" low="0" high="31" shr="0"/>
+	</reg32>
+	<reg64 offset="1" name="EXT_SRC_ADDR" type="address"/>
+</domain>
+
+<bitset name="vgt_draw_initiator" inline="yes">
+	<bitfield name="PRIM_TYPE" low="0" high="5" type="pc_di_primtype"/>
+	<bitfield name="SOURCE_SELECT" low="6" high="7" type="pc_di_src_sel"/>
+	<bitfield name="VIS_CULL" low="9" high="10" type="pc_di_vis_cull_mode"/>
+	<bitfield name="INDEX_SIZE" pos="11" type="pc_di_index_size"/>
+	<bitfield name="NOT_EOP" pos="12" type="boolean"/>
+	<bitfield name="SMALL_INDEX" pos="13" type="boolean"/>
+	<bitfield name="PRE_DRAW_INITIATOR_ENABLE" pos="14" type="boolean"/>
+	<bitfield name="NUM_INSTANCES" low="24" high="31" type="uint"/>
+</bitset>
+
+<!-- changed on a4xx: -->
+<enum name="a4xx_index_size">
+	<value name="INDEX4_SIZE_8_BIT" value="0"/>
+	<value name="INDEX4_SIZE_16_BIT" value="1"/>
+	<value name="INDEX4_SIZE_32_BIT" value="2"/>
+</enum>
+
+<enum name="a6xx_patch_type">
+  <value name="TESS_QUADS" value="0"/>
+  <value name="TESS_TRIANGLES" value="1"/>
+  <value name="TESS_ISOLINES" value="2"/>
+</enum>
+
+<bitset name="vgt_draw_initiator_a4xx" inline="yes">
+	<!-- When the 0x20 bit is set, it's the number of patch vertices - 1 -->
+	<bitfield name="PRIM_TYPE" low="0" high="5" type="pc_di_primtype"/>
+	<bitfield name="SOURCE_SELECT" low="6" high="7" type="pc_di_src_sel"/>
+	<bitfield name="VIS_CULL" low="8" high="9" type="pc_di_vis_cull_mode"/>
+	<bitfield name="INDEX_SIZE" low="10" high="11" type="a4xx_index_size"/>
+	<bitfield name="PATCH_TYPE" low="12" high="13" type="a6xx_patch_type"/>
+	<bitfield name="GS_ENABLE" pos="16" type="boolean"/>
+	<bitfield name="TESS_ENABLE" pos="17" type="boolean"/>
+</bitset>
+
+<domain name="CP_DRAW_INDX" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="VIZ_QUERY" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="1" name="1" type="vgt_draw_initiator"/>
+	<reg32 offset="2" name="2">
+		<bitfield name="NUM_INDICES" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="INDX_BASE" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="4" name="4">
+		<bitfield name="INDX_SIZE" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain name="CP_DRAW_INDX_2" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="VIZ_QUERY" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="1" name="1" type="vgt_draw_initiator"/>
+	<reg32 offset="2" name="2">
+		<bitfield name="NUM_INDICES" low="0" high="31" type="uint"/>
+	</reg32>
+	<!-- followed by NUM_INDICES indices.. -->
+</domain>
+
+<domain name="CP_DRAW_INDX_OFFSET" width="32">
+	<reg32 offset="0" name="0" type="vgt_draw_initiator_a4xx"/>
+	<reg32 offset="1" name="1">
+		<bitfield name="NUM_INSTANCES" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="NUM_INDICES" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="FIRST_INDX" low="0" high="31"/>
+	</reg32>
+
+	<stripe varset="chip" variants="A5XX-">
+		<reg32 offset="4" name="4">
+			<bitfield name="INDX_BASE_LO" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="5" name="5">
+			<bitfield name="INDX_BASE_HI" low="0" high="31"/>
+		</reg32>
+		<reg64 offset="4" name="INDX_BASE" type="address"/>
+		<reg32 offset="6" name="6">
+			<!-- max # of elements in index buffer -->
+			<bitfield name="MAX_INDICES" low="0" high="31"/>
+		</reg32>
+	</stripe>
+
+	<reg32 offset="4" name="4">
+		<bitfield name="INDX_BASE" low="0" high="31" type="address"/>
+	</reg32>
+
+	<reg32 offset="5" name="5">
+		<bitfield name="INDX_SIZE" low="0" high="31" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_DRAW_INDIRECT" width="32" varset="chip" prefix="chip" variants="A4XX-">
+	<reg32 offset="0" name="0" type="vgt_draw_initiator_a4xx"/>
+	<stripe varset="chip" variants="A4XX">
+		<reg32 offset="1" name="1">
+			<bitfield name="INDIRECT" low="0" high="31"/>
+		</reg32>
+	</stripe>
+	<stripe varset="chip" variants="A5XX-">
+		<reg32 offset="1" name="1">
+			<bitfield name="INDIRECT_LO" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="2" name="2">
+			<bitfield name="INDIRECT_HI" low="0" high="31"/>
+		</reg32>
+		<reg64 offset="1" name="INDIRECT" type="address"/>
+	</stripe>
+</domain>
+
+<domain name="CP_DRAW_INDX_INDIRECT" width="32" varset="chip" prefix="chip" variants="A4XX-">
+	<reg32 offset="0" name="0" type="vgt_draw_initiator_a4xx"/>
+	<stripe varset="chip" variants="A4XX">
+		<reg32 offset="1" name="1">
+			<bitfield name="INDX_BASE" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="2" name="2">
+			<!-- max # of bytes in index buffer -->
+			<bitfield name="INDX_SIZE" low="0" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="3" name="3">
+			<bitfield name="INDIRECT" low="0" high="31"/>
+		</reg32>
+	</stripe>
+	<stripe varset="chip" variants="A5XX-">
+		<reg32 offset="1" name="1">
+			<bitfield name="INDX_BASE_LO" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="2" name="2">
+			<bitfield name="INDX_BASE_HI" low="0" high="31"/>
+		</reg32>
+		<reg64 offset="1" name="INDX_BASE" type="address"/>
+		<reg32 offset="3" name="3">
+			<!-- max # of elements in index buffer -->
+			<bitfield name="MAX_INDICES" low="0" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="4" name="4">
+			<bitfield name="INDIRECT_LO" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="5" name="5">
+			<bitfield name="INDIRECT_HI" low="0" high="31"/>
+		</reg32>
+		<reg64 offset="4" name="INDIRECT" type="address"/>
+	</stripe>
+</domain>
+
+<domain name="CP_DRAW_INDIRECT_MULTI" width="32" varset="chip" prefix="chip" variants="A6XX-">
+	<enum name="a6xx_draw_indirect_opcode">
+		<value name="INDIRECT_OP_NORMAL"  value="0x2"/>
+		<value name="INDIRECT_OP_INDEXED" value="0x4"/>
+		<value name="INDIRECT_OP_INDIRECT_COUNT" value="0x6"/>
+		<value name="INDIRECT_OP_INDIRECT_COUNT_INDEXED" value="0x7"/>
+	</enum>
+	<reg32 offset="0" name="0" type="vgt_draw_initiator_a4xx"/>
+	<reg32 offset="1" name="1">
+		<bitfield name="OPCODE" low="0" high="3" type="a6xx_draw_indirect_opcode" addvariant="yes"/>
+		<doc>
+		DST_OFF same as in CP_LOAD_STATE6 - vec4 VS const at this offset will
+		be updated for each draw to {draw_id, first_vertex, first_instance, 0}
+		value of 0 disables it
+		</doc>
+		<bitfield name="DST_OFF" low="8" high="21" type="hex"/>
+	</reg32>
+	<reg32 offset="2" name="DRAW_COUNT" type="uint"/>
+	<stripe varset="a6xx_draw_indirect_opcode" variants="INDIRECT_OP_NORMAL">
+		<reg64 offset="3" name="INDIRECT" type="address"/>
+		<reg32 offset="5" name="STRIDE" type="uint"/>
+	</stripe>
+	<stripe varset="a6xx_draw_indirect_opcode" variants="INDIRECT_OP_INDEXED" prefix="INDEXED">
+		<reg64 offset="3" name="INDEX" type="address"/>
+		<reg32 offset="5" name="MAX_INDICES" type="uint"/>
+		<reg64 offset="6" name="INDIRECT" type="address"/>
+		<reg32 offset="8" name="STRIDE" type="uint"/>
+	</stripe>
+	<stripe varset="a6xx_draw_indirect_opcode" variants="INDIRECT_OP_INDIRECT_COUNT" prefix="INDIRECT">
+		<reg64 offset="3" name="INDIRECT" type="address"/>
+		<reg64 offset="5" name="INDIRECT_COUNT" type="address"/>
+		<reg32 offset="7" name="STRIDE" type="uint"/>
+	</stripe>
+	<stripe varset="a6xx_draw_indirect_opcode" variants="INDIRECT_OP_INDIRECT_COUNT_INDEXED" prefix="INDIRECT_INDEXED">
+		<reg64 offset="3" name="INDEX" type="address"/>
+		<reg32 offset="5" name="MAX_INDICES" type="uint"/>
+		<reg64 offset="6" name="INDIRECT" type="address"/>
+		<reg64 offset="8" name="INDIRECT_COUNT" type="address"/>
+		<reg32 offset="10" name="STRIDE" type="uint"/>
+	</stripe>
+</domain>
+
+<domain name="CP_DRAW_AUTO" width="32">
+	<reg32 offset="0" name="0" type="vgt_draw_initiator_a4xx"/>
+	<reg32 offset="1" name="1">
+		<bitfield name="NUM_INSTANCES" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg64 offset="2" name="NUM_VERTICES_BASE" type="address"/>
+	<reg32 offset="4" name="4">
+		<bitfield name="NUM_VERTICES_OFFSET" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="5" name="5">
+		<bitfield name="STRIDE" low="0" high="31" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_DRAW_PRED_ENABLE_GLOBAL" width="32" varset="chip">
+	<reg32 offset="0" name="0">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+	</reg32>
+</domain>
+
+<domain name="CP_DRAW_PRED_ENABLE_LOCAL" width="32" varset="chip">
+	<reg32 offset="0" name="0">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+	</reg32>
+</domain>
+
+<domain name="CP_DRAW_PRED_SET" width="32" varset="chip">
+	<enum name="cp_draw_pred_src">
+		<!--
+			Sources 1-4 seem to be about combining reading
+			SO/primitive queries and setting the predicate, which is
+			a DX11-specific optimization (since in DX11 you can only
+			predicate on the result of queries).
+		-->
+		<value name="PRED_SRC_MEM" value="5">
+			<doc>
+				Read a 64-bit value at the given address and
+				test if it equals/doesn't equal 0.
+			</doc>
+		</value>
+	</enum>
+	<enum name="cp_draw_pred_test">
+		<value name="NE_0_PASS" value="0"/>
+		<value name="EQ_0_PASS" value="1"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="SRC" low="4" high="7" type="cp_draw_pred_src"/>
+		<bitfield name="TEST" pos="8" type="cp_draw_pred_test"/>
+	</reg32>
+	<reg64 offset="1" name="MEM_ADDR" type="address"/>
+</domain>
+
+<domain name="CP_SET_DRAW_STATE" width="32" varset="chip" variants="A4XX-">
+	<array offset="0" stride="3" length="100">
+		<reg32 offset="0" name="0">
+			<bitfield name="COUNT" low="0" high="15" type="uint"/>
+			<bitfield name="DIRTY" pos="16" type="boolean"/>
+			<bitfield name="DISABLE" pos="17" type="boolean"/>
+			<bitfield name="DISABLE_ALL_GROUPS" pos="18" type="boolean"/>
+			<bitfield name="LOAD_IMMED" pos="19" type="boolean"/>
+			<bitfield name="BINNING" pos="20" varset="chip" variants="A6XX-" type="boolean"/>
+			<bitfield name="GMEM" pos="21" varset="chip" variants="A6XX-" type="boolean"/>
+			<bitfield name="SYSMEM" pos="22" varset="chip" variants="A6XX-" type="boolean"/>
+			<bitfield name="GROUP_ID" low="24" high="28" type="uint"/>
+		</reg32>
+		<reg32 offset="1" name="1">
+			<bitfield name="ADDR_LO" low="0" high="31" type="hex"/>
+		</reg32>
+		<reg32 offset="2" name="2" varset="chip" variants="A5XX-">
+			<bitfield name="ADDR_HI" low="0" high="31" type="hex"/>
+		</reg32>
+	</array>
+</domain>
+
+<domain name="CP_SET_BIN" width="32">
+	<doc>value at offset 0 always seems to be 0x00000000..</doc>
+	<reg32 offset="0" name="0"/>
+	<reg32 offset="1" name="1">
+		<bitfield name="X1" low="0" high="15" type="uint"/>
+		<bitfield name="Y1" low="16" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="X2" low="0" high="15" type="uint"/>
+		<bitfield name="Y2" low="16" high="31" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_SET_BIN_DATA" width="32">
+	<reg32 offset="0" name="0">
+		<!-- corresponds to VSC_PIPE[n].DATA_ADDR -->
+		<bitfield name="BIN_DATA_ADDR" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<!-- seesm to correspond to VSC_SIZE_ADDRESS -->
+		<bitfield name="BIN_SIZE_ADDRESS" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain name="CP_SET_BIN_DATA5" width="32">
+	<reg32 offset="0" name="0">
+		<!-- equiv to PC_VSTREAM_CONTROL.SIZE on a3xx/a4xx: -->
+		<bitfield name="VSC_SIZE" low="16" high="21" type="uint"/>
+		<!-- equiv to PC_VSTREAM_CONTROL.N on a3xx/a4xx: -->
+		<bitfield name="VSC_N" low="22" high="26" type="uint"/>
+	</reg32>
+	<!-- BIN_DATA_ADDR -> VSC_PIPE[p].DATA_ADDRESS -->
+	<reg32 offset="1" name="1">
+		<bitfield name="BIN_DATA_ADDR_LO" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="BIN_DATA_ADDR_HI" low="0" high="31" type="hex"/>
+	</reg32>
+	<!-- BIN_SIZE_ADDRESS -> VSC_SIZE_ADDRESS + (p * 4)-->
+	<reg32 offset="3" name="3">
+		<bitfield name="BIN_SIZE_ADDRESS_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="4" name="4">
+		<bitfield name="BIN_SIZE_ADDRESS_HI" low="0" high="31"/>
+	</reg32>
+	<!-- new on a6xx, where BIN_DATA_ADDR is the DRAW_STRM: -->
+	<reg32 offset="5" name="5">
+		<bitfield name="BIN_PRIM_STRM_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="6" name="6">
+		<bitfield name="BIN_PRIM_STRM_HI" low="0" high="31"/>
+	</reg32>
+	<!--
+		a7xx adds a few more addresses to the end of the pkt
+	 -->
+	<reg64 offset="7" name="7"/>
+	<reg64 offset="9" name="9"/>
+</domain>
+
+<domain name="CP_SET_BIN_DATA5_OFFSET" width="32">
+	<doc>
+                Like CP_SET_BIN_DATA5, but set the pointers as offsets from the
+                pointers stored in VSC_PIPE_{DATA,DATA2,SIZE}_ADDRESS. Useful
+                for Vulkan where these values aren't known when the command
+                stream is recorded.
+	</doc>
+	<reg32 offset="0" name="0">
+		<!-- equiv to PC_VSTREAM_CONTROL.SIZE on a3xx/a4xx: -->
+		<bitfield name="VSC_SIZE" low="16" high="21" type="uint"/>
+		<!-- equiv to PC_VSTREAM_CONTROL.N on a3xx/a4xx: -->
+		<bitfield name="VSC_N" low="22" high="26" type="uint"/>
+	</reg32>
+	<!-- BIN_DATA_ADDR -> VSC_PIPE[p].DATA_ADDRESS -->
+	<reg32 offset="1" name="1">
+		<bitfield name="BIN_DATA_OFFSET" low="0" high="31" type="uint"/>
+	</reg32>
+	<!-- BIN_SIZE_ADDRESS -> VSC_SIZE_ADDRESS + (p * 4)-->
+	<reg32 offset="2" name="2">
+		<bitfield name="BIN_SIZE_OFFSET" low="0" high="31" type="uint"/>
+	</reg32>
+	<!-- BIN_DATA2_ADDR -> VSC_PIPE[p].DATA2_ADDRESS -->
+	<reg32 offset="3" name="3">
+		<bitfield name="BIN_DATA2_OFFSET" low="0" high="31" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_REG_RMW" width="32">
+	<doc>
+                Modifies DST_REG using two sources that can either be registers
+                or immediates. If SRC1_ADD is set, then do the following:
+
+			$dst = (($dst &amp; $src0) rot $rotate) + $src1
+
+		Otherwise:
+
+			$dst = (($dst &amp; $src0) rot $rotate) | $src1
+
+		Here "rot" means rotate left.
+	</doc>
+	<reg32 offset="0" name="0">
+		<bitfield name="DST_REG" low="0" high="17" type="hex"/>
+		<bitfield name="ROTATE" low="24" high="28" type="uint"/>
+		<bitfield name="SRC1_ADD" pos="29" type="boolean"/>
+		<bitfield name="SRC1_IS_REG" pos="30" type="boolean"/>
+		<bitfield name="SRC0_IS_REG" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="SRC0" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="SRC1" low="0" high="31" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_REG_TO_MEM" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="REG" low="0" high="17" type="hex"/>
+		<!-- number of registers/dwords copied is max(CNT, 1). -->
+		<bitfield name="CNT" low="18" high="29" type="uint"/>
+		<bitfield name="64B" pos="30" type="boolean"/>
+		<bitfield name="ACCUMULATE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="DEST" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="2" name="2" varset="chip" variants="A5XX-">
+		<bitfield name="DEST_HI" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain name="CP_REG_TO_MEM_OFFSET_REG" width="32">
+	<doc>
+                Like CP_REG_TO_MEM, but the memory address to write to can be
+                offsetted using either one or two registers or scratch
+                registers.
+	</doc>
+	<reg32 offset="0" name="0">
+		<bitfield name="REG" low="0" high="17" type="hex"/>
+		<!-- number of registers/dwords copied is max(CNT, 1). -->
+		<bitfield name="CNT" low="18" high="29" type="uint"/>
+		<bitfield name="64B" pos="30" type="boolean"/>
+		<bitfield name="ACCUMULATE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="DEST" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="2" name="2" varset="chip" variants="A5XX-">
+		<bitfield name="DEST_HI" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="OFFSET0" low="0" high="17" type="hex"/>
+		<bitfield name="OFFSET0_SCRATCH" pos="19" type="boolean"/>
+	</reg32>
+	<!-- followed by an optional identical OFFSET1 dword -->
+</domain>
+
+<domain name="CP_REG_TO_MEM_OFFSET_MEM" width="32">
+	<doc>
+                Like CP_REG_TO_MEM, but the memory address to write to can be
+                offsetted using a DWORD in memory.
+	</doc>
+	<reg32 offset="0" name="0">
+		<bitfield name="REG" low="0" high="17" type="hex"/>
+		<!-- number of registers/dwords copied is max(CNT, 1). -->
+		<bitfield name="CNT" low="18" high="29" type="uint"/>
+		<bitfield name="64B" pos="30" type="boolean"/>
+		<bitfield name="ACCUMULATE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="DEST" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="2" name="2" varset="chip" variants="A5XX-">
+		<bitfield name="DEST_HI" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="OFFSET_LO" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="4" name="4">
+		<bitfield name="OFFSET_HI" low="0" high="31" type="hex"/>
+	</reg32>
+</domain>
+
+<domain name="CP_MEM_TO_REG" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="REG" low="0" high="17" type="hex"/>
+		<!-- number of registers/dwords copied is max(CNT, 1). -->
+		<bitfield name="CNT" low="19" high="29" type="uint"/>
+		<!-- shift each DWORD left by 2 while copying -->
+		<bitfield name="SHIFT_BY_2" pos="30" type="boolean"/>
+		<!-- does the same thing as CP_MEM_TO_MEM::UNK31 -->
+		<bitfield name="UNK31" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="SRC" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="2" name="2" varset="chip" variants="A5XX-">
+		<bitfield name="SRC_HI" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain name="CP_MEM_TO_MEM" width="32">
+	<reg32 offset="0" name="0">
+		<!--
+		not sure how many src operands we have, but the low
+		bits negate the n'th src argument.
+		 -->
+		<bitfield name="NEG_A" pos="0" type="boolean"/>
+		<bitfield name="NEG_B" pos="1" type="boolean"/>
+		<bitfield name="NEG_C" pos="2" type="boolean"/>
+
+		<!-- if set treat src/dst as 64bit values -->
+		<bitfield name="DOUBLE" pos="29" type="boolean"/>
+		<!-- execute CP_WAIT_FOR_MEM_WRITES beforehand -->
+		<bitfield name="WAIT_FOR_MEM_WRITES" pos="30" type="boolean"/>
+		<!-- some other kind of wait -->
+		<bitfield name="UNK31" pos="31" type="boolean"/>
+	</reg32>
+	<!--
+	followed by sequence of addresses.. the first is the
+	destination and the rest are N src addresses which are
+	summed (after being negated if NEG_x bit set) allowing
+	to do things like 'result += end - start' (which turns
+	out to be useful for queries and accumulating results
+	across multiple tiles)
+	 -->
+</domain>
+
+<domain name="CP_MEMCPY" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="DWORDS" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="SRC_LO" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="SRC_HI" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="DST_LO" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="4" name="4">
+		<bitfield name="DST_HI" low="0" high="31" type="hex"/>
+	</reg32>
+</domain>
+
+<domain name="CP_REG_TO_SCRATCH" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="REG" low="0" high="17" type="hex"/>
+		<bitfield name="SCRATCH" low="20" high="22" type="uint"/>
+		<!-- number of registers/dwords copied is CNT + 1. -->
+		<bitfield name="CNT" low="24" high="26" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_SCRATCH_TO_REG" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="REG" low="0" high="17" type="hex"/>
+		<!-- note: CP_MEM_TO_REG always sets this when writing to the register -->
+		<bitfield name="UNK18" pos="18" type="boolean"/>
+		<bitfield name="SCRATCH" low="20" high="22" type="uint"/>
+		<!-- number of registers/dwords copied is CNT + 1. -->
+		<bitfield name="CNT" low="24" high="26" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_SCRATCH_WRITE" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="SCRATCH" low="20" high="22" type="uint"/>
+	</reg32>
+	<!-- followed by one or more DWORDs to write to scratch registers -->
+</domain>
+
+<domain name="CP_MEM_WRITE" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="ADDR_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="ADDR_HI" low="0" high="31"/>
+	</reg32>
+	<!-- followed by the DWORDs to write -->
+</domain>
+
+<enum name="cp_cond_function">
+	<value value="0" name="WRITE_ALWAYS"/>
+	<value value="1" name="WRITE_LT"/>
+	<value value="2" name="WRITE_LE"/>
+	<value value="3" name="WRITE_EQ"/>
+	<value value="4" name="WRITE_NE"/>
+	<value value="5" name="WRITE_GE"/>
+	<value value="6" name="WRITE_GT"/>
+</enum>
+
+<domain name="CP_COND_WRITE" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="FUNCTION" low="0" high="2" type="cp_cond_function"/>
+		<bitfield name="POLL_MEMORY" pos="4" type="boolean"/>
+		<bitfield name="WRITE_MEMORY" pos="8" type="boolean"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="POLL_ADDR" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="REF" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="MASK" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="4" name="4">
+		<bitfield name="WRITE_ADDR" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="5" name="5">
+		<bitfield name="WRITE_DATA" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<enum name="poll_memory_type">
+	<value value="0" name="POLL_REGISTER"/>
+	<value value="1" name="POLL_MEMORY"/>
+	<value value="2" name="POLL_SCRATCH"/>
+	<value value="3" name="POLL_ON_CHIP" varset="chip" variants="A7XX-"/>
+</enum>
+
+<domain name="CP_COND_WRITE5" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="FUNCTION" low="0" high="2" type="cp_cond_function"/>
+		<bitfield name="SIGNED_COMPARE" pos="3" type="boolean"/>
+		<!-- POLL_REGISTER polls a register at POLL_ADDR_LO. -->
+		<bitfield name="POLL" low="4" high="5" type="poll_memory_type"/>
+		<bitfield name="WRITE_MEMORY" pos="8" type="boolean"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="POLL_ADDR_LO" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="POLL_ADDR_HI" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="REF" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="4" name="4">
+		<bitfield name="MASK" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="5" name="5">
+		<bitfield name="WRITE_ADDR_LO" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="6" name="6">
+		<bitfield name="WRITE_ADDR_HI" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="7" name="7">
+		<bitfield name="WRITE_DATA" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain name="CP_WAIT_MEM_GTE" width="32">
+        <doc>
+                Wait until a memory value is greater than or equal to the
+                reference, using signed comparison.
+	</doc>
+	<reg32 offset="0" name="0">
+		<!-- Reserved for flags, presumably? Unused in FW -->
+		<bitfield name="RESERVED" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="POLL_ADDR_LO" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="POLL_ADDR_HI" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="REF" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain name="CP_WAIT_REG_MEM" width="32">
+        <doc>
+                This uses the same internal comparison as CP_COND_WRITE,
+                but waits until the comparison is true instead. It busy-loops in
+                the CP for the given number of cycles before trying again.
+	</doc>
+	<reg32 offset="0" name="0">
+		<bitfield name="FUNCTION" low="0" high="2" type="cp_cond_function"/>
+		<bitfield name="SIGNED_COMPARE" pos="3" type="boolean"/>
+		<bitfield name="POLL" low="4" high="5" type="poll_memory_type"/>
+		<bitfield name="WRITE_MEMORY" pos="8" type="boolean"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="POLL_ADDR_LO" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="POLL_ADDR_HI" low="0" high="31" type="hex"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="REF" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="4" name="4">
+		<bitfield name="MASK" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="5" name="5">
+		<bitfield name="DELAY_LOOP_CYCLES" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain name="CP_WAIT_TWO_REGS" width="32">
+	<doc>
+		Waits for REG0 to not be 0 or REG1 to not equal REF
+	</doc>
+	<reg32 offset="0" name="0">
+		<bitfield name="REG0" low="0" high="17" type="hex"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="REG1" low="0" high="17" type="hex"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="REF" low="0" high="31" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_DISPATCH_COMPUTE" width="32">
+	<reg32 offset="0" name="0"/>
+	<reg32 offset="1" name="1">
+		<bitfield name="X" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="Y" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="Z" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain name="CP_SET_RENDER_MODE" width="32">
+	<enum name="render_mode_cmd">
+		<value value="1" name="BYPASS"/>
+		<value value="2" name="BINNING"/>
+		<value value="3" name="GMEM"/>
+		<value value="5" name="BLIT2D"/>
+		<!-- placeholder name.. used when CP_BLIT packets with BLIT_OP_SCALE?? -->
+		<value value="7" name="BLIT2DSCALE"/>
+		<!-- 8 set before going back to BYPASS exiting 2D -->
+		<value value="8" name="END2D"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="MODE" low="0" high="8" type="render_mode_cmd"/>
+		<!--
+		normally 0x1/0x3, sometimes see 0x5/0x8 with unknown registers in
+		0x21xx range.. possibly (at least some) a5xx variants have a
+		2d core?
+		 -->
+	</reg32>
+	<!-- I think first buffer is for GPU to save context in case of ctx switch? -->
+	<reg32 offset="1" name="1">
+		<bitfield name="ADDR_0_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="ADDR_0_HI" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<!--
+		set when in GMEM.. maybe indicates GMEM contents need to be
+		preserved on ctx switch?
+		 -->
+		<bitfield name="VSC_ENABLE" pos="3" type="boolean"/>
+		<bitfield name="GMEM_ENABLE" pos="4" type="boolean"/>
+	</reg32>
+	<reg32 offset="4" name="4"/>
+	<!-- second buffer looks like some cmdstream.. length in dwords: -->
+	<reg32 offset="5" name="5">
+		<bitfield name="ADDR_1_LEN" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="6" name="6">
+		<bitfield name="ADDR_1_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="7" name="7">
+		<bitfield name="ADDR_1_HI" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<!-- this looks fairly similar to CP_SET_RENDER_MODE minus first dword -->
+<domain name="CP_COMPUTE_CHECKPOINT" width="32">
+	<!-- I think first buffer is for GPU to save context in case of ctx switch? -->
+	<reg32 offset="0" name="0">
+		<bitfield name="ADDR_0_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="ADDR_0_HI" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+	</reg32>
+	<reg32 offset="3" name="3"/>
+	<!-- second buffer looks like some cmdstream.. length in dwords: -->
+	<reg32 offset="4" name="4">
+		<bitfield name="ADDR_1_LEN" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="5" name="5">
+		<bitfield name="ADDR_1_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="6" name="6">
+		<bitfield name="ADDR_1_HI" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="7" name="7"/>
+</domain>
+
+<domain name="CP_PERFCOUNTER_ACTION" width="32">
+	<reg32 offset="0" name="0">
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="ADDR_0_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="ADDR_0_HI" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain varset="chip" name="CP_EVENT_WRITE" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="EVENT" low="0" high="7" type="vgt_event_type"/>
+		<!-- when set, write back timestamp instead of value from packet: -->
+		<bitfield name="TIMESTAMP" pos="30" type="boolean"/>
+		<bitfield name="IRQ" pos="31" type="boolean"/>
+	</reg32>
+	<!--
+	TODO what is gpuaddr for, seems to be all 0's.. maybe needed for
+	context switch?
+	 -->
+	<reg32 offset="1" name="1">
+		<bitfield name="ADDR_0_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="ADDR_0_HI" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<!-- ??? -->
+	</reg32>
+</domain>
+
+<domain varset="chip" name="CP_EVENT_WRITE7" width="32">
+	<enum name="event_write_src">
+		<!-- Write payload[0] -->
+		<value value="0" name="EV_WRITE_USER_32B"/>
+		<!-- Write payload[0] payload[1] -->
+		<value value="1" name="EV_WRITE_USER_64B"/>
+		<!-- Write (TIMESTAMP_GLOBAL + TIMESTAMP_LOCAL) -->
+		<value value="2" name="EV_WRITE_TIMESTAMP_SUM"/>
+		<value value="3" name="EV_WRITE_ALWAYSON"/>
+		<!-- Write payload[1] regs starting at payload[0] offset -->
+		<value value="4" name="EV_WRITE_REGS_CONTENT"/>
+	</enum>
+
+	<enum name="event_write_dst">
+		<value value="0" name="EV_DST_RAM"/>
+		<value value="1" name="EV_DST_ONCHIP"/>
+	</enum>
+
+	<reg32 offset="0" name="0">
+		<bitfield name="EVENT" low="0" high="7" type="vgt_event_type"/>
+		<bitfield name="WRITE_SAMPLE_COUNT" pos="12" type="boolean"/>
+		<!-- Write sample count at (iova + 16) -->
+		<bitfield name="SAMPLE_COUNT_END_OFFSET" pos="13" type="boolean"/>
+		<!-- *(iova + 8) = *(iova + 16) - *iova -->
+		<bitfield name="WRITE_SAMPLE_COUNT_DIFF" pos="14" type="boolean"/>
+
+		<!-- Next 4 flags are valid to set only when concurrent binning is enabled -->
+		<!-- Increment 16b BV counter. Valid only in BV pipe -->
+		<bitfield name="INC_BV_COUNT" pos="16" type="boolean"/>
+		<!-- Increment 16b BR counter. Valid only in BR pipe -->
+		<bitfield name="INC_BR_COUNT" pos="17" type="boolean"/>
+		<bitfield name="CLEAR_RENDER_RESOURCE" pos="18" type="boolean"/>
+		<bitfield name="CLEAR_LRZ_RESOURCE" pos="19" type="boolean"/>
+
+		<bitfield name="WRITE_SRC" low="20" high="22" type="event_write_src"/>
+		<bitfield name="WRITE_DST" pos="24" type="event_write_dst" addvariant="yes"/>
+		<!-- Writes into WRITE_DST from WRITE_SRC. RB_DONE_TS requires WRITE_ENABLED. -->
+		<bitfield name="WRITE_ENABLED" pos="27" type="boolean"/>
+	</reg32>
+
+	<stripe varset="event_write_dst" variants="EV_DST_RAM">
+		<reg32 offset="1" name="1">
+			<bitfield name="ADDR_0_LO" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="2" name="2">
+			<bitfield name="ADDR_0_HI" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="3" name="3">
+			<bitfield name="PAYLOAD_0" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="4" name="4">
+			<bitfield name="PAYLOAD_1" low="0" high="31"/>
+		</reg32>
+	</stripe>
+
+	<stripe varset="event_write_dst" variants="EV_DST_ONCHIP">
+		<reg32 offset="1" name="1">
+			<bitfield name="ONCHIP_ADDR_0" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="3" name="3">
+			<bitfield name="PAYLOAD_0" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="4" name="4">
+			<bitfield name="PAYLOAD_1" low="0" high="31"/>
+		</reg32>
+	</stripe>
+</domain>
+
+<domain name="CP_BLIT" width="32">
+	<enum name="cp_blit_cmd">
+		<value value="0" name="BLIT_OP_FILL"/>
+		<value value="1" name="BLIT_OP_COPY"/>
+		<value value="3" name="BLIT_OP_SCALE"/> <!-- used for mipmap generation -->
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="OP" low="0" high="3" type="cp_blit_cmd"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="SRC_X1" low="0" high="13" type="uint"/>
+		<bitfield name="SRC_Y1" low="16" high="29" type="uint"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="SRC_X2" low="0" high="13" type="uint"/>
+		<bitfield name="SRC_Y2" low="16" high="29" type="uint"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="DST_X1" low="0" high="13" type="uint"/>
+		<bitfield name="DST_Y1" low="16" high="29" type="uint"/>
+	</reg32>
+	<reg32 offset="4" name="4">
+		<bitfield name="DST_X2" low="0" high="13" type="uint"/>
+		<bitfield name="DST_Y2" low="16" high="29" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_EXEC_CS" width="32">
+	<reg32 offset="0" name="0">
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="NGROUPS_X" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="NGROUPS_Y" low="0" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="NGROUPS_Z" low="0" high="31" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_EXEC_CS_INDIRECT" width="32" varset="chip" prefix="chip" variants="A4XX-">
+	<reg32 offset="0" name="0">
+	</reg32>
+	<stripe varset="chip" variants="A4XX">
+		<reg32 offset="1" name="1">
+			<bitfield name="ADDR" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="2" name="2">
+			<!-- localsize is value minus one: -->
+			<bitfield name="LOCALSIZEX" low="2" high="11" type="uint"/>
+			<bitfield name="LOCALSIZEY" low="12" high="21" type="uint"/>
+			<bitfield name="LOCALSIZEZ" low="22" high="31" type="uint"/>
+		</reg32>
+	</stripe>
+	<stripe varset="chip" variants="A5XX-">
+		<reg32 offset="1" name="1">
+			<bitfield name="ADDR_LO" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="2" name="2">
+			<bitfield name="ADDR_HI" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="3" name="3">
+			<!-- localsize is value minus one: -->
+			<bitfield name="LOCALSIZEX" low="2" high="11" type="uint"/>
+			<bitfield name="LOCALSIZEY" low="12" high="21" type="uint"/>
+			<bitfield name="LOCALSIZEZ" low="22" high="31" type="uint"/>
+		</reg32>
+	</stripe>
+</domain>
+
+<domain name="CP_SET_MARKER" width="32" varset="chip" prefix="chip" variants="A6XX-">
+	<doc>Tell CP the current operation mode, indicates save and restore procedure</doc>
+	<enum name="a6xx_marker">
+		<value value="1" name="RM6_BYPASS"/>
+		<value value="2" name="RM6_BINNING"/>
+		<value value="4" name="RM6_GMEM"/>
+		<value value="5" name="RM6_ENDVIS"/>
+		<value value="6" name="RM6_RESOLVE"/>
+		<value value="7" name="RM6_YIELD"/>
+		<value value="8" name="RM6_COMPUTE"/>
+		<value value="0xc" name="RM6_BLIT2DSCALE"/>  <!-- no-op (at least on current sqe fw) -->
+
+		<!--
+			These values come from a6xx_set_marker() in the
+			downstream kernel, and they can only be set by the kernel
+		-->
+		<value value="0xd" name="RM6_IB1LIST_START"/>
+		<value value="0xe" name="RM6_IB1LIST_END"/>
+		<!-- IFPC - inter-frame power collapse -->
+		<value value="0x100" name="RM6_IFPC_ENABLE"/>
+		<value value="0x101" name="RM6_IFPC_DISABLE"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<!--
+			NOTE: blob driver and some versions of freedreno/turnip set
+			b4, which is unused (at least by current sqe fw), but interferes
+			with parsing if we extend the size of the bitfield to include
+			b8 (only sent by kernel mode driver).  Really, the way the
+			parsing works in the firmware, only b0-b3 are considered, but
+			if b8 is set, the low bits are interpreted differently.  To
+			model this, without getting confused by spurious b4, this is
+			described as two overlapping bitfields:
+		 -->
+		<bitfield name="MODE" low="0" high="8" type="a6xx_marker"/>
+		<bitfield name="MARKER" low="0" high="3" type="a6xx_marker"/>
+	</reg32>
+</domain>
+
+<domain name="CP_SET_PSEUDO_REG" width="32" varset="chip" prefix="chip" variants="A6XX-">
+	<doc>Set internal CP registers, used to indicate context save data addresses</doc>
+	<enum name="pseudo_reg">
+		<value value="0" name="SMMU_INFO"/>
+		<value value="1" name="NON_SECURE_SAVE_ADDR"/>
+		<value value="2" name="SECURE_SAVE_ADDR"/>
+		<value value="3" name="NON_PRIV_SAVE_ADDR"/>
+		<value value="4" name="COUNTER"/>
+
+		<!--
+			On a6xx the registers are set directly and CP_SET_BIN_DATA5_OFFSET reads them,
+			but that doesn't work with concurrent binning because BR will be reading from
+			a different set of streams than BV is writing, so on a7xx we have these
+			pseudo-regs instead, which do the right thing.
+
+			The corresponding VSC registers exist, and they're written by BV when it
+			encounters CP_SET_PSEUDO_REG. When BR later encounters the same CP_SET_PSEUDO_REG
+			it will only write some private scratch registers which are read by
+			CP_SET_BIN_DATA5_OFFSET.
+
+			If concurrent binning is disabled then BR also does binning so it will also
+			write the "real" registers in BR.
+		-->
+		<value value="8" name="DRAW_STRM_ADDRESS"/>
+		<value value="9" name="DRAW_STRM_SIZE_ADDRESS"/>
+		<value value="10" name="PRIM_STRM_ADDRESS"/>
+		<value value="11" name="UNK_STRM_ADDRESS"/>
+		<value value="12" name="UNK_STRM_SIZE_ADDRESS"/>
+
+		<value value="16" name="BINDLESS_BASE_0_ADDR"/>
+		<value value="17" name="BINDLESS_BASE_1_ADDR"/>
+		<value value="18" name="BINDLESS_BASE_2_ADDR"/>
+		<value value="19" name="BINDLESS_BASE_3_ADDR"/>
+		<value value="20" name="BINDLESS_BASE_4_ADDR"/>
+		<value value="21" name="BINDLESS_BASE_5_ADDR"/>
+		<value value="22" name="BINDLESS_BASE_6_ADDR"/>
+	</enum>
+	<array offset="0" stride="3" length="100">
+		<reg32 offset="0" name="0">
+			<bitfield name="PSEUDO_REG" low="0" high="10" type="pseudo_reg"/>
+		</reg32>
+		<reg32 offset="1" name="1">
+			<bitfield name="LO" low="0" high="31"/>
+		</reg32>
+		<reg32 offset="2" name="2">
+			<bitfield name="HI" low="0" high="31"/>
+		</reg32>
+	</array>
+</domain>
+
+<domain name="CP_REG_TEST" width="32" varset="chip" prefix="chip" variants="A6XX-">
+	<doc>
+		Tests bit in specified register and sets predicate for CP_COND_REG_EXEC.
+		So:
+
+			opcode: CP_REG_TEST (39) (2 dwords)
+			        { REG = 0xc10 | BIT = 0 }
+			               0000: 70b90001 00000c10
+			opcode: CP_COND_REG_EXEC (47) (3 dwords)
+			               0000: 70c70002 10000000 00000004
+			opcode: CP_INDIRECT_BUFFER (3f) (4 dwords)
+
+		Will execute the CP_INDIRECT_BUFFER only if b0 in the register at
+		offset 0x0c10 is 1
+	</doc>
+	<enum name="source_type">
+		<value value="0" name="SOURCE_REG"/>
+		<!-- Don't confuse with scratch registers, this is a separate memory
+			 written into by CP_MEM_TO_SCRATCH_MEM. -->
+		<value value="1" name="SOURCE_SCRATCH_MEM" varset="chip" variants="A7XX-"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<!-- the register to test -->
+		<bitfield name="REG" low="0" high="17" varset="source_type" variants="SOURCE_REG"/>
+		<bitfield name="SCRATCH_MEM_OFFSET" low="0" high="17" varset="source_type" variants="SOURCE_SCRATCH_MEM"/>
+		<bitfield name="SOURCE" pos="18" type="source_type" addvariant="yes"/>
+		<!-- the bit to test -->
+		<bitfield name="BIT" low="20" high="24" type="uint"/>
+		<!-- skip implied CP_WAIT_FOR_ME -->
+		<bitfield name="SKIP_WAIT_FOR_ME" pos="25" type="boolean"/>
+		<!-- the predicate bit to set (new in gen3+) -->
+		<bitfield name="PRED_BIT" low="26" high="30" type="uint"/>
+		<!-- update the predicate reg directly (new in gen3+) -->
+		<bitfield name="PRED_UPDATE" pos="31" type="boolean"/>
+	</reg32>
+
+        <!--
+		In PRED_UPDATE mode, the predicate reg is updated directly using two
+		more dwords, ignoring other bits:
+
+			PRED_REG = (PRED_REG & ~PRED_MASK) | (PRED_VAL & PRED_MASK);
+	-->
+	<reg32 offset="1" name="PRED_MASK" type="hex"/>
+	<reg32 offset="2" name="PRED_VAL" type="hex"/>
+</domain>
+
+<!-- I *think* this existed at least as far back as a4xx -->
+<domain name="CP_COND_REG_EXEC" width="32">
+	<enum name="compare_mode">
+		<!-- use the predicate bit set by CP_REG_TEST -->
+		<value value="1" name="PRED_TEST"/>
+		<!-- compare two registers directly for equality -->
+		<value value="2" name="REG_COMPARE"/>
+		<!-- test if certain render modes are set via CP_SET_MARKER -->
+		<value value="3" name="RENDER_MODE" varset="chip" variants="A6XX-"/>
+		<!-- compare REG0 for equality with immediate -->
+		<value value="4" name="REG_COMPARE_IMM" varset="chip" variants="A7XX-"/>
+		<!-- test which of BR/BV are enabled -->
+		<value value="5" name="THREAD_MODE" varset="chip" variants="A7XX-"/>
+	</enum>
+	<reg32 offset="0" name="0" varset="compare_mode">
+		<bitfield name="REG0" low="0" high="17" variants="REG_COMPARE" type="hex"/>
+
+		<!-- the predicate bit to test (new in gen3+) -->
+		<bitfield name="PRED_BIT" low="18" high="22" variants="PRED_TEST" type="uint"/>
+		<bitfield name="SKIP_WAIT_FOR_ME" pos="23" varset="chip" variants="A7XX-" type="boolean"/>
+		<!-- With REG_COMPARE instead of register read from ONCHIP memory -->
+		<bitfield name="ONCHIP_MEM" pos="24" varset="chip" variants="A7XX-" type="boolean"/>
+
+		<!--
+			Note: these bits have the same meaning, and use the same
+			internal mechanism as the bits in CP_SET_DRAW_STATE.
+			When RENDER_MODE is selected, they're used as
+			a bitmask of which modes pass the test.
+		-->
+
+		<!-- RM6_BINNING -->
+		<bitfield name="BINNING" pos="25" variants="RENDER_MODE" type="boolean"/>
+		<!-- all others -->
+		<bitfield name="GMEM" pos="26" variants="RENDER_MODE" type="boolean"/>
+		<!-- RM6_BYPASS -->
+		<bitfield name="SYSMEM" pos="27" variants="RENDER_MODE" type="boolean"/>
+
+		<bitfield name="BV" pos="25" variants="THREAD_MODE" type="boolean"/>
+		<bitfield name="BR" pos="26" variants="THREAD_MODE" type="boolean"/>
+		<bitfield name="LPAC" pos="27" variants="THREAD_MODE" type="boolean"/>
+
+		<bitfield name="MODE" low="28" high="31" type="compare_mode" addvariant="yes"/>
+	</reg32>
+
+	<stripe varset="compare_mode" variants="PRED_TEST">
+		<reg32 offset="1" name="1">
+			<bitfield name="DWORDS" low="0" high="23" type="uint"/>
+		</reg32>
+	</stripe>
+
+	<stripe varset="compare_mode" variants="REG_COMPARE">
+		<reg32 offset="1" name="1">
+			<bitfield name="REG1" low="0" high="17" type="hex"/>
+			<!-- Instead of register read from ONCHIP memory -->
+			<bitfield name="ONCHIP_MEM" pos="24" varset="chip" variants="A7XX-" type="boolean"/>
+		</reg32>
+	</stripe>
+
+	<stripe varset="compare_mode" variants="RENDER_MODE">
+		<reg32 offset="1" name="1">
+			<bitfield name="DWORDS" low="0" high="23" type="uint"/>
+		</reg32>
+	</stripe>
+
+	<stripe varset="compare_mode" variants="REG_COMPARE_IMM">
+		<reg32 offset="1" name="1">
+			<bitfield name="IMM" low="0" high="31"/>
+		</reg32>
+	</stripe>
+
+	<stripe varset="compare_mode" variants="THREAD_MODE">
+		<reg32 offset="1" name="1">
+			<bitfield name="DWORDS" low="0" high="23" type="uint"/>
+		</reg32>
+	</stripe>
+
+	<reg32 offset="2" name="2">
+		<bitfield name="DWORDS" low="0" high="23" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_COND_EXEC" width="32">
+	<doc>
+                Executes the following DWORDs of commands if the dword at ADDR0
+                is not equal to 0 and the dword at ADDR1 is less than REF
+                (signed comparison).
+	</doc>
+	<reg32 offset="0" name="0">
+		<bitfield name="ADDR0_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="ADDR0_HI" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="ADDR1_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="ADDR1_HI" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="4" name="4">
+		<bitfield name="REF" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="5" name="5">
+		<bitfield name="DWORDS" low="0" high="31" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_SET_CTXSWITCH_IB" width="32">
+	<doc>
+                Used by the userspace driver to set various IB's which are
+                executed during context save/restore for handling
+                state that isn't restored by the
+                context switch routine itself.
+	</doc>
+	<enum name="ctxswitch_ib">
+		<value name="RESTORE_IB" value="0">
+			<doc>Executed unconditionally when switching back to the context.</doc>
+		</value>
+		<value name="YIELD_RESTORE_IB" value="1">
+                        <doc>
+				Executed when switching back after switching
+				away during execution of
+				a CP_SET_MARKER packet with RM6_YIELD as the
+				payload *and* the normal save routine was
+				bypassed for a shorter one. I think this is
+				connected to the "skipsaverestore" bit set by
+				the kernel when preempting.
+			</doc>
+		</value>
+		<value name="SAVE_IB" value="2">
+                        <doc>
+				Executed when switching away from the context,
+				except for context switches initiated via
+				CP_YIELD.
+                        </doc>
+		</value>
+		<value name="RB_SAVE_IB" value="3">
+			<doc>
+				This can only be set by the RB (i.e. the kernel)
+				and executes with protected mode off, but
+				is otherwise similar to SAVE_IB.
+
+				Note, kgsl calls this CP_KMD_AMBLE_TYPE
+			</doc>
+		</value>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="ADDR_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="ADDR_HI" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="DWORDS" low="0" high="19" type="uint"/>
+		<bitfield name="TYPE" low="20" high="21" type="ctxswitch_ib"/>
+	</reg32>
+</domain>
+
+<domain name="CP_REG_WRITE" width="32">
+	<enum name="reg_tracker">
+		<doc>
+			Keep shadow copies of these registers and only set them
+			when drawing, avoiding redundant writes:
+			- VPC_CNTL_0
+			- HLSQ_CONTROL_1_REG
+			- HLSQ_UNKNOWN_B980
+		</doc>
+		<value name="TRACK_CNTL_REG" value="0x1"/>
+		<doc>
+			Track RB_RENDER_CNTL, and insert a WFI in the following
+			situation:
+			- There is a write that disables binning
+			- There was a draw with binning left enabled, but in
+			  BYPASS mode
+			Presumably this is a hang workaround?
+		</doc>
+		<value name="TRACK_RENDER_CNTL" value="0x2"/>
+		<doc>
+			Do a mysterious CP_EVENT_WRITE 0x3f when the low bit of
+			the data to write is 0. Used by the Vulkan blob with
+			PC_MULTIVIEW_CNTL, but this isn't predicated on particular
+			register(s) like the others.
+		</doc>
+		<value name="UNK_EVENT_WRITE" value="0x4"/>
+		<doc>
+			Tracks GRAS_LRZ_CNTL::GREATER, GRAS_LRZ_CNTL::DIR, and
+			GRAS_LRZ_DEPTH_VIEW with previous values, and if one of
+			the following is true:
+			- GRAS_LRZ_CNTL::GREATER has changed
+			- GRAS_LRZ_CNTL::DIR has changed, the old value is not
+			  CUR_DIR_GE, and the new value is not CUR_DIR_DISABLED
+			- GRAS_LRZ_DEPTH_VIEW has changed
+			then it does a LRZ_FLUSH with GRAS_LRZ_CNTL::ENABLE
+			forced to 1.
+			Only exists in a650_sqe.fw.
+		</doc>
+		<value name="TRACK_LRZ" value="0x8"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="TRACKER" low="0" high="3" type="reg_tracker"/>
+	</reg32>
+	<reg32 offset="1" name="1"/>
+	<reg32 offset="2" name="2"/>
+</domain>
+
+<domain name="CP_SMMU_TABLE_UPDATE" width="32">
+	<doc>
+		Note that the SMMU's definition of TTBRn can take different forms
+		depending on the pgtable format.  But a5xx+ only uses aarch64
+		format.
+	</doc>
+	<reg32 offset="0" name="0">
+		<bitfield name="TTBR0_LO" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="TTBR0_HI" low="0" high="15"/>
+		<bitfield name="ASID" low="16" high="31"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<doc>Unused, does not apply to aarch64 pgtable format</doc>
+		<bitfield name="CONTEXTIDR" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="CONTEXTBANK" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain name="CP_START_BIN" width="32">
+	<reg32 offset="0" name="BIN_COUNT" type="uint"/>
+	<reg64 offset="1" name="PREFIX_ADDR" type="address"/>
+	<reg32 offset="3" name="PREFIX_DWORDS">
+		<doc>
+			Size of prefix for each bin. For each bin index i, the
+			prefix commands at PREFIX_ADDR + i * PREFIX_DWORDS are
+			executed in an IB2 before the IB1 commands following
+			this packet.
+		</doc>
+	</reg32>
+	<reg32 offset="4" name="BODY_DWORDS">
+		<doc>Number of dwords after this packet until CP_END_BIN</doc>
+	</reg32>
+</domain>
+
+<domain name="CP_WAIT_TIMESTAMP" width="32">
+	<enum name="ts_wait_value_src">
+		<!-- Wait for value at memory address to be >= SRC_0 (signed comparison) -->
+		<value value="0" name="TS_WAIT_GE_32B"/>
+		<!-- Wait for value at memory address to be >= SRC_0 (unsigned) -->
+		<value value="1" name="TS_WAIT_GE_64B"/>
+		<!-- Write (TIMESTAMP_GLOBAL + TIMESTAMP_LOCAL) -->
+		<value value="2" name="TS_WAIT_GE_TIMESTAMP_SUM"/>
+	</enum>
+
+	<enum name="ts_wait_type">
+		<value value="0" name="TS_WAIT_RAM"/>
+		<value value="1" name="TS_WAIT_ONCHIP"/>
+	</enum>
+
+	<reg32 offset="0" name="0">
+		<bitfield name="WAIT_VALUE_SRC" low="0" high="1" type="ts_wait_value_src"/>
+		<bitfield name="WAIT_DST" pos="4" type="ts_wait_type" addvariant="yes"/>
+	</reg32>
+
+	<stripe varset="ts_wait_type" variants="TS_WAIT_RAM">
+		<reg64 offset="1" name="ADDR" type="address"/>
+	</stripe>
+
+	<stripe varset="ts_wait_type" variants="TS_WAIT_ONCHIP">
+		<reg32 offset="1" name="ONCHIP_ADDR_0" low="0" high="31"/>
+	</stripe>
+
+	<reg32 offset="3" name="SRC_0"/>
+	<reg32 offset="4" name="SRC_1"/>
+</domain>
+
+<domain name="CP_BV_BR_COUNT_OPS" width="32">
+	<enum name="pipe_count_op">
+		<value name="PIPE_CLEAR_BV_BR" value="0x1"/>
+		<value name="PIPE_SET_BR_OFFSET" value="0x2"/>
+		<!-- Wait until for BV_counter > BR_counter -->
+		<value name="PIPE_BR_WAIT_FOR_BV" value="0x3"/>
+		<!-- Wait until (BR_counter + BR_OFFSET) > BV_counter -->
+		<value name="PIPE_BV_WAIT_FOR_BR" value="0x4"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="OP" low="0" high="3" type="pipe_count_op"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<bitfield name="BR_OFFSET" low="0" high="15" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="CP_MODIFY_TIMESTAMP" width="32">
+	<enum name="timestamp_op">
+		<value name="MODIFY_TIMESTAMP_CLEAR" value="0"/>
+		<value name="MODIFY_TIMESTAMP_ADD_GLOBAL" value="1"/>
+		<value name="MODIFY_TIMESTAMP_ADD_LOCAL" value="2"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield name="ADD" low="0" high="7" type="uint"/>
+		<bitfield name="OP" low="28" high="31" type="timestamp_op"/>
+	</reg32>
+</domain>
+
+<domain name="CP_MEM_TO_SCRATCH_MEM" width="32">
+	<doc>
+		Best guess is that it is a faster way to fetch all the VSC_STATE registers
+		and keep them in a local scratch memory instead of fetching every time
+		when skipping IBs.
+	</doc>
+	<reg32 offset="0" name="0">
+		<bitfield name="CNT" low="0" high="5" type="uint"/>
+	</reg32>
+	<reg32 offset="1" name="1">
+		<doc>Scratch memory size is 48 dwords`</doc>
+		<bitfield name="OFFSET" low="0" high="5" type="uint"/>
+	</reg32>
+	<reg32 offset="2" name="2">
+		<bitfield name="SRC" low="0" high="31"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="SRC_HI" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain name="CP_THREAD_CONTROL" width="32">
+	<enum name="cp_thread">
+		<value name="CP_SET_THREAD_BR" value="1"/>    <!-- Render -->
+		<value name="CP_SET_THREAD_BV" value="2"/>    <!-- Visibility -->
+		<value name="CP_SET_THREAD_BOTH" value="3"/>
+	</enum>
+	<reg32 offset="0" name="0">
+		<bitfield low="0" high="1" name="THREAD" type="cp_thread"/>
+		<bitfield pos="27" name="CONCURRENT_BIN_DISABLE" type="boolean"/>
+		<bitfield pos="31" name="SYNC_THREADS" type="boolean"/>
+	</reg32>
+</domain>
+
+<domain name="CP_FIXED_STRIDE_DRAW_TABLE" width="32">
+	<reg64 offset="0" name="IB_BASE"/>
+	<reg32 offset="2" name="2">
+		<!-- STRIDE * COUNT -->
+		<bitfield name="IB_SIZE" low="0" high="11"/>
+		<bitfield name="STRIDE" low="20" high="31"/>
+	</reg32>
+	<reg32 offset="3" name="3">
+		<bitfield name="COUNT" low="0" high="31"/>
+	</reg32>
+</domain>
+
+<domain name="CP_RESET_CONTEXT_STATE" width="32">
+	<reg32 offset="0" name="0">
+		<bitfield name="CLEAR_ON_CHIP_TS" pos="0" type="boolean"/>
+		<bitfield name="CLEAR_RESOURCE_TABLE" pos="1" type="boolean"/>
+		<bitfield name="CLEAR_GLOBAL_LOCAL_TS" pos="2" type="boolean"/>
+	</reg32>
+</domain>
+
+</database>
+
diff --git a/drivers/gpu/drm/msm/registers/display/dsi.xml b/drivers/gpu/drm/msm/registers/display/dsi.xml
new file mode 100644
index 000000000000..501ffc585a9f
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/dsi.xml
@@ -0,0 +1,390 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+
+<domain name="DSI" width="32">
+	<enum name="dsi_traffic_mode">
+		<value name="NON_BURST_SYNCH_PULSE" value="0"/>
+		<value name="NON_BURST_SYNCH_EVENT" value="1"/>
+		<value name="BURST_MODE" value="2"/>
+	</enum>
+	<enum name="dsi_vid_dst_format">
+		<value name="VID_DST_FORMAT_RGB565" value="0"/>
+		<value name="VID_DST_FORMAT_RGB666" value="1"/>
+		<value name="VID_DST_FORMAT_RGB666_LOOSE" value="2"/>
+		<value name="VID_DST_FORMAT_RGB888" value="3"/>
+	</enum>
+	<enum name="dsi_rgb_swap">
+		<value name="SWAP_RGB" value="0"/>
+		<value name="SWAP_RBG" value="1"/>
+		<value name="SWAP_BGR" value="2"/>
+		<value name="SWAP_BRG" value="3"/>
+		<value name="SWAP_GRB" value="4"/>
+		<value name="SWAP_GBR" value="5"/>
+	</enum>
+	<enum name="dsi_cmd_trigger">
+		<value name="TRIGGER_NONE" value="0"/>
+		<value name="TRIGGER_SEOF" value="1"/>
+		<value name="TRIGGER_TE" value="2"/>
+		<value name="TRIGGER_SW" value="4"/>
+		<value name="TRIGGER_SW_SEOF" value="5"/>
+		<value name="TRIGGER_SW_TE" value="6"/>
+	</enum>
+	<enum name="dsi_cmd_dst_format">
+		<value name="CMD_DST_FORMAT_RGB111" value="0"/>
+		<value name="CMD_DST_FORMAT_RGB332" value="3"/>
+		<value name="CMD_DST_FORMAT_RGB444" value="4"/>
+		<value name="CMD_DST_FORMAT_RGB565" value="6"/>
+		<value name="CMD_DST_FORMAT_RGB666" value="7"/>
+		<value name="CMD_DST_FORMAT_RGB888" value="8"/>
+	</enum>
+	<enum name="dsi_lane_swap">
+		<value name="LANE_SWAP_0123" value="0"/>
+		<value name="LANE_SWAP_3012" value="1"/>
+		<value name="LANE_SWAP_2301" value="2"/>
+		<value name="LANE_SWAP_1230" value="3"/>
+		<value name="LANE_SWAP_0321" value="4"/>
+		<value name="LANE_SWAP_1032" value="5"/>
+		<value name="LANE_SWAP_2103" value="6"/>
+		<value name="LANE_SWAP_3210" value="7"/>
+	</enum>
+		<enum name="video_config_bpp">
+		<value name="VIDEO_CONFIG_18BPP" value="0"/>
+		<value name="VIDEO_CONFIG_24BPP" value="1"/>
+	</enum>
+	<enum name="video_pattern_sel">
+		<value name="VID_PRBS" value="0"/>
+		<value name="VID_INCREMENTAL" value="1"/>
+		<value name="VID_FIXED" value="2"/>
+		<value name="VID_MDSS_GENERAL_PATTERN" value="3"/>
+	</enum>
+	<enum name="cmd_mdp_stream0_pattern_sel">
+		<value name="CMD_MDP_PRBS" value="0"/>
+		<value name="CMD_MDP_INCREMENTAL" value="1"/>
+		<value name="CMD_MDP_FIXED" value="2"/>
+		<value name="CMD_MDP_MDSS_GENERAL_PATTERN" value="3"/>
+	</enum>
+	<enum name="cmd_dma_pattern_sel">
+		<value name="CMD_DMA_PRBS" value="0"/>
+		<value name="CMD_DMA_INCREMENTAL" value="1"/>
+		<value name="CMD_DMA_FIXED" value="2"/>
+		<value name="CMD_DMA_CUSTOM_PATTERN_DMA_FIFO" value="3"/>
+	</enum>
+	<bitset name="DSI_IRQ">
+		<bitfield name="CMD_DMA_DONE" pos="0" type="boolean"/>
+		<bitfield name="MASK_CMD_DMA_DONE" pos="1" type="boolean"/>
+		<bitfield name="CMD_MDP_DONE" pos="8" type="boolean"/>
+		<bitfield name="MASK_CMD_MDP_DONE" pos="9" type="boolean"/>
+		<bitfield name="VIDEO_DONE" pos="16" type="boolean"/>
+		<bitfield name="MASK_VIDEO_DONE" pos="17" type="boolean"/>
+		<bitfield name="BTA_DONE" pos="20" type="boolean"/>
+		<bitfield name="MASK_BTA_DONE" pos="21" type="boolean"/>
+		<bitfield name="ERROR" pos="24" type="boolean"/>
+		<bitfield name="MASK_ERROR" pos="25" type="boolean"/>
+	</bitset>
+
+	<reg32 offset="0x00000" name="6G_HW_VERSION">
+		<bitfield name="MAJOR" low="28" high="31" type="uint"/>
+		<bitfield name="MINOR" low="16" high="27" type="uint"/>
+		<bitfield name="STEP" low="0" high="15" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x00000" name="CTRL">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+		<bitfield name="VID_MODE_EN" pos="1" type="boolean"/>
+		<bitfield name="CMD_MODE_EN" pos="2" type="boolean"/>
+		<bitfield name="LANE0" pos="4" type="boolean"/>
+		<bitfield name="LANE1" pos="5" type="boolean"/>
+		<bitfield name="LANE2" pos="6" type="boolean"/>
+		<bitfield name="LANE3" pos="7" type="boolean"/>
+		<bitfield name="CLK_EN" pos="8" type="boolean"/>
+		<bitfield name="ECC_CHECK" pos="20" type="boolean"/>
+		<bitfield name="CRC_CHECK" pos="24" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x00004" name="STATUS0">
+		<bitfield name="CMD_MODE_ENGINE_BUSY" pos="0" type="boolean"/>
+		<bitfield name="CMD_MODE_DMA_BUSY" pos="1" type="boolean"/>
+		<bitfield name="CMD_MODE_MDP_BUSY" pos="2" type="boolean"/>
+		<bitfield name="VIDEO_MODE_ENGINE_BUSY" pos="3" type="boolean"/>
+		<bitfield name="DSI_BUSY" pos="4" type="boolean"/>  <!-- see mipi_dsi_cmd_bta_sw_trigger() -->
+		<bitfield name="INTERLEAVE_OP_CONTENTION" pos="31" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x00008" name="FIFO_STATUS">
+		<bitfield name="VIDEO_MDP_FIFO_OVERFLOW" pos="0" type="boolean"/>
+		<bitfield name="VIDEO_MDP_FIFO_UNDERFLOW" pos="3" type="boolean"/>
+		<bitfield name="CMD_MDP_FIFO_UNDERFLOW" pos="7" type="boolean"/>
+		<bitfield name="CMD_DMA_FIFO_RD_WATERMARK_REACH" pos="8" type="boolean"/>
+		<bitfield name="CMD_DMA_FIFO_WR_WATERMARK_REACH" pos="9" type="boolean"/>
+		<bitfield name="CMD_DMA_FIFO_UNDERFLOW" pos="10" type="boolean"/>
+		<bitfield name="DLN0_LP_FIFO_EMPTY"     pos="12" type="boolean"/>
+		<bitfield name="DLN0_LP_FIFO_FULL"      pos="13" type="boolean"/>
+		<bitfield name="DLN0_LP_FIFO_OVERFLOW"  pos="14" type="boolean"/>
+		<bitfield name="DLN0_HS_FIFO_EMPTY"     pos="16" type="boolean"/>
+		<bitfield name="DLN0_HS_FIFO_FULL"      pos="17" type="boolean"/>
+		<bitfield name="DLN0_HS_FIFO_OVERFLOW"  pos="18" type="boolean"/>
+		<bitfield name="DLN0_HS_FIFO_UNDERFLOW" pos="19" type="boolean"/>
+		<bitfield name="DLN1_HS_FIFO_EMPTY"     pos="20" type="boolean"/>
+		<bitfield name="DLN1_HS_FIFO_FULL"      pos="21" type="boolean"/>
+		<bitfield name="DLN1_HS_FIFO_OVERFLOW"  pos="22" type="boolean"/>
+		<bitfield name="DLN1_HS_FIFO_UNDERFLOW" pos="23" type="boolean"/>
+		<bitfield name="DLN2_HS_FIFO_EMPTY"     pos="24" type="boolean"/>
+		<bitfield name="DLN2_HS_FIFO_FULL"      pos="25" type="boolean"/>
+		<bitfield name="DLN2_HS_FIFO_OVERFLOW"  pos="26" type="boolean"/>
+		<bitfield name="DLN2_HS_FIFO_UNDERFLOW" pos="27" type="boolean"/>
+		<bitfield name="DLN3_HS_FIFO_EMPTY"     pos="28" type="boolean"/>
+		<bitfield name="DLN3_HS_FIFO_FULL"      pos="29" type="boolean"/>
+		<bitfield name="DLN3_HS_FIFO_OVERFLOW"  pos="30" type="boolean"/>
+		<bitfield name="DLN3_HS_FIFO_UNDERFLOW" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0000c" name="VID_CFG0">
+		<bitfield name="VIRT_CHANNEL" low="0" high="1" type="uint"/>  <!-- always zero? -->
+		<bitfield name="DST_FORMAT" low="4" high="5" type="dsi_vid_dst_format"/>
+		<bitfield name="TRAFFIC_MODE" low="8" high="9" type="dsi_traffic_mode"/>
+		<bitfield name="BLLP_POWER_STOP" pos="12" type="boolean"/>
+		<bitfield name="EOF_BLLP_POWER_STOP" pos="15" type="boolean"/>
+		<bitfield name="HSA_POWER_STOP" pos="16" type="boolean"/>
+		<bitfield name="HBP_POWER_STOP" pos="20" type="boolean"/>
+		<bitfield name="HFP_POWER_STOP" pos="24" type="boolean"/>
+		<bitfield name="DATABUS_WIDEN" pos="25" type="boolean"/>
+		<bitfield name="PULSE_MODE_HSA_HE" pos="28" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0001c" name="VID_CFG1">
+		<bitfield name="R_SEL" pos="0" type="boolean"/>
+		<bitfield name="G_SEL" pos="4" type="boolean"/>
+		<bitfield name="B_SEL" pos="8" type="boolean"/>
+		<bitfield name="RGB_SWAP" low="12" high="14" type="dsi_rgb_swap"/>
+	</reg32>
+	<reg32 offset="0x00020" name="ACTIVE_H">
+		<bitfield name="START" low="0" high="11" type="uint"/>
+		<bitfield name="END" low="16" high="27" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00024" name="ACTIVE_V">
+		<bitfield name="START" low="0" high="11" type="uint"/>
+		<bitfield name="END" low="16" high="27" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00028" name="TOTAL">
+		<bitfield name="H_TOTAL" low="0" high="11" type="uint"/>
+		<bitfield name="V_TOTAL" low="16" high="27" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0002c" name="ACTIVE_HSYNC">
+		<bitfield name="START" low="0" high="11" type="uint"/>
+		<bitfield name="END" low="16" high="27" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00030" name="ACTIVE_VSYNC_HPOS">
+		<bitfield name="START" low="0" high="11" type="uint"/>
+		<bitfield name="END" low="16" high="27" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00034" name="ACTIVE_VSYNC_VPOS">
+		<bitfield name="START" low="0" high="11" type="uint"/>
+		<bitfield name="END" low="16" high="27" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x00038" name="CMD_DMA_CTRL">
+		<bitfield name="BROADCAST_EN" pos="31" type="boolean"/>
+		<bitfield name="FROM_FRAME_BUFFER" pos="28" type="boolean"/>
+		<bitfield name="LOW_POWER" pos="26" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0003c" name="CMD_CFG0">
+		<bitfield name="DST_FORMAT" low="0" high="3" type="dsi_cmd_dst_format"/>
+		<bitfield name="R_SEL" pos="4" type="boolean"/>
+		<bitfield name="G_SEL" pos="8" type="boolean"/>
+		<bitfield name="B_SEL" pos="12" type="boolean"/>
+		<bitfield name="INTERLEAVE_MAX" low="20" high="23" type="uint"/>
+		<bitfield name="RGB_SWAP" low="16" high="18" type="dsi_rgb_swap"/>
+	</reg32>
+	<reg32 offset="0x00040" name="CMD_CFG1">
+		<bitfield name="WR_MEM_START" low="0" high="7" type="uint"/>
+		<bitfield name="WR_MEM_CONTINUE" low="8" high="15" type="uint"/>
+		<bitfield name="INSERT_DCS_COMMAND" pos="16" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00044" name="DMA_BASE"/>
+	<reg32 offset="0x00048" name="DMA_LEN"/>
+	<reg32 offset="0x00054" name="CMD_MDP_STREAM0_CTRL">
+		<bitfield name="DATA_TYPE" low="0" high="5" type="uint"/>
+		<bitfield name="VIRTUAL_CHANNEL" low="8" high="9" type="uint"/>
+		<bitfield name="WORD_COUNT" low="16" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00058" name="CMD_MDP_STREAM0_TOTAL">
+		<bitfield name="H_TOTAL" low="0" high="11" type="uint"/>
+		<bitfield name="V_TOTAL" low="16" high="27" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0005c" name="CMD_MDP_STREAM1_CTRL">
+		<bitfield name="DATA_TYPE" low="0" high="5" type="uint"/>
+		<bitfield name="VIRTUAL_CHANNEL" low="8" high="9" type="uint"/>
+		<bitfield name="WORD_COUNT" low="16" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00060" name="CMD_MDP_STREAM1_TOTAL">
+		<bitfield name="H_TOTAL" low="0" high="15" type="uint"/>
+		<bitfield name="V_TOTAL" low="16" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00064" name="ACK_ERR_STATUS"/>
+	<array offset="0x00068" name="RDBK" length="4" stride="4">
+		<reg32 offset="0x0" name="DATA"/>
+	</array>
+	<reg32 offset="0x00080" name="TRIG_CTRL">
+		<bitfield name="DMA_TRIGGER" low="0" high="2" type="dsi_cmd_trigger"/>
+		<bitfield name="MDP_TRIGGER" low="4" high="6" type="dsi_cmd_trigger"/>
+		<bitfield name="STREAM" low="8" high="9" type="uint"/>
+		<bitfield name="BLOCK_DMA_WITHIN_FRAME" pos="12" type="boolean"/>
+		<bitfield name="TE" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0008c" name="TRIG_DMA"/>
+	<reg32 offset="0x000b0" name="DLN0_PHY_ERR">
+		<bitfield name="DLN0_ERR_ESC" pos="0" type="boolean"/>
+		<bitfield name="DLN0_ERR_SYNC_ESC" pos="4" type="boolean"/>
+		<bitfield name="DLN0_ERR_CONTROL" pos="8" type="boolean"/>
+		<bitfield name="DLN0_ERR_CONTENTION_LP0" pos="12" type="boolean"/>
+		<bitfield name="DLN0_ERR_CONTENTION_LP1" pos="16" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x000b4" name="LP_TIMER_CTRL">
+		<bitfield name="LP_RX_TO" low="0" high="15" type="uint"/>
+		<bitfield name="BTA_TO" low="16" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x000b8" name="HS_TIMER_CTRL">
+		<bitfield name="HS_TX_TO" low="0" high="15" type="uint"/>
+		<bitfield name="TIMER_RESOLUTION" low="16" high="19" type="uint"/>
+		<bitfield name="HS_TX_TO_STOP_EN" pos="28" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x000bc" name="TIMEOUT_STATUS"/>
+	<reg32 offset="0x000c0" name="CLKOUT_TIMING_CTRL">
+		<bitfield name="T_CLK_PRE" low="0" high="5" type="uint"/>
+		<bitfield name="T_CLK_POST" low="8" high="13" type="uint"/>
+	</reg32>
+	<reg32 offset="0x000c8" name="EOT_PACKET_CTRL">
+		<bitfield name="TX_EOT_APPEND" pos="0" type="boolean"/>
+		<bitfield name="RX_EOT_IGNORE" pos="4" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x000a4" name="LANE_STATUS">
+		<bitfield name="DLN0_STOPSTATE" pos="0" type="boolean"/>
+		<bitfield name="DLN1_STOPSTATE" pos="1" type="boolean"/>
+		<bitfield name="DLN2_STOPSTATE" pos="2" type="boolean"/>
+		<bitfield name="DLN3_STOPSTATE" pos="3" type="boolean"/>
+		<bitfield name="CLKLN_STOPSTATE" pos="4" type="boolean"/>
+		<bitfield name="DLN0_ULPS_ACTIVE_NOT" pos="8" type="boolean"/>
+		<bitfield name="DLN1_ULPS_ACTIVE_NOT" pos="9" type="boolean"/>
+		<bitfield name="DLN2_ULPS_ACTIVE_NOT" pos="10" type="boolean"/>
+		<bitfield name="DLN3_ULPS_ACTIVE_NOT" pos="11" type="boolean"/>
+		<bitfield name="CLKLN_ULPS_ACTIVE_NOT" pos="12" type="boolean"/>
+		<bitfield name="DLN0_DIRECTION" pos="16" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x000a8" name="LANE_CTRL">
+		<bitfield name="HS_REQ_SEL_PHY" pos="24" type="boolean"/>
+		<bitfield name="CLKLN_HS_FORCE_REQUEST" pos="28" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x000ac" name="LANE_SWAP_CTRL">
+		<bitfield name="DLN_SWAP_SEL" low="0" high="2" type="dsi_lane_swap"/>
+	</reg32>
+	<reg32 offset="0x00108" name="ERR_INT_MASK0"/>
+	<reg32 offset="0x0010c" name="INTR_CTRL" type="DSI_IRQ"/>
+	<reg32 offset="0x00114" name="RESET"/>
+	<reg32 offset="0x00118" name="CLK_CTRL">
+		<bitfield name="AHBS_HCLK_ON" pos="0" type="boolean"/>
+		<bitfield name="AHBM_SCLK_ON" pos="1" type="boolean"/>
+		<bitfield name="PCLK_ON" pos="2" type="boolean"/>
+		<bitfield name="DSICLK_ON" pos="3" type="boolean"/>
+		<bitfield name="BYTECLK_ON" pos="4" type="boolean"/>
+		<bitfield name="ESCCLK_ON" pos="5" type="boolean"/>
+		<bitfield name="FORCE_ON_DYN_AHBM_HCLK" pos="9" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0011c" name="CLK_STATUS">
+		<bitfield name="DSI_AON_AHBM_HCLK_ACTIVE" pos="0" type="boolean"/>
+		<bitfield name="DSI_DYN_AHBM_HCLK_ACTIVE" pos="1" type="boolean"/>
+		<bitfield name="DSI_AON_AHBS_HCLK_ACTIVE" pos="2" type="boolean"/>
+		<bitfield name="DSI_DYN_AHBS_HCLK_ACTIVE" pos="3" type="boolean"/>
+		<bitfield name="DSI_AON_DSICLK_ACTIVE" pos="4" type="boolean"/>
+		<bitfield name="DSI_DYN_DSICLK_ACTIVE" pos="5" type="boolean"/>
+		<bitfield name="DSI_AON_BYTECLK_ACTIVE" pos="6" type="boolean"/>
+		<bitfield name="DSI_DYN_BYTECLK_ACTIVE" pos="7" type="boolean"/>
+		<bitfield name="DSI_AON_ESCCLK_ACTIVE" pos="8" type="boolean"/>
+		<bitfield name="DSI_AON_PCLK_ACTIVE" pos="9" type="boolean"/>
+		<bitfield name="DSI_DYN_PCLK_ACTIVE" pos="10" type="boolean"/>
+		<bitfield name="DSI_DYN_CMD_PCLK_ACTIVE" pos="12" type="boolean"/>
+		<bitfield name="DSI_CMD_PCLK_ACTIVE" pos="13" type="boolean"/>
+		<bitfield name="DSI_VID_PCLK_ACTIVE" pos="14" type="boolean"/>
+		<bitfield name="DSI_CAM_BIST_PCLK_ACT" pos="15" type="boolean"/>
+		<bitfield name="PLL_UNLOCKED" pos="16" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00128" name="PHY_RESET">
+		<bitfield name="RESET" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00160" name="TEST_PATTERN_GEN_VIDEO_INIT_VAL"/>
+	<reg32 offset="0x00198" name="TPG_MAIN_CONTROL">
+		<bitfield name="CHECKERED_RECTANGLE_PATTERN" pos="8" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x001a0" name="TPG_VIDEO_CONFIG">
+		<bitfield name="BPP" low="0" high="1" type="video_config_bpp"/>
+		<bitfield name="RGB" pos="2" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00158" name="TEST_PATTERN_GEN_CTRL">
+		<bitfield name="CMD_DMA_PATTERN_SEL" low="16" high="17" type="cmd_dma_pattern_sel"/>
+		<bitfield name="CMD_MDP_STREAM0_PATTERN_SEL" low="8" high="9" type="cmd_mdp_stream0_pattern_sel"/>
+		<bitfield name="VIDEO_PATTERN_SEL" low="4" high="5" type="video_pattern_sel"/>
+		<bitfield name="TPG_DMA_FIFO_MODE" pos="2" type="boolean"/>
+		<bitfield name="CMD_DMA_TPG_EN" pos="1" type="boolean"/>
+		<bitfield name="EN" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00168" name="TEST_PATTERN_GEN_CMD_MDP_INIT_VAL0"/>
+	<reg32 offset="0x00180" name="TEST_PATTERN_GEN_CMD_STREAM0_TRIGGER">
+		<bitfield name="SW_TRIGGER" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0019c" name="TPG_MAIN_CONTROL2">
+		<bitfield name="CMD_MDP0_CHECKERED_RECTANGLE_PATTERN" pos="7" type="boolean"/>
+		<bitfield name="CMD_MDP1_CHECKERED_RECTANGLE_PATTERN" pos="16" type="boolean"/>
+		<bitfield name="CMD_MDP2_CHECKERED_RECTANGLE_PATTERN" pos="25" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0017c" name="T_CLK_PRE_EXTEND">
+		<bitfield name="INC_BY_2_BYTECLK" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x001b4" name="CMD_MODE_MDP_CTRL2">
+		<bitfield name="DST_FORMAT2" low="0" high="3" type="dsi_cmd_dst_format"/>
+		<bitfield name="R_SEL" pos="4" type="boolean"/>
+		<bitfield name="G_SEL" pos="5" type="boolean"/>
+		<bitfield name="B_SEL" pos="6" type="boolean"/>
+		<bitfield name="BYTE_MSB_LSB_FLIP" pos="7" type="boolean"/>
+		<bitfield name="RGB_SWAP" low="8" high="10" type="dsi_rgb_swap"/>
+		<bitfield name="INPUT_RGB_SWAP" low="12" high="14" type="dsi_rgb_swap"/>
+		<bitfield name="BURST_MODE" pos="16" type="boolean"/>
+		<bitfield name="DATABUS_WIDEN" pos="20" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x001b8" name="CMD_MODE_MDP_STREAM2_CTRL">
+		<bitfield name="DATA_TYPE" low="0" high="5" type="uint"/>
+		<bitfield name="VIRTUAL_CHANNEL" low="8" high="9" type="uint"/>
+		<bitfield name="WORD_COUNT" low="16" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x001d0" name="RDBK_DATA_CTRL">
+		<bitfield name="COUNT" low="16" high="23" type="uint"/>
+		<bitfield name="CLR" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x001f0" name="VERSION">
+		<bitfield name="MAJOR" low="24" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x002d4" name="CPHY_MODE_CTRL"/>
+	<reg32 offset="0x0029c" name="VIDEO_COMPRESSION_MODE_CTRL">
+		<bitfield name="WC" low="16" high="31" type="uint"/>
+		<bitfield name="DATATYPE" low="8" high="13" type="uint"/>
+		<bitfield name="PKT_PER_LINE" low="6" high="7" type="uint"/>
+		<bitfield name="EOL_BYTE_NUM" low="4" high="5" type="uint"/>
+		<bitfield name="EN" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x002a4" name="COMMAND_COMPRESSION_MODE_CTRL">
+		<bitfield name="STREAM1_DATATYPE" low="24" high="29" type="uint"/>
+		<bitfield name="STREAM1_PKT_PER_LINE" low="22" high="23" type="uint"/>
+		<bitfield name="STREAM1_EOL_BYTE_NUM" low="20" high="21" type="uint"/>
+		<bitfield name="STREAM1_EN" pos="16" type="boolean"/>
+		<bitfield name="STREAM0_DATATYPE" low="8" high="13" type="uint"/>
+		<bitfield name="STREAM0_PKT_PER_LINE" low="6" high="7" type="uint"/>
+		<bitfield name="STREAM0_EOL_BYTE_NUM" low="4" high="5" type="uint"/>
+		<bitfield name="STREAM0_EN" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x002a8" name="COMMAND_COMPRESSION_MODE_CTRL2">
+		<bitfield name="STREAM1_SLICE_WIDTH" low="16" high="31" type="uint"/>
+		<bitfield name="STREAM0_SLICE_WIDTH" low="0" high="15" type="uint"/>
+	</reg32>
+
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/display/dsi_phy_10nm.xml b/drivers/gpu/drm/msm/registers/display/dsi_phy_10nm.xml
new file mode 100644
index 000000000000..874c3db3e126
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/dsi_phy_10nm.xml
@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+
+<domain name="DSI_10nm_PHY_CMN" width="32">
+	<reg32 offset="0x00000" name="REVISION_ID0"/>
+	<reg32 offset="0x00004" name="REVISION_ID1"/>
+	<reg32 offset="0x00008" name="REVISION_ID2"/>
+	<reg32 offset="0x0000c" name="REVISION_ID3"/>
+	<reg32 offset="0x00010" name="CLK_CFG0"/>
+	<reg32 offset="0x00014" name="CLK_CFG1"/>
+	<reg32 offset="0x00018" name="GLBL_CTRL"/>
+	<reg32 offset="0x0001c" name="RBUF_CTRL"/>
+	<reg32 offset="0x00020" name="VREG_CTRL"/>
+	<reg32 offset="0x00024" name="CTRL_0"/>
+	<reg32 offset="0x00028" name="CTRL_1"/>
+	<reg32 offset="0x0002c" name="CTRL_2"/>
+	<reg32 offset="0x00030" name="LANE_CFG0"/>
+	<reg32 offset="0x00034" name="LANE_CFG1"/>
+	<reg32 offset="0x00038" name="PLL_CNTRL"/>
+	<reg32 offset="0x00098" name="LANE_CTRL0"/>
+	<reg32 offset="0x0009c" name="LANE_CTRL1"/>
+	<reg32 offset="0x000a0" name="LANE_CTRL2"/>
+	<reg32 offset="0x000a4" name="LANE_CTRL3"/>
+	<reg32 offset="0x000a8" name="LANE_CTRL4"/>
+	<reg32 offset="0x000ac" name="TIMING_CTRL_0"/>
+	<reg32 offset="0x000b0" name="TIMING_CTRL_1"/>
+	<reg32 offset="0x000b4" name="TIMING_CTRL_2"/>
+	<reg32 offset="0x000b8" name="TIMING_CTRL_3"/>
+	<reg32 offset="0x000bc" name="TIMING_CTRL_4"/>
+	<reg32 offset="0x000c0" name="TIMING_CTRL_5"/>
+	<reg32 offset="0x000c4" name="TIMING_CTRL_6"/>
+	<reg32 offset="0x000c8" name="TIMING_CTRL_7"/>
+	<reg32 offset="0x000cc" name="TIMING_CTRL_8"/>
+	<reg32 offset="0x000d0" name="TIMING_CTRL_9"/>
+	<reg32 offset="0x000d4" name="TIMING_CTRL_10"/>
+	<reg32 offset="0x000d8" name="TIMING_CTRL_11"/>
+	<reg32 offset="0x000ec" name="PHY_STATUS"/>
+	<reg32 offset="0x000f4" name="LANE_STATUS0"/>
+	<reg32 offset="0x000f8" name="LANE_STATUS1"/>
+</domain>
+
+<domain name="DSI_10nm_PHY" width="32">
+	<array offset="0x00000" name="LN" length="5" stride="0x80">
+		<reg32 offset="0x00" name="CFG0"/>
+		<reg32 offset="0x04" name="CFG1"/>
+		<reg32 offset="0x08" name="CFG2"/>
+		<reg32 offset="0x0c" name="CFG3"/>
+		<reg32 offset="0x10" name="TEST_DATAPATH"/>
+		<reg32 offset="0x14" name="PIN_SWAP"/>
+		<reg32 offset="0x18" name="HSTX_STR_CTRL"/>
+		<reg32 offset="0x1c" name="OFFSET_TOP_CTRL"/>
+		<reg32 offset="0x20" name="OFFSET_BOT_CTRL"/>
+		<reg32 offset="0x24" name="LPTX_STR_CTRL"/>
+		<reg32 offset="0x28" name="LPRX_CTRL"/>
+		<reg32 offset="0x2c" name="TX_DCTRL"/>
+	</array>
+</domain>
+
+<domain name="DSI_10nm_PHY_PLL" width="32">
+	<reg32 offset="0x0000" name="ANALOG_CONTROLS_ONE"/>
+	<reg32 offset="0x0004" name="ANALOG_CONTROLS_TWO"/>
+	<reg32 offset="0x0010" name="ANALOG_CONTROLS_THREE"/>
+	<reg32 offset="0x001c" name="DSM_DIVIDER"/>
+	<reg32 offset="0x0020" name="FEEDBACK_DIVIDER"/>
+	<reg32 offset="0x0024" name="SYSTEM_MUXES"/>
+	<reg32 offset="0x002c" name="CMODE"/>
+	<reg32 offset="0x0030" name="CALIBRATION_SETTINGS"/>
+	<reg32 offset="0x0054" name="BAND_SEL_CAL_SETTINGS_THREE"/>
+	<reg32 offset="0x0064" name="FREQ_DETECT_SETTINGS_ONE"/>
+	<reg32 offset="0x007c" name="PFILT"/>
+	<reg32 offset="0x0080" name="IFILT"/>
+	<reg32 offset="0x0094" name="OUTDIV"/>
+	<reg32 offset="0x00a4" name="CORE_OVERRIDE"/>
+	<reg32 offset="0x00a8" name="CORE_INPUT_OVERRIDE"/>
+	<reg32 offset="0x00b4" name="PLL_DIGITAL_TIMERS_TWO"/>
+	<reg32 offset="0x00cc" name="DECIMAL_DIV_START_1"/>
+	<reg32 offset="0x00d0" name="FRAC_DIV_START_LOW_1"/>
+	<reg32 offset="0x00d4" name="FRAC_DIV_START_MID_1"/>
+	<reg32 offset="0x00d8" name="FRAC_DIV_START_HIGH_1"/>
+	<reg32 offset="0x010c" name="SSC_STEPSIZE_LOW_1"/>
+	<reg32 offset="0x0110" name="SSC_STEPSIZE_HIGH_1"/>
+	<reg32 offset="0x0114" name="SSC_DIV_PER_LOW_1"/>
+	<reg32 offset="0x0118" name="SSC_DIV_PER_HIGH_1"/>
+	<reg32 offset="0x011c" name="SSC_DIV_ADJPER_LOW_1"/>
+	<reg32 offset="0x0120" name="SSC_DIV_ADJPER_HIGH_1"/>
+	<reg32 offset="0x013c" name="SSC_CONTROL"/>
+	<reg32 offset="0x0140" name="PLL_OUTDIV_RATE"/>
+	<reg32 offset="0x0144" name="PLL_LOCKDET_RATE_1"/>
+	<reg32 offset="0x014c" name="PLL_PROP_GAIN_RATE_1"/>
+	<reg32 offset="0x0154" name="PLL_BAND_SET_RATE_1"/>
+	<reg32 offset="0x015c" name="PLL_INT_GAIN_IFILT_BAND_1"/>
+	<reg32 offset="0x0164" name="PLL_FL_INT_GAIN_PFILT_BAND_1"/>
+	<reg32 offset="0x0180" name="PLL_LOCK_OVERRIDE"/>
+	<reg32 offset="0x0184" name="PLL_LOCK_DELAY"/>
+	<reg32 offset="0x018c" name="CLOCK_INVERTERS"/>
+	<reg32 offset="0x01a0" name="COMMON_STATUS_ONE"/>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/display/dsi_phy_14nm.xml b/drivers/gpu/drm/msm/registers/display/dsi_phy_14nm.xml
new file mode 100644
index 000000000000..314b74489d49
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/dsi_phy_14nm.xml
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+
+<domain name="DSI_14nm_PHY_CMN" width="32">
+	<reg32 offset="0x00000" name="REVISION_ID0"/>
+	<reg32 offset="0x00004" name="REVISION_ID1"/>
+	<reg32 offset="0x00008" name="REVISION_ID2"/>
+	<reg32 offset="0x0000c" name="REVISION_ID3"/>
+	<reg32 offset="0x00010" name="CLK_CFG0">
+		<bitfield name="DIV_CTRL_3_0" low="4" high="7" type="uint"/>
+		<bitfield name="DIV_CTRL_7_4" low="4" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00014" name="CLK_CFG1">
+		<bitfield name="DSICLK_SEL" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00018" name="GLBL_TEST_CTRL">
+		<bitfield name="BITCLK_HS_SEL" pos="2" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0001C" name="CTRL_0"/>
+	<reg32 offset="0x00020" name="CTRL_1">
+	</reg32>
+	<reg32 offset="0x00024" name="HW_TRIGGER"/>
+	<reg32 offset="0x00028" name="SW_CFG0"/>
+	<reg32 offset="0x0002C" name="SW_CFG1"/>
+	<reg32 offset="0x00030" name="SW_CFG2"/>
+	<reg32 offset="0x00034" name="HW_CFG0"/>
+	<reg32 offset="0x00038" name="HW_CFG1"/>
+	<reg32 offset="0x0003C" name="HW_CFG2"/>
+	<reg32 offset="0x00040" name="HW_CFG3"/>
+	<reg32 offset="0x00044" name="HW_CFG4"/>
+	<reg32 offset="0x00048" name="PLL_CNTRL">
+		<bitfield name="PLL_START" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0004C" name="LDO_CNTRL">
+		<bitfield name="VREG_CTRL" low="0" high="5" type="uint"/>
+	</reg32>
+</domain>
+
+<domain name="DSI_14nm_PHY" width="32">
+	<array offset="0x00000" name="LN" length="5" stride="0x80">
+		<reg32 offset="0x00" name="CFG0">
+			<bitfield name="PREPARE_DLY" low="6" high="7" type="uint"/>
+		</reg32>
+		<reg32 offset="0x04" name="CFG1">
+			<bitfield name="HALFBYTECLK_EN" pos="0" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x08" name="CFG2"/>
+		<reg32 offset="0x0c" name="CFG3"/>
+		<reg32 offset="0x10" name="TEST_DATAPATH"/>
+		<reg32 offset="0x14" name="TEST_STR"/>
+		<reg32 offset="0x18" name="TIMING_CTRL_4">
+			<bitfield name="HS_EXIT" low="0" high="7" type="uint"/>
+		</reg32>
+		<reg32 offset="0x1c" name="TIMING_CTRL_5">
+			<bitfield name="HS_ZERO" low="0" high="7" type="uint"/>
+		</reg32>
+		<reg32 offset="0x20" name="TIMING_CTRL_6">
+			<bitfield name="HS_PREPARE" low="0" high="7" type="uint"/>
+		</reg32>
+		<reg32 offset="0x24" name="TIMING_CTRL_7">
+			<bitfield name="HS_TRAIL" low="0" high="7" type="uint"/>
+		</reg32>
+		<reg32 offset="0x28" name="TIMING_CTRL_8">
+			<bitfield name="HS_RQST" low="0" high="7" type="uint"/>
+		</reg32>
+		<reg32 offset="0x2c" name="TIMING_CTRL_9">
+			<bitfield name="TA_GO" low="0" high="2" type="uint"/>
+			<bitfield name="TA_SURE" low="4" high="6" type="uint"/>
+		</reg32>
+		<reg32 offset="0x30" name="TIMING_CTRL_10">
+			<bitfield name="TA_GET" low="0" high="2" type="uint"/>
+		</reg32>
+		<reg32 offset="0x34" name="TIMING_CTRL_11">
+			<bitfield name="TRIG3_CMD" low="0" high="7" type="uint"/>
+		</reg32>
+		<reg32 offset="0x38" name="STRENGTH_CTRL_0"/>
+		<reg32 offset="0x3c" name="STRENGTH_CTRL_1"/>
+		<reg32 offset="0x64" name="VREG_CNTRL"/>
+	</array>
+</domain>
+
+<domain name="DSI_14nm_PHY_PLL" width="32">
+	<reg32 offset="0x000" name="IE_TRIM"/>
+	<reg32 offset="0x004" name="IP_TRIM"/>
+	<reg32 offset="0x010" name="IPTAT_TRIM"/>
+	<reg32 offset="0x01c" name="CLKBUFLR_EN"/>
+	<reg32 offset="0x028" name="SYSCLK_EN_RESET"/>
+	<reg32 offset="0x02c" name="RESETSM_CNTRL"/>
+	<reg32 offset="0x030" name="RESETSM_CNTRL2"/>
+	<reg32 offset="0x034" name="RESETSM_CNTRL3"/>
+	<reg32 offset="0x038" name="RESETSM_CNTRL4"/>
+	<reg32 offset="0x03c" name="RESETSM_CNTRL5"/>
+	<reg32 offset="0x040" name="KVCO_DIV_REF1"/>
+	<reg32 offset="0x044" name="KVCO_DIV_REF2"/>
+	<reg32 offset="0x048" name="KVCO_COUNT1"/>
+	<reg32 offset="0x04c" name="KVCO_COUNT2"/>
+	<reg32 offset="0x05c" name="VREF_CFG1"/>
+	<reg32 offset="0x058" name="KVCO_CODE"/>
+	<reg32 offset="0x06c" name="VCO_DIV_REF1"/>
+	<reg32 offset="0x070" name="VCO_DIV_REF2"/>
+	<reg32 offset="0x074" name="VCO_COUNT1"/>
+	<reg32 offset="0x078" name="VCO_COUNT2"/>
+	<reg32 offset="0x07c" name="PLLLOCK_CMP1"/>
+	<reg32 offset="0x080" name="PLLLOCK_CMP2"/>
+	<reg32 offset="0x084" name="PLLLOCK_CMP3"/>
+	<reg32 offset="0x088" name="PLLLOCK_CMP_EN"/>
+	<reg32 offset="0x08c" name="PLL_VCO_TUNE"/>
+	<reg32 offset="0x090" name="DEC_START"/>
+	<reg32 offset="0x094" name="SSC_EN_CENTER"/>
+	<reg32 offset="0x098" name="SSC_ADJ_PER1"/>
+	<reg32 offset="0x09c" name="SSC_ADJ_PER2"/>
+	<reg32 offset="0x0a0" name="SSC_PER1"/>
+	<reg32 offset="0x0a4" name="SSC_PER2"/>
+	<reg32 offset="0x0a8" name="SSC_STEP_SIZE1"/>
+	<reg32 offset="0x0ac" name="SSC_STEP_SIZE2"/>
+	<reg32 offset="0x0b4" name="DIV_FRAC_START1"/>
+	<reg32 offset="0x0b8" name="DIV_FRAC_START2"/>
+	<reg32 offset="0x0bc" name="DIV_FRAC_START3"/>
+	<reg32 offset="0x0c0" name="TXCLK_EN"/>
+	<reg32 offset="0x0c4" name="PLL_CRCTRL"/>
+	<reg32 offset="0x0cc" name="RESET_SM_READY_STATUS"/>
+	<reg32 offset="0x0e8" name="PLL_MISC1"/>
+	<reg32 offset="0x0f0" name="CP_SET_CUR"/>
+	<reg32 offset="0x0f4" name="PLL_ICPMSET"/>
+	<reg32 offset="0x0f8" name="PLL_ICPCSET"/>
+	<reg32 offset="0x0fc" name="PLL_ICP_SET"/>
+	<reg32 offset="0x100" name="PLL_LPF1"/>
+	<reg32 offset="0x104" name="PLL_LPF2_POSTDIV"/>
+	<reg32 offset="0x108" name="PLL_BANDGAP"/>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/display/dsi_phy_20nm.xml b/drivers/gpu/drm/msm/registers/display/dsi_phy_20nm.xml
new file mode 100644
index 000000000000..99e9deb361b6
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/dsi_phy_20nm.xml
@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+
+<domain name="DSI_20nm_PHY" width="32">
+	<array offset="0x00000" name="LN" length="4" stride="0x40">
+		<reg32 offset="0x00" name="CFG_0"/>
+		<reg32 offset="0x04" name="CFG_1"/>
+		<reg32 offset="0x08" name="CFG_2"/>
+		<reg32 offset="0x0c" name="CFG_3"/>
+		<reg32 offset="0x10" name="CFG_4"/>
+		<reg32 offset="0x14" name="TEST_DATAPATH"/>
+		<reg32 offset="0x18" name="DEBUG_SEL"/>
+		<reg32 offset="0x1c" name="TEST_STR_0"/>
+		<reg32 offset="0x20" name="TEST_STR_1"/>
+	</array>
+
+	<reg32 offset="0x00100" name="LNCK_CFG_0"/>
+	<reg32 offset="0x00104" name="LNCK_CFG_1"/>
+	<reg32 offset="0x00108" name="LNCK_CFG_2"/>
+	<reg32 offset="0x0010c" name="LNCK_CFG_3"/>
+	<reg32 offset="0x00110" name="LNCK_CFG_4"/>
+	<reg32 offset="0x00114" name="LNCK_TEST_DATAPATH"/>
+	<reg32 offset="0x00118" name="LNCK_DEBUG_SEL"/>
+	<reg32 offset="0x0011c" name="LNCK_TEST_STR0"/>
+	<reg32 offset="0x00120" name="LNCK_TEST_STR1"/>
+
+	<reg32 offset="0x00140" name="TIMING_CTRL_0">
+		<bitfield name="CLK_ZERO" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00144" name="TIMING_CTRL_1">
+		<bitfield name="CLK_TRAIL" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00148" name="TIMING_CTRL_2">
+		<bitfield name="CLK_PREPARE" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0014c" name="TIMING_CTRL_3">
+		<bitfield name="CLK_ZERO_8" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00150" name="TIMING_CTRL_4">
+		<bitfield name="HS_EXIT" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00154" name="TIMING_CTRL_5">
+		<bitfield name="HS_ZERO" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00158" name="TIMING_CTRL_6">
+		<bitfield name="HS_PREPARE" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0015c" name="TIMING_CTRL_7">
+		<bitfield name="HS_TRAIL" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00160" name="TIMING_CTRL_8">
+		<bitfield name="HS_RQST" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00164" name="TIMING_CTRL_9">
+		<bitfield name="TA_GO" low="0" high="2" type="uint"/>
+		<bitfield name="TA_SURE" low="4" high="6" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00168" name="TIMING_CTRL_10">
+		<bitfield name="TA_GET" low="0" high="2" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0016c" name="TIMING_CTRL_11">
+		<bitfield name="TRIG3_CMD" low="0" high="7" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x00170" name="CTRL_0"/>
+	<reg32 offset="0x00174" name="CTRL_1"/>
+	<reg32 offset="0x00178" name="CTRL_2"/>
+	<reg32 offset="0x0017c" name="CTRL_3"/>
+	<reg32 offset="0x00180" name="CTRL_4"/>
+
+	<reg32 offset="0x00184" name="STRENGTH_0"/>
+	<reg32 offset="0x00188" name="STRENGTH_1"/>
+
+	<reg32 offset="0x001b4" name="BIST_CTRL_0"/>
+	<reg32 offset="0x001b8" name="BIST_CTRL_1"/>
+	<reg32 offset="0x001bc" name="BIST_CTRL_2"/>
+	<reg32 offset="0x001c0" name="BIST_CTRL_3"/>
+	<reg32 offset="0x001c4" name="BIST_CTRL_4"/>
+	<reg32 offset="0x001c8" name="BIST_CTRL_5"/>
+
+	<reg32 offset="0x001d4" name="GLBL_TEST_CTRL">
+		<bitfield name="BITCLK_HS_SEL" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x001dc" name="LDO_CNTRL"/>
+</domain>
+
+<domain name="DSI_20nm_PHY_REGULATOR" width="32">
+	<reg32 offset="0x00000" name="CTRL_0"/>
+	<reg32 offset="0x00004" name="CTRL_1"/>
+	<reg32 offset="0x00008" name="CTRL_2"/>
+	<reg32 offset="0x0000c" name="CTRL_3"/>
+	<reg32 offset="0x00010" name="CTRL_4"/>
+	<reg32 offset="0x00014" name="CTRL_5"/>
+	<reg32 offset="0x00018" name="CAL_PWR_CFG"/>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/display/dsi_phy_28nm.xml b/drivers/gpu/drm/msm/registers/display/dsi_phy_28nm.xml
new file mode 100644
index 000000000000..81d5b96f18c4
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/dsi_phy_28nm.xml
@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+
+<domain name="DSI_28nm_PHY" width="32">
+	<array offset="0x00000" name="LN" length="4" stride="0x40">
+		<reg32 offset="0x00" name="CFG_0"/>
+		<reg32 offset="0x04" name="CFG_1"/>
+		<reg32 offset="0x08" name="CFG_2"/>
+		<reg32 offset="0x0c" name="CFG_3"/>
+		<reg32 offset="0x10" name="CFG_4"/>
+		<reg32 offset="0x14" name="TEST_DATAPATH"/>
+		<reg32 offset="0x18" name="DEBUG_SEL"/>
+		<reg32 offset="0x1c" name="TEST_STR_0"/>
+		<reg32 offset="0x20" name="TEST_STR_1"/>
+	</array>
+
+	<reg32 offset="0x00100" name="LNCK_CFG_0"/>
+	<reg32 offset="0x00104" name="LNCK_CFG_1"/>
+	<reg32 offset="0x00108" name="LNCK_CFG_2"/>
+	<reg32 offset="0x0010c" name="LNCK_CFG_3"/>
+	<reg32 offset="0x00110" name="LNCK_CFG_4"/>
+	<reg32 offset="0x00114" name="LNCK_TEST_DATAPATH"/>
+	<reg32 offset="0x00118" name="LNCK_DEBUG_SEL"/>
+	<reg32 offset="0x0011c" name="LNCK_TEST_STR0"/>
+	<reg32 offset="0x00120" name="LNCK_TEST_STR1"/>
+
+	<reg32 offset="0x00140" name="TIMING_CTRL_0">
+		<bitfield name="CLK_ZERO" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00144" name="TIMING_CTRL_1">
+		<bitfield name="CLK_TRAIL" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00148" name="TIMING_CTRL_2">
+		<bitfield name="CLK_PREPARE" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0014c" name="TIMING_CTRL_3">
+		<bitfield name="CLK_ZERO_8" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00150" name="TIMING_CTRL_4">
+		<bitfield name="HS_EXIT" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00154" name="TIMING_CTRL_5">
+		<bitfield name="HS_ZERO" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00158" name="TIMING_CTRL_6">
+		<bitfield name="HS_PREPARE" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0015c" name="TIMING_CTRL_7">
+		<bitfield name="HS_TRAIL" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00160" name="TIMING_CTRL_8">
+		<bitfield name="HS_RQST" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00164" name="TIMING_CTRL_9">
+		<bitfield name="TA_GO" low="0" high="2" type="uint"/>
+		<bitfield name="TA_SURE" low="4" high="6" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00168" name="TIMING_CTRL_10">
+		<bitfield name="TA_GET" low="0" high="2" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0016c" name="TIMING_CTRL_11">
+		<bitfield name="TRIG3_CMD" low="0" high="7" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x00170" name="CTRL_0"/>
+	<reg32 offset="0x00174" name="CTRL_1"/>
+	<reg32 offset="0x00178" name="CTRL_2"/>
+	<reg32 offset="0x0017c" name="CTRL_3"/>
+	<reg32 offset="0x00180" name="CTRL_4"/>
+
+	<reg32 offset="0x00184" name="STRENGTH_0"/>
+	<reg32 offset="0x00188" name="STRENGTH_1"/>
+
+	<reg32 offset="0x001b4" name="BIST_CTRL_0"/>
+	<reg32 offset="0x001b8" name="BIST_CTRL_1"/>
+	<reg32 offset="0x001bc" name="BIST_CTRL_2"/>
+	<reg32 offset="0x001c0" name="BIST_CTRL_3"/>
+	<reg32 offset="0x001c4" name="BIST_CTRL_4"/>
+	<reg32 offset="0x001c8" name="BIST_CTRL_5"/>
+
+	<reg32 offset="0x001d4" name="GLBL_TEST_CTRL">
+		<bitfield name="BITCLK_HS_SEL" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x001dc" name="LDO_CNTRL"/>
+</domain>
+
+<domain name="DSI_28nm_PHY_REGULATOR" width="32">
+	<reg32 offset="0x00000" name="CTRL_0"/>
+	<reg32 offset="0x00004" name="CTRL_1"/>
+	<reg32 offset="0x00008" name="CTRL_2"/>
+	<reg32 offset="0x0000c" name="CTRL_3"/>
+	<reg32 offset="0x00010" name="CTRL_4"/>
+	<reg32 offset="0x00014" name="CTRL_5"/>
+	<reg32 offset="0x00018" name="CAL_PWR_CFG"/>
+</domain>
+
+<domain name="DSI_28nm_PHY_PLL" width="32">
+	<reg32 offset="0x00000" name="REFCLK_CFG">
+		<bitfield name="DBLR" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00004" name="POSTDIV1_CFG"/>
+	<reg32 offset="0x00008" name="CHGPUMP_CFG"/>
+	<reg32 offset="0x0000C" name="VCOLPF_CFG"/>
+	<reg32 offset="0x00010" name="VREG_CFG">
+		<bitfield name="POSTDIV1_BYPASS_B" pos="1" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00014" name="PWRGEN_CFG"/>
+	<reg32 offset="0x00018" name="DMUX_CFG"/>
+	<reg32 offset="0x0001C" name="AMUX_CFG"/>
+	<reg32 offset="0x00020" name="GLB_CFG">
+		<bitfield name="PLL_PWRDN_B" pos="0" type="boolean"/>
+		<bitfield name="PLL_LDO_PWRDN_B" pos="1" type="boolean"/>
+		<bitfield name="PLL_PWRGEN_PWRDN_B" pos="2" type="boolean"/>
+		<bitfield name="PLL_ENABLE" pos="3" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00024" name="POSTDIV2_CFG"/>
+	<reg32 offset="0x00028" name="POSTDIV3_CFG"/>
+	<reg32 offset="0x0002C" name="LPFR_CFG"/>
+	<reg32 offset="0x00030" name="LPFC1_CFG"/>
+	<reg32 offset="0x00034" name="LPFC2_CFG"/>
+	<reg32 offset="0x00038" name="SDM_CFG0">
+		<bitfield name="BYP_DIV" low="0" high="5" type="uint"/>
+		<bitfield name="BYP" pos="6" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0003C" name="SDM_CFG1">
+		<bitfield name="DC_OFFSET" low="0" high="5" type="uint"/>
+		<bitfield name="DITHER_EN" pos="6" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00040" name="SDM_CFG2">
+		<bitfield name="FREQ_SEED_7_0" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00044" name="SDM_CFG3">
+		<bitfield name="FREQ_SEED_15_8" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00048" name="SDM_CFG4"/>
+	<reg32 offset="0x0004C" name="SSC_CFG0"/>
+	<reg32 offset="0x00050" name="SSC_CFG1"/>
+	<reg32 offset="0x00054" name="SSC_CFG2"/>
+	<reg32 offset="0x00058" name="SSC_CFG3"/>
+	<reg32 offset="0x0005C" name="LKDET_CFG0"/>
+	<reg32 offset="0x00060" name="LKDET_CFG1"/>
+	<reg32 offset="0x00064" name="LKDET_CFG2"/>
+	<reg32 offset="0x00068" name="TEST_CFG">
+		<bitfield name="PLL_SW_RESET" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0006C" name="CAL_CFG0"/>
+	<reg32 offset="0x00070" name="CAL_CFG1"/>
+	<reg32 offset="0x00074" name="CAL_CFG2"/>
+	<reg32 offset="0x00078" name="CAL_CFG3"/>
+	<reg32 offset="0x0007C" name="CAL_CFG4"/>
+	<reg32 offset="0x00080" name="CAL_CFG5"/>
+	<reg32 offset="0x00084" name="CAL_CFG6"/>
+	<reg32 offset="0x00088" name="CAL_CFG7"/>
+	<reg32 offset="0x0008C" name="CAL_CFG8"/>
+	<reg32 offset="0x00090" name="CAL_CFG9"/>
+	<reg32 offset="0x00094" name="CAL_CFG10"/>
+	<reg32 offset="0x00098" name="CAL_CFG11"/>
+	<reg32 offset="0x0009C" name="EFUSE_CFG"/>
+	<reg32 offset="0x000A0" name="DEBUG_BUS_SEL"/>
+	<reg32 offset="0x000A4" name="CTRL_42"/>
+	<reg32 offset="0x000A8" name="CTRL_43"/>
+	<reg32 offset="0x000AC" name="CTRL_44"/>
+	<reg32 offset="0x000B0" name="CTRL_45"/>
+	<reg32 offset="0x000B4" name="CTRL_46"/>
+	<reg32 offset="0x000B8" name="CTRL_47"/>
+	<reg32 offset="0x000BC" name="CTRL_48"/>
+	<reg32 offset="0x000C0" name="STATUS">
+		<bitfield name="PLL_RDY" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x000C4" name="DEBUG_BUS0"/>
+	<reg32 offset="0x000C8" name="DEBUG_BUS1"/>
+	<reg32 offset="0x000CC" name="DEBUG_BUS2"/>
+	<reg32 offset="0x000D0" name="DEBUG_BUS3"/>
+	<reg32 offset="0x000D4" name="CTRL_54"/>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/display/dsi_phy_28nm_8960.xml b/drivers/gpu/drm/msm/registers/display/dsi_phy_28nm_8960.xml
new file mode 100644
index 000000000000..4c4de4dda640
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/dsi_phy_28nm_8960.xml
@@ -0,0 +1,134 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+
+<domain name="DSI_28nm_8960_PHY" width="32">
+
+	<array offset="0x00000" name="LN" length="4" stride="0x40">
+		<reg32 offset="0x00" name="CFG_0"/>
+		<reg32 offset="0x04" name="CFG_1"/>
+		<reg32 offset="0x08" name="CFG_2"/>
+		<reg32 offset="0x0c" name="TEST_DATAPATH"/>
+		<reg32 offset="0x14" name="TEST_STR_0"/>
+		<reg32 offset="0x18" name="TEST_STR_1"/>
+	</array>
+
+	<reg32 offset="0x00100" name="LNCK_CFG_0"/>
+	<reg32 offset="0x00104" name="LNCK_CFG_1"/>
+	<reg32 offset="0x00108" name="LNCK_CFG_2"/>
+
+	<reg32 offset="0x0010c" name="LNCK_TEST_DATAPATH"/>
+	<reg32 offset="0x00114" name="LNCK_TEST_STR0"/>
+	<reg32 offset="0x00118" name="LNCK_TEST_STR1"/>
+
+	<reg32 offset="0x00140" name="TIMING_CTRL_0">
+		<bitfield name="CLK_ZERO" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00144" name="TIMING_CTRL_1">
+		<bitfield name="CLK_TRAIL" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00148" name="TIMING_CTRL_2">
+		<bitfield name="CLK_PREPARE" low="0" high="7" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x0014c" name="TIMING_CTRL_3"/>
+
+	<reg32 offset="0x00150" name="TIMING_CTRL_4">
+		<bitfield name="HS_EXIT" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00154" name="TIMING_CTRL_5">
+		<bitfield name="HS_ZERO" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00158" name="TIMING_CTRL_6">
+		<bitfield name="HS_PREPARE" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0015c" name="TIMING_CTRL_7">
+		<bitfield name="HS_TRAIL" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00160" name="TIMING_CTRL_8">
+		<bitfield name="HS_RQST" low="0" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00164" name="TIMING_CTRL_9">
+		<bitfield name="TA_GO" low="0" high="2" type="uint"/>
+		<bitfield name="TA_SURE" low="4" high="6" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00168" name="TIMING_CTRL_10">
+		<bitfield name="TA_GET" low="0" high="2" type="uint"/>
+	</reg32>
+	<reg32 offset="0x0016c" name="TIMING_CTRL_11">
+		<bitfield name="TRIG3_CMD" low="0" high="7" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x00170" name="CTRL_0"/>
+	<reg32 offset="0x00174" name="CTRL_1"/>
+	<reg32 offset="0x00178" name="CTRL_2"/>
+	<reg32 offset="0x0017c" name="CTRL_3"/>
+
+	<reg32 offset="0x00180" name="STRENGTH_0"/>
+	<reg32 offset="0x00184" name="STRENGTH_1"/>
+	<reg32 offset="0x00188" name="STRENGTH_2"/>
+
+	<reg32 offset="0x0018c" name="BIST_CTRL_0"/>
+	<reg32 offset="0x00190" name="BIST_CTRL_1"/>
+	<reg32 offset="0x00194" name="BIST_CTRL_2"/>
+	<reg32 offset="0x00198" name="BIST_CTRL_3"/>
+	<reg32 offset="0x0019c" name="BIST_CTRL_4"/>
+
+	<reg32 offset="0x001b0" name="LDO_CTRL"/>
+</domain>
+
+<domain name="DSI_28nm_8960_PHY_MISC" width="32">
+	<reg32 offset="0x00000" name="REGULATOR_CTRL_0"/>
+	<reg32 offset="0x00004" name="REGULATOR_CTRL_1"/>
+	<reg32 offset="0x00008" name="REGULATOR_CTRL_2"/>
+	<reg32 offset="0x0000c" name="REGULATOR_CTRL_3"/>
+	<reg32 offset="0x00010" name="REGULATOR_CTRL_4"/>
+	<reg32 offset="0x00014" name="REGULATOR_CTRL_5"/>
+	<reg32 offset="0x00018" name="REGULATOR_CAL_PWR_CFG"/>
+	<reg32 offset="0x00028" name="CAL_HW_TRIGGER"/>
+	<reg32 offset="0x0002c" name="CAL_SW_CFG_0"/>
+	<reg32 offset="0x00030" name="CAL_SW_CFG_1"/>
+	<reg32 offset="0x00034" name="CAL_SW_CFG_2"/>
+	<reg32 offset="0x00038" name="CAL_HW_CFG_0"/>
+	<reg32 offset="0x0003c" name="CAL_HW_CFG_1"/>
+	<reg32 offset="0x00040" name="CAL_HW_CFG_2"/>
+	<reg32 offset="0x00044" name="CAL_HW_CFG_3"/>
+	<reg32 offset="0x00048" name="CAL_HW_CFG_4"/>
+	<reg32 offset="0x00050" name="CAL_STATUS">
+		<bitfield name="CAL_BUSY" pos="4" type="boolean"/>
+	</reg32>
+</domain>
+
+<domain name="DSI_28nm_8960_PHY_PLL" width="32">
+	<reg32 offset="0x00000" name="CTRL_0">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00004" name="CTRL_1"/>
+	<reg32 offset="0x00008" name="CTRL_2"/>
+	<reg32 offset="0x0000c" name="CTRL_3"/>
+	<reg32 offset="0x00010" name="CTRL_4"/>
+	<reg32 offset="0x00014" name="CTRL_5"/>
+	<reg32 offset="0x00018" name="CTRL_6"/>
+	<reg32 offset="0x0001c" name="CTRL_7"/>
+	<reg32 offset="0x00020" name="CTRL_8"/>
+	<reg32 offset="0x00024" name="CTRL_9"/>
+	<reg32 offset="0x00028" name="CTRL_10"/>
+	<reg32 offset="0x0002c" name="CTRL_11"/>
+	<reg32 offset="0x00030" name="CTRL_12"/>
+	<reg32 offset="0x00034" name="CTRL_13"/>
+	<reg32 offset="0x00038" name="CTRL_14"/>
+	<reg32 offset="0x0003c" name="CTRL_15"/>
+	<reg32 offset="0x00040" name="CTRL_16"/>
+	<reg32 offset="0x00044" name="CTRL_17"/>
+	<reg32 offset="0x00048" name="CTRL_18"/>
+	<reg32 offset="0x0004c" name="CTRL_19"/>
+	<reg32 offset="0x00050" name="CTRL_20"/>
+
+	<reg32 offset="0x00080" name="RDY">
+		<bitfield name="PLL_RDY" pos="0" type="boolean"/>
+	</reg32>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/display/dsi_phy_7nm.xml b/drivers/gpu/drm/msm/registers/display/dsi_phy_7nm.xml
new file mode 100644
index 000000000000..d54b72f92449
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/dsi_phy_7nm.xml
@@ -0,0 +1,230 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+
+<domain name="DSI_7nm_PHY_CMN" width="32">
+	<reg32 offset="0x00000" name="REVISION_ID0"/>
+	<reg32 offset="0x00004" name="REVISION_ID1"/>
+	<reg32 offset="0x00008" name="REVISION_ID2"/>
+	<reg32 offset="0x0000c" name="REVISION_ID3"/>
+	<reg32 offset="0x00010" name="CLK_CFG0"/>
+	<reg32 offset="0x00014" name="CLK_CFG1"/>
+	<reg32 offset="0x00018" name="GLBL_CTRL"/>
+	<reg32 offset="0x0001c" name="RBUF_CTRL"/>
+	<reg32 offset="0x00020" name="VREG_CTRL_0"/>
+	<reg32 offset="0x00024" name="CTRL_0"/>
+	<reg32 offset="0x00028" name="CTRL_1"/>
+	<reg32 offset="0x0002c" name="CTRL_2"/>
+	<reg32 offset="0x00030" name="CTRL_3"/>
+	<reg32 offset="0x00034" name="LANE_CFG0"/>
+	<reg32 offset="0x00038" name="LANE_CFG1"/>
+	<reg32 offset="0x0003c" name="PLL_CNTRL"/>
+	<reg32 offset="0x00040" name="DPHY_SOT"/>
+	<reg32 offset="0x000a0" name="LANE_CTRL0"/>
+	<reg32 offset="0x000a4" name="LANE_CTRL1"/>
+	<reg32 offset="0x000a8" name="LANE_CTRL2"/>
+	<reg32 offset="0x000ac" name="LANE_CTRL3"/>
+	<reg32 offset="0x000b0" name="LANE_CTRL4"/>
+	<reg32 offset="0x000b4" name="TIMING_CTRL_0"/>
+	<reg32 offset="0x000b8" name="TIMING_CTRL_1"/>
+	<reg32 offset="0x000bc" name="TIMING_CTRL_2"/>
+	<reg32 offset="0x000c0" name="TIMING_CTRL_3"/>
+	<reg32 offset="0x000c4" name="TIMING_CTRL_4"/>
+	<reg32 offset="0x000c8" name="TIMING_CTRL_5"/>
+	<reg32 offset="0x000cc" name="TIMING_CTRL_6"/>
+	<reg32 offset="0x000d0" name="TIMING_CTRL_7"/>
+	<reg32 offset="0x000d4" name="TIMING_CTRL_8"/>
+	<reg32 offset="0x000d8" name="TIMING_CTRL_9"/>
+	<reg32 offset="0x000dc" name="TIMING_CTRL_10"/>
+	<reg32 offset="0x000e0" name="TIMING_CTRL_11"/>
+	<reg32 offset="0x000e4" name="TIMING_CTRL_12"/>
+	<reg32 offset="0x000e8" name="TIMING_CTRL_13"/>
+	<reg32 offset="0x000ec" name="GLBL_HSTX_STR_CTRL_0"/>
+	<reg32 offset="0x000f0" name="GLBL_HSTX_STR_CTRL_1"/>
+	<reg32 offset="0x000f4" name="GLBL_RESCODE_OFFSET_TOP_CTRL"/>
+	<reg32 offset="0x000f8" name="GLBL_RESCODE_OFFSET_BOT_CTRL"/>
+	<reg32 offset="0x000fc" name="GLBL_RESCODE_OFFSET_MID_CTRL"/>
+	<reg32 offset="0x00100" name="GLBL_LPTX_STR_CTRL"/>
+	<reg32 offset="0x00104" name="GLBL_PEMPH_CTRL_0"/>
+	<reg32 offset="0x00108" name="GLBL_PEMPH_CTRL_1"/>
+	<reg32 offset="0x0010c" name="GLBL_STR_SWI_CAL_SEL_CTRL"/>
+	<reg32 offset="0x00110" name="VREG_CTRL_1"/>
+	<reg32 offset="0x00114" name="CTRL_4"/>
+	<reg32 offset="0x00128" name="GLBL_DIGTOP_SPARE4"/>
+	<reg32 offset="0x00140" name="PHY_STATUS"/>
+	<reg32 offset="0x00148" name="LANE_STATUS0"/>
+	<reg32 offset="0x0014c" name="LANE_STATUS1"/>
+	<reg32 offset="0x001ac" name="GLBL_DIGTOP_SPARE10"/>
+</domain>
+
+<domain name="DSI_7nm_PHY" width="32">
+	<array offset="0x00000" name="LN" length="5" stride="0x80">
+		<reg32 offset="0x00" name="CFG0"/>
+		<reg32 offset="0x04" name="CFG1"/>
+		<reg32 offset="0x08" name="CFG2"/>
+		<reg32 offset="0x0c" name="TEST_DATAPATH"/>
+		<reg32 offset="0x10" name="PIN_SWAP"/>
+		<reg32 offset="0x14" name="LPRX_CTRL"/>
+		<reg32 offset="0x18" name="TX_DCTRL"/>
+	</array>
+</domain>
+
+<domain name="DSI_7nm_PHY_PLL" width="32">
+	<reg32 offset="0x0000" name="ANALOG_CONTROLS_ONE"/>
+	<reg32 offset="0x0004" name="ANALOG_CONTROLS_TWO"/>
+	<reg32 offset="0x0008" name="INT_LOOP_SETTINGS"/>
+	<reg32 offset="0x000c" name="INT_LOOP_SETTINGS_TWO"/>
+	<reg32 offset="0x0010" name="ANALOG_CONTROLS_THREE"/>
+	<reg32 offset="0x0014" name="ANALOG_CONTROLS_FOUR"/>
+	<reg32 offset="0x0018" name="ANALOG_CONTROLS_FIVE"/>
+	<reg32 offset="0x001c" name="INT_LOOP_CONTROLS"/>
+	<reg32 offset="0x0020" name="DSM_DIVIDER"/>
+	<reg32 offset="0x0024" name="FEEDBACK_DIVIDER"/>
+	<reg32 offset="0x0028" name="SYSTEM_MUXES"/>
+	<reg32 offset="0x002c" name="FREQ_UPDATE_CONTROL_OVERRIDES"/>
+	<reg32 offset="0x0030" name="CMODE"/>
+	<reg32 offset="0x0034" name="PSM_CTRL"/>
+	<reg32 offset="0x0038" name="RSM_CTRL"/>
+	<reg32 offset="0x003c" name="VCO_TUNE_MAP"/>
+	<reg32 offset="0x0040" name="PLL_CNTRL"/>
+	<reg32 offset="0x0044" name="CALIBRATION_SETTINGS"/>
+	<reg32 offset="0x0048" name="BAND_SEL_CAL_TIMER_LOW"/>
+	<reg32 offset="0x004c" name="BAND_SEL_CAL_TIMER_HIGH"/>
+	<reg32 offset="0x0050" name="BAND_SEL_CAL_SETTINGS"/>
+	<reg32 offset="0x0054" name="BAND_SEL_MIN"/>
+	<reg32 offset="0x0058" name="BAND_SEL_MAX"/>
+	<reg32 offset="0x005c" name="BAND_SEL_PFILT"/>
+	<reg32 offset="0x0060" name="BAND_SEL_IFILT"/>
+	<reg32 offset="0x0064" name="BAND_SEL_CAL_SETTINGS_TWO"/>
+	<reg32 offset="0x0068" name="BAND_SEL_CAL_SETTINGS_THREE"/>
+	<reg32 offset="0x006c" name="BAND_SEL_CAL_SETTINGS_FOUR"/>
+	<reg32 offset="0x0070" name="BAND_SEL_ICODE_HIGH"/>
+	<reg32 offset="0x0074" name="BAND_SEL_ICODE_LOW"/>
+	<reg32 offset="0x0078" name="FREQ_DETECT_SETTINGS_ONE"/>
+	<reg32 offset="0x007c" name="FREQ_DETECT_THRESH"/>
+	<reg32 offset="0x0080" name="FREQ_DET_REFCLK_HIGH"/>
+	<reg32 offset="0x0084" name="FREQ_DET_REFCLK_LOW"/>
+	<reg32 offset="0x0088" name="FREQ_DET_PLLCLK_HIGH"/>
+	<reg32 offset="0x008c" name="FREQ_DET_PLLCLK_LOW"/>
+	<reg32 offset="0x0090" name="PFILT"/>
+	<reg32 offset="0x0094" name="IFILT"/>
+	<reg32 offset="0x0098" name="PLL_GAIN"/>
+	<reg32 offset="0x009c" name="ICODE_LOW"/>
+	<reg32 offset="0x00a0" name="ICODE_HIGH"/>
+	<reg32 offset="0x00a4" name="LOCKDET"/>
+	<reg32 offset="0x00a8" name="OUTDIV"/>
+	<reg32 offset="0x00ac" name="FASTLOCK_CONTROL"/>
+	<reg32 offset="0x00b0" name="PASS_OUT_OVERRIDE_ONE"/>
+	<reg32 offset="0x00b4" name="PASS_OUT_OVERRIDE_TWO"/>
+	<reg32 offset="0x00b8" name="CORE_OVERRIDE"/>
+	<reg32 offset="0x00bc" name="CORE_INPUT_OVERRIDE"/>
+	<reg32 offset="0x00c0" name="RATE_CHANGE"/>
+	<reg32 offset="0x00c4" name="PLL_DIGITAL_TIMERS"/>
+	<reg32 offset="0x00c8" name="PLL_DIGITAL_TIMERS_TWO"/>
+	<reg32 offset="0x00cc" name="DECIMAL_DIV_START"/>
+	<reg32 offset="0x00d0" name="FRAC_DIV_START_LOW"/>
+	<reg32 offset="0x00d4" name="FRAC_DIV_START_MID"/>
+	<reg32 offset="0x00d8" name="FRAC_DIV_START_HIGH"/>
+	<reg32 offset="0x00dc" name="DEC_FRAC_MUXES"/>
+	<reg32 offset="0x00e0" name="DECIMAL_DIV_START_1"/>
+	<reg32 offset="0x00e4" name="FRAC_DIV_START_LOW_1"/>
+	<reg32 offset="0x00e8" name="FRAC_DIV_START_MID_1"/>
+	<reg32 offset="0x00ec" name="FRAC_DIV_START_HIGH_1"/>
+	<reg32 offset="0x00f0" name="DECIMAL_DIV_START_2"/>
+	<reg32 offset="0x00f4" name="FRAC_DIV_START_LOW_2"/>
+	<reg32 offset="0x00f8" name="FRAC_DIV_START_MID_2"/>
+	<reg32 offset="0x00fc" name="FRAC_DIV_START_HIGH_2"/>
+	<reg32 offset="0x0100" name="MASH_CONTROL"/>
+	<reg32 offset="0x0104" name="SSC_STEPSIZE_LOW"/>
+	<reg32 offset="0x0108" name="SSC_STEPSIZE_HIGH"/>
+	<reg32 offset="0x010c" name="SSC_DIV_PER_LOW"/>
+	<reg32 offset="0x0110" name="SSC_DIV_PER_HIGH"/>
+	<reg32 offset="0x0114" name="SSC_ADJPER_LOW"/>
+	<reg32 offset="0x0118" name="SSC_ADJPER_HIGH"/>
+	<reg32 offset="0x011c" name="SSC_MUX_CONTROL"/>
+	<reg32 offset="0x0120" name="SSC_STEPSIZE_LOW_1"/>
+	<reg32 offset="0x0124" name="SSC_STEPSIZE_HIGH_1"/>
+	<reg32 offset="0x0128" name="SSC_DIV_PER_LOW_1"/>
+	<reg32 offset="0x012c" name="SSC_DIV_PER_HIGH_1"/>
+	<reg32 offset="0x0130" name="SSC_ADJPER_LOW_1"/>
+	<reg32 offset="0x0134" name="SSC_ADJPER_HIGH_1"/>
+	<reg32 offset="0x0138" name="SSC_STEPSIZE_LOW_2"/>
+	<reg32 offset="0x013c" name="SSC_STEPSIZE_HIGH_2"/>
+	<reg32 offset="0x0140" name="SSC_DIV_PER_LOW_2"/>
+	<reg32 offset="0x0144" name="SSC_DIV_PER_HIGH_2"/>
+	<reg32 offset="0x0148" name="SSC_ADJPER_LOW_2"/>
+	<reg32 offset="0x014c" name="SSC_ADJPER_HIGH_2"/>
+	<reg32 offset="0x0150" name="SSC_CONTROL"/>
+	<reg32 offset="0x0154" name="PLL_OUTDIV_RATE"/>
+	<reg32 offset="0x0158" name="PLL_LOCKDET_RATE_1"/>
+	<reg32 offset="0x015c" name="PLL_LOCKDET_RATE_2"/>
+	<reg32 offset="0x0160" name="PLL_PROP_GAIN_RATE_1"/>
+	<reg32 offset="0x0164" name="PLL_PROP_GAIN_RATE_2"/>
+	<reg32 offset="0x0168" name="PLL_BAND_SEL_RATE_1"/>
+	<reg32 offset="0x016c" name="PLL_BAND_SEL_RATE_2"/>
+	<reg32 offset="0x0170" name="PLL_INT_GAIN_IFILT_BAND_1"/>
+	<reg32 offset="0x0174" name="PLL_INT_GAIN_IFILT_BAND_2"/>
+	<reg32 offset="0x0178" name="PLL_FL_INT_GAIN_PFILT_BAND_1"/>
+	<reg32 offset="0x017c" name="PLL_FL_INT_GAIN_PFILT_BAND_2"/>
+	<reg32 offset="0x0180" name="PLL_FASTLOCK_EN_BAND"/>
+	<reg32 offset="0x0184" name="FREQ_TUNE_ACCUM_INIT_MID"/>
+	<reg32 offset="0x0188" name="FREQ_TUNE_ACCUM_INIT_HIGH"/>
+	<reg32 offset="0x018c" name="FREQ_TUNE_ACCUM_INIT_MUX"/>
+	<reg32 offset="0x0190" name="PLL_LOCK_OVERRIDE"/>
+	<reg32 offset="0x0194" name="PLL_LOCK_DELAY"/>
+	<reg32 offset="0x0198" name="PLL_LOCK_MIN_DELAY"/>
+	<reg32 offset="0x019c" name="CLOCK_INVERTERS"/>
+	<reg32 offset="0x01a0" name="SPARE_AND_JPC_OVERRIDES"/>
+	<reg32 offset="0x01a4" name="BIAS_CONTROL_1"/>
+	<reg32 offset="0x01a8" name="BIAS_CONTROL_2"/>
+	<reg32 offset="0x01ac" name="ALOG_OBSV_BUS_CTRL_1"/>
+	<reg32 offset="0x01b0" name="COMMON_STATUS_ONE"/>
+	<reg32 offset="0x01b4" name="COMMON_STATUS_TWO"/>
+	<reg32 offset="0x01b8" name="BAND_SEL_CAL"/>
+	<reg32 offset="0x01bc" name="ICODE_ACCUM_STATUS_LOW"/>
+	<reg32 offset="0x01c0" name="ICODE_ACCUM_STATUS_HIGH"/>
+	<reg32 offset="0x01c4" name="FD_OUT_LOW"/>
+	<reg32 offset="0x01c8" name="FD_OUT_HIGH"/>
+	<reg32 offset="0x01cc" name="ALOG_OBSV_BUS_STATUS_1"/>
+	<reg32 offset="0x01d0" name="PLL_MISC_CONFIG"/>
+	<reg32 offset="0x01d4" name="FLL_CONFIG"/>
+	<reg32 offset="0x01d8" name="FLL_FREQ_ACQ_TIME"/>
+	<reg32 offset="0x01dc" name="FLL_CODE0"/>
+	<reg32 offset="0x01e0" name="FLL_CODE1"/>
+	<reg32 offset="0x01e4" name="FLL_GAIN0"/>
+	<reg32 offset="0x01e8" name="FLL_GAIN1"/>
+	<reg32 offset="0x01ec" name="SW_RESET"/>
+	<reg32 offset="0x01f0" name="FAST_PWRUP"/>
+	<reg32 offset="0x01f4" name="LOCKTIME0"/>
+	<reg32 offset="0x01f8" name="LOCKTIME1"/>
+	<reg32 offset="0x01fc" name="DEBUG_BUS_SEL"/>
+	<reg32 offset="0x0200" name="DEBUG_BUS0"/>
+	<reg32 offset="0x0204" name="DEBUG_BUS1"/>
+	<reg32 offset="0x0208" name="DEBUG_BUS2"/>
+	<reg32 offset="0x020c" name="DEBUG_BUS3"/>
+	<reg32 offset="0x0210" name="ANALOG_FLL_CONTROL_OVERRIDES"/>
+	<reg32 offset="0x0214" name="VCO_CONFIG"/>
+	<reg32 offset="0x0218" name="VCO_CAL_CODE1_MODE0_STATUS"/>
+	<reg32 offset="0x021c" name="VCO_CAL_CODE1_MODE1_STATUS"/>
+	<reg32 offset="0x0220" name="RESET_SM_STATUS"/>
+	<reg32 offset="0x0224" name="TDC_OFFSET"/>
+	<reg32 offset="0x0228" name="PS3_PWRDOWN_CONTROLS"/>
+	<reg32 offset="0x022c" name="PS4_PWRDOWN_CONTROLS"/>
+	<reg32 offset="0x0230" name="PLL_RST_CONTROLS"/>
+	<reg32 offset="0x0234" name="GEAR_BAND_SELECT_CONTROLS"/>
+	<reg32 offset="0x0238" name="PSM_CLK_CONTROLS"/>
+	<reg32 offset="0x023c" name="SYSTEM_MUXES_2"/>
+	<reg32 offset="0x0240" name="VCO_CONFIG_1"/>
+	<reg32 offset="0x0244" name="VCO_CONFIG_2"/>
+	<reg32 offset="0x0248" name="CLOCK_INVERTERS_1"/>
+	<reg32 offset="0x024c" name="CLOCK_INVERTERS_2"/>
+	<reg32 offset="0x0250" name="CMODE_1"/>
+	<reg32 offset="0x0254" name="CMODE_2"/>
+	<reg32 offset="0x0258" name="ANALOG_CONTROLS_FIVE_1"/>
+	<reg32 offset="0x025c" name="ANALOG_CONTROLS_FIVE_2"/>
+	<reg32 offset="0x0260" name="PERF_OPTIMIZE"/>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/display/edp.xml b/drivers/gpu/drm/msm/registers/display/edp.xml
new file mode 100644
index 000000000000..354f90eb6de5
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/edp.xml
@@ -0,0 +1,239 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+
+<domain name="EDP" width="32">
+	<enum name="edp_color_depth">
+		<value name="EDP_6BIT"  value="0"/>
+		<value name="EDP_8BIT"  value="1"/>
+		<value name="EDP_10BIT" value="2"/>
+		<value name="EDP_12BIT" value="3"/>
+		<value name="EDP_16BIT" value="4"/>
+	</enum>
+
+	<enum name="edp_component_format">
+		<value name="EDP_RGB" value="0"/>
+		<value name="EDP_YUV422" value="1"/>
+		<value name="EDP_YUV444" value="2"/>
+	</enum>
+
+	<reg32 offset="0x0004" name="MAINLINK_CTRL">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+		<bitfield name="RESET"  pos="1" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x0008" name="STATE_CTRL">
+		<bitfield name="TRAIN_PATTERN_1"       pos="0" type="boolean"/>
+		<bitfield name="TRAIN_PATTERN_2"       pos="1" type="boolean"/>
+		<bitfield name="TRAIN_PATTERN_3"       pos="2" type="boolean"/>
+		<bitfield name="SYMBOL_ERR_RATE_MEAS"  pos="3" type="boolean"/>
+		<bitfield name="PRBS7"                 pos="4" type="boolean"/>
+		<bitfield name="CUSTOM_80_BIT_PATTERN" pos="5" type="boolean"/>
+		<bitfield name="SEND_VIDEO"            pos="6" type="boolean"/>
+		<bitfield name="PUSH_IDLE"             pos="7" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x000c" name="CONFIGURATION_CTRL">
+		<!-- next two may be swapped? -->
+		<bitfield name="SYNC_CLK" pos="0" type="boolean"/>
+		<bitfield name="STATIC_MVID" pos="1" type="boolean"/>
+		<bitfield name="PROGRESSIVE" pos="2" type="boolean"/>
+		<!-- # of lanes minus one: -->
+		<bitfield name="LANES" low="4" high="5" type="uint"/>
+		<bitfield name="ENHANCED_FRAMING" pos="6" type="boolean"/>
+		<!--
+		   NOTE: only 6bit and 8bit valid
+		 -->
+		<bitfield name="COLOR" pos="8" type="edp_color_depth"/>
+	</reg32>
+
+	<reg32 offset="0x0014" name="SOFTWARE_MVID" type="uint"/>
+	<reg32 offset="0x0018" name="SOFTWARE_NVID" type="uint"/>
+
+	<reg32 offset="0x001c" name="TOTAL_HOR_VER">
+		<bitfield name="HORIZ" low="0" high="15" type="uint"/>
+		<bitfield name="VERT"  low="16" high="31" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x0020" name="START_HOR_VER_FROM_SYNC">
+		<bitfield name="HORIZ" low="0" high="15" type="uint"/>
+		<bitfield name="VERT"  low="16" high="31" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x0024" name="HSYNC_VSYNC_WIDTH_POLARITY">
+		<bitfield name="HORIZ"  low="0" high="14" type="uint"/>
+		<bitfield name="NHSYNC" pos="15" type="boolean"/>
+		<bitfield name="VERT"   low="16" high="30" type="uint"/>
+		<bitfield name="NVSYNC" pos="31" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x0028" name="ACTIVE_HOR_VER">
+		<bitfield name="HORIZ" low="0" high="15" type="uint"/>
+		<bitfield name="VERT"  low="16" high="31" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x002c" name="MISC1_MISC0">
+		<!-- MISC0 from DisplayPort v1.2 spec: -->
+		<bitfield name="MISC0" low="0" high="7"/>
+		<!-- aliased MISC0 bitfields: -->
+		<bitfield name="SYNC" pos="0" type="boolean"/>
+		<bitfield name="COMPONENT_FORMAT" low="1" high="2" type="edp_component_format"/>
+		<!-- CEA (vs VESA) color range: -->
+		<bitfield name="CEA" pos="3" type="boolean"/>
+		<!-- YCbCr Colorimetry ITU-R BT709-5 (vs ITU-R BT601-5): -->
+		<bitfield name="BT709_5" pos="4" type="boolean"/>
+		<bitfield name="COLOR" low="5" high="7" type="edp_color_depth"/>
+
+		<!-- MISC1 from DisplayPort v1.2 spec: -->
+		<bitfield name="MISC1" low="8" high="15"/>
+		<!-- aliased MISC1 bitfields: -->
+		<bitfield name="INTERLACED_ODD" pos="8" type="boolean"/>
+		<bitfield name="STEREO" low="9" high="10" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x0074" name="PHY_CTRL">
+		<bitfield name="SW_RESET_PLL" pos="0" type="boolean"/>
+		<bitfield name="SW_RESET" pos="2" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0084" name="MAINLINK_READY">
+		<bitfield name="TRAIN_PATTERN_1_READY" pos="3" type="boolean"/>
+		<bitfield name="TRAIN_PATTERN_2_READY" pos="4" type="boolean"/>
+		<bitfield name="TRAIN_PATTERN_3_READY" pos="5" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x0300" name="AUX_CTRL">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+		<bitfield name="RESET"  pos="1" type="boolean"/>
+	</reg32>
+
+	<!-- interrupt registers come in sets of 3 bits, status/ack/en -->
+	<reg32 offset="0x0308" name="INTERRUPT_REG_1">
+		<bitfield name="HPD"                    pos="0"  type="boolean"/>
+		<bitfield name="HPD_ACK"                pos="1"  type="boolean"/>
+		<bitfield name="HPD_EN"                 pos="2"  type="boolean"/>
+		<bitfield name="AUX_I2C_DONE"           pos="3"  type="boolean"/>
+		<bitfield name="AUX_I2C_DONE_ACK"       pos="4"  type="boolean"/>
+		<bitfield name="AUX_I2C_DONE_EN"        pos="5"  type="boolean"/>
+		<bitfield name="WRONG_ADDR"             pos="6"  type="boolean"/>
+		<bitfield name="WRONG_ADDR_ACK"         pos="7"  type="boolean"/>
+		<bitfield name="WRONG_ADDR_EN"          pos="8"  type="boolean"/>
+		<bitfield name="TIMEOUT"                pos="9"  type="boolean"/>
+		<bitfield name="TIMEOUT_ACK"            pos="10" type="boolean"/>
+		<bitfield name="TIMEOUT_EN"             pos="11" type="boolean"/>
+		<bitfield name="NACK_DEFER"             pos="12" type="boolean"/>
+		<bitfield name="NACK_DEFER_ACK"         pos="13" type="boolean"/>
+		<bitfield name="NACK_DEFER_EN"          pos="14" type="boolean"/>
+		<bitfield name="WRONG_DATA_CNT"         pos="15" type="boolean"/>
+		<bitfield name="WRONG_DATA_CNT_ACK"     pos="16" type="boolean"/>
+		<bitfield name="WRONG_DATA_CNT_EN"      pos="17" type="boolean"/>
+		<bitfield name="I2C_NACK"               pos="18" type="boolean"/>
+		<bitfield name="I2C_NACK_ACK"           pos="19" type="boolean"/>
+		<bitfield name="I2C_NACK_EN"            pos="20" type="boolean"/>
+		<bitfield name="I2C_DEFER"              pos="21" type="boolean"/>
+		<bitfield name="I2C_DEFER_ACK"          pos="22" type="boolean"/>
+		<bitfield name="I2C_DEFER_EN"           pos="23" type="boolean"/>
+		<bitfield name="PLL_UNLOCK"             pos="24" type="boolean"/>
+		<bitfield name="PLL_UNLOCK_ACK"         pos="25" type="boolean"/>
+		<bitfield name="PLL_UNLOCK_EN"          pos="26" type="boolean"/>
+		<bitfield name="AUX_ERROR"              pos="27" type="boolean"/>
+		<bitfield name="AUX_ERROR_ACK"          pos="28" type="boolean"/>
+		<bitfield name="AUX_ERROR_EN"           pos="29" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x030c" name="INTERRUPT_REG_2">
+		<bitfield name="READY_FOR_VIDEO"        pos="0"  type="boolean"/>
+		<bitfield name="READY_FOR_VIDEO_ACK"    pos="1"  type="boolean"/>
+		<bitfield name="READY_FOR_VIDEO_EN"     pos="2"  type="boolean"/>
+		<bitfield name="IDLE_PATTERNs_SENT"     pos="3"  type="boolean"/>
+		<bitfield name="IDLE_PATTERNs_SENT_ACK" pos="4"  type="boolean"/>
+		<bitfield name="IDLE_PATTERNs_SENT_EN"  pos="5"  type="boolean"/>
+		<bitfield name="FRAME_END"              pos="9"  type="boolean"/>
+		<bitfield name="FRAME_END_ACK"          pos="7"  type="boolean"/>
+		<bitfield name="FRAME_END_EN"           pos="8"  type="boolean"/>
+		<bitfield name="CRC_UPDATED"            pos="9"  type="boolean"/>
+		<bitfield name="CRC_UPDATED_ACK"        pos="10" type="boolean"/>
+		<bitfield name="CRC_UPDATED_EN"         pos="11" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x0310" name="INTERRUPT_TRANS_NUM"/>
+	<reg32 offset="0x0314" name="AUX_DATA">
+		<bitfield name="READ" pos="0" type="boolean"/>
+		<bitfield name="DATA" low="8" high="15"/>
+		<bitfield name="INDEX" low="16" high="23"/>
+		<bitfield name="INDEX_WRITE" pos="31" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x0318" name="AUX_TRANS_CTRL">
+		<bitfield name="I2C" pos="8" type="boolean"/>
+		<bitfield name="GO"  pos="9" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x0324" name="AUX_STATUS"/>
+</domain>
+
+<domain name="EDP_PHY" width="32">
+	<array offset="0x0400" name="LN" length="4" stride="0x40">
+		<reg32 offset="0x04" name="PD_CTL"/>
+	</array>
+	<reg32 offset="0x0510" name="GLB_VM_CFG0"/>
+	<reg32 offset="0x0514" name="GLB_VM_CFG1"/>
+	<reg32 offset="0x0518" name="GLB_MISC9"/>
+	<reg32 offset="0x0528" name="GLB_CFG"/>
+	<reg32 offset="0x052c" name="GLB_PD_CTL"/>
+	<reg32 offset="0x0598" name="GLB_PHY_STATUS"/>
+</domain>
+
+<domain name="EDP_28nm_PHY_PLL" width="32">
+	<reg32 offset="0x00000" name="REFCLK_CFG"/>
+	<reg32 offset="0x00004" name="POSTDIV1_CFG"/>
+	<reg32 offset="0x00008" name="CHGPUMP_CFG"/>
+	<reg32 offset="0x0000C" name="VCOLPF_CFG"/>
+	<reg32 offset="0x00010" name="VREG_CFG"/>
+	<reg32 offset="0x00014" name="PWRGEN_CFG"/>
+	<reg32 offset="0x00018" name="DMUX_CFG"/>
+	<reg32 offset="0x0001C" name="AMUX_CFG"/>
+	<reg32 offset="0x00020" name="GLB_CFG">
+		<bitfield name="PLL_PWRDN_B" pos="0" type="boolean"/>
+		<bitfield name="PLL_LDO_PWRDN_B" pos="1" type="boolean"/>
+		<bitfield name="PLL_PWRGEN_PWRDN_B" pos="2" type="boolean"/>
+		<bitfield name="PLL_ENABLE" pos="3" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00024" name="POSTDIV2_CFG"/>
+	<reg32 offset="0x00028" name="POSTDIV3_CFG"/>
+	<reg32 offset="0x0002C" name="LPFR_CFG"/>
+	<reg32 offset="0x00030" name="LPFC1_CFG"/>
+	<reg32 offset="0x00034" name="LPFC2_CFG"/>
+	<reg32 offset="0x00038" name="SDM_CFG0"/>
+	<reg32 offset="0x0003C" name="SDM_CFG1"/>
+	<reg32 offset="0x00040" name="SDM_CFG2"/>
+	<reg32 offset="0x00044" name="SDM_CFG3"/>
+	<reg32 offset="0x00048" name="SDM_CFG4"/>
+	<reg32 offset="0x0004C" name="SSC_CFG0"/>
+	<reg32 offset="0x00050" name="SSC_CFG1"/>
+	<reg32 offset="0x00054" name="SSC_CFG2"/>
+	<reg32 offset="0x00058" name="SSC_CFG3"/>
+	<reg32 offset="0x0005C" name="LKDET_CFG0"/>
+	<reg32 offset="0x00060" name="LKDET_CFG1"/>
+	<reg32 offset="0x00064" name="LKDET_CFG2"/>
+	<reg32 offset="0x00068" name="TEST_CFG">
+		<bitfield name="PLL_SW_RESET" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0006C" name="CAL_CFG0"/>
+	<reg32 offset="0x00070" name="CAL_CFG1"/>
+	<reg32 offset="0x00074" name="CAL_CFG2"/>
+	<reg32 offset="0x00078" name="CAL_CFG3"/>
+	<reg32 offset="0x0007C" name="CAL_CFG4"/>
+	<reg32 offset="0x00080" name="CAL_CFG5"/>
+	<reg32 offset="0x00084" name="CAL_CFG6"/>
+	<reg32 offset="0x00088" name="CAL_CFG7"/>
+	<reg32 offset="0x0008C" name="CAL_CFG8"/>
+	<reg32 offset="0x00090" name="CAL_CFG9"/>
+	<reg32 offset="0x00094" name="CAL_CFG10"/>
+	<reg32 offset="0x00098" name="CAL_CFG11"/>
+	<reg32 offset="0x0009C" name="EFUSE_CFG"/>
+	<reg32 offset="0x000A0" name="DEBUG_BUS_SEL"/>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/display/hdmi.xml b/drivers/gpu/drm/msm/registers/display/hdmi.xml
new file mode 100644
index 000000000000..6c81581016c7
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/hdmi.xml
@@ -0,0 +1,1015 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+
+<!--
+	NOTE: also see mdss_hdmi_util.h.. newer devices using MDSS appear
+	to have the same HDMI block (or maybe a newer version?) but for
+	some reason duplicate the code under drivers/video/msm/mdss
+ -->
+
+<domain name="HDMI" width="32">
+	<enum name="hdmi_hdcp_key_state">
+		<value name="HDCP_KEYS_STATE_NO_KEYS" value="0"/>
+		<value name="HDCP_KEYS_STATE_NOT_CHECKED" value="1"/>
+		<value name="HDCP_KEYS_STATE_CHECKING" value="2"/>
+		<value name="HDCP_KEYS_STATE_VALID" value="3"/>
+		<value name="HDCP_KEYS_STATE_AKSV_NOT_VALID" value="4"/>
+		<value name="HDCP_KEYS_STATE_CHKSUM_MISMATCH" value="5"/>
+		<value name="HDCP_KEYS_STATE_PROD_AKSV" value="6"/>
+		<value name="HDCP_KEYS_STATE_RESERVED" value="7"/>
+	</enum>
+	<enum name="hdmi_ddc_read_write">
+		<value name="DDC_WRITE" value="0"/>
+		<value name="DDC_READ" value="1"/>
+	</enum>
+	<enum name="hdmi_acr_cts">
+		<value name="ACR_NONE" value="0"/>
+		<value name="ACR_32" value="1"/>
+		<value name="ACR_44" value="2"/>
+		<value name="ACR_48" value="3"/>
+	</enum>
+
+	<enum name="hdmi_cec_tx_status">
+		<value name="CEC_TX_OK" value="0"/>
+		<value name="CEC_TX_NACK" value="1"/>
+		<value name="CEC_TX_ARB_LOSS" value="2"/>
+		<value name="CEC_TX_MAX_RETRIES" value="3"/>
+	</enum>
+
+	<reg32 offset="0x00000" name="CTRL">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+		<bitfield name="HDMI" pos="1" type="boolean"/>
+		<bitfield name="ENCRYPTED" pos="2" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00020" name="AUDIO_PKT_CTRL1">
+		<bitfield name="AUDIO_SAMPLE_SEND" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00024" name="ACR_PKT_CTRL">
+		<!--
+			Guessing on order of bitfields from these comments:
+				/* AUDIO_PRIORITY | SOURCE */
+				acr_pck_ctrl_reg |= 0x80000100;
+				/* N_MULTIPLE(multiplier) */
+				acr_pck_ctrl_reg |= (multiplier & 7) << 16;
+				/* SEND | CONT */
+				acr_pck_ctrl_reg |= 0x00000003;
+		 -->
+		<bitfield name="CONT" pos="0" type="boolean"/>
+		<bitfield name="SEND" pos="1" type="boolean"/>
+		<bitfield name="SELECT" low="4" high="5" type="hdmi_acr_cts"/>
+		<bitfield name="SOURCE" pos="8" type="boolean"/>
+		<bitfield name="N_MULTIPLIER" low="16" high="18" type="uint"/>
+		<bitfield name="AUDIO_PRIORITY" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0028" name="VBI_PKT_CTRL">
+		<!--
+			Guessing on the order of bits from:
+				/* GC packet enable (every frame) */
+				/* HDMI_VBI_PKT_CTRL[0x0028] */
+				hdmi_msm_rmw32or(0x0028, 3 << 4);
+				/* HDMI_VBI_PKT_CTRL[0x0028] */
+				/* ISRC Send + Continuous */
+				hdmi_msm_rmw32or(0x0028, 3 << 8);
+				/* HDMI_VBI_PKT_CTRL[0x0028] */
+				/* ACP send, s/w source */
+				hdmi_msm_rmw32or(0x0028, 3 << 12);
+		 -->
+		<bitfield name="GC_ENABLE" pos="4" type="boolean"/>
+		<bitfield name="GC_EVERY_FRAME" pos="5" type="boolean"/>
+		<bitfield name="ISRC_SEND" pos="8" type="boolean"/>
+		<bitfield name="ISRC_CONTINUOUS" pos="9" type="boolean"/>
+		<bitfield name="ACP_SEND" pos="12" type="boolean"/>
+		<bitfield name="ACP_SRC_SW" pos="13" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0002c" name="INFOFRAME_CTRL0">
+		<!--
+			Guessing on the order of these flags, from this comment:
+				/* Set these flags */
+				/* AUDIO_INFO_UPDATE | AUDIO_INFO_SOURCE | AUDIO_INFO_CONT
+				 | AUDIO_INFO_SEND */
+				audio_info_ctrl_reg |= 0x000000F0;
+				/* 0x3 for AVI InfFrame enable (every frame) */
+				HDMI_OUTP(0x002C, HDMI_INP(0x002C) | 0x00000003L);
+		 -->
+		<bitfield name="AVI_SEND" pos="0" type="boolean"/>
+		<bitfield name="AVI_CONT" pos="1" type="boolean"/>           <!-- every frame -->
+		<bitfield name="AUDIO_INFO_SEND" pos="4" type="boolean"/>
+		<bitfield name="AUDIO_INFO_CONT" pos="5" type="boolean"/>    <!-- every frame -->
+		<bitfield name="AUDIO_INFO_SOURCE" pos="6" type="boolean"/>
+		<bitfield name="AUDIO_INFO_UPDATE" pos="7" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00030" name="INFOFRAME_CTRL1">
+		<bitfield name="AVI_INFO_LINE" low="0" high="5" type="uint"/>
+		<bitfield name="AUDIO_INFO_LINE" low="8" high="13" type="uint"/>
+		<bitfield name="MPEG_INFO_LINE" low="16" high="21" type="uint"/>
+		<bitfield name="VENSPEC_INFO_LINE" low="24" high="29" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00034" name="GEN_PKT_CTRL">
+		<!--
+			0x0034 GEN_PKT_CTRL
+			  GENERIC0_SEND   0      0 = Disable Generic0 Packet Transmission
+			                         1 = Enable Generic0 Packet Transmission
+			  GENERIC0_CONT   1      0 = Send Generic0 Packet on next frame only
+			                         1 = Send Generic0 Packet on every frame
+			  GENERIC0_UPDATE 2      NUM
+			  GENERIC1_SEND   4      0 = Disable Generic1 Packet Transmission
+			                         1 = Enable Generic1 Packet Transmission
+			  GENERIC1_CONT   5      0 = Send Generic1 Packet on next frame only
+			                         1 = Send Generic1 Packet on every frame
+			  GENERIC0_LINE   21:16  NUM
+			  GENERIC1_LINE   29:24  NUM
+			
+			GENERIC0_LINE | GENERIC0_UPDATE | GENERIC0_CONT | GENERIC0_SEND
+			Setup HDMI TX generic packet control
+			Enable this packet to transmit every frame
+			Enable this packet to transmit every frame
+			Enable HDMI TX engine to transmit Generic packet 0
+			  HDMI_OUTP(0x0034, (1 << 16) | (1 << 2) | BIT(1) | BIT(0));
+		 -->
+		<bitfield name="GENERIC0_SEND" pos="0" type="boolean"/>
+		<bitfield name="GENERIC0_CONT" pos="1" type="boolean"/>
+		<bitfield name="GENERIC0_UPDATE" low="2" high="3" type="uint"/> <!-- ??? -->
+		<bitfield name="GENERIC1_SEND" pos="4" type="boolean"/>
+		<bitfield name="GENERIC1_CONT" pos="5" type="boolean"/>
+		<bitfield name="GENERIC0_LINE" low="16" high="21" type="uint"/>
+		<bitfield name="GENERIC1_LINE" low="24" high="29" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00040" name="GC">
+		<bitfield name="MUTE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00044" name="AUDIO_PKT_CTRL2">
+		<bitfield name="OVERRIDE" pos="0" type="boolean"/>
+		<bitfield name="LAYOUT" pos="1" type="boolean"/> <!-- 1 for >2 channels -->
+	</reg32>
+
+	<!--
+		AVI_INFO appears to be the infoframe in a slightly weird order..
+		starts with PB0 (checksum), and ends with version..
+	-->
+	<reg32 offset="0x0006c" name="AVI_INFO" stride="4" length="4"/>
+
+	<reg32 offset="0x00084" name="GENERIC0_HDR"/>
+	<reg32 offset="0x00088" name="GENERIC0" stride="4" length="7"/>
+
+	<reg32 offset="0x000a4" name="GENERIC1_HDR"/>
+	<reg32 offset="0x000a8" name="GENERIC1" stride="4" length="7"/>
+
+	<!--
+		TODO add a way to show symbolic offsets into array: hdmi_acr_cts-1
+	 -->
+	<array offset="0x00c4" name="ACR" length="3" stride="8" index="hdmi_acr_cts">
+		<reg32 offset="0" name="0">
+			<bitfield name="CTS" low="12" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="4" name="1">
+			<!-- not sure the actual # of bits.. -->
+			<bitfield name="N" low="0" high="31" type="uint"/>
+		</reg32>
+	</array>
+
+	<reg32 offset="0x000e4" name="AUDIO_INFO0">
+		<bitfield name="CHECKSUM" low="0" high="7"/>
+		<bitfield name="CC" low="8" high="10" type="uint"/> <!-- channel count -->
+	</reg32>
+	<reg32 offset="0x000e8" name="AUDIO_INFO1">
+		<bitfield name="CA" low="0" high="7"/>        <!-- Channel Allocation -->
+		<bitfield name="LSV" low="11" high="14"/>     <!-- Level Shift -->
+		<bitfield name="DM_INH" pos="15" type="boolean"/>  <!-- down-mix inhibit flag -->
+	</reg32>
+	<reg32 offset="0x00110" name="HDCP_CTRL">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+		<bitfield name="ENCRYPTION_ENABLE" pos="8" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00114" name="HDCP_DEBUG_CTRL">
+		<bitfield name="RNG_CIPHER" pos="2" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00118" name="HDCP_INT_CTRL">
+		<bitfield name="AUTH_SUCCESS_INT" pos="0" type="boolean"/>
+		<bitfield name="AUTH_SUCCESS_ACK" pos="1" type="boolean"/>
+		<bitfield name="AUTH_SUCCESS_MASK" pos="2" type="boolean"/>
+		<bitfield name="AUTH_FAIL_INT" pos="4" type="boolean"/>
+		<bitfield name="AUTH_FAIL_ACK" pos="5" type="boolean"/>
+		<bitfield name="AUTH_FAIL_MASK" pos="6" type="boolean"/>
+		<bitfield name="AUTH_FAIL_INFO_ACK" pos="7" type="boolean"/>
+		<bitfield name="AUTH_XFER_REQ_INT" pos="8" type="boolean"/>
+		<bitfield name="AUTH_XFER_REQ_ACK" pos="9" type="boolean"/>
+		<bitfield name="AUTH_XFER_REQ_MASK" pos="10" type="boolean"/>
+		<bitfield name="AUTH_XFER_DONE_INT" pos="12" type="boolean"/>
+		<bitfield name="AUTH_XFER_DONE_ACK" pos="13" type="boolean"/>
+		<bitfield name="AUTH_XFER_DONE_MASK" pos="14" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0011c" name="HDCP_LINK0_STATUS">
+		<bitfield name="AN_0_READY" pos="8" type="boolean"/>
+		<bitfield name="AN_1_READY" pos="9" type="boolean"/>
+		<bitfield name="RI_MATCHES" pos="12" type="boolean"/>
+		<bitfield name="V_MATCHES" pos="20" type="boolean"/>
+		<bitfield name="KEY_STATE" low="28" high="30" type="hdmi_hdcp_key_state"/>
+	</reg32>
+	<reg32 offset="0x00120" name="HDCP_DDC_CTRL_0">
+		<bitfield name="DISABLE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00124" name="HDCP_DDC_CTRL_1">
+		<bitfield name="FAILED_ACK" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00128" name="HDCP_DDC_STATUS">
+		<bitfield name="XFER_REQ" pos="4" type="boolean"/>
+		<bitfield name="XFER_DONE" pos="10" type="boolean"/>
+		<bitfield name="ABORTED" pos="12" type="boolean"/>
+		<bitfield name="TIMEOUT" pos="13" type="boolean"/>
+		<bitfield name="NACK0" pos="14" type="boolean"/>
+		<bitfield name="NACK1" pos="15" type="boolean"/>
+		<bitfield name="FAILED" pos="16" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x0012c" name="HDCP_ENTROPY_CTRL0"/>
+	<reg32 offset="0x0025c" name="HDCP_ENTROPY_CTRL1"/>
+
+	<reg32 offset="0x00130" name="HDCP_RESET">
+		<bitfield name="LINK0_DEAUTHENTICATE" pos="0" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x00134" name="HDCP_RCVPORT_DATA0"/>
+	<reg32 offset="0x00138" name="HDCP_RCVPORT_DATA1"/>
+	<reg32 offset="0x0013C" name="HDCP_RCVPORT_DATA2_0"/>
+	<reg32 offset="0x00140" name="HDCP_RCVPORT_DATA2_1"/>
+	<reg32 offset="0x00144" name="HDCP_RCVPORT_DATA3"/>
+	<reg32 offset="0x00148" name="HDCP_RCVPORT_DATA4"/>
+	<reg32 offset="0x0014c" name="HDCP_RCVPORT_DATA5"/>
+	<reg32 offset="0x00150" name="HDCP_RCVPORT_DATA6"/>
+	<reg32 offset="0x00154" name="HDCP_RCVPORT_DATA7"/>
+	<reg32 offset="0x00158" name="HDCP_RCVPORT_DATA8"/>
+	<reg32 offset="0x0015c" name="HDCP_RCVPORT_DATA9"/>
+	<reg32 offset="0x00160" name="HDCP_RCVPORT_DATA10"/>
+	<reg32 offset="0x00164" name="HDCP_RCVPORT_DATA11"/>
+	<reg32 offset="0x00168" name="HDCP_RCVPORT_DATA12"/>
+
+	<reg32 offset="0x0016c" name="VENSPEC_INFO0"/>
+	<reg32 offset="0x00170" name="VENSPEC_INFO1"/>
+	<reg32 offset="0x00174" name="VENSPEC_INFO2"/>
+	<reg32 offset="0x00178" name="VENSPEC_INFO3"/>
+	<reg32 offset="0x0017c" name="VENSPEC_INFO4"/>
+	<reg32 offset="0x00180" name="VENSPEC_INFO5"/>
+	<reg32 offset="0x00184" name="VENSPEC_INFO6"/>
+
+	<reg32 offset="0x001d0" name="AUDIO_CFG">
+		<bitfield name="ENGINE_ENABLE" pos="0" type="boolean"/>
+		<bitfield name="FIFO_WATERMARK" low="4" high="7" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x00208" name="USEC_REFTIMER"/>
+	<reg32 offset="0x0020c" name="DDC_CTRL">
+		<!--
+			 0x020C HDMI_DDC_CTRL
+			[21:20] TRANSACTION_CNT
+				Number of transactions to be done in current transfer.
+				* 0x0: transaction0 only
+				* 0x1: transaction0, transaction1
+				* 0x2: transaction0, transaction1, transaction2
+				* 0x3: transaction0, transaction1, transaction2, transaction3
+			[3] SW_STATUS_RESET
+				Write 1 to reset HDMI_DDC_SW_STATUS flags, will reset SW_DONE,
+				ABORTED, TIMEOUT, SW_INTERRUPTED, BUFFER_OVERFLOW,
+				STOPPED_ON_NACK, NACK0, NACK1, NACK2, NACK3
+			[2] SEND_RESET Set to 1 to send reset sequence (9 clocks with no
+				data) at start of transfer.  This sequence is sent after GO is
+				written to 1, before the first transaction only.
+			[1] SOFT_RESET Write 1 to reset DDC controller
+			[0] GO WRITE ONLY. Write 1 to start DDC transfer.
+		 -->
+		<bitfield name="GO" pos="0" type="boolean"/>
+		<bitfield name="SOFT_RESET" pos="1" type="boolean"/>
+		<bitfield name="SEND_RESET" pos="2" type="boolean"/>
+		<bitfield name="SW_STATUS_RESET" pos="3" type="boolean"/>
+		<bitfield name="TRANSACTION_CNT" low="20" high="21" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00210" name="DDC_ARBITRATION">
+		<bitfield name="HW_ARBITRATION" pos="4" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00214" name="DDC_INT_CTRL">
+		<!--
+			HDMI_DDC_INT_CTRL[0x0214]
+			   [2] SW_DONE_MK Mask bit for SW_DONE_INT. Set to 1 to enable
+			       interrupt.
+			   [1] SW_DONE_ACK WRITE ONLY. Acknowledge bit for SW_DONE_INT.
+			       Write 1 to clear interrupt.
+			   [0] SW_DONE_INT READ ONLY. SW_DONE interrupt status */
+		 -->
+		<bitfield name="SW_DONE_INT" pos="0" type="boolean"/>
+		<bitfield name="SW_DONE_ACK" pos="1" type="boolean"/>
+		<bitfield name="SW_DONE_MASK" pos="2" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00218" name="DDC_SW_STATUS">
+		<bitfield name="NACK0" pos="12" type="boolean"/>
+		<bitfield name="NACK1" pos="13" type="boolean"/>
+		<bitfield name="NACK2" pos="14" type="boolean"/>
+		<bitfield name="NACK3" pos="15" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0021c" name="DDC_HW_STATUS">
+		<bitfield name="DONE" pos="3" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00220" name="DDC_SPEED">
+		<!--
+		   0x0220 HDMI_DDC_SPEED
+		   [31:16] PRESCALE prescale = (m * xtal_frequency) /
+			(desired_i2c_speed), where m is multiply
+			factor, default: m = 1
+		   [1:0]   THRESHOLD Select threshold to use to determine whether value
+			sampled on SDA is a 1 or 0. Specified in terms of the ratio
+			between the number of sampled ones and the total number of times
+			SDA is sampled.
+			* 0x0: >0
+			* 0x1: 1/4 of total samples
+			* 0x2: 1/2 of total samples
+			* 0x3: 3/4 of total samples */
+		 -->
+		<bitfield name="THRESHOLD" low="0" high="1" type="uint"/>
+		<bitfield name="PRESCALE" low="16" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00224" name="DDC_SETUP">
+		<!--
+			 * 0x0224 HDMI_DDC_SETUP
+			 * Setting 31:24 bits : Time units to wait before timeout
+			 * when clock is being stalled by external sink device
+		 -->
+		<bitfield name="TIMEOUT" low="24" high="31" type="uint"/>
+	</reg32>
+	<!-- Guessing length is 4, as elsewhere the are references to trans0 thru trans3 -->
+	<array offset="0x00228" name="I2C_TRANSACTION" length="4" stride="4">
+		<reg32 offset="0" name="REG">
+			<!--
+				0x0228 HDMI_DDC_TRANS0
+				[23:16] CNT0 Byte count for first transaction (excluding the first
+					byte, which is usually the address).
+				[13] STOP0 Determines whether a stop bit will be sent after the first
+					transaction
+					* 0: NO STOP
+					* 1: STOP
+				[12] START0 Determines whether a start bit will be sent before the
+					first transaction
+					* 0: NO START
+					* 1: START
+				[8] STOP_ON_NACK0 Determines whether the current transfer will stop
+					if a NACK is received during the first transaction (current
+					transaction always stops).
+					* 0: STOP CURRENT TRANSACTION, GO TO NEXT TRANSACTION
+					* 1: STOP ALL TRANSACTIONS, SEND STOP BIT
+				[0] RW0 Read/write indicator for first transaction - set to 0 for
+					write, 1 for read. This bit only controls HDMI_DDC behaviour -
+					the R/W bit in the transaction is programmed into the DDC buffer
+					as the LSB of the address byte.
+					* 0: WRITE
+					* 1: READ
+			 -->
+			<bitfield name="RW" pos="0" type="hdmi_ddc_read_write"/>
+			<bitfield name="STOP_ON_NACK" pos="8" type="boolean"/>
+			<bitfield name="START" pos="12" type="boolean"/>
+			<bitfield name="STOP" pos="13" type="boolean"/>
+			<bitfield name="CNT" low="16" high="23" type="uint"/>
+		</reg32>
+	</array>
+	<reg32 offset="0x00238" name="DDC_DATA">
+		<!--
+			0x0238 HDMI_DDC_DATA
+			[31] INDEX_WRITE WRITE ONLY. To write index field, set this bit to
+				1 while writing HDMI_DDC_DATA.
+			[23:16] INDEX Use to set index into DDC buffer for next read or
+				current write, or to read index of current read or next write.
+				Writable only when INDEX_WRITE=1.
+			[15:8] DATA Use to fill or read the DDC buffer
+			[0] DATA_RW Select whether buffer access will be a read or write.
+				For writes, address auto-increments on write to HDMI_DDC_DATA.
+				For reads, address autoincrements on reads to HDMI_DDC_DATA.
+				* 0: Write
+				* 1: Read
+		 -->
+		<bitfield name="DATA_RW" pos="0" type="hdmi_ddc_read_write"/>
+		<bitfield name="DATA" low="8" high="15" type="uint"/>
+		<bitfield name="INDEX" low="16" high="23" type="uint"/>
+		<bitfield name="INDEX_WRITE" pos="31" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x0023c" name="HDCP_SHA_CTRL"/>
+	<reg32 offset="0x00240" name="HDCP_SHA_STATUS">
+		<bitfield name="BLOCK_DONE" pos="0" type="boolean"/>
+		<bitfield name="COMP_DONE" pos="4" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00244" name="HDCP_SHA_DATA">
+		<bitfield name="DONE" pos="0" type="boolean"/>
+	</reg32>
+
+	<reg32 offset="0x00250" name="HPD_INT_STATUS">
+		<bitfield name="INT" pos="0" type="boolean"/>  <!-- an irq has occurred -->
+		<bitfield name="CABLE_DETECTED" pos="1" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00254" name="HPD_INT_CTRL">
+		<!-- (this useful comment was removed in df6b645.. git archaeology is fun)
+			HPD_INT_CTRL[0x0254]
+			31:10 Reserved
+			9     RCV_PLUGIN_DET_MASK  receiver plug in interrupt mask.
+			                           When programmed to 1,
+			                           RCV_PLUGIN_DET_INT will toggle
+			                           the interrupt line
+			8:6   Reserved
+			5     RX_INT_EN            Panel RX interrupt enable
+			      0: Disable
+			      1: Enable
+			4     RX_INT_ACK           WRITE ONLY. Panel RX interrupt
+			                           ack
+			3     Reserved
+			2     INT_EN               Panel interrupt control
+			      0: Disable
+			      1: Enable
+			1     INT_POLARITY         Panel interrupt polarity
+			      0: generate interrupt on disconnect
+			      1: generate interrupt on connect
+			0     INT_ACK              WRITE ONLY. Panel interrupt ack
+		 -->
+		<bitfield name="INT_ACK" pos="0" type="boolean"/>
+		<bitfield name="INT_CONNECT" pos="1" type="boolean"/>
+		<bitfield name="INT_EN" pos="2" type="boolean"/>
+		<bitfield name="RX_INT_ACK" pos="4" type="boolean"/>
+		<bitfield name="RX_INT_EN" pos="5" type="boolean"/>
+		<bitfield name="RCV_PLUGIN_DET_MASK" pos="9" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00258" name="HPD_CTRL">
+		<bitfield name="TIMEOUT" low="0" high="12" type="uint"/>
+		<bitfield name="ENABLE" pos="28" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0027c" name="DDC_REF">
+		<!--
+			0x027C HDMI_DDC_REF
+			[16] REFTIMER_ENABLE	Enable the timer
+				* 0: Disable
+				* 1: Enable
+			[15:0] REFTIMER	Value to set the register in order to generate
+				DDC strobe. This register counts on HDCP application clock
+
+			/* Enable reference timer
+			 * 27 micro-seconds */
+			HDMI_OUTP_ND(0x027C, (1 << 16) | (27 << 0));
+		 -->
+		<bitfield name="REFTIMER_ENABLE" pos="16" type="boolean"/>
+		<bitfield name="REFTIMER" low="0" high="15" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x00284" name="HDCP_SW_UPPER_AKSV"/>
+	<reg32 offset="0x00288" name="HDCP_SW_LOWER_AKSV"/>
+
+	<reg32 offset="0x0028c" name="CEC_CTRL">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+		<bitfield name="SEND_TRIGGER" pos="1" type="boolean"/>
+		<bitfield name="FRAME_SIZE" low="4" high="8" type="uint"/>
+		<bitfield name="LINE_OE" pos="9" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00290" name="CEC_WR_DATA">
+		<bitfield name="BROADCAST" pos="0" type="boolean"/>
+		<bitfield name="DATA" low="8" high="15" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00294" name="CEC_RETRANSMIT">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+		<bitfield name="COUNT" low="1" high="7" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00298" name="CEC_STATUS">
+		<bitfield name="BUSY" pos="0" type="boolean"/>
+		<bitfield name="TX_FRAME_DONE" pos="3" type="boolean"/>
+		<bitfield name="TX_STATUS" low="4" high="7" type="hdmi_cec_tx_status"/>
+	</reg32>
+	<reg32 offset="0x0029c" name="CEC_INT">
+		<bitfield name="TX_DONE" pos="0" type="boolean"/>
+		<bitfield name="TX_DONE_MASK" pos="1" type="boolean"/>
+		<bitfield name="TX_ERROR" pos="2" type="boolean"/>
+		<bitfield name="TX_ERROR_MASK" pos="3" type="boolean"/>
+		<bitfield name="MONITOR" pos="4" type="boolean"/>
+		<bitfield name="MONITOR_MASK" pos="5" type="boolean"/>
+		<bitfield name="RX_DONE" pos="6" type="boolean"/>
+		<bitfield name="RX_DONE_MASK" pos="7" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x002a0" name="CEC_ADDR"/>
+	<reg32 offset="0x002a4" name="CEC_TIME">
+		<bitfield name="ENABLE" pos="0" type="boolean"/>
+		<bitfield name="SIGNAL_FREE_TIME" low="7" high="15" type="uint"/>
+	</reg32>
+	<reg32 offset="0x002a8" name="CEC_REFTIMER">
+		<bitfield name="REFTIMER" low="0" high="15" type="uint"/>
+		<bitfield name="ENABLE" pos="16" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x002ac" name="CEC_RD_DATA">
+		<bitfield name="DATA" low="0" high="7" type="uint"/>
+		<bitfield name="SIZE" low="8" high="12" type="uint"/>
+	</reg32>
+	<reg32 offset="0x002b0" name="CEC_RD_FILTER"/>
+
+	<reg32 offset="0x002b4" name="ACTIVE_HSYNC">
+		<bitfield name="START" low="0" high="12" type="uint"/>
+		<bitfield name="END" low="16" high="27" type="uint"/>
+	</reg32>
+	<reg32 offset="0x002b8" name="ACTIVE_VSYNC">
+		<bitfield name="START" low="0" high="12" type="uint"/>
+		<bitfield name="END" low="16" high="28" type="uint"/>
+	</reg32>
+	<reg32 offset="0x002bc" name="VSYNC_ACTIVE_F2">
+		<!-- interlaced, frame 2 -->
+		<bitfield name="START" low="0" high="12" type="uint"/>
+		<bitfield name="END" low="16" high="28" type="uint"/>
+	</reg32>
+	<reg32 offset="0x002c0" name="TOTAL">
+		<bitfield name="H_TOTAL" low="0" high="12" type="uint"/>
+		<bitfield name="V_TOTAL" low="16" high="28" type="uint"/>
+	</reg32>
+	<reg32 offset="0x002c4" name="VSYNC_TOTAL_F2">
+		<!-- interlaced, frame 2 -->
+		<bitfield name="V_TOTAL" low="0" high="12" type="uint"/>
+	</reg32>
+	<reg32 offset="0x002c8" name="FRAME_CTRL">
+		<bitfield name="RGB_MUX_SEL_BGR" pos="12" type="boolean"/>
+		<bitfield name="VSYNC_LOW" pos="28" type="boolean"/>
+		<bitfield name="HSYNC_LOW" pos="29" type="boolean"/>
+		<bitfield name="INTERLACED_EN" pos="31" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x002cc" name="AUD_INT">
+		<!--
+			HDMI_AUD_INT[0x02CC]
+			[3] AUD_SAM_DROP_MASK [R/W]
+			[2] AUD_SAM_DROP_ACK [W], AUD_SAM_DROP_INT [R]
+			[1] AUD_FIFO_URUN_MASK [R/W]
+			[0] AUD_FIFO_URUN_ACK [W], AUD_FIFO_URUN_INT [R]
+		 -->
+		<bitfield name="AUD_FIFO_URUN_INT" pos="0" type="boolean"/>  <!-- write to ack irq -->
+		<bitfield name="AUD_FIFO_URAN_MASK" pos="1" type="boolean"/> <!-- r/w, enables irq -->
+		<bitfield name="AUD_SAM_DROP_INT" pos="2" type="boolean"/>   <!-- write to ack irq -->
+		<bitfield name="AUD_SAM_DROP_MASK" pos="3" type="boolean"/>  <!-- r/w, enables irq -->
+	</reg32>
+	<reg32 offset="0x002d4" name="PHY_CTRL">
+		<!--
+			in hdmi_phy_reset() it appears to be toggling SW_RESET/
+			SW_RESET_PLL based on the value of the bit above, so
+			I'm guessing the bit above is a polarit bit
+		 -->
+		<bitfield name="SW_RESET_PLL" pos="0" type="boolean"/>
+		<bitfield name="SW_RESET_PLL_LOW" pos="1" type="boolean"/>
+		<bitfield name="SW_RESET" pos="2" type="boolean"/>
+		<bitfield name="SW_RESET_LOW" pos="3" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x002dc" name="CEC_WR_RANGE"/>
+	<reg32 offset="0x002e0" name="CEC_RD_RANGE"/>
+	<reg32 offset="0x002e4" name="VERSION"/>
+	<reg32 offset="0x00360" name="CEC_COMPL_CTL"/>
+	<reg32 offset="0x00364" name="CEC_RD_START_RANGE"/>
+	<reg32 offset="0x00368" name="CEC_RD_TOTAL_RANGE"/>
+	<reg32 offset="0x0036c" name="CEC_RD_ERR_RESP_LO"/>
+	<reg32 offset="0x00370" name="CEC_WR_CHECK_CONFIG"/>
+
+</domain>
+
+<domain name="HDMI_8x60" width="32">
+	<reg32 offset="0x00000" name="PHY_REG0">
+		<bitfield name="DESER_DEL_CTRL" low="2" high="4" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00004" name="PHY_REG1">
+		<bitfield name="DTEST_MUX_SEL" low="4" high="7" type="uint"/>
+		<bitfield name="OUTVOL_SWING_CTRL" low="0" high="3" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00008" name="PHY_REG2">
+		<bitfield name="PD_DESER" pos="0" type="boolean"/>
+		<bitfield name="PD_DRIVE_1" pos="1" type="boolean"/>
+		<bitfield name="PD_DRIVE_2" pos="2" type="boolean"/>
+		<bitfield name="PD_DRIVE_3" pos="3" type="boolean"/>
+		<bitfield name="PD_DRIVE_4" pos="4" type="boolean"/>
+		<bitfield name="PD_PLL" pos="5" type="boolean"/>
+		<bitfield name="PD_PWRGEN" pos="6" type="boolean"/>
+		<bitfield name="RCV_SENSE_EN" pos="7" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0000c" name="PHY_REG3">
+		<bitfield name="PLL_ENABLE" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00010" name="PHY_REG4"/>
+	<reg32 offset="0x00014" name="PHY_REG5"/>
+	<reg32 offset="0x00018" name="PHY_REG6"/>
+	<reg32 offset="0x0001c" name="PHY_REG7"/>
+	<reg32 offset="0x00020" name="PHY_REG8"/>
+	<reg32 offset="0x00024" name="PHY_REG9"/>
+	<reg32 offset="0x00028" name="PHY_REG10"/>
+	<reg32 offset="0x0002c" name="PHY_REG11"/>
+	<reg32 offset="0x00030" name="PHY_REG12">
+		<bitfield name="RETIMING_EN" pos="0" type="boolean"/>
+		<bitfield name="PLL_LOCK_DETECT_EN" pos="1" type="boolean"/>
+		<bitfield name="FORCE_LOCK" pos="4" type="boolean"/>
+	</reg32>
+</domain>
+
+<domain name="HDMI_8960" width="32">
+	<!--
+		some of the bitfields may be same as 8x60.. but no helpful comments
+		in msm_dss_io_8960.c
+	 -->
+	<reg32 offset="0x00000" name="PHY_REG0"/>
+	<reg32 offset="0x00004" name="PHY_REG1"/>
+	<reg32 offset="0x00008" name="PHY_REG2"/>
+	<reg32 offset="0x0000c" name="PHY_REG3"/>
+	<reg32 offset="0x00010" name="PHY_REG4"/>
+	<reg32 offset="0x00014" name="PHY_REG5"/>
+	<reg32 offset="0x00018" name="PHY_REG6"/>
+	<reg32 offset="0x0001c" name="PHY_REG7"/>
+	<reg32 offset="0x00020" name="PHY_REG8"/>
+	<reg32 offset="0x00024" name="PHY_REG9"/>
+	<reg32 offset="0x00028" name="PHY_REG10"/>
+	<reg32 offset="0x0002c" name="PHY_REG11"/>
+	<reg32 offset="0x00030" name="PHY_REG12">
+		<bitfield name="SW_RESET" pos="5" type="boolean"/>
+		<bitfield name="PWRDN_B" pos="7" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00034" name="PHY_REG_BIST_CFG"/>
+	<reg32 offset="0x00038" name="PHY_DEBUG_BUS_SEL"/>
+	<reg32 offset="0x0003c" name="PHY_REG_MISC0"/>
+	<reg32 offset="0x00040" name="PHY_REG13"/>
+	<reg32 offset="0x00044" name="PHY_REG14"/>
+	<reg32 offset="0x00048" name="PHY_REG15"/>
+</domain>
+
+<domain name="HDMI_8960_PHY_PLL" width="32">
+	<reg32 offset="0x00000" name="REFCLK_CFG"/>
+	<reg32 offset="0x00004" name="CHRG_PUMP_CFG"/>
+	<reg32 offset="0x00008" name="LOOP_FLT_CFG0"/>
+	<reg32 offset="0x0000c" name="LOOP_FLT_CFG1"/>
+	<reg32 offset="0x00010" name="IDAC_ADJ_CFG"/>
+	<reg32 offset="0x00014" name="I_VI_KVCO_CFG"/>
+	<reg32 offset="0x00018" name="PWRDN_B">
+		<bitfield name="PD_PLL" pos="1" type="boolean"/>
+		<bitfield name="PLL_PWRDN_B" pos="3" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0001c" name="SDM_CFG0"/>
+	<reg32 offset="0x00020" name="SDM_CFG1"/>
+	<reg32 offset="0x00024" name="SDM_CFG2"/>
+	<reg32 offset="0x00028" name="SDM_CFG3"/>
+	<reg32 offset="0x0002c" name="SDM_CFG4"/>
+	<reg32 offset="0x00030" name="SSC_CFG0"/>
+	<reg32 offset="0x00034" name="SSC_CFG1"/>
+	<reg32 offset="0x00038" name="SSC_CFG2"/>
+	<reg32 offset="0x0003c" name="SSC_CFG3"/>
+	<reg32 offset="0x00040" name="LOCKDET_CFG0"/>
+	<reg32 offset="0x00044" name="LOCKDET_CFG1"/>
+	<reg32 offset="0x00048" name="LOCKDET_CFG2"/>
+	<reg32 offset="0x0004c" name="VCOCAL_CFG0"/>
+	<reg32 offset="0x00050" name="VCOCAL_CFG1"/>
+	<reg32 offset="0x00054" name="VCOCAL_CFG2"/>
+	<reg32 offset="0x00058" name="VCOCAL_CFG3"/>
+	<reg32 offset="0x0005c" name="VCOCAL_CFG4"/>
+	<reg32 offset="0x00060" name="VCOCAL_CFG5"/>
+	<reg32 offset="0x00064" name="VCOCAL_CFG6"/>
+	<reg32 offset="0x00068" name="VCOCAL_CFG7"/>
+	<reg32 offset="0x0006c" name="DEBUG_SEL"/>
+	<reg32 offset="0x00070" name="MISC0"/>
+	<reg32 offset="0x00074" name="MISC1"/>
+	<reg32 offset="0x00078" name="MISC2"/>
+	<reg32 offset="0x0007c" name="MISC3"/>
+	<reg32 offset="0x00080" name="MISC4"/>
+	<reg32 offset="0x00084" name="MISC5"/>
+	<reg32 offset="0x00088" name="MISC6"/>
+	<reg32 offset="0x0008c" name="DEBUG_BUS0"/>
+	<reg32 offset="0x00090" name="DEBUG_BUS1"/>
+	<reg32 offset="0x00094" name="DEBUG_BUS2"/>
+	<reg32 offset="0x00098" name="STATUS0">
+		<bitfield name="PLL_LOCK" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0009c" name="STATUS1"/>
+</domain>
+
+<domain name="HDMI_8x74" width="32">
+	<!--
+		seems to be all mdp5+ have same?
+	 -->
+	<reg32 offset="0x00000" name="ANA_CFG0"/>
+	<reg32 offset="0x00004" name="ANA_CFG1"/>
+	<reg32 offset="0x00008" name="ANA_CFG2"/>
+	<reg32 offset="0x0000c" name="ANA_CFG3"/>
+	<reg32 offset="0x00010" name="PD_CTRL0"/>
+	<reg32 offset="0x00014" name="PD_CTRL1"/>
+	<reg32 offset="0x00018" name="GLB_CFG"/>
+	<reg32 offset="0x0001c" name="DCC_CFG0"/>
+	<reg32 offset="0x00020" name="DCC_CFG1"/>
+	<reg32 offset="0x00024" name="TXCAL_CFG0"/>
+	<reg32 offset="0x00028" name="TXCAL_CFG1"/>
+	<reg32 offset="0x0002c" name="TXCAL_CFG2"/>
+	<reg32 offset="0x00030" name="TXCAL_CFG3"/>
+	<reg32 offset="0x00034" name="BIST_CFG0"/>
+	<reg32 offset="0x0003c" name="BIST_PATN0"/>
+	<reg32 offset="0x00040" name="BIST_PATN1"/>
+	<reg32 offset="0x00044" name="BIST_PATN2"/>
+	<reg32 offset="0x00048" name="BIST_PATN3"/>
+	<reg32 offset="0x0005c" name="STATUS"/>
+</domain>
+
+<domain name="HDMI_28nm_PHY_PLL" width="32">
+	<reg32 offset="0x00000" name="REFCLK_CFG"/>
+	<reg32 offset="0x00004" name="POSTDIV1_CFG"/>
+	<reg32 offset="0x00008" name="CHGPUMP_CFG"/>
+	<reg32 offset="0x0000C" name="VCOLPF_CFG"/>
+	<reg32 offset="0x00010" name="VREG_CFG"/>
+	<reg32 offset="0x00014" name="PWRGEN_CFG"/>
+	<reg32 offset="0x00018" name="DMUX_CFG"/>
+	<reg32 offset="0x0001C" name="AMUX_CFG"/>
+	<reg32 offset="0x00020" name="GLB_CFG">
+		<bitfield name="PLL_PWRDN_B" pos="0" type="boolean"/>
+		<bitfield name="PLL_LDO_PWRDN_B" pos="1" type="boolean"/>
+		<bitfield name="PLL_PWRGEN_PWRDN_B" pos="2" type="boolean"/>
+		<bitfield name="PLL_ENABLE" pos="3" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x00024" name="POSTDIV2_CFG"/>
+	<reg32 offset="0x00028" name="POSTDIV3_CFG"/>
+	<reg32 offset="0x0002C" name="LPFR_CFG"/>
+	<reg32 offset="0x00030" name="LPFC1_CFG"/>
+	<reg32 offset="0x00034" name="LPFC2_CFG"/>
+	<reg32 offset="0x00038" name="SDM_CFG0"/>
+	<reg32 offset="0x0003C" name="SDM_CFG1"/>
+	<reg32 offset="0x00040" name="SDM_CFG2"/>
+	<reg32 offset="0x00044" name="SDM_CFG3"/>
+	<reg32 offset="0x00048" name="SDM_CFG4"/>
+	<reg32 offset="0x0004C" name="SSC_CFG0"/>
+	<reg32 offset="0x00050" name="SSC_CFG1"/>
+	<reg32 offset="0x00054" name="SSC_CFG2"/>
+	<reg32 offset="0x00058" name="SSC_CFG3"/>
+	<reg32 offset="0x0005C" name="LKDET_CFG0"/>
+	<reg32 offset="0x00060" name="LKDET_CFG1"/>
+	<reg32 offset="0x00064" name="LKDET_CFG2"/>
+	<reg32 offset="0x00068" name="TEST_CFG">
+		<bitfield name="PLL_SW_RESET" pos="0" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0006C" name="CAL_CFG0"/>
+	<reg32 offset="0x00070" name="CAL_CFG1"/>
+	<reg32 offset="0x00074" name="CAL_CFG2"/>
+	<reg32 offset="0x00078" name="CAL_CFG3"/>
+	<reg32 offset="0x0007C" name="CAL_CFG4"/>
+	<reg32 offset="0x00080" name="CAL_CFG5"/>
+	<reg32 offset="0x00084" name="CAL_CFG6"/>
+	<reg32 offset="0x00088" name="CAL_CFG7"/>
+	<reg32 offset="0x0008C" name="CAL_CFG8"/>
+	<reg32 offset="0x00090" name="CAL_CFG9"/>
+	<reg32 offset="0x00094" name="CAL_CFG10"/>
+	<reg32 offset="0x00098" name="CAL_CFG11"/>
+	<reg32 offset="0x0009C" name="EFUSE_CFG"/>
+	<reg32 offset="0x000A0" name="DEBUG_BUS_SEL"/>
+	<reg32 offset="0x000C0" name="STATUS"/>
+</domain>
+
+<domain name="HDMI_8996_PHY" width="32">
+	<reg32 offset="0x00000" name="CFG"/>
+	<reg32 offset="0x00004" name="PD_CTL"/>
+	<reg32 offset="0x00008" name="MODE"/>
+	<reg32 offset="0x0000C" name="MISR_CLEAR"/>
+	<reg32 offset="0x00010" name="TX0_TX1_BIST_CFG0"/>
+	<reg32 offset="0x00014" name="TX0_TX1_BIST_CFG1"/>
+	<reg32 offset="0x00018" name="TX0_TX1_PRBS_SEED_BYTE0"/>
+	<reg32 offset="0x0001C" name="TX0_TX1_PRBS_SEED_BYTE1"/>
+	<reg32 offset="0x00020" name="TX0_TX1_BIST_PATTERN0"/>
+	<reg32 offset="0x00024" name="TX0_TX1_BIST_PATTERN1"/>
+	<reg32 offset="0x00028" name="TX2_TX3_BIST_CFG0"/>
+	<reg32 offset="0x0002C" name="TX2_TX3_BIST_CFG1"/>
+	<reg32 offset="0x00030" name="TX2_TX3_PRBS_SEED_BYTE0"/>
+	<reg32 offset="0x00034" name="TX2_TX3_PRBS_SEED_BYTE1"/>
+	<reg32 offset="0x00038" name="TX2_TX3_BIST_PATTERN0"/>
+	<reg32 offset="0x0003C" name="TX2_TX3_BIST_PATTERN1"/>
+	<reg32 offset="0x00040" name="DEBUG_BUS_SEL"/>
+	<reg32 offset="0x00044" name="TXCAL_CFG0"/>
+	<reg32 offset="0x00048" name="TXCAL_CFG1"/>
+	<reg32 offset="0x0004C" name="TX0_TX1_LANE_CTL"/>
+	<reg32 offset="0x00050" name="TX2_TX3_LANE_CTL"/>
+	<reg32 offset="0x00054" name="LANE_BIST_CONFIG"/>
+	<reg32 offset="0x00058" name="CLOCK"/>
+	<reg32 offset="0x0005C" name="MISC1"/>
+	<reg32 offset="0x00060" name="MISC2"/>
+	<reg32 offset="0x00064" name="TX0_TX1_BIST_STATUS0"/>
+	<reg32 offset="0x00068" name="TX0_TX1_BIST_STATUS1"/>
+	<reg32 offset="0x0006C" name="TX0_TX1_BIST_STATUS2"/>
+	<reg32 offset="0x00070" name="TX2_TX3_BIST_STATUS0"/>
+	<reg32 offset="0x00074" name="TX2_TX3_BIST_STATUS1"/>
+	<reg32 offset="0x00078" name="TX2_TX3_BIST_STATUS2"/>
+	<reg32 offset="0x0007C" name="PRE_MISR_STATUS0"/>
+	<reg32 offset="0x00080" name="PRE_MISR_STATUS1"/>
+	<reg32 offset="0x00084" name="PRE_MISR_STATUS2"/>
+	<reg32 offset="0x00088" name="PRE_MISR_STATUS3"/>
+	<reg32 offset="0x0008C" name="POST_MISR_STATUS0"/>
+	<reg32 offset="0x00090" name="POST_MISR_STATUS1"/>
+	<reg32 offset="0x00094" name="POST_MISR_STATUS2"/>
+	<reg32 offset="0x00098" name="POST_MISR_STATUS3"/>
+	<reg32 offset="0x0009C" name="STATUS"/>
+	<reg32 offset="0x000A0" name="MISC3_STATUS"/>
+	<reg32 offset="0x000A4" name="MISC4_STATUS"/>
+	<reg32 offset="0x000A8" name="DEBUG_BUS0"/>
+	<reg32 offset="0x000AC" name="DEBUG_BUS1"/>
+	<reg32 offset="0x000B0" name="DEBUG_BUS2"/>
+	<reg32 offset="0x000B4" name="DEBUG_BUS3"/>
+	<reg32 offset="0x000B8" name="PHY_REVISION_ID0"/>
+	<reg32 offset="0x000BC" name="PHY_REVISION_ID1"/>
+	<reg32 offset="0x000C0" name="PHY_REVISION_ID2"/>
+	<reg32 offset="0x000C4" name="PHY_REVISION_ID3"/>
+</domain>
+
+<domain name="HDMI_PHY_QSERDES_COM" width="32">
+	<reg32 offset="0x00000" name="ATB_SEL1"/>
+	<reg32 offset="0x00004" name="ATB_SEL2"/>
+	<reg32 offset="0x00008" name="FREQ_UPDATE"/>
+	<reg32 offset="0x0000C" name="BG_TIMER"/>
+	<reg32 offset="0x00010" name="SSC_EN_CENTER"/>
+	<reg32 offset="0x00014" name="SSC_ADJ_PER1"/>
+	<reg32 offset="0x00018" name="SSC_ADJ_PER2"/>
+	<reg32 offset="0x0001C" name="SSC_PER1"/>
+	<reg32 offset="0x00020" name="SSC_PER2"/>
+	<reg32 offset="0x00024" name="SSC_STEP_SIZE1"/>
+	<reg32 offset="0x00028" name="SSC_STEP_SIZE2"/>
+	<reg32 offset="0x0002C" name="POST_DIV"/>
+	<reg32 offset="0x00030" name="POST_DIV_MUX"/>
+	<reg32 offset="0x00034" name="BIAS_EN_CLKBUFLR_EN"/>
+	<reg32 offset="0x00038" name="CLK_ENABLE1"/>
+	<reg32 offset="0x0003C" name="SYS_CLK_CTRL"/>
+	<reg32 offset="0x00040" name="SYSCLK_BUF_ENABLE"/>
+	<reg32 offset="0x00044" name="PLL_EN"/>
+	<reg32 offset="0x00048" name="PLL_IVCO"/>
+	<reg32 offset="0x0004C" name="LOCK_CMP1_MODE0"/>
+	<reg32 offset="0x00050" name="LOCK_CMP2_MODE0"/>
+	<reg32 offset="0x00054" name="LOCK_CMP3_MODE0"/>
+	<reg32 offset="0x00058" name="LOCK_CMP1_MODE1"/>
+	<reg32 offset="0x0005C" name="LOCK_CMP2_MODE1"/>
+	<reg32 offset="0x00060" name="LOCK_CMP3_MODE1"/>
+	<reg32 offset="0x00064" name="LOCK_CMP1_MODE2"/>
+	<reg32 offset="0x00064" name="CMN_RSVD0"/>
+	<reg32 offset="0x00068" name="LOCK_CMP2_MODE2"/>
+	<reg32 offset="0x00068" name="EP_CLOCK_DETECT_CTRL"/>
+	<reg32 offset="0x0006C" name="LOCK_CMP3_MODE2"/>
+	<reg32 offset="0x0006C" name="SYSCLK_DET_COMP_STATUS"/>
+	<reg32 offset="0x00070" name="BG_TRIM"/>
+	<reg32 offset="0x00074" name="CLK_EP_DIV"/>
+	<reg32 offset="0x00078" name="CP_CTRL_MODE0"/>
+	<reg32 offset="0x0007C" name="CP_CTRL_MODE1"/>
+	<reg32 offset="0x00080" name="CP_CTRL_MODE2"/>
+	<reg32 offset="0x00080" name="CMN_RSVD1"/>
+	<reg32 offset="0x00084" name="PLL_RCTRL_MODE0"/>
+	<reg32 offset="0x00088" name="PLL_RCTRL_MODE1"/>
+	<reg32 offset="0x0008C" name="PLL_RCTRL_MODE2"/>
+	<reg32 offset="0x0008C" name="CMN_RSVD2"/>
+	<reg32 offset="0x00090" name="PLL_CCTRL_MODE0"/>
+	<reg32 offset="0x00094" name="PLL_CCTRL_MODE1"/>
+	<reg32 offset="0x00098" name="PLL_CCTRL_MODE2"/>
+	<reg32 offset="0x00098" name="CMN_RSVD3"/>
+	<reg32 offset="0x0009C" name="PLL_CNTRL"/>
+	<reg32 offset="0x000A0" name="PHASE_SEL_CTRL"/>
+	<reg32 offset="0x000A4" name="PHASE_SEL_DC"/>
+	<reg32 offset="0x000A8" name="CORE_CLK_IN_SYNC_SEL"/>
+	<reg32 offset="0x000A8" name="BIAS_EN_CTRL_BY_PSM"/>
+	<reg32 offset="0x000AC" name="SYSCLK_EN_SEL"/>
+	<reg32 offset="0x000B0" name="CML_SYSCLK_SEL"/>
+	<reg32 offset="0x000B4" name="RESETSM_CNTRL"/>
+	<reg32 offset="0x000B8" name="RESETSM_CNTRL2"/>
+	<reg32 offset="0x000BC" name="RESTRIM_CTRL"/>
+	<reg32 offset="0x000C0" name="RESTRIM_CTRL2"/>
+	<reg32 offset="0x000C4" name="RESCODE_DIV_NUM"/>
+	<reg32 offset="0x000C8" name="LOCK_CMP_EN"/>
+	<reg32 offset="0x000CC" name="LOCK_CMP_CFG"/>
+	<reg32 offset="0x000D0" name="DEC_START_MODE0"/>
+	<reg32 offset="0x000D4" name="DEC_START_MODE1"/>
+	<reg32 offset="0x000D8" name="DEC_START_MODE2"/>
+	<reg32 offset="0x000D8" name="VCOCAL_DEADMAN_CTRL"/>
+	<reg32 offset="0x000DC" name="DIV_FRAC_START1_MODE0"/>
+	<reg32 offset="0x000E0" name="DIV_FRAC_START2_MODE0"/>
+	<reg32 offset="0x000E4" name="DIV_FRAC_START3_MODE0"/>
+	<reg32 offset="0x000E8" name="DIV_FRAC_START1_MODE1"/>
+	<reg32 offset="0x000EC" name="DIV_FRAC_START2_MODE1"/>
+	<reg32 offset="0x000F0" name="DIV_FRAC_START3_MODE1"/>
+	<reg32 offset="0x000F4" name="DIV_FRAC_START1_MODE2"/>
+	<reg32 offset="0x000F4" name="VCO_TUNE_MINVAL1"/>
+	<reg32 offset="0x000F8" name="DIV_FRAC_START2_MODE2"/>
+	<reg32 offset="0x000F8" name="VCO_TUNE_MINVAL2"/>
+	<reg32 offset="0x000FC" name="DIV_FRAC_START3_MODE2"/>
+	<reg32 offset="0x000FC" name="CMN_RSVD4"/>
+	<reg32 offset="0x00100" name="INTEGLOOP_INITVAL"/>
+	<reg32 offset="0x00104" name="INTEGLOOP_EN"/>
+	<reg32 offset="0x00108" name="INTEGLOOP_GAIN0_MODE0"/>
+	<reg32 offset="0x0010C" name="INTEGLOOP_GAIN1_MODE0"/>
+	<reg32 offset="0x00110" name="INTEGLOOP_GAIN0_MODE1"/>
+	<reg32 offset="0x00114" name="INTEGLOOP_GAIN1_MODE1"/>
+	<reg32 offset="0x00118" name="INTEGLOOP_GAIN0_MODE2"/>
+	<reg32 offset="0x00118" name="VCO_TUNE_MAXVAL1"/>
+	<reg32 offset="0x0011C" name="INTEGLOOP_GAIN1_MODE2"/>
+	<reg32 offset="0x0011C" name="VCO_TUNE_MAXVAL2"/>
+	<reg32 offset="0x00120" name="RES_TRIM_CONTROL2"/>
+	<reg32 offset="0x00124" name="VCO_TUNE_CTRL"/>
+	<reg32 offset="0x00128" name="VCO_TUNE_MAP"/>
+	<reg32 offset="0x0012C" name="VCO_TUNE1_MODE0"/>
+	<reg32 offset="0x00130" name="VCO_TUNE2_MODE0"/>
+	<reg32 offset="0x00134" name="VCO_TUNE1_MODE1"/>
+	<reg32 offset="0x00138" name="VCO_TUNE2_MODE1"/>
+	<reg32 offset="0x0013C" name="VCO_TUNE1_MODE2"/>
+	<reg32 offset="0x0013C" name="VCO_TUNE_INITVAL1"/>
+	<reg32 offset="0x00140" name="VCO_TUNE2_MODE2"/>
+	<reg32 offset="0x00140" name="VCO_TUNE_INITVAL2"/>
+	<reg32 offset="0x00144" name="VCO_TUNE_TIMER1"/>
+	<reg32 offset="0x00148" name="VCO_TUNE_TIMER2"/>
+	<reg32 offset="0x0014C" name="SAR"/>
+	<reg32 offset="0x00150" name="SAR_CLK"/>
+	<reg32 offset="0x00154" name="SAR_CODE_OUT_STATUS"/>
+	<reg32 offset="0x00158" name="SAR_CODE_READY_STATUS"/>
+	<reg32 offset="0x0015C" name="CMN_STATUS"/>
+	<reg32 offset="0x00160" name="RESET_SM_STATUS"/>
+	<reg32 offset="0x00164" name="RESTRIM_CODE_STATUS"/>
+	<reg32 offset="0x00168" name="PLLCAL_CODE1_STATUS"/>
+	<reg32 offset="0x0016C" name="PLLCAL_CODE2_STATUS"/>
+	<reg32 offset="0x00170" name="BG_CTRL"/>
+	<reg32 offset="0x00174" name="CLK_SELECT"/>
+	<reg32 offset="0x00178" name="HSCLK_SEL"/>
+	<reg32 offset="0x0017C" name="INTEGLOOP_BINCODE_STATUS"/>
+	<reg32 offset="0x00180" name="PLL_ANALOG"/>
+	<reg32 offset="0x00184" name="CORECLK_DIV"/>
+	<reg32 offset="0x00188" name="SW_RESET"/>
+	<reg32 offset="0x0018C" name="CORE_CLK_EN"/>
+	<reg32 offset="0x00190" name="C_READY_STATUS"/>
+	<reg32 offset="0x00194" name="CMN_CONFIG"/>
+	<reg32 offset="0x00198" name="CMN_RATE_OVERRIDE"/>
+	<reg32 offset="0x0019C" name="SVS_MODE_CLK_SEL"/>
+	<reg32 offset="0x001A0" name="DEBUG_BUS0"/>
+	<reg32 offset="0x001A4" name="DEBUG_BUS1"/>
+	<reg32 offset="0x001A8" name="DEBUG_BUS2"/>
+	<reg32 offset="0x001AC" name="DEBUG_BUS3"/>
+	<reg32 offset="0x001B0" name="DEBUG_BUS_SEL"/>
+	<reg32 offset="0x001B4" name="CMN_MISC1"/>
+	<reg32 offset="0x001B8" name="CMN_MISC2"/>
+	<reg32 offset="0x001BC" name="CORECLK_DIV_MODE1"/>
+	<reg32 offset="0x001C0" name="CORECLK_DIV_MODE2"/>
+	<reg32 offset="0x001C4" name="CMN_RSVD5"/>
+</domain>
+
+
+<domain name="HDMI_PHY_QSERDES_TX_LX" width="32">
+		<reg32 offset="0x00000" name="BIST_MODE_LANENO"/>
+		<reg32 offset="0x00004" name="BIST_INVERT"/>
+		<reg32 offset="0x00008" name="CLKBUF_ENABLE"/>
+		<reg32 offset="0x0000C" name="CMN_CONTROL_ONE"/>
+		<reg32 offset="0x00010" name="CMN_CONTROL_TWO"/>
+		<reg32 offset="0x00014" name="CMN_CONTROL_THREE"/>
+		<reg32 offset="0x00018" name="TX_EMP_POST1_LVL"/>
+		<reg32 offset="0x0001C" name="TX_POST2_EMPH"/>
+		<reg32 offset="0x00020" name="TX_BOOST_LVL_UP_DN"/>
+		<reg32 offset="0x00024" name="HP_PD_ENABLES"/>
+		<reg32 offset="0x00028" name="TX_IDLE_LVL_LARGE_AMP"/>
+		<reg32 offset="0x0002C" name="TX_DRV_LVL"/>
+		<reg32 offset="0x00030" name="TX_DRV_LVL_OFFSET"/>
+		<reg32 offset="0x00034" name="RESET_TSYNC_EN"/>
+		<reg32 offset="0x00038" name="PRE_STALL_LDO_BOOST_EN"/>
+		<reg32 offset="0x0003C" name="TX_BAND"/>
+		<reg32 offset="0x00040" name="SLEW_CNTL"/>
+		<reg32 offset="0x00044" name="INTERFACE_SELECT"/>
+		<reg32 offset="0x00048" name="LPB_EN"/>
+		<reg32 offset="0x0004C" name="RES_CODE_LANE_TX"/>
+		<reg32 offset="0x00050" name="RES_CODE_LANE_RX"/>
+		<reg32 offset="0x00054" name="RES_CODE_LANE_OFFSET"/>
+		<reg32 offset="0x00058" name="PERL_LENGTH1"/>
+		<reg32 offset="0x0005C" name="PERL_LENGTH2"/>
+		<reg32 offset="0x00060" name="SERDES_BYP_EN_OUT"/>
+		<reg32 offset="0x00064" name="DEBUG_BUS_SEL"/>
+		<reg32 offset="0x00068" name="HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN"/>
+		<reg32 offset="0x0006C" name="TX_POL_INV"/>
+		<reg32 offset="0x00070" name="PARRATE_REC_DETECT_IDLE_EN"/>
+		<reg32 offset="0x00074" name="BIST_PATTERN1"/>
+		<reg32 offset="0x00078" name="BIST_PATTERN2"/>
+		<reg32 offset="0x0007C" name="BIST_PATTERN3"/>
+		<reg32 offset="0x00080" name="BIST_PATTERN4"/>
+		<reg32 offset="0x00084" name="BIST_PATTERN5"/>
+		<reg32 offset="0x00088" name="BIST_PATTERN6"/>
+		<reg32 offset="0x0008C" name="BIST_PATTERN7"/>
+		<reg32 offset="0x00090" name="BIST_PATTERN8"/>
+		<reg32 offset="0x00094" name="LANE_MODE"/>
+		<reg32 offset="0x00098" name="IDAC_CAL_LANE_MODE"/>
+		<reg32 offset="0x0009C" name="IDAC_CAL_LANE_MODE_CONFIGURATION"/>
+		<reg32 offset="0x000A0" name="ATB_SEL1"/>
+		<reg32 offset="0x000A4" name="ATB_SEL2"/>
+		<reg32 offset="0x000A8" name="RCV_DETECT_LVL"/>
+		<reg32 offset="0x000AC" name="RCV_DETECT_LVL_2"/>
+		<reg32 offset="0x000B0" name="PRBS_SEED1"/>
+		<reg32 offset="0x000B4" name="PRBS_SEED2"/>
+		<reg32 offset="0x000B8" name="PRBS_SEED3"/>
+		<reg32 offset="0x000BC" name="PRBS_SEED4"/>
+		<reg32 offset="0x000C0" name="RESET_GEN"/>
+		<reg32 offset="0x000C4" name="RESET_GEN_MUXES"/>
+		<reg32 offset="0x000C8" name="TRAN_DRVR_EMP_EN"/>
+		<reg32 offset="0x000CC" name="TX_INTERFACE_MODE"/>
+		<reg32 offset="0x000D0" name="PWM_CTRL"/>
+		<reg32 offset="0x000D4" name="PWM_ENCODED_OR_DATA"/>
+		<reg32 offset="0x000D8" name="PWM_GEAR_1_DIVIDER_BAND2"/>
+		<reg32 offset="0x000DC" name="PWM_GEAR_2_DIVIDER_BAND2"/>
+		<reg32 offset="0x000E0" name="PWM_GEAR_3_DIVIDER_BAND2"/>
+		<reg32 offset="0x000E4" name="PWM_GEAR_4_DIVIDER_BAND2"/>
+		<reg32 offset="0x000E8" name="PWM_GEAR_1_DIVIDER_BAND0_1"/>
+		<reg32 offset="0x000EC" name="PWM_GEAR_2_DIVIDER_BAND0_1"/>
+		<reg32 offset="0x000F0" name="PWM_GEAR_3_DIVIDER_BAND0_1"/>
+		<reg32 offset="0x000F4" name="PWM_GEAR_4_DIVIDER_BAND0_1"/>
+		<reg32 offset="0x000F8" name="VMODE_CTRL1"/>
+		<reg32 offset="0x000FC" name="VMODE_CTRL2"/>
+		<reg32 offset="0x00100" name="TX_ALOG_INTF_OBSV_CNTL"/>
+		<reg32 offset="0x00104" name="BIST_STATUS"/>
+		<reg32 offset="0x00108" name="BIST_ERROR_COUNT1"/>
+		<reg32 offset="0x0010C" name="BIST_ERROR_COUNT2"/>
+		<reg32 offset="0x00110" name="TX_ALOG_INTF_OBSV"/>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/display/mdp4.xml b/drivers/gpu/drm/msm/registers/display/mdp4.xml
new file mode 100644
index 000000000000..6abb4a3c04da
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/mdp4.xml
@@ -0,0 +1,504 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+<import file="display/mdp_common.xml"/>
+
+<domain name="MDP4" width="32">
+	<enum name="mdp4_pipe">
+		<brief>pipe names, index into PIPE[]</brief>
+		<value name="VG1" value="0"/>
+		<value name="VG2" value="1"/>
+		<value name="RGB1" value="2"/>
+		<value name="RGB2" value="3"/>
+		<value name="RGB3" value="4"/>
+		<value name="VG3" value="5"/>
+		<value name="VG4" value="6"/>
+	</enum>
+
+	<enum name="mdp4_mixer">
+		<value name="MIXER0" value="0"/>
+		<value name="MIXER1" value="1"/>
+		<value name="MIXER2" value="2"/>
+	</enum>
+
+	<enum name="mdp4_intf">
+		<!--
+			A bit confusing the enums for interface selection:
+				enum {
+					LCDC_RGB_INTF,			/* 0 */
+					DTV_INTF = LCDC_RGB_INTF,	/* 0 */
+					MDDI_LCDC_INTF,			/* 1 */
+					MDDI_INTF,			/* 2 */
+					EBI2_INTF,			/* 3 */
+					TV_INTF = EBI2_INTF,		/* 3 */
+					DSI_VIDEO_INTF,
+					DSI_CMD_INTF
+				};
+			there is some overlap, and not all the values end up getting
+			written to hw (mdp4_display_intf_sel() remaps the last two
+			values to MDDI_LCDC_INTF/MDDI_INTF with extra bits set).. so
+			taking some liberties in guessing the actual meanings/names:
+		 -->
+		<value name="INTF_LCDC_DTV" value="0"/>  <!-- LCDC RGB or DTV (external) -->
+		<value name="INTF_DSI_VIDEO" value="1"/>
+		<value name="INTF_DSI_CMD" value="2"/>
+		<value name="INTF_EBI2_TV" value="3"/>   <!-- EBI2 or TV (external) -->
+	</enum>
+	<enum name="mdp4_cursor_format">
+		<value name="CURSOR_ARGB" value="1"/>
+		<value name="CURSOR_XRGB" value="2"/>
+	</enum>
+	<enum name="mdp4_frame_format">
+		<value name="FRAME_LINEAR" value="0"/>
+		<value name="FRAME_TILE_ARGB_4X4" value="1"/>
+		<value name="FRAME_TILE_YCBCR_420" value="2"/>
+	</enum>
+	<enum name="mdp4_scale_unit">
+		<value name="SCALE_FIR" value="0"/>
+		<value name="SCALE_MN_PHASE" value="1"/>
+		<value name="SCALE_PIXEL_RPT" value="2"/>
+	</enum>
+
+	<bitset name="mdp4_layermixer_in_cfg" inline="yes">
+		<brief>appears to map pipe to mixer stage</brief>
+		<bitfield name="PIPE0" low="0"  high="2"  type="mdp_mixer_stage_id"/>
+		<bitfield name="PIPE0_MIXER1" pos="3" type="boolean"/>
+		<bitfield name="PIPE1" low="4"  high="6"  type="mdp_mixer_stage_id"/>
+		<bitfield name="PIPE1_MIXER1" pos="7" type="boolean"/>
+		<bitfield name="PIPE2" low="8"  high="10" type="mdp_mixer_stage_id"/>
+		<bitfield name="PIPE2_MIXER1" pos="11" type="boolean"/>
+		<bitfield name="PIPE3" low="12" high="14" type="mdp_mixer_stage_id"/>
+		<bitfield name="PIPE3_MIXER1" pos="15" type="boolean"/>
+		<bitfield name="PIPE4" low="16" high="18" type="mdp_mixer_stage_id"/>
+		<bitfield name="PIPE4_MIXER1" pos="19" type="boolean"/>
+		<bitfield name="PIPE5" low="20" high="22" type="mdp_mixer_stage_id"/>
+		<bitfield name="PIPE5_MIXER1" pos="23" type="boolean"/>
+		<bitfield name="PIPE6" low="24" high="26" type="mdp_mixer_stage_id"/>
+		<bitfield name="PIPE6_MIXER1" pos="27" type="boolean"/>
+		<bitfield name="PIPE7" low="28" high="30" type="mdp_mixer_stage_id"/>
+		<bitfield name="PIPE7_MIXER1" pos="31" type="boolean"/>
+	</bitset>
+
+	<bitset name="MDP4_IRQ">
+		<bitfield name="OVERLAY0_DONE" pos="0" type="boolean"/>
+		<bitfield name="OVERLAY1_DONE" pos="1" type="boolean"/>
+		<bitfield name="DMA_S_DONE" pos="2" type="boolean"/>
+		<bitfield name="DMA_E_DONE" pos="3" type="boolean"/>
+		<bitfield name="DMA_P_DONE" pos="4" type="boolean"/>
+		<bitfield name="VG1_HISTOGRAM" pos="5" type="boolean"/>
+		<bitfield name="VG2_HISTOGRAM" pos="6" type="boolean"/>
+		<bitfield name="PRIMARY_VSYNC" pos="7" type="boolean"/>
+		<bitfield name="PRIMARY_INTF_UDERRUN" pos="8" type="boolean"/>
+		<bitfield name="EXTERNAL_VSYNC" pos="9" type="boolean"/>
+		<bitfield name="EXTERNAL_INTF_UDERRUN" pos="10" type="boolean"/>
+		<bitfield name="PRIMARY_RDPTR" pos="11" type="boolean"/>  <!-- read pointer -->
+		<bitfield name="DMA_P_HISTOGRAM" pos="17" type="boolean"/>
+		<bitfield name="DMA_S_HISTOGRAM" pos="26" type="boolean"/>
+		<bitfield name="OVERLAY2_DONE" pos="30" type="boolean"/>
+	</bitset>
+
+	<reg32 offset="0x00000" name="VERSION">
+		<!--
+			from mdp_probe() we can see minor rev starts at 16.. assume
+			major is above that.. not sure the rest of bits but doesn't
+			really seem to matter
+		 -->
+		<bitfield name="MINOR" low="16" high="23" type="uint"/>
+		<bitfield name="MAJOR" low="24" high="31" type="uint"/>
+	</reg32>
+	<reg32 offset="0x00004" name="OVLP0_KICK"/>
+	<reg32 offset="0x00008" name="OVLP1_KICK"/>
+	<reg32 offset="0x000d0" name="OVLP2_KICK"/>
+	<reg32 offset="0x0000c" name="DMA_P_KICK"/>
+	<reg32 offset="0x00010" name="DMA_S_KICK"/>
+	<reg32 offset="0x00014" name="DMA_E_KICK"/>
+	<reg32 offset="0x00018" name="DISP_STATUS"/>
+
+	<reg32 offset="0x00038" name="DISP_INTF_SEL">
+		<bitfield name="PRIM" low="0" high="1" type="mdp4_intf"/>
+		<bitfield name="SEC" low="2" high="3" type="mdp4_intf"/>
+		<bitfield name="EXT" low="4" high="5" type="mdp4_intf"/>
+		<bitfield name="DSI_VIDEO" pos="6" type="boolean"/>
+		<bitfield name="DSI_CMD" pos="7" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x0003c" name="RESET_STATUS"/>  <!-- only mdp4 >v2.1 -->
+	<reg32 offset="0x0004c" name="READ_CNFG"/>  <!-- something about # of pending requests.. -->
+	<reg32 offset="0x00050" name="INTR_ENABLE" type="MDP4_IRQ"/>
+	<reg32 offset="0x00054" name="INTR_STATUS" type="MDP4_IRQ"/>
+	<reg32 offset="0x00058" name="INTR_CLEAR" type="MDP4_IRQ"/>
+	<reg32 offset="0x00060" name="EBI2_LCD0"/>
+	<reg32 offset="0x00064" name="EBI2_LCD1"/>
+	<reg32 offset="0x00070" name="PORTMAP_MODE"/>
+
+	<!-- mdp chip-select controller: -->
+	<reg32 offset="0x000c0" name="CS_CONTROLLER0"/>
+	<reg32 offset="0x000c4" name="CS_CONTROLLER1"/>
+
+	<reg32 offset="0x100f0" name="LAYERMIXER2_IN_CFG" type="mdp4_layermixer_in_cfg"/>
+	<reg32 offset="0x100fc" name="LAYERMIXER_IN_CFG_UPDATE_METHOD"/>
+	<reg32 offset="0x10100" name="LAYERMIXER_IN_CFG" type="mdp4_layermixer_in_cfg"/>
+
+	<reg32 offset="0x30050" name="VG2_SRC_FORMAT"/>
+	<reg32 offset="0x31008" name="VG2_CONST_COLOR"/>
+
+	<reg32 offset="0x18000" name="OVERLAY_FLUSH">
+		<bitfield name="OVLP0" pos="0" type="boolean"/>
+		<bitfield name="OVLP1" pos="1" type="boolean"/>
+		<bitfield name="VG1" pos="2" type="boolean"/>
+		<bitfield name="VG2" pos="3" type="boolean"/>
+		<bitfield name="RGB1" pos="4" type="boolean"/>
+		<bitfield name="RGB2" pos="5" type="boolean"/>
+	</reg32>
+
+	<array offsets="0x10000,0x18000,0x88000" name="OVLP" length="3" stride="0x8000">
+		<reg32 offset="0x0004" name="CFG"/>
+		<reg32 offset="0x0008" name="SIZE" type="reg_wh"/>
+		<reg32 offset="0x000c" name="BASE"/>
+		<reg32 offset="0x0010" name="STRIDE" type="uint"/>
+		<reg32 offset="0x0014" name="OPMODE"/>
+
+		<array offsets="0x0104,0x0124,0x0144,0x0160" name="STAGE" length="4" stride="0x1c">
+			<reg32 offset="0x00" name="OP">
+				<bitfield name="FG_ALPHA" low="0" high="1" type="mdp_alpha_type"/>
+				<bitfield name="FG_INV_ALPHA" pos="2" type="boolean"/>
+				<bitfield name="FG_MOD_ALPHA" pos="3" type="boolean"/>
+				<bitfield name="BG_ALPHA" low="4" high="5" type="mdp_alpha_type"/>
+				<bitfield name="BG_INV_ALPHA" pos="6" type="boolean"/>
+				<bitfield name="BG_MOD_ALPHA" pos="7" type="boolean"/>
+				<bitfield name="FG_TRANSP" pos="8" type="boolean"/>
+				<bitfield name="BG_TRANSP" pos="9" type="boolean"/>
+			</reg32>
+			<reg32 offset="0x04" name="FG_ALPHA"/>
+			<reg32 offset="0x08" name="BG_ALPHA"/>
+			<reg32 offset="0x0c" name="TRANSP_LOW0"/>
+			<reg32 offset="0x10" name="TRANSP_LOW1"/>
+			<reg32 offset="0x14" name="TRANSP_HIGH0"/>
+			<reg32 offset="0x18" name="TRANSP_HIGH1"/>
+		</array>
+
+		<array offsets="0x1004,0x1404,0x1804,0x1b84" name="STAGE_CO3" length="4" stride="4">
+			<reg32 offset="0" name="SEL">
+				<bitfield name="FG_ALPHA" pos="0" type="boolean"/> <!-- otherwise bg alpha -->
+			</reg32>
+		</array>
+
+		<reg32 offset="0x0180" name="TRANSP_LOW0"/>
+		<reg32 offset="0x0184" name="TRANSP_LOW1"/>
+		<reg32 offset="0x0188" name="TRANSP_HIGH0"/>
+		<reg32 offset="0x018c" name="TRANSP_HIGH1"/>
+
+		<reg32 offset="0x0200" name="CSC_CONFIG"/>
+
+		<array offset="0x2000" name="CSC" length="1" stride="0x700">
+			<array offset="0x400" name="MV" length="9" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+			<array offset="0x500" name="PRE_BV" length="3" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+			<array offset="0x580" name="POST_BV" length="3" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+			<array offset="0x600" name="PRE_LV" length="6" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+			<array offset="0x680" name="POST_LV" length="6" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+		</array>
+	</array>
+
+	<enum name="mdp4_dma">
+		<value name="DMA_P" value="0"/>
+		<value name="DMA_S" value="1"/>
+		<value name="DMA_E" value="2"/>
+	</enum>
+	<reg32 offset="0x90070" name="DMA_P_OP_MODE"/>
+	<array offset="0x94800" name="LUTN" length="2" stride="0x400">
+		<array offset="0" name="LUT" length="0x100" stride="4">
+			<reg32 offset="0" name="VAL"/>
+		</array>
+	</array>
+	<reg32 offset="0xa0028" name="DMA_S_OP_MODE"/>
+	<!-- I guess if DMA_S has an OP_MODE, it must have a LUT too.. -->
+	<reg32 offset="0xb0070" name="DMA_E_QUANT" length="3" stride="4"/>
+	<array offsets="0x90000,0xa0000,0xb0000" name="DMA" length="3" stride="0x10000" index="mdp4_dma">
+		<reg32 offset="0x0000" name="CONFIG">
+			<bitfield name="G_BPC" low="0" high="1" type="mdp_bpc"/>
+			<bitfield name="B_BPC" low="2" high="3" type="mdp_bpc"/>
+			<bitfield name="R_BPC" low="4" high="5" type="mdp_bpc"/>
+			<bitfield name="PACK_ALIGN_MSB" pos="7" type="boolean"/>
+			<bitfield name="PACK" low="8" high="15"/>
+			<!-- bit 24 is DITHER_EN on DMA_P, DEFLKR_EN on DMA_E -->
+			<bitfield name="DEFLKR_EN" pos="24" type="boolean"/>
+			<bitfield name="DITHER_EN" pos="24" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x0004" name="SRC_SIZE" type="reg_wh"/>
+		<reg32 offset="0x0008" name="SRC_BASE"/>
+		<reg32 offset="0x000c" name="SRC_STRIDE" type="uint"/>
+		<reg32 offset="0x0010" name="DST_SIZE" type="reg_wh"/>
+
+		<reg32 offset="0x0044" name="CURSOR_SIZE">
+			<!-- seems the limit is 64x64: -->
+			<bitfield name="WIDTH" low="0" high="6" type="uint"/>
+			<bitfield name="HEIGHT" low="16" high="22" type="uint"/>
+		</reg32>
+		<reg32 offset="0x0048" name="CURSOR_BASE"/>
+		<reg32 offset="0x004c" name="CURSOR_POS">
+			<bitfield name="X" low="0" high="15" type="uint"/>
+			<bitfield name="Y" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x0060" name="CURSOR_BLEND_CONFIG">
+			<bitfield name="CURSOR_EN" pos="0" type="boolean"/>
+			<bitfield name="FORMAT" low="1" high="2" type="mdp4_cursor_format"/>
+			<bitfield name="TRANSP_EN" pos="3" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x0064" name="CURSOR_BLEND_PARAM"/>
+		<reg32 offset="0x0068" name="BLEND_TRANS_LOW"/>
+		<reg32 offset="0x006c" name="BLEND_TRANS_HIGH"/>
+
+		<reg32 offset="0x1004" name="FETCH_CONFIG"/>
+		<array offset="0x3000" name="CSC" length="1" stride="0x700">
+			<array offset="0x400" name="MV" length="9" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+			<array offset="0x500" name="PRE_BV" length="3" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+			<array offset="0x580" name="POST_BV" length="3" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+			<array offset="0x600" name="PRE_LV" length="6" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+			<array offset="0x680" name="POST_LV" length="6" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+		</array>
+	</array>
+
+	<!--
+		TODO length should be 7, but that would collide w/ OVLP2..!?!
+		this register map is a bit strange..
+	 -->
+	<array offset="0x20000" name="PIPE" length="6" stride="0x10000" index="mdp4_pipe">
+		<reg32 offset="0x0000" name="SRC_SIZE" type="reg_wh"/>
+		<reg32 offset="0x0004" name="SRC_XY" type="reg_xy"/>
+		<reg32 offset="0x0008" name="DST_SIZE" type="reg_wh"/>
+		<reg32 offset="0x000c" name="DST_XY" type="reg_xy"/>
+		<reg32 offset="0x0010" name="SRCP0_BASE"/>
+		<reg32 offset="0x0014" name="SRCP1_BASE"/>
+		<reg32 offset="0x0018" name="SRCP2_BASE"/>
+		<reg32 offset="0x001c" name="SRCP3_BASE"/>
+		<reg32 offset="0x0040" name="SRC_STRIDE_A">
+			<bitfield name="P0" low="0" high="15" type="uint"/>
+			<bitfield name="P1" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x0044" name="SRC_STRIDE_B">
+			<bitfield name="P2" low="0" high="15" type="uint"/>
+			<bitfield name="P3" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x0048" name="SSTILE_FRAME_SIZE" type="reg_wh"/>
+		<reg32 offset="0x0050" name="SRC_FORMAT">
+			<bitfield name="G_BPC" low="0" high="1" type="mdp_bpc"/>
+			<bitfield name="B_BPC" low="2" high="3" type="mdp_bpc"/>
+			<bitfield name="R_BPC" low="4" high="5" type="mdp_bpc"/>
+			<bitfield name="A_BPC" low="6" high="7" type="mdp_bpc_alpha"/>
+			<bitfield name="ALPHA_ENABLE" pos="8" type="boolean"/>
+			<bitfield name="CPP" low="9" high="10" type="uint">
+				<brief>8bit characters per pixel minus 1</brief>
+			</bitfield>
+			<bitfield name="ROTATED_90" pos="12" type="boolean"/>
+			<bitfield name="UNPACK_COUNT" low="13" high="14" type="uint"/>
+			<bitfield name="UNPACK_TIGHT" pos="17" type="boolean"/>
+			<bitfield name="UNPACK_ALIGN_MSB" pos="18" type="boolean"/>
+			<bitfield name="FETCH_PLANES" low="19" high="20" type="uint"/>
+			<bitfield name="SOLID_FILL" pos="22" type="boolean"/>
+			<bitfield name="CHROMA_SAMP" low="26" high="27" type="mdp_chroma_samp_type"/>
+			<bitfield name="FRAME_FORMAT" low="29" high="30" type="mdp4_frame_format"/>
+		</reg32>
+		<reg32 offset="0x0054" name="SRC_UNPACK" type="mdp_unpack_pattern"/>
+		<reg32 offset="0x0058" name="OP_MODE">
+			<bitfield name="SCALEX_EN" pos="0" type="boolean"/>
+			<bitfield name="SCALEY_EN" pos="1" type="boolean"/>
+			<bitfield name="SCALEX_UNIT_SEL" low="2" high="3" type="mdp4_scale_unit"/>
+			<bitfield name="SCALEY_UNIT_SEL" low="4" high="5" type="mdp4_scale_unit"/>
+			<bitfield name="SRC_YCBCR" pos="9" type="boolean"/>
+			<bitfield name="DST_YCBCR" pos="10" type="boolean"/>
+			<bitfield name="CSC_EN" pos="11" type="boolean"/>
+			<bitfield name="FLIP_LR" pos="13" type="boolean"/>
+			<bitfield name="FLIP_UD" pos="14" type="boolean"/>
+			<bitfield name="DITHER_EN" pos="15" type="boolean"/>
+			<bitfield name="IGC_LUT_EN" pos="16" type="boolean"/>
+			<bitfield name="DEINT_EN" pos="18" type="boolean"/>
+			<bitfield name="DEINT_ODD_REF" pos="19" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x005c" name="PHASEX_STEP"/>
+		<reg32 offset="0x0060" name="PHASEY_STEP"/>
+		<reg32 offset="0x1004" name="FETCH_CONFIG"/>
+		<reg32 offset="0x1008" name="SOLID_COLOR"/>
+
+		<array offset="0x4000" name="CSC" length="1" stride="0x700">
+			<array offset="0x400" name="MV" length="9" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+			<array offset="0x500" name="PRE_BV" length="3" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+			<array offset="0x580" name="POST_BV" length="3" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+			<array offset="0x600" name="PRE_LV" length="6" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+			<array offset="0x680" name="POST_LV" length="6" stride="4">
+				<reg32 offset="0" name="VAL"/>
+			</array>
+		</array>
+	</array>
+
+	<!--
+		ENCODERS
+			LCDC and DSI seem the same, DTV is just slightly different..
+	 -->
+
+	<bitset name="mdp4_ctrl_polarity" inline="yes">
+		<!-- not entirely sure if these bits mean hi or low.. -->
+		<bitfield name="HSYNC_LOW" pos="0" type="boolean"/>
+		<bitfield name="VSYNC_LOW" pos="1" type="boolean"/>
+		<bitfield name="DATA_EN_LOW" pos="2" type="boolean"/>
+	</bitset>
+
+	<bitset name="mdp4_active_hctl" inline="yes">
+		<bitfield name="START" low="0" high="14" type="uint"/>
+		<bitfield name="END" low="16" high="30" type="uint"/>
+		<bitfield name="ACTIVE_START_X" pos="31" type="boolean"/>
+	</bitset>
+
+	<bitset name="mdp4_display_hctl" inline="yes">
+		<bitfield name="START" low="0" high="15" type="uint"/>
+		<bitfield name="END" low="16" high="31" type="uint"/>
+	</bitset>
+
+	<bitset name="mdp4_hsync_ctrl" inline="yes">
+		<bitfield name="PULSEW" low="0" high="15" type="uint"/>
+		<bitfield name="PERIOD" low="16" high="31" type="uint"/>
+	</bitset>
+
+	<bitset name="mdp4_underflow_clr" inline="yes">
+		<bitfield name="COLOR" low="0" high="23"/>
+		<bitfield name="ENABLE_RECOVERY" pos="31" type="boolean"/>
+	</bitset>
+
+	<!-- offset is 0xe0000 on !mdp4.. -->
+	<array offset="0xc0000" name="LCDC" length="1" stride="0x1000">
+		<reg32 offset="0x0000" name="ENABLE"/>
+		<reg32 offset="0x0004" name="HSYNC_CTRL" type="mdp4_hsync_ctrl"/>
+		<reg32 offset="0x0008" name="VSYNC_PERIOD" type="uint"/>
+		<reg32 offset="0x000c" name="VSYNC_LEN" type="uint"/>
+		<reg32 offset="0x0010" name="DISPLAY_HCTRL" type="mdp4_display_hctl"/>
+		<reg32 offset="0x0014" name="DISPLAY_VSTART" type="uint"/>
+		<reg32 offset="0x0018" name="DISPLAY_VEND" type="uint"/>
+		<reg32 offset="0x001c" name="ACTIVE_HCTL" type="mdp4_active_hctl"/>
+		<reg32 offset="0x0020" name="ACTIVE_VSTART" type="uint"/>
+		<reg32 offset="0x0024" name="ACTIVE_VEND" type="uint"/>
+		<reg32 offset="0x0028" name="BORDER_CLR"/>
+		<reg32 offset="0x002c" name="UNDERFLOW_CLR" type="mdp4_underflow_clr"/>
+		<reg32 offset="0x0030" name="HSYNC_SKEW"/>
+		<reg32 offset="0x0034" name="TEST_CNTL"/>
+		<reg32 offset="0x0038" name="CTRL_POLARITY" type="mdp4_ctrl_polarity"/>
+	</array>
+
+	<reg32 offset="0xc2000" name="LCDC_LVDS_INTF_CTL">
+		<bitfield name="MODE_SEL"           pos="2"  type="boolean"/>
+		<bitfield name="RGB_OUT"            pos="3"  type="boolean"/>
+		<bitfield name="CH_SWAP"            pos="4"  type="boolean"/>
+		<bitfield name="CH1_RES_BIT"        pos="5"  type="boolean"/>
+		<bitfield name="CH2_RES_BIT"        pos="6"  type="boolean"/>
+		<bitfield name="ENABLE"             pos="7"  type="boolean"/>
+		<bitfield name="CH1_DATA_LANE0_EN"  pos="8"  type="boolean"/>
+		<bitfield name="CH1_DATA_LANE1_EN"  pos="9"  type="boolean"/>
+		<bitfield name="CH1_DATA_LANE2_EN"  pos="10" type="boolean"/>
+		<bitfield name="CH1_DATA_LANE3_EN"  pos="11" type="boolean"/>
+		<bitfield name="CH2_DATA_LANE0_EN"  pos="12" type="boolean"/>
+		<bitfield name="CH2_DATA_LANE1_EN"  pos="13" type="boolean"/>
+		<bitfield name="CH2_DATA_LANE2_EN"  pos="14" type="boolean"/>
+		<bitfield name="CH2_DATA_LANE3_EN"  pos="15" type="boolean"/>
+		<bitfield name="CH1_CLK_LANE_EN"    pos="16" type="boolean"/>
+		<bitfield name="CH2_CLK_LANE_EN"    pos="17" type="boolean"/>
+	</reg32>
+
+	<array offset="0xc2014" name="LCDC_LVDS_MUX_CTL" length="4" stride="0x8">
+		<reg32 offset="0x0" name="3_TO_0">
+			<bitfield name="BIT0" low="0"  high="7"/>
+			<bitfield name="BIT1" low="8"  high="15"/>
+			<bitfield name="BIT2" low="16" high="23"/>
+			<bitfield name="BIT3" low="24" high="31"/>
+		</reg32>
+		<reg32 offset="0x4" name="6_TO_4">
+			<bitfield name="BIT4" low="0"  high="7"/>
+			<bitfield name="BIT5" low="8"  high="15"/>
+			<bitfield name="BIT6" low="16" high="23"/>
+		</reg32>
+	</array>
+
+	<reg32 offset="0xc2034" name="LCDC_LVDS_PHY_RESET"/>
+
+	<reg32 offset="0xc3000" name="LVDS_PHY_PLL_CTRL_0"/>
+	<reg32 offset="0xc3004" name="LVDS_PHY_PLL_CTRL_1"/>
+	<reg32 offset="0xc3008" name="LVDS_PHY_PLL_CTRL_2"/>
+	<reg32 offset="0xc300c" name="LVDS_PHY_PLL_CTRL_3"/>
+	<reg32 offset="0xc3014" name="LVDS_PHY_PLL_CTRL_5"/>
+	<reg32 offset="0xc3018" name="LVDS_PHY_PLL_CTRL_6"/>
+	<reg32 offset="0xc301c" name="LVDS_PHY_PLL_CTRL_7"/>
+	<reg32 offset="0xc3020" name="LVDS_PHY_PLL_CTRL_8"/>
+	<reg32 offset="0xc3024" name="LVDS_PHY_PLL_CTRL_9"/>
+	<reg32 offset="0xc3080" name="LVDS_PHY_PLL_LOCKED"/>
+	<reg32 offset="0xc3108" name="LVDS_PHY_CFG2"/>
+
+	<reg32 offset="0xc3100" name="LVDS_PHY_CFG0">
+		<bitfield name="SERIALIZATION_ENBLE" pos="4" type="boolean"/>
+		<bitfield name="CHANNEL0" pos="6" type="boolean"/>
+		<bitfield name="CHANNEL1" pos="7" type="boolean"/>
+	</reg32>
+
+	<array offset="0xd0000" name="DTV" length="1" stride="0x1000">
+		<reg32 offset="0x0000" name="ENABLE"/>
+		<reg32 offset="0x0004" name="HSYNC_CTRL" type="mdp4_hsync_ctrl"/>
+		<reg32 offset="0x0008" name="VSYNC_PERIOD" type="uint"/>
+		<reg32 offset="0x000c" name="VSYNC_LEN" type="uint"/>
+		<reg32 offset="0x0018" name="DISPLAY_HCTRL" type="mdp4_display_hctl"/>
+		<reg32 offset="0x001c" name="DISPLAY_VSTART" type="uint"/>
+		<reg32 offset="0x0020" name="DISPLAY_VEND" type="uint"/>
+		<reg32 offset="0x002c" name="ACTIVE_HCTL" type="mdp4_active_hctl"/>
+		<reg32 offset="0x0030" name="ACTIVE_VSTART" type="uint"/>
+		<reg32 offset="0x0038" name="ACTIVE_VEND" type="uint"/>
+		<reg32 offset="0x0040" name="BORDER_CLR"/>
+		<reg32 offset="0x0044" name="UNDERFLOW_CLR" type="mdp4_underflow_clr"/>
+		<reg32 offset="0x0048" name="HSYNC_SKEW"/>
+		<reg32 offset="0x004c" name="TEST_CNTL"/>
+		<reg32 offset="0x0050" name="CTRL_POLARITY" type="mdp4_ctrl_polarity"/>
+	</array>
+
+	<array offset="0xe0000" name="DSI" length="1" stride="0x1000">
+		<reg32 offset="0x0000" name="ENABLE"/>
+		<reg32 offset="0x0004" name="HSYNC_CTRL" type="mdp4_hsync_ctrl"/>
+		<reg32 offset="0x0008" name="VSYNC_PERIOD" type="uint"/>
+		<reg32 offset="0x000c" name="VSYNC_LEN" type="uint"/>
+		<reg32 offset="0x0010" name="DISPLAY_HCTRL" type="mdp4_display_hctl"/>
+		<reg32 offset="0x0014" name="DISPLAY_VSTART" type="uint"/>
+		<reg32 offset="0x0018" name="DISPLAY_VEND" type="uint"/>
+		<reg32 offset="0x001c" name="ACTIVE_HCTL" type="mdp4_active_hctl"/>
+		<reg32 offset="0x0020" name="ACTIVE_VSTART" type="uint"/>
+		<reg32 offset="0x0024" name="ACTIVE_VEND" type="uint"/>
+		<reg32 offset="0x0028" name="BORDER_CLR"/>
+		<reg32 offset="0x002c" name="UNDERFLOW_CLR" type="mdp4_underflow_clr"/>
+		<reg32 offset="0x0030" name="HSYNC_SKEW"/>
+		<reg32 offset="0x0034" name="TEST_CNTL"/>
+		<reg32 offset="0x0038" name="CTRL_POLARITY" type="mdp4_ctrl_polarity"/>
+	</array>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/display/mdp5.xml b/drivers/gpu/drm/msm/registers/display/mdp5.xml
new file mode 100644
index 000000000000..92f3263af170
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/mdp5.xml
@@ -0,0 +1,806 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+<import file="display/mdp_common.xml"/>
+
+<!-- where does this belong? -->
+<domain name="VBIF" width="32">
+</domain>
+
+<domain name="MDSS" width="32">
+	<reg32 offset="0x00000" name="HW_VERSION">
+		<bitfield name="STEP" low="0" high="15" type="uint"/>
+		<bitfield name="MINOR" low="16" high="27" type="uint"/>
+		<bitfield name="MAJOR" low="28" high="31" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x00010" name="HW_INTR_STATUS">
+		<bitfield name="INTR_MDP"  pos="0"  type="boolean"/>
+		<bitfield name="INTR_DSI0" pos="4"  type="boolean"/>
+		<bitfield name="INTR_DSI1" pos="5"  type="boolean"/>
+		<bitfield name="INTR_HDMI" pos="8"  type="boolean"/>
+		<bitfield name="INTR_EDP"  pos="12" type="boolean"/>
+	</reg32>
+</domain>
+
+<domain name="MDP5" width="32">
+
+	<enum name="mdp5_intf_type">
+		<value name="INTF_DISABLED" value="0x0"/>
+		<value name="INTF_DSI"  value="0x1"/>
+		<value name="INTF_HDMI" value="0x3"/>
+		<value name="INTF_LCDC" value="0x5"/>
+		<value name="INTF_eDP"  value="0x9"/>
+		<value name="INTF_VIRTUAL"  value="0x64"/>
+		<!-- non-display interfaces are listed below: -->
+		<value name="INTF_WB"  value="0x65"/>
+	</enum>
+
+	<enum name="mdp5_intfnum">
+		<value name="NO_INTF" value="0"/>
+		<value name="INTF0"   value="1"/>
+		<value name="INTF1"   value="2"/>
+		<value name="INTF2"   value="3"/>
+		<value name="INTF3"   value="4"/>
+	</enum>
+
+	<enum name="mdp5_pipe">
+		<value name="SSPP_NONE" value="0"/>
+		<value name="SSPP_VIG0" value="1"/>
+		<value name="SSPP_VIG1" value="2"/>
+		<value name="SSPP_VIG2" value="3"/>
+		<value name="SSPP_RGB0" value="4"/>
+		<value name="SSPP_RGB1" value="5"/>
+		<value name="SSPP_RGB2" value="6"/>
+		<value name="SSPP_DMA0" value="7"/>
+		<value name="SSPP_DMA1" value="8"/>
+		<value name="SSPP_VIG3" value="9"/>
+		<value name="SSPP_RGB3" value="10"/>
+		<value name="SSPP_CURSOR0" value="11"/>
+		<value name="SSPP_CURSOR1" value="12"/>
+	</enum>
+
+	<enum name="mdp5_format">
+		<!-- TODO -->
+		<value name="DUMMY" value="0"/>
+	</enum>
+
+	<enum name="mdp5_ctl_mode">
+		<value name="MODE_NONE" value="0"/>
+		<value name="MODE_WB_0_BLOCK" value="1"/>
+		<value name="MODE_WB_1_BLOCK" value="2"/>
+		<value name="MODE_WB_0_LINE" value="3"/>
+		<value name="MODE_WB_1_LINE" value="4"/>
+		<value name="MODE_WB_2_LINE" value="5"/>
+	</enum>
+
+	<enum name="mdp5_pack_3d">
+		<value name="PACK_3D_FRAME_INT" value="0"/>
+		<value name="PACK_3D_H_ROW_INT" value="1"/>
+		<value name="PACK_3D_V_ROW_INT" value="2"/>
+		<value name="PACK_3D_COL_INT"   value="3"/>
+	</enum>
+
+	<enum name="mdp5_scale_filter">
+		<value name="SCALE_FILTER_NEAREST" value="0"/>
+		<value name="SCALE_FILTER_BIL" value="1"/>
+		<value name="SCALE_FILTER_PCMN" value="2"/>
+		<value name="SCALE_FILTER_CA" value="3"/>
+	</enum>
+
+	<enum name="mdp5_pipe_bwc">
+		<value name="BWC_LOSSLESS" value="0"/>
+		<value name="BWC_Q_HIGH"   value="1"/>
+		<value name="BWC_Q_MED"    value="2"/>
+	</enum>
+
+	<enum name="mdp5_cursor_format">
+		<value name="CURSOR_FMT_ARGB8888" value="0"/>
+		<value name="CURSOR_FMT_ARGB1555" value="2"/>
+		<value name="CURSOR_FMT_ARGB4444" value="4"/>
+	</enum>
+
+	<enum name="mdp5_cursor_alpha">
+		<value name="CURSOR_ALPHA_CONST" value="0"/>
+		<value name="CURSOR_ALPHA_PER_PIXEL" value="2"/>
+	</enum>
+
+	<bitset name="MDP5_IRQ">
+		<bitfield name="WB_0_DONE"                pos="0"  type="boolean"/>
+		<bitfield name="WB_1_DONE"                pos="1"  type="boolean"/>
+		<bitfield name="WB_2_DONE"                pos="4"  type="boolean"/>
+		<bitfield name="PING_PONG_0_DONE"         pos="8"  type="boolean"/>
+		<bitfield name="PING_PONG_1_DONE"         pos="9"  type="boolean"/>
+		<bitfield name="PING_PONG_2_DONE"         pos="10" type="boolean"/>
+		<bitfield name="PING_PONG_3_DONE"         pos="11" type="boolean"/>
+		<bitfield name="PING_PONG_0_RD_PTR"       pos="12" type="boolean"/>
+		<bitfield name="PING_PONG_1_RD_PTR"       pos="13" type="boolean"/>
+		<bitfield name="PING_PONG_2_RD_PTR"       pos="14" type="boolean"/>
+		<bitfield name="PING_PONG_3_RD_PTR"       pos="15" type="boolean"/>
+		<bitfield name="PING_PONG_0_WR_PTR"       pos="16" type="boolean"/>
+		<bitfield name="PING_PONG_1_WR_PTR"       pos="17" type="boolean"/>
+		<bitfield name="PING_PONG_2_WR_PTR"       pos="18" type="boolean"/>
+		<bitfield name="PING_PONG_3_WR_PTR"       pos="19" type="boolean"/>
+		<bitfield name="PING_PONG_0_AUTO_REF"     pos="20" type="boolean"/>
+		<bitfield name="PING_PONG_1_AUTO_REF"     pos="21" type="boolean"/>
+		<bitfield name="PING_PONG_2_AUTO_REF"     pos="22" type="boolean"/>
+		<bitfield name="PING_PONG_3_AUTO_REF"     pos="23" type="boolean"/>
+		<bitfield name="INTF0_UNDER_RUN"          pos="24" type="boolean"/>
+		<bitfield name="INTF0_VSYNC"              pos="25" type="boolean"/>
+		<bitfield name="INTF1_UNDER_RUN"          pos="26" type="boolean"/>
+		<bitfield name="INTF1_VSYNC"              pos="27" type="boolean"/>
+		<bitfield name="INTF2_UNDER_RUN"          pos="28" type="boolean"/>
+		<bitfield name="INTF2_VSYNC"              pos="29" type="boolean"/>
+		<bitfield name="INTF3_UNDER_RUN"          pos="30" type="boolean"/>
+		<bitfield name="INTF3_VSYNC"              pos="31" type="boolean"/>
+	</bitset>
+
+	<bitset name="mdp5_smp_alloc" inline="yes">
+        <!-- Use "mdp5_cfg->mdp.smp.clients[enum mdp5_pipe]" instead -->
+		<bitfield name="CLIENT0" low="0"  high="7"  type="uint"/>
+		<bitfield name="CLIENT1" low="8"  high="15" type="uint"/>
+		<bitfield name="CLIENT2" low="16" high="23" type="uint"/>
+	</bitset>
+
+	<reg32 offset="0x00000" name="HW_VERSION">
+		<bitfield name="STEP" low="0" high="15" type="uint"/>
+		<bitfield name="MINOR" low="16" high="27" type="uint"/>
+		<bitfield name="MAJOR" low="28" high="31" type="uint"/>
+	</reg32>
+
+	<reg32 offset="0x00004" name="DISP_INTF_SEL">
+		<bitfield name="INTF0" low="0"  high="7"  type="mdp5_intf_type"/>
+		<bitfield name="INTF1" low="8"  high="15" type="mdp5_intf_type"/>
+		<bitfield name="INTF2" low="16" high="23" type="mdp5_intf_type"/>
+		<bitfield name="INTF3" low="24" high="31" type="mdp5_intf_type"/>
+	</reg32>
+	<reg32 offset="0x00010" name="INTR_EN" type="MDP5_IRQ"/>
+	<reg32 offset="0x00014" name="INTR_STATUS" type="MDP5_IRQ"/>
+	<reg32 offset="0x00018" name="INTR_CLEAR" type="MDP5_IRQ"/>
+	<reg32 offset="0x0001C" name="HIST_INTR_EN"/>
+	<reg32 offset="0x00020" name="HIST_INTR_STATUS"/>
+	<reg32 offset="0x00024" name="HIST_INTR_CLEAR"/>
+	<reg32 offset="0x00028" name="SPARE_0">
+		<bitfield name="SPLIT_DPL_SINGLE_FLUSH_EN" pos="0"/>
+	</reg32>
+
+	<array offset="0x00080" name="SMP_ALLOC_W" length="8" stride="4">
+		<reg32 offset="0" name="REG" type="mdp5_smp_alloc"/>
+	</array>
+	<array offset="0x00130" name="SMP_ALLOC_R" length="8" stride="4">
+		<reg32 offset="0" name="REG" type="mdp5_smp_alloc"/>
+	</array>
+
+	<enum name="mdp5_igc_type">
+		<value name="IGC_VIG" value="0"/>		<!-- 0x200 -->
+		<value name="IGC_RGB" value="1"/>		<!-- 0x210 -->
+		<value name="IGC_DMA" value="2"/>		<!-- 0x220 -->
+		<value name="IGC_DSPP" value="3"/>		<!-- 0x300 -->
+	</enum>
+	<array offsets="0x00200,0x00210,0x00220,0x00300" name="IGC" length="3" stride="0x10" index="mdp5_igc_type">
+		<array offset="0x00" name="LUT" length="3" stride="4">
+			<reg32 offset="0" name="REG">
+				<bitfield name="VAL" low="0" high="11"/>
+				<bitfield name="INDEX_UPDATE" pos="25" type="boolean"/>
+				<!--
+					not sure about these:
+						/* INDEX_UPDATE */
+						data = (1 << 25) | (((~(1 << blk_idx)) & 0x7) << 28);
+						MDSS_MDP_REG_WRITE(offset, (cfg->c0_c1_data[0] & 0xFFF) | data);
+				-->
+				<bitfield name="DISABLE_PIPE_0" pos="28" type="boolean"/>
+				<bitfield name="DISABLE_PIPE_1" pos="29" type="boolean"/>
+				<bitfield name="DISABLE_PIPE_2" pos="30" type="boolean"/>
+			</reg32>
+		</array>
+	</array>
+	<reg32 offset="0x002f4" name="SPLIT_DPL_EN"/>
+	<reg32 offset="0x002f8" name="SPLIT_DPL_UPPER">
+		<bitfield name="SMART_PANEL" pos="1" type="boolean"/>
+		<bitfield name="SMART_PANEL_FREE_RUN" pos="2" type="boolean"/>
+		<bitfield name="INTF1_SW_TRG_MUX" pos="4" type="boolean"/>
+		<bitfield name="INTF2_SW_TRG_MUX" pos="8" type="boolean"/>
+	</reg32>
+	<reg32 offset="0x003f0" name="SPLIT_DPL_LOWER">
+		<bitfield name="SMART_PANEL" pos="1" type="boolean"/>
+		<bitfield name="SMART_PANEL_FREE_RUN" pos="2" type="boolean"/>
+		<bitfield name="INTF1_TG_SYNC" pos="4" type="boolean"/>
+		<bitfield name="INTF2_TG_SYNC" pos="8" type="boolean"/>
+	</reg32>
+
+<!-- check length/index.. -->
+	<array doffsets="mdp5_cfg->ctl.base[0],mdp5_cfg->ctl.base[1],mdp5_cfg->ctl.base[2],mdp5_cfg->ctl.base[3],mdp5_cfg->ctl.base[4]" name="CTL" length="5" stride="0x400">
+		<array offsets="0x000,0x004,0x008,0x00C,0x010,0x024" name="LAYER" length="6" stride="4">
+			<!--
+			NOTE: for backwards compat (from when there were fewer stages),
+			this register has the low three bits of mdp_mixer_stage_id, with
+			the high bit coming from LAYER_EXT
+			 -->
+			<reg32 offset="0" name="REG">
+				<bitfield name="VIG0"  low="0"  high="2"  type="uint"/>
+				<bitfield name="VIG1"  low="3"  high="5"  type="uint"/>
+				<bitfield name="VIG2"  low="6"  high="8"  type="uint"/>
+				<bitfield name="RGB0"  low="9"  high="11" type="uint"/>
+				<bitfield name="RGB1"  low="12" high="14" type="uint"/>
+				<bitfield name="RGB2"  low="15" high="17" type="uint"/>
+				<bitfield name="DMA0"  low="18" high="20" type="uint"/>
+				<bitfield name="DMA1"  low="21" high="23" type="uint"/>
+				<bitfield name="BORDER_COLOR" pos="24" type="boolean"/>
+				<bitfield name="CURSOR_OUT"   pos="25" type="boolean"/>
+				<bitfield name="VIG3"  low="26"  high="28"  type="uint"/>
+				<bitfield name="RGB3"  low="29" high="31" type="uint"/>
+			</reg32>
+		</array>
+		<reg32 offset="0x014" name="OP">
+			<bitfield name="MODE" low="0" high="3" type="mdp5_ctl_mode"/>
+			<bitfield name="INTF_NUM" low="4" high="6" type="mdp5_intfnum"/>
+			<bitfield name="CMD_MODE" pos="17" type="boolean"/>
+			<bitfield name="PACK_3D_ENABLE" pos="19" type="boolean"/>
+			<bitfield name="PACK_3D" low="20" high="21" type="mdp5_pack_3d"/>
+		</reg32>
+		<reg32 offset="0x018" name="FLUSH">
+			<bitfield name="VIG0" pos="0"  type="boolean"/>
+			<bitfield name="VIG1" pos="1"  type="boolean"/>
+			<bitfield name="VIG2" pos="2"  type="boolean"/>
+			<bitfield name="RGB0" pos="3"  type="boolean"/>
+			<bitfield name="RGB1" pos="4"  type="boolean"/>
+			<bitfield name="RGB2" pos="5"  type="boolean"/>
+			<bitfield name="LM0"  pos="6"  type="boolean"/>
+			<bitfield name="LM1"  pos="7"  type="boolean"/>
+			<bitfield name="LM2"  pos="8"  type="boolean"/>
+			<bitfield name="LM3"  pos="9"  type="boolean"/>
+			<bitfield name="LM4"  pos="10"  type="boolean"/>
+			<bitfield name="DMA0" pos="11" type="boolean"/>
+			<bitfield name="DMA1" pos="12" type="boolean"/>
+			<bitfield name="DSPP0" pos="13" type="boolean"/>
+			<bitfield name="DSPP1" pos="14" type="boolean"/>
+			<bitfield name="DSPP2" pos="15" type="boolean"/>
+			<bitfield name="WB"   pos="16" type="boolean"/>
+			<bitfield name="CTL"   pos="17" type="boolean"/>
+			<bitfield name="VIG3" pos="18"  type="boolean"/>
+			<bitfield name="RGB3" pos="19"  type="boolean"/>
+			<bitfield name="LM5"  pos="20"  type="boolean"/>
+			<bitfield name="DSPP3" pos="21" type="boolean"/>
+			<bitfield name="CURSOR_0" pos="22" type="boolean"/>
+			<bitfield name="CURSOR_1" pos="23" type="boolean"/>
+			<bitfield name="CHROMADOWN_0" pos="26" type="boolean"/>
+			<bitfield name="TIMING_3" pos="28" type="boolean"/>
+			<bitfield name="TIMING_2" pos="29" type="boolean"/>
+			<bitfield name="TIMING_1" pos="30" type="boolean"/>
+			<bitfield name="TIMING_0" pos="31" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x01C" name="START"/>
+		<reg32 offset="0x020" name="PACK_3D"/>
+		<array offsets="0x040,0x044,0x048,0x04C,0x050,0x054" name="LAYER_EXT" length="6" stride="4">
+			<reg32 offset="0" name="REG">
+				<bitfield name="VIG0_BIT3"  pos="0"  type="boolean"/>
+				<bitfield name="VIG1_BIT3"  pos="2"  type="boolean"/>
+				<bitfield name="VIG2_BIT3"  pos="4"  type="boolean"/>
+				<bitfield name="VIG3_BIT3"  pos="6"  type="boolean"/>
+				<bitfield name="RGB0_BIT3"  pos="8"  type="boolean"/>
+				<bitfield name="RGB1_BIT3"  pos="10"  type="boolean"/>
+				<bitfield name="RGB2_BIT3"  pos="12"  type="boolean"/>
+				<bitfield name="RGB3_BIT3"  pos="14"  type="boolean"/>
+				<bitfield name="DMA0_BIT3"  pos="16"  type="boolean"/>
+				<bitfield name="DMA1_BIT3"  pos="18"  type="boolean"/>
+				<bitfield name="CURSOR0" low="20"  high="23"  type="mdp_mixer_stage_id"/>
+				<bitfield name="CURSOR1" low="26"  high="29"  type="mdp_mixer_stage_id"/>
+			</reg32>
+		</array>
+	</array>
+
+	<enum name="mdp5_data_format">
+		<value name="DATA_FORMAT_RGB" value="0"/>
+		<value name="DATA_FORMAT_YUV" value="1"/>
+	</enum>
+
+	<array doffsets="INVALID_IDX(idx),mdp5_cfg->pipe_vig.base[0],mdp5_cfg->pipe_vig.base[1],mdp5_cfg->pipe_vig.base[2],mdp5_cfg->pipe_rgb.base[0],mdp5_cfg->pipe_rgb.base[1],mdp5_cfg->pipe_rgb.base[2],mdp5_cfg->pipe_dma.base[0],mdp5_cfg->pipe_dma.base[1],mdp5_cfg->pipe_vig.base[3],mdp5_cfg->pipe_rgb.base[3],mdp5_cfg->pipe_cursor.base[0],mdp5_cfg->pipe_cursor.base[1]" name="PIPE" length="10" stride="0x400" index="mdp5_pipe">
+		<reg32 offset="0x200" name="OP_MODE">
+			<bitfield name="CSC_DST_DATA_FORMAT" pos="19" type="mdp5_data_format"/>
+			<bitfield name="CSC_SRC_DATA_FORMAT" pos="18" type="mdp5_data_format"/>
+			<bitfield name="CSC_1_EN" pos="17" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x2C4" name="HIST_CTL_BASE"/>
+		<reg32 offset="0x2F0" name="HIST_LUT_BASE"/>
+		<reg32 offset="0x300" name="HIST_LUT_SWAP"/>
+		<reg32 offset="0x320" name="CSC_1_MATRIX_COEFF_0">
+			<bitfield name="COEFF_11" low="0" high="12" type="uint"/>
+			<bitfield name="COEFF_12" low="16" high="28" type="uint"/>
+		</reg32>
+		<reg32 offset="0x324" name="CSC_1_MATRIX_COEFF_1">
+			<bitfield name="COEFF_13" low="0" high="12" type="uint"/>
+			<bitfield name="COEFF_21" low="16" high="28" type="uint"/>
+		</reg32>
+		<reg32 offset="0x328" name="CSC_1_MATRIX_COEFF_2">
+			<bitfield name="COEFF_22" low="0" high="12" type="uint"/>
+			<bitfield name="COEFF_23" low="16" high="28" type="uint"/>
+		</reg32>
+		<reg32 offset="0x32c" name="CSC_1_MATRIX_COEFF_3">
+			<bitfield name="COEFF_31" low="0" high="12" type="uint"/>
+			<bitfield name="COEFF_32" low="16" high="28" type="uint"/>
+		</reg32>
+		<reg32 offset="0x330" name="CSC_1_MATRIX_COEFF_4">
+			<bitfield name="COEFF_33" low="0" high="12" type="uint"/>
+		</reg32>
+		<array offset="0x334" name="CSC_1_PRE_CLAMP" length="3" stride="4">
+			<reg32 offset="0" name="REG">
+				<bitfield name="HIGH"  low="0"  high="7"  type="uint"/>
+				<bitfield name="LOW"  low="8"  high="15"  type="uint"/>
+			</reg32>
+		</array>
+		<array offset="0x340" name="CSC_1_POST_CLAMP" length="3" stride="4">
+			<reg32 offset="0" name="REG">
+				<bitfield name="HIGH"  low="0"  high="7"  type="uint"/>
+				<bitfield name="LOW"  low="8"  high="15"  type="uint"/>
+			</reg32>
+		</array>
+		<array offset="0x34c" name="CSC_1_PRE_BIAS" length="3" stride="4">
+			<reg32 offset="0" name="REG">
+				<bitfield name="VALUE"  low="0"  high="8"  type="uint"/>
+			</reg32>
+		</array>
+		<array offset="0x358" name="CSC_1_POST_BIAS" length="3" stride="4">
+			<reg32 offset="0" name="REG">
+				<bitfield name="VALUE"  low="0"  high="8"  type="uint"/>
+			</reg32>
+		</array>
+		<!-- SSPP: -->
+		<reg32 offset="0x000" name="SRC_SIZE" type="reg_wh"/>
+		<reg32 offset="0x004" name="SRC_IMG_SIZE" type="reg_wh"/>
+		<reg32 offset="0x008" name="SRC_XY" type="reg_xy"/>
+		<reg32 offset="0x00C" name="OUT_SIZE" type="reg_wh"/>
+		<reg32 offset="0x010" name="OUT_XY" type="reg_xy"/>
+		<reg32 offset="0x014" name="SRC0_ADDR"/>
+		<reg32 offset="0x018" name="SRC1_ADDR"/>
+		<reg32 offset="0x01C" name="SRC2_ADDR"/>
+		<reg32 offset="0x020" name="SRC3_ADDR"/>
+		<reg32 offset="0x024" name="SRC_STRIDE_A">
+			<bitfield name="P0" low="0" high="15" type="uint"/>
+			<bitfield name="P1" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x028" name="SRC_STRIDE_B">
+			<bitfield name="P2" low="0" high="15" type="uint"/>
+			<bitfield name="P3" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x02C" name="STILE_FRAME_SIZE"/>
+		<reg32 offset="0x030" name="SRC_FORMAT">
+			<bitfield name="G_BPC" low="0" high="1" type="mdp_bpc"/>
+			<bitfield name="B_BPC" low="2" high="3" type="mdp_bpc"/>
+			<bitfield name="R_BPC" low="4" high="5" type="mdp_bpc"/>
+			<bitfield name="A_BPC" low="6" high="7" type="mdp_bpc_alpha"/>
+			<bitfield name="ALPHA_ENABLE" pos="8" type="boolean"/>
+			<bitfield name="CPP" low="9" high="10" type="uint">
+				<brief>8bit characters per pixel minus 1</brief>
+			</bitfield>
+			<bitfield name="ROT90" pos="11" type="boolean"/>
+			<bitfield name="UNPACK_COUNT" low="12" high="13" type="uint"/>
+			<bitfield name="UNPACK_TIGHT" pos="17" type="boolean"/>
+			<bitfield name="UNPACK_ALIGN_MSB" pos="18" type="boolean"/>
+			<bitfield name="FETCH_TYPE" low="19" high="20" type="mdp_fetch_type"/>
+			<bitfield name="CHROMA_SAMP" low="23" high="24" type="mdp_chroma_samp_type"/>
+		</reg32>
+		<reg32 offset="0x034" name="SRC_UNPACK" type="mdp_unpack_pattern"/>
+		<reg32 offset="0x038" name="SRC_OP_MODE">
+			<bitfield name="BWC_EN" pos="0" type="boolean"/>
+			<bitfield name="BWC" low="1" high="2" type="mdp5_pipe_bwc"/>
+			<bitfield name="FLIP_LR" pos="13" type="boolean"/>
+			<bitfield name="FLIP_UD" pos="14" type="boolean"/>
+			<bitfield name="IGC_EN" pos="16" type="boolean"/>
+			<bitfield name="IGC_ROM_0" pos="17" type="boolean"/>
+			<bitfield name="IGC_ROM_1" pos="18" type="boolean"/>
+			<bitfield name="DEINTERLACE" pos="22" type="boolean"/>
+			<bitfield name="DEINTERLACE_ODD" pos="23" type="boolean"/>
+			<bitfield name="SW_PIX_EXT_OVERRIDE" pos="31" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x03c" name="SRC_CONSTANT_COLOR"/>
+		<reg32 offset="0x048" name="FETCH_CONFIG"/>
+		<reg32 offset="0x04c" name="VC1_RANGE"/>
+		<reg32 offset="0x050" name="REQPRIO_FIFO_WM_0"/>
+		<reg32 offset="0x054" name="REQPRIO_FIFO_WM_1"/>
+		<reg32 offset="0x058" name="REQPRIO_FIFO_WM_2"/>
+		<reg32 offset="0x070" name="SRC_ADDR_SW_STATUS"/>
+		<reg32 offset="0x0a4" name="CURRENT_SRC0_ADDR"/>
+		<reg32 offset="0x0a8" name="CURRENT_SRC1_ADDR"/>
+		<reg32 offset="0x0ac" name="CURRENT_SRC2_ADDR"/>
+		<reg32 offset="0x0b0" name="CURRENT_SRC3_ADDR"/>
+		<reg32 offset="0x0b4" name="DECIMATION">
+			<bitfield name="VERT" low="0" high="7" type="uint"/>
+			<bitfield name="HORZ" low="8" high="15" type="uint"/>
+		</reg32>
+		<array offsets="0x100,0x110,0x120" name="SW_PIX_EXT" length="3" stride="0x10" index="mdp_component_type">
+			<!--
+				Notes:
+				o These value only take effect if SW_PIX_EXT_OVERRIDE is set in SRC_OP_MODE register
+				o For signed values (int): + indicates overfetch, - indicates line drop
+			-->
+                        <reg32 offset="0x00" name="LR">
+				<bitfield name="LEFT_RPT" low="0" high="7" type="uint"/>
+				<bitfield name="LEFT_OVF" low="8" high="15" type="int"/>
+				<bitfield name="RIGHT_RPT" low="16" high="23" type="uint"/>
+				<bitfield name="RIGHT_OVF" low="24" high="31" type="int"/>
+			</reg32>
+			<reg32 offset="0x04" name="TB">
+				<bitfield name="TOP_RPT" low="0" high="7" type="uint"/>
+				<bitfield name="TOP_OVF" low="8" high="15" type="int"/>
+				<bitfield name="BOTTOM_RPT" low="16" high="23" type="uint"/>
+				<bitfield name="BOTTOM_OVF" low="24" high="31" type="int"/>
+			</reg32>
+			<reg32 offset="0x08" name="REQ_PIXELS">
+				<bitfield name="LEFT_RIGHT" low="0" high="15" type="uint"/>
+				<bitfield name="TOP_BOTTOM" low="16" high="31" type="uint"/>
+			</reg32>
+		</array>
+		<reg32 offset="0x204" name="SCALE_CONFIG">
+			<bitfield name="SCALEX_EN" pos="0" type="boolean"/>
+			<bitfield name="SCALEY_EN" pos="1" type="boolean"/>
+			<bitfield name="SCALEX_FILTER_COMP_0" low="8"  high="9"  type="mdp5_scale_filter"/>
+			<bitfield name="SCALEY_FILTER_COMP_0" low="10" high="11" type="mdp5_scale_filter"/>
+			<bitfield name="SCALEX_FILTER_COMP_1_2"  low="12" high="13" type="mdp5_scale_filter"/>
+			<bitfield name="SCALEY_FILTER_COMP_1_2"  low="14" high="15" type="mdp5_scale_filter"/>
+			<bitfield name="SCALEX_FILTER_COMP_3" low="16" high="17" type="mdp5_scale_filter"/>
+			<bitfield name="SCALEY_FILTER_COMP_3" low="18" high="19" type="mdp5_scale_filter"/>
+		</reg32>
+		<reg32 offset="0x210" name="SCALE_PHASE_STEP_X"/>
+		<reg32 offset="0x214" name="SCALE_PHASE_STEP_Y"/>
+		<reg32 offset="0x218" name="SCALE_CR_PHASE_STEP_X"/>
+		<reg32 offset="0x21c" name="SCALE_CR_PHASE_STEP_Y"/>
+		<reg32 offset="0x220" name="SCALE_INIT_PHASE_X"/>
+		<reg32 offset="0x224" name="SCALE_INIT_PHASE_Y"/>
+	</array>
+
+	<array doffsets="mdp5_cfg->lm.base[0],mdp5_cfg->lm.base[1],mdp5_cfg->lm.base[2],mdp5_cfg->lm.base[3],mdp5_cfg->lm.base[4],mdp5_cfg->lm.base[5]" name="LM" length="6" stride="0x400">
+		<reg32 offset="0x000" name="BLEND_COLOR_OUT">
+			<bitfield name="STAGE0_FG_ALPHA" pos="1" type="boolean"/>
+			<bitfield name="STAGE1_FG_ALPHA" pos="2" type="boolean"/>
+			<bitfield name="STAGE2_FG_ALPHA" pos="3" type="boolean"/>
+			<bitfield name="STAGE3_FG_ALPHA" pos="4" type="boolean"/>
+			<bitfield name="STAGE4_FG_ALPHA" pos="5" type="boolean"/>
+			<bitfield name="STAGE5_FG_ALPHA" pos="6" type="boolean"/>
+			<bitfield name="STAGE6_FG_ALPHA" pos="7" type="boolean"/>
+			<bitfield name="SPLIT_LEFT_RIGHT" pos="31" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x004" name="OUT_SIZE" type="reg_wh"/>
+		<reg32 offset="0x008" name="BORDER_COLOR_0"/>
+		<reg32 offset="0x010" name="BORDER_COLOR_1"/>
+		<array offsets="0x020,0x050,0x080,0x0B0,0x230,0x260,0x290" name="BLEND" length="7" stride="0x30">
+			<reg32 offset="0x00" name="OP_MODE">
+				<bitfield name="FG_ALPHA" low="0" high="1" type="mdp_alpha_type"/>
+				<bitfield name="FG_INV_ALPHA"     pos="2"  type="boolean"/>
+				<bitfield name="FG_MOD_ALPHA"     pos="3"  type="boolean"/>
+				<bitfield name="FG_INV_MOD_ALPHA" pos="4"  type="boolean"/>
+				<bitfield name="FG_TRANSP_EN"     pos="5"  type="boolean"/>
+				<bitfield name="BG_ALPHA" low="8" high="9" type="mdp_alpha_type"/>
+				<bitfield name="BG_INV_ALPHA"     pos="10" type="boolean"/>
+				<bitfield name="BG_MOD_ALPHA"     pos="11" type="boolean"/>
+				<bitfield name="BG_INV_MOD_ALPHA" pos="12" type="boolean"/>
+				<bitfield name="BG_TRANSP_EN"     pos="13" type="boolean"/>
+			</reg32>
+			<reg32 offset="0x04" name="FG_ALPHA"/>
+			<reg32 offset="0x08" name="BG_ALPHA"/>
+			<reg32 offset="0x0c" name="FG_TRANSP_LOW0"/>
+			<reg32 offset="0x10" name="FG_TRANSP_LOW1"/>
+			<reg32 offset="0x14" name="FG_TRANSP_HIGH0"/>
+			<reg32 offset="0x18" name="FG_TRANSP_HIGH1"/>
+			<reg32 offset="0x1c" name="BG_TRANSP_LOW0"/>
+			<reg32 offset="0x20" name="BG_TRANSP_LOW1"/>
+			<reg32 offset="0x24" name="BG_TRANSP_HIGH0"/>
+			<reg32 offset="0x28" name="BG_TRANSP_HIGH1"/>
+		</array>
+		<reg32 offset="0x0e0" name="CURSOR_IMG_SIZE">
+			<bitfield name="SRC_W" low="0" high="15" type="uint"/>
+			<bitfield name="SRC_H" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x0e4" name="CURSOR_SIZE">
+			<bitfield name="ROI_W" low="0" high="15" type="uint"/>
+			<bitfield name="ROI_H" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x0e8" name="CURSOR_XY">
+			<bitfield name="SRC_X" low="0" high="15" type="uint"/>
+			<bitfield name="SRC_Y" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x0dc" name="CURSOR_STRIDE">
+			<bitfield name="STRIDE" low="0"  high="15"  type="uint"/>
+		</reg32>
+		<reg32 offset="0x0ec" name="CURSOR_FORMAT">
+			<bitfield name="FORMAT" low="0"  high="2"  type="mdp5_cursor_format"/>
+		</reg32>
+		<reg32 offset="0x0f0" name="CURSOR_BASE_ADDR"/>
+		<reg32 offset="0x0f4" name="CURSOR_START_XY">
+			<bitfield name="X_START" low="0" high="15" type="uint"/>
+			<bitfield name="Y_START" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x0f8" name="CURSOR_BLEND_CONFIG">
+			<bitfield name="BLEND_EN" pos="0" type="boolean"/>
+			<bitfield name="BLEND_ALPHA_SEL" low="1"  high="2"  type="mdp5_cursor_alpha"/>
+			<bitfield name="BLEND_TRANSP_EN" pos="3" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x0fc" name="CURSOR_BLEND_PARAM"/>
+		<reg32 offset="0x100" name="CURSOR_BLEND_TRANSP_LOW0"/>
+		<reg32 offset="0x104" name="CURSOR_BLEND_TRANSP_LOW1"/>
+		<reg32 offset="0x108" name="CURSOR_BLEND_TRANSP_HIGH0"/>
+		<reg32 offset="0x10c" name="CURSOR_BLEND_TRANSP_HIGH1"/>
+		<reg32 offset="0x110" name="GC_LUT_BASE"/>
+	</array>
+
+	<array doffsets="mdp5_cfg->dspp.base[0],mdp5_cfg->dspp.base[1],mdp5_cfg->dspp.base[2],mdp5_cfg->dspp.base[3]" name="DSPP" length="4" stride="0x400">
+		<reg32 offset="0x000" name="OP_MODE">
+			<bitfield name="IGC_LUT_EN" pos="0" type="boolean"/>
+			<bitfield name="IGC_TBL_IDX" low="1" high="3" type="uint"/>
+			<bitfield name="PCC_EN" pos="4" type="boolean"/>
+			<bitfield name="DITHER_EN" pos="8" type="boolean"/>
+			<bitfield name="HIST_EN" pos="16" type="boolean"/>
+			<bitfield name="AUTO_CLEAR" pos="17" type="boolean"/>
+			<bitfield name="HIST_LUT_EN" pos="19" type="boolean"/>
+			<bitfield name="PA_EN" pos="20" type="boolean"/>
+			<bitfield name="GAMUT_EN" pos="23" type="boolean"/>
+			<bitfield name="GAMUT_ORDER" pos="24" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x030" name="PCC_BASE"/>
+		<reg32 offset="0x150" name="DITHER_DEPTH"/>
+		<reg32 offset="0x210" name="HIST_CTL_BASE"/>
+		<reg32 offset="0x230" name="HIST_LUT_BASE"/>
+		<reg32 offset="0x234" name="HIST_LUT_SWAP"/>
+		<reg32 offset="0x238" name="PA_BASE"/>
+		<reg32 offset="0x2dc" name="GAMUT_BASE"/>
+		<reg32 offset="0x2b0" name="GC_BASE"/>
+	</array>
+
+	<array doffsets="mdp5_cfg->pp.base[0],mdp5_cfg->pp.base[1],mdp5_cfg->pp.base[2],mdp5_cfg->pp.base[3]" name="PP" length="4" stride="0x100">
+		<reg32 offset="0x000" name="TEAR_CHECK_EN"/>
+		<reg32 offset="0x004" name="SYNC_CONFIG_VSYNC">
+			<bitfield name="COUNT" low="0" high="18" type="uint"/>
+			<bitfield name="COUNTER_EN" pos="19" type="boolean"/>
+			<bitfield name="IN_EN" pos="20" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x008" name="SYNC_CONFIG_HEIGHT"/>
+		<reg32 offset="0x00c" name="SYNC_WRCOUNT">
+			<bitfield name="LINE_COUNT" low="0" high="15" type="uint"/>
+			<bitfield name="FRAME_COUNT" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x010" name="VSYNC_INIT_VAL"/>
+		<reg32 offset="0x014" name="INT_COUNT_VAL">
+			<bitfield name="LINE_COUNT" low="0" high="15" type="uint"/>
+			<bitfield name="FRAME_COUNT" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x018" name="SYNC_THRESH">
+			<bitfield name="START" low="0" high="15" type="uint"/>
+			<bitfield name="CONTINUE" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x01c" name="START_POS"/>
+		<reg32 offset="0x020" name="RD_PTR_IRQ"/>
+		<reg32 offset="0x024" name="WR_PTR_IRQ"/>
+		<reg32 offset="0x028" name="OUT_LINE_COUNT"/>
+		<reg32 offset="0x02c" name="PP_LINE_COUNT"/>
+		<reg32 offset="0x030" name="AUTOREFRESH_CONFIG"/>
+		<reg32 offset="0x034" name="FBC_MODE"/>
+		<reg32 offset="0x038" name="FBC_BUDGET_CTL"/>
+		<reg32 offset="0x03c" name="FBC_LOSSY_MODE"/>
+	</array>
+
+	<enum name="mdp5_block_size">
+		<value name="BLOCK_SIZE_64" value="0"/>
+		<value name="BLOCK_SIZE_128" value="1"/>
+	</enum>
+
+	<enum name="mdp5_rotate_mode">
+		<value name="ROTATE_0" value="0"/>
+		<value name="ROTATE_90" value="1"/>
+	</enum>
+
+	<enum name="mdp5_chroma_downsample_method">
+		<value name="DS_MTHD_NO_PIXEL_DROP" value="0"/>
+		<value name="DS_MTHD_PIXEL_DROP" value="1"/>
+	</enum>
+
+	<array doffsets="mdp5_cfg->wb.base[0],mdp5_cfg->wb.base[1],mdp5_cfg->wb.base[2],mdp5_cfg->wb.base[3],mdp5_cfg->wb.base[4]" name="WB" length="5" stride="0x400">
+		<reg32 offset="0x000" name="DST_FORMAT">
+			<bitfield name="DSTC0_OUT" low="0" high="1" type="uint"/>
+			<bitfield name="DSTC1_OUT" low="2" high="3" type="uint"/>
+			<bitfield name="DSTC2_OUT" low="4" high="5" type="uint"/>
+			<bitfield name="DSTC3_OUT" low="6" high="7" type="uint"/>
+			<bitfield name="DSTC3_EN" pos="8" type="boolean"/>
+			<bitfield name="DST_BPP" low="9" high="10" type="uint"/>
+			<bitfield name="PACK_COUNT" low="12" high="13" type="uint"/>
+			<bitfield name="DST_ALPHA_X" pos="14" type="boolean"/>
+			<bitfield name="PACK_TIGHT" pos="17" type="boolean"/>
+			<bitfield name="PACK_ALIGN_MSB" pos="18" type="boolean"/>
+			<bitfield name="WRITE_PLANES" low="19" high="20" type="uint"/>
+			<bitfield name="DST_DITHER_EN" pos="22" type="boolean"/>
+			<bitfield name="DST_CHROMA_SAMP" low="23" high="25" type="uint"/>
+			<bitfield name="DST_CHROMA_SITE" low="26" high="29" type="uint"/>
+			<bitfield name="FRAME_FORMAT" low="30" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x004" name="DST_OP_MODE">
+			<bitfield name="BWC_ENC_EN" pos="0" type="boolean"/>
+			<bitfield name="BWC_ENC_OP" low="1" high="2" type="uint"/>
+			<bitfield name="BLOCK_SIZE" low="4" high="4" type="uint"/>
+			<bitfield name="ROT_MODE" low="5" high="5" type="uint"/>
+			<bitfield name="ROT_EN" pos="6" type="boolean"/>
+			<bitfield name="CSC_EN" pos="8" type="boolean"/>
+			<bitfield name="CSC_SRC_DATA_FORMAT" low="9" high="9" type="uint"/>
+			<bitfield name="CSC_DST_DATA_FORMAT" low="10" high="10" type="uint"/>
+			<bitfield name="CHROMA_DWN_SAMPLE_EN" pos="11" type="boolean"/>
+			<bitfield name="CHROMA_DWN_SAMPLE_FORMAT" low="12" high="12" type="uint"/>
+			<bitfield name="CHROMA_DWN_SAMPLE_H_MTHD" low="13" high="13" type="uint"/>
+			<bitfield name="CHROMA_DWN_SAMPLE_V_MTHD" low="14" high="14" type="uint"/>
+		</reg32>
+		<reg32 offset="0x008" name="DST_PACK_PATTERN">
+			<bitfield name="ELEMENT0" low="0" high="1" type="uint"/>
+			<bitfield name="ELEMENT1" low="8" high="9" type="uint"/>
+			<bitfield name="ELEMENT2" low="16" high="17" type="uint"/>
+			<bitfield name="ELEMENT3" low="24" high="25" type="uint"/>
+		</reg32>
+		<reg32 offset="0x00c" name="DST0_ADDR"/>
+		<reg32 offset="0x010" name="DST1_ADDR"/>
+		<reg32 offset="0x014" name="DST2_ADDR"/>
+		<reg32 offset="0x018" name="DST3_ADDR"/>
+		<reg32 offset="0x01c" name="DST_YSTRIDE0">
+			<bitfield name="DST0_YSTRIDE" low="0" high="15" type="uint"/>
+			<bitfield name="DST1_YSTRIDE" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x020" name="DST_YSTRIDE1">
+			<bitfield name="DST2_YSTRIDE" low="0" high="15" type="uint"/>
+			<bitfield name="DST3_YSTRIDE" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x024" name="DST_DITHER_BITDEPTH"/>
+		<reg32 offset="0x030" name="DITHER_MATRIX_ROW0"/>
+		<reg32 offset="0x034" name="DITHER_MATRIX_ROW1"/>
+		<reg32 offset="0x038" name="DITHER_MATRIX_ROW2"/>
+		<reg32 offset="0x03c" name="DITHER_MATRIX_ROW3"/>
+		<reg32 offset="0x048" name="DST_WRITE_CONFIG"/>
+		<reg32 offset="0x050" name="ROTATION_DNSCALER"/>
+		<reg32 offset="0x060" name="N16_INIT_PHASE_X_0_3"/>
+		<reg32 offset="0x064" name="N16_INIT_PHASE_X_1_2"/>
+		<reg32 offset="0x068" name="N16_INIT_PHASE_Y_0_3"/>
+		<reg32 offset="0x06c" name="N16_INIT_PHASE_Y_1_2"/>
+		<reg32 offset="0x074" name="OUT_SIZE">
+			<bitfield name="DST_W" low="0" high="15" type="uint"/>
+			<bitfield name="DST_H" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x078" name="ALPHA_X_VALUE"/>
+		<reg32 offset="0x260" name="CSC_MATRIX_COEFF_0">
+			<bitfield name="COEFF_11" low="0" high="12" type="uint"/>
+			<bitfield name="COEFF_12" low="16" high="28" type="uint"/>
+		</reg32>
+		<reg32 offset="0x264" name="CSC_MATRIX_COEFF_1">
+			<bitfield name="COEFF_13" low="0" high="12" type="uint"/>
+			<bitfield name="COEFF_21" low="16" high="28" type="uint"/>
+		</reg32>
+		<reg32 offset="0x268" name="CSC_MATRIX_COEFF_2">
+			<bitfield name="COEFF_22" low="0" high="12" type="uint"/>
+			<bitfield name="COEFF_23" low="16" high="28" type="uint"/>
+		</reg32>
+		<reg32 offset="0x26c" name="CSC_MATRIX_COEFF_3">
+			<bitfield name="COEFF_31" low="0" high="12" type="uint"/>
+			<bitfield name="COEFF_32" low="16" high="28" type="uint"/>
+		</reg32>
+		<reg32 offset="0x270" name="CSC_MATRIX_COEFF_4">
+			<bitfield name="COEFF_33" low="0" high="12" type="uint"/>
+		</reg32>
+		<array offset="0x274" name="CSC_COMP_PRECLAMP" length="3" stride="4">
+			<reg32 offset="0" name="REG">
+				<bitfield name="HIGH"  low="0"  high="7"  type="uint"/>
+				<bitfield name="LOW"  low="8"  high="15"  type="uint"/>
+			</reg32>
+		</array>
+		<array offset="0x280" name="CSC_COMP_POSTCLAMP" length="3" stride="4">
+			<reg32 offset="0" name="REG">
+				<bitfield name="HIGH"  low="0"  high="7"  type="uint"/>
+				<bitfield name="LOW"  low="8"  high="15"  type="uint"/>
+			</reg32>
+		</array>
+		<array offset="0x28c" name="CSC_COMP_PREBIAS" length="3" stride="4">
+			<reg32 offset="0" name="REG">
+				<bitfield name="VALUE"  low="0"  high="8"  type="uint"/>
+			</reg32>
+		</array>
+		<array offset="0x298" name="CSC_COMP_POSTBIAS" length="3" stride="4">
+			<reg32 offset="0" name="REG">
+				<bitfield name="VALUE"  low="0"  high="8"  type="uint"/>
+			</reg32>
+		</array>
+	</array>
+
+	<array doffsets="mdp5_cfg->intf.base[0],mdp5_cfg->intf.base[1],mdp5_cfg->intf.base[2],mdp5_cfg->intf.base[3],mdp5_cfg->intf.base[4]" name="INTF" length="5" stride="0x200">
+		<reg32 offset="0x000" name="TIMING_ENGINE_EN"/>
+		<reg32 offset="0x004" name="CONFIG"/>
+		<reg32 offset="0x008" name="HSYNC_CTL">
+			<bitfield name="PULSEW" low="0" high="15" type="uint"/>
+			<bitfield name="PERIOD" low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x00c" name="VSYNC_PERIOD_F0" type="uint"/>
+		<reg32 offset="0x010" name="VSYNC_PERIOD_F1" type="uint"/>
+		<reg32 offset="0x014" name="VSYNC_LEN_F0" type="uint"/>
+		<reg32 offset="0x018" name="VSYNC_LEN_F1" type="uint"/>
+		<reg32 offset="0x01c" name="DISPLAY_VSTART_F0" type="uint"/>
+		<reg32 offset="0x020" name="DISPLAY_VSTART_F1" type="uint"/>
+		<reg32 offset="0x024" name="DISPLAY_VEND_F0" type="uint"/>
+		<reg32 offset="0x028" name="DISPLAY_VEND_F1" type="uint"/>
+		<reg32 offset="0x02c" name="ACTIVE_VSTART_F0">
+			<bitfield name="VAL" low="0" high="30" type="uint"/>
+			<bitfield name="ACTIVE_V_ENABLE" pos="31" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x030" name="ACTIVE_VSTART_F1">
+			<bitfield name="VAL" low="0" high="30" type="uint"/>
+		</reg32>
+		<reg32 offset="0x034" name="ACTIVE_VEND_F0" type="uint"/>
+		<reg32 offset="0x038" name="ACTIVE_VEND_F1" type="uint"/>
+		<reg32 offset="0x03c" name="DISPLAY_HCTL">
+			<bitfield name="START" low="0"  high="15" type="uint"/>
+			<bitfield name="END"   low="16" high="31" type="uint"/>
+		</reg32>
+		<reg32 offset="0x040" name="ACTIVE_HCTL">
+			<bitfield name="START" low="0"  high="14" type="uint"/>
+			<bitfield name="END"   low="16" high="30" type="uint"/>
+			<bitfield name="ACTIVE_H_ENABLE" pos="31" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x044" name="BORDER_COLOR"/>
+		<reg32 offset="0x048" name="UNDERFLOW_COLOR"/>
+		<reg32 offset="0x04c" name="HSYNC_SKEW"/>
+		<reg32 offset="0x050" name="POLARITY_CTL">
+			<bitfield name="HSYNC_LOW" pos="0" type="boolean"/>
+			<bitfield name="VSYNC_LOW" pos="1" type="boolean"/>
+			<bitfield name="DATA_EN_LOW" pos="2" type="boolean"/>
+		</reg32>
+		<reg32 offset="0x054" name="TEST_CTL"/>
+		<reg32 offset="0x058" name="TP_COLOR0"/>
+		<reg32 offset="0x05c" name="TP_COLOR1"/>
+		<reg32 offset="0x084" name="DSI_CMD_MODE_TRIGGER_EN"/>
+		<reg32 offset="0x090" name="PANEL_FORMAT" type="mdp5_format"/>
+		<reg32 offset="0x0a8" name="FRAME_LINE_COUNT_EN"/>
+		<reg32 offset="0x0ac" name="FRAME_COUNT"/>
+		<reg32 offset="0x0b0" name="LINE_COUNT"/>
+		<reg32 offset="0x0f0" name="DEFLICKER_CONFIG"/>
+		<reg32 offset="0x0f4" name="DEFLICKER_STRNG_COEFF"/>
+		<reg32 offset="0x0f8" name="DEFLICKER_WEAK_COEFF"/>
+		<reg32 offset="0x100" name="TPG_ENABLE"/>
+		<reg32 offset="0x104" name="TPG_MAIN_CONTROL"/>
+		<reg32 offset="0x108" name="TPG_VIDEO_CONFIG"/>
+		<reg32 offset="0x10c" name="TPG_COMPONENT_LIMITS"/>
+		<reg32 offset="0x110" name="TPG_RECTANGLE"/>
+		<reg32 offset="0x114" name="TPG_INITIAL_VALUE"/>
+		<reg32 offset="0x118" name="TPG_BLK_WHITE_PATTERN_FRAME"/>
+		<reg32 offset="0x11c" name="TPG_RGB_MAPPING"/>
+	</array>
+
+	<array doffsets="mdp5_cfg->ad.base[0],mdp5_cfg->ad.base[1]" name="AD" length="2" stride="0x200">
+		<reg32 offset="0x000" name="BYPASS"/>
+		<reg32 offset="0x004" name="CTRL_0"/>
+		<reg32 offset="0x008" name="CTRL_1"/>
+		<reg32 offset="0x00c" name="FRAME_SIZE"/>
+		<reg32 offset="0x010" name="CON_CTRL_0"/>
+		<reg32 offset="0x014" name="CON_CTRL_1"/>
+		<reg32 offset="0x018" name="STR_MAN"/>
+		<reg32 offset="0x01c" name="VAR"/>
+		<reg32 offset="0x020" name="DITH"/>
+		<reg32 offset="0x024" name="DITH_CTRL"/>
+		<reg32 offset="0x028" name="AMP_LIM"/>
+		<reg32 offset="0x02c" name="SLOPE"/>
+		<reg32 offset="0x030" name="BW_LVL"/>
+		<reg32 offset="0x034" name="LOGO_POS"/>
+		<reg32 offset="0x038" name="LUT_FI"/>
+		<reg32 offset="0x07c" name="LUT_CC"/>
+		<reg32 offset="0x0c8" name="STR_LIM"/>
+		<reg32 offset="0x0cc" name="CALIB_AB"/>
+		<reg32 offset="0x0d0" name="CALIB_CD"/>
+		<reg32 offset="0x0d4" name="MODE_SEL"/>
+		<reg32 offset="0x0d8" name="TFILT_CTRL"/>
+		<reg32 offset="0x0dc" name="BL_MINMAX"/>
+		<reg32 offset="0x0e0" name="BL"/>
+		<reg32 offset="0x0e8" name="BL_MAX"/>
+		<reg32 offset="0x0ec" name="AL"/>
+		<reg32 offset="0x0f0" name="AL_MIN"/>
+		<reg32 offset="0x0f4" name="AL_FILT"/>
+		<reg32 offset="0x0f8" name="CFG_BUF"/>
+		<reg32 offset="0x100" name="LUT_AL"/>
+		<reg32 offset="0x144" name="TARG_STR"/>
+		<reg32 offset="0x148" name="START_CALC"/>
+		<reg32 offset="0x14c" name="STR_OUT"/>
+		<reg32 offset="0x154" name="BL_OUT"/>
+		<reg32 offset="0x158" name="CALC_DONE"/>
+	</array>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/display/mdp_common.xml b/drivers/gpu/drm/msm/registers/display/mdp_common.xml
new file mode 100644
index 000000000000..f1b6345c1323
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/mdp_common.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+
+
+<!-- random bits that seem same between mdp4 and mdp5 (ie. not much) -->
+
+<enum name="mdp_chroma_samp_type">
+    <value name="CHROMA_FULL"  value="0"/>
+    <value name="CHROMA_H2V1" value="1"/>
+    <value name="CHROMA_H1V2" value="2"/>
+    <value name="CHROMA_420"  value="3"/>
+</enum>
+
+<enum name="mdp_fetch_type">
+    <value name="MDP_PLANE_INTERLEAVED"   value="0"/>
+    <value name="MDP_PLANE_PLANAR"        value="1"/>
+    <value name="MDP_PLANE_PSEUDO_PLANAR" value="2"/>
+</enum>
+
+<enum name="mdp_mixer_stage_id">
+	<value name="STAGE_UNUSED" value="0"/>
+	<value name="STAGE_BASE" value="1"/>
+	<value name="STAGE0" value="2"/>   <!-- zorder 0 -->
+	<value name="STAGE1" value="3"/>   <!-- zorder 1 -->
+	<value name="STAGE2" value="4"/>   <!-- zorder 2 -->
+	<value name="STAGE3" value="5"/>   <!-- zorder 3 -->
+	<value name="STAGE4" value="6"/>   <!-- zorder 4 -->
+	<value name="STAGE5" value="7"/>   <!-- zorder 5 -->
+	<value name="STAGE6" value="8"/>   <!-- zorder 6 -->
+	<value name="STAGE_MAX" value="8"/> <!-- maximum zorder -->
+</enum>
+
+<enum name="mdp_alpha_type">
+	<value name="FG_CONST" value="0"/>
+	<value name="BG_CONST" value="1"/>
+	<value name="FG_PIXEL" value="2"/>
+	<value name="BG_PIXEL" value="3"/>
+</enum>
+
+<enum name="mdp_component_type">
+        <value name="COMP_0" value="0"/>	<!-- Y component -->
+        <value name="COMP_1_2" value="1"/>	<!-- Cb/Cr comp. -->
+        <value name="COMP_3" value="2"/>	<!-- Trans comp. -->
+        <value name="COMP_MAX" value="3"/>
+</enum>
+
+<enum name="mdp_bpc">
+	<brief>bits per component (non-alpha channel)</brief>
+	<value name="BPC4" value="0"/> <!-- 4 bits -->
+	<value name="BPC5" value="1"/> <!-- 5 bits -->
+	<value name="BPC6" value="2"/> <!-- 6 bits -->
+	<value name="BPC8" value="3"/> <!-- 8 bits -->
+</enum>
+
+<enum name="mdp_bpc_alpha">
+	<brief>bits per component (alpha channel)</brief>
+	<value name="BPC1A" value="0"/> <!-- 1 bit -->
+	<value name="BPC4A" value="1"/> <!-- 4 bits -->
+	<value name="BPC6A" value="2"/> <!-- 6 bits -->
+	<value name="BPC8A" value="3"/> <!-- 8 bits -->
+</enum>
+
+<enum name="mdp_fetch_mode">
+	<value name="MDP_FETCH_LINEAR" value="0"/>
+	<value name="MDP_FETCH_TILE" value="1"/>
+	<value name="MDP_FETCH_UBWC" value="2"/>
+</enum>
+
+<bitset name="reg_wh" inline="yes">
+    <bitfield name="HEIGHT" low="16" high="31" type="uint"/>
+    <bitfield name="WIDTH" low="0" high="15" type="uint"/>
+</bitset>
+
+<bitset name="reg_xy" inline="yes">
+    <bitfield name="Y" low="16" high="31" type="uint"/>
+    <bitfield name="X" low="0" high="15" type="uint"/>
+</bitset>
+
+<bitset name="mdp_unpack_pattern" inline="yes">
+	<bitfield name="ELEM0" low="0"  high="7"/>
+	<bitfield name="ELEM1" low="8"  high="15"/>
+	<bitfield name="ELEM2" low="16" high="23"/>
+	<bitfield name="ELEM3" low="24" high="31"/>
+</bitset>
+
+</database>
+
diff --git a/drivers/gpu/drm/msm/registers/display/msm.xml b/drivers/gpu/drm/msm/registers/display/msm.xml
new file mode 100644
index 000000000000..429c35b73bad
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/msm.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="http://nouveau.freedesktop.org/ rules-ng.xsd">
+<import file="freedreno_copyright.xml"/>
+
+<doc>
+	Register definitions for the display related hw blocks on
+	msm/snapdragon
+</doc>
+
+<!--
+<enum name="chipset">
+	<value name="MDP40"/>
+	<value name="MDP50"/>
+</enum>
+-->
+
+<import file="mdp4.xml"/>
+<import file="mdp5.xml"/>
+<import file="dsi.xml"/>
+<import file="dsi_phy_28nm_8960.xml"/>
+<import file="dsi_phy_28nm.xml"/>
+<import file="dsi_phy_20nm.xml"/>
+<import file="dsi_phy_14nm.xml"/>
+<import file="dsi_phy_10nm.xml"/>
+<import file="dsi_phy_7nm.xml"/>
+<import file="sfpb.xml"/>
+<import file="hdmi.xml"/>
+<import file="edp.xml"/>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/display/sfpb.xml b/drivers/gpu/drm/msm/registers/display/sfpb.xml
new file mode 100644
index 000000000000..de1cf43c131f
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/display/sfpb.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+<import file="freedreno_copyright.xml"/>
+
+<domain name="SFPB" width="32">
+	<enum name="sfpb_ahb_arb_master_port_en">
+		<value name="SFPB_MASTER_PORT_ENABLE" value="3"/>
+		<value name="SFPB_MASTER_PORT_DISABLE" value="0"/>
+	</enum>
+	<reg32 offset="0x0058" name="GPREG">
+		<bitfield name="MASTER_PORT_EN" low="11" high="12" type="sfpb_ahb_arb_master_port_en"/>
+	</reg32>
+</domain>
+
+</database>
diff --git a/drivers/gpu/drm/msm/registers/freedreno_copyright.xml b/drivers/gpu/drm/msm/registers/freedreno_copyright.xml
new file mode 100644
index 000000000000..854efdd2e5fc
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/freedreno_copyright.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<database xmlns="http://nouveau.freedesktop.org/"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
+
+<copyright year="2013">
+
+<author name="Rob Clark" email="robdclark@gmail.com"><nick name="robclark"/>
+Initial Author.
+</author>
+
+<author name="Ilia Mirkin" email="imirkin@alum.mit.edu"><nick name="imirkin"/>
+many a3xx/a4xx contributions
+</author>
+
+<license>
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice (including the
+next paragraph) shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+</license>
+
+</copyright>
+</database>
+
diff --git a/drivers/gpu/drm/msm/registers/gen_header.py b/drivers/gpu/drm/msm/registers/gen_header.py
new file mode 100644
index 000000000000..3926485bb197
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/gen_header.py
@@ -0,0 +1,971 @@
+#!/usr/bin/python3
+#
+# Copyright © 2019-2024 Google, Inc.
+#
+# SPDX-License-Identifier: MIT
+
+import xml.parsers.expat
+import sys
+import os
+import collections
+import argparse
+import time
+import datetime
+
+class Error(Exception):
+	def __init__(self, message):
+		self.message = message
+
+class Enum(object):
+	def __init__(self, name):
+		self.name = name
+		self.values = []
+
+	def has_name(self, name):
+		for (n, value) in self.values:
+			if n == name:
+				return True
+		return False
+
+	def names(self):
+		return [n for (n, value) in self.values]
+
+	def dump(self):
+		use_hex = False
+		for (name, value) in self.values:
+			if value > 0x1000:
+				use_hex = True
+
+		print("enum %s {" % self.name)
+		for (name, value) in self.values:
+			if use_hex:
+				print("\t%s = 0x%08x," % (name, value))
+			else:
+				print("\t%s = %d," % (name, value))
+		print("};\n")
+
+	def dump_pack_struct(self):
+		pass
+
+class Field(object):
+	def __init__(self, name, low, high, shr, type, parser):
+		self.name = name
+		self.low = low
+		self.high = high
+		self.shr = shr
+		self.type = type
+
+		builtin_types = [ None, "a3xx_regid", "boolean", "uint", "hex", "int", "fixed", "ufixed", "float", "address", "waddress" ]
+
+		maxpos = parser.current_bitsize - 1
+
+		if low < 0 or low > maxpos:
+			raise parser.error("low attribute out of range: %d" % low)
+		if high < 0 or high > maxpos:
+			raise parser.error("high attribute out of range: %d" % high)
+		if high < low:
+			raise parser.error("low is greater than high: low=%d, high=%d" % (low, high))
+		if self.type == "boolean" and not low == high:
+			raise parser.error("booleans should be 1 bit fields")
+		elif self.type == "float" and not (high - low == 31 or high - low == 15):
+			raise parser.error("floats should be 16 or 32 bit fields")
+		elif not self.type in builtin_types and not self.type in parser.enums:
+			raise parser.error("unknown type '%s'" % self.type)
+
+	def ctype(self, var_name):
+		if self.type == None:
+			type = "uint32_t"
+			val = var_name
+		elif self.type == "boolean":
+			type = "bool"
+			val = var_name
+		elif self.type == "uint" or self.type == "hex" or self.type == "a3xx_regid":
+			type = "uint32_t"
+			val = var_name
+		elif self.type == "int":
+			type = "int32_t"
+			val = var_name
+		elif self.type == "fixed":
+			type = "float"
+			val = "((int32_t)(%s * %d.0))" % (var_name, 1 << self.radix)
+		elif self.type == "ufixed":
+			type = "float"
+			val = "((uint32_t)(%s * %d.0))" % (var_name, 1 << self.radix)
+		elif self.type == "float" and self.high - self.low == 31:
+			type = "float"
+			val = "fui(%s)" % var_name
+		elif self.type == "float" and self.high - self.low == 15:
+			type = "float"
+			val = "_mesa_float_to_half(%s)" % var_name
+		elif self.type in [ "address", "waddress" ]:
+			type = "uint64_t"
+			val = var_name
+		else:
+			type = "enum %s" % self.type
+			val = var_name
+
+		if self.shr > 0:
+			val = "(%s >> %d)" % (val, self.shr)
+
+		return (type, val)
+
+def tab_to(name, value):
+	tab_count = (68 - (len(name) & ~7)) // 8
+	if tab_count <= 0:
+		tab_count = 1
+	print(name + ('\t' * tab_count) + value)
+
+def mask(low, high):
+	return ((0xffffffffffffffff >> (64 - (high + 1 - low))) << low)
+
+def field_name(reg, f):
+	if f.name:
+		name = f.name.lower()
+	else:
+		# We hit this path when a reg is defined with no bitset fields, ie.
+		# 	<reg32 offset="0x88db" name="RB_BLIT_DST_ARRAY_PITCH" low="0" high="28" shr="6" type="uint"/>
+		name = reg.name.lower()
+
+	if (name in [ "double", "float", "int" ]) or not (name[0].isalpha()):
+			name = "_" + name
+
+	return name
+
+# indices - array of (ctype, stride, __offsets_NAME)
+def indices_varlist(indices):
+	return ", ".join(["i%d" % i for i in range(len(indices))])
+
+def indices_prototype(indices):
+	return ", ".join(["%s i%d" % (ctype, idx)
+			for (idx, (ctype, stride, offset)) in  enumerate(indices)])
+
+def indices_strides(indices):
+	return " + ".join(["0x%x*i%d" % (stride, idx)
+					if stride else
+					"%s(i%d)" % (offset, idx)
+			for (idx, (ctype, stride, offset)) in  enumerate(indices)])
+
+class Bitset(object):
+	def __init__(self, name, template):
+		self.name = name
+		self.inline = False
+		if template:
+			self.fields = template.fields[:]
+		else:
+			self.fields = []
+
+	# Get address field if there is one in the bitset, else return None:
+	def get_address_field(self):
+		for f in self.fields:
+			if f.type in [ "address", "waddress" ]:
+				return f
+		return None
+
+	def dump_regpair_builder(self, reg):
+		print("#ifndef NDEBUG")
+		known_mask = 0
+		for f in self.fields:
+			known_mask |= mask(f.low, f.high)
+			if f.type in [ "boolean", "address", "waddress" ]:
+				continue
+			type, val = f.ctype("fields.%s" % field_name(reg, f))
+			print("    assert((%-40s & 0x%08x) == 0);" % (val, 0xffffffff ^ mask(0 , f.high - f.low)))
+		print("    assert((%-40s & 0x%08x) == 0);" % ("fields.unknown", known_mask))
+		print("#endif\n")
+
+		print("    return (struct fd_reg_pair) {")
+		if reg.array:
+			print("        .reg = REG_%s(__i)," % reg.full_name)
+		else:
+			print("        .reg = REG_%s," % reg.full_name)
+
+		print("        .value =")
+		for f in self.fields:
+			if f.type in [ "address", "waddress" ]:
+				continue
+			else:
+				type, val = f.ctype("fields.%s" % field_name(reg, f))
+				print("            (%-40s << %2d) |" % (val, f.low))
+		value_name = "dword"
+		if reg.bit_size == 64:
+			value_name = "qword"
+		print("            fields.unknown | fields.%s," % (value_name,))
+
+		address = self.get_address_field()
+		if address:
+			print("        .bo = fields.bo,")
+			print("        .is_address = true,")
+			if f.type == "waddress":
+				print("        .bo_write = true,")
+			print("        .bo_offset = fields.bo_offset,")
+			print("        .bo_shift = %d," % address.shr)
+			print("        .bo_low = %d," % address.low)
+
+		print("    };")
+
+	def dump_pack_struct(self, reg=None):
+		if not reg:
+			return
+
+		prefix = reg.full_name
+
+		print("struct %s {" % prefix)
+		for f in self.fields:
+			if f.type in [ "address", "waddress" ]:
+				tab_to("    __bo_type", "bo;")
+				tab_to("    uint32_t", "bo_offset;")
+				continue
+			name = field_name(reg, f)
+
+			type, val = f.ctype("var")
+
+			tab_to("    %s" % type, "%s;" % name)
+		if reg.bit_size == 64:
+			tab_to("    uint64_t", "unknown;")
+			tab_to("    uint64_t", "qword;")
+		else:
+			tab_to("    uint32_t", "unknown;")
+			tab_to("    uint32_t", "dword;")
+		print("};\n")
+
+		if reg.array:
+			print("static inline struct fd_reg_pair\npack_%s(uint32_t __i, struct %s fields)\n{" %
+				  (prefix, prefix))
+		else:
+			print("static inline struct fd_reg_pair\npack_%s(struct %s fields)\n{" %
+				  (prefix, prefix))
+
+		self.dump_regpair_builder(reg)
+
+		print("\n}\n")
+
+		if self.get_address_field():
+			skip = ", { .reg = 0 }"
+		else:
+			skip = ""
+
+		if reg.array:
+			print("#define %s(__i, ...) pack_%s(__i, __struct_cast(%s) { __VA_ARGS__ })%s\n" %
+				  (prefix, prefix, prefix, skip))
+		else:
+			print("#define %s(...) pack_%s(__struct_cast(%s) { __VA_ARGS__ })%s\n" %
+				  (prefix, prefix, prefix, skip))
+
+
+	def dump(self, prefix=None):
+		if prefix == None:
+			prefix = self.name
+		for f in self.fields:
+			if f.name:
+				name = prefix + "_" + f.name
+			else:
+				name = prefix
+
+			if not f.name and f.low == 0 and f.shr == 0 and not f.type in ["float", "fixed", "ufixed"]:
+				pass
+			elif f.type == "boolean" or (f.type == None and f.low == f.high):
+				tab_to("#define %s" % name, "0x%08x" % (1 << f.low))
+			else:
+				tab_to("#define %s__MASK" % name, "0x%08x" % mask(f.low, f.high))
+				tab_to("#define %s__SHIFT" % name, "%d" % f.low)
+				type, val = f.ctype("val")
+
+				print("static inline uint32_t %s(%s val)\n{" % (name, type))
+				if f.shr > 0:
+					print("\tassert(!(val & 0x%x));" % mask(0, f.shr - 1))
+				print("\treturn ((%s) << %s__SHIFT) & %s__MASK;\n}" % (val, name, name))
+		print()
+
+class Array(object):
+	def __init__(self, attrs, domain, variant, parent, index_type):
+		if "name" in attrs:
+			self.local_name = attrs["name"]
+		else:
+			self.local_name = ""
+		self.domain = domain
+		self.variant = variant
+		self.parent = parent
+		if self.parent:
+			self.name = self.parent.name + "_" + self.local_name
+		else:
+			self.name = self.local_name
+		if "offsets" in attrs:
+			self.offsets = map(lambda i: "0x%08x" % int(i, 0), attrs["offsets"].split(","))
+			self.fixed_offsets = True
+		elif "doffsets" in attrs:
+			self.offsets = map(lambda s: "(%s)" % s , attrs["doffsets"].split(","))
+			self.fixed_offsets = True
+		else:
+			self.offset = int(attrs["offset"], 0)
+			self.stride = int(attrs["stride"], 0)
+			self.fixed_offsets = False
+		if "index" in attrs:
+			self.index_type = index_type
+		else:
+			self.index_type = None
+		self.length = int(attrs["length"], 0)
+		if "usage" in attrs:
+			self.usages = attrs["usage"].split(',')
+		else:
+			self.usages = None
+
+	def index_ctype(self):
+		if not self.index_type:
+			return "uint32_t"
+		else:
+			return "enum %s" % self.index_type.name
+
+	# Generate array of (ctype, stride, __offsets_NAME)
+	def indices(self):
+		if self.parent:
+			indices = self.parent.indices()
+		else:
+			indices = []
+		if self.length != 1:
+			if self.fixed_offsets:
+				indices.append((self.index_ctype(), None, "__offset_%s" % self.local_name))
+			else:
+				indices.append((self.index_ctype(), self.stride, None))
+		return indices
+
+	def total_offset(self):
+		offset = 0
+		if not self.fixed_offsets:
+			offset += self.offset
+		if self.parent:
+			offset += self.parent.total_offset()
+		return offset
+
+	def dump(self):
+		proto = indices_varlist(self.indices())
+		strides = indices_strides(self.indices())
+		array_offset = self.total_offset()
+		if self.fixed_offsets:
+			print("static inline uint32_t __offset_%s(%s idx)" % (self.local_name, self.index_ctype()))
+			print("{\n\tswitch (idx) {")
+			if self.index_type:
+				for val, offset in zip(self.index_type.names(), self.offsets):
+					print("\t\tcase %s: return %s;" % (val, offset))
+			else:
+				for idx, offset in enumerate(self.offsets):
+					print("\t\tcase %d: return %s;" % (idx, offset))
+			print("\t\tdefault: return INVALID_IDX(idx);")
+			print("\t}\n}")
+		if proto == '':
+			tab_to("#define REG_%s_%s" % (self.domain, self.name), "0x%08x\n" % array_offset)
+		else:
+			tab_to("#define REG_%s_%s(%s)" % (self.domain, self.name, proto), "(0x%08x + %s )\n" % (array_offset, strides))
+
+	def dump_pack_struct(self):
+		pass
+
+	def dump_regpair_builder(self):
+		pass
+
+class Reg(object):
+	def __init__(self, attrs, domain, array, bit_size):
+		self.name = attrs["name"]
+		self.domain = domain
+		self.array = array
+		self.offset = int(attrs["offset"], 0)
+		self.type = None
+		self.bit_size = bit_size
+		if array:
+			self.name = array.name + "_" + self.name
+		self.full_name = self.domain + "_" + self.name
+		if "stride" in attrs:
+			self.stride = int(attrs["stride"], 0)
+			self.length = int(attrs["length"], 0)
+		else:
+			self.stride = None
+			self.length = None
+
+	# Generate array of (ctype, stride, __offsets_NAME)
+	def indices(self):
+		if self.array:
+			indices = self.array.indices()
+		else:
+			indices = []
+		if self.stride:
+			indices.append(("uint32_t", self.stride, None))
+		return indices
+
+	def total_offset(self):
+		if self.array:
+			return self.array.total_offset() + self.offset
+		else:
+			return self.offset
+
+	def dump(self):
+		proto = indices_prototype(self.indices())
+		strides = indices_strides(self.indices())
+		offset = self.total_offset()
+		if proto == '':
+			tab_to("#define REG_%s" % self.full_name, "0x%08x" % offset)
+		else:
+			print("static inline uint32_t REG_%s(%s) { return 0x%08x + %s; }" % (self.full_name, proto, offset, strides))
+
+		if self.bitset.inline:
+			self.bitset.dump(self.full_name)
+
+	def dump_pack_struct(self):
+		if self.bitset.inline:
+			self.bitset.dump_pack_struct(self)
+
+	def dump_regpair_builder(self):
+		if self.bitset.inline:
+			self.bitset.dump_regpair_builder(self)
+
+	def dump_py(self):
+		print("\tREG_%s = 0x%08x" % (self.full_name, self.offset))
+
+
+class Parser(object):
+	def __init__(self):
+		self.current_array = None
+		self.current_domain = None
+		self.current_prefix = None
+		self.current_prefix_type = None
+		self.current_stripe = None
+		self.current_bitset = None
+		self.current_bitsize = 32
+		# The varset attribute on the domain specifies the enum which
+		# specifies all possible hw variants:
+		self.current_varset = None
+		# Regs that have multiple variants.. we only generated the C++
+		# template based struct-packers for these
+		self.variant_regs = {}
+		# Information in which contexts regs are used, to be used in
+		# debug options
+		self.usage_regs = collections.defaultdict(list)
+		self.bitsets = {}
+		self.enums = {}
+		self.variants = set()
+		self.file = []
+		self.xml_files = []
+		self.copyright_year = None
+		self.authors = []
+		self.license = None
+
+	def error(self, message):
+		parser, filename = self.stack[-1]
+		return Error("%s:%d:%d: %s" % (filename, parser.CurrentLineNumber, parser.CurrentColumnNumber, message))
+
+	def prefix(self, variant=None):
+		if self.current_prefix_type == "variant" and variant:
+			return variant
+		elif self.current_stripe:
+			return self.current_stripe + "_" + self.current_domain
+		elif self.current_prefix:
+			return self.current_prefix + "_" + self.current_domain
+		else:
+			return self.current_domain
+
+	def parse_field(self, name, attrs):
+		try:
+			if "pos" in attrs:
+				high = low = int(attrs["pos"], 0)
+			elif "high" in attrs and "low" in attrs:
+				high = int(attrs["high"], 0)
+				low = int(attrs["low"], 0)
+			else:
+				low = 0
+				high = self.current_bitsize - 1
+
+			if "type" in attrs:
+				type = attrs["type"]
+			else:
+				type = None
+
+			if "shr" in attrs:
+				shr = int(attrs["shr"], 0)
+			else:
+				shr = 0
+
+			b = Field(name, low, high, shr, type, self)
+
+			if type == "fixed" or type == "ufixed":
+				b.radix = int(attrs["radix"], 0)
+
+			self.current_bitset.fields.append(b)
+		except ValueError as e:
+			raise self.error(e)
+
+	def parse_varset(self, attrs):
+		# Inherit the varset from the enclosing domain if not overriden:
+		varset = self.current_varset
+		if "varset" in attrs:
+			varset = self.enums[attrs["varset"]]
+		return varset
+
+	def parse_variants(self, attrs):
+		if not "variants" in attrs:
+				return None
+		variant = attrs["variants"].split(",")[0]
+		if "-" in variant:
+			variant = variant[:variant.index("-")]
+
+		varset = self.parse_varset(attrs)
+
+		assert varset.has_name(variant)
+
+		return variant
+
+	def add_all_variants(self, reg, attrs, parent_variant):
+		# TODO this should really handle *all* variants, including dealing
+		# with open ended ranges (ie. "A2XX,A4XX-") (we have the varset
+		# enum now to make that possible)
+		variant = self.parse_variants(attrs)
+		if not variant:
+			variant = parent_variant
+
+		if reg.name not in self.variant_regs:
+			self.variant_regs[reg.name] = {}
+		else:
+			# All variants must be same size:
+			v = next(iter(self.variant_regs[reg.name]))
+			assert self.variant_regs[reg.name][v].bit_size == reg.bit_size
+
+		self.variant_regs[reg.name][variant] = reg
+
+	def add_all_usages(self, reg, usages):
+		if not usages:
+			return
+
+		for usage in usages:
+			self.usage_regs[usage].append(reg)
+
+		self.variants.add(reg.domain)
+
+	def do_validate(self, schemafile):
+		if not self.validate:
+			return
+
+		try:
+			from lxml import etree
+
+			parser, filename = self.stack[-1]
+			dirname = os.path.dirname(filename)
+
+			# we expect this to look like <namespace url> schema.xsd.. I think
+			# technically it is supposed to be just a URL, but that doesn't
+			# quite match up to what we do.. Just skip over everything up to
+			# and including the first whitespace character:
+			schemafile = schemafile[schemafile.rindex(" ")+1:]
+
+			# this is a bit cheezy, but the xml file to validate could be
+			# in a child director, ie. we don't really know where the schema
+			# file is, the way the rnn C code does.  So if it doesn't exist
+			# just look one level up
+			if not os.path.exists(dirname + "/" + schemafile):
+				schemafile = "../" + schemafile
+
+			if not os.path.exists(dirname + "/" + schemafile):
+				raise self.error("Cannot find schema for: " + filename)
+
+			xmlschema_doc = etree.parse(dirname + "/" + schemafile)
+			xmlschema = etree.XMLSchema(xmlschema_doc)
+
+			xml_doc = etree.parse(filename)
+			if not xmlschema.validate(xml_doc):
+				error_str = str(xmlschema.error_log.filter_from_errors()[0])
+				raise self.error("Schema validation failed for: " + filename + "\n" + error_str)
+		except ImportError as e:
+			if self.validate:
+				raise e
+
+			print("lxml not found, skipping validation", file=sys.stderr)
+
+	def do_parse(self, filename):
+		filepath = os.path.abspath(filename)
+		if filepath in self.xml_files:
+			return
+		self.xml_files.append(filepath)
+		file = open(filename, "rb")
+		parser = xml.parsers.expat.ParserCreate()
+		self.stack.append((parser, filename))
+		parser.StartElementHandler = self.start_element
+		parser.EndElementHandler = self.end_element
+		parser.CharacterDataHandler = self.character_data
+		parser.buffer_text = True
+		parser.ParseFile(file)
+		self.stack.pop()
+		file.close()
+
+	def parse(self, rnn_path, filename, validate):
+		self.path = rnn_path
+		self.stack = []
+		self.validate = validate
+		self.do_parse(filename)
+
+	def parse_reg(self, attrs, bit_size):
+		self.current_bitsize = bit_size
+		if "type" in attrs and attrs["type"] in self.bitsets:
+			bitset = self.bitsets[attrs["type"]]
+			if bitset.inline:
+				self.current_bitset = Bitset(attrs["name"], bitset)
+				self.current_bitset.inline = True
+			else:
+				self.current_bitset = bitset
+		else:
+			self.current_bitset = Bitset(attrs["name"], None)
+			self.current_bitset.inline = True
+			if "type" in attrs:
+				self.parse_field(None, attrs)
+
+		variant = self.parse_variants(attrs)
+		if not variant and self.current_array:
+			variant = self.current_array.variant
+
+		self.current_reg = Reg(attrs, self.prefix(variant), self.current_array, bit_size)
+		self.current_reg.bitset = self.current_bitset
+
+		if len(self.stack) == 1:
+			self.file.append(self.current_reg)
+
+		if variant is not None:
+			self.add_all_variants(self.current_reg, attrs, variant)
+
+		usages = None
+		if "usage" in attrs:
+			usages = attrs["usage"].split(',')
+		elif self.current_array:
+			usages = self.current_array.usages
+
+		self.add_all_usages(self.current_reg, usages)
+
+	def start_element(self, name, attrs):
+		self.cdata = ""
+		if name == "import":
+			filename = attrs["file"]
+			self.do_parse(os.path.join(self.path, filename))
+		elif name == "domain":
+			self.current_domain = attrs["name"]
+			if "prefix" in attrs:
+				self.current_prefix = self.parse_variants(attrs)
+				self.current_prefix_type = attrs["prefix"]
+			else:
+				self.current_prefix = None
+				self.current_prefix_type = None
+			if "varset" in attrs:
+				self.current_varset = self.enums[attrs["varset"]]
+		elif name == "stripe":
+			self.current_stripe = self.parse_variants(attrs)
+		elif name == "enum":
+			self.current_enum_value = 0
+			self.current_enum = Enum(attrs["name"])
+			self.enums[attrs["name"]] = self.current_enum
+			if len(self.stack) == 1:
+				self.file.append(self.current_enum)
+		elif name == "value":
+			if "value" in attrs:
+				value = int(attrs["value"], 0)
+			else:
+				value = self.current_enum_value
+			self.current_enum.values.append((attrs["name"], value))
+		elif name == "reg32":
+			self.parse_reg(attrs, 32)
+		elif name == "reg64":
+			self.parse_reg(attrs, 64)
+		elif name == "array":
+			self.current_bitsize = 32
+			variant = self.parse_variants(attrs)
+			index_type = self.enums[attrs["index"]] if "index" in attrs else None
+			self.current_array = Array(attrs, self.prefix(variant), variant, self.current_array, index_type)
+			if len(self.stack) == 1:
+				self.file.append(self.current_array)
+		elif name == "bitset":
+			self.current_bitset = Bitset(attrs["name"], None)
+			if "inline" in attrs and attrs["inline"] == "yes":
+				self.current_bitset.inline = True
+			self.bitsets[self.current_bitset.name] = self.current_bitset
+			if len(self.stack) == 1 and not self.current_bitset.inline:
+				self.file.append(self.current_bitset)
+		elif name == "bitfield" and self.current_bitset:
+			self.parse_field(attrs["name"], attrs)
+		elif name == "database":
+			self.do_validate(attrs["xsi:schemaLocation"])
+		elif name == "copyright":
+			self.copyright_year = attrs["year"]
+		elif name == "author":
+			self.authors.append(attrs["name"] + " <" + attrs["email"] + "> " + attrs["name"])
+
+	def end_element(self, name):
+		if name == "domain":
+			self.current_domain = None
+			self.current_prefix = None
+			self.current_prefix_type = None
+		elif name == "stripe":
+			self.current_stripe = None
+		elif name == "bitset":
+			self.current_bitset = None
+		elif name == "reg32":
+			self.current_reg = None
+		elif name == "array":
+			self.current_array = self.current_array.parent
+		elif name == "enum":
+			self.current_enum = None
+		elif name == "license":
+			self.license = self.cdata
+
+	def character_data(self, data):
+		self.cdata += data
+
+	def dump_reg_usages(self):
+		d = collections.defaultdict(list)
+		for usage, regs in self.usage_regs.items():
+			for reg in regs:
+				variants = self.variant_regs.get(reg.name)
+				if variants:
+					for variant, vreg in variants.items():
+						if reg == vreg:
+							d[(usage, variant)].append(reg)
+				else:
+					for variant in self.variants:
+						d[(usage, variant)].append(reg)
+
+		print("#ifdef __cplusplus")
+
+		for usage, regs in self.usage_regs.items():
+			print("template<chip CHIP> constexpr inline uint16_t %s_REGS[] = {};" % (usage.upper()))
+
+		for (usage, variant), regs in d.items():
+			offsets = []
+
+			for reg in regs:
+				if reg.array:
+					for i in range(reg.array.length):
+						offsets.append(reg.array.offset + reg.offset + i * reg.array.stride)
+						if reg.bit_size == 64:
+							offsets.append(offsets[-1] + 1)
+				else:
+					offsets.append(reg.offset)
+					if reg.bit_size == 64:
+						offsets.append(offsets[-1] + 1)
+
+			offsets.sort()
+
+			print("template<> constexpr inline uint16_t %s_REGS<%s>[] = {" % (usage.upper(), variant))
+			for offset in offsets:
+				print("\t%s," % hex(offset))
+			print("};")
+
+		print("#endif")
+
+	def dump(self):
+		enums = []
+		bitsets = []
+		regs = []
+		for e in self.file:
+			if isinstance(e, Enum):
+				enums.append(e)
+			elif isinstance(e, Bitset):
+				bitsets.append(e)
+			else:
+				regs.append(e)
+
+		for e in enums + bitsets + regs:
+			e.dump()
+
+		self.dump_reg_usages()
+
+
+	def dump_regs_py(self):
+		regs = []
+		for e in self.file:
+			if isinstance(e, Reg):
+				regs.append(e)
+
+		for e in regs:
+			e.dump_py()
+
+
+	def dump_reg_variants(self, regname, variants):
+		# Don't bother for things that only have a single variant:
+		if len(variants) == 1:
+			return
+		print("#ifdef __cplusplus")
+		print("struct __%s {" % regname)
+		# TODO be more clever.. we should probably figure out which
+		# fields have the same type in all variants (in which they
+		# appear) and stuff everything else in a variant specific
+		# sub-structure.
+		seen_fields = []
+		bit_size = 32
+		array = False
+		address = None
+		for variant in variants.keys():
+			print("    /* %s fields: */" % variant)
+			reg = variants[variant]
+			bit_size = reg.bit_size
+			array = reg.array
+			for f in reg.bitset.fields:
+				fld_name = field_name(reg, f)
+				if fld_name in seen_fields:
+					continue
+				seen_fields.append(fld_name)
+				name = fld_name.lower()
+				if f.type in [ "address", "waddress" ]:
+					if address:
+						continue
+					address = f
+					tab_to("    __bo_type", "bo;")
+					tab_to("    uint32_t", "bo_offset;")
+					continue
+				type, val = f.ctype("var")
+				tab_to("    %s" %type, "%s;" %name)
+		print("    /* fallback fields: */")
+		if bit_size == 64:
+			tab_to("    uint64_t", "unknown;")
+			tab_to("    uint64_t", "qword;")
+		else:
+			tab_to("    uint32_t", "unknown;")
+			tab_to("    uint32_t", "dword;")
+		print("};")
+		# TODO don't hardcode the varset enum name
+		varenum = "chip"
+		print("template <%s %s>" % (varenum, varenum.upper()))
+		print("static inline struct fd_reg_pair")
+		xtra = ""
+		xtravar = ""
+		if array:
+			xtra = "int __i, "
+			xtravar = "__i, "
+		print("__%s(%sstruct __%s fields) {" % (regname, xtra, regname))
+		for variant in variants.keys():
+			print("  if (%s == %s) {" % (varenum.upper(), variant))
+			reg = variants[variant]
+			reg.dump_regpair_builder()
+			print("  } else")
+		print("    assert(!\"invalid variant\");")
+		print("}")
+
+		if bit_size == 64:
+			skip = ", { .reg = 0 }"
+		else:
+			skip = ""
+
+		print("#define %s(VARIANT, %s...) __%s<VARIANT>(%s{__VA_ARGS__})%s" % (regname, xtravar, regname, xtravar, skip))
+		print("#endif /* __cplusplus */")
+
+	def dump_structs(self):
+		for e in self.file:
+			e.dump_pack_struct()
+
+		for regname in self.variant_regs:
+			self.dump_reg_variants(regname, self.variant_regs[regname])
+
+
+def dump_c(args, guard, func):
+	p = Parser()
+
+	try:
+		p.parse(args.rnn, args.xml, args.validate)
+	except Error as e:
+		print(e, file=sys.stderr)
+		exit(1)
+
+	print("#ifndef %s\n#define %s\n" % (guard, guard))
+
+	print("""/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng gen_header.py tool in this git repository:
+http://gitlab.freedesktop.org/mesa/mesa/
+git clone https://gitlab.freedesktop.org/mesa/mesa.git
+
+The rules-ng-ng source files this header was generated from are:
+""")
+	maxlen = 0
+	for filepath in p.xml_files:
+		maxlen = max(maxlen, len(filepath))
+	for filepath in p.xml_files:
+		pad = " " * (maxlen - len(filepath))
+		filesize = str(os.path.getsize(filepath))
+		filesize = " " * (7 - len(filesize)) + filesize
+		filetime = time.ctime(os.path.getmtime(filepath))
+		print("- " + filepath + pad + " (" + filesize + " bytes, from " + filetime + ")")
+	if p.copyright_year:
+		current_year = str(datetime.date.today().year)
+		print()
+		print("Copyright (C) %s-%s by the following authors:" % (p.copyright_year, current_year))
+		for author in p.authors:
+			print("- " + author)
+	if p.license:
+		print(p.license)
+	print("*/")
+
+	print()
+	print("#ifdef __KERNEL__")
+	print("#include <linux/bug.h>")
+	print("#define assert(x) BUG_ON(!(x))")
+	print("#else")
+	print("#include <assert.h>")
+	print("#endif")
+	print()
+
+	print("#ifdef __cplusplus")
+	print("#define __struct_cast(X)")
+	print("#else")
+	print("#define __struct_cast(X) (struct X)")
+	print("#endif")
+	print()
+
+	func(p)
+
+	print("\n#endif /* %s */" % guard)
+
+
+def dump_c_defines(args):
+	guard = str.replace(os.path.basename(args.xml), '.', '_').upper()
+	dump_c(args, guard, lambda p: p.dump())
+
+
+def dump_c_pack_structs(args):
+	guard = str.replace(os.path.basename(args.xml), '.', '_').upper() + '_STRUCTS'
+	dump_c(args, guard, lambda p: p.dump_structs())
+
+
+def dump_py_defines(args):
+	p = Parser()
+
+	try:
+		p.parse(args.rnn, args.xml)
+	except Error as e:
+		print(e, file=sys.stderr)
+		exit(1)
+
+	file_name = os.path.splitext(os.path.basename(args.xml))[0]
+
+	print("from enum import IntEnum")
+	print("class %sRegs(IntEnum):" % file_name.upper())
+
+	os.path.basename(args.xml)
+
+	p.dump_regs_py()
+
+
+def main():
+	parser = argparse.ArgumentParser()
+	parser.add_argument('--rnn', type=str, required=True)
+	parser.add_argument('--xml', type=str, required=True)
+	parser.add_argument('--validate', default=False, action='store_true')
+	parser.add_argument('--no-validate', dest='validate', action='store_false')
+
+	subparsers = parser.add_subparsers()
+	subparsers.required = True
+
+	parser_c_defines = subparsers.add_parser('c-defines')
+	parser_c_defines.set_defaults(func=dump_c_defines)
+
+	parser_c_pack_structs = subparsers.add_parser('c-pack-structs')
+	parser_c_pack_structs.set_defaults(func=dump_c_pack_structs)
+
+	parser_py_defines = subparsers.add_parser('py-defines')
+	parser_py_defines.set_defaults(func=dump_py_defines)
+
+	args = parser.parse_args()
+	args.func(args)
+
+
+if __name__ == '__main__':
+	main()
diff --git a/drivers/gpu/drm/msm/registers/rules-fd.xsd b/drivers/gpu/drm/msm/registers/rules-fd.xsd
new file mode 100644
index 000000000000..2eedb099a4eb
--- /dev/null
+++ b/drivers/gpu/drm/msm/registers/rules-fd.xsd
@@ -0,0 +1,404 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<schema xmlns="http://www.w3.org/2001/XMLSchema"
+	targetNamespace="http://nouveau.freedesktop.org/"
+	xmlns:rng="http://nouveau.freedesktop.org/"
+	elementFormDefault="qualified">
+
+	<annotation>
+		<documentation>
+			An updated version of the old rules.xml file from the
+			RivaTV project. Specifications by Pekka Paalanen,
+			preliminary attempt by KoalaBR,
+			first working version by Jakob Bornecrantz.
+			For specifications, see the file rules-ng-format.txt
+			in Nouveau CVS module 'rules-ng'.
+		</documentation>
+		<documentation>Version 0.1</documentation>
+	</annotation>
+
+
+	<!-- Elements -->
+
+	<element name="database"       type="rng:databaseType" />
+	<element name="import"         type="rng:importType" />
+	<element name="copyright"      type="rng:copyrightType" />
+	<element name="domain"         type="rng:domainType" />
+	<element name="array"          type="rng:arrayType" />
+	<element name="stripe"         type="rng:stripeType" />
+	<element name="reg64"          type="rng:registerType" />
+	<element name="reg32"          type="rng:registerType" />
+	<element name="bitset"         type="rng:bitsetType" />
+	<element name="bitfield"       type="rng:bitfieldType" />
+	<element name="enum"           type="rng:enumType" />
+	<element name="value"          type="rng:valueType" />
+
+	<!-- Copyright elements -->
+	<element name="author"         type="rng:authorType" />
+	<element name="nick"           type="rng:nickType" />
+	<element name="license"        type="rng:docType" />
+
+	<!-- Documentation elements -->
+	
+	<!-- FIXME: allowed only one  per parent element -->
+	<element name="brief" type="rng:briefType" />
+	
+	<element name="doc"  type="rng:docType" />
+	<element name="b"    type="rng:textformatType" />
+	<element name="i"    type="rng:textformatType" />
+	<element name="u"    type="rng:textformatType" />
+	<element name="code" type="rng:textcodeType" />
+	<element name="ul"   type="rng:listType" />
+	<element name="ol"   type="rng:listType" />
+	<element name="li"   type="rng:listitemType" />
+
+	<!-- Copyright element types -->
+
+	<complexType name="authorType" mixed="true">
+		<annotation>
+			<documentation>
+				register database author
+			</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<element ref="rng:nick" />
+		</choice>
+		<attribute name="name" type="string" use="required" />
+		<attribute name="email" type="string" use="required" />
+	</complexType>
+
+	<complexType name="nickType">
+		<annotation>
+			<documentation>nickType</documentation>
+		</annotation>
+		<attribute name="name" type="string" use="required" />
+	</complexType>
+
+	<!-- Database element types -->
+
+	<complexType name="databaseType">
+		<annotation>
+			<documentation>databaseType</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<group ref="rng:docGroup" />
+			<group ref="rng:topGroup" />
+		</choice>
+	</complexType>
+
+	<complexType name="importType">
+		<annotation>
+			<documentation>importType</documentation>
+		</annotation>
+		<attribute name="file" type="string" use="required" />
+	</complexType>
+
+	<complexType name="copyrightType">
+		<annotation>
+			<documentation>copyrightType</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<group ref="rng:docGroup" />
+			<group ref="rng:topGroup" />
+			<element ref="rng:author" />
+			<element ref="rng:license" />
+		</choice>
+		<attribute name="year" type="nonNegativeInteger" use="optional" />
+	</complexType>
+
+	<complexType name="domainType">
+		<annotation>
+			<documentation>domainType</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<group ref="rng:docGroup" />
+			<group ref="rng:topGroup" />
+			<group ref="rng:regarrayGroup" />
+		</choice>
+		<attribute name="name" type="NMTOKEN" use="required" />
+		<attribute name="prefix" type="NMTOKENS" use="optional" />
+		<attribute name="width" type="rng:DomainWidth" use="optional" />
+		<attribute name="varset" type="NMTOKEN" use="optional" />
+		<attribute name="variants" type="string" use="optional" />
+	</complexType>
+
+	<complexType name="arrayType">
+		<annotation>
+			<documentation>arrayType</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<group ref="rng:docGroup" />
+			<group ref="rng:topGroup" />
+			<group ref="rng:regarrayGroup" />
+		</choice>
+		<attribute name="name" type="NMTOKEN" use="optional" />
+		<attribute name="offset" type="rng:HexOrNumber" use="optional" />
+		<attribute name="offsets" type="string" use="optional"/>
+		<attribute name="doffsets" type="string" use="optional"/>
+		<attribute name="index" type="NMTOKENS" use="optional"/>
+		<attribute name="stride" type="rng:HexOrNumber" use="required" />
+		<attribute name="length" type="rng:HexOrNumber" use="required" />
+		<attribute name="varset" type="NMTOKEN" use="optional" />
+		<attribute name="variants" type="string" use="optional" />
+		<attribute name="usage" type="string" use="optional" />
+	</complexType>
+
+	<complexType name="stripeType">
+		<annotation>
+			<documentation>stripeType</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<group ref="rng:docGroup" />
+			<group ref="rng:topGroup" />
+			<group ref="rng:regarrayGroup" minOccurs="0" />
+		</choice>
+		<attribute name="varset" type="NMTOKEN" use="optional" />
+		<attribute name="variants" type="string" use="optional" />
+		<attribute name="prefix" type="NMTOKENS" use="optional" />
+	</complexType>
+
+	<complexType name="registerType">
+		<annotation>
+			<documentation>
+				registerType used by reg32, reg64
+			</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<group ref="rng:docGroup" />
+			<group ref="rng:topGroup" />
+			<element ref="rng:value" />
+			<element ref="rng:bitfield" />
+		</choice>
+		<attribute name="name" type="NMTOKEN" use="required" />
+		<attribute name="offset" type="rng:HexOrNumber" use="required" />
+		<attribute name="type" type="NMTOKENS" use="optional" />
+		<attribute name="shr" type="nonNegativeInteger" use="optional" />
+		<attribute name="varset" type="NMTOKEN" use="optional" />
+		<attribute name="variants" type="string" use="optional" />
+		<attribute name="stride" type="rng:HexOrNumber" use="optional" />
+		<attribute name="length" type="rng:HexOrNumber" use="optional" />
+		<attribute name="high" type="nonNegativeInteger" use="optional" />
+		<attribute name="low" type="nonNegativeInteger" use="optional" />
+		<attribute name="pos" type="nonNegativeInteger" use="optional" />
+		<attribute name="align" type="nonNegativeInteger" use="optional" />
+		<attribute name="radix" type="nonNegativeInteger" use="optional" />
+		<attribute name="usage" type="string" use="optional" />
+	</complexType>
+
+	<complexType name="bitsetType">
+		<annotation>
+			<documentation>bitsetType</documentation>
+		</annotation>
+		<choice maxOccurs="unbounded">
+			<element ref="rng:bitfield" />
+			<group ref="rng:docGroup" />
+			<group ref="rng:topGroup" />
+		</choice>
+		<attribute name="name" type="NMTOKEN" use="required" />
+		<attribute name="inline" type="rng:Boolean" use="optional" />
+		<attribute name="varset" type="NMTOKEN" use="optional" />
+	</complexType>
+
+	<complexType name="bitfieldType">
+		<annotation>
+			<documentation>bitfieldType</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<element ref="rng:value" maxOccurs="unbounded" />
+			<group ref="rng:docGroup" />
+			<group ref="rng:topGroup" />
+		</choice>
+		<attribute name="name" type="NMTOKEN" use="required" />
+		<attribute name="high" type="nonNegativeInteger" use="optional" />
+		<attribute name="low" type="nonNegativeInteger" use="optional" />
+		<attribute name="pos" type="nonNegativeInteger" use="optional" />
+		<attribute name="radix" type="nonNegativeInteger" use="optional" />
+		<attribute name="type" type="NMTOKENS" use="optional" />
+		<attribute name="varset" type="NMTOKEN" use="optional" />
+		<attribute name="variants" type="string" use="optional" />
+		<attribute name="addvariant" type="rng:Boolean" use="optional" />
+		<attribute name="shr" type="nonNegativeInteger" use="optional" />
+	</complexType>
+
+	<complexType name="enumType">
+		<annotation>
+			<documentation>enumType</documentation>
+		</annotation>
+		<choice maxOccurs="unbounded">
+			<element ref="rng:value" />
+			<group ref="rng:docGroup" />
+			<group ref="rng:topGroup" />
+		</choice>
+		<attribute name="name" type="NMTOKEN" use="required" />
+		<attribute name="bare" type="rng:Boolean" use="optional" />
+		<attribute name="prefix" type="NMTOKENS" use="optional" />
+		<attribute name="varset" type="NMTOKEN" use="optional" />
+	</complexType>
+
+	<complexType name="valueType">
+		<annotation>
+			<documentation>valueType</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<group ref="rng:docGroup" />
+			<group ref="rng:topGroup" />
+		</choice>
+		<attribute name="name" type="NMTOKEN" use="required" />
+		<attribute name="value" type="string" use="optional" />
+		<attribute name="varset" type="NMTOKEN" use="optional" />
+		<attribute name="variants" type="string" use="optional" />
+	</complexType>
+
+	<!-- Documentation element types -->
+
+	<complexType name="briefType">
+		<annotation>
+			<documentation>
+				brief documentation, no markup
+			</documentation>
+		</annotation>
+		<simpleContent>
+			<extension base="string" />
+		</simpleContent>
+	</complexType>
+	
+	<complexType name="docType" mixed="true">
+		<annotation>
+			<documentation>
+				root element of documentation sub-tree
+			</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<group ref="rng:textformatGroup" />
+			<group ref="rng:listGroup" />
+			<element ref="rng:code" />
+		</choice>
+	</complexType>
+	
+	<complexType name="textformatType" mixed="true">
+		<annotation>
+			<documentation>
+				for bold, underline, italics
+			</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<group ref="rng:textformatGroup" />
+		</choice>
+	</complexType>
+	
+	<complexType name="textcodeType">
+		<simpleContent>
+			<extension base="string">
+				<attribute name="title" type="string" />
+			</extension>
+		</simpleContent>
+	</complexType>
+	
+	<complexType name="listType">
+		<annotation>
+			<documentation>
+				definition of a list, ordered or unordered
+			</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<element ref="rng:li" />
+		</choice>
+	</complexType>
+	
+	<complexType name="listitemType" mixed="true">
+		<annotation>
+			<documentation>
+				items of a list
+			</documentation>
+		</annotation>
+		<choice minOccurs="0" maxOccurs="unbounded">
+			<group ref="rng:textformatGroup" />
+			<group ref="rng:listGroup" />
+			<element ref="rng:code" />
+		</choice>
+	</complexType>
+
+
+
+	<!-- Attribute value types -->
+
+	<simpleType name="Hexadecimal">
+		<restriction base="string">
+			<pattern value="0x[0-9a-f]+" />
+			<pattern value="0x[0-9A-F]+" />
+			<pattern value="[0-9]" />
+		</restriction>
+	</simpleType>
+
+	<simpleType name="HexOrNumber">
+		<annotation>
+			<documentation>HexOrNumber</documentation>
+		</annotation>
+		<union memberTypes="rng:Hexadecimal nonNegativeInteger" />
+	</simpleType>
+
+	<simpleType name="Boolean">
+		<restriction base="string">
+			<enumeration value="true" />
+			<enumeration value="1" />
+			<enumeration value="yes" />
+			<enumeration value="false" />
+			<enumeration value="0" />
+			<enumeration value="no" />
+		</restriction>
+	</simpleType>
+
+	<simpleType name="DomainWidth">
+		<annotation>
+			<documentation>DomainWidth</documentation>
+		</annotation>
+		<restriction base="string">
+			<enumeration value="32" />
+		</restriction>
+	</simpleType>
+
+
+
+	<!-- Element groups -->
+
+	<group name="topGroup">
+		<choice>
+			<element ref="rng:copyright" />
+			<element ref="rng:domain" />
+			<element ref="rng:enum" />
+			<element ref="rng:bitset" />
+			<element ref="rng:import" />
+		</choice>
+	</group>
+	
+	<group name="regarrayGroup">
+		<choice>
+			<element ref="rng:reg64" />
+			<element ref="rng:reg32" />
+			<element ref="rng:array" />
+			<element ref="rng:stripe" />
+		</choice>
+	</group>
+	
+	<group name="docGroup">
+		<choice>
+			<element ref="rng:brief" />
+			<element ref="rng:doc" />
+		</choice>
+	</group>
+	
+	<group name="textformatGroup">
+		<choice>
+			<element ref="rng:b" />
+			<element ref="rng:i" />
+			<element ref="rng:u" />
+		</choice>
+	</group>
+	
+	<group name="listGroup">
+		<choice>
+			<element ref="rng:ul" />
+			<element ref="rng:ol" />
+		</choice>
+	</group>
+
+</schema>
diff --git a/drivers/gpu/drm/mxsfb/lcdif_drv.c b/drivers/gpu/drm/mxsfb/lcdif_drv.c
index ea10bf81582e..0f895b8a99d6 100644
--- a/drivers/gpu/drm/mxsfb/lcdif_drv.c
+++ b/drivers/gpu/drm/mxsfb/lcdif_drv.c
@@ -343,6 +343,9 @@ static int __maybe_unused lcdif_suspend(struct device *dev)
 	if (ret)
 		return ret;
 
+	if (pm_runtime_suspended(dev))
+		return 0;
+
 	return lcdif_rpm_suspend(dev);
 }
 
@@ -350,7 +353,8 @@ static int __maybe_unused lcdif_resume(struct device *dev)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
 
-	lcdif_rpm_resume(dev);
+	if (!pm_runtime_suspended(dev))
+		lcdif_rpm_resume(dev);
 
 	return drm_mode_config_helper_resume(drm);
 }
diff --git a/drivers/gpu/drm/nouveau/Kbuild b/drivers/gpu/drm/nouveau/Kbuild
index cf6b3a80c0c8..c32c01827c1d 100644
--- a/drivers/gpu/drm/nouveau/Kbuild
+++ b/drivers/gpu/drm/nouveau/Kbuild
@@ -1,10 +1,8 @@
-NOUVEAU_PATH ?= $(srctree)
-
 # SPDX-License-Identifier: MIT
-ccflags-y += -I $(NOUVEAU_PATH)/$(src)/include
-ccflags-y += -I $(NOUVEAU_PATH)/$(src)/include/nvkm
-ccflags-y += -I $(NOUVEAU_PATH)/$(src)/nvkm
-ccflags-y += -I $(NOUVEAU_PATH)/$(src)
+ccflags-y += -I $(src)/include
+ccflags-y += -I $(src)/include/nvkm
+ccflags-y += -I $(src)/nvkm
+ccflags-y += -I $(src)
 
 # NVKM - HW resource manager
 #- code also used by various userspace tools/tests
diff --git a/drivers/gpu/drm/nouveau/dispnv50/crc.c b/drivers/gpu/drm/nouveau/dispnv50/crc.c
index 9c942fbd836d..5936b6b3b15d 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/crc.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/crc.c
@@ -1,5 +1,7 @@
 // SPDX-License-Identifier: MIT
+#include <linux/debugfs.h>
 #include <linux/string.h>
+
 #include <drm/drm_crtc.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_vblank.h>
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c
index 0c3d88ad0b0e..88728a0b2c25 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c
@@ -915,7 +915,7 @@ nv50_msto_cleanup(struct drm_atomic_state *state,
 		msto->disabled = false;
 		drm_dp_remove_payload_part2(mgr, new_mst_state, old_payload, new_payload);
 	} else if (msto->enabled) {
-		drm_dp_add_payload_part2(mgr, state, new_payload);
+		drm_dp_add_payload_part2(mgr, new_payload);
 		msto->enabled = false;
 	}
 }
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h
index 6f5d376d8fcc..a11d16a16c3b 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h
@@ -15,7 +15,9 @@ struct nvkm_gsp_mem {
 };
 
 struct nvkm_gsp_radix3 {
-	struct nvkm_gsp_mem mem[3];
+	struct nvkm_gsp_mem lvl0;
+	struct nvkm_gsp_mem lvl1;
+	struct sg_table lvl2;
 };
 
 int nvkm_gsp_sg(struct nvkm_device *, u64 size, struct sg_table *);
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index 80f74ee0fc78..d56909071de6 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -272,6 +272,9 @@ nouveau_abi16_ioctl_getparam(ABI16_IOCTL_ARGS)
 		getparam->value = (u64)ttm_resource_manager_usage(vram_mgr);
 		break;
 	}
+	case NOUVEAU_GETPARAM_HAS_VMA_TILEMODE:
+		getparam->value = 1;
+		break;
 	default:
 		NV_PRINTK(dbg, cli, "unknown parameter %lld\n", getparam->param);
 		return -EINVAL;
@@ -312,11 +315,21 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
 	if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) {
 		if (init->fb_ctxdma_handle == ~0) {
 			switch (init->tt_ctxdma_handle) {
-			case 0x01: engine = NV_DEVICE_HOST_RUNLIST_ENGINES_GR    ; break;
-			case 0x02: engine = NV_DEVICE_HOST_RUNLIST_ENGINES_MSPDEC; break;
-			case 0x04: engine = NV_DEVICE_HOST_RUNLIST_ENGINES_MSPPP ; break;
-			case 0x08: engine = NV_DEVICE_HOST_RUNLIST_ENGINES_MSVLD ; break;
-			case 0x30: engine = NV_DEVICE_HOST_RUNLIST_ENGINES_CE    ; break;
+			case NOUVEAU_FIFO_ENGINE_GR:
+				engine = NV_DEVICE_HOST_RUNLIST_ENGINES_GR;
+				break;
+			case NOUVEAU_FIFO_ENGINE_VP:
+				engine = NV_DEVICE_HOST_RUNLIST_ENGINES_MSPDEC;
+				break;
+			case NOUVEAU_FIFO_ENGINE_PPP:
+				engine = NV_DEVICE_HOST_RUNLIST_ENGINES_MSPPP;
+				break;
+			case NOUVEAU_FIFO_ENGINE_BSP:
+				engine = NV_DEVICE_HOST_RUNLIST_ENGINES_MSVLD;
+				break;
+			case NOUVEAU_FIFO_ENGINE_CE:
+				engine = NV_DEVICE_HOST_RUNLIST_ENGINES_CE;
+				break;
 			default:
 				return nouveau_abi16_put(abi16, -ENOSYS);
 			}
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.h b/drivers/gpu/drm/nouveau/nouveau_abi16.h
index 11c8c4a80079..661b901d8ecc 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.h
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.h
@@ -50,18 +50,6 @@ struct drm_nouveau_grobj_alloc {
 	int      class;
 };
 
-struct drm_nouveau_notifierobj_alloc {
-	uint32_t channel;
-	uint32_t handle;
-	uint32_t size;
-	uint32_t offset;
-};
-
-struct drm_nouveau_gpuobj_free {
-	int      channel;
-	uint32_t handle;
-};
-
 struct drm_nouveau_setparam {
 	uint64_t param;
 	uint64_t value;
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index 479effcf607e..79cfab53f80e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -23,6 +23,7 @@
  */
 
 #include "nouveau_drv.h"
+#include "nouveau_bios.h"
 #include "nouveau_reg.h"
 #include "dispnv04/hw.h"
 #include "nouveau_encoder.h"
@@ -1677,7 +1678,7 @@ apply_dcb_encoder_quirks(struct drm_device *dev, int idx, u32 *conn, u32 *conf)
 	 */
 	if (nv_match_device(dev, 0x0201, 0x1462, 0x8851)) {
 		if (*conn == 0xf2005014 && *conf == 0xffffffff) {
-			fabricate_dcb_output(dcb, DCB_OUTPUT_TMDS, 1, 1, 1);
+			fabricate_dcb_output(dcb, DCB_OUTPUT_TMDS, 1, 1, DCB_OUTPUT_B);
 			return false;
 		}
 	}
@@ -1763,26 +1764,26 @@ fabricate_dcb_encoder_table(struct drm_device *dev, struct nvbios *bios)
 #ifdef __powerpc__
 	/* Apple iMac G4 NV17 */
 	if (of_machine_is_compatible("PowerMac4,5")) {
-		fabricate_dcb_output(dcb, DCB_OUTPUT_TMDS, 0, all_heads, 1);
-		fabricate_dcb_output(dcb, DCB_OUTPUT_ANALOG, 1, all_heads, 2);
+		fabricate_dcb_output(dcb, DCB_OUTPUT_TMDS, 0, all_heads, DCB_OUTPUT_B);
+		fabricate_dcb_output(dcb, DCB_OUTPUT_ANALOG, 1, all_heads, DCB_OUTPUT_C);
 		return;
 	}
 #endif
 
 	/* Make up some sane defaults */
 	fabricate_dcb_output(dcb, DCB_OUTPUT_ANALOG,
-			     bios->legacy.i2c_indices.crt, 1, 1);
+			     bios->legacy.i2c_indices.crt, 1, DCB_OUTPUT_B);
 
 	if (nv04_tv_identify(dev, bios->legacy.i2c_indices.tv) >= 0)
 		fabricate_dcb_output(dcb, DCB_OUTPUT_TV,
 				     bios->legacy.i2c_indices.tv,
-				     all_heads, 0);
+				     all_heads, DCB_OUTPUT_A);
 
 	else if (bios->tmds.output0_script_ptr ||
 		 bios->tmds.output1_script_ptr)
 		fabricate_dcb_output(dcb, DCB_OUTPUT_TMDS,
 				     bios->legacy.i2c_indices.panel,
-				     all_heads, 1);
+				     all_heads, DCB_OUTPUT_B);
 }
 
 static int
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index db8cbf615112..70fb003a6666 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -241,28 +241,28 @@ nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 domain,
 	}
 
 	nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG);
-	if (!nouveau_cli_uvmm(cli) || internal) {
-		/* for BO noVM allocs, don't assign kinds */
-		if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) {
-			nvbo->kind = (tile_flags & 0x0000ff00) >> 8;
-			if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
-				kfree(nvbo);
-				return ERR_PTR(-EINVAL);
-			}
 
-			nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind;
-		} else if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
-			nvbo->kind = (tile_flags & 0x00007f00) >> 8;
-			nvbo->comp = (tile_flags & 0x00030000) >> 16;
-			if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
-				kfree(nvbo);
-				return ERR_PTR(-EINVAL);
-			}
-		} else {
-			nvbo->zeta = (tile_flags & 0x00000007);
+	if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) {
+		nvbo->kind = (tile_flags & 0x0000ff00) >> 8;
+		if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
+			kfree(nvbo);
+			return ERR_PTR(-EINVAL);
 		}
-		nvbo->mode = tile_mode;
 
+		nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind;
+	} else if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
+		nvbo->kind = (tile_flags & 0x00007f00) >> 8;
+		nvbo->comp = (tile_flags & 0x00030000) >> 16;
+		if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
+			kfree(nvbo);
+			return ERR_PTR(-EINVAL);
+		}
+	} else {
+		nvbo->zeta = (tile_flags & 0x00000007);
+	}
+	nvbo->mode = tile_mode;
+
+	if (!nouveau_cli_uvmm(cli) || internal) {
 		/* Determine the desirable target GPU page size for the buffer. */
 		for (i = 0; i < vmm->page_nr; i++) {
 			/* Because we cannot currently allow VMM maps to fail
@@ -304,12 +304,6 @@ nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 domain,
 		}
 		nvbo->page = vmm->page[pi].shift;
 	} else {
-		/* reject other tile flags when in VM mode. */
-		if (tile_mode)
-			return ERR_PTR(-EINVAL);
-		if (tile_flags & ~NOUVEAU_GEM_TILE_NONCONTIG)
-			return ERR_PTR(-EINVAL);
-
 		/* Determine the desirable target GPU page size for the buffer. */
 		for (i = 0; i < vmm->page_nr; i++) {
 			/* Because we cannot currently allow VMM maps to fail
@@ -467,17 +461,14 @@ nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t domain,
 	set_placement_range(nvbo, domain);
 }
 
-int
-nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig)
+int nouveau_bo_pin_locked(struct nouveau_bo *nvbo, uint32_t domain, bool contig)
 {
 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
 	struct ttm_buffer_object *bo = &nvbo->bo;
 	bool force = false, evict = false;
-	int ret;
+	int ret = 0;
 
-	ret = ttm_bo_reserve(bo, false, false, NULL);
-	if (ret)
-		return ret;
+	dma_resv_assert_held(bo->base.resv);
 
 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
 	    domain == NOUVEAU_GEM_DOMAIN_VRAM && contig) {
@@ -540,20 +531,15 @@ nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig)
 out:
 	if (force && ret)
 		nvbo->contig = false;
-	ttm_bo_unreserve(bo);
 	return ret;
 }
 
-int
-nouveau_bo_unpin(struct nouveau_bo *nvbo)
+void nouveau_bo_unpin_locked(struct nouveau_bo *nvbo)
 {
 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
 	struct ttm_buffer_object *bo = &nvbo->bo;
-	int ret;
 
-	ret = ttm_bo_reserve(bo, false, false, NULL);
-	if (ret)
-		return ret;
+	dma_resv_assert_held(bo->base.resv);
 
 	ttm_bo_unpin(&nvbo->bo);
 	if (!nvbo->bo.pin_count) {
@@ -568,8 +554,33 @@ nouveau_bo_unpin(struct nouveau_bo *nvbo)
 			break;
 		}
 	}
+}
+
+int nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig)
+{
+	struct ttm_buffer_object *bo = &nvbo->bo;
+	int ret;
+
+	ret = ttm_bo_reserve(bo, false, false, NULL);
+	if (ret)
+		return ret;
+	ret = nouveau_bo_pin_locked(nvbo, domain, contig);
+	ttm_bo_unreserve(bo);
+
+	return ret;
+}
+
+int nouveau_bo_unpin(struct nouveau_bo *nvbo)
+{
+	struct ttm_buffer_object *bo = &nvbo->bo;
+	int ret;
 
+	ret = ttm_bo_reserve(bo, false, false, NULL);
+	if (ret)
+		return ret;
+	nouveau_bo_unpin_locked(nvbo);
 	ttm_bo_unreserve(bo);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.h b/drivers/gpu/drm/nouveau/nouveau_bo.h
index e9dfab6a8156..4e891752c255 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.h
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.h
@@ -85,6 +85,8 @@ int  nouveau_bo_new(struct nouveau_cli *, u64 size, int align, u32 domain,
 		    u32 tile_mode, u32 tile_flags, struct sg_table *sg,
 		    struct dma_resv *robj,
 		    struct nouveau_bo **);
+int  nouveau_bo_pin_locked(struct nouveau_bo *nvbo, uint32_t domain, bool contig);
+void nouveau_bo_unpin_locked(struct nouveau_bo *nvbo);
 int  nouveau_bo_pin(struct nouveau_bo *, u32 flags, bool contig);
 int  nouveau_bo_unpin(struct nouveau_bo *);
 int  nouveau_bo_map(struct nouveau_bo *);
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index f28f9a857458..aed5d5b51b43 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -83,7 +83,7 @@ static bool
 nouveau_display_scanoutpos_head(struct drm_crtc *crtc, int *vpos, int *hpos,
 				ktime_t *stime, ktime_t *etime)
 {
-	struct drm_vblank_crtc *vblank = &crtc->dev->vblank[drm_crtc_index(crtc)];
+	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
 	struct nvif_head *head = &nouveau_crtc(crtc)->head;
 	struct nvif_head_scanoutpos_v0 args;
 	int retry = 20;
diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c
index 7de7707ec6a8..bcda0105160f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dp.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dp.c
@@ -181,7 +181,7 @@ nouveau_dp_probe_dpcd(struct nouveau_connector *nv_connector,
 	if (nouveau_mst) {
 		mstm = outp->dp.mstm;
 		if (mstm)
-			mstm->can_mst = drm_dp_read_mst_cap(aux, dpcd);
+			mstm->can_mst = drm_dp_read_mst_cap(aux, dpcd) == DRM_DP_MST;
 	}
 
 	if (nouveau_dp_has_sink_count(connector, outp)) {
@@ -225,12 +225,18 @@ nouveau_dp_detect(struct nouveau_connector *nv_connector,
 	u8 *dpcd = nv_encoder->dp.dpcd;
 	int ret = NOUVEAU_DP_NONE, hpd;
 
-	/* If we've already read the DPCD on an eDP device, we don't need to
-	 * reread it as it won't change
+	/* eDP ports don't support hotplugging - so there's no point in probing eDP ports unless we
+	 * haven't probed them once before.
 	 */
-	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP &&
-	    dpcd[DP_DPCD_REV] != 0)
-		return NOUVEAU_DP_SST;
+	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
+		if (connector->status == connector_status_connected)
+			return NOUVEAU_DP_SST;
+		else if (connector->status == connector_status_disconnected)
+			return NOUVEAU_DP_NONE;
+	}
+
+	// Ensure that the aux bus is enabled for probing
+	drm_dp_dpcd_set_powered(&nv_connector->aux, true);
 
 	mutex_lock(&nv_encoder->dp.hpd_irq_lock);
 	if (mstm) {
@@ -293,6 +299,13 @@ out:
 	if (mstm && !mstm->suspended && ret != NOUVEAU_DP_MST)
 		nv50_mstm_remove(mstm);
 
+	/* GSP doesn't like when we try to do aux transactions on a port it considers disconnected,
+	 * and since we don't really have a usecase for that anyway - just disable the aux bus here
+	 * if we've decided the connector is disconnected
+	 */
+	if (ret == NOUVEAU_DP_NONE)
+		drm_dp_dpcd_set_powered(&nv_connector->aux, false);
+
 	mutex_unlock(&nv_encoder->dp.hpd_irq_lock);
 	return ret;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_prime.c b/drivers/gpu/drm/nouveau/nouveau_prime.c
index 1b2ff0c40fc1..b58ab595faf8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_prime.c
+++ b/drivers/gpu/drm/nouveau/nouveau_prime.c
@@ -89,18 +89,18 @@ int nouveau_gem_prime_pin(struct drm_gem_object *obj)
 	int ret;
 
 	/* pin buffer into GTT */
-	ret = nouveau_bo_pin(nvbo, NOUVEAU_GEM_DOMAIN_GART, false);
+	ret = nouveau_bo_pin_locked(nvbo, NOUVEAU_GEM_DOMAIN_GART, false);
 	if (ret)
-		return -EINVAL;
+		ret = -EINVAL;
 
-	return 0;
+	return ret;
 }
 
 void nouveau_gem_prime_unpin(struct drm_gem_object *obj)
 {
 	struct nouveau_bo *nvbo = nouveau_gem_object(obj);
 
-	nouveau_bo_unpin(nvbo);
+	nouveau_bo_unpin_locked(nvbo);
 }
 
 struct dma_buf *nouveau_gem_prime_export(struct drm_gem_object *gobj,
diff --git a/drivers/gpu/drm/nouveau/nvif/object.c b/drivers/gpu/drm/nouveau/nvif/object.c
index 4d1aaee8fe15..1d19c87eaec1 100644
--- a/drivers/gpu/drm/nouveau/nvif/object.c
+++ b/drivers/gpu/drm/nouveau/nvif/object.c
@@ -142,11 +142,16 @@ nvif_object_mthd(struct nvif_object *object, u32 mthd, void *data, u32 size)
 		struct nvif_ioctl_v0 ioctl;
 		struct nvif_ioctl_mthd_v0 mthd;
 	} *args;
+	u32 args_size;
 	u8 stack[128];
 	int ret;
 
-	if (sizeof(*args) + size > sizeof(stack)) {
-		if (!(args = kmalloc(sizeof(*args) + size, GFP_KERNEL)))
+	if (check_add_overflow(sizeof(*args), size, &args_size))
+		return -ENOMEM;
+
+	if (args_size > sizeof(stack)) {
+		args = kmalloc(args_size, GFP_KERNEL);
+		if (!args)
 			return -ENOMEM;
 	} else {
 		args = (void *)stack;
@@ -157,7 +162,7 @@ nvif_object_mthd(struct nvif_object *object, u32 mthd, void *data, u32 size)
 	args->mthd.method = mthd;
 
 	memcpy(args->mthd.data, data, size);
-	ret = nvif_object_ioctl(object, args, sizeof(*args) + size, NULL);
+	ret = nvif_object_ioctl(object, args, args_size, NULL);
 	memcpy(data, args->mthd.data, size);
 	if (args != (void *)stack)
 		kfree(args);
@@ -276,7 +281,15 @@ nvif_object_ctor(struct nvif_object *parent, const char *name, u32 handle,
 	object->map.size = 0;
 
 	if (parent) {
-		if (!(args = kmalloc(sizeof(*args) + size, GFP_KERNEL))) {
+		u32 args_size;
+
+		if (check_add_overflow(sizeof(*args), size, &args_size)) {
+			nvif_object_dtor(object);
+			return -ENOMEM;
+		}
+
+		args = kmalloc(args_size, GFP_KERNEL);
+		if (!args) {
 			nvif_object_dtor(object);
 			return -ENOMEM;
 		}
@@ -293,8 +306,7 @@ nvif_object_ctor(struct nvif_object *parent, const char *name, u32 handle,
 		args->new.oclass = oclass;
 
 		memcpy(args->new.data, data, size);
-		ret = nvif_object_ioctl(parent, args, sizeof(*args) + size,
-					&object->priv);
+		ret = nvif_object_ioctl(parent, args, args_size, &object->priv);
 		memcpy(data, args->new.data, size);
 		kfree(args);
 		if (ret == 0)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/r535.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/r535.c
index 6a0a4d3b8902..027867c2a8c5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/r535.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/r535.c
@@ -1080,7 +1080,7 @@ r535_dp_aux_xfer(struct nvkm_outp *outp, u8 type, u32 addr, u8 *data, u8 *psize)
 	ret = nvkm_gsp_rm_ctrl_push(&disp->rm.objcom, &ctrl, sizeof(*ctrl));
 	if (ret) {
 		nvkm_gsp_rm_ctrl_done(&disp->rm.objcom, ctrl);
-		return PTR_ERR(ctrl);
+		return ret;
 	}
 
 	memcpy(data, ctrl->data, size);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c
index 9858c1438aa7..abe41f7a3404 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c
@@ -1624,7 +1624,7 @@ r535_gsp_wpr_meta_init(struct nvkm_gsp *gsp)
 	meta->magic = GSP_FW_WPR_META_MAGIC;
 	meta->revision = GSP_FW_WPR_META_REVISION;
 
-	meta->sysmemAddrOfRadix3Elf = gsp->radix3.mem[0].addr;
+	meta->sysmemAddrOfRadix3Elf = gsp->radix3.lvl0.addr;
 	meta->sizeOfRadix3Elf = gsp->fb.wpr2.elf.size;
 
 	meta->sysmemAddrOfBootloader = gsp->boot.fw.addr;
@@ -1919,8 +1919,9 @@ nvkm_gsp_sg(struct nvkm_device *device, u64 size, struct sg_table *sgt)
 static void
 nvkm_gsp_radix3_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_radix3 *rx3)
 {
-	for (int i = ARRAY_SIZE(rx3->mem) - 1; i >= 0; i--)
-		nvkm_gsp_mem_dtor(gsp, &rx3->mem[i]);
+	nvkm_gsp_sg_free(gsp->subdev.device, &rx3->lvl2);
+	nvkm_gsp_mem_dtor(gsp, &rx3->lvl1);
+	nvkm_gsp_mem_dtor(gsp, &rx3->lvl0);
 }
 
 /**
@@ -1960,36 +1961,60 @@ static int
 nvkm_gsp_radix3_sg(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size,
 		   struct nvkm_gsp_radix3 *rx3)
 {
-	u64 addr;
+	struct sg_dma_page_iter sg_dma_iter;
+	struct scatterlist *sg;
+	size_t bufsize;
+	u64 *pte;
+	int ret, i, page_idx = 0;
 
-	for (int i = ARRAY_SIZE(rx3->mem) - 1; i >= 0; i--) {
-		u64 *ptes;
-		size_t bufsize;
-		int ret, idx;
+	ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl0);
+	if (ret)
+		return ret;
 
-		bufsize = ALIGN((size / GSP_PAGE_SIZE) * sizeof(u64), GSP_PAGE_SIZE);
-		ret = nvkm_gsp_mem_ctor(gsp, bufsize, &rx3->mem[i]);
-		if (ret)
-			return ret;
+	ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl1);
+	if (ret)
+		goto lvl1_fail;
 
-		ptes = rx3->mem[i].data;
-		if (i == 2) {
-			struct scatterlist *sgl;
+	// Allocate level 2
+	bufsize = ALIGN((size / GSP_PAGE_SIZE) * sizeof(u64), GSP_PAGE_SIZE);
+	ret = nvkm_gsp_sg(gsp->subdev.device, bufsize, &rx3->lvl2);
+	if (ret)
+		goto lvl2_fail;
 
-			for_each_sgtable_dma_sg(sgt, sgl, idx) {
-				for (int j = 0; j < sg_dma_len(sgl) / GSP_PAGE_SIZE; j++)
-					*ptes++ = sg_dma_address(sgl) + (GSP_PAGE_SIZE * j);
-			}
-		} else {
-			for (int j = 0; j < size / GSP_PAGE_SIZE; j++)
-				*ptes++ = addr + GSP_PAGE_SIZE * j;
+	// Write the bus address of level 1 to level 0
+	pte = rx3->lvl0.data;
+	*pte = rx3->lvl1.addr;
+
+	// Write the bus address of each page in level 2 to level 1
+	pte = rx3->lvl1.data;
+	for_each_sgtable_dma_page(&rx3->lvl2, &sg_dma_iter, 0)
+		*pte++ = sg_page_iter_dma_address(&sg_dma_iter);
+
+	// Finally, write the bus address of each page in sgt to level 2
+	for_each_sgtable_sg(&rx3->lvl2, sg, i) {
+		void *sgl_end;
+
+		pte = sg_virt(sg);
+		sgl_end = (void *)pte + sg->length;
+
+		for_each_sgtable_dma_page(sgt, &sg_dma_iter, page_idx) {
+			*pte++ = sg_page_iter_dma_address(&sg_dma_iter);
+			page_idx++;
+
+			// Go to the next scatterlist for level 2 if we've reached the end
+			if ((void *)pte >= sgl_end)
+				break;
 		}
+	}
 
-		size = rx3->mem[i].size;
-		addr = rx3->mem[i].addr;
+	if (ret) {
+lvl2_fail:
+		nvkm_gsp_mem_dtor(gsp, &rx3->lvl1);
+lvl1_fail:
+		nvkm_gsp_mem_dtor(gsp, &rx3->lvl0);
 	}
 
-	return 0;
+	return ret;
 }
 
 int
@@ -2021,7 +2046,7 @@ r535_gsp_fini(struct nvkm_gsp *gsp, bool suspend)
 		sr = gsp->sr.meta.data;
 		sr->magic = GSP_FW_SR_META_MAGIC;
 		sr->revision = GSP_FW_SR_META_REVISION;
-		sr->sysmemAddrOfSuspendResumeData = gsp->sr.radix3.mem[0].addr;
+		sr->sysmemAddrOfSuspendResumeData = gsp->sr.radix3.lvl0.addr;
 		sr->sizeOfSuspendResumeData = len;
 
 		mbox0 = lower_32_bits(gsp->sr.meta.addr);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c
index a7f3fc342d87..dd5b5a17ece0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c
@@ -222,8 +222,11 @@ nv50_instobj_acquire(struct nvkm_memory *memory)
 	void __iomem *map = NULL;
 
 	/* Already mapped? */
-	if (refcount_inc_not_zero(&iobj->maps))
+	if (refcount_inc_not_zero(&iobj->maps)) {
+		/* read barrier match the wmb on refcount set */
+		smp_rmb();
 		return iobj->map;
+	}
 
 	/* Take the lock, and re-check that another thread hasn't
 	 * already mapped the object in the meantime.
@@ -250,6 +253,8 @@ nv50_instobj_acquire(struct nvkm_memory *memory)
 			iobj->base.memory.ptrs = &nv50_instobj_fast;
 		else
 			iobj->base.memory.ptrs = &nv50_instobj_slow;
+		/* barrier to ensure the ptrs are written before refcount is set */
+		smp_wmb();
 		refcount_set(&iobj->maps, 1);
 	}
 
diff --git a/drivers/gpu/drm/omapdrm/Kconfig b/drivers/gpu/drm/omapdrm/Kconfig
index b715301ec79f..6c49270cb290 100644
--- a/drivers/gpu/drm/omapdrm/Kconfig
+++ b/drivers/gpu/drm/omapdrm/Kconfig
@@ -4,7 +4,7 @@ config DRM_OMAP
 	depends on DRM && OF
 	depends on ARCH_OMAP2PLUS
 	select DRM_KMS_HELPER
-	select FB_DMAMEM_HELPERS if DRM_FBDEV_EMULATION
+	select FB_DMAMEM_HELPERS_DEFERRED if DRM_FBDEV_EMULATION
 	select VIDEOMODE_HELPERS
 	select HDMI
 	default n
diff --git a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
index 9753c1e1f994..1aca3060333e 100644
--- a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
+++ b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
@@ -1212,7 +1212,6 @@ struct platform_driver omap_dmm_driver = {
 	.probe = omap_dmm_probe,
 	.remove_new = omap_dmm_remove,
 	.driver = {
-		.owner = THIS_MODULE,
 		.name = DMM_DRIVER_NAME,
 		.of_match_table = of_match_ptr(dmm_of_match),
 		.pm = &omap_dmm_pm_ops,
diff --git a/drivers/gpu/drm/omapdrm/omap_fb.c b/drivers/gpu/drm/omapdrm/omap_fb.c
index 1d414b33fee3..449d521c78fe 100644
--- a/drivers/gpu/drm/omapdrm/omap_fb.c
+++ b/drivers/gpu/drm/omapdrm/omap_fb.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/dma-mapping.h>
+#include <linux/seq_file.h>
 
 #include <drm/drm_blend.h>
 #include <drm/drm_modeset_helper.h>
diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c
index 6b08b137af1a..523be34682ca 100644
--- a/drivers/gpu/drm/omapdrm/omap_fbdev.c
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c
@@ -51,6 +51,10 @@ static void pan_worker(struct work_struct *work)
 	omap_gem_roll(bo, fbi->var.yoffset * npages);
 }
 
+FB_GEN_DEFAULT_DEFERRED_DMAMEM_OPS(omap_fbdev,
+				   drm_fb_helper_damage_range,
+				   drm_fb_helper_damage_area)
+
 static int omap_fbdev_pan_display(struct fb_var_screeninfo *var,
 		struct fb_info *fbi)
 {
@@ -78,11 +82,9 @@ fallback:
 
 static int omap_fbdev_fb_mmap(struct fb_info *info, struct vm_area_struct *vma)
 {
-	struct drm_fb_helper *helper = info->par;
-	struct drm_framebuffer *fb = helper->fb;
-	struct drm_gem_object *bo = drm_gem_fb_get_obj(fb, 0);
+	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
 
-	return drm_gem_mmap_obj(bo, omap_gem_mmap_size(bo), vma);
+	return fb_deferred_io_mmap(info, vma);
 }
 
 static void omap_fbdev_fb_destroy(struct fb_info *info)
@@ -94,6 +96,7 @@ static void omap_fbdev_fb_destroy(struct fb_info *info)
 
 	DBG();
 
+	fb_deferred_io_cleanup(info);
 	drm_fb_helper_fini(helper);
 
 	omap_gem_unpin(bo);
@@ -104,15 +107,19 @@ static void omap_fbdev_fb_destroy(struct fb_info *info)
 	kfree(fbdev);
 }
 
+/*
+ * For now, we cannot use FB_DEFAULT_DEFERRED_OPS and fb_deferred_io_mmap()
+ * because we use write-combine.
+ */
 static const struct fb_ops omap_fb_ops = {
 	.owner = THIS_MODULE,
-	__FB_DEFAULT_DMAMEM_OPS_RDWR,
+	__FB_DEFAULT_DEFERRED_OPS_RDWR(omap_fbdev),
 	.fb_check_var	= drm_fb_helper_check_var,
 	.fb_set_par	= drm_fb_helper_set_par,
 	.fb_setcmap	= drm_fb_helper_setcmap,
 	.fb_blank	= drm_fb_helper_blank,
 	.fb_pan_display = omap_fbdev_pan_display,
-	__FB_DEFAULT_DMAMEM_OPS_DRAW,
+	__FB_DEFAULT_DEFERRED_OPS_DRAW(omap_fbdev),
 	.fb_ioctl	= drm_fb_helper_ioctl,
 	.fb_mmap	= omap_fbdev_fb_mmap,
 	.fb_destroy	= omap_fbdev_fb_destroy,
@@ -213,6 +220,15 @@ static int omap_fbdev_create(struct drm_fb_helper *helper,
 	fbi->fix.smem_start = dma_addr;
 	fbi->fix.smem_len = bo->size;
 
+	/* deferred I/O */
+	helper->fbdefio.delay = HZ / 20;
+	helper->fbdefio.deferred_io = drm_fb_helper_deferred_io;
+
+	fbi->fbdefio = &helper->fbdefio;
+	ret = fb_deferred_io_init(fbi);
+	if (ret)
+		goto fail;
+
 	/* if we have DMM, then we can use it for scrolling by just
 	 * shuffling pages around in DMM rather than doing sw blit.
 	 */
@@ -238,8 +254,20 @@ fail:
 	return ret;
 }
 
+static int omap_fbdev_dirty(struct drm_fb_helper *helper, struct drm_clip_rect *clip)
+{
+	if (!(clip->x1 < clip->x2 && clip->y1 < clip->y2))
+		return 0;
+
+	if (helper->fb->funcs->dirty)
+		return helper->fb->funcs->dirty(helper->fb, NULL, 0, 0, clip, 1);
+
+	return 0;
+}
+
 static const struct drm_fb_helper_funcs omap_fb_helper_funcs = {
 	.fb_probe = omap_fbdev_create,
+	.fb_dirty = omap_fbdev_dirty,
 };
 
 static struct drm_fb_helper *get_fb(struct fb_info *fbi)
diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c
index 3421e8389222..9ea0c64c26b5 100644
--- a/drivers/gpu/drm/omapdrm/omap_gem.c
+++ b/drivers/gpu/drm/omapdrm/omap_gem.c
@@ -9,6 +9,7 @@
 #include <linux/shmem_fs.h>
 #include <linux/spinlock.h>
 #include <linux/pfn_t.h>
+#include <linux/vmalloc.h>
 
 #include <drm/drm_prime.h>
 #include <drm/drm_vma_manager.h>
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index d037b3b8b999..2ae0eb0638f3 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -177,7 +177,7 @@ config DRM_PANEL_ILITEK_IL9322
 
 config DRM_PANEL_ILITEK_ILI9341
 	tristate "Ilitek ILI9341 240x320 QVGA panels"
-	depends on OF && SPI
+	depends on SPI
 	select DRM_KMS_HELPER
 	select DRM_GEM_DMA_HELPER
 	depends on BACKLIGHT_CLASS_DEVICE
@@ -335,6 +335,19 @@ config DRM_PANEL_LG_LG4573
 	  Say Y here if you want to enable support for LG4573 RGB panel.
 	  To compile this driver as a module, choose M here.
 
+config DRM_PANEL_LG_SW43408
+	tristate "LG SW43408 panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	select DRM_DISPLAY_DP_HELPER
+	select DRM_DISPLAY_HELPER
+	help
+	  Say Y here if you want to enable support for LG sw43408 panel.
+	  The panel has a 1080x2160@60Hz resolution and uses 24 bit RGB per
+	  pixel. It provides a MIPI DSI interface to the host and has a
+	  built-in LED backlight.
+
 config DRM_PANEL_MAGNACHIP_D53E6EA8966
 	tristate "Magnachip D53E6EA8966 DSI panel"
 	depends on OF && SPI
@@ -542,6 +555,18 @@ config DRM_PANEL_RAYDIUM_RM692E5
 	  Say Y here if you want to enable support for Raydium RM692E5-based
 	  display panels, such as the one found in the Fairphone 5 smartphone.
 
+config DRM_PANEL_RAYDIUM_RM69380
+	tristate "Raydium RM69380-based DSI panel"
+	depends on OF && GPIOLIB
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y here if you want to enable support for Raydium RM69380-based
+	  display panels.
+
+	  This panel controller can be found in the Lenovo Xiaoxin Pad Pro 2021
+	  in combination with an EDO OLED panel.
+
 config DRM_PANEL_RONBO_RB070D30
 	tristate "Ronbo Electronics RB070D30 panel"
 	depends on OF
@@ -564,7 +589,7 @@ config DRM_PANEL_SAMSUNG_ATNA33XC20
 	depends on PM
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HELPER
-	select DRM_DP_AUX_BUS
+	select DRM_DISPLAY_DP_AUX_BUS
 	help
 	  DRM panel driver for the Samsung ATNA33XC20 panel. This panel can't
 	  be handled by the DRM_PANEL_SIMPLE driver because its power
@@ -586,6 +611,15 @@ config DRM_PANEL_SAMSUNG_LD9040
 	depends on BACKLIGHT_CLASS_DEVICE
 	select VIDEOMODE_HELPERS
 
+config DRM_PANEL_SAMSUNG_S6E3FA7
+	tristate "Samsung S6E3FA7 panel driver"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y here if you want to enable support for the Samsung S6E3FA7
+	  1920x2220 panel.
+
 config DRM_PANEL_SAMSUNG_S6D16D0
 	tristate "Samsung S6D16D0 DSI video mode panel"
 	depends on OF
@@ -796,7 +830,7 @@ config DRM_PANEL_EDP
 	select VIDEOMODE_HELPERS
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HELPER
-	select DRM_DP_AUX_BUS
+	select DRM_DISPLAY_DP_AUX_BUS
 	select DRM_KMS_HELPER
 	help
 	  DRM panel driver for dumb eDP panels that need at most a regulator and
diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
index f156d7fa0bcc..f0203f6e02f4 100644
--- a/drivers/gpu/drm/panel/Makefile
+++ b/drivers/gpu/drm/panel/Makefile
@@ -34,6 +34,7 @@ obj-$(CONFIG_DRM_PANEL_LEADTEK_LTK050H3146W) += panel-leadtek-ltk050h3146w.o
 obj-$(CONFIG_DRM_PANEL_LEADTEK_LTK500HD1829) += panel-leadtek-ltk500hd1829.o
 obj-$(CONFIG_DRM_PANEL_LG_LB035Q02) += panel-lg-lb035q02.o
 obj-$(CONFIG_DRM_PANEL_LG_LG4573) += panel-lg-lg4573.o
+obj-$(CONFIG_DRM_PANEL_LG_SW43408) += panel-lg-sw43408.o
 obj-$(CONFIG_DRM_PANEL_MAGNACHIP_D53E6EA8966) += panel-magnachip-d53e6ea8966.o
 obj-$(CONFIG_DRM_PANEL_NEC_NL8048HL11) += panel-nec-nl8048hl11.o
 obj-$(CONFIG_DRM_PANEL_NEWVISION_NV3051D) += panel-newvision-nv3051d.o
@@ -55,6 +56,7 @@ obj-$(CONFIG_DRM_PANEL_RASPBERRYPI_TOUCHSCREEN) += panel-raspberrypi-touchscreen
 obj-$(CONFIG_DRM_PANEL_RAYDIUM_RM67191) += panel-raydium-rm67191.o
 obj-$(CONFIG_DRM_PANEL_RAYDIUM_RM68200) += panel-raydium-rm68200.o
 obj-$(CONFIG_DRM_PANEL_RAYDIUM_RM692E5) += panel-raydium-rm692e5.o
+obj-$(CONFIG_DRM_PANEL_RAYDIUM_RM69380) += panel-raydium-rm69380.o
 obj-$(CONFIG_DRM_PANEL_RONBO_RB070D30) += panel-ronbo-rb070d30.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_ATNA33XC20) += panel-samsung-atna33xc20.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_DB7430) += panel-samsung-db7430.o
@@ -62,6 +64,7 @@ obj-$(CONFIG_DRM_PANEL_SAMSUNG_LD9040) += panel-samsung-ld9040.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6D16D0) += panel-samsung-s6d16d0.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6D27A1) += panel-samsung-s6d27a1.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6D7AA0) += panel-samsung-s6d7aa0.o
+obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6E3FA7) += panel-samsung-s6e3fa7.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6E3HA2) += panel-samsung-s6e3ha2.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6E63J0X03) += panel-samsung-s6e63j0x03.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6E63M0) += panel-samsung-s6e63m0.o
diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c
index d58f90bc48fb..6db277efcbb7 100644
--- a/drivers/gpu/drm/panel/panel-edp.c
+++ b/drivers/gpu/drm/panel/panel-edp.c
@@ -210,15 +210,12 @@ struct panel_desc {
  * struct edp_panel_entry - Maps panel ID to delay / panel name.
  */
 struct edp_panel_entry {
-	/** @panel_id: 32-bit ID for panel, encoded with drm_edid_encode_panel_id(). */
-	u32 panel_id;
+	/** @ident: edid identity used for panel matching. */
+	const struct drm_edid_ident ident;
 
 	/** @delay: The power sequencing delays needed for this panel. */
 	const struct panel_delay *delay;
 
-	/** @name: Name of this panel (for printing to logs). */
-	const char *name;
-
 	/** @override_edid_mode: Override the mode obtained by edid. */
 	const struct drm_display_mode *override_edid_mode;
 };
@@ -245,7 +242,7 @@ struct panel_edp {
 
 	const struct edp_panel_entry *detected_panel;
 
-	struct edid *edid;
+	const struct drm_edid *drm_edid;
 
 	struct drm_display_mode override_mode;
 
@@ -620,13 +617,16 @@ static int panel_edp_get_modes(struct drm_panel *panel,
 	if (p->ddc) {
 		pm_runtime_get_sync(panel->dev);
 
-		if (!p->edid)
-			p->edid = drm_get_edid(connector, p->ddc);
+		if (!p->drm_edid)
+			p->drm_edid = drm_edid_read_ddc(connector, p->ddc);
+
+		drm_edid_connector_update(connector, p->drm_edid);
+
 		/*
 		 * If both edid and hard-coded modes exists, skip edid modes to
 		 * avoid multiple preferred modes.
 		 */
-		if (p->edid && !has_hard_coded_modes) {
+		if (p->drm_edid && !has_hard_coded_modes) {
 			if (has_override_edid_mode) {
 				/*
 				 * override_edid_mode is specified. Use
@@ -635,7 +635,7 @@ static int panel_edp_get_modes(struct drm_panel *panel,
 				num += panel_edp_override_edid_mode(p, connector,
 						p->detected_panel->override_edid_mode);
 			} else {
-				num += drm_add_edid_modes(connector, p->edid);
+				num += drm_edid_connector_add_modes(connector);
 			}
 		}
 
@@ -691,7 +691,7 @@ static int detected_panel_show(struct seq_file *s, void *data)
 	else if (!p->detected_panel)
 		seq_puts(s, "HARDCODED\n");
 	else
-		seq_printf(s, "%s\n", p->detected_panel->name);
+		seq_printf(s, "%s\n", p->detected_panel->ident.name);
 
 	return 0;
 }
@@ -761,11 +761,31 @@ static void panel_edp_parse_panel_timing_node(struct device *dev,
 		dev_err(dev, "Reject override mode: No display_timing found\n");
 }
 
-static const struct edp_panel_entry *find_edp_panel(u32 panel_id);
+static const struct edp_panel_entry *find_edp_panel(u32 panel_id, const struct drm_edid *edid);
+
+static void panel_edp_set_conservative_timings(struct panel_edp *panel, struct panel_desc *desc)
+{
+	/*
+	 * It's highly likely that the panel will work if we use very
+	 * conservative timings, so let's do that.
+	 *
+	 * Nearly all panels have a "unprepare" delay of 500 ms though
+	 * there are a few with 1000. Let's stick 2000 in just to be
+	 * super conservative.
+	 *
+	 * An "enable" delay of 80 ms seems the most common, but we'll
+	 * throw in 200 ms to be safe.
+	 */
+	desc->delay.unprepare = 2000;
+	desc->delay.enable = 200;
+
+	panel->detected_panel = ERR_PTR(-EINVAL);
+}
 
 static int generic_edp_panel_probe(struct device *dev, struct panel_edp *panel)
 {
 	struct panel_desc *desc;
+	const struct drm_edid *base_block;
 	u32 panel_id;
 	char vend[4];
 	u16 product_id;
@@ -791,19 +811,26 @@ static int generic_edp_panel_probe(struct device *dev, struct panel_edp *panel)
 	/* Power the panel on so we can read the EDID */
 	ret = pm_runtime_get_sync(dev);
 	if (ret < 0) {
-		dev_err(dev, "Couldn't power on panel to read EDID: %d\n", ret);
+		dev_err(dev,
+			"Couldn't power on panel to ID it; using conservative timings: %d\n",
+			ret);
+		panel_edp_set_conservative_timings(panel, desc);
 		goto exit;
 	}
 
-	panel_id = drm_edid_get_panel_id(panel->ddc);
-	if (!panel_id) {
-		dev_err(dev, "Couldn't identify panel via EDID\n");
-		ret = -EIO;
+	base_block = drm_edid_read_base_block(panel->ddc);
+	if (base_block) {
+		panel_id = drm_edid_get_panel_id(base_block);
+	} else {
+		dev_err(dev, "Couldn't read EDID for ID; using conservative timings\n");
+		panel_edp_set_conservative_timings(panel, desc);
 		goto exit;
 	}
 	drm_edid_decode_panel_id(panel_id, vend, &product_id);
 
-	panel->detected_panel = find_edp_panel(panel_id);
+	panel->detected_panel = find_edp_panel(panel_id, base_block);
+
+	drm_edid_free(base_block);
 
 	/*
 	 * We're using non-optimized timings and want it really obvious that
@@ -814,40 +841,20 @@ static int generic_edp_panel_probe(struct device *dev, struct panel_edp *panel)
 		dev_warn(dev,
 			 "Unknown panel %s %#06x, using conservative timings\n",
 			 vend, product_id);
-
-		/*
-		 * It's highly likely that the panel will work if we use very
-		 * conservative timings, so let's do that. We already know that
-		 * the HPD-related delays must have worked since we got this
-		 * far, so we really just need the "unprepare" / "enable"
-		 * delays. We don't need "prepare_to_enable" since that
-		 * overlaps the "enable" delay anyway.
-		 *
-		 * Nearly all panels have a "unprepare" delay of 500 ms though
-		 * there are a few with 1000. Let's stick 2000 in just to be
-		 * super conservative.
-		 *
-		 * An "enable" delay of 80 ms seems the most common, but we'll
-		 * throw in 200 ms to be safe.
-		 */
-		desc->delay.unprepare = 2000;
-		desc->delay.enable = 200;
-
-		panel->detected_panel = ERR_PTR(-EINVAL);
+		panel_edp_set_conservative_timings(panel, desc);
 	} else {
 		dev_info(dev, "Detected %s %s (%#06x)\n",
-			 vend, panel->detected_panel->name, product_id);
+			 vend, panel->detected_panel->ident.name, product_id);
 
 		/* Update the delay; everything else comes from EDID */
 		desc->delay = *panel->detected_panel->delay;
 	}
 
-	ret = 0;
 exit:
 	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_autosuspend(dev);
 
-	return ret;
+	return 0;
 }
 
 static int panel_edp_probe(struct device *dev, const struct panel_desc *desc,
@@ -940,8 +947,14 @@ static int panel_edp_probe(struct device *dev, const struct panel_desc *desc,
 		err = drm_panel_dp_aux_backlight(&panel->base, panel->aux);
 		pm_runtime_mark_last_busy(dev);
 		pm_runtime_put_autosuspend(dev);
+
+		/*
+		 * Warn if we get an error, but don't consider it fatal. Having
+		 * a panel where we can't control the backlight is better than
+		 * no panel.
+		 */
 		if (err)
-			goto err_finished_pm_runtime;
+			dev_warn(dev, "failed to register dp aux backlight: %d\n", err);
 	}
 
 	drm_panel_add(&panel->base);
@@ -971,8 +984,8 @@ static void panel_edp_remove(struct device *dev)
 	if (panel->ddc && (!panel->aux || panel->ddc != &panel->aux->ddc))
 		put_device(&panel->ddc->dev);
 
-	kfree(panel->edid);
-	panel->edid = NULL;
+	drm_edid_free(panel->drm_edid);
+	panel->drm_edid = NULL;
 }
 
 static void panel_edp_shutdown(struct device *dev)
@@ -1005,6 +1018,19 @@ static const struct panel_desc auo_b101ean01 = {
 	},
 };
 
+static const struct drm_display_mode auo_b116xa3_mode = {
+	.clock = 70589,
+	.hdisplay = 1366,
+	.hsync_start = 1366 + 40,
+	.hsync_end = 1366 + 40 + 40,
+	.htotal = 1366 + 40 + 40 + 32,
+	.vdisplay = 768,
+	.vsync_start = 768 + 10,
+	.vsync_end = 768 + 10 + 12,
+	.vtotal = 768 + 10 + 12 + 6,
+	.flags = DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
+};
+
 static const struct drm_display_mode auo_b116xak01_mode = {
 	.clock = 69300,
 	.hdisplay = 1366,
@@ -1865,6 +1891,13 @@ static const struct panel_delay delay_200_500_e50 = {
 	.enable = 50,
 };
 
+static const struct panel_delay delay_200_500_e50_p2e200 = {
+	.hpd_absent = 200,
+	.unprepare = 500,
+	.enable = 50,
+	.prepare_to_enable = 200,
+};
+
 static const struct panel_delay delay_200_500_e80 = {
 	.hpd_absent = 200,
 	.unprepare = 500,
@@ -1919,17 +1952,21 @@ static const struct panel_delay delay_200_500_e50_po2e200 = {
 
 #define EDP_PANEL_ENTRY(vend_chr_0, vend_chr_1, vend_chr_2, product_id, _delay, _name) \
 { \
-	.name = _name, \
-	.panel_id = drm_edid_encode_panel_id(vend_chr_0, vend_chr_1, vend_chr_2, \
-					     product_id), \
+	.ident = { \
+		.name = _name, \
+		.panel_id = drm_edid_encode_panel_id(vend_chr_0, vend_chr_1, vend_chr_2, \
+						     product_id), \
+	}, \
 	.delay = _delay \
 }
 
 #define EDP_PANEL_ENTRY2(vend_chr_0, vend_chr_1, vend_chr_2, product_id, _delay, _name, _mode) \
 { \
-	.name = _name, \
-	.panel_id = drm_edid_encode_panel_id(vend_chr_0, vend_chr_1, vend_chr_2, \
-					     product_id), \
+	.ident = { \
+		.name = _name, \
+		.panel_id = drm_edid_encode_panel_id(vend_chr_0, vend_chr_1, vend_chr_2, \
+						     product_id), \
+	}, \
 	.delay = _delay, \
 	.override_edid_mode = _mode \
 }
@@ -1953,7 +1990,9 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x239b, &delay_200_500_e50, "B116XAN06.1"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x255c, &delay_200_500_e50, "B116XTN02.5"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x403d, &delay_200_500_e50, "B140HAN04.0"),
-	EDP_PANEL_ENTRY('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAK01.0"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAN04.0"),
+	EDP_PANEL_ENTRY2('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAK01.0",
+			 &auo_b116xa3_mode),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x435c, &delay_200_500_e50, "Unknown"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x582d, &delay_200_500_e50, "B133UAN01.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x615c, &delay_200_500_e50, "B116XAN06.1"),
@@ -1961,6 +2000,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x639c, &delay_200_500_e50, "B140HAK02.7"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x723c, &delay_200_500_e50, "B140XTN07.2"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x8594, &delay_200_500_e50, "B133UAN01.0"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0xd497, &delay_200_500_e50, "B120XAN01.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0xf390, &delay_200_500_e50, "B140XTN07.7"),
 
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0607, &delay_200_500_e200, "Unknown"),
@@ -2010,6 +2050,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b43, &delay_200_500_e200, "NV140FHM-T09"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b56, &delay_200_500_e80, "NT140FHM-N47"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c20, &delay_200_500_e80, "NT140FHM-N47"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cb6, &delay_200_500_e200, "NT116WHM-N44"),
 
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1130, &delay_200_500_e50, "N116BGE-EB2"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1132, &delay_200_500_e80_d50, "N116BGE-EA2"),
@@ -2025,6 +2066,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1156, &delay_200_500_e80_d50, "Unknown"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1157, &delay_200_500_e80_d50, "N116BGE-EA2"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x115b, &delay_200_500_e80_d50, "N116BCN-EB1"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x115e, &delay_200_500_e80_d50, "N116BCA-EA1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1247, &delay_200_500_e80_d50, "N120ACA-EA1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x142b, &delay_200_500_e80_d50, "N140HCA-EAC"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x142e, &delay_200_500_e80_d50, "N140BGA-EA4"),
@@ -2034,7 +2076,9 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14d6, &delay_200_500_e80_d50, "N140BGA-EA4"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14e5, &delay_200_500_e80_d50, "N140HGA-EA1"),
 
-	EDP_PANEL_ENTRY('C', 'S', 'O', 0x1200, &delay_200_500_e50, "MNC207QS1-1"),
+	EDP_PANEL_ENTRY('C', 'S', 'O', 0x1200, &delay_200_500_e50_p2e200, "MNC207QS1-1"),
+
+	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1100, &delay_200_500_e80_d50, "MNB601LS1-1"),
 
 	EDP_PANEL_ENTRY('H', 'K', 'C', 0x2d51, &delay_200_500_e200, "Unknown"),
 	EDP_PANEL_ENTRY('H', 'K', 'C', 0x2d5b, &delay_200_500_e200, "Unknown"),
@@ -2076,15 +2120,25 @@ static const struct edp_panel_entry edp_panels[] = {
 	{ /* sentinal */ }
 };
 
-static const struct edp_panel_entry *find_edp_panel(u32 panel_id)
+static const struct edp_panel_entry *find_edp_panel(u32 panel_id, const struct drm_edid *edid)
 {
 	const struct edp_panel_entry *panel;
 
 	if (!panel_id)
 		return NULL;
 
-	for (panel = edp_panels; panel->panel_id; panel++)
-		if (panel->panel_id == panel_id)
+	/*
+	 * Match with identity first. This allows handling the case where
+	 * vendors incorrectly reused the same panel ID for multiple panels that
+	 * need different settings. If there's no match, try again with panel
+	 * ID, which should be unique.
+	 */
+	for (panel = edp_panels; panel->ident.panel_id; panel++)
+		if (drm_edid_match(edid, &panel->ident))
+			return panel;
+
+	for (panel = edp_panels; panel->ident.panel_id; panel++)
+		if (panel->ident.panel_id == panel_id)
 			return panel;
 
 	return NULL;
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9341.c b/drivers/gpu/drm/panel/panel-ilitek-ili9341.c
index 3574681891e8..b933380b7eb7 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9341.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9341.c
@@ -22,8 +22,9 @@
 #include <linux/bitops.h>
 #include <linux/delay.h>
 #include <linux/gpio/consumer.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
+#include <linux/property.h>
 #include <linux/regulator/consumer.h>
 #include <linux/spi/spi.h>
 
@@ -421,7 +422,7 @@ static int ili9341_dpi_prepare(struct drm_panel *panel)
 
 	ili9341_dpi_init(ili);
 
-	return ret;
+	return 0;
 }
 
 static int ili9341_dpi_enable(struct drm_panel *panel)
@@ -691,7 +692,7 @@ static int ili9341_dpi_probe(struct spi_device *spi, struct gpio_desc *dc,
 	 * Every new incarnation of this display must have a unique
 	 * data entry for the system in this driver.
 	 */
-	ili->conf = of_device_get_match_data(dev);
+	ili->conf = device_get_match_data(dev);
 	if (!ili->conf) {
 		dev_err(dev, "missing device configuration\n");
 		return -ENODEV;
@@ -714,18 +715,18 @@ static int ili9341_probe(struct spi_device *spi)
 
 	reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(reset))
-		dev_err(dev, "Failed to get gpio 'reset'\n");
+		return dev_err_probe(dev, PTR_ERR(reset), "Failed to get gpio 'reset'\n");
 
 	dc = devm_gpiod_get_optional(dev, "dc", GPIOD_OUT_LOW);
 	if (IS_ERR(dc))
-		dev_err(dev, "Failed to get gpio 'dc'\n");
+		return dev_err_probe(dev, PTR_ERR(dc), "Failed to get gpio 'dc'\n");
 
 	if (!strcmp(id->name, "sf-tc240t-9370-t"))
 		return ili9341_dpi_probe(spi, dc, reset);
 	else if (!strcmp(id->name, "yx240qv29"))
 		return ili9341_dbi_probe(spi, dc, reset);
 
-	return -1;
+	return -ENODEV;
 }
 
 static void ili9341_remove(struct spi_device *spi)
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
index 2ffe5f68a890..084c37fa7348 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
@@ -455,6 +455,202 @@ static const struct ili9881c_instr k101_im2byl02_init[] = {
 	ILI9881C_COMMAND_INSTR(0xD3, 0x3F), /* VN0 */
 };
 
+static const struct ili9881c_instr kd050hdfia020_init[] = {
+	ILI9881C_SWITCH_PAGE_INSTR(3),
+	ILI9881C_COMMAND_INSTR(0x01, 0x00),
+	ILI9881C_COMMAND_INSTR(0x02, 0x00),
+	ILI9881C_COMMAND_INSTR(0x03, 0x72),
+	ILI9881C_COMMAND_INSTR(0x04, 0x00),
+	ILI9881C_COMMAND_INSTR(0x05, 0x00),
+	ILI9881C_COMMAND_INSTR(0x06, 0x09),
+	ILI9881C_COMMAND_INSTR(0x07, 0x00),
+	ILI9881C_COMMAND_INSTR(0x08, 0x00),
+	ILI9881C_COMMAND_INSTR(0x09, 0x01),
+	ILI9881C_COMMAND_INSTR(0x0a, 0x00),
+	ILI9881C_COMMAND_INSTR(0x0b, 0x00),
+	ILI9881C_COMMAND_INSTR(0x0c, 0x01),
+	ILI9881C_COMMAND_INSTR(0x0d, 0x00),
+	ILI9881C_COMMAND_INSTR(0x0e, 0x00),
+	ILI9881C_COMMAND_INSTR(0x0f, 0x00),
+	ILI9881C_COMMAND_INSTR(0x10, 0x00),
+	ILI9881C_COMMAND_INSTR(0x11, 0x00),
+	ILI9881C_COMMAND_INSTR(0x12, 0x00),
+	ILI9881C_COMMAND_INSTR(0x13, 0x00),
+	ILI9881C_COMMAND_INSTR(0x14, 0x00),
+	ILI9881C_COMMAND_INSTR(0x15, 0x00),
+	ILI9881C_COMMAND_INSTR(0x16, 0x00),
+	ILI9881C_COMMAND_INSTR(0x17, 0x00),
+	ILI9881C_COMMAND_INSTR(0x18, 0x00),
+	ILI9881C_COMMAND_INSTR(0x19, 0x00),
+	ILI9881C_COMMAND_INSTR(0x1a, 0x00),
+	ILI9881C_COMMAND_INSTR(0x1b, 0x00),
+	ILI9881C_COMMAND_INSTR(0x1c, 0x00),
+	ILI9881C_COMMAND_INSTR(0x1d, 0x00),
+	ILI9881C_COMMAND_INSTR(0x1e, 0x40),
+	ILI9881C_COMMAND_INSTR(0x1f, 0x80),
+	ILI9881C_COMMAND_INSTR(0x20, 0x05),
+	ILI9881C_COMMAND_INSTR(0x20, 0x05),
+	ILI9881C_COMMAND_INSTR(0x21, 0x02),
+	ILI9881C_COMMAND_INSTR(0x22, 0x00),
+	ILI9881C_COMMAND_INSTR(0x23, 0x00),
+	ILI9881C_COMMAND_INSTR(0x24, 0x00),
+	ILI9881C_COMMAND_INSTR(0x25, 0x00),
+	ILI9881C_COMMAND_INSTR(0x26, 0x00),
+	ILI9881C_COMMAND_INSTR(0x27, 0x00),
+	ILI9881C_COMMAND_INSTR(0x28, 0x33),
+	ILI9881C_COMMAND_INSTR(0x29, 0x02),
+	ILI9881C_COMMAND_INSTR(0x2a, 0x00),
+	ILI9881C_COMMAND_INSTR(0x2b, 0x00),
+	ILI9881C_COMMAND_INSTR(0x2c, 0x00),
+	ILI9881C_COMMAND_INSTR(0x2d, 0x00),
+	ILI9881C_COMMAND_INSTR(0x2e, 0x00),
+	ILI9881C_COMMAND_INSTR(0x2f, 0x00),
+	ILI9881C_COMMAND_INSTR(0x30, 0x00),
+	ILI9881C_COMMAND_INSTR(0x31, 0x00),
+	ILI9881C_COMMAND_INSTR(0x32, 0x00),
+	ILI9881C_COMMAND_INSTR(0x32, 0x00),
+	ILI9881C_COMMAND_INSTR(0x33, 0x00),
+	ILI9881C_COMMAND_INSTR(0x34, 0x04),
+	ILI9881C_COMMAND_INSTR(0x35, 0x00),
+	ILI9881C_COMMAND_INSTR(0x36, 0x00),
+	ILI9881C_COMMAND_INSTR(0x37, 0x00),
+	ILI9881C_COMMAND_INSTR(0x38, 0x3C),
+	ILI9881C_COMMAND_INSTR(0x39, 0x00),
+	ILI9881C_COMMAND_INSTR(0x3a, 0x40),
+	ILI9881C_COMMAND_INSTR(0x3b, 0x40),
+	ILI9881C_COMMAND_INSTR(0x3c, 0x00),
+	ILI9881C_COMMAND_INSTR(0x3d, 0x00),
+	ILI9881C_COMMAND_INSTR(0x3e, 0x00),
+	ILI9881C_COMMAND_INSTR(0x3f, 0x00),
+	ILI9881C_COMMAND_INSTR(0x40, 0x00),
+	ILI9881C_COMMAND_INSTR(0x41, 0x00),
+	ILI9881C_COMMAND_INSTR(0x42, 0x00),
+	ILI9881C_COMMAND_INSTR(0x43, 0x00),
+	ILI9881C_COMMAND_INSTR(0x44, 0x00),
+	ILI9881C_COMMAND_INSTR(0x50, 0x01),
+	ILI9881C_COMMAND_INSTR(0x51, 0x23),
+	ILI9881C_COMMAND_INSTR(0x52, 0x45),
+	ILI9881C_COMMAND_INSTR(0x53, 0x67),
+	ILI9881C_COMMAND_INSTR(0x54, 0x89),
+	ILI9881C_COMMAND_INSTR(0x55, 0xab),
+	ILI9881C_COMMAND_INSTR(0x56, 0x01),
+	ILI9881C_COMMAND_INSTR(0x57, 0x23),
+	ILI9881C_COMMAND_INSTR(0x58, 0x45),
+	ILI9881C_COMMAND_INSTR(0x59, 0x67),
+	ILI9881C_COMMAND_INSTR(0x5a, 0x89),
+	ILI9881C_COMMAND_INSTR(0x5b, 0xab),
+	ILI9881C_COMMAND_INSTR(0x5c, 0xcd),
+	ILI9881C_COMMAND_INSTR(0x5d, 0xef),
+	ILI9881C_COMMAND_INSTR(0x5e, 0x11),
+	ILI9881C_COMMAND_INSTR(0x5f, 0x01),
+	ILI9881C_COMMAND_INSTR(0x60, 0x00),
+	ILI9881C_COMMAND_INSTR(0x61, 0x15),
+	ILI9881C_COMMAND_INSTR(0x62, 0x14),
+	ILI9881C_COMMAND_INSTR(0x63, 0x0E),
+	ILI9881C_COMMAND_INSTR(0x64, 0x0F),
+	ILI9881C_COMMAND_INSTR(0x65, 0x0C),
+	ILI9881C_COMMAND_INSTR(0x66, 0x0D),
+	ILI9881C_COMMAND_INSTR(0x67, 0x06),
+	ILI9881C_COMMAND_INSTR(0x68, 0x02),
+	ILI9881C_COMMAND_INSTR(0x69, 0x07),
+	ILI9881C_COMMAND_INSTR(0x6a, 0x02),
+	ILI9881C_COMMAND_INSTR(0x6b, 0x02),
+	ILI9881C_COMMAND_INSTR(0x6c, 0x02),
+	ILI9881C_COMMAND_INSTR(0x6d, 0x02),
+	ILI9881C_COMMAND_INSTR(0x6e, 0x02),
+	ILI9881C_COMMAND_INSTR(0x6f, 0x02),
+	ILI9881C_COMMAND_INSTR(0x70, 0x02),
+	ILI9881C_COMMAND_INSTR(0x71, 0x02),
+	ILI9881C_COMMAND_INSTR(0x72, 0x02),
+	ILI9881C_COMMAND_INSTR(0x73, 0x02),
+	ILI9881C_COMMAND_INSTR(0x74, 0x02),
+	ILI9881C_COMMAND_INSTR(0x75, 0x01),
+	ILI9881C_COMMAND_INSTR(0x76, 0x00),
+	ILI9881C_COMMAND_INSTR(0x77, 0x14),
+	ILI9881C_COMMAND_INSTR(0x78, 0x15),
+	ILI9881C_COMMAND_INSTR(0x79, 0x0E),
+	ILI9881C_COMMAND_INSTR(0x7a, 0x0F),
+	ILI9881C_COMMAND_INSTR(0x7b, 0x0C),
+	ILI9881C_COMMAND_INSTR(0x7c, 0x0D),
+	ILI9881C_COMMAND_INSTR(0x7d, 0x06),
+	ILI9881C_COMMAND_INSTR(0x7e, 0x02),
+	ILI9881C_COMMAND_INSTR(0x7f, 0x07),
+	ILI9881C_COMMAND_INSTR(0x80, 0x02),
+	ILI9881C_COMMAND_INSTR(0x81, 0x02),
+	ILI9881C_COMMAND_INSTR(0x83, 0x02),
+	ILI9881C_COMMAND_INSTR(0x84, 0x02),
+	ILI9881C_COMMAND_INSTR(0x85, 0x02),
+	ILI9881C_COMMAND_INSTR(0x86, 0x02),
+	ILI9881C_COMMAND_INSTR(0x87, 0x02),
+	ILI9881C_COMMAND_INSTR(0x88, 0x02),
+	ILI9881C_COMMAND_INSTR(0x89, 0x02),
+	ILI9881C_COMMAND_INSTR(0x8A, 0x02),
+	ILI9881C_SWITCH_PAGE_INSTR(0x4),
+	ILI9881C_COMMAND_INSTR(0x6C, 0x15),
+	ILI9881C_COMMAND_INSTR(0x6E, 0x2A),
+	ILI9881C_COMMAND_INSTR(0x6F, 0x33),
+	ILI9881C_COMMAND_INSTR(0x3A, 0x94),
+	ILI9881C_COMMAND_INSTR(0x8D, 0x15),
+	ILI9881C_COMMAND_INSTR(0x87, 0xBA),
+	ILI9881C_COMMAND_INSTR(0x26, 0x76),
+	ILI9881C_COMMAND_INSTR(0xB2, 0xD1),
+	ILI9881C_COMMAND_INSTR(0xB5, 0x06),
+	ILI9881C_SWITCH_PAGE_INSTR(0x1),
+	ILI9881C_COMMAND_INSTR(0x22, 0x0A),
+	ILI9881C_COMMAND_INSTR(0x31, 0x00),
+	ILI9881C_COMMAND_INSTR(0x53, 0x90),
+	ILI9881C_COMMAND_INSTR(0x55, 0xA2),
+	ILI9881C_COMMAND_INSTR(0x50, 0xB7),
+	ILI9881C_COMMAND_INSTR(0x51, 0xB7),
+	ILI9881C_COMMAND_INSTR(0x60, 0x22),
+	ILI9881C_COMMAND_INSTR(0x61, 0x00),
+	ILI9881C_COMMAND_INSTR(0x62, 0x19),
+	ILI9881C_COMMAND_INSTR(0x63, 0x10),
+	ILI9881C_COMMAND_INSTR(0xA0, 0x08),
+	ILI9881C_COMMAND_INSTR(0xA1, 0x1A),
+	ILI9881C_COMMAND_INSTR(0xA2, 0x27),
+	ILI9881C_COMMAND_INSTR(0xA3, 0x15),
+	ILI9881C_COMMAND_INSTR(0xA4, 0x17),
+	ILI9881C_COMMAND_INSTR(0xA5, 0x2A),
+	ILI9881C_COMMAND_INSTR(0xA6, 0x1E),
+	ILI9881C_COMMAND_INSTR(0xA7, 0x1F),
+	ILI9881C_COMMAND_INSTR(0xA8, 0x8B),
+	ILI9881C_COMMAND_INSTR(0xA9, 0x1B),
+	ILI9881C_COMMAND_INSTR(0xAA, 0x27),
+	ILI9881C_COMMAND_INSTR(0xAB, 0x78),
+	ILI9881C_COMMAND_INSTR(0xAC, 0x18),
+	ILI9881C_COMMAND_INSTR(0xAD, 0x18),
+	ILI9881C_COMMAND_INSTR(0xAE, 0x4C),
+	ILI9881C_COMMAND_INSTR(0xAF, 0x21),
+	ILI9881C_COMMAND_INSTR(0xB0, 0x27),
+	ILI9881C_COMMAND_INSTR(0xB1, 0x54),
+	ILI9881C_COMMAND_INSTR(0xB2, 0x67),
+	ILI9881C_COMMAND_INSTR(0xB3, 0x39),
+	ILI9881C_COMMAND_INSTR(0xC0, 0x08),
+	ILI9881C_COMMAND_INSTR(0xC1, 0x1A),
+	ILI9881C_COMMAND_INSTR(0xC2, 0x27),
+	ILI9881C_COMMAND_INSTR(0xC3, 0x15),
+	ILI9881C_COMMAND_INSTR(0xC4, 0x17),
+	ILI9881C_COMMAND_INSTR(0xC5, 0x2A),
+	ILI9881C_COMMAND_INSTR(0xC6, 0x1E),
+	ILI9881C_COMMAND_INSTR(0xC7, 0x1F),
+	ILI9881C_COMMAND_INSTR(0xC8, 0x8B),
+	ILI9881C_COMMAND_INSTR(0xC9, 0x1B),
+	ILI9881C_COMMAND_INSTR(0xCA, 0x27),
+	ILI9881C_COMMAND_INSTR(0xCB, 0x78),
+	ILI9881C_COMMAND_INSTR(0xCC, 0x18),
+	ILI9881C_COMMAND_INSTR(0xCD, 0x18),
+	ILI9881C_COMMAND_INSTR(0xCE, 0x4C),
+	ILI9881C_COMMAND_INSTR(0xCF, 0x21),
+	ILI9881C_COMMAND_INSTR(0xD0, 0x27),
+	ILI9881C_COMMAND_INSTR(0xD1, 0x54),
+	ILI9881C_COMMAND_INSTR(0xD2, 0x67),
+	ILI9881C_COMMAND_INSTR(0xD3, 0x39),
+	ILI9881C_SWITCH_PAGE_INSTR(0),
+	ILI9881C_COMMAND_INSTR(0x35, 0x00),
+	ILI9881C_COMMAND_INSTR(0x3A, 0x7),
+};
+
 static const struct ili9881c_instr tl050hdv35_init[] = {
 	ILI9881C_SWITCH_PAGE_INSTR(3),
 	ILI9881C_COMMAND_INSTR(0x01, 0x00),
@@ -1080,10 +1276,10 @@ static int ili9881c_prepare(struct drm_panel *panel)
 	msleep(5);
 
 	/* And reset it */
-	gpiod_set_value(ctx->reset, 1);
+	gpiod_set_value_cansleep(ctx->reset, 1);
 	msleep(20);
 
-	gpiod_set_value(ctx->reset, 0);
+	gpiod_set_value_cansleep(ctx->reset, 0);
 	msleep(20);
 
 	for (i = 0; i < ctx->desc->init_length; i++) {
@@ -1138,7 +1334,7 @@ static int ili9881c_unprepare(struct drm_panel *panel)
 
 	mipi_dsi_dcs_enter_sleep_mode(ctx->dsi);
 	regulator_disable(ctx->power);
-	gpiod_set_value(ctx->reset, 1);
+	gpiod_set_value_cansleep(ctx->reset, 1);
 
 	return 0;
 }
@@ -1177,6 +1373,23 @@ static const struct drm_display_mode k101_im2byl02_default_mode = {
 	.height_mm	= 217,
 };
 
+static const struct drm_display_mode kd050hdfia020_default_mode = {
+	.clock		= 62000,
+
+	.hdisplay	= 720,
+	.hsync_start	= 720 + 10,
+	.hsync_end	= 720 + 10 + 20,
+	.htotal		= 720 + 10 + 20 + 30,
+
+	.vdisplay	= 1280,
+	.vsync_start	= 1280 + 10,
+	.vsync_end	= 1280 + 10 + 10,
+	.vtotal		= 1280 + 10 + 10 + 20,
+
+	.width_mm	= 62,
+	.height_mm	= 110,
+};
+
 static const struct drm_display_mode tl050hdv35_default_mode = {
 	.clock		= 59400,
 
@@ -1345,6 +1558,14 @@ static const struct ili9881c_desc k101_im2byl02_desc = {
 	.mode_flags = MIPI_DSI_MODE_VIDEO_SYNC_PULSE,
 };
 
+static const struct ili9881c_desc kd050hdfia020_desc = {
+	.init = kd050hdfia020_init,
+	.init_length = ARRAY_SIZE(kd050hdfia020_init),
+	.mode = &kd050hdfia020_default_mode,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_SYNC_PULSE |
+		      MIPI_DSI_MODE_LPM,
+};
+
 static const struct ili9881c_desc tl050hdv35_desc = {
 	.init = tl050hdv35_init,
 	.init_length = ARRAY_SIZE(tl050hdv35_init),
@@ -1372,6 +1593,7 @@ static const struct ili9881c_desc am8001280g_desc = {
 static const struct of_device_id ili9881c_of_match[] = {
 	{ .compatible = "bananapi,lhr050h41", .data = &lhr050h41_desc },
 	{ .compatible = "feixin,k101-im2byl02", .data = &k101_im2byl02_desc },
+	{ .compatible = "startek,kd050hdfia020", .data = &kd050hdfia020_desc },
 	{ .compatible = "tdo,tl050hdv35", .data = &tl050hdv35_desc },
 	{ .compatible = "wanchanglong,w552946aba", .data = &w552946aba_desc },
 	{ .compatible = "ampire,am8001280g", .data = &am8001280g_desc },
diff --git a/drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c b/drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c
index 3e0a8e0d58a0..483dc88d16d8 100644
--- a/drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c
+++ b/drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c
@@ -247,6 +247,7 @@ static int jdi_fhd_r63452_probe(struct mipi_dsi_device *dsi)
 
 	drm_panel_init(&ctx->panel, dev, &jdi_fhd_r63452_panel_funcs,
 		       DRM_MODE_CONNECTOR_DSI);
+	ctx->panel.prepare_prev_first = true;
 
 	ret = drm_panel_of_backlight(&ctx->panel);
 	if (ret)
diff --git a/drivers/gpu/drm/panel/panel-khadas-ts050.c b/drivers/gpu/drm/panel/panel-khadas-ts050.c
index b942a0162274..c54be0cc3f08 100644
--- a/drivers/gpu/drm/panel/panel-khadas-ts050.c
+++ b/drivers/gpu/drm/panel/panel-khadas-ts050.c
@@ -25,6 +25,7 @@ struct khadas_ts050_panel {
 	struct regulator *supply;
 	struct gpio_desc *reset_gpio;
 	struct gpio_desc *enable_gpio;
+	struct khadas_ts050_panel_data *panel_data;
 
 	bool prepared;
 	bool enabled;
@@ -32,544 +33,601 @@ struct khadas_ts050_panel {
 
 struct khadas_ts050_panel_cmd {
 	u8 cmd;
-	u8 data;
+	u8 data[55];
+	u8 size;
+};
+
+struct khadas_ts050_panel_data {
+	struct khadas_ts050_panel_cmd *init_code;
+	int len;
+};
+
+static const struct khadas_ts050_panel_cmd ts050v2_init_code[] = {
+	{0xB9, {0xFF, 0x83, 0x99}, 0x03},
+	{0xBA, {0x63, 0x23, 0x68, 0xCF}, 0x04},
+	{0xD2, {0x55}, 0x01},
+	{0xB1, {0x02, 0x04, 0x70, 0x90, 0x01, 0x32, 0x33,
+			0x11, 0x11, 0x4D, 0x57, 0x56, 0x73, 0x02, 0x02}, 0x0f},
+	{0xB2, {0x00, 0x80, 0x80, 0xAE, 0x0A, 0x0E, 0x75, 0x11, 0x00, 0x00, 0x00}, 0x0b},
+	{0xB4, {0x00, 0xFF, 0x04, 0xA4, 0x02, 0xA0, 0x00, 0x00, 0x10, 0x00, 0x00, 0x02,
+			0x00, 0x24,	0x02, 0x04, 0x0A, 0x21, 0x03, 0x00, 0x00, 0x08, 0xA6, 0x88,
+			0x04, 0xA4, 0x02, 0xA0,	0x00, 0x00,	0x10, 0x00, 0x00, 0x02, 0x00, 0x24,
+			0x02, 0x04, 0x0A, 0x00, 0x00, 0x08,	0xA6, 0x00, 0x08, 0x11}, 0x2e},
+	{0xD3, {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18,
+			0x18, 0x32, 0x10, 0x09, 0x00, 0x09, 0x32,
+			0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x11, 0x00, 0x02, 0x02, 0x03, 0x00, 0x00, 0x00, 0x0A,
+			0x40}, 0x21},
+	{0xD5, {0x18, 0x18, 0x18, 0x18, 0x21, 0x20, 0x18, 0x18, 0x19, 0x19, 0x19,
+			0x19, 0x18, 0x18, 0x18, 0x18, 0x03, 0x02, 0x01, 0x00, 0x2F, 0x2F,
+			0x30, 0x30, 0x31, 0x31, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18}, 0x20},
+	{0xD6, {0x18, 0x18, 0x18, 0x18, 0x20, 0x21, 0x19, 0x19, 0x18, 0x18, 0x19,
+			0x19, 0x18, 0x18, 0x18, 0x18, 0x00, 0x01, 0x02, 0x03, 0x2F, 0x2F,
+			0x30, 0x30, 0x31, 0x31, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18}, 0x20},
+	{0xD8, {0x0A, 0xBE, 0xFA, 0xA0, 0x0A, 0xBE, 0xFA, 0xA0}, 0x08},
+	{0xBD, {0x01}, 0x01},
+	{0xD8, {0x0F, 0xFF, 0xFF, 0xE0, 0x0F, 0xFF, 0xFF, 0xE0}, 0x08},
+	{0xBD, {0x02}, 0x01},
+	{0xD8, {0x0F, 0xFF, 0xFF, 0xE0, 0x0F, 0xFF, 0xFF, 0xE0}, 0x08},
+	{0xBD, {0x00}, 0x01},
+	{0xE0, {0x01, 0x35, 0x41, 0x3B, 0x79, 0x81, 0x8C, 0x85, 0x8E,
+			0x95, 0x9B, 0xA0, 0xA4, 0xAB, 0xB1, 0xB3, 0xB7, 0xC5, 0xBD, 0xC5,
+			0xB6, 0xC2, 0xC2, 0x62, 0x5D, 0x66, 0x73, 0x01, 0x35, 0x41, 0x3B,
+			0x79, 0x81, 0x8C, 0x85, 0x8E, 0x95, 0x9B, 0xA0, 0xA4, 0xAB, 0xB1,
+			0xB3, 0xB7, 0xB5, 0xBD, 0xC5, 0xB6, 0xC2, 0xC2, 0x62, 0x5D, 0x66,
+			0x73}, 0x36},
+	{0xB6, {0x97, 0x97}, 0x02},
+	{0xCC, {0xC8}, 0x02},
+	{0xBF, {0x40, 0x41, 0x50, 0x19}, 0x04},
+	{0xC6, {0xFF, 0xF9}, 0x02},
+	{0xC0, {0x25, 0x5A}, 0x02},
 };
 
 /* Only the CMD1 User Command set is documented */
-static const struct khadas_ts050_panel_cmd init_code[] = {
+static const struct khadas_ts050_panel_cmd ts050_init_code[] = {
 	/* Select Unknown CMD Page (Undocumented) */
-	{0xff, 0xee},
+	{0xff, {0xee}, 0x01},
 	/* Reload CMD1: Don't reload default value to register */
-	{0xfb, 0x01},
-	{0x1f, 0x45},
-	{0x24, 0x4f},
-	{0x38, 0xc8},
-	{0x39, 0x27},
-	{0x1e, 0x77},
-	{0x1d, 0x0f},
-	{0x7e, 0x71},
-	{0x7c, 0x03},
-	{0xff, 0x00},
-	{0xfb, 0x01},
-	{0x35, 0x01},
+	{0xfb, {0x01}, 0x01},
+	{0x1f, {0x45}, 0x01},
+	{0x24, {0x4f}, 0x01},
+	{0x38, {0xc8}, 0x01},
+	{0x39, {0x27}, 0x01},
+	{0x1e, {0x77}, 0x01},
+	{0x1d, {0x0f}, 0x01},
+	{0x7e, {0x71}, 0x01},
+	{0x7c, {0x03}, 0x01},
+	{0xff, {0x00}, 0x01},
+	{0xfb, {0x01}, 0x01},
+	{0x35, {0x01}, 0x01},
 	/* Select CMD2 Page0 (Undocumented) */
-	{0xff, 0x01},
+	{0xff, {0x01}, 0x01},
 	/* Reload CMD1: Don't reload default value to register */
-	{0xfb, 0x01},
-	{0x00, 0x01},
-	{0x01, 0x55},
-	{0x02, 0x40},
-	{0x05, 0x40},
-	{0x06, 0x4a},
-	{0x07, 0x24},
-	{0x08, 0x0c},
-	{0x0b, 0x7d},
-	{0x0c, 0x7d},
-	{0x0e, 0xb0},
-	{0x0f, 0xae},
-	{0x11, 0x10},
-	{0x12, 0x10},
-	{0x13, 0x03},
-	{0x14, 0x4a},
-	{0x15, 0x12},
-	{0x16, 0x12},
-	{0x18, 0x00},
-	{0x19, 0x77},
-	{0x1a, 0x55},
-	{0x1b, 0x13},
-	{0x1c, 0x00},
-	{0x1d, 0x00},
-	{0x1e, 0x13},
-	{0x1f, 0x00},
-	{0x23, 0x00},
-	{0x24, 0x00},
-	{0x25, 0x00},
-	{0x26, 0x00},
-	{0x27, 0x00},
-	{0x28, 0x00},
-	{0x35, 0x00},
-	{0x66, 0x00},
-	{0x58, 0x82},
-	{0x59, 0x02},
-	{0x5a, 0x02},
-	{0x5b, 0x02},
-	{0x5c, 0x82},
-	{0x5d, 0x82},
-	{0x5e, 0x02},
-	{0x5f, 0x02},
-	{0x72, 0x31},
+	{0xfb, {0x01}, 0x01},
+	{0x00, {0x01}, 0x01},
+	{0x01, {0x55}, 0x01},
+	{0x02, {0x40}, 0x01},
+	{0x05, {0x40}, 0x01},
+	{0x06, {0x4a}, 0x01},
+	{0x07, {0x24}, 0x01},
+	{0x08, {0x0c}, 0x01},
+	{0x0b, {0x7d}, 0x01},
+	{0x0c, {0x7d}, 0x01},
+	{0x0e, {0xb0}, 0x01},
+	{0x0f, {0xae}, 0x01},
+	{0x11, {0x10}, 0x01},
+	{0x12, {0x10}, 0x01},
+	{0x13, {0x03}, 0x01},
+	{0x14, {0x4a}, 0x01},
+	{0x15, {0x12}, 0x01},
+	{0x16, {0x12}, 0x01},
+	{0x18, {0x00}, 0x01},
+	{0x19, {0x77}, 0x01},
+	{0x1a, {0x55}, 0x01},
+	{0x1b, {0x13}, 0x01},
+	{0x1c, {0x00}, 0x01},
+	{0x1d, {0x00}, 0x01},
+	{0x1e, {0x13}, 0x01},
+	{0x1f, {0x00}, 0x01},
+	{0x23, {0x00}, 0x01},
+	{0x24, {0x00}, 0x01},
+	{0x25, {0x00}, 0x01},
+	{0x26, {0x00}, 0x01},
+	{0x27, {0x00}, 0x01},
+	{0x28, {0x00}, 0x01},
+	{0x35, {0x00}, 0x01},
+	{0x66, {0x00}, 0x01},
+	{0x58, {0x82}, 0x01},
+	{0x59, {0x02}, 0x01},
+	{0x5a, {0x02}, 0x01},
+	{0x5b, {0x02}, 0x01},
+	{0x5c, {0x82}, 0x01},
+	{0x5d, {0x82}, 0x01},
+	{0x5e, {0x02}, 0x01},
+	{0x5f, {0x02}, 0x01},
+	{0x72, {0x31}, 0x01},
 	/* Select CMD2 Page4 (Undocumented) */
-	{0xff, 0x05},
+	{0xff, {0x05}, 0x01},
 	/* Reload CMD1: Don't reload default value to register */
-	{0xfb, 0x01},
-	{0x00, 0x01},
-	{0x01, 0x0b},
-	{0x02, 0x0c},
-	{0x03, 0x09},
-	{0x04, 0x0a},
-	{0x05, 0x00},
-	{0x06, 0x0f},
-	{0x07, 0x10},
-	{0x08, 0x00},
-	{0x09, 0x00},
-	{0x0a, 0x00},
-	{0x0b, 0x00},
-	{0x0c, 0x00},
-	{0x0d, 0x13},
-	{0x0e, 0x15},
-	{0x0f, 0x17},
-	{0x10, 0x01},
-	{0x11, 0x0b},
-	{0x12, 0x0c},
-	{0x13, 0x09},
-	{0x14, 0x0a},
-	{0x15, 0x00},
-	{0x16, 0x0f},
-	{0x17, 0x10},
-	{0x18, 0x00},
-	{0x19, 0x00},
-	{0x1a, 0x00},
-	{0x1b, 0x00},
-	{0x1c, 0x00},
-	{0x1d, 0x13},
-	{0x1e, 0x15},
-	{0x1f, 0x17},
-	{0x20, 0x00},
-	{0x21, 0x03},
-	{0x22, 0x01},
-	{0x23, 0x40},
-	{0x24, 0x40},
-	{0x25, 0xed},
-	{0x29, 0x58},
-	{0x2a, 0x12},
-	{0x2b, 0x01},
-	{0x4b, 0x06},
-	{0x4c, 0x11},
-	{0x4d, 0x20},
-	{0x4e, 0x02},
-	{0x4f, 0x02},
-	{0x50, 0x20},
-	{0x51, 0x61},
-	{0x52, 0x01},
-	{0x53, 0x63},
-	{0x54, 0x77},
-	{0x55, 0xed},
-	{0x5b, 0x00},
-	{0x5c, 0x00},
-	{0x5d, 0x00},
-	{0x5e, 0x00},
-	{0x5f, 0x15},
-	{0x60, 0x75},
-	{0x61, 0x00},
-	{0x62, 0x00},
-	{0x63, 0x00},
-	{0x64, 0x00},
-	{0x65, 0x00},
-	{0x66, 0x00},
-	{0x67, 0x00},
-	{0x68, 0x04},
-	{0x69, 0x00},
-	{0x6a, 0x00},
-	{0x6c, 0x40},
-	{0x75, 0x01},
-	{0x76, 0x01},
-	{0x7a, 0x80},
-	{0x7b, 0xa3},
-	{0x7c, 0xd8},
-	{0x7d, 0x60},
-	{0x7f, 0x15},
-	{0x80, 0x81},
-	{0x83, 0x05},
-	{0x93, 0x08},
-	{0x94, 0x10},
-	{0x8a, 0x00},
-	{0x9b, 0x0f},
-	{0xea, 0xff},
-	{0xec, 0x00},
+	{0xfb, {0x01}, 0x01},
+	{0x00, {0x01}, 0x01},
+	{0x01, {0x0b}, 0x01},
+	{0x02, {0x0c}, 0x01},
+	{0x03, {0x09}, 0x01},
+	{0x04, {0x0a}, 0x01},
+	{0x05, {0x00}, 0x01},
+	{0x06, {0x0f}, 0x01},
+	{0x07, {0x10}, 0x01},
+	{0x08, {0x00}, 0x01},
+	{0x09, {0x00}, 0x01},
+	{0x0a, {0x00}, 0x01},
+	{0x0b, {0x00}, 0x01},
+	{0x0c, {0x00}, 0x01},
+	{0x0d, {0x13}, 0x01},
+	{0x0e, {0x15}, 0x01},
+	{0x0f, {0x17}, 0x01},
+	{0x10, {0x01}, 0x01},
+	{0x11, {0x0b}, 0x01},
+	{0x12, {0x0c}, 0x01},
+	{0x13, {0x09}, 0x01},
+	{0x14, {0x0a}, 0x01},
+	{0x15, {0x00}, 0x01},
+	{0x16, {0x0f}, 0x01},
+	{0x17, {0x10}, 0x01},
+	{0x18, {0x00}, 0x01},
+	{0x19, {0x00}, 0x01},
+	{0x1a, {0x00}, 0x01},
+	{0x1b, {0x00}, 0x01},
+	{0x1c, {0x00}, 0x01},
+	{0x1d, {0x13}, 0x01},
+	{0x1e, {0x15}, 0x01},
+	{0x1f, {0x17}, 0x01},
+	{0x20, {0x00}, 0x01},
+	{0x21, {0x03}, 0x01},
+	{0x22, {0x01}, 0x01},
+	{0x23, {0x40}, 0x01},
+	{0x24, {0x40}, 0x01},
+	{0x25, {0xed}, 0x01},
+	{0x29, {0x58}, 0x01},
+	{0x2a, {0x12}, 0x01},
+	{0x2b, {0x01}, 0x01},
+	{0x4b, {0x06}, 0x01},
+	{0x4c, {0x11}, 0x01},
+	{0x4d, {0x20}, 0x01},
+	{0x4e, {0x02}, 0x01},
+	{0x4f, {0x02}, 0x01},
+	{0x50, {0x20}, 0x01},
+	{0x51, {0x61}, 0x01},
+	{0x52, {0x01}, 0x01},
+	{0x53, {0x63}, 0x01},
+	{0x54, {0x77}, 0x01},
+	{0x55, {0xed}, 0x01},
+	{0x5b, {0x00}, 0x01},
+	{0x5c, {0x00}, 0x01},
+	{0x5d, {0x00}, 0x01},
+	{0x5e, {0x00}, 0x01},
+	{0x5f, {0x15}, 0x01},
+	{0x60, {0x75}, 0x01},
+	{0x61, {0x00}, 0x01},
+	{0x62, {0x00}, 0x01},
+	{0x63, {0x00}, 0x01},
+	{0x64, {0x00}, 0x01},
+	{0x65, {0x00}, 0x01},
+	{0x66, {0x00}, 0x01},
+	{0x67, {0x00}, 0x01},
+	{0x68, {0x04}, 0x01},
+	{0x69, {0x00}, 0x01},
+	{0x6a, {0x00}, 0x01},
+	{0x6c, {0x40}, 0x01},
+	{0x75, {0x01}, 0x01},
+	{0x76, {0x01}, 0x01},
+	{0x7a, {0x80}, 0x01},
+	{0x7b, {0xa3}, 0x01},
+	{0x7c, {0xd8}, 0x01},
+	{0x7d, {0x60}, 0x01},
+	{0x7f, {0x15}, 0x01},
+	{0x80, {0x81}, 0x01},
+	{0x83, {0x05}, 0x01},
+	{0x93, {0x08}, 0x01},
+	{0x94, {0x10}, 0x01},
+	{0x8a, {0x00}, 0x01},
+	{0x9b, {0x0f}, 0x01},
+	{0xea, {0xff}, 0x01},
+	{0xec, {0x00}, 0x01},
 	/* Select CMD2 Page0 (Undocumented) */
-	{0xff, 0x01},
+	{0xff, {0x01}, 0x01},
 	/* Reload CMD1: Don't reload default value to register */
-	{0xfb, 0x01},
-	{0x75, 0x00},
-	{0x76, 0xdf},
-	{0x77, 0x00},
-	{0x78, 0xe4},
-	{0x79, 0x00},
-	{0x7a, 0xed},
-	{0x7b, 0x00},
-	{0x7c, 0xf6},
-	{0x7d, 0x00},
-	{0x7e, 0xff},
-	{0x7f, 0x01},
-	{0x80, 0x07},
-	{0x81, 0x01},
-	{0x82, 0x10},
-	{0x83, 0x01},
-	{0x84, 0x18},
-	{0x85, 0x01},
-	{0x86, 0x20},
-	{0x87, 0x01},
-	{0x88, 0x3d},
-	{0x89, 0x01},
-	{0x8a, 0x56},
-	{0x8b, 0x01},
-	{0x8c, 0x84},
-	{0x8d, 0x01},
-	{0x8e, 0xab},
-	{0x8f, 0x01},
-	{0x90, 0xec},
-	{0x91, 0x02},
-	{0x92, 0x22},
-	{0x93, 0x02},
-	{0x94, 0x23},
-	{0x95, 0x02},
-	{0x96, 0x55},
-	{0x97, 0x02},
-	{0x98, 0x8b},
-	{0x99, 0x02},
-	{0x9a, 0xaf},
-	{0x9b, 0x02},
-	{0x9c, 0xdf},
-	{0x9d, 0x03},
-	{0x9e, 0x01},
-	{0x9f, 0x03},
-	{0xa0, 0x2c},
-	{0xa2, 0x03},
-	{0xa3, 0x39},
-	{0xa4, 0x03},
-	{0xa5, 0x47},
-	{0xa6, 0x03},
-	{0xa7, 0x56},
-	{0xa9, 0x03},
-	{0xaa, 0x66},
-	{0xab, 0x03},
-	{0xac, 0x76},
-	{0xad, 0x03},
-	{0xae, 0x85},
-	{0xaf, 0x03},
-	{0xb0, 0x90},
-	{0xb1, 0x03},
-	{0xb2, 0xcb},
-	{0xb3, 0x00},
-	{0xb4, 0xdf},
-	{0xb5, 0x00},
-	{0xb6, 0xe4},
-	{0xb7, 0x00},
-	{0xb8, 0xed},
-	{0xb9, 0x00},
-	{0xba, 0xf6},
-	{0xbb, 0x00},
-	{0xbc, 0xff},
-	{0xbd, 0x01},
-	{0xbe, 0x07},
-	{0xbf, 0x01},
-	{0xc0, 0x10},
-	{0xc1, 0x01},
-	{0xc2, 0x18},
-	{0xc3, 0x01},
-	{0xc4, 0x20},
-	{0xc5, 0x01},
-	{0xc6, 0x3d},
-	{0xc7, 0x01},
-	{0xc8, 0x56},
-	{0xc9, 0x01},
-	{0xca, 0x84},
-	{0xcb, 0x01},
-	{0xcc, 0xab},
-	{0xcd, 0x01},
-	{0xce, 0xec},
-	{0xcf, 0x02},
-	{0xd0, 0x22},
-	{0xd1, 0x02},
-	{0xd2, 0x23},
-	{0xd3, 0x02},
-	{0xd4, 0x55},
-	{0xd5, 0x02},
-	{0xd6, 0x8b},
-	{0xd7, 0x02},
-	{0xd8, 0xaf},
-	{0xd9, 0x02},
-	{0xda, 0xdf},
-	{0xdb, 0x03},
-	{0xdc, 0x01},
-	{0xdd, 0x03},
-	{0xde, 0x2c},
-	{0xdf, 0x03},
-	{0xe0, 0x39},
-	{0xe1, 0x03},
-	{0xe2, 0x47},
-	{0xe3, 0x03},
-	{0xe4, 0x56},
-	{0xe5, 0x03},
-	{0xe6, 0x66},
-	{0xe7, 0x03},
-	{0xe8, 0x76},
-	{0xe9, 0x03},
-	{0xea, 0x85},
-	{0xeb, 0x03},
-	{0xec, 0x90},
-	{0xed, 0x03},
-	{0xee, 0xcb},
-	{0xef, 0x00},
-	{0xf0, 0xbb},
-	{0xf1, 0x00},
-	{0xf2, 0xc0},
-	{0xf3, 0x00},
-	{0xf4, 0xcc},
-	{0xf5, 0x00},
-	{0xf6, 0xd6},
-	{0xf7, 0x00},
-	{0xf8, 0xe1},
-	{0xf9, 0x00},
-	{0xfa, 0xea},
+	{0xfb, {0x01}, 0x01},
+	{0x75, {0x00}, 0x01},
+	{0x76, {0xdf}, 0x01},
+	{0x77, {0x00}, 0x01},
+	{0x78, {0xe4}, 0x01},
+	{0x79, {0x00}, 0x01},
+	{0x7a, {0xed}, 0x01},
+	{0x7b, {0x00}, 0x01},
+	{0x7c, {0xf6}, 0x01},
+	{0x7d, {0x00}, 0x01},
+	{0x7e, {0xff}, 0x01},
+	{0x7f, {0x01}, 0x01},
+	{0x80, {0x07}, 0x01},
+	{0x81, {0x01}, 0x01},
+	{0x82, {0x10}, 0x01},
+	{0x83, {0x01}, 0x01},
+	{0x84, {0x18}, 0x01},
+	{0x85, {0x01}, 0x01},
+	{0x86, {0x20}, 0x01},
+	{0x87, {0x01}, 0x01},
+	{0x88, {0x3d}, 0x01},
+	{0x89, {0x01}, 0x01},
+	{0x8a, {0x56}, 0x01},
+	{0x8b, {0x01}, 0x01},
+	{0x8c, {0x84}, 0x01},
+	{0x8d, {0x01}, 0x01},
+	{0x8e, {0xab}, 0x01},
+	{0x8f, {0x01}, 0x01},
+	{0x90, {0xec}, 0x01},
+	{0x91, {0x02}, 0x01},
+	{0x92, {0x22}, 0x01},
+	{0x93, {0x02}, 0x01},
+	{0x94, {0x23}, 0x01},
+	{0x95, {0x02}, 0x01},
+	{0x96, {0x55}, 0x01},
+	{0x97, {0x02}, 0x01},
+	{0x98, {0x8b}, 0x01},
+	{0x99, {0x02}, 0x01},
+	{0x9a, {0xaf}, 0x01},
+	{0x9b, {0x02}, 0x01},
+	{0x9c, {0xdf}, 0x01},
+	{0x9d, {0x03}, 0x01},
+	{0x9e, {0x01}, 0x01},
+	{0x9f, {0x03}, 0x01},
+	{0xa0, {0x2c}, 0x01},
+	{0xa2, {0x03}, 0x01},
+	{0xa3, {0x39}, 0x01},
+	{0xa4, {0x03}, 0x01},
+	{0xa5, {0x47}, 0x01},
+	{0xa6, {0x03}, 0x01},
+	{0xa7, {0x56}, 0x01},
+	{0xa9, {0x03}, 0x01},
+	{0xaa, {0x66}, 0x01},
+	{0xab, {0x03}, 0x01},
+	{0xac, {0x76}, 0x01},
+	{0xad, {0x03}, 0x01},
+	{0xae, {0x85}, 0x01},
+	{0xaf, {0x03}, 0x01},
+	{0xb0, {0x90}, 0x01},
+	{0xb1, {0x03}, 0x01},
+	{0xb2, {0xcb}, 0x01},
+	{0xb3, {0x00}, 0x01},
+	{0xb4, {0xdf}, 0x01},
+	{0xb5, {0x00}, 0x01},
+	{0xb6, {0xe4}, 0x01},
+	{0xb7, {0x00}, 0x01},
+	{0xb8, {0xed}, 0x01},
+	{0xb9, {0x00}, 0x01},
+	{0xba, {0xf6}, 0x01},
+	{0xbb, {0x00}, 0x01},
+	{0xbc, {0xff}, 0x01},
+	{0xbd, {0x01}, 0x01},
+	{0xbe, {0x07}, 0x01},
+	{0xbf, {0x01}, 0x01},
+	{0xc0, {0x10}, 0x01},
+	{0xc1, {0x01}, 0x01},
+	{0xc2, {0x18}, 0x01},
+	{0xc3, {0x01}, 0x01},
+	{0xc4, {0x20}, 0x01},
+	{0xc5, {0x01}, 0x01},
+	{0xc6, {0x3d}, 0x01},
+	{0xc7, {0x01}, 0x01},
+	{0xc8, {0x56}, 0x01},
+	{0xc9, {0x01}, 0x01},
+	{0xca, {0x84}, 0x01},
+	{0xcb, {0x01}, 0x01},
+	{0xcc, {0xab}, 0x01},
+	{0xcd, {0x01}, 0x01},
+	{0xce, {0xec}, 0x01},
+	{0xcf, {0x02}, 0x01},
+	{0xd0, {0x22}, 0x01},
+	{0xd1, {0x02}, 0x01},
+	{0xd2, {0x23}, 0x01},
+	{0xd3, {0x02}, 0x01},
+	{0xd4, {0x55}, 0x01},
+	{0xd5, {0x02}, 0x01},
+	{0xd6, {0x8b}, 0x01},
+	{0xd7, {0x02}, 0x01},
+	{0xd8, {0xaf}, 0x01},
+	{0xd9, {0x02}, 0x01},
+	{0xda, {0xdf}, 0x01},
+	{0xdb, {0x03}, 0x01},
+	{0xdc, {0x01}, 0x01},
+	{0xdd, {0x03}, 0x01},
+	{0xde, {0x2c}, 0x01},
+	{0xdf, {0x03}, 0x01},
+	{0xe0, {0x39}, 0x01},
+	{0xe1, {0x03}, 0x01},
+	{0xe2, {0x47}, 0x01},
+	{0xe3, {0x03}, 0x01},
+	{0xe4, {0x56}, 0x01},
+	{0xe5, {0x03}, 0x01},
+	{0xe6, {0x66}, 0x01},
+	{0xe7, {0x03}, 0x01},
+	{0xe8, {0x76}, 0x01},
+	{0xe9, {0x03}, 0x01},
+	{0xea, {0x85}, 0x01},
+	{0xeb, {0x03}, 0x01},
+	{0xec, {0x90}, 0x01},
+	{0xed, {0x03}, 0x01},
+	{0xee, {0xcb}, 0x01},
+	{0xef, {0x00}, 0x01},
+	{0xf0, {0xbb}, 0x01},
+	{0xf1, {0x00}, 0x01},
+	{0xf2, {0xc0}, 0x01},
+	{0xf3, {0x00}, 0x01},
+	{0xf4, {0xcc}, 0x01},
+	{0xf5, {0x00}, 0x01},
+	{0xf6, {0xd6}, 0x01},
+	{0xf7, {0x00}, 0x01},
+	{0xf8, {0xe1}, 0x01},
+	{0xf9, {0x00}, 0x01},
+	{0xfa, {0xea}, 0x01},
 	/* Select CMD2 Page2 (Undocumented) */
-	{0xff, 0x02},
+	{0xff, {0x02}, 0x01},
 	/* Reload CMD1: Don't reload default value to register */
-	{0xfb, 0x01},
-	{0x00, 0x00},
-	{0x01, 0xf4},
-	{0x02, 0x00},
-	{0x03, 0xef},
-	{0x04, 0x01},
-	{0x05, 0x07},
-	{0x06, 0x01},
-	{0x07, 0x28},
-	{0x08, 0x01},
-	{0x09, 0x44},
-	{0x0a, 0x01},
-	{0x0b, 0x76},
-	{0x0c, 0x01},
-	{0x0d, 0xa0},
-	{0x0e, 0x01},
-	{0x0f, 0xe7},
-	{0x10, 0x02},
-	{0x11, 0x1f},
-	{0x12, 0x02},
-	{0x13, 0x22},
-	{0x14, 0x02},
-	{0x15, 0x54},
-	{0x16, 0x02},
-	{0x17, 0x8b},
-	{0x18, 0x02},
-	{0x19, 0xaf},
-	{0x1a, 0x02},
-	{0x1b, 0xe0},
-	{0x1c, 0x03},
-	{0x1d, 0x01},
-	{0x1e, 0x03},
-	{0x1f, 0x2d},
-	{0x20, 0x03},
-	{0x21, 0x39},
-	{0x22, 0x03},
-	{0x23, 0x47},
-	{0x24, 0x03},
-	{0x25, 0x57},
-	{0x26, 0x03},
-	{0x27, 0x65},
-	{0x28, 0x03},
-	{0x29, 0x77},
-	{0x2a, 0x03},
-	{0x2b, 0x85},
-	{0x2d, 0x03},
-	{0x2f, 0x8f},
-	{0x30, 0x03},
-	{0x31, 0xcb},
-	{0x32, 0x00},
-	{0x33, 0xbb},
-	{0x34, 0x00},
-	{0x35, 0xc0},
-	{0x36, 0x00},
-	{0x37, 0xcc},
-	{0x38, 0x00},
-	{0x39, 0xd6},
-	{0x3a, 0x00},
-	{0x3b, 0xe1},
-	{0x3d, 0x00},
-	{0x3f, 0xea},
-	{0x40, 0x00},
-	{0x41, 0xf4},
-	{0x42, 0x00},
-	{0x43, 0xfe},
-	{0x44, 0x01},
-	{0x45, 0x07},
-	{0x46, 0x01},
-	{0x47, 0x28},
-	{0x48, 0x01},
-	{0x49, 0x44},
-	{0x4a, 0x01},
-	{0x4b, 0x76},
-	{0x4c, 0x01},
-	{0x4d, 0xa0},
-	{0x4e, 0x01},
-	{0x4f, 0xe7},
-	{0x50, 0x02},
-	{0x51, 0x1f},
-	{0x52, 0x02},
-	{0x53, 0x22},
-	{0x54, 0x02},
-	{0x55, 0x54},
-	{0x56, 0x02},
-	{0x58, 0x8b},
-	{0x59, 0x02},
-	{0x5a, 0xaf},
-	{0x5b, 0x02},
-	{0x5c, 0xe0},
-	{0x5d, 0x03},
-	{0x5e, 0x01},
-	{0x5f, 0x03},
-	{0x60, 0x2d},
-	{0x61, 0x03},
-	{0x62, 0x39},
-	{0x63, 0x03},
-	{0x64, 0x47},
-	{0x65, 0x03},
-	{0x66, 0x57},
-	{0x67, 0x03},
-	{0x68, 0x65},
-	{0x69, 0x03},
-	{0x6a, 0x77},
-	{0x6b, 0x03},
-	{0x6c, 0x85},
-	{0x6d, 0x03},
-	{0x6e, 0x8f},
-	{0x6f, 0x03},
-	{0x70, 0xcb},
-	{0x71, 0x00},
-	{0x72, 0x00},
-	{0x73, 0x00},
-	{0x74, 0x21},
-	{0x75, 0x00},
-	{0x76, 0x4c},
-	{0x77, 0x00},
-	{0x78, 0x6b},
-	{0x79, 0x00},
-	{0x7a, 0x85},
-	{0x7b, 0x00},
-	{0x7c, 0x9a},
-	{0x7d, 0x00},
-	{0x7e, 0xad},
-	{0x7f, 0x00},
-	{0x80, 0xbe},
-	{0x81, 0x00},
-	{0x82, 0xcd},
-	{0x83, 0x01},
-	{0x84, 0x01},
-	{0x85, 0x01},
-	{0x86, 0x29},
-	{0x87, 0x01},
-	{0x88, 0x68},
-	{0x89, 0x01},
-	{0x8a, 0x98},
-	{0x8b, 0x01},
-	{0x8c, 0xe5},
-	{0x8d, 0x02},
-	{0x8e, 0x1e},
-	{0x8f, 0x02},
-	{0x90, 0x30},
-	{0x91, 0x02},
-	{0x92, 0x52},
-	{0x93, 0x02},
-	{0x94, 0x88},
-	{0x95, 0x02},
-	{0x96, 0xaa},
-	{0x97, 0x02},
-	{0x98, 0xd7},
-	{0x99, 0x02},
-	{0x9a, 0xf7},
-	{0x9b, 0x03},
-	{0x9c, 0x21},
-	{0x9d, 0x03},
-	{0x9e, 0x2e},
-	{0x9f, 0x03},
-	{0xa0, 0x3d},
-	{0xa2, 0x03},
-	{0xa3, 0x4c},
-	{0xa4, 0x03},
-	{0xa5, 0x5e},
-	{0xa6, 0x03},
-	{0xa7, 0x71},
-	{0xa9, 0x03},
-	{0xaa, 0x86},
-	{0xab, 0x03},
-	{0xac, 0x94},
-	{0xad, 0x03},
-	{0xae, 0xfa},
-	{0xaf, 0x00},
-	{0xb0, 0x00},
-	{0xb1, 0x00},
-	{0xb2, 0x21},
-	{0xb3, 0x00},
-	{0xb4, 0x4c},
-	{0xb5, 0x00},
-	{0xb6, 0x6b},
-	{0xb7, 0x00},
-	{0xb8, 0x85},
-	{0xb9, 0x00},
-	{0xba, 0x9a},
-	{0xbb, 0x00},
-	{0xbc, 0xad},
-	{0xbd, 0x00},
-	{0xbe, 0xbe},
-	{0xbf, 0x00},
-	{0xc0, 0xcd},
-	{0xc1, 0x01},
-	{0xc2, 0x01},
-	{0xc3, 0x01},
-	{0xc4, 0x29},
-	{0xc5, 0x01},
-	{0xc6, 0x68},
-	{0xc7, 0x01},
-	{0xc8, 0x98},
-	{0xc9, 0x01},
-	{0xca, 0xe5},
-	{0xcb, 0x02},
-	{0xcc, 0x1e},
-	{0xcd, 0x02},
-	{0xce, 0x20},
-	{0xcf, 0x02},
-	{0xd0, 0x52},
-	{0xd1, 0x02},
-	{0xd2, 0x88},
-	{0xd3, 0x02},
-	{0xd4, 0xaa},
-	{0xd5, 0x02},
-	{0xd6, 0xd7},
-	{0xd7, 0x02},
-	{0xd8, 0xf7},
-	{0xd9, 0x03},
-	{0xda, 0x21},
-	{0xdb, 0x03},
-	{0xdc, 0x2e},
-	{0xdd, 0x03},
-	{0xde, 0x3d},
-	{0xdf, 0x03},
-	{0xe0, 0x4c},
-	{0xe1, 0x03},
-	{0xe2, 0x5e},
-	{0xe3, 0x03},
-	{0xe4, 0x71},
-	{0xe5, 0x03},
-	{0xe6, 0x86},
-	{0xe7, 0x03},
-	{0xe8, 0x94},
-	{0xe9, 0x03},
-	{0xea, 0xfa},
+	{0xfb, {0x01}, 0x01},
+	{0x00, {0x00}, 0x01},
+	{0x01, {0xf4}, 0x01},
+	{0x02, {0x00}, 0x01},
+	{0x03, {0xef}, 0x01},
+	{0x04, {0x01}, 0x01},
+	{0x05, {0x07}, 0x01},
+	{0x06, {0x01}, 0x01},
+	{0x07, {0x28}, 0x01},
+	{0x08, {0x01}, 0x01},
+	{0x09, {0x44}, 0x01},
+	{0x0a, {0x01}, 0x01},
+	{0x0b, {0x76}, 0x01},
+	{0x0c, {0x01}, 0x01},
+	{0x0d, {0xa0}, 0x01},
+	{0x0e, {0x01}, 0x01},
+	{0x0f, {0xe7}, 0x01},
+	{0x10, {0x02}, 0x01},
+	{0x11, {0x1f}, 0x01},
+	{0x12, {0x02}, 0x01},
+	{0x13, {0x22}, 0x01},
+	{0x14, {0x02}, 0x01},
+	{0x15, {0x54}, 0x01},
+	{0x16, {0x02}, 0x01},
+	{0x17, {0x8b}, 0x01},
+	{0x18, {0x02}, 0x01},
+	{0x19, {0xaf}, 0x01},
+	{0x1a, {0x02}, 0x01},
+	{0x1b, {0xe0}, 0x01},
+	{0x1c, {0x03}, 0x01},
+	{0x1d, {0x01}, 0x01},
+	{0x1e, {0x03}, 0x01},
+	{0x1f, {0x2d}, 0x01},
+	{0x20, {0x03}, 0x01},
+	{0x21, {0x39}, 0x01},
+	{0x22, {0x03}, 0x01},
+	{0x23, {0x47}, 0x01},
+	{0x24, {0x03}, 0x01},
+	{0x25, {0x57}, 0x01},
+	{0x26, {0x03}, 0x01},
+	{0x27, {0x65}, 0x01},
+	{0x28, {0x03}, 0x01},
+	{0x29, {0x77}, 0x01},
+	{0x2a, {0x03}, 0x01},
+	{0x2b, {0x85}, 0x01},
+	{0x2d, {0x03}, 0x01},
+	{0x2f, {0x8f}, 0x01},
+	{0x30, {0x03}, 0x01},
+	{0x31, {0xcb}, 0x01},
+	{0x32, {0x00}, 0x01},
+	{0x33, {0xbb}, 0x01},
+	{0x34, {0x00}, 0x01},
+	{0x35, {0xc0}, 0x01},
+	{0x36, {0x00}, 0x01},
+	{0x37, {0xcc}, 0x01},
+	{0x38, {0x00}, 0x01},
+	{0x39, {0xd6}, 0x01},
+	{0x3a, {0x00}, 0x01},
+	{0x3b, {0xe1}, 0x01},
+	{0x3d, {0x00}, 0x01},
+	{0x3f, {0xea}, 0x01},
+	{0x40, {0x00}, 0x01},
+	{0x41, {0xf4}, 0x01},
+	{0x42, {0x00}, 0x01},
+	{0x43, {0xfe}, 0x01},
+	{0x44, {0x01}, 0x01},
+	{0x45, {0x07}, 0x01},
+	{0x46, {0x01}, 0x01},
+	{0x47, {0x28}, 0x01},
+	{0x48, {0x01}, 0x01},
+	{0x49, {0x44}, 0x01},
+	{0x4a, {0x01}, 0x01},
+	{0x4b, {0x76}, 0x01},
+	{0x4c, {0x01}, 0x01},
+	{0x4d, {0xa0}, 0x01},
+	{0x4e, {0x01}, 0x01},
+	{0x4f, {0xe7}, 0x01},
+	{0x50, {0x02}, 0x01},
+	{0x51, {0x1f}, 0x01},
+	{0x52, {0x02}, 0x01},
+	{0x53, {0x22}, 0x01},
+	{0x54, {0x02}, 0x01},
+	{0x55, {0x54}, 0x01},
+	{0x56, {0x02}, 0x01},
+	{0x58, {0x8b}, 0x01},
+	{0x59, {0x02}, 0x01},
+	{0x5a, {0xaf}, 0x01},
+	{0x5b, {0x02}, 0x01},
+	{0x5c, {0xe0}, 0x01},
+	{0x5d, {0x03}, 0x01},
+	{0x5e, {0x01}, 0x01},
+	{0x5f, {0x03}, 0x01},
+	{0x60, {0x2d}, 0x01},
+	{0x61, {0x03}, 0x01},
+	{0x62, {0x39}, 0x01},
+	{0x63, {0x03}, 0x01},
+	{0x64, {0x47}, 0x01},
+	{0x65, {0x03}, 0x01},
+	{0x66, {0x57}, 0x01},
+	{0x67, {0x03}, 0x01},
+	{0x68, {0x65}, 0x01},
+	{0x69, {0x03}, 0x01},
+	{0x6a, {0x77}, 0x01},
+	{0x6b, {0x03}, 0x01},
+	{0x6c, {0x85}, 0x01},
+	{0x6d, {0x03}, 0x01},
+	{0x6e, {0x8f}, 0x01},
+	{0x6f, {0x03}, 0x01},
+	{0x70, {0xcb}, 0x01},
+	{0x71, {0x00}, 0x01},
+	{0x72, {0x00}, 0x01},
+	{0x73, {0x00}, 0x01},
+	{0x74, {0x21}, 0x01},
+	{0x75, {0x00}, 0x01},
+	{0x76, {0x4c}, 0x01},
+	{0x77, {0x00}, 0x01},
+	{0x78, {0x6b}, 0x01},
+	{0x79, {0x00}, 0x01},
+	{0x7a, {0x85}, 0x01},
+	{0x7b, {0x00}, 0x01},
+	{0x7c, {0x9a}, 0x01},
+	{0x7d, {0x00}, 0x01},
+	{0x7e, {0xad}, 0x01},
+	{0x7f, {0x00}, 0x01},
+	{0x80, {0xbe}, 0x01},
+	{0x81, {0x00}, 0x01},
+	{0x82, {0xcd}, 0x01},
+	{0x83, {0x01}, 0x01},
+	{0x84, {0x01}, 0x01},
+	{0x85, {0x01}, 0x01},
+	{0x86, {0x29}, 0x01},
+	{0x87, {0x01}, 0x01},
+	{0x88, {0x68}, 0x01},
+	{0x89, {0x01}, 0x01},
+	{0x8a, {0x98}, 0x01},
+	{0x8b, {0x01}, 0x01},
+	{0x8c, {0xe5}, 0x01},
+	{0x8d, {0x02}, 0x01},
+	{0x8e, {0x1e}, 0x01},
+	{0x8f, {0x02}, 0x01},
+	{0x90, {0x30}, 0x01},
+	{0x91, {0x02}, 0x01},
+	{0x92, {0x52}, 0x01},
+	{0x93, {0x02}, 0x01},
+	{0x94, {0x88}, 0x01},
+	{0x95, {0x02}, 0x01},
+	{0x96, {0xaa}, 0x01},
+	{0x97, {0x02}, 0x01},
+	{0x98, {0xd7}, 0x01},
+	{0x99, {0x02}, 0x01},
+	{0x9a, {0xf7}, 0x01},
+	{0x9b, {0x03}, 0x01},
+	{0x9c, {0x21}, 0x01},
+	{0x9d, {0x03}, 0x01},
+	{0x9e, {0x2e}, 0x01},
+	{0x9f, {0x03}, 0x01},
+	{0xa0, {0x3d}, 0x01},
+	{0xa2, {0x03}, 0x01},
+	{0xa3, {0x4c}, 0x01},
+	{0xa4, {0x03}, 0x01},
+	{0xa5, {0x5e}, 0x01},
+	{0xa6, {0x03}, 0x01},
+	{0xa7, {0x71}, 0x01},
+	{0xa9, {0x03}, 0x01},
+	{0xaa, {0x86}, 0x01},
+	{0xab, {0x03}, 0x01},
+	{0xac, {0x94}, 0x01},
+	{0xad, {0x03}, 0x01},
+	{0xae, {0xfa}, 0x01},
+	{0xaf, {0x00}, 0x01},
+	{0xb0, {0x00}, 0x01},
+	{0xb1, {0x00}, 0x01},
+	{0xb2, {0x21}, 0x01},
+	{0xb3, {0x00}, 0x01},
+	{0xb4, {0x4c}, 0x01},
+	{0xb5, {0x00}, 0x01},
+	{0xb6, {0x6b}, 0x01},
+	{0xb7, {0x00}, 0x01},
+	{0xb8, {0x85}, 0x01},
+	{0xb9, {0x00}, 0x01},
+	{0xba, {0x9a}, 0x01},
+	{0xbb, {0x00}, 0x01},
+	{0xbc, {0xad}, 0x01},
+	{0xbd, {0x00}, 0x01},
+	{0xbe, {0xbe}, 0x01},
+	{0xbf, {0x00}, 0x01},
+	{0xc0, {0xcd}, 0x01},
+	{0xc1, {0x01}, 0x01},
+	{0xc2, {0x01}, 0x01},
+	{0xc3, {0x01}, 0x01},
+	{0xc4, {0x29}, 0x01},
+	{0xc5, {0x01}, 0x01},
+	{0xc6, {0x68}, 0x01},
+	{0xc7, {0x01}, 0x01},
+	{0xc8, {0x98}, 0x01},
+	{0xc9, {0x01}, 0x01},
+	{0xca, {0xe5}, 0x01},
+	{0xcb, {0x02}, 0x01},
+	{0xcc, {0x1e}, 0x01},
+	{0xcd, {0x02}, 0x01},
+	{0xce, {0x20}, 0x01},
+	{0xcf, {0x02}, 0x01},
+	{0xd0, {0x52}, 0x01},
+	{0xd1, {0x02}, 0x01},
+	{0xd2, {0x88}, 0x01},
+	{0xd3, {0x02}, 0x01},
+	{0xd4, {0xaa}, 0x01},
+	{0xd5, {0x02}, 0x01},
+	{0xd6, {0xd7}, 0x01},
+	{0xd7, {0x02}, 0x01},
+	{0xd8, {0xf7}, 0x01},
+	{0xd9, {0x03}, 0x01},
+	{0xda, {0x21}, 0x01},
+	{0xdb, {0x03}, 0x01},
+	{0xdc, {0x2e}, 0x01},
+	{0xdd, {0x03}, 0x01},
+	{0xde, {0x3d}, 0x01},
+	{0xdf, {0x03}, 0x01},
+	{0xe0, {0x4c}, 0x01},
+	{0xe1, {0x03}, 0x01},
+	{0xe2, {0x5e}, 0x01},
+	{0xe3, {0x03}, 0x01},
+	{0xe4, {0x71}, 0x01},
+	{0xe5, {0x03}, 0x01},
+	{0xe6, {0x86}, 0x01},
+	{0xe7, {0x03}, 0x01},
+	{0xe8, {0x94}, 0x01},
+	{0xe9, {0x03}, 0x01},
+	{0xea, {0xfa}, 0x01},
 	/* Select CMD2 Page0 (Undocumented) */
-	{0xff, 0x01},
+	{0xff, {0x01}, 0x01},
 	/* Reload CMD1: Don't reload default value to register */
-	{0xfb, 0x01},
+	{0xfb, {0x01}, 0x01},
 	/* Select CMD2 Page1 (Undocumented) */
-	{0xff, 0x02},
+	{0xff, {0x02}, 0x01},
 	/* Reload CMD1: Don't reload default value to register */
-	{0xfb, 0x01},
+	{0xfb, {0x01}, 0x01},
 	/* Select CMD2 Page3 (Undocumented) */
-	{0xff, 0x04},
+	{0xff, {0x04}, 0x01},
 	/* Reload CMD1: Don't reload default value to register */
-	{0xfb, 0x01},
+	{0xfb, {0x01}, 0x01},
 	/* Select CMD1 */
-	{0xff, 0x00},
-	{0xd3, 0x22}, /* RGBMIPICTRL: VSYNC back porch = 34 */
-	{0xd4, 0x04}, /* RGBMIPICTRL: VSYNC front porch = 4 */
+	{0xff, {0x00}, 0x01},
+	{0xd3, {0x22}, 0x01}, /* RGBMIPICTRL: VSYNC back porch = 34 */
+	{0xd4, {0x04}, 0x01}, /* RGBMIPICTRL: VSYNC front porch = 4 */
+};
+
+struct khadas_ts050_panel_data ts050_panel_data = {
+	.init_code = (struct khadas_ts050_panel_cmd *)ts050_init_code,
+	.len = ARRAY_SIZE(ts050_init_code)
+};
+
+struct khadas_ts050_panel_data ts050v2_panel_data = {
+	.init_code = (struct khadas_ts050_panel_cmd *)ts050v2_init_code,
+	.len = ARRAY_SIZE(ts050v2_init_code)
 };
 
 static inline
@@ -613,10 +671,11 @@ static int khadas_ts050_panel_prepare(struct drm_panel *panel)
 
 	msleep(100);
 
-	for (i = 0; i < ARRAY_SIZE(init_code); i++) {
+	for (i = 0; i < khadas_ts050->panel_data->len; i++) {
 		err = mipi_dsi_dcs_write(khadas_ts050->link,
-					 init_code[i].cmd,
-					 &init_code[i].data, 1);
+						khadas_ts050->panel_data->init_code[i].cmd,
+						&khadas_ts050->panel_data->init_code[i].data,
+						khadas_ts050->panel_data->init_code[i].size);
 		if (err < 0) {
 			dev_err(panel->dev, "failed write cmds: %d\n", err);
 			goto poweroff;
@@ -762,7 +821,8 @@ static const struct drm_panel_funcs khadas_ts050_panel_funcs = {
 };
 
 static const struct of_device_id khadas_ts050_of_match[] = {
-	{ .compatible = "khadas,ts050", },
+	{ .compatible = "khadas,ts050",    .data = &ts050_panel_data, },
+	{ .compatible = "khadas,ts050v2",  .data = &ts050v2_panel_data, },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, khadas_ts050_of_match);
@@ -806,6 +866,13 @@ static int khadas_ts050_panel_probe(struct mipi_dsi_device *dsi)
 	struct khadas_ts050_panel *khadas_ts050;
 	int err;
 
+	const void *data = of_device_get_match_data(&dsi->dev);
+
+	if (!data) {
+		dev_err(&dsi->dev, "No matching data\n");
+		return -ENODEV;
+	}
+
 	dsi->lanes = 4;
 	dsi->format = MIPI_DSI_FMT_RGB888;
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
@@ -816,6 +883,7 @@ static int khadas_ts050_panel_probe(struct mipi_dsi_device *dsi)
 	if (!khadas_ts050)
 		return -ENOMEM;
 
+	khadas_ts050->panel_data = (struct khadas_ts050_panel_data *)data;
 	mipi_dsi_set_drvdata(dsi, khadas_ts050);
 	khadas_ts050->link = dsi;
 
diff --git a/drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c b/drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c
index 9d87cc1a357e..1a26205701b5 100644
--- a/drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c
+++ b/drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c
@@ -295,8 +295,6 @@ static int ltk050h3148w_init_sequence(struct ltk050h3146w *ctx)
 	mipi_dsi_dcs_write_seq(dsi, 0xbd, 0x00);
 	mipi_dsi_dcs_write_seq(dsi, 0xc6, 0xef);
 	mipi_dsi_dcs_write_seq(dsi, 0xd4, 0x02);
-	mipi_dsi_dcs_write_seq(dsi, 0x11);
-	mipi_dsi_dcs_write_seq(dsi, 0x29);
 
 	ret = mipi_dsi_dcs_set_tear_on(dsi, 1);
 	if (ret < 0) {
@@ -326,7 +324,8 @@ static const struct drm_display_mode ltk050h3148w_mode = {
 static const struct ltk050h3146w_desc ltk050h3148w_data = {
 	.mode = &ltk050h3148w_mode,
 	.init = ltk050h3148w_init_sequence,
-	.mode_flags = MIPI_DSI_MODE_VIDEO_SYNC_PULSE | MIPI_DSI_MODE_VIDEO_BURST,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_SYNC_PULSE |
+		      MIPI_DSI_MODE_VIDEO_BURST,
 };
 
 static int ltk050h3146w_init_sequence(struct ltk050h3146w *ctx)
diff --git a/drivers/gpu/drm/panel/panel-lg-sw43408.c b/drivers/gpu/drm/panel/panel-lg-sw43408.c
new file mode 100644
index 000000000000..2b3a73696dce
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-lg-sw43408.c
@@ -0,0 +1,320 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019-2024 Linaro Ltd
+ * Author: Sumit Semwal <sumit.semwal@linaro.org>
+ *	 Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
+ */
+
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/display/drm_dsc.h>
+#include <drm/display/drm_dsc_helper.h>
+
+#define NUM_SUPPLIES 2
+
+struct sw43408_panel {
+	struct drm_panel base;
+	struct mipi_dsi_device *link;
+
+	struct regulator_bulk_data supplies[NUM_SUPPLIES];
+
+	struct gpio_desc *reset_gpio;
+
+	struct drm_dsc_config dsc;
+};
+
+static inline struct sw43408_panel *to_panel_info(struct drm_panel *panel)
+{
+	return container_of(panel, struct sw43408_panel, base);
+}
+
+static int sw43408_unprepare(struct drm_panel *panel)
+{
+	struct sw43408_panel *ctx = to_panel_info(panel);
+	int ret;
+
+	ret = mipi_dsi_dcs_set_display_off(ctx->link);
+	if (ret < 0)
+		dev_err(panel->dev, "set_display_off cmd failed ret = %d\n", ret);
+
+	ret = mipi_dsi_dcs_enter_sleep_mode(ctx->link);
+	if (ret < 0)
+		dev_err(panel->dev, "enter_sleep cmd failed ret = %d\n", ret);
+
+	msleep(100);
+
+	gpiod_set_value(ctx->reset_gpio, 1);
+
+	return regulator_bulk_disable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+}
+
+static int sw43408_program(struct drm_panel *panel)
+{
+	struct sw43408_panel *ctx = to_panel_info(panel);
+	struct drm_dsc_picture_parameter_set pps;
+
+	mipi_dsi_dcs_write_seq(ctx->link, MIPI_DCS_SET_GAMMA_CURVE, 0x02);
+
+	mipi_dsi_dcs_set_tear_on(ctx->link, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
+
+	mipi_dsi_dcs_write_seq(ctx->link, 0x53, 0x0c, 0x30);
+	mipi_dsi_dcs_write_seq(ctx->link, 0x55, 0x00, 0x70, 0xdf, 0x00, 0x70, 0xdf);
+	mipi_dsi_dcs_write_seq(ctx->link, 0xf7, 0x01, 0x49, 0x0c);
+
+	mipi_dsi_dcs_exit_sleep_mode(ctx->link);
+
+	msleep(135);
+
+	/* COMPRESSION_MODE moved after setting the PPS */
+
+	mipi_dsi_dcs_write_seq(ctx->link, 0xb0, 0xac);
+	mipi_dsi_dcs_write_seq(ctx->link, 0xe5,
+			       0x00, 0x3a, 0x00, 0x3a, 0x00, 0x0e, 0x10);
+	mipi_dsi_dcs_write_seq(ctx->link, 0xb5,
+			       0x75, 0x60, 0x2d, 0x5d, 0x80, 0x00, 0x0a, 0x0b,
+			       0x00, 0x05, 0x0b, 0x00, 0x80, 0x0d, 0x0e, 0x40,
+			       0x00, 0x0c, 0x00, 0x16, 0x00, 0xb8, 0x00, 0x80,
+			       0x0d, 0x0e, 0x40, 0x00, 0x0c, 0x00, 0x16, 0x00,
+			       0xb8, 0x00, 0x81, 0x00, 0x03, 0x03, 0x03, 0x01,
+			       0x01);
+	msleep(85);
+	mipi_dsi_dcs_write_seq(ctx->link, 0xcd,
+			       0x00, 0x00, 0x00, 0x19, 0x19, 0x19, 0x19, 0x19,
+			       0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19,
+			       0x16, 0x16);
+	mipi_dsi_dcs_write_seq(ctx->link, 0xcb, 0x80, 0x5c, 0x07, 0x03, 0x28);
+	mipi_dsi_dcs_write_seq(ctx->link, 0xc0, 0x02, 0x02, 0x0f);
+	mipi_dsi_dcs_write_seq(ctx->link, 0x55, 0x04, 0x61, 0xdb, 0x04, 0x70, 0xdb);
+	mipi_dsi_dcs_write_seq(ctx->link, 0xb0, 0xca);
+
+	mipi_dsi_dcs_set_display_on(ctx->link);
+
+	msleep(50);
+
+	ctx->link->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	drm_dsc_pps_payload_pack(&pps, ctx->link->dsc);
+	mipi_dsi_picture_parameter_set(ctx->link, &pps);
+
+	ctx->link->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	/*
+	 * This panel uses PPS selectors with offset:
+	 * PPS 1 if pps_identifier is 0
+	 * PPS 2 if pps_identifier is 1
+	 */
+	mipi_dsi_compression_mode_ext(ctx->link, true,
+				      MIPI_DSI_COMPRESSION_DSC, 1);
+
+	return 0;
+}
+
+static int sw43408_prepare(struct drm_panel *panel)
+{
+	struct sw43408_panel *ctx = to_panel_info(panel);
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+	if (ret < 0)
+		return ret;
+
+	usleep_range(5000, 6000);
+
+	gpiod_set_value(ctx->reset_gpio, 0);
+	usleep_range(9000, 10000);
+	gpiod_set_value(ctx->reset_gpio, 1);
+	usleep_range(1000, 2000);
+	gpiod_set_value(ctx->reset_gpio, 0);
+	usleep_range(9000, 10000);
+
+	ret = sw43408_program(panel);
+	if (ret)
+		goto poweroff;
+
+	return 0;
+
+poweroff:
+	gpiod_set_value(ctx->reset_gpio, 1);
+	regulator_bulk_disable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+	return ret;
+}
+
+static const struct drm_display_mode sw43408_mode = {
+	.clock = (1080 + 20 + 32 + 20) * (2160 + 20 + 4 + 20) * 60 / 1000,
+
+	.hdisplay = 1080,
+	.hsync_start = 1080 + 20,
+	.hsync_end = 1080 + 20 + 32,
+	.htotal = 1080 + 20 + 32 + 20,
+
+	.vdisplay = 2160,
+	.vsync_start = 2160 + 20,
+	.vsync_end = 2160 + 20 + 4,
+	.vtotal = 2160 + 20 + 4 + 20,
+
+	.width_mm = 62,
+	.height_mm = 124,
+
+	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+};
+
+static int sw43408_get_modes(struct drm_panel *panel,
+			     struct drm_connector *connector)
+{
+	return drm_connector_helper_get_modes_fixed(connector, &sw43408_mode);
+}
+
+static int sw43408_backlight_update_status(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	u16 brightness = backlight_get_brightness(bl);
+
+	return mipi_dsi_dcs_set_display_brightness_large(dsi, brightness);
+}
+
+static const struct backlight_ops sw43408_backlight_ops = {
+	.update_status = sw43408_backlight_update_status,
+};
+
+static int sw43408_backlight_init(struct sw43408_panel *ctx)
+{
+	struct device *dev = &ctx->link->dev;
+	const struct backlight_properties props = {
+		.type = BACKLIGHT_PLATFORM,
+		.brightness = 255,
+		.max_brightness = 255,
+	};
+
+	ctx->base.backlight = devm_backlight_device_register(dev, dev_name(dev), dev,
+							     ctx->link,
+							     &sw43408_backlight_ops,
+							     &props);
+
+	if (IS_ERR(ctx->base.backlight))
+		return dev_err_probe(dev, PTR_ERR(ctx->base.backlight),
+				     "Failed to create backlight\n");
+
+	return 0;
+}
+
+static const struct drm_panel_funcs sw43408_funcs = {
+	.unprepare = sw43408_unprepare,
+	.prepare = sw43408_prepare,
+	.get_modes = sw43408_get_modes,
+};
+
+static const struct of_device_id sw43408_of_match[] = {
+	{ .compatible = "lg,sw43408", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, sw43408_of_match);
+
+static int sw43408_add(struct sw43408_panel *ctx)
+{
+	struct device *dev = &ctx->link->dev;
+	int ret;
+
+	ctx->supplies[0].supply = "vddi"; /* 1.88 V */
+	ctx->supplies[0].init_load_uA = 62000;
+	ctx->supplies[1].supply = "vpnl"; /* 3.0 V */
+	ctx->supplies[1].init_load_uA = 857000;
+
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ctx->supplies),
+				      ctx->supplies);
+	if (ret < 0)
+		return ret;
+
+	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->reset_gpio)) {
+		ret = PTR_ERR(ctx->reset_gpio);
+		return dev_err_probe(dev, ret, "cannot get reset gpio\n");
+	}
+
+	ret = sw43408_backlight_init(ctx);
+	if (ret < 0)
+		return ret;
+
+	ctx->base.prepare_prev_first = true;
+
+	drm_panel_init(&ctx->base, dev, &sw43408_funcs, DRM_MODE_CONNECTOR_DSI);
+
+	drm_panel_add(&ctx->base);
+	return ret;
+}
+
+static int sw43408_probe(struct mipi_dsi_device *dsi)
+{
+	struct sw43408_panel *ctx;
+	int ret;
+
+	ctx = devm_kzalloc(&dsi->dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	dsi->mode_flags = MIPI_DSI_MODE_LPM;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->lanes = 4;
+
+	ctx->link = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	ret = sw43408_add(ctx);
+	if (ret < 0)
+		return ret;
+
+	/* The panel works only in the DSC mode. Set DSC params. */
+	ctx->dsc.dsc_version_major = 0x1;
+	ctx->dsc.dsc_version_minor = 0x1;
+
+	/* slice_count * slice_width == width */
+	ctx->dsc.slice_height = 16;
+	ctx->dsc.slice_width = 540;
+	ctx->dsc.slice_count = 2;
+	ctx->dsc.bits_per_component = 8;
+	ctx->dsc.bits_per_pixel = 8 << 4;
+	ctx->dsc.block_pred_enable = true;
+
+	dsi->dsc = &ctx->dsc;
+
+	return mipi_dsi_attach(dsi);
+}
+
+static void sw43408_remove(struct mipi_dsi_device *dsi)
+{
+	struct sw43408_panel *ctx = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = sw43408_unprepare(&ctx->base);
+	if (ret < 0)
+		dev_err(&dsi->dev, "failed to unprepare panel: %d\n", ret);
+
+	ret = mipi_dsi_detach(dsi);
+	if (ret < 0)
+		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
+
+	drm_panel_remove(&ctx->base);
+}
+
+static struct mipi_dsi_driver sw43408_driver = {
+	.driver = {
+		.name = "panel-lg-sw43408",
+		.of_match_table = sw43408_of_match,
+	},
+	.probe = sw43408_probe,
+	.remove = sw43408_remove,
+};
+module_mipi_dsi_driver(sw43408_driver);
+
+MODULE_AUTHOR("Sumit Semwal <sumit.semwal@linaro.org>");
+MODULE_DESCRIPTION("LG SW436408 MIPI-DSI LED panel");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt35950.c b/drivers/gpu/drm/panel/panel-novatek-nt35950.c
index 648ce9201426..028fdac293f7 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt35950.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt35950.c
@@ -556,10 +556,8 @@ static int nt35950_probe(struct mipi_dsi_device *dsi)
 		}
 		dsi_r_host = of_find_mipi_dsi_host_by_node(dsi_r);
 		of_node_put(dsi_r);
-		if (!dsi_r_host) {
-			dev_err(dev, "Cannot get secondary DSI host\n");
-			return -EPROBE_DEFER;
-		}
+		if (!dsi_r_host)
+			return dev_err_probe(dev, -EPROBE_DEFER, "Cannot get secondary DSI host\n");
 
 		nt->dsi[1] = mipi_dsi_device_register_full(dsi_r_host, info);
 		if (!nt->dsi[1]) {
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt36672a.c b/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
index 33fb3d715e54..3886372415c2 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
@@ -605,21 +605,16 @@ static int nt36672a_panel_add(struct nt36672a_panel *pinfo)
 	struct device *dev = &pinfo->link->dev;
 	int i, ret;
 
-	for (i = 0; i < ARRAY_SIZE(pinfo->supplies); i++)
+	for (i = 0; i < ARRAY_SIZE(pinfo->supplies); i++) {
 		pinfo->supplies[i].supply = nt36672a_regulator_names[i];
+		pinfo->supplies[i].init_load_uA = nt36672a_regulator_enable_loads[i];
+	}
 
 	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(pinfo->supplies),
 				      pinfo->supplies);
 	if (ret < 0)
 		return dev_err_probe(dev, ret, "failed to get regulators\n");
 
-	for (i = 0; i < ARRAY_SIZE(pinfo->supplies); i++) {
-		ret = regulator_set_load(pinfo->supplies[i].consumer,
-					 nt36672a_regulator_enable_loads[i]);
-		if (ret)
-			return dev_err_probe(dev, ret, "failed to set regulator enable loads\n");
-	}
-
 	pinfo->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
 	if (IS_ERR(pinfo->reset_gpio))
 		return dev_err_probe(dev, PTR_ERR(pinfo->reset_gpio),
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt36672e.c b/drivers/gpu/drm/panel/panel-novatek-nt36672e.c
index cb7406d74466..20b7bfe4aa12 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt36672e.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt36672e.c
@@ -25,12 +25,6 @@ static const unsigned long regulator_enable_loads[] = {
 	100000,
 };
 
-static const unsigned long regulator_disable_loads[] = {
-	80,
-	100,
-	100,
-};
-
 struct panel_desc {
 	const struct drm_display_mode *display_mode;
 	u32 width_mm;
@@ -349,17 +343,7 @@ static int nt36672e_1080x2408_60hz_init(struct mipi_dsi_device *dsi)
 static int nt36672e_power_on(struct nt36672e_panel *ctx)
 {
 	struct mipi_dsi_device *dsi = ctx->dsi;
-	int ret, i;
-
-	for (i = 0; i < ARRAY_SIZE(ctx->supplies); i++) {
-		ret = regulator_set_load(ctx->supplies[i].consumer,
-				regulator_enable_loads[i]);
-		if (ret) {
-			dev_err(&dsi->dev, "regulator set load failed for supply %s: %d\n",
-				ctx->supplies[i].supply, ret);
-			return ret;
-		}
-	}
+	int ret;
 
 	ret = regulator_bulk_enable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
 	if (ret < 0) {
@@ -385,20 +369,9 @@ static int nt36672e_power_off(struct nt36672e_panel *ctx)
 {
 	struct mipi_dsi_device *dsi = ctx->dsi;
 	int ret = 0;
-	int i;
 
 	gpiod_set_value(ctx->reset_gpio, 0);
 
-	for (i = 0; i < ARRAY_SIZE(ctx->supplies); i++) {
-		ret = regulator_set_load(ctx->supplies[i].consumer,
-				regulator_disable_loads[i]);
-		if (ret) {
-			dev_err(&dsi->dev, "regulator set load failed for supply %s: %d\n",
-				ctx->supplies[i].supply, ret);
-			return ret;
-		}
-	}
-
 	ret = regulator_bulk_disable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
 	if (ret)
 		dev_err(&dsi->dev, "regulator bulk disable failed: %d\n", ret);
@@ -567,8 +540,10 @@ static int nt36672e_panel_probe(struct mipi_dsi_device *dsi)
 		return -ENODEV;
 	}
 
-	for (i = 0; i < ARRAY_SIZE(ctx->supplies); i++)
+	for (i = 0; i < ARRAY_SIZE(ctx->supplies); i++) {
 		ctx->supplies[i].supply = regulator_names[i];
+		ctx->supplies[i].init_load_uA = regulator_enable_loads[i];
+	}
 
 	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ctx->supplies),
 			ctx->supplies);
@@ -614,8 +589,6 @@ static void nt36672e_panel_remove(struct mipi_dsi_device *dsi)
 	struct nt36672e_panel *ctx = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(ctx->dsi);
-	mipi_dsi_device_unregister(ctx->dsi);
-
 	drm_panel_remove(&ctx->panel);
 }
 
diff --git a/drivers/gpu/drm/panel/panel-raydium-rm69380.c b/drivers/gpu/drm/panel/panel-raydium-rm69380.c
new file mode 100644
index 000000000000..4dca6802faef
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-raydium-rm69380.c
@@ -0,0 +1,344 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Generated with linux-mdss-dsi-panel-driver-generator from vendor device tree.
+ * Copyright (c) 2024 David Wronek <david@mainlining.org>
+ */
+
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_graph.h>
+#include <linux/regulator/consumer.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+struct rm69380_panel {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi[2];
+	struct regulator_bulk_data supplies[2];
+	struct gpio_desc *reset_gpio;
+};
+
+static inline
+struct rm69380_panel *to_rm69380_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct rm69380_panel, panel);
+}
+
+static void rm69380_reset(struct rm69380_panel *ctx)
+{
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	usleep_range(15000, 16000);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	usleep_range(10000, 11000);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	msleep(30);
+}
+
+static int rm69380_on(struct rm69380_panel *ctx)
+{
+	struct mipi_dsi_device *dsi = ctx->dsi[0];
+	struct device *dev = &dsi->dev;
+	int ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+	if (ctx->dsi[1])
+		ctx->dsi[1]->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	mipi_dsi_dcs_write_seq(dsi, 0xfe, 0xd4);
+	mipi_dsi_dcs_write_seq(dsi, 0x00, 0x80);
+	mipi_dsi_dcs_write_seq(dsi, 0xfe, 0xd0);
+	mipi_dsi_dcs_write_seq(dsi, 0x48, 0x00);
+	mipi_dsi_dcs_write_seq(dsi, 0xfe, 0x26);
+	mipi_dsi_dcs_write_seq(dsi, 0x75, 0x3f);
+	mipi_dsi_dcs_write_seq(dsi, 0x1d, 0x1a);
+	mipi_dsi_dcs_write_seq(dsi, 0xfe, 0x00);
+	mipi_dsi_dcs_write_seq(dsi, MIPI_DCS_WRITE_CONTROL_DISPLAY, 0x28);
+	mipi_dsi_dcs_write_seq(dsi, 0xc2, 0x08);
+
+	ret = mipi_dsi_dcs_set_tear_on(dsi, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set tear on: %d\n", ret);
+		return ret;
+	}
+
+	ret = mipi_dsi_dcs_exit_sleep_mode(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to exit sleep mode: %d\n", ret);
+		return ret;
+	}
+	msleep(20);
+
+	ret = mipi_dsi_dcs_set_display_on(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set display on: %d\n", ret);
+		return ret;
+	}
+	msleep(36);
+
+	return 0;
+}
+
+static int rm69380_off(struct rm69380_panel *ctx)
+{
+	struct mipi_dsi_device *dsi = ctx->dsi[0];
+	struct device *dev = &dsi->dev;
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+	if (ctx->dsi[1])
+		ctx->dsi[1]->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_off(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set display off: %d\n", ret);
+		return ret;
+	}
+	msleep(35);
+
+	ret = mipi_dsi_dcs_enter_sleep_mode(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enter sleep mode: %d\n", ret);
+		return ret;
+	}
+	msleep(20);
+
+	return 0;
+}
+
+static int rm69380_prepare(struct drm_panel *panel)
+{
+	struct rm69380_panel *ctx = to_rm69380_panel(panel);
+	struct device *dev = &ctx->dsi[0]->dev;
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enable regulators: %d\n", ret);
+		return ret;
+	}
+
+	rm69380_reset(ctx);
+
+	ret = rm69380_on(ctx);
+	if (ret < 0) {
+		dev_err(dev, "Failed to initialize panel: %d\n", ret);
+		gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+		regulator_bulk_disable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int rm69380_unprepare(struct drm_panel *panel)
+{
+	struct rm69380_panel *ctx = to_rm69380_panel(panel);
+	struct device *dev = &ctx->dsi[0]->dev;
+	int ret;
+
+	ret = rm69380_off(ctx);
+	if (ret < 0)
+		dev_err(dev, "Failed to un-initialize panel: %d\n", ret);
+
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	regulator_bulk_disable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+
+	return 0;
+}
+
+static const struct drm_display_mode rm69380_mode = {
+	.clock = (2560 + 32 + 12 + 38) * (1600 + 20 + 4 + 8) * 90 / 1000,
+	.hdisplay = 2560,
+	.hsync_start = 2560 + 32,
+	.hsync_end = 2560 + 32 + 12,
+	.htotal = 2560 + 32 + 12 + 38,
+	.vdisplay = 1600,
+	.vsync_start = 1600 + 20,
+	.vsync_end = 1600 + 20 + 4,
+	.vtotal = 1600 + 20 + 4 + 8,
+	.width_mm = 248,
+	.height_mm = 155,
+	.type = DRM_MODE_TYPE_DRIVER,
+};
+
+static int rm69380_get_modes(struct drm_panel *panel,
+					struct drm_connector *connector)
+{
+	return drm_connector_helper_get_modes_fixed(connector, &rm69380_mode);
+}
+
+static const struct drm_panel_funcs rm69380_panel_funcs = {
+	.prepare = rm69380_prepare,
+	.unprepare = rm69380_unprepare,
+	.get_modes = rm69380_get_modes,
+};
+
+static int rm69380_bl_update_status(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	u16 brightness = backlight_get_brightness(bl);
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_brightness_large(dsi, brightness);
+	if (ret < 0)
+		return ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	return 0;
+}
+
+static int rm69380_bl_get_brightness(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	u16 brightness;
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_get_display_brightness_large(dsi, &brightness);
+	if (ret < 0)
+		return ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	return brightness;
+}
+
+static const struct backlight_ops rm69380_bl_ops = {
+	.update_status = rm69380_bl_update_status,
+	.get_brightness = rm69380_bl_get_brightness,
+};
+
+static struct backlight_device *
+rm69380_create_backlight(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	const struct backlight_properties props = {
+		.type = BACKLIGHT_RAW,
+		.brightness = 511,
+		.max_brightness = 2047,
+	};
+
+	return devm_backlight_device_register(dev, dev_name(dev), dev, dsi,
+					      &rm69380_bl_ops, &props);
+}
+
+static int rm69380_probe(struct mipi_dsi_device *dsi)
+{
+	struct mipi_dsi_host *dsi_sec_host;
+	struct rm69380_panel *ctx;
+	struct device *dev = &dsi->dev;
+	struct device_node *dsi_sec;
+	int ret, i;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->supplies[0].supply = "vddio";
+	ctx->supplies[1].supply = "avdd";
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ctx->supplies),
+				      ctx->supplies);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to get regulators\n");
+
+	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(ctx->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
+				     "Failed to get reset-gpios\n");
+
+	dsi_sec = of_graph_get_remote_node(dsi->dev.of_node, 1, -1);
+
+	if (dsi_sec) {
+		const struct mipi_dsi_device_info info = { "RM69380 DSI1", 0,
+							   dsi_sec };
+
+		dsi_sec_host = of_find_mipi_dsi_host_by_node(dsi_sec);
+		of_node_put(dsi_sec);
+		if (!dsi_sec_host)
+			return dev_err_probe(dev, -EPROBE_DEFER,
+					     "Cannot get secondary DSI host\n");
+
+		ctx->dsi[1] =
+			devm_mipi_dsi_device_register_full(dev, dsi_sec_host, &info);
+		if (IS_ERR(ctx->dsi[1]))
+			return dev_err_probe(dev, PTR_ERR(ctx->dsi[1]),
+					     "Cannot get secondary DSI node\n");
+
+		mipi_dsi_set_drvdata(ctx->dsi[1], ctx);
+	}
+
+	ctx->dsi[0] = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	drm_panel_init(&ctx->panel, dev, &rm69380_panel_funcs,
+		       DRM_MODE_CONNECTOR_DSI);
+	ctx->panel.prepare_prev_first = true;
+
+	ctx->panel.backlight = rm69380_create_backlight(dsi);
+	if (IS_ERR(ctx->panel.backlight))
+		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
+				     "Failed to create backlight\n");
+
+	drm_panel_add(&ctx->panel);
+
+	for (i = 0; i < ARRAY_SIZE(ctx->dsi); i++) {
+		if (!ctx->dsi[i])
+			continue;
+
+		dev_dbg(&ctx->dsi[i]->dev, "Binding DSI %d\n", i);
+
+		ctx->dsi[i]->lanes = 4;
+		ctx->dsi[i]->format = MIPI_DSI_FMT_RGB888;
+		ctx->dsi[i]->mode_flags = MIPI_DSI_MODE_VIDEO_BURST |
+					  MIPI_DSI_CLOCK_NON_CONTINUOUS;
+
+		ret = devm_mipi_dsi_attach(dev, ctx->dsi[i]);
+		if (ret < 0) {
+			drm_panel_remove(&ctx->panel);
+			return dev_err_probe(dev, ret,
+					     "Failed to attach to DSI%d\n", i);
+		}
+	}
+
+	return 0;
+}
+
+static void rm69380_remove(struct mipi_dsi_device *dsi)
+{
+	struct rm69380_panel *ctx = mipi_dsi_get_drvdata(dsi);
+
+	drm_panel_remove(&ctx->panel);
+}
+
+static const struct of_device_id rm69380_of_match[] = {
+	{ .compatible = "lenovo,j716f-edo-rm69380" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, rm69380_of_match);
+
+static struct mipi_dsi_driver rm69380_panel_driver = {
+	.probe = rm69380_probe,
+	.remove = rm69380_remove,
+	.driver = {
+		.name = "panel-raydium-rm69380",
+		.of_match_table = rm69380_of_match,
+	},
+};
+module_mipi_dsi_driver(rm69380_panel_driver);
+
+MODULE_AUTHOR("David Wronek <david@mainlining.org");
+MODULE_DESCRIPTION("DRM driver for Raydium RM69380-equipped DSI panels");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-samsung-atna33xc20.c b/drivers/gpu/drm/panel/panel-samsung-atna33xc20.c
index 76c2a8f6718c..a9f0d214a900 100644
--- a/drivers/gpu/drm/panel/panel-samsung-atna33xc20.c
+++ b/drivers/gpu/drm/panel/panel-samsung-atna33xc20.c
@@ -36,7 +36,7 @@ struct atana33xc20_panel {
 	struct gpio_desc *el_on3_gpio;
 	struct drm_dp_aux *aux;
 
-	struct edid *edid;
+	const struct drm_edid *drm_edid;
 
 	ktime_t powered_off_time;
 	ktime_t powered_on_time;
@@ -109,19 +109,17 @@ static int atana33xc20_resume(struct device *dev)
 		if (hpd_asserted < 0)
 			ret = hpd_asserted;
 
-		if (ret)
+		if (ret) {
 			dev_warn(dev, "Error waiting for HPD GPIO: %d\n", ret);
-
-		return ret;
-	}
-
-	if (p->aux->wait_hpd_asserted) {
+			goto error;
+		}
+	} else if (p->aux->wait_hpd_asserted) {
 		ret = p->aux->wait_hpd_asserted(p->aux, HPD_MAX_US);
 
-		if (ret)
+		if (ret) {
 			dev_warn(dev, "Controller error waiting for HPD: %d\n", ret);
-
-		return ret;
+			goto error;
+		}
 	}
 
 	/*
@@ -133,6 +131,12 @@ static int atana33xc20_resume(struct device *dev)
 	 * right times.
 	 */
 	return 0;
+
+error:
+	drm_dp_dpcd_set_powered(p->aux, false);
+	regulator_disable(p->supply);
+
+	return ret;
 }
 
 static int atana33xc20_disable(struct drm_panel *panel)
@@ -249,9 +253,12 @@ static int atana33xc20_get_modes(struct drm_panel *panel,
 
 	pm_runtime_get_sync(panel->dev);
 
-	if (!p->edid)
-		p->edid = drm_get_edid(connector, &aux_ep->aux->ddc);
-	num = drm_add_edid_modes(connector, p->edid);
+	if (!p->drm_edid)
+		p->drm_edid = drm_edid_read_ddc(connector, &aux_ep->aux->ddc);
+
+	drm_edid_connector_update(connector, p->drm_edid);
+
+	num = drm_edid_connector_add_modes(connector);
 
 	pm_runtime_mark_last_busy(panel->dev);
 	pm_runtime_put_autosuspend(panel->dev);
@@ -324,9 +331,14 @@ static int atana33xc20_probe(struct dp_aux_ep_device *aux_ep)
 	ret = drm_panel_dp_aux_backlight(&panel->base, aux_ep->aux);
 	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_autosuspend(dev);
+
+	/*
+	 * Warn if we get an error, but don't consider it fatal. Having
+	 * a panel where we can't control the backlight is better than
+	 * no panel.
+	 */
 	if (ret)
-		return dev_err_probe(dev, ret,
-				     "failed to register dp aux backlight\n");
+		dev_warn(dev, "failed to register dp aux backlight: %d\n", ret);
 
 	drm_panel_add(&panel->base);
 
@@ -342,7 +354,7 @@ static void atana33xc20_remove(struct dp_aux_ep_device *aux_ep)
 	drm_panel_disable(&panel->base);
 	drm_panel_unprepare(&panel->base);
 
-	kfree(panel->edid);
+	drm_edid_free(panel->drm_edid);
 }
 
 static void atana33xc20_shutdown(struct dp_aux_ep_device *aux_ep)
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e3fa7.c b/drivers/gpu/drm/panel/panel-samsung-s6e3fa7.c
new file mode 100644
index 000000000000..10bc8fb5f1f9
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e3fa7.c
@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Driver for the Samsung S6E3FA7 panel.
+ *
+ * Copyright (c) 2022-2024, The Linux Foundation. All rights reserved.
+ * Generated with linux-mdss-dsi-panel-driver-generator from vendor device tree:
+ * Copyright (c) 2013, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+
+struct s6e3fa7_panel {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi;
+	struct gpio_desc *reset_gpio;
+};
+
+static inline struct s6e3fa7_panel *to_s6e3fa7_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct s6e3fa7_panel, panel);
+}
+
+static void s6e3fa7_panel_reset(struct s6e3fa7_panel *ctx)
+{
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	usleep_range(1000, 2000);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	usleep_range(10000, 11000);
+}
+
+static int s6e3fa7_panel_on(struct s6e3fa7_panel *ctx)
+{
+	struct mipi_dsi_device *dsi = ctx->dsi;
+	struct device *dev = &dsi->dev;
+	int ret;
+
+	ret = mipi_dsi_dcs_exit_sleep_mode(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to exit sleep mode: %d\n", ret);
+		return ret;
+	}
+	msleep(120);
+
+	ret = mipi_dsi_dcs_set_tear_on(dsi, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set tear on: %d\n", ret);
+		return ret;
+	}
+
+	mipi_dsi_dcs_write_seq(dsi, 0xf0, 0x5a, 0x5a);
+	mipi_dsi_dcs_write_seq(dsi, 0xf4,
+			       0xbb, 0x23, 0x19, 0x3a, 0x9f, 0x0f, 0x09, 0xc0,
+			       0x00, 0xb4, 0x37, 0x70, 0x79, 0x69);
+	mipi_dsi_dcs_write_seq(dsi, 0xf0, 0xa5, 0xa5);
+	mipi_dsi_dcs_write_seq(dsi, MIPI_DCS_WRITE_CONTROL_DISPLAY, 0x20);
+
+	ret = mipi_dsi_dcs_set_display_on(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set display on: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int s6e3fa7_panel_prepare(struct drm_panel *panel)
+{
+	struct s6e3fa7_panel *ctx = to_s6e3fa7_panel(panel);
+	struct device *dev = &ctx->dsi->dev;
+	int ret;
+
+	s6e3fa7_panel_reset(ctx);
+
+	ret = s6e3fa7_panel_on(ctx);
+	if (ret < 0) {
+		dev_err(dev, "Failed to initialize panel: %d\n", ret);
+		gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int s6e3fa7_panel_unprepare(struct drm_panel *panel)
+{
+	struct s6e3fa7_panel *ctx = to_s6e3fa7_panel(panel);
+
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+
+	return 0;
+}
+
+static int s6e3fa7_panel_disable(struct drm_panel *panel)
+{
+	struct s6e3fa7_panel *ctx = to_s6e3fa7_panel(panel);
+	struct mipi_dsi_device *dsi = ctx->dsi;
+	struct device *dev = &dsi->dev;
+	int ret;
+
+	ret = mipi_dsi_dcs_set_display_off(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set display off: %d\n", ret);
+		return ret;
+	}
+
+	ret = mipi_dsi_dcs_enter_sleep_mode(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enter sleep mode: %d\n", ret);
+		return ret;
+	}
+	msleep(120);
+
+	return 0;
+}
+
+static const struct drm_display_mode s6e3fa7_panel_mode = {
+	.clock = (1080 + 32 + 32 + 78) * (2220 + 32 + 4 + 78) * 60 / 1000,
+	.hdisplay = 1080,
+	.hsync_start = 1080 + 32,
+	.hsync_end = 1080 + 32 + 32,
+	.htotal = 1080 + 32 + 32 + 78,
+	.vdisplay = 2220,
+	.vsync_start = 2220 + 32,
+	.vsync_end = 2220 + 32 + 4,
+	.vtotal = 2220 + 32 + 4 + 78,
+	.width_mm = 62,
+	.height_mm = 127,
+};
+
+static int s6e3fa7_panel_get_modes(struct drm_panel *panel,
+				 struct drm_connector *connector)
+{
+	struct drm_display_mode *mode;
+
+	mode = drm_mode_duplicate(connector->dev, &s6e3fa7_panel_mode);
+	if (!mode)
+		return -ENOMEM;
+
+	drm_mode_set_name(mode);
+
+	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+	connector->display_info.width_mm = mode->width_mm;
+	connector->display_info.height_mm = mode->height_mm;
+	drm_mode_probed_add(connector, mode);
+
+	return 1;
+}
+
+static const struct drm_panel_funcs s6e3fa7_panel_funcs = {
+	.prepare = s6e3fa7_panel_prepare,
+	.unprepare = s6e3fa7_panel_unprepare,
+	.disable = s6e3fa7_panel_disable,
+	.get_modes = s6e3fa7_panel_get_modes,
+};
+
+static int s6e3fa7_panel_bl_update_status(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	u16 brightness = backlight_get_brightness(bl);
+	int ret;
+
+	ret = mipi_dsi_dcs_set_display_brightness_large(dsi, brightness);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int s6e3fa7_panel_bl_get_brightness(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	u16 brightness;
+	int ret;
+
+	ret = mipi_dsi_dcs_get_display_brightness_large(dsi, &brightness);
+	if (ret < 0)
+		return ret;
+
+	return brightness;
+}
+
+static const struct backlight_ops s6e3fa7_panel_bl_ops = {
+	.update_status = s6e3fa7_panel_bl_update_status,
+	.get_brightness = s6e3fa7_panel_bl_get_brightness,
+};
+
+static struct backlight_device *
+s6e3fa7_panel_create_backlight(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	const struct backlight_properties props = {
+		.type = BACKLIGHT_RAW,
+		.brightness = 1023,
+		.max_brightness = 1023,
+	};
+
+	return devm_backlight_device_register(dev, dev_name(dev), dev, dsi,
+					      &s6e3fa7_panel_bl_ops, &props);
+}
+
+static int s6e3fa7_panel_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct s6e3fa7_panel *ctx;
+	int ret;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(ctx->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
+				     "Failed to get reset-gpios\n");
+
+	ctx->dsi = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	dsi->lanes = 4;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO_BURST |
+			  MIPI_DSI_CLOCK_NON_CONTINUOUS | MIPI_DSI_MODE_LPM;
+
+	drm_panel_init(&ctx->panel, dev, &s6e3fa7_panel_funcs,
+		       DRM_MODE_CONNECTOR_DSI);
+	ctx->panel.prepare_prev_first = true;
+
+	ctx->panel.backlight = s6e3fa7_panel_create_backlight(dsi);
+	if (IS_ERR(ctx->panel.backlight))
+		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
+				     "Failed to create backlight\n");
+
+	drm_panel_add(&ctx->panel);
+
+	ret = mipi_dsi_attach(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
+		drm_panel_remove(&ctx->panel);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void s6e3fa7_panel_remove(struct mipi_dsi_device *dsi)
+{
+	struct s6e3fa7_panel *ctx = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = mipi_dsi_detach(dsi);
+	if (ret < 0)
+		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
+
+	drm_panel_remove(&ctx->panel);
+}
+
+static const struct of_device_id s6e3fa7_panel_of_match[] = {
+	{ .compatible = "samsung,s6e3fa7-ams559nk06" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, s6e3fa7_panel_of_match);
+
+static struct mipi_dsi_driver s6e3fa7_panel_driver = {
+	.probe = s6e3fa7_panel_probe,
+	.remove = s6e3fa7_panel_remove,
+	.driver = {
+		.name = "panel-samsung-s6e3fa7",
+		.of_match_table = s6e3fa7_panel_of_match,
+	},
+};
+module_mipi_dsi_driver(s6e3fa7_panel_driver);
+
+MODULE_AUTHOR("Richard Acayan <mailingradian@gmail.com>");
+MODULE_DESCRIPTION("DRM driver for Samsung S6E3FA7 command mode DSI panel");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 20e3df1c59d4..dcb6d0b6ced0 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -151,7 +151,7 @@ struct panel_simple {
 
 	struct gpio_desc *enable_gpio;
 
-	struct edid *edid;
+	const struct drm_edid *drm_edid;
 
 	struct drm_display_mode override_mode;
 
@@ -309,8 +309,8 @@ static int panel_simple_suspend(struct device *dev)
 	regulator_disable(p->supply);
 	p->unprepared_time = ktime_get_boottime();
 
-	kfree(p->edid);
-	p->edid = NULL;
+	drm_edid_free(p->drm_edid);
+	p->drm_edid = NULL;
 
 	return 0;
 }
@@ -399,11 +399,12 @@ static int panel_simple_get_modes(struct drm_panel *panel,
 	if (p->ddc) {
 		pm_runtime_get_sync(panel->dev);
 
-		if (!p->edid)
-			p->edid = drm_get_edid(connector, p->ddc);
+		if (!p->drm_edid)
+			p->drm_edid = drm_edid_read_ddc(connector, p->ddc);
 
-		if (p->edid)
-			num += drm_add_edid_modes(connector, p->edid);
+		drm_edid_connector_update(connector, p->drm_edid);
+
+		num += drm_edid_connector_add_modes(connector);
 
 		pm_runtime_mark_last_busy(panel->dev);
 		pm_runtime_put_autosuspend(panel->dev);
@@ -1457,6 +1458,32 @@ static const struct panel_desc boe_hv070wsa = {
 	.connector_type = DRM_MODE_CONNECTOR_LVDS,
 };
 
+static const struct display_timing cct_cmt430b19n00_timing = {
+	.pixelclock = { 8000000, 9000000, 12000000 },
+	.hactive = { 480, 480, 480 },
+	.hfront_porch = { 2, 8, 75 },
+	.hback_porch = { 3, 43, 43 },
+	.hsync_len = { 2, 4, 75 },
+	.vactive = { 272, 272, 272 },
+	.vfront_porch = { 2, 8, 37 },
+	.vback_porch = { 2, 12, 12 },
+	.vsync_len = { 2, 4, 37 },
+	.flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW
+};
+
+static const struct panel_desc cct_cmt430b19n00 = {
+	.timings = &cct_cmt430b19n00_timing,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 95,
+		.height = 53,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
+	.connector_type = DRM_MODE_CONNECTOR_DPI,
+};
+
 static const struct drm_display_mode cdtech_s043wq26h_ct7_mode = {
 	.clock = 9000,
 	.hdisplay = 480,
@@ -2565,22 +2592,22 @@ static const struct panel_desc innolux_g121i1_l01 = {
 	.connector_type = DRM_MODE_CONNECTOR_LVDS,
 };
 
-static const struct drm_display_mode innolux_g121x1_l03_mode = {
-	.clock = 65000,
-	.hdisplay = 1024,
-	.hsync_start = 1024 + 0,
-	.hsync_end = 1024 + 1,
-	.htotal = 1024 + 0 + 1 + 320,
-	.vdisplay = 768,
-	.vsync_start = 768 + 38,
-	.vsync_end = 768 + 38 + 1,
-	.vtotal = 768 + 38 + 1 + 0,
-	.flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+static const struct display_timing innolux_g121x1_l03_timings = {
+	.pixelclock = { 57500000, 64900000, 74400000 },
+	.hactive = { 1024, 1024, 1024 },
+	.hfront_porch = { 90, 140, 190 },
+	.hback_porch = { 90, 140, 190 },
+	.hsync_len = { 36, 40, 60 },
+	.vactive = { 768, 768, 768 },
+	.vfront_porch = { 2, 15, 30 },
+	.vback_porch = { 2, 15, 30 },
+	.vsync_len = { 2, 8, 20 },
+	.flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW,
 };
 
 static const struct panel_desc innolux_g121x1_l03 = {
-	.modes = &innolux_g121x1_l03_mode,
-	.num_modes = 1,
+	.timings = &innolux_g121x1_l03_timings,
+	.num_timings = 1,
 	.bpc = 6,
 	.size = {
 		.width = 246,
@@ -2591,6 +2618,27 @@ static const struct panel_desc innolux_g121x1_l03 = {
 		.unprepare = 200,
 		.disable = 400,
 	},
+	.bus_format = MEDIA_BUS_FMT_RGB666_1X7X3_SPWG,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
+static const struct panel_desc innolux_g121xce_l01 = {
+	.timings = &innolux_g121x1_l03_timings,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 246,
+		.height = 185,
+	},
+	.delay = {
+		.enable = 200,
+		.unprepare = 200,
+		.disable = 400,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
 };
 
 static const struct display_timing innolux_g156hce_l01_timings = {
@@ -3465,6 +3513,32 @@ static const struct panel_desc pda_91_00156_a0  = {
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
 };
 
+static const struct drm_display_mode powertip_ph128800t006_zhc01_mode = {
+	.clock = 66500,
+	.hdisplay = 1280,
+	.hsync_start = 1280 + 12,
+	.hsync_end = 1280 + 12 + 20,
+	.htotal = 1280 + 12 + 20 + 56,
+	.vdisplay = 800,
+	.vsync_start = 800 + 1,
+	.vsync_end = 800 + 1 + 3,
+	.vtotal = 800 + 1 + 3 + 20,
+	.flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC,
+};
+
+static const struct panel_desc powertip_ph128800t006_zhc01 = {
+	.modes = &powertip_ph128800t006_zhc01_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 216,
+		.height = 135,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
 static const struct drm_display_mode powertip_ph800480t013_idf02_mode = {
 	.clock = 24750,
 	.hdisplay = 800,
@@ -4403,6 +4477,9 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible = "boe,hv070wsa-100",
 		.data = &boe_hv070wsa
 	}, {
+		.compatible = "cct,cmt430b19n00",
+		.data = &cct_cmt430b19n00,
+	}, {
 		.compatible = "cdtech,s043wq26h-ct7",
 		.data = &cdtech_s043wq26h_ct7,
 	}, {
@@ -4538,6 +4615,9 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible = "innolux,g121x1-l03",
 		.data = &innolux_g121x1_l03,
 	}, {
+		.compatible = "innolux,g121xce-l01",
+		.data = &innolux_g121xce_l01,
+	}, {
 		.compatible = "innolux,g156hce-l01",
 		.data = &innolux_g156hce_l01,
 	}, {
@@ -4640,6 +4720,9 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible = "pda,91-00156-a0",
 		.data = &pda_91_00156_a0,
 	}, {
+		.compatible = "powertip,ph128800t006-zhc01",
+		.data = &powertip_ph128800t006_zhc01,
+	}, {
 		.compatible = "powertip,ph800480t013-idf02",
 		.data = &powertip_ph800480t013_idf02,
 	}, {
diff --git a/drivers/gpu/drm/panel/panel-sitronix-st7703.c b/drivers/gpu/drm/panel/panel-sitronix-st7703.c
index a3e142f156d5..7d8302cca091 100644
--- a/drivers/gpu/drm/panel/panel-sitronix-st7703.c
+++ b/drivers/gpu/drm/panel/panel-sitronix-st7703.c
@@ -612,6 +612,92 @@ static const struct st7703_panel_desc rgb10max3_panel_desc = {
 	.init_sequence = rgb10max3_panel_init_sequence,
 };
 
+static int gameforcechi_init_sequence(struct st7703 *ctx)
+{
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+
+	/*
+	 * Init sequence was supplied by the panel vendor. Panel will not
+	 * respond to commands until it is brought out of sleep mode first.
+	 */
+
+	mipi_dsi_dcs_exit_sleep_mode(dsi);
+	msleep(250);
+
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETEXTC, 0xf1, 0x12, 0x83);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETMIPI, 0x31, 0x81, 0x05, 0xf9,
+			       0x0e, 0x0e, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00,
+			       0x00, 0x00, 0x44, 0x25, 0x00, 0x91, 0x0a, 0x00,
+			       0x00, 0x02, 0x4f, 0xd1, 0x00, 0x00, 0x37);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETPOWER_EXT, 0x25);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_UNKNOWN_BF, 0x02, 0x11, 0x00);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETRGBIF, 0x0c, 0x10, 0x0a,
+			       0x50, 0x03, 0xff, 0x00, 0x00, 0x00, 0x00);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETSCR, 0x73, 0x73, 0x50, 0x50,
+			       0x00, 0x00, 0x08, 0x70, 0x00);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETVDC, 0x46);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETPANEL, 0x0b);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETCYC, 0x80);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETDISP, 0x00, 0x13, 0xf0);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETEQ, 0x07, 0x07, 0x0b, 0x0b,
+			       0x03, 0x0b, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00,
+			       0xc0, 0x10);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETPOWER, 0x53, 0x00, 0x1e,
+			       0x1e, 0x77, 0xe1, 0xcc, 0xdd, 0x67, 0x77, 0x33,
+			       0x33);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETBGP, 0x10, 0x10);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETVCOM, 0x6c, 0x7c);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETGIP1, 0x08, 0x00, 0x0e, 0x00,
+			       0x00, 0xb0, 0xb1, 0x11, 0x31, 0x23, 0x28, 0x10,
+			       0xb0, 0xb1, 0x27, 0x08, 0x00, 0x04, 0x02, 0x00,
+			       0x00, 0x00, 0x00, 0x04, 0x02, 0x00, 0x00, 0x00,
+			       0x88, 0x88, 0xba, 0x60, 0x24, 0x08, 0x88, 0x88,
+			       0x88, 0x88, 0x88, 0x88, 0x88, 0xba, 0x71, 0x35,
+			       0x18, 0x88, 0x88, 0x88, 0x88, 0x88, 0x00, 0x00,
+			       0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			       0x00, 0x00, 0x00);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETGIP2, 0x97, 0x0a, 0x82, 0x02,
+			       0x13, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			       0x80, 0x88, 0xba, 0x17, 0x53, 0x88, 0x88, 0x88,
+			       0x88, 0x88, 0x88, 0x81, 0x88, 0xba, 0x06, 0x42,
+			       0x88, 0x88, 0x88, 0x88, 0x88, 0x88, 0x23, 0x10,
+			       0x00, 0x02, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00,
+			       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			       0x00);
+	mipi_dsi_dcs_write_seq(dsi, ST7703_CMD_SETGAMMA, 0x00, 0x07, 0x0b,
+			       0x27, 0x2d, 0x3f, 0x3b, 0x37, 0x05, 0x0a, 0x0b,
+			       0x0f, 0x11, 0x0f, 0x12, 0x12, 0x18, 0x00, 0x07,
+			       0x0b, 0x27, 0x2d, 0x3f, 0x3b, 0x37, 0x05, 0xa0,
+			       0x0b, 0x0f, 0x11, 0x0f, 0x12, 0x12, 0x18);
+
+	return 0;
+}
+
+static const struct drm_display_mode gameforcechi_mode = {
+	.hdisplay	= 640,
+	.hsync_start	= 640 + 40,
+	.hsync_end	= 640 + 40 + 2,
+	.htotal		= 640 + 40 + 2 + 80,
+	.vdisplay	= 480,
+	.vsync_start	= 480 + 17,
+	.vsync_end	= 480 + 17 + 5,
+	.vtotal		= 480 + 17 + 5 + 13,
+	.clock		= 23546,
+	.flags		= DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+	.width_mm	= 71,
+	.height_mm	= 53,
+};
+
+static const struct st7703_panel_desc gameforcechi_desc = {
+	.mode = &gameforcechi_mode,
+	.lanes = 2,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+		      MIPI_DSI_MODE_NO_EOT_PACKET | MIPI_DSI_MODE_LPM,
+	.format = MIPI_DSI_FMT_RGB888,
+	.init_sequence = gameforcechi_init_sequence,
+};
+
 static int st7703_enable(struct drm_panel *panel)
 {
 	struct st7703 *ctx = panel_to_st7703(panel);
@@ -887,6 +973,7 @@ static void st7703_remove(struct mipi_dsi_device *dsi)
 
 static const struct of_device_id st7703_of_match[] = {
 	{ .compatible = "anbernic,rg353v-panel-v2", .data = &rg353v2_desc },
+	{ .compatible = "gameforce,chi-panel", .data = &gameforcechi_desc },
 	{ .compatible = "powkiddy,rgb10max3-panel", .data = &rgb10max3_panel_desc },
 	{ .compatible = "powkiddy,rgb30-panel", .data = &rgb30panel_desc },
 	{ .compatible = "rocktech,jh057n00900", .data = &jh057n00900_panel_desc },
diff --git a/drivers/gpu/drm/panel/panel-sitronix-st7789v.c b/drivers/gpu/drm/panel/panel-sitronix-st7789v.c
index 88e80fe98112..e8f385b9c618 100644
--- a/drivers/gpu/drm/panel/panel-sitronix-st7789v.c
+++ b/drivers/gpu/drm/panel/panel-sitronix-st7789v.c
@@ -282,15 +282,15 @@ static const struct drm_display_mode et028013dma_mode = {
 static const struct drm_display_mode jt240mhqs_hwt_ek_e3_mode = {
 	.clock = 6000,
 	.hdisplay = 240,
-	.hsync_start = 240 + 28,
-	.hsync_end = 240 + 28 + 10,
-	.htotal = 240 + 28 + 10 + 10,
+	.hsync_start = 240 + 38,
+	.hsync_end = 240 + 38 + 10,
+	.htotal = 240 + 38 + 10 + 10,
 	.vdisplay = 280,
-	.vsync_start = 280 + 8,
-	.vsync_end = 280 + 8 + 4,
-	.vtotal = 280 + 8 + 4 + 4,
-	.width_mm = 43,
-	.height_mm = 37,
+	.vsync_start = 280 + 48,
+	.vsync_end = 280 + 48 + 4,
+	.vtotal = 280 + 48 + 4 + 4,
+	.width_mm = 37,
+	.height_mm = 43,
 	.flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC,
 };
 
diff --git a/drivers/gpu/drm/panel/panel-truly-nt35597.c b/drivers/gpu/drm/panel/panel-truly-nt35597.c
index b73448cf349d..d447db912a61 100644
--- a/drivers/gpu/drm/panel/panel-truly-nt35597.c
+++ b/drivers/gpu/drm/panel/panel-truly-nt35597.c
@@ -550,10 +550,8 @@ static int truly_nt35597_probe(struct mipi_dsi_device *dsi)
 
 	dsi1_host = of_find_mipi_dsi_host_by_node(dsi1);
 	of_node_put(dsi1);
-	if (!dsi1_host) {
-		dev_err(dev, "failed to find dsi host\n");
-		return -EPROBE_DEFER;
-	}
+	if (!dsi1_host)
+		return dev_err_probe(dev, -EPROBE_DEFER, "failed to find dsi host\n");
 
 	/* register the second DSI device */
 	dsi1_device = mipi_dsi_device_register_full(dsi1_host, &info);
diff --git a/drivers/gpu/drm/panel/panel-visionox-rm69299.c b/drivers/gpu/drm/panel/panel-visionox-rm69299.c
index 775144695283..272490b9565b 100644
--- a/drivers/gpu/drm/panel/panel-visionox-rm69299.c
+++ b/drivers/gpu/drm/panel/panel-visionox-rm69299.c
@@ -197,7 +197,9 @@ static int visionox_rm69299_probe(struct mipi_dsi_device *dsi)
 	ctx->dsi = dsi;
 
 	ctx->supplies[0].supply = "vdda";
+	ctx->supplies[0].init_load_uA = 32000;
 	ctx->supplies[1].supply = "vdd3p3";
+	ctx->supplies[1].init_load_uA = 13200;
 
 	ret = devm_regulator_bulk_get(ctx->panel.dev, ARRAY_SIZE(ctx->supplies),
 				      ctx->supplies);
@@ -227,22 +229,8 @@ static int visionox_rm69299_probe(struct mipi_dsi_device *dsi)
 		goto err_dsi_attach;
 	}
 
-	ret = regulator_set_load(ctx->supplies[0].consumer, 32000);
-	if (ret) {
-		dev_err(dev, "regulator set load failed for vdda supply ret = %d\n", ret);
-		goto err_set_load;
-	}
-
-	ret = regulator_set_load(ctx->supplies[1].consumer, 13200);
-	if (ret) {
-		dev_err(dev, "regulator set load failed for vdd3p3 supply ret = %d\n", ret);
-		goto err_set_load;
-	}
-
 	return 0;
 
-err_set_load:
-	mipi_dsi_detach(dsi);
 err_dsi_attach:
 	drm_panel_remove(&ctx->panel);
 	return ret;
@@ -253,8 +241,6 @@ static void visionox_rm69299_remove(struct mipi_dsi_device *dsi)
 	struct visionox_rm69299 *ctx = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(ctx->dsi);
-	mipi_dsi_device_unregister(ctx->dsi);
-
 	drm_panel_remove(&ctx->panel);
 }
 
diff --git a/drivers/gpu/drm/panfrost/Makefile b/drivers/gpu/drm/panfrost/Makefile
index 2c01c1e7523e..7da2b3f02ed9 100644
--- a/drivers/gpu/drm/panfrost/Makefile
+++ b/drivers/gpu/drm/panfrost/Makefile
@@ -12,6 +12,4 @@ panfrost-y := \
 	panfrost_perfcnt.o \
 	panfrost_dump.o
 
-panfrost-$(CONFIG_DEBUG_FS) += panfrost_debugfs.o
-
 obj-$(CONFIG_DRM_PANFROST) += panfrost.o
diff --git a/drivers/gpu/drm/panfrost/panfrost_debugfs.c b/drivers/gpu/drm/panfrost/panfrost_debugfs.c
deleted file mode 100644
index 72d4286a6bf7..000000000000
--- a/drivers/gpu/drm/panfrost/panfrost_debugfs.c
+++ /dev/null
@@ -1,21 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Copyright 2023 Collabora ltd. */
-/* Copyright 2023 Amazon.com, Inc. or its affiliates. */
-
-#include <linux/debugfs.h>
-#include <linux/platform_device.h>
-#include <drm/drm_debugfs.h>
-#include <drm/drm_file.h>
-#include <drm/panfrost_drm.h>
-
-#include "panfrost_device.h"
-#include "panfrost_gpu.h"
-#include "panfrost_debugfs.h"
-
-void panfrost_debugfs_init(struct drm_minor *minor)
-{
-	struct drm_device *dev = minor->dev;
-	struct panfrost_device *pfdev = platform_get_drvdata(to_platform_device(dev->dev));
-
-	debugfs_create_atomic_t("profile", 0600, minor->debugfs_root, &pfdev->profile_mode);
-}
diff --git a/drivers/gpu/drm/panfrost/panfrost_debugfs.h b/drivers/gpu/drm/panfrost/panfrost_debugfs.h
deleted file mode 100644
index c5af5f35877f..000000000000
--- a/drivers/gpu/drm/panfrost/panfrost_debugfs.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright 2023 Collabora ltd.
- * Copyright 2023 Amazon.com, Inc. or its affiliates.
- */
-
-#ifndef PANFROST_DEBUGFS_H
-#define PANFROST_DEBUGFS_H
-
-#ifdef CONFIG_DEBUG_FS
-void panfrost_debugfs_init(struct drm_minor *minor);
-#endif
-
-#endif  /* PANFROST_DEBUGFS_H */
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h
index 62f7e3527385..cffcb0ac7c11 100644
--- a/drivers/gpu/drm/panfrost/panfrost_device.h
+++ b/drivers/gpu/drm/panfrost/panfrost_device.h
@@ -130,7 +130,7 @@ struct panfrost_device {
 	struct list_head scheduled_jobs;
 
 	struct panfrost_perfcnt *perfcnt;
-	atomic_t profile_mode;
+	bool profile_mode;
 
 	struct mutex sched_lock;
 
diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
index a926d71e8131..ef9f6c0716d5 100644
--- a/drivers/gpu/drm/panfrost/panfrost_drv.c
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
@@ -20,7 +20,6 @@
 #include "panfrost_job.h"
 #include "panfrost_gpu.h"
 #include "panfrost_perfcnt.h"
-#include "panfrost_debugfs.h"
 
 static bool unstable_ioctls;
 module_param_unsafe(unstable_ioctls, bool, 0600);
@@ -551,10 +550,12 @@ static void panfrost_gpu_show_fdinfo(struct panfrost_device *pfdev,
 	BUILD_BUG_ON(ARRAY_SIZE(engine_names) != NUM_JOB_SLOTS);
 
 	for (i = 0; i < NUM_JOB_SLOTS - 1; i++) {
-		drm_printf(p, "drm-engine-%s:\t%llu ns\n",
-			   engine_names[i], panfrost_priv->engine_usage.elapsed_ns[i]);
-		drm_printf(p, "drm-cycles-%s:\t%llu\n",
-			   engine_names[i], panfrost_priv->engine_usage.cycles[i]);
+		if (pfdev->profile_mode) {
+			drm_printf(p, "drm-engine-%s:\t%llu ns\n",
+				   engine_names[i], panfrost_priv->engine_usage.elapsed_ns[i]);
+			drm_printf(p, "drm-cycles-%s:\t%llu\n",
+				   engine_names[i], panfrost_priv->engine_usage.cycles[i]);
+		}
 		drm_printf(p, "drm-maxfreq-%s:\t%lu Hz\n",
 			   engine_names[i], pfdev->pfdevfreq.fast_rate);
 		drm_printf(p, "drm-curfreq-%s:\t%lu Hz\n",
@@ -600,10 +601,6 @@ static const struct drm_driver panfrost_drm_driver = {
 
 	.gem_create_object	= panfrost_gem_create_object,
 	.gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table,
-
-#ifdef CONFIG_DEBUG_FS
-	.debugfs_init		= panfrost_debugfs_init,
-#endif
 };
 
 static int panfrost_probe(struct platform_device *pdev)
@@ -692,6 +689,40 @@ static void panfrost_remove(struct platform_device *pdev)
 	drm_dev_put(ddev);
 }
 
+static ssize_t profiling_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	struct panfrost_device *pfdev = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "%d\n", pfdev->profile_mode);
+}
+
+static ssize_t profiling_store(struct device *dev,
+			       struct device_attribute *attr,
+			       const char *buf, size_t len)
+{
+	struct panfrost_device *pfdev = dev_get_drvdata(dev);
+	bool value;
+	int err;
+
+	err = kstrtobool(buf, &value);
+	if (err)
+		return err;
+
+	pfdev->profile_mode = value;
+
+	return len;
+}
+
+static DEVICE_ATTR_RW(profiling);
+
+static struct attribute *panfrost_attrs[] = {
+	&dev_attr_profiling.attr,
+	NULL,
+};
+
+ATTRIBUTE_GROUPS(panfrost);
+
 /*
  * The OPP core wants the supply names to be NULL terminated, but we need the
  * correct num_supplies value for regulator core. Hence, we NULL terminate here
@@ -789,6 +820,7 @@ static struct platform_driver panfrost_driver = {
 		.name	= "panfrost",
 		.pm	= pm_ptr(&panfrost_pm_ops),
 		.of_match_table = dt_match,
+		.dev_groups = panfrost_groups,
 	},
 };
 module_platform_driver(panfrost_driver);
diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c
index d47b40b82b0b..8e0ff3efede7 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gem.c
+++ b/drivers/gpu/drm/panfrost/panfrost_gem.c
@@ -192,7 +192,7 @@ static int panfrost_gem_pin(struct drm_gem_object *obj)
 	if (bo->is_heap)
 		return -EINVAL;
 
-	return drm_gem_shmem_pin(&bo->base);
+	return drm_gem_shmem_pin_locked(&bo->base);
 }
 
 static enum drm_gem_object_status panfrost_gem_status(struct drm_gem_object *obj)
diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c
index 0c2dbf6ef2a5..a61ef0af9a4e 100644
--- a/drivers/gpu/drm/panfrost/panfrost_job.c
+++ b/drivers/gpu/drm/panfrost/panfrost_job.c
@@ -243,7 +243,7 @@ static void panfrost_job_hw_submit(struct panfrost_job *job, int js)
 	subslot = panfrost_enqueue_job(pfdev, js, job);
 	/* Don't queue the job if a reset is in progress */
 	if (!atomic_read(&pfdev->reset.pending)) {
-		if (atomic_read(&pfdev->profile_mode)) {
+		if (pfdev->profile_mode) {
 			panfrost_cycle_counter_get(pfdev);
 			job->is_profiled = true;
 			job->start_time = ktime_get();
diff --git a/drivers/gpu/drm/panthor/Kconfig b/drivers/gpu/drm/panthor/Kconfig
new file mode 100644
index 000000000000..55b40ad07f3b
--- /dev/null
+++ b/drivers/gpu/drm/panthor/Kconfig
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: GPL-2.0 or MIT
+
+config DRM_PANTHOR
+	tristate "Panthor (DRM support for ARM Mali CSF-based GPUs)"
+	depends on DRM
+	depends on ARM || ARM64 || COMPILE_TEST
+	depends on !GENERIC_ATOMIC64  # for IOMMU_IO_PGTABLE_LPAE
+	depends on MMU
+	select DEVFREQ_GOV_SIMPLE_ONDEMAND
+	select DRM_EXEC
+	select DRM_GEM_SHMEM_HELPER
+	select DRM_GPUVM
+	select DRM_SCHED
+	select IOMMU_IO_PGTABLE_LPAE
+	select IOMMU_SUPPORT
+	select PM_DEVFREQ
+	help
+	  DRM driver for ARM Mali CSF-based GPUs.
+
+	  This driver is for Mali (or Immortalis) Valhall Gxxx GPUs.
+
+	  Note that the Mali-G68 and Mali-G78, while Valhall architecture, will
+	  be supported with the panfrost driver as they are not CSF GPUs.
diff --git a/drivers/gpu/drm/panthor/Makefile b/drivers/gpu/drm/panthor/Makefile
new file mode 100644
index 000000000000..15294719b09c
--- /dev/null
+++ b/drivers/gpu/drm/panthor/Makefile
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0 or MIT
+
+panthor-y := \
+	panthor_devfreq.o \
+	panthor_device.o \
+	panthor_drv.o \
+	panthor_fw.o \
+	panthor_gem.o \
+	panthor_gpu.o \
+	panthor_heap.o \
+	panthor_mmu.o \
+	panthor_sched.o
+
+obj-$(CONFIG_DRM_PANTHOR) += panthor.o
diff --git a/drivers/gpu/drm/panthor/panthor_devfreq.c b/drivers/gpu/drm/panthor/panthor_devfreq.c
new file mode 100644
index 000000000000..c6d3c327cc24
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_devfreq.c
@@ -0,0 +1,283 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/* Copyright 2019 Collabora ltd. */
+
+#include <linux/clk.h>
+#include <linux/devfreq.h>
+#include <linux/devfreq_cooling.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+
+#include <drm/drm_managed.h>
+
+#include "panthor_devfreq.h"
+#include "panthor_device.h"
+
+/**
+ * struct panthor_devfreq - Device frequency management
+ */
+struct panthor_devfreq {
+	/** @devfreq: devfreq device. */
+	struct devfreq *devfreq;
+
+	/** @gov_data: Governor data. */
+	struct devfreq_simple_ondemand_data gov_data;
+
+	/** @busy_time: Busy time. */
+	ktime_t busy_time;
+
+	/** @idle_time: Idle time. */
+	ktime_t idle_time;
+
+	/** @time_last_update: Last update time. */
+	ktime_t time_last_update;
+
+	/** @last_busy_state: True if the GPU was busy last time we updated the state. */
+	bool last_busy_state;
+
+	/**
+	 * @lock: Lock used to protect busy_time, idle_time, time_last_update and
+	 * last_busy_state.
+	 *
+	 * These fields can be accessed concurrently by panthor_devfreq_get_dev_status()
+	 * and panthor_devfreq_record_{busy,idle}().
+	 */
+	spinlock_t lock;
+};
+
+static void panthor_devfreq_update_utilization(struct panthor_devfreq *pdevfreq)
+{
+	ktime_t now, last;
+
+	now = ktime_get();
+	last = pdevfreq->time_last_update;
+
+	if (pdevfreq->last_busy_state)
+		pdevfreq->busy_time += ktime_sub(now, last);
+	else
+		pdevfreq->idle_time += ktime_sub(now, last);
+
+	pdevfreq->time_last_update = now;
+}
+
+static int panthor_devfreq_target(struct device *dev, unsigned long *freq,
+				  u32 flags)
+{
+	struct dev_pm_opp *opp;
+
+	opp = devfreq_recommended_opp(dev, freq, flags);
+	if (IS_ERR(opp))
+		return PTR_ERR(opp);
+	dev_pm_opp_put(opp);
+
+	return dev_pm_opp_set_rate(dev, *freq);
+}
+
+static void panthor_devfreq_reset(struct panthor_devfreq *pdevfreq)
+{
+	pdevfreq->busy_time = 0;
+	pdevfreq->idle_time = 0;
+	pdevfreq->time_last_update = ktime_get();
+}
+
+static int panthor_devfreq_get_dev_status(struct device *dev,
+					  struct devfreq_dev_status *status)
+{
+	struct panthor_device *ptdev = dev_get_drvdata(dev);
+	struct panthor_devfreq *pdevfreq = ptdev->devfreq;
+	unsigned long irqflags;
+
+	status->current_frequency = clk_get_rate(ptdev->clks.core);
+
+	spin_lock_irqsave(&pdevfreq->lock, irqflags);
+
+	panthor_devfreq_update_utilization(pdevfreq);
+
+	status->total_time = ktime_to_ns(ktime_add(pdevfreq->busy_time,
+						   pdevfreq->idle_time));
+
+	status->busy_time = ktime_to_ns(pdevfreq->busy_time);
+
+	panthor_devfreq_reset(pdevfreq);
+
+	spin_unlock_irqrestore(&pdevfreq->lock, irqflags);
+
+	drm_dbg(&ptdev->base, "busy %lu total %lu %lu %% freq %lu MHz\n",
+		status->busy_time, status->total_time,
+		status->busy_time / (status->total_time / 100),
+		status->current_frequency / 1000 / 1000);
+
+	return 0;
+}
+
+static struct devfreq_dev_profile panthor_devfreq_profile = {
+	.timer = DEVFREQ_TIMER_DELAYED,
+	.polling_ms = 50, /* ~3 frames */
+	.target = panthor_devfreq_target,
+	.get_dev_status = panthor_devfreq_get_dev_status,
+};
+
+int panthor_devfreq_init(struct panthor_device *ptdev)
+{
+	/* There's actually 2 regulators (mali and sram), but the OPP core only
+	 * supports one.
+	 *
+	 * We assume the sram regulator is coupled with the mali one and let
+	 * the coupling logic deal with voltage updates.
+	 */
+	static const char * const reg_names[] = { "mali", NULL };
+	struct thermal_cooling_device *cooling;
+	struct device *dev = ptdev->base.dev;
+	struct panthor_devfreq *pdevfreq;
+	struct dev_pm_opp *opp;
+	unsigned long cur_freq;
+	int ret;
+
+	pdevfreq = drmm_kzalloc(&ptdev->base, sizeof(*ptdev->devfreq), GFP_KERNEL);
+	if (!pdevfreq)
+		return -ENOMEM;
+
+	ptdev->devfreq = pdevfreq;
+
+	ret = devm_pm_opp_set_regulators(dev, reg_names);
+	if (ret) {
+		if (ret != -EPROBE_DEFER)
+			DRM_DEV_ERROR(dev, "Couldn't set OPP regulators\n");
+
+		return ret;
+	}
+
+	ret = devm_pm_opp_of_add_table(dev);
+	if (ret)
+		return ret;
+
+	spin_lock_init(&pdevfreq->lock);
+
+	panthor_devfreq_reset(pdevfreq);
+
+	cur_freq = clk_get_rate(ptdev->clks.core);
+
+	opp = devfreq_recommended_opp(dev, &cur_freq, 0);
+	if (IS_ERR(opp))
+		return PTR_ERR(opp);
+
+	panthor_devfreq_profile.initial_freq = cur_freq;
+
+	/* Regulator coupling only takes care of synchronizing/balancing voltage
+	 * updates, but the coupled regulator needs to be enabled manually.
+	 *
+	 * We use devm_regulator_get_enable_optional() and keep the sram supply
+	 * enabled until the device is removed, just like we do for the mali
+	 * supply, which is enabled when dev_pm_opp_set_opp(dev, opp) is called,
+	 * and disabled when the opp_table is torn down, using the devm action.
+	 *
+	 * If we really care about disabling regulators on suspend, we should:
+	 * - use devm_regulator_get_optional() here
+	 * - call dev_pm_opp_set_opp(dev, NULL) before leaving this function
+	 *   (this disables the regulator passed to the OPP layer)
+	 * - call dev_pm_opp_set_opp(dev, NULL) and
+	 *   regulator_disable(ptdev->regulators.sram) in
+	 *   panthor_devfreq_suspend()
+	 * - call dev_pm_opp_set_opp(dev, default_opp) and
+	 *   regulator_enable(ptdev->regulators.sram) in
+	 *   panthor_devfreq_resume()
+	 *
+	 * But without knowing if it's beneficial or not (in term of power
+	 * consumption), or how much it slows down the suspend/resume steps,
+	 * let's just keep regulators enabled for the device lifetime.
+	 */
+	ret = devm_regulator_get_enable_optional(dev, "sram");
+	if (ret && ret != -ENODEV) {
+		if (ret != -EPROBE_DEFER)
+			DRM_DEV_ERROR(dev, "Couldn't retrieve/enable sram supply\n");
+		return ret;
+	}
+
+	/*
+	 * Set the recommend OPP this will enable and configure the regulator
+	 * if any and will avoid a switch off by regulator_late_cleanup()
+	 */
+	ret = dev_pm_opp_set_opp(dev, opp);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "Couldn't set recommended OPP\n");
+		return ret;
+	}
+
+	dev_pm_opp_put(opp);
+
+	/*
+	 * Setup default thresholds for the simple_ondemand governor.
+	 * The values are chosen based on experiments.
+	 */
+	pdevfreq->gov_data.upthreshold = 45;
+	pdevfreq->gov_data.downdifferential = 5;
+
+	pdevfreq->devfreq = devm_devfreq_add_device(dev, &panthor_devfreq_profile,
+						    DEVFREQ_GOV_SIMPLE_ONDEMAND,
+						    &pdevfreq->gov_data);
+	if (IS_ERR(pdevfreq->devfreq)) {
+		DRM_DEV_ERROR(dev, "Couldn't initialize GPU devfreq\n");
+		ret = PTR_ERR(pdevfreq->devfreq);
+		pdevfreq->devfreq = NULL;
+		return ret;
+	}
+
+	cooling = devfreq_cooling_em_register(pdevfreq->devfreq, NULL);
+	if (IS_ERR(cooling))
+		DRM_DEV_INFO(dev, "Failed to register cooling device\n");
+
+	return 0;
+}
+
+int panthor_devfreq_resume(struct panthor_device *ptdev)
+{
+	struct panthor_devfreq *pdevfreq = ptdev->devfreq;
+
+	if (!pdevfreq->devfreq)
+		return 0;
+
+	panthor_devfreq_reset(pdevfreq);
+
+	return devfreq_resume_device(pdevfreq->devfreq);
+}
+
+int panthor_devfreq_suspend(struct panthor_device *ptdev)
+{
+	struct panthor_devfreq *pdevfreq = ptdev->devfreq;
+
+	if (!pdevfreq->devfreq)
+		return 0;
+
+	return devfreq_suspend_device(pdevfreq->devfreq);
+}
+
+void panthor_devfreq_record_busy(struct panthor_device *ptdev)
+{
+	struct panthor_devfreq *pdevfreq = ptdev->devfreq;
+	unsigned long irqflags;
+
+	if (!pdevfreq->devfreq)
+		return;
+
+	spin_lock_irqsave(&pdevfreq->lock, irqflags);
+
+	panthor_devfreq_update_utilization(pdevfreq);
+	pdevfreq->last_busy_state = true;
+
+	spin_unlock_irqrestore(&pdevfreq->lock, irqflags);
+}
+
+void panthor_devfreq_record_idle(struct panthor_device *ptdev)
+{
+	struct panthor_devfreq *pdevfreq = ptdev->devfreq;
+	unsigned long irqflags;
+
+	if (!pdevfreq->devfreq)
+		return;
+
+	spin_lock_irqsave(&pdevfreq->lock, irqflags);
+
+	panthor_devfreq_update_utilization(pdevfreq);
+	pdevfreq->last_busy_state = false;
+
+	spin_unlock_irqrestore(&pdevfreq->lock, irqflags);
+}
diff --git a/drivers/gpu/drm/panthor/panthor_devfreq.h b/drivers/gpu/drm/panthor/panthor_devfreq.h
new file mode 100644
index 000000000000..83a5c9522493
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_devfreq.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2019 Collabora ltd. */
+
+#ifndef __PANTHOR_DEVFREQ_H__
+#define __PANTHOR_DEVFREQ_H__
+
+struct devfreq;
+struct thermal_cooling_device;
+
+struct panthor_device;
+struct panthor_devfreq;
+
+int panthor_devfreq_init(struct panthor_device *ptdev);
+
+int panthor_devfreq_resume(struct panthor_device *ptdev);
+int panthor_devfreq_suspend(struct panthor_device *ptdev);
+
+void panthor_devfreq_record_busy(struct panthor_device *ptdev);
+void panthor_devfreq_record_idle(struct panthor_device *ptdev);
+
+#endif /* __PANTHOR_DEVFREQ_H__ */
diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c
new file mode 100644
index 000000000000..4082c8f2951d
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_device.c
@@ -0,0 +1,557 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+/* Copyright 2023 Collabora ltd. */
+
+#include <linux/clk.h>
+#include <linux/mm.h>
+#include <linux/platform_device.h>
+#include <linux/pm_domain.h>
+#include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+
+#include <drm/drm_drv.h>
+#include <drm/drm_managed.h>
+
+#include "panthor_devfreq.h"
+#include "panthor_device.h"
+#include "panthor_fw.h"
+#include "panthor_gpu.h"
+#include "panthor_mmu.h"
+#include "panthor_regs.h"
+#include "panthor_sched.h"
+
+static int panthor_clk_init(struct panthor_device *ptdev)
+{
+	ptdev->clks.core = devm_clk_get(ptdev->base.dev, NULL);
+	if (IS_ERR(ptdev->clks.core))
+		return dev_err_probe(ptdev->base.dev,
+				     PTR_ERR(ptdev->clks.core),
+				     "get 'core' clock failed");
+
+	ptdev->clks.stacks = devm_clk_get_optional(ptdev->base.dev, "stacks");
+	if (IS_ERR(ptdev->clks.stacks))
+		return dev_err_probe(ptdev->base.dev,
+				     PTR_ERR(ptdev->clks.stacks),
+				     "get 'stacks' clock failed");
+
+	ptdev->clks.coregroup = devm_clk_get_optional(ptdev->base.dev, "coregroup");
+	if (IS_ERR(ptdev->clks.coregroup))
+		return dev_err_probe(ptdev->base.dev,
+				     PTR_ERR(ptdev->clks.coregroup),
+				     "get 'coregroup' clock failed");
+
+	drm_info(&ptdev->base, "clock rate = %lu\n", clk_get_rate(ptdev->clks.core));
+	return 0;
+}
+
+void panthor_device_unplug(struct panthor_device *ptdev)
+{
+	/* This function can be called from two different path: the reset work
+	 * and the platform device remove callback. drm_dev_unplug() doesn't
+	 * deal with concurrent callers, so we have to protect drm_dev_unplug()
+	 * calls with our own lock, and bail out if the device is already
+	 * unplugged.
+	 */
+	mutex_lock(&ptdev->unplug.lock);
+	if (drm_dev_is_unplugged(&ptdev->base)) {
+		/* Someone beat us, release the lock and wait for the unplug
+		 * operation to be reported as done.
+		 **/
+		mutex_unlock(&ptdev->unplug.lock);
+		wait_for_completion(&ptdev->unplug.done);
+		return;
+	}
+
+	/* Call drm_dev_unplug() so any access to HW blocks happening after
+	 * that point get rejected.
+	 */
+	drm_dev_unplug(&ptdev->base);
+
+	/* We do the rest of the unplug with the unplug lock released,
+	 * future callers will wait on ptdev->unplug.done anyway.
+	 */
+	mutex_unlock(&ptdev->unplug.lock);
+
+	drm_WARN_ON(&ptdev->base, pm_runtime_get_sync(ptdev->base.dev) < 0);
+
+	/* Now, try to cleanly shutdown the GPU before the device resources
+	 * get reclaimed.
+	 */
+	panthor_sched_unplug(ptdev);
+	panthor_fw_unplug(ptdev);
+	panthor_mmu_unplug(ptdev);
+	panthor_gpu_unplug(ptdev);
+
+	pm_runtime_dont_use_autosuspend(ptdev->base.dev);
+	pm_runtime_put_sync_suspend(ptdev->base.dev);
+
+	/* If PM is disabled, we need to call the suspend handler manually. */
+	if (!IS_ENABLED(CONFIG_PM))
+		panthor_device_suspend(ptdev->base.dev);
+
+	/* Report the unplug operation as done to unblock concurrent
+	 * panthor_device_unplug() callers.
+	 */
+	complete_all(&ptdev->unplug.done);
+}
+
+static void panthor_device_reset_cleanup(struct drm_device *ddev, void *data)
+{
+	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
+
+	cancel_work_sync(&ptdev->reset.work);
+	destroy_workqueue(ptdev->reset.wq);
+}
+
+static void panthor_device_reset_work(struct work_struct *work)
+{
+	struct panthor_device *ptdev = container_of(work, struct panthor_device, reset.work);
+	int ret = 0, cookie;
+
+	if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE) {
+		/*
+		 * No need for a reset as the device has been (or will be)
+		 * powered down
+		 */
+		atomic_set(&ptdev->reset.pending, 0);
+		return;
+	}
+
+	if (!drm_dev_enter(&ptdev->base, &cookie))
+		return;
+
+	panthor_sched_pre_reset(ptdev);
+	panthor_fw_pre_reset(ptdev, true);
+	panthor_mmu_pre_reset(ptdev);
+	panthor_gpu_soft_reset(ptdev);
+	panthor_gpu_l2_power_on(ptdev);
+	panthor_mmu_post_reset(ptdev);
+	ret = panthor_fw_post_reset(ptdev);
+	atomic_set(&ptdev->reset.pending, 0);
+	panthor_sched_post_reset(ptdev, ret != 0);
+	drm_dev_exit(cookie);
+
+	if (ret) {
+		panthor_device_unplug(ptdev);
+		drm_err(&ptdev->base, "Failed to boot MCU after reset, making device unusable.");
+	}
+}
+
+static bool panthor_device_is_initialized(struct panthor_device *ptdev)
+{
+	return !!ptdev->scheduler;
+}
+
+static void panthor_device_free_page(struct drm_device *ddev, void *data)
+{
+	__free_page(data);
+}
+
+int panthor_device_init(struct panthor_device *ptdev)
+{
+	u32 *dummy_page_virt;
+	struct resource *res;
+	struct page *p;
+	int ret;
+
+	ptdev->coherent = device_get_dma_attr(ptdev->base.dev) == DEV_DMA_COHERENT;
+
+	init_completion(&ptdev->unplug.done);
+	ret = drmm_mutex_init(&ptdev->base, &ptdev->unplug.lock);
+	if (ret)
+		return ret;
+
+	ret = drmm_mutex_init(&ptdev->base, &ptdev->pm.mmio_lock);
+	if (ret)
+		return ret;
+
+	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
+	p = alloc_page(GFP_KERNEL | __GFP_ZERO);
+	if (!p)
+		return -ENOMEM;
+
+	ptdev->pm.dummy_latest_flush = p;
+	dummy_page_virt = page_address(p);
+	ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_free_page,
+				       ptdev->pm.dummy_latest_flush);
+	if (ret)
+		return ret;
+
+	/*
+	 * Set the dummy page holding the latest flush to 1. This will cause the
+	 * flush to avoided as we know it isn't necessary if the submission
+	 * happens while the dummy page is mapped. Zero cannot be used because
+	 * that means 'always flush'.
+	 */
+	*dummy_page_virt = 1;
+
+	INIT_WORK(&ptdev->reset.work, panthor_device_reset_work);
+	ptdev->reset.wq = alloc_ordered_workqueue("panthor-reset-wq", 0);
+	if (!ptdev->reset.wq)
+		return -ENOMEM;
+
+	ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_reset_cleanup, NULL);
+	if (ret)
+		return ret;
+
+	ret = panthor_clk_init(ptdev);
+	if (ret)
+		return ret;
+
+	ret = panthor_devfreq_init(ptdev);
+	if (ret)
+		return ret;
+
+	ptdev->iomem = devm_platform_get_and_ioremap_resource(to_platform_device(ptdev->base.dev),
+							      0, &res);
+	if (IS_ERR(ptdev->iomem))
+		return PTR_ERR(ptdev->iomem);
+
+	ptdev->phys_addr = res->start;
+
+	ret = devm_pm_runtime_enable(ptdev->base.dev);
+	if (ret)
+		return ret;
+
+	ret = pm_runtime_resume_and_get(ptdev->base.dev);
+	if (ret)
+		return ret;
+
+	/* If PM is disabled, we need to call panthor_device_resume() manually. */
+	if (!IS_ENABLED(CONFIG_PM)) {
+		ret = panthor_device_resume(ptdev->base.dev);
+		if (ret)
+			return ret;
+	}
+
+	ret = panthor_gpu_init(ptdev);
+	if (ret)
+		goto err_rpm_put;
+
+	ret = panthor_mmu_init(ptdev);
+	if (ret)
+		goto err_unplug_gpu;
+
+	ret = panthor_fw_init(ptdev);
+	if (ret)
+		goto err_unplug_mmu;
+
+	ret = panthor_sched_init(ptdev);
+	if (ret)
+		goto err_unplug_fw;
+
+	/* ~3 frames */
+	pm_runtime_set_autosuspend_delay(ptdev->base.dev, 50);
+	pm_runtime_use_autosuspend(ptdev->base.dev);
+
+	ret = drm_dev_register(&ptdev->base, 0);
+	if (ret)
+		goto err_disable_autosuspend;
+
+	pm_runtime_put_autosuspend(ptdev->base.dev);
+	return 0;
+
+err_disable_autosuspend:
+	pm_runtime_dont_use_autosuspend(ptdev->base.dev);
+	panthor_sched_unplug(ptdev);
+
+err_unplug_fw:
+	panthor_fw_unplug(ptdev);
+
+err_unplug_mmu:
+	panthor_mmu_unplug(ptdev);
+
+err_unplug_gpu:
+	panthor_gpu_unplug(ptdev);
+
+err_rpm_put:
+	pm_runtime_put_sync_suspend(ptdev->base.dev);
+	return ret;
+}
+
+#define PANTHOR_EXCEPTION(id) \
+	[DRM_PANTHOR_EXCEPTION_ ## id] = { \
+		.name = #id, \
+	}
+
+struct panthor_exception_info {
+	const char *name;
+};
+
+static const struct panthor_exception_info panthor_exception_infos[] = {
+	PANTHOR_EXCEPTION(OK),
+	PANTHOR_EXCEPTION(TERMINATED),
+	PANTHOR_EXCEPTION(KABOOM),
+	PANTHOR_EXCEPTION(EUREKA),
+	PANTHOR_EXCEPTION(ACTIVE),
+	PANTHOR_EXCEPTION(CS_RES_TERM),
+	PANTHOR_EXCEPTION(CS_CONFIG_FAULT),
+	PANTHOR_EXCEPTION(CS_UNRECOVERABLE),
+	PANTHOR_EXCEPTION(CS_ENDPOINT_FAULT),
+	PANTHOR_EXCEPTION(CS_BUS_FAULT),
+	PANTHOR_EXCEPTION(CS_INSTR_INVALID),
+	PANTHOR_EXCEPTION(CS_CALL_STACK_OVERFLOW),
+	PANTHOR_EXCEPTION(CS_INHERIT_FAULT),
+	PANTHOR_EXCEPTION(INSTR_INVALID_PC),
+	PANTHOR_EXCEPTION(INSTR_INVALID_ENC),
+	PANTHOR_EXCEPTION(INSTR_BARRIER_FAULT),
+	PANTHOR_EXCEPTION(DATA_INVALID_FAULT),
+	PANTHOR_EXCEPTION(TILE_RANGE_FAULT),
+	PANTHOR_EXCEPTION(ADDR_RANGE_FAULT),
+	PANTHOR_EXCEPTION(IMPRECISE_FAULT),
+	PANTHOR_EXCEPTION(OOM),
+	PANTHOR_EXCEPTION(CSF_FW_INTERNAL_ERROR),
+	PANTHOR_EXCEPTION(CSF_RES_EVICTION_TIMEOUT),
+	PANTHOR_EXCEPTION(GPU_BUS_FAULT),
+	PANTHOR_EXCEPTION(GPU_SHAREABILITY_FAULT),
+	PANTHOR_EXCEPTION(SYS_SHAREABILITY_FAULT),
+	PANTHOR_EXCEPTION(GPU_CACHEABILITY_FAULT),
+	PANTHOR_EXCEPTION(TRANSLATION_FAULT_0),
+	PANTHOR_EXCEPTION(TRANSLATION_FAULT_1),
+	PANTHOR_EXCEPTION(TRANSLATION_FAULT_2),
+	PANTHOR_EXCEPTION(TRANSLATION_FAULT_3),
+	PANTHOR_EXCEPTION(TRANSLATION_FAULT_4),
+	PANTHOR_EXCEPTION(PERM_FAULT_0),
+	PANTHOR_EXCEPTION(PERM_FAULT_1),
+	PANTHOR_EXCEPTION(PERM_FAULT_2),
+	PANTHOR_EXCEPTION(PERM_FAULT_3),
+	PANTHOR_EXCEPTION(ACCESS_FLAG_1),
+	PANTHOR_EXCEPTION(ACCESS_FLAG_2),
+	PANTHOR_EXCEPTION(ACCESS_FLAG_3),
+	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_IN),
+	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT0),
+	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT1),
+	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT2),
+	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT3),
+	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_0),
+	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_1),
+	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_2),
+	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_3),
+};
+
+const char *panthor_exception_name(struct panthor_device *ptdev, u32 exception_code)
+{
+	if (exception_code >= ARRAY_SIZE(panthor_exception_infos) ||
+	    !panthor_exception_infos[exception_code].name)
+		return "Unknown exception type";
+
+	return panthor_exception_infos[exception_code].name;
+}
+
+static vm_fault_t panthor_mmio_vm_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct panthor_device *ptdev = vma->vm_private_data;
+	u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
+	unsigned long pfn;
+	pgprot_t pgprot;
+	vm_fault_t ret;
+	bool active;
+	int cookie;
+
+	if (!drm_dev_enter(&ptdev->base, &cookie))
+		return VM_FAULT_SIGBUS;
+
+	mutex_lock(&ptdev->pm.mmio_lock);
+	active = atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE;
+
+	switch (offset) {
+	case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:
+		if (active)
+			pfn = __phys_to_pfn(ptdev->phys_addr + CSF_GPU_LATEST_FLUSH_ID);
+		else
+			pfn = page_to_pfn(ptdev->pm.dummy_latest_flush);
+		break;
+
+	default:
+		ret = VM_FAULT_SIGBUS;
+		goto out_unlock;
+	}
+
+	pgprot = vma->vm_page_prot;
+	if (active)
+		pgprot = pgprot_noncached(pgprot);
+
+	ret = vmf_insert_pfn_prot(vma, vmf->address, pfn, pgprot);
+
+out_unlock:
+	mutex_unlock(&ptdev->pm.mmio_lock);
+	drm_dev_exit(cookie);
+	return ret;
+}
+
+static const struct vm_operations_struct panthor_mmio_vm_ops = {
+	.fault = panthor_mmio_vm_fault,
+};
+
+int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct *vma)
+{
+	u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
+
+	switch (offset) {
+	case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:
+		if (vma->vm_end - vma->vm_start != PAGE_SIZE ||
+		    (vma->vm_flags & (VM_WRITE | VM_EXEC)))
+			return -EINVAL;
+
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	/* Defer actual mapping to the fault handler. */
+	vma->vm_private_data = ptdev;
+	vma->vm_ops = &panthor_mmio_vm_ops;
+	vm_flags_set(vma,
+		     VM_IO | VM_DONTCOPY | VM_DONTEXPAND |
+		     VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
+	return 0;
+}
+
+int panthor_device_resume(struct device *dev)
+{
+	struct panthor_device *ptdev = dev_get_drvdata(dev);
+	int ret, cookie;
+
+	if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_SUSPENDED)
+		return -EINVAL;
+
+	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_RESUMING);
+
+	ret = clk_prepare_enable(ptdev->clks.core);
+	if (ret)
+		goto err_set_suspended;
+
+	ret = clk_prepare_enable(ptdev->clks.stacks);
+	if (ret)
+		goto err_disable_core_clk;
+
+	ret = clk_prepare_enable(ptdev->clks.coregroup);
+	if (ret)
+		goto err_disable_stacks_clk;
+
+	ret = panthor_devfreq_resume(ptdev);
+	if (ret)
+		goto err_disable_coregroup_clk;
+
+	if (panthor_device_is_initialized(ptdev) &&
+	    drm_dev_enter(&ptdev->base, &cookie)) {
+		panthor_gpu_resume(ptdev);
+		panthor_mmu_resume(ptdev);
+		ret = drm_WARN_ON(&ptdev->base, panthor_fw_resume(ptdev));
+		if (!ret) {
+			panthor_sched_resume(ptdev);
+		} else {
+			panthor_mmu_suspend(ptdev);
+			panthor_gpu_suspend(ptdev);
+		}
+
+		drm_dev_exit(cookie);
+
+		if (ret)
+			goto err_suspend_devfreq;
+	}
+
+	if (atomic_read(&ptdev->reset.pending))
+		queue_work(ptdev->reset.wq, &ptdev->reset.work);
+
+	/* Clear all IOMEM mappings pointing to this device after we've
+	 * resumed. This way the fake mappings pointing to the dummy pages
+	 * are removed and the real iomem mapping will be restored on next
+	 * access.
+	 */
+	mutex_lock(&ptdev->pm.mmio_lock);
+	unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
+			    DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
+	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE);
+	mutex_unlock(&ptdev->pm.mmio_lock);
+	return 0;
+
+err_suspend_devfreq:
+	panthor_devfreq_suspend(ptdev);
+
+err_disable_coregroup_clk:
+	clk_disable_unprepare(ptdev->clks.coregroup);
+
+err_disable_stacks_clk:
+	clk_disable_unprepare(ptdev->clks.stacks);
+
+err_disable_core_clk:
+	clk_disable_unprepare(ptdev->clks.core);
+
+err_set_suspended:
+	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
+	return ret;
+}
+
+int panthor_device_suspend(struct device *dev)
+{
+	struct panthor_device *ptdev = dev_get_drvdata(dev);
+	int ret, cookie;
+
+	if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE)
+		return -EINVAL;
+
+	/* Clear all IOMEM mappings pointing to this device before we
+	 * shutdown the power-domain and clocks. Failing to do that results
+	 * in external aborts when the process accesses the iomem region.
+	 * We change the state and call unmap_mapping_range() with the
+	 * mmio_lock held to make sure the vm_fault handler won't set up
+	 * invalid mappings.
+	 */
+	mutex_lock(&ptdev->pm.mmio_lock);
+	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDING);
+	unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
+			    DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
+	mutex_unlock(&ptdev->pm.mmio_lock);
+
+	if (panthor_device_is_initialized(ptdev) &&
+	    drm_dev_enter(&ptdev->base, &cookie)) {
+		cancel_work_sync(&ptdev->reset.work);
+
+		/* We prepare everything as if we were resetting the GPU.
+		 * The end of the reset will happen in the resume path though.
+		 */
+		panthor_sched_suspend(ptdev);
+		panthor_fw_suspend(ptdev);
+		panthor_mmu_suspend(ptdev);
+		panthor_gpu_suspend(ptdev);
+		drm_dev_exit(cookie);
+	}
+
+	ret = panthor_devfreq_suspend(ptdev);
+	if (ret) {
+		if (panthor_device_is_initialized(ptdev) &&
+		    drm_dev_enter(&ptdev->base, &cookie)) {
+			panthor_gpu_resume(ptdev);
+			panthor_mmu_resume(ptdev);
+			drm_WARN_ON(&ptdev->base, panthor_fw_resume(ptdev));
+			panthor_sched_resume(ptdev);
+			drm_dev_exit(cookie);
+		}
+
+		goto err_set_active;
+	}
+
+	clk_disable_unprepare(ptdev->clks.coregroup);
+	clk_disable_unprepare(ptdev->clks.stacks);
+	clk_disable_unprepare(ptdev->clks.core);
+	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
+	return 0;
+
+err_set_active:
+	/* If something failed and we have to revert back to an
+	 * active state, we also need to clear the MMIO userspace
+	 * mappings, so any dumb pages that were mapped while we
+	 * were trying to suspend gets invalidated.
+	 */
+	mutex_lock(&ptdev->pm.mmio_lock);
+	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE);
+	unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
+			    DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
+	mutex_unlock(&ptdev->pm.mmio_lock);
+	return ret;
+}
diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h
new file mode 100644
index 000000000000..e388c0472ba7
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_device.h
@@ -0,0 +1,358 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+/* Copyright 2023 Collabora ltd. */
+
+#ifndef __PANTHOR_DEVICE_H__
+#define __PANTHOR_DEVICE_H__
+
+#include <linux/atomic.h>
+#include <linux/io-pgtable.h>
+#include <linux/regulator/consumer.h>
+#include <linux/sched.h>
+#include <linux/spinlock.h>
+
+#include <drm/drm_device.h>
+#include <drm/drm_mm.h>
+#include <drm/gpu_scheduler.h>
+#include <drm/panthor_drm.h>
+
+struct panthor_csf;
+struct panthor_csf_ctx;
+struct panthor_device;
+struct panthor_gpu;
+struct panthor_group_pool;
+struct panthor_heap_pool;
+struct panthor_job;
+struct panthor_mmu;
+struct panthor_fw;
+struct panthor_perfcnt;
+struct panthor_vm;
+struct panthor_vm_pool;
+
+/**
+ * enum panthor_device_pm_state - PM state
+ */
+enum panthor_device_pm_state {
+	/** @PANTHOR_DEVICE_PM_STATE_SUSPENDED: Device is suspended. */
+	PANTHOR_DEVICE_PM_STATE_SUSPENDED = 0,
+
+	/** @PANTHOR_DEVICE_PM_STATE_RESUMING: Device is being resumed. */
+	PANTHOR_DEVICE_PM_STATE_RESUMING,
+
+	/** @PANTHOR_DEVICE_PM_STATE_ACTIVE: Device is active. */
+	PANTHOR_DEVICE_PM_STATE_ACTIVE,
+
+	/** @PANTHOR_DEVICE_PM_STATE_SUSPENDING: Device is being suspended. */
+	PANTHOR_DEVICE_PM_STATE_SUSPENDING,
+};
+
+/**
+ * struct panthor_irq - IRQ data
+ *
+ * Used to automate IRQ handling for the 3 different IRQs we have in this driver.
+ */
+struct panthor_irq {
+	/** @ptdev: Panthor device */
+	struct panthor_device *ptdev;
+
+	/** @irq: IRQ number. */
+	int irq;
+
+	/** @mask: Current mask being applied to xxx_INT_MASK. */
+	u32 mask;
+
+	/** @suspended: Set to true when the IRQ is suspended. */
+	atomic_t suspended;
+};
+
+/**
+ * struct panthor_device - Panthor device
+ */
+struct panthor_device {
+	/** @base: Base drm_device. */
+	struct drm_device base;
+
+	/** @phys_addr: Physical address of the iomem region. */
+	phys_addr_t phys_addr;
+
+	/** @iomem: CPU mapping of the IOMEM region. */
+	void __iomem *iomem;
+
+	/** @clks: GPU clocks. */
+	struct {
+		/** @core: Core clock. */
+		struct clk *core;
+
+		/** @stacks: Stacks clock. This clock is optional. */
+		struct clk *stacks;
+
+		/** @coregroup: Core group clock. This clock is optional. */
+		struct clk *coregroup;
+	} clks;
+
+	/** @coherent: True if the CPU/GPU are memory coherent. */
+	bool coherent;
+
+	/** @gpu_info: GPU information. */
+	struct drm_panthor_gpu_info gpu_info;
+
+	/** @csif_info: Command stream interface information. */
+	struct drm_panthor_csif_info csif_info;
+
+	/** @gpu: GPU management data. */
+	struct panthor_gpu *gpu;
+
+	/** @fw: FW management data. */
+	struct panthor_fw *fw;
+
+	/** @mmu: MMU management data. */
+	struct panthor_mmu *mmu;
+
+	/** @scheduler: Scheduler management data. */
+	struct panthor_scheduler *scheduler;
+
+	/** @devfreq: Device frequency scaling management data. */
+	struct panthor_devfreq *devfreq;
+
+	/** @unplug: Device unplug related fields. */
+	struct {
+		/** @lock: Lock used to serialize unplug operations. */
+		struct mutex lock;
+
+		/**
+		 * @done: Completion object signaled when the unplug
+		 * operation is done.
+		 */
+		struct completion done;
+	} unplug;
+
+	/** @reset: Reset related fields. */
+	struct {
+		/** @wq: Ordered worqueud used to schedule reset operations. */
+		struct workqueue_struct *wq;
+
+		/** @work: Reset work. */
+		struct work_struct work;
+
+		/** @pending: Set to true if a reset is pending. */
+		atomic_t pending;
+	} reset;
+
+	/** @pm: Power management related data. */
+	struct {
+		/** @state: Power state. */
+		atomic_t state;
+
+		/**
+		 * @mmio_lock: Lock protecting MMIO userspace CPU mappings.
+		 *
+		 * This is needed to ensure we map the dummy IO pages when
+		 * the device is being suspended, and the real IO pages when
+		 * the device is being resumed. We can't just do with the
+		 * state atomicity to deal with this race.
+		 */
+		struct mutex mmio_lock;
+
+		/**
+		 * @dummy_latest_flush: Dummy LATEST_FLUSH page.
+		 *
+		 * Used to replace the real LATEST_FLUSH page when the GPU
+		 * is suspended.
+		 */
+		struct page *dummy_latest_flush;
+	} pm;
+};
+
+/**
+ * struct panthor_file - Panthor file
+ */
+struct panthor_file {
+	/** @ptdev: Device attached to this file. */
+	struct panthor_device *ptdev;
+
+	/** @vms: VM pool attached to this file. */
+	struct panthor_vm_pool *vms;
+
+	/** @groups: Scheduling group pool attached to this file. */
+	struct panthor_group_pool *groups;
+};
+
+int panthor_device_init(struct panthor_device *ptdev);
+void panthor_device_unplug(struct panthor_device *ptdev);
+
+/**
+ * panthor_device_schedule_reset() - Schedules a reset operation
+ */
+static inline void panthor_device_schedule_reset(struct panthor_device *ptdev)
+{
+	if (!atomic_cmpxchg(&ptdev->reset.pending, 0, 1) &&
+	    atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE)
+		queue_work(ptdev->reset.wq, &ptdev->reset.work);
+}
+
+/**
+ * panthor_device_reset_is_pending() - Checks if a reset is pending.
+ *
+ * Return: true if a reset is pending, false otherwise.
+ */
+static inline bool panthor_device_reset_is_pending(struct panthor_device *ptdev)
+{
+	return atomic_read(&ptdev->reset.pending) != 0;
+}
+
+int panthor_device_mmap_io(struct panthor_device *ptdev,
+			   struct vm_area_struct *vma);
+
+int panthor_device_resume(struct device *dev);
+int panthor_device_suspend(struct device *dev);
+
+enum drm_panthor_exception_type {
+	DRM_PANTHOR_EXCEPTION_OK = 0x00,
+	DRM_PANTHOR_EXCEPTION_TERMINATED = 0x04,
+	DRM_PANTHOR_EXCEPTION_KABOOM = 0x05,
+	DRM_PANTHOR_EXCEPTION_EUREKA = 0x06,
+	DRM_PANTHOR_EXCEPTION_ACTIVE = 0x08,
+	DRM_PANTHOR_EXCEPTION_CS_RES_TERM = 0x0f,
+	DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT = 0x3f,
+	DRM_PANTHOR_EXCEPTION_CS_CONFIG_FAULT = 0x40,
+	DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE = 0x41,
+	DRM_PANTHOR_EXCEPTION_CS_ENDPOINT_FAULT = 0x44,
+	DRM_PANTHOR_EXCEPTION_CS_BUS_FAULT = 0x48,
+	DRM_PANTHOR_EXCEPTION_CS_INSTR_INVALID = 0x49,
+	DRM_PANTHOR_EXCEPTION_CS_CALL_STACK_OVERFLOW = 0x4a,
+	DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT = 0x4b,
+	DRM_PANTHOR_EXCEPTION_INSTR_INVALID_PC = 0x50,
+	DRM_PANTHOR_EXCEPTION_INSTR_INVALID_ENC = 0x51,
+	DRM_PANTHOR_EXCEPTION_INSTR_BARRIER_FAULT = 0x55,
+	DRM_PANTHOR_EXCEPTION_DATA_INVALID_FAULT = 0x58,
+	DRM_PANTHOR_EXCEPTION_TILE_RANGE_FAULT = 0x59,
+	DRM_PANTHOR_EXCEPTION_ADDR_RANGE_FAULT = 0x5a,
+	DRM_PANTHOR_EXCEPTION_IMPRECISE_FAULT = 0x5b,
+	DRM_PANTHOR_EXCEPTION_OOM = 0x60,
+	DRM_PANTHOR_EXCEPTION_CSF_FW_INTERNAL_ERROR = 0x68,
+	DRM_PANTHOR_EXCEPTION_CSF_RES_EVICTION_TIMEOUT = 0x69,
+	DRM_PANTHOR_EXCEPTION_GPU_BUS_FAULT = 0x80,
+	DRM_PANTHOR_EXCEPTION_GPU_SHAREABILITY_FAULT = 0x88,
+	DRM_PANTHOR_EXCEPTION_SYS_SHAREABILITY_FAULT = 0x89,
+	DRM_PANTHOR_EXCEPTION_GPU_CACHEABILITY_FAULT = 0x8a,
+	DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_0 = 0xc0,
+	DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_1 = 0xc1,
+	DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_2 = 0xc2,
+	DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_3 = 0xc3,
+	DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_4 = 0xc4,
+	DRM_PANTHOR_EXCEPTION_PERM_FAULT_0 = 0xc8,
+	DRM_PANTHOR_EXCEPTION_PERM_FAULT_1 = 0xc9,
+	DRM_PANTHOR_EXCEPTION_PERM_FAULT_2 = 0xca,
+	DRM_PANTHOR_EXCEPTION_PERM_FAULT_3 = 0xcb,
+	DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_1 = 0xd9,
+	DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_2 = 0xda,
+	DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_3 = 0xdb,
+	DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_IN = 0xe0,
+	DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT0 = 0xe4,
+	DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT1 = 0xe5,
+	DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT2 = 0xe6,
+	DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT3 = 0xe7,
+	DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_0 = 0xe8,
+	DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_1 = 0xe9,
+	DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_2 = 0xea,
+	DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_3 = 0xeb,
+};
+
+/**
+ * panthor_exception_is_fault() - Checks if an exception is a fault.
+ *
+ * Return: true if the exception is a fault, false otherwise.
+ */
+static inline bool
+panthor_exception_is_fault(u32 exception_code)
+{
+	return exception_code > DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT;
+}
+
+const char *panthor_exception_name(struct panthor_device *ptdev,
+				   u32 exception_code);
+
+/**
+ * PANTHOR_IRQ_HANDLER() - Define interrupt handlers and the interrupt
+ * registration function.
+ *
+ * The boiler-plate to gracefully deal with shared interrupts is
+ * auto-generated. All you have to do is call PANTHOR_IRQ_HANDLER()
+ * just after the actual handler. The handler prototype is:
+ *
+ * void (*handler)(struct panthor_device *, u32 status);
+ */
+#define PANTHOR_IRQ_HANDLER(__name, __reg_prefix, __handler)					\
+static irqreturn_t panthor_ ## __name ## _irq_raw_handler(int irq, void *data)			\
+{												\
+	struct panthor_irq *pirq = data;							\
+	struct panthor_device *ptdev = pirq->ptdev;						\
+												\
+	if (atomic_read(&pirq->suspended))							\
+		return IRQ_NONE;								\
+	if (!gpu_read(ptdev, __reg_prefix ## _INT_STAT))					\
+		return IRQ_NONE;								\
+												\
+	gpu_write(ptdev, __reg_prefix ## _INT_MASK, 0);						\
+	return IRQ_WAKE_THREAD;									\
+}												\
+												\
+static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *data)		\
+{												\
+	struct panthor_irq *pirq = data;							\
+	struct panthor_device *ptdev = pirq->ptdev;						\
+	irqreturn_t ret = IRQ_NONE;								\
+												\
+	while (true) {										\
+		u32 status = gpu_read(ptdev, __reg_prefix ## _INT_RAWSTAT) & pirq->mask;	\
+												\
+		if (!status)									\
+			break;									\
+												\
+		gpu_write(ptdev, __reg_prefix ## _INT_CLEAR, status);				\
+												\
+		__handler(ptdev, status);							\
+		ret = IRQ_HANDLED;								\
+	}											\
+												\
+	if (!atomic_read(&pirq->suspended))							\
+		gpu_write(ptdev, __reg_prefix ## _INT_MASK, pirq->mask);			\
+												\
+	return ret;										\
+}												\
+												\
+static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq)			\
+{												\
+	pirq->mask = 0;										\
+	gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0);					\
+	synchronize_irq(pirq->irq);								\
+	atomic_set(&pirq->suspended, true);							\
+}												\
+												\
+static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq, u32 mask)	\
+{												\
+	atomic_set(&pirq->suspended, false);							\
+	pirq->mask = mask;									\
+	gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, mask);				\
+	gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, mask);				\
+}												\
+												\
+static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev,			\
+					      struct panthor_irq *pirq,				\
+					      int irq, u32 mask)				\
+{												\
+	pirq->ptdev = ptdev;									\
+	pirq->irq = irq;									\
+	panthor_ ## __name ## _irq_resume(pirq, mask);						\
+												\
+	return devm_request_threaded_irq(ptdev->base.dev, irq,					\
+					 panthor_ ## __name ## _irq_raw_handler,		\
+					 panthor_ ## __name ## _irq_threaded_handler,		\
+					 IRQF_SHARED, KBUILD_MODNAME "-" # __name,		\
+					 pirq);							\
+}
+
+extern struct workqueue_struct *panthor_cleanup_wq;
+
+#endif
diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c
new file mode 100644
index 000000000000..b8a84f26b3ef
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_drv.c
@@ -0,0 +1,1488 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
+/* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
+/* Copyright 2019 Collabora ltd. */
+
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/pagemap.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#include <drm/drm_debugfs.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_exec.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_syncobj.h>
+#include <drm/drm_utils.h>
+#include <drm/gpu_scheduler.h>
+#include <drm/panthor_drm.h>
+
+#include "panthor_device.h"
+#include "panthor_fw.h"
+#include "panthor_gem.h"
+#include "panthor_gpu.h"
+#include "panthor_heap.h"
+#include "panthor_mmu.h"
+#include "panthor_regs.h"
+#include "panthor_sched.h"
+
+/**
+ * DOC: user <-> kernel object copy helpers.
+ */
+
+/**
+ * panthor_set_uobj() - Copy kernel object to user object.
+ * @usr_ptr: Users pointer.
+ * @usr_size: Size of the user object.
+ * @min_size: Minimum size for this object.
+ * @kern_size: Size of the kernel object.
+ * @in: Address of the kernel object to copy.
+ *
+ * Helper automating kernel -> user object copies.
+ *
+ * Don't use this function directly, use PANTHOR_UOBJ_SET() instead.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int
+panthor_set_uobj(u64 usr_ptr, u32 usr_size, u32 min_size, u32 kern_size, const void *in)
+{
+	/* User size shouldn't be smaller than the minimal object size. */
+	if (usr_size < min_size)
+		return -EINVAL;
+
+	if (copy_to_user(u64_to_user_ptr(usr_ptr), in, min_t(u32, usr_size, kern_size)))
+		return -EFAULT;
+
+	/* When the kernel object is smaller than the user object, we fill the gap with
+	 * zeros.
+	 */
+	if (usr_size > kern_size &&
+	    clear_user(u64_to_user_ptr(usr_ptr + kern_size), usr_size - kern_size)) {
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_get_uobj_array() - Copy a user object array into a kernel accessible object array.
+ * @in: The object array to copy.
+ * @min_stride: Minimum array stride.
+ * @obj_size: Kernel object size.
+ *
+ * Helper automating user -> kernel object copies.
+ *
+ * Don't use this function directly, use PANTHOR_UOBJ_GET_ARRAY() instead.
+ *
+ * Return: newly allocated object array or an ERR_PTR on error.
+ */
+static void *
+panthor_get_uobj_array(const struct drm_panthor_obj_array *in, u32 min_stride,
+		       u32 obj_size)
+{
+	int ret = 0;
+	void *out_alloc;
+
+	/* User stride must be at least the minimum object size, otherwise it might
+	 * lack useful information.
+	 */
+	if (in->stride < min_stride)
+		return ERR_PTR(-EINVAL);
+
+	if (!in->count)
+		return NULL;
+
+	out_alloc = kvmalloc_array(in->count, obj_size, GFP_KERNEL);
+	if (!out_alloc)
+		return ERR_PTR(-ENOMEM);
+
+	if (obj_size == in->stride) {
+		/* Fast path when user/kernel have the same uAPI header version. */
+		if (copy_from_user(out_alloc, u64_to_user_ptr(in->array),
+				   (unsigned long)obj_size * in->count))
+			ret = -EFAULT;
+	} else {
+		void __user *in_ptr = u64_to_user_ptr(in->array);
+		void *out_ptr = out_alloc;
+
+		/* If the sizes differ, we need to copy elements one by one. */
+		for (u32 i = 0; i < in->count; i++) {
+			ret = copy_struct_from_user(out_ptr, obj_size, in_ptr, in->stride);
+			if (ret)
+				break;
+
+			out_ptr += obj_size;
+			in_ptr += in->stride;
+		}
+	}
+
+	if (ret) {
+		kvfree(out_alloc);
+		return ERR_PTR(ret);
+	}
+
+	return out_alloc;
+}
+
+/**
+ * PANTHOR_UOBJ_MIN_SIZE_INTERNAL() - Get the minimum user object size
+ * @_typename: Object type.
+ * @_last_mandatory_field: Last mandatory field.
+ *
+ * Get the minimum user object size based on the last mandatory field name,
+ * A.K.A, the name of the last field of the structure at the time this
+ * structure was added to the uAPI.
+ *
+ * Don't use directly, use PANTHOR_UOBJ_DECL() instead.
+ */
+#define PANTHOR_UOBJ_MIN_SIZE_INTERNAL(_typename, _last_mandatory_field) \
+	(offsetof(_typename, _last_mandatory_field) + \
+	 sizeof(((_typename *)NULL)->_last_mandatory_field))
+
+/**
+ * PANTHOR_UOBJ_DECL() - Declare a new uAPI object whose subject to
+ * evolutions.
+ * @_typename: Object type.
+ * @_last_mandatory_field: Last mandatory field.
+ *
+ * Should be used to extend the PANTHOR_UOBJ_MIN_SIZE() list.
+ */
+#define PANTHOR_UOBJ_DECL(_typename, _last_mandatory_field) \
+	_typename : PANTHOR_UOBJ_MIN_SIZE_INTERNAL(_typename, _last_mandatory_field)
+
+/**
+ * PANTHOR_UOBJ_MIN_SIZE() - Get the minimum size of a given uAPI object
+ * @_obj_name: Object to get the minimum size of.
+ *
+ * Don't use this macro directly, it's automatically called by
+ * PANTHOR_UOBJ_{SET,GET_ARRAY}().
+ */
+#define PANTHOR_UOBJ_MIN_SIZE(_obj_name) \
+	_Generic(_obj_name, \
+		 PANTHOR_UOBJ_DECL(struct drm_panthor_gpu_info, tiler_present), \
+		 PANTHOR_UOBJ_DECL(struct drm_panthor_csif_info, pad), \
+		 PANTHOR_UOBJ_DECL(struct drm_panthor_sync_op, timeline_value), \
+		 PANTHOR_UOBJ_DECL(struct drm_panthor_queue_submit, syncs), \
+		 PANTHOR_UOBJ_DECL(struct drm_panthor_queue_create, ringbuf_size), \
+		 PANTHOR_UOBJ_DECL(struct drm_panthor_vm_bind_op, syncs))
+
+/**
+ * PANTHOR_UOBJ_SET() - Copy a kernel object to a user object.
+ * @_dest_usr_ptr: User pointer to copy to.
+ * @_usr_size: Size of the user object.
+ * @_src_obj: Kernel object to copy (not a pointer).
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+#define PANTHOR_UOBJ_SET(_dest_usr_ptr, _usr_size, _src_obj) \
+	panthor_set_uobj(_dest_usr_ptr, _usr_size, \
+			 PANTHOR_UOBJ_MIN_SIZE(_src_obj), \
+			 sizeof(_src_obj), &(_src_obj))
+
+/**
+ * PANTHOR_UOBJ_GET_ARRAY() - Copy a user object array to a kernel accessible
+ * object array.
+ * @_dest_array: Local variable that will hold the newly allocated kernel
+ * object array.
+ * @_uobj_array: The drm_panthor_obj_array object describing the user object
+ * array.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+#define PANTHOR_UOBJ_GET_ARRAY(_dest_array, _uobj_array) \
+	({ \
+		typeof(_dest_array) _tmp; \
+		_tmp = panthor_get_uobj_array(_uobj_array, \
+					      PANTHOR_UOBJ_MIN_SIZE((_dest_array)[0]), \
+					      sizeof((_dest_array)[0])); \
+		if (!IS_ERR(_tmp)) \
+			_dest_array = _tmp; \
+		PTR_ERR_OR_ZERO(_tmp); \
+	})
+
+/**
+ * struct panthor_sync_signal - Represent a synchronization object point to attach
+ * our job fence to.
+ *
+ * This structure is here to keep track of fences that are currently bound to
+ * a specific syncobj point.
+ *
+ * At the beginning of a job submission, the fence
+ * is retrieved from the syncobj itself, and can be NULL if no fence was attached
+ * to this point.
+ *
+ * At the end, it points to the fence of the last job that had a
+ * %DRM_PANTHOR_SYNC_OP_SIGNAL on this syncobj.
+ *
+ * With jobs being submitted in batches, the fence might change several times during
+ * the process, allowing one job to wait on a job that's part of the same submission
+ * but appears earlier in the drm_panthor_group_submit::queue_submits array.
+ */
+struct panthor_sync_signal {
+	/** @node: list_head to track signal ops within a submit operation */
+	struct list_head node;
+
+	/** @handle: The syncobj handle. */
+	u32 handle;
+
+	/**
+	 * @point: The syncobj point.
+	 *
+	 * Zero for regular syncobjs, and non-zero for timeline syncobjs.
+	 */
+	u64 point;
+
+	/**
+	 * @syncobj: The sync object pointed by @handle.
+	 */
+	struct drm_syncobj *syncobj;
+
+	/**
+	 * @chain: Chain object used to link the new fence to an existing
+	 * timeline syncobj.
+	 *
+	 * NULL for regular syncobj, non-NULL for timeline syncobjs.
+	 */
+	struct dma_fence_chain *chain;
+
+	/**
+	 * @fence: The fence to assign to the syncobj or syncobj-point.
+	 */
+	struct dma_fence *fence;
+};
+
+/**
+ * struct panthor_job_ctx - Job context
+ */
+struct panthor_job_ctx {
+	/** @job: The job that is about to be submitted to drm_sched. */
+	struct drm_sched_job *job;
+
+	/** @syncops: Array of sync operations. */
+	struct drm_panthor_sync_op *syncops;
+
+	/** @syncop_count: Number of sync operations. */
+	u32 syncop_count;
+};
+
+/**
+ * struct panthor_submit_ctx - Submission context
+ *
+ * Anything that's related to a submission (%DRM_IOCTL_PANTHOR_VM_BIND or
+ * %DRM_IOCTL_PANTHOR_GROUP_SUBMIT) is kept here, so we can automate the
+ * initialization and cleanup steps.
+ */
+struct panthor_submit_ctx {
+	/** @file: DRM file this submission happens on. */
+	struct drm_file *file;
+
+	/**
+	 * @signals: List of struct panthor_sync_signal.
+	 *
+	 * %DRM_PANTHOR_SYNC_OP_SIGNAL operations will be recorded here,
+	 * and %DRM_PANTHOR_SYNC_OP_WAIT will first check if an entry
+	 * matching the syncobj+point exists before calling
+	 * drm_syncobj_find_fence(). This allows us to describe dependencies
+	 * existing between jobs that are part of the same batch.
+	 */
+	struct list_head signals;
+
+	/** @jobs: Array of jobs. */
+	struct panthor_job_ctx *jobs;
+
+	/** @job_count: Number of entries in the @jobs array. */
+	u32 job_count;
+
+	/** @exec: drm_exec context used to acquire and prepare resv objects. */
+	struct drm_exec exec;
+};
+
+#define PANTHOR_SYNC_OP_FLAGS_MASK \
+	(DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_MASK | DRM_PANTHOR_SYNC_OP_SIGNAL)
+
+static bool sync_op_is_signal(const struct drm_panthor_sync_op *sync_op)
+{
+	return !!(sync_op->flags & DRM_PANTHOR_SYNC_OP_SIGNAL);
+}
+
+static bool sync_op_is_wait(const struct drm_panthor_sync_op *sync_op)
+{
+	/* Note that DRM_PANTHOR_SYNC_OP_WAIT == 0 */
+	return !(sync_op->flags & DRM_PANTHOR_SYNC_OP_SIGNAL);
+}
+
+/**
+ * panthor_check_sync_op() - Check drm_panthor_sync_op fields
+ * @sync_op: The sync operation to check.
+ *
+ * Return: 0 on success, -EINVAL otherwise.
+ */
+static int
+panthor_check_sync_op(const struct drm_panthor_sync_op *sync_op)
+{
+	u8 handle_type;
+
+	if (sync_op->flags & ~PANTHOR_SYNC_OP_FLAGS_MASK)
+		return -EINVAL;
+
+	handle_type = sync_op->flags & DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_MASK;
+	if (handle_type != DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_SYNCOBJ &&
+	    handle_type != DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_TIMELINE_SYNCOBJ)
+		return -EINVAL;
+
+	if (handle_type == DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_SYNCOBJ &&
+	    sync_op->timeline_value != 0)
+		return -EINVAL;
+
+	return 0;
+}
+
+/**
+ * panthor_sync_signal_free() - Release resources and free a panthor_sync_signal object
+ * @sig_sync: Signal object to free.
+ */
+static void
+panthor_sync_signal_free(struct panthor_sync_signal *sig_sync)
+{
+	if (!sig_sync)
+		return;
+
+	drm_syncobj_put(sig_sync->syncobj);
+	dma_fence_chain_free(sig_sync->chain);
+	dma_fence_put(sig_sync->fence);
+	kfree(sig_sync);
+}
+
+/**
+ * panthor_submit_ctx_add_sync_signal() - Add a signal operation to a submit context
+ * @ctx: Context to add the signal operation to.
+ * @handle: Syncobj handle.
+ * @point: Syncobj point.
+ *
+ * Return: 0 on success, otherwise negative error value.
+ */
+static int
+panthor_submit_ctx_add_sync_signal(struct panthor_submit_ctx *ctx, u32 handle, u64 point)
+{
+	struct panthor_sync_signal *sig_sync;
+	struct dma_fence *cur_fence;
+	int ret;
+
+	sig_sync = kzalloc(sizeof(*sig_sync), GFP_KERNEL);
+	if (!sig_sync)
+		return -ENOMEM;
+
+	sig_sync->handle = handle;
+	sig_sync->point = point;
+
+	if (point > 0) {
+		sig_sync->chain = dma_fence_chain_alloc();
+		if (!sig_sync->chain) {
+			ret = -ENOMEM;
+			goto err_free_sig_sync;
+		}
+	}
+
+	sig_sync->syncobj = drm_syncobj_find(ctx->file, handle);
+	if (!sig_sync->syncobj) {
+		ret = -EINVAL;
+		goto err_free_sig_sync;
+	}
+
+	/* Retrieve the current fence attached to that point. It's
+	 * perfectly fine to get a NULL fence here, it just means there's
+	 * no fence attached to that point yet.
+	 */
+	if (!drm_syncobj_find_fence(ctx->file, handle, point, 0, &cur_fence))
+		sig_sync->fence = cur_fence;
+
+	list_add_tail(&sig_sync->node, &ctx->signals);
+
+	return 0;
+
+err_free_sig_sync:
+	panthor_sync_signal_free(sig_sync);
+	return ret;
+}
+
+/**
+ * panthor_submit_ctx_search_sync_signal() - Search an existing signal operation in a
+ * submit context.
+ * @ctx: Context to search the signal operation in.
+ * @handle: Syncobj handle.
+ * @point: Syncobj point.
+ *
+ * Return: A valid panthor_sync_signal object if found, NULL otherwise.
+ */
+static struct panthor_sync_signal *
+panthor_submit_ctx_search_sync_signal(struct panthor_submit_ctx *ctx, u32 handle, u64 point)
+{
+	struct panthor_sync_signal *sig_sync;
+
+	list_for_each_entry(sig_sync, &ctx->signals, node) {
+		if (handle == sig_sync->handle && point == sig_sync->point)
+			return sig_sync;
+	}
+
+	return NULL;
+}
+
+/**
+ * panthor_submit_ctx_add_job() - Add a job to a submit context
+ * @ctx: Context to search the signal operation in.
+ * @idx: Index of the job in the context.
+ * @job: Job to add.
+ * @syncs: Sync operations provided by userspace.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int
+panthor_submit_ctx_add_job(struct panthor_submit_ctx *ctx, u32 idx,
+			   struct drm_sched_job *job,
+			   const struct drm_panthor_obj_array *syncs)
+{
+	int ret;
+
+	ctx->jobs[idx].job = job;
+
+	ret = PANTHOR_UOBJ_GET_ARRAY(ctx->jobs[idx].syncops, syncs);
+	if (ret)
+		return ret;
+
+	ctx->jobs[idx].syncop_count = syncs->count;
+	return 0;
+}
+
+/**
+ * panthor_submit_ctx_get_sync_signal() - Search signal operation and add one if none was found.
+ * @ctx: Context to search the signal operation in.
+ * @handle: Syncobj handle.
+ * @point: Syncobj point.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int
+panthor_submit_ctx_get_sync_signal(struct panthor_submit_ctx *ctx, u32 handle, u64 point)
+{
+	struct panthor_sync_signal *sig_sync;
+
+	sig_sync = panthor_submit_ctx_search_sync_signal(ctx, handle, point);
+	if (sig_sync)
+		return 0;
+
+	return panthor_submit_ctx_add_sync_signal(ctx, handle, point);
+}
+
+/**
+ * panthor_submit_ctx_update_job_sync_signal_fences() - Update fences
+ * on the signal operations specified by a job.
+ * @ctx: Context to search the signal operation in.
+ * @job_idx: Index of the job to operate on.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int
+panthor_submit_ctx_update_job_sync_signal_fences(struct panthor_submit_ctx *ctx,
+						 u32 job_idx)
+{
+	struct panthor_device *ptdev = container_of(ctx->file->minor->dev,
+						    struct panthor_device,
+						    base);
+	struct dma_fence *done_fence = &ctx->jobs[job_idx].job->s_fence->finished;
+	const struct drm_panthor_sync_op *sync_ops = ctx->jobs[job_idx].syncops;
+	u32 sync_op_count = ctx->jobs[job_idx].syncop_count;
+
+	for (u32 i = 0; i < sync_op_count; i++) {
+		struct dma_fence *old_fence;
+		struct panthor_sync_signal *sig_sync;
+
+		if (!sync_op_is_signal(&sync_ops[i]))
+			continue;
+
+		sig_sync = panthor_submit_ctx_search_sync_signal(ctx, sync_ops[i].handle,
+								 sync_ops[i].timeline_value);
+		if (drm_WARN_ON(&ptdev->base, !sig_sync))
+			return -EINVAL;
+
+		old_fence = sig_sync->fence;
+		sig_sync->fence = dma_fence_get(done_fence);
+		dma_fence_put(old_fence);
+
+		if (drm_WARN_ON(&ptdev->base, !sig_sync->fence))
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_submit_ctx_collect_job_signal_ops() - Iterate over all job signal operations
+ * and add them to the context.
+ * @ctx: Context to search the signal operation in.
+ * @job_idx: Index of the job to operate on.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int
+panthor_submit_ctx_collect_job_signal_ops(struct panthor_submit_ctx *ctx,
+					  u32 job_idx)
+{
+	const struct drm_panthor_sync_op *sync_ops = ctx->jobs[job_idx].syncops;
+	u32 sync_op_count = ctx->jobs[job_idx].syncop_count;
+
+	for (u32 i = 0; i < sync_op_count; i++) {
+		int ret;
+
+		if (!sync_op_is_signal(&sync_ops[i]))
+			continue;
+
+		ret = panthor_check_sync_op(&sync_ops[i]);
+		if (ret)
+			return ret;
+
+		ret = panthor_submit_ctx_get_sync_signal(ctx,
+							 sync_ops[i].handle,
+							 sync_ops[i].timeline_value);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_submit_ctx_push_fences() - Iterate over the signal array, and for each entry, push
+ * the currently assigned fence to the associated syncobj.
+ * @ctx: Context to push fences on.
+ *
+ * This is the last step of a submission procedure, and is done once we know the submission
+ * is effective and job fences are guaranteed to be signaled in finite time.
+ */
+static void
+panthor_submit_ctx_push_fences(struct panthor_submit_ctx *ctx)
+{
+	struct panthor_sync_signal *sig_sync;
+
+	list_for_each_entry(sig_sync, &ctx->signals, node) {
+		if (sig_sync->chain) {
+			drm_syncobj_add_point(sig_sync->syncobj, sig_sync->chain,
+					      sig_sync->fence, sig_sync->point);
+			sig_sync->chain = NULL;
+		} else {
+			drm_syncobj_replace_fence(sig_sync->syncobj, sig_sync->fence);
+		}
+	}
+}
+
+/**
+ * panthor_submit_ctx_add_sync_deps_to_job() - Add sync wait operations as
+ * job dependencies.
+ * @ctx: Submit context.
+ * @job_idx: Index of the job to operate on.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int
+panthor_submit_ctx_add_sync_deps_to_job(struct panthor_submit_ctx *ctx,
+					u32 job_idx)
+{
+	struct panthor_device *ptdev = container_of(ctx->file->minor->dev,
+						    struct panthor_device,
+						    base);
+	const struct drm_panthor_sync_op *sync_ops = ctx->jobs[job_idx].syncops;
+	struct drm_sched_job *job = ctx->jobs[job_idx].job;
+	u32 sync_op_count = ctx->jobs[job_idx].syncop_count;
+	int ret = 0;
+
+	for (u32 i = 0; i < sync_op_count; i++) {
+		struct panthor_sync_signal *sig_sync;
+		struct dma_fence *fence;
+
+		if (!sync_op_is_wait(&sync_ops[i]))
+			continue;
+
+		ret = panthor_check_sync_op(&sync_ops[i]);
+		if (ret)
+			return ret;
+
+		sig_sync = panthor_submit_ctx_search_sync_signal(ctx, sync_ops[i].handle,
+								 sync_ops[i].timeline_value);
+		if (sig_sync) {
+			if (drm_WARN_ON(&ptdev->base, !sig_sync->fence))
+				return -EINVAL;
+
+			fence = dma_fence_get(sig_sync->fence);
+		} else {
+			ret = drm_syncobj_find_fence(ctx->file, sync_ops[i].handle,
+						     sync_ops[i].timeline_value,
+						     0, &fence);
+			if (ret)
+				return ret;
+		}
+
+		ret = drm_sched_job_add_dependency(job, fence);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_submit_ctx_collect_jobs_signal_ops() - Collect all signal operations
+ * and add them to the submit context.
+ * @ctx: Submit context.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int
+panthor_submit_ctx_collect_jobs_signal_ops(struct panthor_submit_ctx *ctx)
+{
+	for (u32 i = 0; i < ctx->job_count; i++) {
+		int ret;
+
+		ret = panthor_submit_ctx_collect_job_signal_ops(ctx, i);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_submit_ctx_add_deps_and_arm_jobs() - Add jobs dependencies and arm jobs
+ * @ctx: Submit context.
+ *
+ * Must be called after the resv preparation has been taken care of.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int
+panthor_submit_ctx_add_deps_and_arm_jobs(struct panthor_submit_ctx *ctx)
+{
+	for (u32 i = 0; i < ctx->job_count; i++) {
+		int ret;
+
+		ret = panthor_submit_ctx_add_sync_deps_to_job(ctx, i);
+		if (ret)
+			return ret;
+
+		drm_sched_job_arm(ctx->jobs[i].job);
+
+		ret = panthor_submit_ctx_update_job_sync_signal_fences(ctx, i);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_submit_ctx_push_jobs() - Push jobs to their scheduling entities.
+ * @ctx: Submit context.
+ * @upd_resvs: Callback used to update reservation objects that were previously
+ * preapred.
+ */
+static void
+panthor_submit_ctx_push_jobs(struct panthor_submit_ctx *ctx,
+			     void (*upd_resvs)(struct drm_exec *, struct drm_sched_job *))
+{
+	for (u32 i = 0; i < ctx->job_count; i++) {
+		upd_resvs(&ctx->exec, ctx->jobs[i].job);
+		drm_sched_entity_push_job(ctx->jobs[i].job);
+
+		/* Job is owned by the scheduler now. */
+		ctx->jobs[i].job = NULL;
+	}
+
+	panthor_submit_ctx_push_fences(ctx);
+}
+
+/**
+ * panthor_submit_ctx_init() - Initializes a submission context
+ * @ctx: Submit context to initialize.
+ * @file: drm_file this submission happens on.
+ * @job_count: Number of jobs that will be submitted.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int panthor_submit_ctx_init(struct panthor_submit_ctx *ctx,
+				   struct drm_file *file, u32 job_count)
+{
+	ctx->jobs = kvmalloc_array(job_count, sizeof(*ctx->jobs),
+				   GFP_KERNEL | __GFP_ZERO);
+	if (!ctx->jobs)
+		return -ENOMEM;
+
+	ctx->file = file;
+	ctx->job_count = job_count;
+	INIT_LIST_HEAD(&ctx->signals);
+	drm_exec_init(&ctx->exec,
+		      DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES,
+		      0);
+	return 0;
+}
+
+/**
+ * panthor_submit_ctx_cleanup() - Cleanup a submission context
+ * @ctx: Submit context to cleanup.
+ * @job_put: Job put callback.
+ */
+static void panthor_submit_ctx_cleanup(struct panthor_submit_ctx *ctx,
+				       void (*job_put)(struct drm_sched_job *))
+{
+	struct panthor_sync_signal *sig_sync, *tmp;
+	unsigned long i;
+
+	drm_exec_fini(&ctx->exec);
+
+	list_for_each_entry_safe(sig_sync, tmp, &ctx->signals, node)
+		panthor_sync_signal_free(sig_sync);
+
+	for (i = 0; i < ctx->job_count; i++) {
+		job_put(ctx->jobs[i].job);
+		kvfree(ctx->jobs[i].syncops);
+	}
+
+	kvfree(ctx->jobs);
+}
+
+static int panthor_ioctl_dev_query(struct drm_device *ddev, void *data, struct drm_file *file)
+{
+	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
+	struct drm_panthor_dev_query *args = data;
+
+	if (!args->pointer) {
+		switch (args->type) {
+		case DRM_PANTHOR_DEV_QUERY_GPU_INFO:
+			args->size = sizeof(ptdev->gpu_info);
+			return 0;
+
+		case DRM_PANTHOR_DEV_QUERY_CSIF_INFO:
+			args->size = sizeof(ptdev->csif_info);
+			return 0;
+
+		default:
+			return -EINVAL;
+		}
+	}
+
+	switch (args->type) {
+	case DRM_PANTHOR_DEV_QUERY_GPU_INFO:
+		return PANTHOR_UOBJ_SET(args->pointer, args->size, ptdev->gpu_info);
+
+	case DRM_PANTHOR_DEV_QUERY_CSIF_INFO:
+		return PANTHOR_UOBJ_SET(args->pointer, args->size, ptdev->csif_info);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+#define PANTHOR_VM_CREATE_FLAGS			0
+
+static int panthor_ioctl_vm_create(struct drm_device *ddev, void *data,
+				   struct drm_file *file)
+{
+	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
+	struct panthor_file *pfile = file->driver_priv;
+	struct drm_panthor_vm_create *args = data;
+	int cookie, ret;
+
+	if (!drm_dev_enter(ddev, &cookie))
+		return -ENODEV;
+
+	ret = panthor_vm_pool_create_vm(ptdev, pfile->vms,  args);
+	if (ret >= 0) {
+		args->id = ret;
+		ret = 0;
+	}
+
+	drm_dev_exit(cookie);
+	return ret;
+}
+
+static int panthor_ioctl_vm_destroy(struct drm_device *ddev, void *data,
+				    struct drm_file *file)
+{
+	struct panthor_file *pfile = file->driver_priv;
+	struct drm_panthor_vm_destroy *args = data;
+
+	if (args->pad)
+		return -EINVAL;
+
+	return panthor_vm_pool_destroy_vm(pfile->vms, args->id);
+}
+
+#define PANTHOR_BO_FLAGS		DRM_PANTHOR_BO_NO_MMAP
+
+static int panthor_ioctl_bo_create(struct drm_device *ddev, void *data,
+				   struct drm_file *file)
+{
+	struct panthor_file *pfile = file->driver_priv;
+	struct drm_panthor_bo_create *args = data;
+	struct panthor_vm *vm = NULL;
+	int cookie, ret;
+
+	if (!drm_dev_enter(ddev, &cookie))
+		return -ENODEV;
+
+	if (!args->size || args->pad ||
+	    (args->flags & ~PANTHOR_BO_FLAGS)) {
+		ret = -EINVAL;
+		goto out_dev_exit;
+	}
+
+	if (args->exclusive_vm_id) {
+		vm = panthor_vm_pool_get_vm(pfile->vms, args->exclusive_vm_id);
+		if (!vm) {
+			ret = -EINVAL;
+			goto out_dev_exit;
+		}
+	}
+
+	ret = panthor_gem_create_with_handle(file, ddev, vm, &args->size,
+					     args->flags, &args->handle);
+
+	panthor_vm_put(vm);
+
+out_dev_exit:
+	drm_dev_exit(cookie);
+	return ret;
+}
+
+static int panthor_ioctl_bo_mmap_offset(struct drm_device *ddev, void *data,
+					struct drm_file *file)
+{
+	struct drm_panthor_bo_mmap_offset *args = data;
+	struct drm_gem_object *obj;
+	int ret;
+
+	if (args->pad)
+		return -EINVAL;
+
+	obj = drm_gem_object_lookup(file, args->handle);
+	if (!obj)
+		return -ENOENT;
+
+	ret = drm_gem_create_mmap_offset(obj);
+	if (ret)
+		goto out;
+
+	args->offset = drm_vma_node_offset_addr(&obj->vma_node);
+
+out:
+	drm_gem_object_put(obj);
+	return ret;
+}
+
+static int panthor_ioctl_group_submit(struct drm_device *ddev, void *data,
+				      struct drm_file *file)
+{
+	struct panthor_file *pfile = file->driver_priv;
+	struct drm_panthor_group_submit *args = data;
+	struct drm_panthor_queue_submit *jobs_args;
+	struct panthor_submit_ctx ctx;
+	int ret = 0, cookie;
+
+	if (args->pad)
+		return -EINVAL;
+
+	if (!drm_dev_enter(ddev, &cookie))
+		return -ENODEV;
+
+	ret = PANTHOR_UOBJ_GET_ARRAY(jobs_args, &args->queue_submits);
+	if (ret)
+		goto out_dev_exit;
+
+	ret = panthor_submit_ctx_init(&ctx, file, args->queue_submits.count);
+	if (ret)
+		goto out_free_jobs_args;
+
+	/* Create jobs and attach sync operations */
+	for (u32 i = 0; i < args->queue_submits.count; i++) {
+		const struct drm_panthor_queue_submit *qsubmit = &jobs_args[i];
+		struct drm_sched_job *job;
+
+		job = panthor_job_create(pfile, args->group_handle, qsubmit);
+		if (IS_ERR(job)) {
+			ret = PTR_ERR(job);
+			goto out_cleanup_submit_ctx;
+		}
+
+		ret = panthor_submit_ctx_add_job(&ctx, i, job, &qsubmit->syncs);
+		if (ret)
+			goto out_cleanup_submit_ctx;
+	}
+
+	/*
+	 * Collect signal operations on all jobs, such that each job can pick
+	 * from it for its dependencies and update the fence to signal when the
+	 * job is submitted.
+	 */
+	ret = panthor_submit_ctx_collect_jobs_signal_ops(&ctx);
+	if (ret)
+		goto out_cleanup_submit_ctx;
+
+	/*
+	 * We acquire/prepare revs on all jobs before proceeding with the
+	 * dependency registration.
+	 *
+	 * This is solving two problems:
+	 * 1. drm_sched_job_arm() and drm_sched_entity_push_job() must be
+	 *    protected by a lock to make sure no concurrent access to the same
+	 *    entity get interleaved, which would mess up with the fence seqno
+	 *    ordering. Luckily, one of the resv being acquired is the VM resv,
+	 *    and a scheduling entity is only bound to a single VM. As soon as
+	 *    we acquire the VM resv, we should be safe.
+	 * 2. Jobs might depend on fences that were issued by previous jobs in
+	 *    the same batch, so we can't add dependencies on all jobs before
+	 *    arming previous jobs and registering the fence to the signal
+	 *    array, otherwise we might miss dependencies, or point to an
+	 *    outdated fence.
+	 */
+	if (args->queue_submits.count > 0) {
+		/* All jobs target the same group, so they also point to the same VM. */
+		struct panthor_vm *vm = panthor_job_vm(ctx.jobs[0].job);
+
+		drm_exec_until_all_locked(&ctx.exec) {
+			ret = panthor_vm_prepare_mapped_bos_resvs(&ctx.exec, vm,
+								  args->queue_submits.count);
+		}
+
+		if (ret)
+			goto out_cleanup_submit_ctx;
+	}
+
+	/*
+	 * Now that resvs are locked/prepared, we can iterate over each job to
+	 * add the dependencies, arm the job fence, register the job fence to
+	 * the signal array.
+	 */
+	ret = panthor_submit_ctx_add_deps_and_arm_jobs(&ctx);
+	if (ret)
+		goto out_cleanup_submit_ctx;
+
+	/* Nothing can fail after that point, so we can make our job fences
+	 * visible to the outside world. Push jobs and set the job fences to
+	 * the resv slots we reserved.  This also pushes the fences to the
+	 * syncobjs that are part of the signal array.
+	 */
+	panthor_submit_ctx_push_jobs(&ctx, panthor_job_update_resvs);
+
+out_cleanup_submit_ctx:
+	panthor_submit_ctx_cleanup(&ctx, panthor_job_put);
+
+out_free_jobs_args:
+	kvfree(jobs_args);
+
+out_dev_exit:
+	drm_dev_exit(cookie);
+	return ret;
+}
+
+static int panthor_ioctl_group_destroy(struct drm_device *ddev, void *data,
+				       struct drm_file *file)
+{
+	struct panthor_file *pfile = file->driver_priv;
+	struct drm_panthor_group_destroy *args = data;
+
+	if (args->pad)
+		return -EINVAL;
+
+	return panthor_group_destroy(pfile, args->group_handle);
+}
+
+static int panthor_ioctl_group_create(struct drm_device *ddev, void *data,
+				      struct drm_file *file)
+{
+	struct panthor_file *pfile = file->driver_priv;
+	struct drm_panthor_group_create *args = data;
+	struct drm_panthor_queue_create *queue_args;
+	int ret;
+
+	if (!args->queues.count)
+		return -EINVAL;
+
+	ret = PANTHOR_UOBJ_GET_ARRAY(queue_args, &args->queues);
+	if (ret)
+		return ret;
+
+	ret = panthor_group_create(pfile, args, queue_args);
+	if (ret >= 0) {
+		args->group_handle = ret;
+		ret = 0;
+	}
+
+	kvfree(queue_args);
+	return ret;
+}
+
+static int panthor_ioctl_group_get_state(struct drm_device *ddev, void *data,
+					 struct drm_file *file)
+{
+	struct panthor_file *pfile = file->driver_priv;
+	struct drm_panthor_group_get_state *args = data;
+
+	return panthor_group_get_state(pfile, args);
+}
+
+static int panthor_ioctl_tiler_heap_create(struct drm_device *ddev, void *data,
+					   struct drm_file *file)
+{
+	struct panthor_file *pfile = file->driver_priv;
+	struct drm_panthor_tiler_heap_create *args = data;
+	struct panthor_heap_pool *pool;
+	struct panthor_vm *vm;
+	int ret;
+
+	vm = panthor_vm_pool_get_vm(pfile->vms, args->vm_id);
+	if (!vm)
+		return -EINVAL;
+
+	pool = panthor_vm_get_heap_pool(vm, true);
+	if (IS_ERR(pool)) {
+		ret = PTR_ERR(pool);
+		goto out_put_vm;
+	}
+
+	ret = panthor_heap_create(pool,
+				  args->initial_chunk_count,
+				  args->chunk_size,
+				  args->max_chunks,
+				  args->target_in_flight,
+				  &args->tiler_heap_ctx_gpu_va,
+				  &args->first_heap_chunk_gpu_va);
+	if (ret < 0)
+		goto out_put_heap_pool;
+
+	/* Heap pools are per-VM. We combine the VM and HEAP id to make
+	 * a unique heap handle.
+	 */
+	args->handle = (args->vm_id << 16) | ret;
+	ret = 0;
+
+out_put_heap_pool:
+	panthor_heap_pool_put(pool);
+
+out_put_vm:
+	panthor_vm_put(vm);
+	return ret;
+}
+
+static int panthor_ioctl_tiler_heap_destroy(struct drm_device *ddev, void *data,
+					    struct drm_file *file)
+{
+	struct panthor_file *pfile = file->driver_priv;
+	struct drm_panthor_tiler_heap_destroy *args = data;
+	struct panthor_heap_pool *pool;
+	struct panthor_vm *vm;
+	int ret;
+
+	if (args->pad)
+		return -EINVAL;
+
+	vm = panthor_vm_pool_get_vm(pfile->vms, args->handle >> 16);
+	if (!vm)
+		return -EINVAL;
+
+	pool = panthor_vm_get_heap_pool(vm, false);
+	if (IS_ERR(pool)) {
+		ret = PTR_ERR(pool);
+		goto out_put_vm;
+	}
+
+	ret = panthor_heap_destroy(pool, args->handle & GENMASK(15, 0));
+	panthor_heap_pool_put(pool);
+
+out_put_vm:
+	panthor_vm_put(vm);
+	return ret;
+}
+
+static int panthor_ioctl_vm_bind_async(struct drm_device *ddev,
+				       struct drm_panthor_vm_bind *args,
+				       struct drm_file *file)
+{
+	struct panthor_file *pfile = file->driver_priv;
+	struct drm_panthor_vm_bind_op *jobs_args;
+	struct panthor_submit_ctx ctx;
+	struct panthor_vm *vm;
+	int ret = 0;
+
+	vm = panthor_vm_pool_get_vm(pfile->vms, args->vm_id);
+	if (!vm)
+		return -EINVAL;
+
+	ret = PANTHOR_UOBJ_GET_ARRAY(jobs_args, &args->ops);
+	if (ret)
+		goto out_put_vm;
+
+	ret = panthor_submit_ctx_init(&ctx, file, args->ops.count);
+	if (ret)
+		goto out_free_jobs_args;
+
+	for (u32 i = 0; i < args->ops.count; i++) {
+		struct drm_panthor_vm_bind_op *op = &jobs_args[i];
+		struct drm_sched_job *job;
+
+		job = panthor_vm_bind_job_create(file, vm, op);
+		if (IS_ERR(job)) {
+			ret = PTR_ERR(job);
+			goto out_cleanup_submit_ctx;
+		}
+
+		ret = panthor_submit_ctx_add_job(&ctx, i, job, &op->syncs);
+		if (ret)
+			goto out_cleanup_submit_ctx;
+	}
+
+	ret = panthor_submit_ctx_collect_jobs_signal_ops(&ctx);
+	if (ret)
+		goto out_cleanup_submit_ctx;
+
+	/* Prepare reservation objects for each VM_BIND job. */
+	drm_exec_until_all_locked(&ctx.exec) {
+		for (u32 i = 0; i < ctx.job_count; i++) {
+			ret = panthor_vm_bind_job_prepare_resvs(&ctx.exec, ctx.jobs[i].job);
+			drm_exec_retry_on_contention(&ctx.exec);
+			if (ret)
+				goto out_cleanup_submit_ctx;
+		}
+	}
+
+	ret = panthor_submit_ctx_add_deps_and_arm_jobs(&ctx);
+	if (ret)
+		goto out_cleanup_submit_ctx;
+
+	/* Nothing can fail after that point. */
+	panthor_submit_ctx_push_jobs(&ctx, panthor_vm_bind_job_update_resvs);
+
+out_cleanup_submit_ctx:
+	panthor_submit_ctx_cleanup(&ctx, panthor_vm_bind_job_put);
+
+out_free_jobs_args:
+	kvfree(jobs_args);
+
+out_put_vm:
+	panthor_vm_put(vm);
+	return ret;
+}
+
+static int panthor_ioctl_vm_bind_sync(struct drm_device *ddev,
+				      struct drm_panthor_vm_bind *args,
+				      struct drm_file *file)
+{
+	struct panthor_file *pfile = file->driver_priv;
+	struct drm_panthor_vm_bind_op *jobs_args;
+	struct panthor_vm *vm;
+	int ret;
+
+	vm = panthor_vm_pool_get_vm(pfile->vms, args->vm_id);
+	if (!vm)
+		return -EINVAL;
+
+	ret = PANTHOR_UOBJ_GET_ARRAY(jobs_args, &args->ops);
+	if (ret)
+		goto out_put_vm;
+
+	for (u32 i = 0; i < args->ops.count; i++) {
+		ret = panthor_vm_bind_exec_sync_op(file, vm, &jobs_args[i]);
+		if (ret) {
+			/* Update ops.count so the user knows where things failed. */
+			args->ops.count = i;
+			break;
+		}
+	}
+
+	kvfree(jobs_args);
+
+out_put_vm:
+	panthor_vm_put(vm);
+	return ret;
+}
+
+#define PANTHOR_VM_BIND_FLAGS DRM_PANTHOR_VM_BIND_ASYNC
+
+static int panthor_ioctl_vm_bind(struct drm_device *ddev, void *data,
+				 struct drm_file *file)
+{
+	struct drm_panthor_vm_bind *args = data;
+	int cookie, ret;
+
+	if (!drm_dev_enter(ddev, &cookie))
+		return -ENODEV;
+
+	if (args->flags & DRM_PANTHOR_VM_BIND_ASYNC)
+		ret = panthor_ioctl_vm_bind_async(ddev, args, file);
+	else
+		ret = panthor_ioctl_vm_bind_sync(ddev, args, file);
+
+	drm_dev_exit(cookie);
+	return ret;
+}
+
+static int panthor_ioctl_vm_get_state(struct drm_device *ddev, void *data,
+				      struct drm_file *file)
+{
+	struct panthor_file *pfile = file->driver_priv;
+	struct drm_panthor_vm_get_state *args = data;
+	struct panthor_vm *vm;
+
+	vm = panthor_vm_pool_get_vm(pfile->vms, args->vm_id);
+	if (!vm)
+		return -EINVAL;
+
+	if (panthor_vm_is_unusable(vm))
+		args->state = DRM_PANTHOR_VM_STATE_UNUSABLE;
+	else
+		args->state = DRM_PANTHOR_VM_STATE_USABLE;
+
+	panthor_vm_put(vm);
+	return 0;
+}
+
+static int
+panthor_open(struct drm_device *ddev, struct drm_file *file)
+{
+	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
+	struct panthor_file *pfile;
+	int ret;
+
+	if (!try_module_get(THIS_MODULE))
+		return -EINVAL;
+
+	pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
+	if (!pfile) {
+		ret = -ENOMEM;
+		goto err_put_mod;
+	}
+
+	pfile->ptdev = ptdev;
+
+	ret = panthor_vm_pool_create(pfile);
+	if (ret)
+		goto err_free_file;
+
+	ret = panthor_group_pool_create(pfile);
+	if (ret)
+		goto err_destroy_vm_pool;
+
+	file->driver_priv = pfile;
+	return 0;
+
+err_destroy_vm_pool:
+	panthor_vm_pool_destroy(pfile);
+
+err_free_file:
+	kfree(pfile);
+
+err_put_mod:
+	module_put(THIS_MODULE);
+	return ret;
+}
+
+static void
+panthor_postclose(struct drm_device *ddev, struct drm_file *file)
+{
+	struct panthor_file *pfile = file->driver_priv;
+
+	panthor_group_pool_destroy(pfile);
+	panthor_vm_pool_destroy(pfile);
+
+	kfree(pfile);
+	module_put(THIS_MODULE);
+}
+
+static const struct drm_ioctl_desc panthor_drm_driver_ioctls[] = {
+#define PANTHOR_IOCTL(n, func, flags) \
+	DRM_IOCTL_DEF_DRV(PANTHOR_##n, panthor_ioctl_##func, flags)
+
+	PANTHOR_IOCTL(DEV_QUERY, dev_query, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(VM_CREATE, vm_create, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(VM_DESTROY, vm_destroy, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(VM_BIND, vm_bind, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(VM_GET_STATE, vm_get_state, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(BO_CREATE, bo_create, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(BO_MMAP_OFFSET, bo_mmap_offset, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(GROUP_CREATE, group_create, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(GROUP_DESTROY, group_destroy, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(GROUP_GET_STATE, group_get_state, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(TILER_HEAP_CREATE, tiler_heap_create, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(TILER_HEAP_DESTROY, tiler_heap_destroy, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(GROUP_SUBMIT, group_submit, DRM_RENDER_ALLOW),
+};
+
+static int panthor_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct drm_file *file = filp->private_data;
+	struct panthor_file *pfile = file->driver_priv;
+	struct panthor_device *ptdev = pfile->ptdev;
+	u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
+	int ret, cookie;
+
+	if (!drm_dev_enter(file->minor->dev, &cookie))
+		return -ENODEV;
+
+#ifdef CONFIG_ARM64
+	/*
+	 * With 32-bit systems being limited by the 32-bit representation of
+	 * mmap2's pgoffset field, we need to make the MMIO offset arch
+	 * specific. This converts a user MMIO offset into something the kernel
+	 * driver understands.
+	 */
+	if (test_tsk_thread_flag(current, TIF_32BIT) &&
+	    offset >= DRM_PANTHOR_USER_MMIO_OFFSET_32BIT) {
+		offset += DRM_PANTHOR_USER_MMIO_OFFSET_64BIT -
+			  DRM_PANTHOR_USER_MMIO_OFFSET_32BIT;
+		vma->vm_pgoff = offset >> PAGE_SHIFT;
+	}
+#endif
+
+	if (offset >= DRM_PANTHOR_USER_MMIO_OFFSET)
+		ret = panthor_device_mmap_io(ptdev, vma);
+	else
+		ret = drm_gem_mmap(filp, vma);
+
+	drm_dev_exit(cookie);
+	return ret;
+}
+
+static const struct file_operations panthor_drm_driver_fops = {
+	.open = drm_open,
+	.release = drm_release,
+	.unlocked_ioctl = drm_ioctl,
+	.compat_ioctl = drm_compat_ioctl,
+	.poll = drm_poll,
+	.read = drm_read,
+	.llseek = noop_llseek,
+	.mmap = panthor_mmap,
+};
+
+#ifdef CONFIG_DEBUG_FS
+static void panthor_debugfs_init(struct drm_minor *minor)
+{
+	panthor_mmu_debugfs_init(minor);
+}
+#endif
+
+/*
+ * PanCSF driver version:
+ * - 1.0 - initial interface
+ */
+static const struct drm_driver panthor_drm_driver = {
+	.driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ |
+			   DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA,
+	.open = panthor_open,
+	.postclose = panthor_postclose,
+	.ioctls = panthor_drm_driver_ioctls,
+	.num_ioctls = ARRAY_SIZE(panthor_drm_driver_ioctls),
+	.fops = &panthor_drm_driver_fops,
+	.name = "panthor",
+	.desc = "Panthor DRM driver",
+	.date = "20230801",
+	.major = 1,
+	.minor = 0,
+
+	.gem_create_object = panthor_gem_create_object,
+	.gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table,
+#ifdef CONFIG_DEBUG_FS
+	.debugfs_init = panthor_debugfs_init,
+#endif
+};
+
+static int panthor_probe(struct platform_device *pdev)
+{
+	struct panthor_device *ptdev;
+
+	ptdev = devm_drm_dev_alloc(&pdev->dev, &panthor_drm_driver,
+				   struct panthor_device, base);
+	if (IS_ERR(ptdev))
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ptdev);
+
+	return panthor_device_init(ptdev);
+}
+
+static void panthor_remove(struct platform_device *pdev)
+{
+	struct panthor_device *ptdev = platform_get_drvdata(pdev);
+
+	panthor_device_unplug(ptdev);
+}
+
+static const struct of_device_id dt_match[] = {
+	{ .compatible = "rockchip,rk3588-mali" },
+	{ .compatible = "arm,mali-valhall-csf" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, dt_match);
+
+static DEFINE_RUNTIME_DEV_PM_OPS(panthor_pm_ops,
+				 panthor_device_suspend,
+				 panthor_device_resume,
+				 NULL);
+
+static struct platform_driver panthor_driver = {
+	.probe = panthor_probe,
+	.remove_new = panthor_remove,
+	.driver = {
+		.name = "panthor",
+		.pm = pm_ptr(&panthor_pm_ops),
+		.of_match_table = dt_match,
+	},
+};
+
+/*
+ * Workqueue used to cleanup stuff.
+ *
+ * We create a dedicated workqueue so we can drain on unplug and
+ * make sure all resources are freed before the module is unloaded.
+ */
+struct workqueue_struct *panthor_cleanup_wq;
+
+static int __init panthor_init(void)
+{
+	int ret;
+
+	ret = panthor_mmu_pt_cache_init();
+	if (ret)
+		return ret;
+
+	panthor_cleanup_wq = alloc_workqueue("panthor-cleanup", WQ_UNBOUND, 0);
+	if (!panthor_cleanup_wq) {
+		pr_err("panthor: Failed to allocate the workqueues");
+		ret = -ENOMEM;
+		goto err_mmu_pt_cache_fini;
+	}
+
+	ret = platform_driver_register(&panthor_driver);
+	if (ret)
+		goto err_destroy_cleanup_wq;
+
+	return 0;
+
+err_destroy_cleanup_wq:
+	destroy_workqueue(panthor_cleanup_wq);
+
+err_mmu_pt_cache_fini:
+	panthor_mmu_pt_cache_fini();
+	return ret;
+}
+module_init(panthor_init);
+
+static void __exit panthor_exit(void)
+{
+	platform_driver_unregister(&panthor_driver);
+	destroy_workqueue(panthor_cleanup_wq);
+	panthor_mmu_pt_cache_fini();
+}
+module_exit(panthor_exit);
+
+MODULE_AUTHOR("Panthor Project Developers");
+MODULE_DESCRIPTION("Panthor DRM Driver");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c
new file mode 100644
index 000000000000..857f3f11258a
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_fw.c
@@ -0,0 +1,1364 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/* Copyright 2023 Collabora ltd. */
+
+#ifdef CONFIG_ARM_ARCH_TIMER
+#include <asm/arch_timer.h>
+#endif
+
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/firmware.h>
+#include <linux/iopoll.h>
+#include <linux/iosys-map.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+
+#include <drm/drm_drv.h>
+#include <drm/drm_managed.h>
+
+#include "panthor_device.h"
+#include "panthor_fw.h"
+#include "panthor_gem.h"
+#include "panthor_gpu.h"
+#include "panthor_mmu.h"
+#include "panthor_regs.h"
+#include "panthor_sched.h"
+
+#define CSF_FW_NAME "mali_csffw.bin"
+
+#define PING_INTERVAL_MS			12000
+#define PROGRESS_TIMEOUT_CYCLES			(5ull * 500 * 1024 * 1024)
+#define PROGRESS_TIMEOUT_SCALE_SHIFT		10
+#define IDLE_HYSTERESIS_US			800
+#define PWROFF_HYSTERESIS_US			10000
+
+/**
+ * struct panthor_fw_binary_hdr - Firmware binary header.
+ */
+struct panthor_fw_binary_hdr {
+	/** @magic: Magic value to check binary validity. */
+	u32 magic;
+#define CSF_FW_BINARY_HEADER_MAGIC		0xc3f13a6e
+
+	/** @minor: Minor FW version. */
+	u8 minor;
+
+	/** @major: Major FW version. */
+	u8 major;
+#define CSF_FW_BINARY_HEADER_MAJOR_MAX		0
+
+	/** @padding1: MBZ. */
+	u16 padding1;
+
+	/** @version_hash: FW version hash. */
+	u32 version_hash;
+
+	/** @padding2: MBZ. */
+	u32 padding2;
+
+	/** @size: FW binary size. */
+	u32 size;
+};
+
+/**
+ * enum panthor_fw_binary_entry_type - Firmware binary entry type
+ */
+enum panthor_fw_binary_entry_type {
+	/** @CSF_FW_BINARY_ENTRY_TYPE_IFACE: Host <-> FW interface. */
+	CSF_FW_BINARY_ENTRY_TYPE_IFACE = 0,
+
+	/** @CSF_FW_BINARY_ENTRY_TYPE_CONFIG: FW config. */
+	CSF_FW_BINARY_ENTRY_TYPE_CONFIG = 1,
+
+	/** @CSF_FW_BINARY_ENTRY_TYPE_FUTF_TEST: Unit-tests. */
+	CSF_FW_BINARY_ENTRY_TYPE_FUTF_TEST = 2,
+
+	/** @CSF_FW_BINARY_ENTRY_TYPE_TRACE_BUFFER: Trace buffer interface. */
+	CSF_FW_BINARY_ENTRY_TYPE_TRACE_BUFFER = 3,
+
+	/** @CSF_FW_BINARY_ENTRY_TYPE_TIMELINE_METADATA: Timeline metadata interface. */
+	CSF_FW_BINARY_ENTRY_TYPE_TIMELINE_METADATA = 4,
+};
+
+#define CSF_FW_BINARY_ENTRY_TYPE(ehdr)					((ehdr) & 0xff)
+#define CSF_FW_BINARY_ENTRY_SIZE(ehdr)					(((ehdr) >> 8) & 0xff)
+#define CSF_FW_BINARY_ENTRY_UPDATE					BIT(30)
+#define CSF_FW_BINARY_ENTRY_OPTIONAL					BIT(31)
+
+#define CSF_FW_BINARY_IFACE_ENTRY_RD_RD					BIT(0)
+#define CSF_FW_BINARY_IFACE_ENTRY_RD_WR					BIT(1)
+#define CSF_FW_BINARY_IFACE_ENTRY_RD_EX					BIT(2)
+#define CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_NONE			(0 << 3)
+#define CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_CACHED			(1 << 3)
+#define CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_UNCACHED_COHERENT	(2 << 3)
+#define CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_CACHED_COHERENT		(3 << 3)
+#define CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_MASK			GENMASK(4, 3)
+#define CSF_FW_BINARY_IFACE_ENTRY_RD_PROT				BIT(5)
+#define CSF_FW_BINARY_IFACE_ENTRY_RD_SHARED				BIT(30)
+#define CSF_FW_BINARY_IFACE_ENTRY_RD_ZERO				BIT(31)
+
+#define CSF_FW_BINARY_IFACE_ENTRY_RD_SUPPORTED_FLAGS			\
+	(CSF_FW_BINARY_IFACE_ENTRY_RD_RD |				\
+	 CSF_FW_BINARY_IFACE_ENTRY_RD_WR |				\
+	 CSF_FW_BINARY_IFACE_ENTRY_RD_EX |				\
+	 CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_MASK |			\
+	 CSF_FW_BINARY_IFACE_ENTRY_RD_PROT |				\
+	 CSF_FW_BINARY_IFACE_ENTRY_RD_SHARED  |				\
+	 CSF_FW_BINARY_IFACE_ENTRY_RD_ZERO)
+
+/**
+ * struct panthor_fw_binary_section_entry_hdr - Describes a section of FW binary
+ */
+struct panthor_fw_binary_section_entry_hdr {
+	/** @flags: Section flags. */
+	u32 flags;
+
+	/** @va: MCU virtual range to map this binary section to. */
+	struct {
+		/** @start: Start address. */
+		u32 start;
+
+		/** @end: End address. */
+		u32 end;
+	} va;
+
+	/** @data: Data to initialize the FW section with. */
+	struct {
+		/** @start: Start offset in the FW binary. */
+		u32 start;
+
+		/** @end: End offset in the FW binary. */
+		u32 end;
+	} data;
+};
+
+/**
+ * struct panthor_fw_binary_iter - Firmware binary iterator
+ *
+ * Used to parse a firmware binary.
+ */
+struct panthor_fw_binary_iter {
+	/** @data: FW binary data. */
+	const void *data;
+
+	/** @size: FW binary size. */
+	size_t size;
+
+	/** @offset: Iterator offset. */
+	size_t offset;
+};
+
+/**
+ * struct panthor_fw_section - FW section
+ */
+struct panthor_fw_section {
+	/** @node: Used to keep track of FW sections. */
+	struct list_head node;
+
+	/** @flags: Section flags, as encoded in the FW binary. */
+	u32 flags;
+
+	/** @mem: Section memory. */
+	struct panthor_kernel_bo *mem;
+
+	/**
+	 * @name: Name of the section, as specified in the binary.
+	 *
+	 * Can be NULL.
+	 */
+	const char *name;
+
+	/**
+	 * @data: Initial data copied to the FW memory.
+	 *
+	 * We keep data around so we can reload sections after a reset.
+	 */
+	struct {
+		/** @buf: Buffed used to store init data. */
+		const void *buf;
+
+		/** @size: Size of @buf in bytes. */
+		size_t size;
+	} data;
+};
+
+#define CSF_MCU_SHARED_REGION_START		0x04000000ULL
+#define CSF_MCU_SHARED_REGION_SIZE		0x04000000ULL
+
+#define MIN_CS_PER_CSG				8
+#define MIN_CSGS				3
+#define MAX_CSG_PRIO				0xf
+
+#define CSF_IFACE_VERSION(major, minor, patch)	\
+	(((major) << 24) | ((minor) << 16) | (patch))
+#define CSF_IFACE_VERSION_MAJOR(v)		((v) >> 24)
+#define CSF_IFACE_VERSION_MINOR(v)		(((v) >> 16) & 0xff)
+#define CSF_IFACE_VERSION_PATCH(v)		((v) & 0xffff)
+
+#define CSF_GROUP_CONTROL_OFFSET		0x1000
+#define CSF_STREAM_CONTROL_OFFSET		0x40
+#define CSF_UNPRESERVED_REG_COUNT		4
+
+/**
+ * struct panthor_fw_iface - FW interfaces
+ */
+struct panthor_fw_iface {
+	/** @global: Global interface. */
+	struct panthor_fw_global_iface global;
+
+	/** @groups: Group slot interfaces. */
+	struct panthor_fw_csg_iface groups[MAX_CSGS];
+
+	/** @streams: Command stream slot interfaces. */
+	struct panthor_fw_cs_iface streams[MAX_CSGS][MAX_CS_PER_CSG];
+};
+
+/**
+ * struct panthor_fw - Firmware management
+ */
+struct panthor_fw {
+	/** @vm: MCU VM. */
+	struct panthor_vm *vm;
+
+	/** @sections: List of FW sections. */
+	struct list_head sections;
+
+	/** @shared_section: The section containing the FW interfaces. */
+	struct panthor_fw_section *shared_section;
+
+	/** @iface: FW interfaces. */
+	struct panthor_fw_iface iface;
+
+	/** @watchdog: Collection of fields relating to the FW watchdog. */
+	struct {
+		/** @ping_work: Delayed work used to ping the FW. */
+		struct delayed_work ping_work;
+	} watchdog;
+
+	/**
+	 * @req_waitqueue: FW request waitqueue.
+	 *
+	 * Everytime a request is sent to a command stream group or the global
+	 * interface, the caller will first busy wait for the request to be
+	 * acknowledged, and then fallback to a sleeping wait.
+	 *
+	 * This wait queue is here to support the sleeping wait flavor.
+	 */
+	wait_queue_head_t req_waitqueue;
+
+	/** @booted: True is the FW is booted */
+	bool booted;
+
+	/**
+	 * @fast_reset: True if the post_reset logic can proceed with a fast reset.
+	 *
+	 * A fast reset is just a reset where the driver doesn't reload the FW sections.
+	 *
+	 * Any time the firmware is properly suspended, a fast reset can take place.
+	 * On the other hand, if the halt operation failed, the driver will reload
+	 * all sections to make sure we start from a fresh state.
+	 */
+	bool fast_reset;
+
+	/** @irq: Job irq data. */
+	struct panthor_irq irq;
+};
+
+struct panthor_vm *panthor_fw_vm(struct panthor_device *ptdev)
+{
+	return ptdev->fw->vm;
+}
+
+/**
+ * panthor_fw_get_glb_iface() - Get the global interface
+ * @ptdev: Device.
+ *
+ * Return: The global interface.
+ */
+struct panthor_fw_global_iface *
+panthor_fw_get_glb_iface(struct panthor_device *ptdev)
+{
+	return &ptdev->fw->iface.global;
+}
+
+/**
+ * panthor_fw_get_csg_iface() - Get a command stream group slot interface
+ * @ptdev: Device.
+ * @csg_slot: Index of the command stream group slot.
+ *
+ * Return: The command stream group slot interface.
+ */
+struct panthor_fw_csg_iface *
+panthor_fw_get_csg_iface(struct panthor_device *ptdev, u32 csg_slot)
+{
+	if (drm_WARN_ON(&ptdev->base, csg_slot >= MAX_CSGS))
+		return NULL;
+
+	return &ptdev->fw->iface.groups[csg_slot];
+}
+
+/**
+ * panthor_fw_get_cs_iface() - Get a command stream slot interface
+ * @ptdev: Device.
+ * @csg_slot: Index of the command stream group slot.
+ * @cs_slot: Index of the command stream slot.
+ *
+ * Return: The command stream slot interface.
+ */
+struct panthor_fw_cs_iface *
+panthor_fw_get_cs_iface(struct panthor_device *ptdev, u32 csg_slot, u32 cs_slot)
+{
+	if (drm_WARN_ON(&ptdev->base, csg_slot >= MAX_CSGS || cs_slot >= MAX_CS_PER_CSG))
+		return NULL;
+
+	return &ptdev->fw->iface.streams[csg_slot][cs_slot];
+}
+
+/**
+ * panthor_fw_conv_timeout() - Convert a timeout into a cycle-count
+ * @ptdev: Device.
+ * @timeout_us: Timeout expressed in micro-seconds.
+ *
+ * The FW has two timer sources: the GPU counter or arch-timer. We need
+ * to express timeouts in term of number of cycles and specify which
+ * timer source should be used.
+ *
+ * Return: A value suitable for timeout fields in the global interface.
+ */
+static u32 panthor_fw_conv_timeout(struct panthor_device *ptdev, u32 timeout_us)
+{
+	bool use_cycle_counter = false;
+	u32 timer_rate = 0;
+	u64 mod_cycles;
+
+#ifdef CONFIG_ARM_ARCH_TIMER
+	timer_rate = arch_timer_get_cntfrq();
+#endif
+
+	if (!timer_rate) {
+		use_cycle_counter = true;
+		timer_rate = clk_get_rate(ptdev->clks.core);
+	}
+
+	if (drm_WARN_ON(&ptdev->base, !timer_rate)) {
+		/* We couldn't get a valid clock rate, let's just pick the
+		 * maximum value so the FW still handles the core
+		 * power on/off requests.
+		 */
+		return GLB_TIMER_VAL(~0) |
+		       GLB_TIMER_SOURCE_GPU_COUNTER;
+	}
+
+	mod_cycles = DIV_ROUND_UP_ULL((u64)timeout_us * timer_rate,
+				      1000000ull << 10);
+	if (drm_WARN_ON(&ptdev->base, mod_cycles > GLB_TIMER_VAL(~0)))
+		mod_cycles = GLB_TIMER_VAL(~0);
+
+	return GLB_TIMER_VAL(mod_cycles) |
+	       (use_cycle_counter ? GLB_TIMER_SOURCE_GPU_COUNTER : 0);
+}
+
+static int panthor_fw_binary_iter_read(struct panthor_device *ptdev,
+				       struct panthor_fw_binary_iter *iter,
+				       void *out, size_t size)
+{
+	size_t new_offset = iter->offset + size;
+
+	if (new_offset > iter->size || new_offset < iter->offset) {
+		drm_err(&ptdev->base, "Firmware too small\n");
+		return -EINVAL;
+	}
+
+	memcpy(out, iter->data + iter->offset, size);
+	iter->offset = new_offset;
+	return 0;
+}
+
+static int panthor_fw_binary_sub_iter_init(struct panthor_device *ptdev,
+					   struct panthor_fw_binary_iter *iter,
+					   struct panthor_fw_binary_iter *sub_iter,
+					   size_t size)
+{
+	size_t new_offset = iter->offset + size;
+
+	if (new_offset > iter->size || new_offset < iter->offset) {
+		drm_err(&ptdev->base, "Firmware entry too long\n");
+		return -EINVAL;
+	}
+
+	sub_iter->offset = 0;
+	sub_iter->data = iter->data + iter->offset;
+	sub_iter->size = size;
+	iter->offset = new_offset;
+	return 0;
+}
+
+static void panthor_fw_init_section_mem(struct panthor_device *ptdev,
+					struct panthor_fw_section *section)
+{
+	bool was_mapped = !!section->mem->kmap;
+	int ret;
+
+	if (!section->data.size &&
+	    !(section->flags & CSF_FW_BINARY_IFACE_ENTRY_RD_ZERO))
+		return;
+
+	ret = panthor_kernel_bo_vmap(section->mem);
+	if (drm_WARN_ON(&ptdev->base, ret))
+		return;
+
+	memcpy(section->mem->kmap, section->data.buf, section->data.size);
+	if (section->flags & CSF_FW_BINARY_IFACE_ENTRY_RD_ZERO) {
+		memset(section->mem->kmap + section->data.size, 0,
+		       panthor_kernel_bo_size(section->mem) - section->data.size);
+	}
+
+	if (!was_mapped)
+		panthor_kernel_bo_vunmap(section->mem);
+}
+
+/**
+ * panthor_fw_alloc_queue_iface_mem() - Allocate a ring-buffer interfaces.
+ * @ptdev: Device.
+ * @input: Pointer holding the input interface on success.
+ * Should be ignored on failure.
+ * @output: Pointer holding the output interface on success.
+ * Should be ignored on failure.
+ * @input_fw_va: Pointer holding the input interface FW VA on success.
+ * Should be ignored on failure.
+ * @output_fw_va: Pointer holding the output interface FW VA on success.
+ * Should be ignored on failure.
+ *
+ * Allocates panthor_fw_ringbuf_{input,out}_iface interfaces. The input
+ * interface is at offset 0, and the output interface at offset 4096.
+ *
+ * Return: A valid pointer in case of success, an ERR_PTR() otherwise.
+ */
+struct panthor_kernel_bo *
+panthor_fw_alloc_queue_iface_mem(struct panthor_device *ptdev,
+				 struct panthor_fw_ringbuf_input_iface **input,
+				 const struct panthor_fw_ringbuf_output_iface **output,
+				 u32 *input_fw_va, u32 *output_fw_va)
+{
+	struct panthor_kernel_bo *mem;
+	int ret;
+
+	mem = panthor_kernel_bo_create(ptdev, ptdev->fw->vm, SZ_8K,
+				       DRM_PANTHOR_BO_NO_MMAP,
+				       DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |
+				       DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,
+				       PANTHOR_VM_KERNEL_AUTO_VA);
+	if (IS_ERR(mem))
+		return mem;
+
+	ret = panthor_kernel_bo_vmap(mem);
+	if (ret) {
+		panthor_kernel_bo_destroy(mem);
+		return ERR_PTR(ret);
+	}
+
+	memset(mem->kmap, 0, panthor_kernel_bo_size(mem));
+	*input = mem->kmap;
+	*output = mem->kmap + SZ_4K;
+	*input_fw_va = panthor_kernel_bo_gpuva(mem);
+	*output_fw_va = *input_fw_va + SZ_4K;
+
+	return mem;
+}
+
+/**
+ * panthor_fw_alloc_suspend_buf_mem() - Allocate a suspend buffer for a command stream group.
+ * @ptdev: Device.
+ * @size: Size of the suspend buffer.
+ *
+ * Return: A valid pointer in case of success, an ERR_PTR() otherwise.
+ */
+struct panthor_kernel_bo *
+panthor_fw_alloc_suspend_buf_mem(struct panthor_device *ptdev, size_t size)
+{
+	return panthor_kernel_bo_create(ptdev, panthor_fw_vm(ptdev), size,
+					DRM_PANTHOR_BO_NO_MMAP,
+					DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC,
+					PANTHOR_VM_KERNEL_AUTO_VA);
+}
+
+static int panthor_fw_load_section_entry(struct panthor_device *ptdev,
+					 const struct firmware *fw,
+					 struct panthor_fw_binary_iter *iter,
+					 u32 ehdr)
+{
+	struct panthor_fw_binary_section_entry_hdr hdr;
+	struct panthor_fw_section *section;
+	u32 section_size;
+	u32 name_len;
+	int ret;
+
+	ret = panthor_fw_binary_iter_read(ptdev, iter, &hdr, sizeof(hdr));
+	if (ret)
+		return ret;
+
+	if (hdr.data.end < hdr.data.start) {
+		drm_err(&ptdev->base, "Firmware corrupted, data.end < data.start (0x%x < 0x%x)\n",
+			hdr.data.end, hdr.data.start);
+		return -EINVAL;
+	}
+
+	if (hdr.va.end < hdr.va.start) {
+		drm_err(&ptdev->base, "Firmware corrupted, hdr.va.end < hdr.va.start (0x%x < 0x%x)\n",
+			hdr.va.end, hdr.va.start);
+		return -EINVAL;
+	}
+
+	if (hdr.data.end > fw->size) {
+		drm_err(&ptdev->base, "Firmware corrupted, file truncated? data_end=0x%x > fw size=0x%zx\n",
+			hdr.data.end, fw->size);
+		return -EINVAL;
+	}
+
+	if ((hdr.va.start & ~PAGE_MASK) != 0 ||
+	    (hdr.va.end & ~PAGE_MASK) != 0) {
+		drm_err(&ptdev->base, "Firmware corrupted, virtual addresses not page aligned: 0x%x-0x%x\n",
+			hdr.va.start, hdr.va.end);
+		return -EINVAL;
+	}
+
+	if (hdr.flags & ~CSF_FW_BINARY_IFACE_ENTRY_RD_SUPPORTED_FLAGS) {
+		drm_err(&ptdev->base, "Firmware contains interface with unsupported flags (0x%x)\n",
+			hdr.flags);
+		return -EINVAL;
+	}
+
+	if (hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_RD_PROT) {
+		drm_warn(&ptdev->base,
+			 "Firmware protected mode entry not be supported, ignoring");
+		return 0;
+	}
+
+	if (hdr.va.start == CSF_MCU_SHARED_REGION_START &&
+	    !(hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_RD_SHARED)) {
+		drm_err(&ptdev->base,
+			"Interface at 0x%llx must be shared", CSF_MCU_SHARED_REGION_START);
+		return -EINVAL;
+	}
+
+	name_len = iter->size - iter->offset;
+
+	section = drmm_kzalloc(&ptdev->base, sizeof(*section), GFP_KERNEL);
+	if (!section)
+		return -ENOMEM;
+
+	list_add_tail(&section->node, &ptdev->fw->sections);
+	section->flags = hdr.flags;
+	section->data.size = hdr.data.end - hdr.data.start;
+
+	if (section->data.size > 0) {
+		void *data = drmm_kmalloc(&ptdev->base, section->data.size, GFP_KERNEL);
+
+		if (!data)
+			return -ENOMEM;
+
+		memcpy(data, fw->data + hdr.data.start, section->data.size);
+		section->data.buf = data;
+	}
+
+	if (name_len > 0) {
+		char *name = drmm_kmalloc(&ptdev->base, name_len + 1, GFP_KERNEL);
+
+		if (!name)
+			return -ENOMEM;
+
+		memcpy(name, iter->data + iter->offset, name_len);
+		name[name_len] = '\0';
+		section->name = name;
+	}
+
+	section_size = hdr.va.end - hdr.va.start;
+	if (section_size) {
+		u32 cache_mode = hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_MASK;
+		struct panthor_gem_object *bo;
+		u32 vm_map_flags = 0;
+		struct sg_table *sgt;
+		u64 va = hdr.va.start;
+
+		if (!(hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_RD_WR))
+			vm_map_flags |= DRM_PANTHOR_VM_BIND_OP_MAP_READONLY;
+
+		if (!(hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_RD_EX))
+			vm_map_flags |= DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC;
+
+		/* TODO: CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_*_COHERENT are mapped to
+		 * non-cacheable for now. We might want to introduce a new
+		 * IOMMU_xxx flag (or abuse IOMMU_MMIO, which maps to device
+		 * memory and is currently not used by our driver) for
+		 * AS_MEMATTR_AARCH64_SHARED memory, so we can take benefit
+		 * of IO-coherent systems.
+		 */
+		if (cache_mode != CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_CACHED)
+			vm_map_flags |= DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED;
+
+		section->mem = panthor_kernel_bo_create(ptdev, panthor_fw_vm(ptdev),
+							section_size,
+							DRM_PANTHOR_BO_NO_MMAP,
+							vm_map_flags, va);
+		if (IS_ERR(section->mem))
+			return PTR_ERR(section->mem);
+
+		if (drm_WARN_ON(&ptdev->base, section->mem->va_node.start != hdr.va.start))
+			return -EINVAL;
+
+		if (section->flags & CSF_FW_BINARY_IFACE_ENTRY_RD_SHARED) {
+			ret = panthor_kernel_bo_vmap(section->mem);
+			if (ret)
+				return ret;
+		}
+
+		panthor_fw_init_section_mem(ptdev, section);
+
+		bo = to_panthor_bo(section->mem->obj);
+		sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
+		if (IS_ERR(sgt))
+			return PTR_ERR(sgt);
+
+		dma_sync_sgtable_for_device(ptdev->base.dev, sgt, DMA_TO_DEVICE);
+	}
+
+	if (hdr.va.start == CSF_MCU_SHARED_REGION_START)
+		ptdev->fw->shared_section = section;
+
+	return 0;
+}
+
+static void
+panthor_reload_fw_sections(struct panthor_device *ptdev, bool full_reload)
+{
+	struct panthor_fw_section *section;
+
+	list_for_each_entry(section, &ptdev->fw->sections, node) {
+		struct sg_table *sgt;
+
+		if (!full_reload && !(section->flags & CSF_FW_BINARY_IFACE_ENTRY_RD_WR))
+			continue;
+
+		panthor_fw_init_section_mem(ptdev, section);
+		sgt = drm_gem_shmem_get_pages_sgt(&to_panthor_bo(section->mem->obj)->base);
+		if (!drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(sgt)))
+			dma_sync_sgtable_for_device(ptdev->base.dev, sgt, DMA_TO_DEVICE);
+	}
+}
+
+static int panthor_fw_load_entry(struct panthor_device *ptdev,
+				 const struct firmware *fw,
+				 struct panthor_fw_binary_iter *iter)
+{
+	struct panthor_fw_binary_iter eiter;
+	u32 ehdr;
+	int ret;
+
+	ret = panthor_fw_binary_iter_read(ptdev, iter, &ehdr, sizeof(ehdr));
+	if (ret)
+		return ret;
+
+	if ((iter->offset % sizeof(u32)) ||
+	    (CSF_FW_BINARY_ENTRY_SIZE(ehdr) % sizeof(u32))) {
+		drm_err(&ptdev->base, "Firmware entry isn't 32 bit aligned, offset=0x%x size=0x%x\n",
+			(u32)(iter->offset - sizeof(u32)), CSF_FW_BINARY_ENTRY_SIZE(ehdr));
+		return -EINVAL;
+	}
+
+	if (panthor_fw_binary_sub_iter_init(ptdev, iter, &eiter,
+					    CSF_FW_BINARY_ENTRY_SIZE(ehdr) - sizeof(ehdr)))
+		return -EINVAL;
+
+	switch (CSF_FW_BINARY_ENTRY_TYPE(ehdr)) {
+	case CSF_FW_BINARY_ENTRY_TYPE_IFACE:
+		return panthor_fw_load_section_entry(ptdev, fw, &eiter, ehdr);
+
+	/* FIXME: handle those entry types? */
+	case CSF_FW_BINARY_ENTRY_TYPE_CONFIG:
+	case CSF_FW_BINARY_ENTRY_TYPE_FUTF_TEST:
+	case CSF_FW_BINARY_ENTRY_TYPE_TRACE_BUFFER:
+	case CSF_FW_BINARY_ENTRY_TYPE_TIMELINE_METADATA:
+		return 0;
+	default:
+		break;
+	}
+
+	if (ehdr & CSF_FW_BINARY_ENTRY_OPTIONAL)
+		return 0;
+
+	drm_err(&ptdev->base,
+		"Unsupported non-optional entry type %u in firmware\n",
+		CSF_FW_BINARY_ENTRY_TYPE(ehdr));
+	return -EINVAL;
+}
+
+static int panthor_fw_load(struct panthor_device *ptdev)
+{
+	const struct firmware *fw = NULL;
+	struct panthor_fw_binary_iter iter = {};
+	struct panthor_fw_binary_hdr hdr;
+	char fw_path[128];
+	int ret;
+
+	snprintf(fw_path, sizeof(fw_path), "arm/mali/arch%d.%d/%s",
+		 (u32)GPU_ARCH_MAJOR(ptdev->gpu_info.gpu_id),
+		 (u32)GPU_ARCH_MINOR(ptdev->gpu_info.gpu_id),
+		 CSF_FW_NAME);
+
+	ret = request_firmware(&fw, fw_path, ptdev->base.dev);
+	if (ret) {
+		drm_err(&ptdev->base, "Failed to load firmware image '%s'\n",
+			CSF_FW_NAME);
+		return ret;
+	}
+
+	iter.data = fw->data;
+	iter.size = fw->size;
+	ret = panthor_fw_binary_iter_read(ptdev, &iter, &hdr, sizeof(hdr));
+	if (ret)
+		goto out;
+
+	if (hdr.magic != CSF_FW_BINARY_HEADER_MAGIC) {
+		ret = -EINVAL;
+		drm_err(&ptdev->base, "Invalid firmware magic\n");
+		goto out;
+	}
+
+	if (hdr.major != CSF_FW_BINARY_HEADER_MAJOR_MAX) {
+		ret = -EINVAL;
+		drm_err(&ptdev->base, "Unsupported firmware binary header version %d.%d (expected %d.x)\n",
+			hdr.major, hdr.minor, CSF_FW_BINARY_HEADER_MAJOR_MAX);
+		goto out;
+	}
+
+	if (hdr.size > iter.size) {
+		drm_err(&ptdev->base, "Firmware image is truncated\n");
+		goto out;
+	}
+
+	iter.size = hdr.size;
+
+	while (iter.offset < hdr.size) {
+		ret = panthor_fw_load_entry(ptdev, fw, &iter);
+		if (ret)
+			goto out;
+	}
+
+	if (!ptdev->fw->shared_section) {
+		drm_err(&ptdev->base, "Shared interface region not found\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+out:
+	release_firmware(fw);
+	return ret;
+}
+
+/**
+ * iface_fw_to_cpu_addr() - Turn an MCU address into a CPU address
+ * @ptdev: Device.
+ * @mcu_va: MCU address.
+ *
+ * Return: NULL if the address is not part of the shared section, non-NULL otherwise.
+ */
+static void *iface_fw_to_cpu_addr(struct panthor_device *ptdev, u32 mcu_va)
+{
+	u64 shared_mem_start = panthor_kernel_bo_gpuva(ptdev->fw->shared_section->mem);
+	u64 shared_mem_end = shared_mem_start +
+			     panthor_kernel_bo_size(ptdev->fw->shared_section->mem);
+	if (mcu_va < shared_mem_start || mcu_va >= shared_mem_end)
+		return NULL;
+
+	return ptdev->fw->shared_section->mem->kmap + (mcu_va - shared_mem_start);
+}
+
+static int panthor_init_cs_iface(struct panthor_device *ptdev,
+				 unsigned int csg_idx, unsigned int cs_idx)
+{
+	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
+	struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, csg_idx);
+	struct panthor_fw_cs_iface *cs_iface = &ptdev->fw->iface.streams[csg_idx][cs_idx];
+	u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem);
+	u32 iface_offset = CSF_GROUP_CONTROL_OFFSET +
+			   (csg_idx * glb_iface->control->group_stride) +
+			   CSF_STREAM_CONTROL_OFFSET +
+			   (cs_idx * csg_iface->control->stream_stride);
+	struct panthor_fw_cs_iface *first_cs_iface =
+		panthor_fw_get_cs_iface(ptdev, 0, 0);
+
+	if (iface_offset + sizeof(*cs_iface) >= shared_section_sz)
+		return -EINVAL;
+
+	spin_lock_init(&cs_iface->lock);
+	cs_iface->control = ptdev->fw->shared_section->mem->kmap + iface_offset;
+	cs_iface->input = iface_fw_to_cpu_addr(ptdev, cs_iface->control->input_va);
+	cs_iface->output = iface_fw_to_cpu_addr(ptdev, cs_iface->control->output_va);
+
+	if (!cs_iface->input || !cs_iface->output) {
+		drm_err(&ptdev->base, "Invalid stream control interface input/output VA");
+		return -EINVAL;
+	}
+
+	if (cs_iface != first_cs_iface) {
+		if (cs_iface->control->features != first_cs_iface->control->features) {
+			drm_err(&ptdev->base, "Expecting identical CS slots");
+			return -EINVAL;
+		}
+	} else {
+		u32 reg_count = CS_FEATURES_WORK_REGS(cs_iface->control->features);
+
+		ptdev->csif_info.cs_reg_count = reg_count;
+		ptdev->csif_info.unpreserved_cs_reg_count = CSF_UNPRESERVED_REG_COUNT;
+	}
+
+	return 0;
+}
+
+static bool compare_csg(const struct panthor_fw_csg_control_iface *a,
+			const struct panthor_fw_csg_control_iface *b)
+{
+	if (a->features != b->features)
+		return false;
+	if (a->suspend_size != b->suspend_size)
+		return false;
+	if (a->protm_suspend_size != b->protm_suspend_size)
+		return false;
+	if (a->stream_num != b->stream_num)
+		return false;
+	return true;
+}
+
+static int panthor_init_csg_iface(struct panthor_device *ptdev,
+				  unsigned int csg_idx)
+{
+	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
+	struct panthor_fw_csg_iface *csg_iface = &ptdev->fw->iface.groups[csg_idx];
+	u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem);
+	u32 iface_offset = CSF_GROUP_CONTROL_OFFSET + (csg_idx * glb_iface->control->group_stride);
+	unsigned int i;
+
+	if (iface_offset + sizeof(*csg_iface) >= shared_section_sz)
+		return -EINVAL;
+
+	spin_lock_init(&csg_iface->lock);
+	csg_iface->control = ptdev->fw->shared_section->mem->kmap + iface_offset;
+	csg_iface->input = iface_fw_to_cpu_addr(ptdev, csg_iface->control->input_va);
+	csg_iface->output = iface_fw_to_cpu_addr(ptdev, csg_iface->control->output_va);
+
+	if (csg_iface->control->stream_num < MIN_CS_PER_CSG ||
+	    csg_iface->control->stream_num > MAX_CS_PER_CSG)
+		return -EINVAL;
+
+	if (!csg_iface->input || !csg_iface->output) {
+		drm_err(&ptdev->base, "Invalid group control interface input/output VA");
+		return -EINVAL;
+	}
+
+	if (csg_idx > 0) {
+		struct panthor_fw_csg_iface *first_csg_iface =
+			panthor_fw_get_csg_iface(ptdev, 0);
+
+		if (!compare_csg(first_csg_iface->control, csg_iface->control)) {
+			drm_err(&ptdev->base, "Expecting identical CSG slots");
+			return -EINVAL;
+		}
+	}
+
+	for (i = 0; i < csg_iface->control->stream_num; i++) {
+		int ret = panthor_init_cs_iface(ptdev, csg_idx, i);
+
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static u32 panthor_get_instr_features(struct panthor_device *ptdev)
+{
+	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
+
+	if (glb_iface->control->version < CSF_IFACE_VERSION(1, 1, 0))
+		return 0;
+
+	return glb_iface->control->instr_features;
+}
+
+static int panthor_fw_init_ifaces(struct panthor_device *ptdev)
+{
+	struct panthor_fw_global_iface *glb_iface = &ptdev->fw->iface.global;
+	unsigned int i;
+
+	if (!ptdev->fw->shared_section->mem->kmap)
+		return -EINVAL;
+
+	spin_lock_init(&glb_iface->lock);
+	glb_iface->control = ptdev->fw->shared_section->mem->kmap;
+
+	if (!glb_iface->control->version) {
+		drm_err(&ptdev->base, "Firmware version is 0. Firmware may have failed to boot");
+		return -EINVAL;
+	}
+
+	glb_iface->input = iface_fw_to_cpu_addr(ptdev, glb_iface->control->input_va);
+	glb_iface->output = iface_fw_to_cpu_addr(ptdev, glb_iface->control->output_va);
+	if (!glb_iface->input || !glb_iface->output) {
+		drm_err(&ptdev->base, "Invalid global control interface input/output VA");
+		return -EINVAL;
+	}
+
+	if (glb_iface->control->group_num > MAX_CSGS ||
+	    glb_iface->control->group_num < MIN_CSGS) {
+		drm_err(&ptdev->base, "Invalid number of control groups");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < glb_iface->control->group_num; i++) {
+		int ret = panthor_init_csg_iface(ptdev, i);
+
+		if (ret)
+			return ret;
+	}
+
+	drm_info(&ptdev->base, "CSF FW v%d.%d.%d, Features %#x Instrumentation features %#x",
+		 CSF_IFACE_VERSION_MAJOR(glb_iface->control->version),
+		 CSF_IFACE_VERSION_MINOR(glb_iface->control->version),
+		 CSF_IFACE_VERSION_PATCH(glb_iface->control->version),
+		 glb_iface->control->features,
+		 panthor_get_instr_features(ptdev));
+	return 0;
+}
+
+static void panthor_fw_init_global_iface(struct panthor_device *ptdev)
+{
+	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
+
+	/* Enable all cores. */
+	glb_iface->input->core_en_mask = ptdev->gpu_info.shader_present;
+
+	/* Setup timers. */
+	glb_iface->input->poweroff_timer = panthor_fw_conv_timeout(ptdev, PWROFF_HYSTERESIS_US);
+	glb_iface->input->progress_timer = PROGRESS_TIMEOUT_CYCLES >> PROGRESS_TIMEOUT_SCALE_SHIFT;
+	glb_iface->input->idle_timer = panthor_fw_conv_timeout(ptdev, IDLE_HYSTERESIS_US);
+
+	/* Enable interrupts we care about. */
+	glb_iface->input->ack_irq_mask = GLB_CFG_ALLOC_EN |
+					 GLB_PING |
+					 GLB_CFG_PROGRESS_TIMER |
+					 GLB_CFG_POWEROFF_TIMER |
+					 GLB_IDLE_EN |
+					 GLB_IDLE;
+
+	panthor_fw_update_reqs(glb_iface, req, GLB_IDLE_EN, GLB_IDLE_EN);
+	panthor_fw_toggle_reqs(glb_iface, req, ack,
+			       GLB_CFG_ALLOC_EN |
+			       GLB_CFG_POWEROFF_TIMER |
+			       GLB_CFG_PROGRESS_TIMER);
+
+	gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1);
+
+	/* Kick the watchdog. */
+	mod_delayed_work(ptdev->reset.wq, &ptdev->fw->watchdog.ping_work,
+			 msecs_to_jiffies(PING_INTERVAL_MS));
+}
+
+static void panthor_job_irq_handler(struct panthor_device *ptdev, u32 status)
+{
+	if (!ptdev->fw->booted && (status & JOB_INT_GLOBAL_IF))
+		ptdev->fw->booted = true;
+
+	wake_up_all(&ptdev->fw->req_waitqueue);
+
+	/* If the FW is not booted, don't process IRQs, just flag the FW as booted. */
+	if (!ptdev->fw->booted)
+		return;
+
+	panthor_sched_report_fw_events(ptdev, status);
+}
+PANTHOR_IRQ_HANDLER(job, JOB, panthor_job_irq_handler);
+
+static int panthor_fw_start(struct panthor_device *ptdev)
+{
+	bool timedout = false;
+
+	ptdev->fw->booted = false;
+	panthor_job_irq_resume(&ptdev->fw->irq, ~0);
+	gpu_write(ptdev, MCU_CONTROL, MCU_CONTROL_AUTO);
+
+	if (!wait_event_timeout(ptdev->fw->req_waitqueue,
+				ptdev->fw->booted,
+				msecs_to_jiffies(1000))) {
+		if (!ptdev->fw->booted &&
+		    !(gpu_read(ptdev, JOB_INT_STAT) & JOB_INT_GLOBAL_IF))
+			timedout = true;
+	}
+
+	if (timedout) {
+		static const char * const status_str[] = {
+			[MCU_STATUS_DISABLED] = "disabled",
+			[MCU_STATUS_ENABLED] = "enabled",
+			[MCU_STATUS_HALT] = "halt",
+			[MCU_STATUS_FATAL] = "fatal",
+		};
+		u32 status = gpu_read(ptdev, MCU_STATUS);
+
+		drm_err(&ptdev->base, "Failed to boot MCU (status=%s)",
+			status < ARRAY_SIZE(status_str) ? status_str[status] : "unknown");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static void panthor_fw_stop(struct panthor_device *ptdev)
+{
+	u32 status;
+
+	gpu_write(ptdev, MCU_CONTROL, MCU_CONTROL_DISABLE);
+	if (readl_poll_timeout(ptdev->iomem + MCU_STATUS, status,
+			       status == MCU_STATUS_DISABLED, 10, 100000))
+		drm_err(&ptdev->base, "Failed to stop MCU");
+}
+
+/**
+ * panthor_fw_pre_reset() - Call before a reset.
+ * @ptdev: Device.
+ * @on_hang: true if the reset was triggered on a GPU hang.
+ *
+ * If the reset is not triggered on a hang, we try to gracefully halt the
+ * MCU, so we can do a fast-reset when panthor_fw_post_reset() is called.
+ */
+void panthor_fw_pre_reset(struct panthor_device *ptdev, bool on_hang)
+{
+	/* Make sure we won't be woken up by a ping. */
+	cancel_delayed_work_sync(&ptdev->fw->watchdog.ping_work);
+
+	ptdev->fw->fast_reset = false;
+
+	if (!on_hang) {
+		struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
+		u32 status;
+
+		panthor_fw_update_reqs(glb_iface, req, GLB_HALT, GLB_HALT);
+		gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1);
+		if (!readl_poll_timeout(ptdev->iomem + MCU_STATUS, status,
+					status == MCU_STATUS_HALT, 10, 100000) &&
+		    glb_iface->output->halt_status == PANTHOR_FW_HALT_OK) {
+			ptdev->fw->fast_reset = true;
+		} else {
+			drm_warn(&ptdev->base, "Failed to cleanly suspend MCU");
+		}
+
+		/* The FW detects 0 -> 1 transitions. Make sure we reset
+		 * the HALT bit before the FW is rebooted.
+		 */
+		panthor_fw_update_reqs(glb_iface, req, 0, GLB_HALT);
+	}
+
+	panthor_job_irq_suspend(&ptdev->fw->irq);
+}
+
+/**
+ * panthor_fw_post_reset() - Call after a reset.
+ * @ptdev: Device.
+ *
+ * Start the FW. If this is not a fast reset, all FW sections are reloaded to
+ * make sure we can recover from a memory corruption.
+ */
+int panthor_fw_post_reset(struct panthor_device *ptdev)
+{
+	int ret;
+
+	/* Make the MCU VM active. */
+	ret = panthor_vm_active(ptdev->fw->vm);
+	if (ret)
+		return ret;
+
+	/* If this is a fast reset, try to start the MCU without reloading
+	 * the FW sections. If it fails, go for a full reset.
+	 */
+	if (ptdev->fw->fast_reset) {
+		ret = panthor_fw_start(ptdev);
+		if (!ret)
+			goto out;
+
+		/* Forcibly reset the MCU and force a slow reset, so we get a
+		 * fresh boot on the next panthor_fw_start() call.
+		 */
+		panthor_fw_stop(ptdev);
+		ptdev->fw->fast_reset = false;
+		drm_err(&ptdev->base, "FW fast reset failed, trying a slow reset");
+	}
+
+	/* Reload all sections, including RO ones. We're not supposed
+	 * to end up here anyway, let's just assume the overhead of
+	 * reloading everything is acceptable.
+	 */
+	panthor_reload_fw_sections(ptdev, true);
+
+	ret = panthor_fw_start(ptdev);
+	if (ret) {
+		drm_err(&ptdev->base, "FW slow reset failed");
+		return ret;
+	}
+
+out:
+	/* We must re-initialize the global interface even on fast-reset. */
+	panthor_fw_init_global_iface(ptdev);
+	return 0;
+}
+
+/**
+ * panthor_fw_unplug() - Called when the device is unplugged.
+ * @ptdev: Device.
+ *
+ * This function must make sure all pending operations are flushed before
+ * will release device resources, thus preventing any interaction with
+ * the HW.
+ *
+ * If there is still FW-related work running after this function returns,
+ * they must use drm_dev_{enter,exit}() and skip any HW access when
+ * drm_dev_enter() returns false.
+ */
+void panthor_fw_unplug(struct panthor_device *ptdev)
+{
+	struct panthor_fw_section *section;
+
+	cancel_delayed_work_sync(&ptdev->fw->watchdog.ping_work);
+
+	/* Make sure the IRQ handler can be called after that point. */
+	if (ptdev->fw->irq.irq)
+		panthor_job_irq_suspend(&ptdev->fw->irq);
+
+	panthor_fw_stop(ptdev);
+
+	list_for_each_entry(section, &ptdev->fw->sections, node)
+		panthor_kernel_bo_destroy(section->mem);
+
+	/* We intentionally don't call panthor_vm_idle() and let
+	 * panthor_mmu_unplug() release the AS we acquired with
+	 * panthor_vm_active() so we don't have to track the VM active/idle
+	 * state to keep the active_refcnt balanced.
+	 */
+	panthor_vm_put(ptdev->fw->vm);
+	ptdev->fw->vm = NULL;
+
+	panthor_gpu_power_off(ptdev, L2, ptdev->gpu_info.l2_present, 20000);
+}
+
+/**
+ * panthor_fw_wait_acks() - Wait for requests to be acknowledged by the FW.
+ * @req_ptr: Pointer to the req register.
+ * @ack_ptr: Pointer to the ack register.
+ * @wq: Wait queue to use for the sleeping wait.
+ * @req_mask: Mask of requests to wait for.
+ * @acked: Pointer to field that's updated with the acked requests.
+ * If the function returns 0, *acked == req_mask.
+ * @timeout_ms: Timeout expressed in milliseconds.
+ *
+ * Return: 0 on success, -ETIMEDOUT otherwise.
+ */
+static int panthor_fw_wait_acks(const u32 *req_ptr, const u32 *ack_ptr,
+				wait_queue_head_t *wq,
+				u32 req_mask, u32 *acked,
+				u32 timeout_ms)
+{
+	u32 ack, req = READ_ONCE(*req_ptr) & req_mask;
+	int ret;
+
+	/* Busy wait for a few µsecs before falling back to a sleeping wait. */
+	*acked = req_mask;
+	ret = read_poll_timeout_atomic(READ_ONCE, ack,
+				       (ack & req_mask) == req,
+				       0, 10, 0,
+				       *ack_ptr);
+	if (!ret)
+		return 0;
+
+	if (wait_event_timeout(*wq, (READ_ONCE(*ack_ptr) & req_mask) == req,
+			       msecs_to_jiffies(timeout_ms)))
+		return 0;
+
+	/* Check one last time, in case we were not woken up for some reason. */
+	ack = READ_ONCE(*ack_ptr);
+	if ((ack & req_mask) == req)
+		return 0;
+
+	*acked = ~(req ^ ack) & req_mask;
+	return -ETIMEDOUT;
+}
+
+/**
+ * panthor_fw_glb_wait_acks() - Wait for global requests to be acknowledged.
+ * @ptdev: Device.
+ * @req_mask: Mask of requests to wait for.
+ * @acked: Pointer to field that's updated with the acked requests.
+ * If the function returns 0, *acked == req_mask.
+ * @timeout_ms: Timeout expressed in milliseconds.
+ *
+ * Return: 0 on success, -ETIMEDOUT otherwise.
+ */
+int panthor_fw_glb_wait_acks(struct panthor_device *ptdev,
+			     u32 req_mask, u32 *acked,
+			     u32 timeout_ms)
+{
+	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
+
+	/* GLB_HALT doesn't get acked through the FW interface. */
+	if (drm_WARN_ON(&ptdev->base, req_mask & (~GLB_REQ_MASK | GLB_HALT)))
+		return -EINVAL;
+
+	return panthor_fw_wait_acks(&glb_iface->input->req,
+				    &glb_iface->output->ack,
+				    &ptdev->fw->req_waitqueue,
+				    req_mask, acked, timeout_ms);
+}
+
+/**
+ * panthor_fw_csg_wait_acks() - Wait for command stream group requests to be acknowledged.
+ * @ptdev: Device.
+ * @csg_slot: CSG slot ID.
+ * @req_mask: Mask of requests to wait for.
+ * @acked: Pointer to field that's updated with the acked requests.
+ * If the function returns 0, *acked == req_mask.
+ * @timeout_ms: Timeout expressed in milliseconds.
+ *
+ * Return: 0 on success, -ETIMEDOUT otherwise.
+ */
+int panthor_fw_csg_wait_acks(struct panthor_device *ptdev, u32 csg_slot,
+			     u32 req_mask, u32 *acked, u32 timeout_ms)
+{
+	struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, csg_slot);
+	int ret;
+
+	if (drm_WARN_ON(&ptdev->base, req_mask & ~CSG_REQ_MASK))
+		return -EINVAL;
+
+	ret = panthor_fw_wait_acks(&csg_iface->input->req,
+				   &csg_iface->output->ack,
+				   &ptdev->fw->req_waitqueue,
+				   req_mask, acked, timeout_ms);
+
+	/*
+	 * Check that all bits in the state field were updated, if any mismatch
+	 * then clear all bits in the state field. This allows code to do
+	 * (acked & CSG_STATE_MASK) and get the right value.
+	 */
+
+	if ((*acked & CSG_STATE_MASK) != CSG_STATE_MASK)
+		*acked &= ~CSG_STATE_MASK;
+
+	return ret;
+}
+
+/**
+ * panthor_fw_ring_csg_doorbells() - Ring command stream group doorbells.
+ * @ptdev: Device.
+ * @csg_mask: Bitmask encoding the command stream group doorbells to ring.
+ *
+ * This function is toggling bits in the doorbell_req and ringing the
+ * global doorbell. It doesn't require a user doorbell to be attached to
+ * the group.
+ */
+void panthor_fw_ring_csg_doorbells(struct panthor_device *ptdev, u32 csg_mask)
+{
+	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
+
+	panthor_fw_toggle_reqs(glb_iface, doorbell_req, doorbell_ack, csg_mask);
+	gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1);
+}
+
+static void panthor_fw_ping_work(struct work_struct *work)
+{
+	struct panthor_fw *fw = container_of(work, struct panthor_fw, watchdog.ping_work.work);
+	struct panthor_device *ptdev = fw->irq.ptdev;
+	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
+	u32 acked;
+	int ret;
+
+	if (panthor_device_reset_is_pending(ptdev))
+		return;
+
+	panthor_fw_toggle_reqs(glb_iface, req, ack, GLB_PING);
+	gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1);
+
+	ret = panthor_fw_glb_wait_acks(ptdev, GLB_PING, &acked, 100);
+	if (ret) {
+		panthor_device_schedule_reset(ptdev);
+		drm_err(&ptdev->base, "FW ping timeout, scheduling a reset");
+	} else {
+		mod_delayed_work(ptdev->reset.wq, &fw->watchdog.ping_work,
+				 msecs_to_jiffies(PING_INTERVAL_MS));
+	}
+}
+
+/**
+ * panthor_fw_init() - Initialize FW related data.
+ * @ptdev: Device.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_fw_init(struct panthor_device *ptdev)
+{
+	struct panthor_fw *fw;
+	int ret, irq;
+
+	fw = drmm_kzalloc(&ptdev->base, sizeof(*fw), GFP_KERNEL);
+	if (!fw)
+		return -ENOMEM;
+
+	ptdev->fw = fw;
+	init_waitqueue_head(&fw->req_waitqueue);
+	INIT_LIST_HEAD(&fw->sections);
+	INIT_DELAYED_WORK(&fw->watchdog.ping_work, panthor_fw_ping_work);
+
+	irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "job");
+	if (irq <= 0)
+		return -ENODEV;
+
+	ret = panthor_request_job_irq(ptdev, &fw->irq, irq, 0);
+	if (ret) {
+		drm_err(&ptdev->base, "failed to request job irq");
+		return ret;
+	}
+
+	ret = panthor_gpu_l2_power_on(ptdev);
+	if (ret)
+		return ret;
+
+	fw->vm = panthor_vm_create(ptdev, true,
+				   0, SZ_4G,
+				   CSF_MCU_SHARED_REGION_START,
+				   CSF_MCU_SHARED_REGION_SIZE);
+	if (IS_ERR(fw->vm)) {
+		ret = PTR_ERR(fw->vm);
+		fw->vm = NULL;
+		goto err_unplug_fw;
+	}
+
+	ret = panthor_fw_load(ptdev);
+	if (ret)
+		goto err_unplug_fw;
+
+	ret = panthor_vm_active(fw->vm);
+	if (ret)
+		goto err_unplug_fw;
+
+	ret = panthor_fw_start(ptdev);
+	if (ret)
+		goto err_unplug_fw;
+
+	ret = panthor_fw_init_ifaces(ptdev);
+	if (ret)
+		goto err_unplug_fw;
+
+	panthor_fw_init_global_iface(ptdev);
+	return 0;
+
+err_unplug_fw:
+	panthor_fw_unplug(ptdev);
+	return ret;
+}
+
+MODULE_FIRMWARE("arm/mali/arch10.8/mali_csffw.bin");
diff --git a/drivers/gpu/drm/panthor/panthor_fw.h b/drivers/gpu/drm/panthor/panthor_fw.h
new file mode 100644
index 000000000000..22448abde992
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_fw.h
@@ -0,0 +1,503 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2023 Collabora ltd. */
+
+#ifndef __PANTHOR_MCU_H__
+#define __PANTHOR_MCU_H__
+
+#include <linux/types.h>
+
+struct panthor_device;
+struct panthor_kernel_bo;
+
+#define MAX_CSGS				31
+#define MAX_CS_PER_CSG                          32
+
+struct panthor_fw_ringbuf_input_iface {
+	u64 insert;
+	u64 extract;
+};
+
+struct panthor_fw_ringbuf_output_iface {
+	u64 extract;
+	u32 active;
+};
+
+struct panthor_fw_cs_control_iface {
+#define CS_FEATURES_WORK_REGS(x)		(((x) & GENMASK(7, 0)) + 1)
+#define CS_FEATURES_SCOREBOARDS(x)		(((x) & GENMASK(15, 8)) >> 8)
+#define CS_FEATURES_COMPUTE			BIT(16)
+#define CS_FEATURES_FRAGMENT			BIT(17)
+#define CS_FEATURES_TILER			BIT(18)
+	u32 features;
+	u32 input_va;
+	u32 output_va;
+};
+
+struct panthor_fw_cs_input_iface {
+#define CS_STATE_MASK				GENMASK(2, 0)
+#define CS_STATE_STOP				0
+#define CS_STATE_START				1
+#define CS_EXTRACT_EVENT			BIT(4)
+#define CS_IDLE_SYNC_WAIT			BIT(8)
+#define CS_IDLE_PROTM_PENDING			BIT(9)
+#define CS_IDLE_EMPTY				BIT(10)
+#define CS_IDLE_RESOURCE_REQ			BIT(11)
+#define CS_TILER_OOM				BIT(26)
+#define CS_PROTM_PENDING			BIT(27)
+#define CS_FATAL				BIT(30)
+#define CS_FAULT				BIT(31)
+#define CS_REQ_MASK				(CS_STATE_MASK | \
+						 CS_EXTRACT_EVENT | \
+						 CS_IDLE_SYNC_WAIT | \
+						 CS_IDLE_PROTM_PENDING | \
+						 CS_IDLE_EMPTY | \
+						 CS_IDLE_RESOURCE_REQ)
+#define CS_EVT_MASK				(CS_TILER_OOM | \
+						 CS_PROTM_PENDING | \
+						 CS_FATAL | \
+						 CS_FAULT)
+	u32 req;
+
+#define CS_CONFIG_PRIORITY(x)			((x) & GENMASK(3, 0))
+#define CS_CONFIG_DOORBELL(x)			(((x) << 8) & GENMASK(15, 8))
+	u32 config;
+	u32 reserved1;
+	u32 ack_irq_mask;
+	u64 ringbuf_base;
+	u32 ringbuf_size;
+	u32 reserved2;
+	u64 heap_start;
+	u64 heap_end;
+	u64 ringbuf_input;
+	u64 ringbuf_output;
+	u32 instr_config;
+	u32 instrbuf_size;
+	u64 instrbuf_base;
+	u64 instrbuf_offset_ptr;
+};
+
+struct panthor_fw_cs_output_iface {
+	u32 ack;
+	u32 reserved1[15];
+	u64 status_cmd_ptr;
+
+#define CS_STATUS_WAIT_SB_MASK			GENMASK(15, 0)
+#define CS_STATUS_WAIT_SB_SRC_MASK		GENMASK(19, 16)
+#define CS_STATUS_WAIT_SB_SRC_NONE		(0 << 16)
+#define CS_STATUS_WAIT_SB_SRC_WAIT		(8 << 16)
+#define CS_STATUS_WAIT_SYNC_COND_LE		(0 << 24)
+#define CS_STATUS_WAIT_SYNC_COND_GT		(1 << 24)
+#define CS_STATUS_WAIT_SYNC_COND_MASK		GENMASK(27, 24)
+#define CS_STATUS_WAIT_PROGRESS			BIT(28)
+#define CS_STATUS_WAIT_PROTM			BIT(29)
+#define CS_STATUS_WAIT_SYNC_64B			BIT(30)
+#define CS_STATUS_WAIT_SYNC			BIT(31)
+	u32 status_wait;
+	u32 status_req_resource;
+	u64 status_wait_sync_ptr;
+	u32 status_wait_sync_value;
+	u32 status_scoreboards;
+
+#define CS_STATUS_BLOCKED_REASON_UNBLOCKED	0
+#define CS_STATUS_BLOCKED_REASON_SB_WAIT	1
+#define CS_STATUS_BLOCKED_REASON_PROGRESS_WAIT	2
+#define CS_STATUS_BLOCKED_REASON_SYNC_WAIT	3
+#define CS_STATUS_BLOCKED_REASON_DEFERRED	5
+#define CS_STATUS_BLOCKED_REASON_RES		6
+#define CS_STATUS_BLOCKED_REASON_FLUSH		7
+#define CS_STATUS_BLOCKED_REASON_MASK		GENMASK(3, 0)
+	u32 status_blocked_reason;
+	u32 status_wait_sync_value_hi;
+	u32 reserved2[6];
+
+#define CS_EXCEPTION_TYPE(x)			((x) & GENMASK(7, 0))
+#define CS_EXCEPTION_DATA(x)			(((x) >> 8) & GENMASK(23, 0))
+	u32 fault;
+	u32 fatal;
+	u64 fault_info;
+	u64 fatal_info;
+	u32 reserved3[10];
+	u32 heap_vt_start;
+	u32 heap_vt_end;
+	u32 reserved4;
+	u32 heap_frag_end;
+	u64 heap_address;
+};
+
+struct panthor_fw_csg_control_iface {
+	u32 features;
+	u32 input_va;
+	u32 output_va;
+	u32 suspend_size;
+	u32 protm_suspend_size;
+	u32 stream_num;
+	u32 stream_stride;
+};
+
+struct panthor_fw_csg_input_iface {
+#define CSG_STATE_MASK				GENMASK(2, 0)
+#define CSG_STATE_TERMINATE			0
+#define CSG_STATE_START				1
+#define CSG_STATE_SUSPEND			2
+#define CSG_STATE_RESUME			3
+#define CSG_ENDPOINT_CONFIG			BIT(4)
+#define CSG_STATUS_UPDATE			BIT(5)
+#define CSG_SYNC_UPDATE				BIT(28)
+#define CSG_IDLE				BIT(29)
+#define CSG_DOORBELL				BIT(30)
+#define CSG_PROGRESS_TIMER_EVENT		BIT(31)
+#define CSG_REQ_MASK				(CSG_STATE_MASK | \
+						 CSG_ENDPOINT_CONFIG | \
+						 CSG_STATUS_UPDATE)
+#define CSG_EVT_MASK				(CSG_SYNC_UPDATE | \
+						 CSG_IDLE | \
+						 CSG_PROGRESS_TIMER_EVENT)
+	u32 req;
+	u32 ack_irq_mask;
+
+	u32 doorbell_req;
+	u32 cs_irq_ack;
+	u32 reserved1[4];
+	u64 allow_compute;
+	u64 allow_fragment;
+	u32 allow_other;
+
+#define CSG_EP_REQ_COMPUTE(x)			((x) & GENMASK(7, 0))
+#define CSG_EP_REQ_FRAGMENT(x)			(((x) << 8) & GENMASK(15, 8))
+#define CSG_EP_REQ_TILER(x)			(((x) << 16) & GENMASK(19, 16))
+#define CSG_EP_REQ_EXCL_COMPUTE			BIT(20)
+#define CSG_EP_REQ_EXCL_FRAGMENT		BIT(21)
+#define CSG_EP_REQ_PRIORITY(x)			(((x) << 28) & GENMASK(31, 28))
+#define CSG_EP_REQ_PRIORITY_MASK		GENMASK(31, 28)
+	u32 endpoint_req;
+	u32 reserved2[2];
+	u64 suspend_buf;
+	u64 protm_suspend_buf;
+	u32 config;
+	u32 iter_trace_config;
+};
+
+struct panthor_fw_csg_output_iface {
+	u32 ack;
+	u32 reserved1;
+	u32 doorbell_ack;
+	u32 cs_irq_req;
+	u32 status_endpoint_current;
+	u32 status_endpoint_req;
+
+#define CSG_STATUS_STATE_IS_IDLE		BIT(0)
+	u32 status_state;
+	u32 resource_dep;
+};
+
+struct panthor_fw_global_control_iface {
+	u32 version;
+	u32 features;
+	u32 input_va;
+	u32 output_va;
+	u32 group_num;
+	u32 group_stride;
+	u32 perfcnt_size;
+	u32 instr_features;
+};
+
+struct panthor_fw_global_input_iface {
+#define GLB_HALT				BIT(0)
+#define GLB_CFG_PROGRESS_TIMER			BIT(1)
+#define GLB_CFG_ALLOC_EN			BIT(2)
+#define GLB_CFG_POWEROFF_TIMER			BIT(3)
+#define GLB_PROTM_ENTER				BIT(4)
+#define GLB_PERFCNT_EN				BIT(5)
+#define GLB_PERFCNT_SAMPLE			BIT(6)
+#define GLB_COUNTER_EN				BIT(7)
+#define GLB_PING				BIT(8)
+#define GLB_FWCFG_UPDATE			BIT(9)
+#define GLB_IDLE_EN				BIT(10)
+#define GLB_SLEEP				BIT(12)
+#define GLB_INACTIVE_COMPUTE			BIT(20)
+#define GLB_INACTIVE_FRAGMENT			BIT(21)
+#define GLB_INACTIVE_TILER			BIT(22)
+#define GLB_PROTM_EXIT				BIT(23)
+#define GLB_PERFCNT_THRESHOLD			BIT(24)
+#define GLB_PERFCNT_OVERFLOW			BIT(25)
+#define GLB_IDLE				BIT(26)
+#define GLB_DBG_CSF				BIT(30)
+#define GLB_DBG_HOST				BIT(31)
+#define GLB_REQ_MASK				GENMASK(10, 0)
+#define GLB_EVT_MASK				GENMASK(26, 20)
+	u32 req;
+	u32 ack_irq_mask;
+	u32 doorbell_req;
+	u32 reserved1;
+	u32 progress_timer;
+
+#define GLB_TIMER_VAL(x)			((x) & GENMASK(30, 0))
+#define GLB_TIMER_SOURCE_GPU_COUNTER		BIT(31)
+	u32 poweroff_timer;
+	u64 core_en_mask;
+	u32 reserved2;
+	u32 perfcnt_as;
+	u64 perfcnt_base;
+	u32 perfcnt_extract;
+	u32 reserved3[3];
+	u32 perfcnt_config;
+	u32 perfcnt_csg_select;
+	u32 perfcnt_fw_enable;
+	u32 perfcnt_csg_enable;
+	u32 perfcnt_csf_enable;
+	u32 perfcnt_shader_enable;
+	u32 perfcnt_tiler_enable;
+	u32 perfcnt_mmu_l2_enable;
+	u32 reserved4[8];
+	u32 idle_timer;
+};
+
+enum panthor_fw_halt_status {
+	PANTHOR_FW_HALT_OK = 0,
+	PANTHOR_FW_HALT_ON_PANIC = 0x4e,
+	PANTHOR_FW_HALT_ON_WATCHDOG_EXPIRATION = 0x4f,
+};
+
+struct panthor_fw_global_output_iface {
+	u32 ack;
+	u32 reserved1;
+	u32 doorbell_ack;
+	u32 reserved2;
+	u32 halt_status;
+	u32 perfcnt_status;
+	u32 perfcnt_insert;
+};
+
+/**
+ * struct panthor_fw_cs_iface - Firmware command stream slot interface
+ */
+struct panthor_fw_cs_iface {
+	/**
+	 * @lock: Lock protecting access to the panthor_fw_cs_input_iface::req
+	 * field.
+	 *
+	 * Needed so we can update the req field concurrently from the interrupt
+	 * handler and the scheduler logic.
+	 *
+	 * TODO: Ideally we'd want to use a cmpxchg() to update the req, but FW
+	 * interface sections are mapped uncached/write-combined right now, and
+	 * using cmpxchg() on such mappings leads to SError faults. Revisit when
+	 * we have 'SHARED' GPU mappings hooked up.
+	 */
+	spinlock_t lock;
+
+	/**
+	 * @control: Command stream slot control interface.
+	 *
+	 * Used to expose command stream slot properties.
+	 *
+	 * This interface is read-only.
+	 */
+	struct panthor_fw_cs_control_iface *control;
+
+	/**
+	 * @input: Command stream slot input interface.
+	 *
+	 * Used for host updates/events.
+	 */
+	struct panthor_fw_cs_input_iface *input;
+
+	/**
+	 * @output: Command stream slot output interface.
+	 *
+	 * Used for FW updates/events.
+	 *
+	 * This interface is read-only.
+	 */
+	const struct panthor_fw_cs_output_iface *output;
+};
+
+/**
+ * struct panthor_fw_csg_iface - Firmware command stream group slot interface
+ */
+struct panthor_fw_csg_iface {
+	/**
+	 * @lock: Lock protecting access to the panthor_fw_csg_input_iface::req
+	 * field.
+	 *
+	 * Needed so we can update the req field concurrently from the interrupt
+	 * handler and the scheduler logic.
+	 *
+	 * TODO: Ideally we'd want to use a cmpxchg() to update the req, but FW
+	 * interface sections are mapped uncached/write-combined right now, and
+	 * using cmpxchg() on such mappings leads to SError faults. Revisit when
+	 * we have 'SHARED' GPU mappings hooked up.
+	 */
+	spinlock_t lock;
+
+	/**
+	 * @control: Command stream group slot control interface.
+	 *
+	 * Used to expose command stream group slot properties.
+	 *
+	 * This interface is read-only.
+	 */
+	const struct panthor_fw_csg_control_iface *control;
+
+	/**
+	 * @input: Command stream slot input interface.
+	 *
+	 * Used for host updates/events.
+	 */
+	struct panthor_fw_csg_input_iface *input;
+
+	/**
+	 * @output: Command stream group slot output interface.
+	 *
+	 * Used for FW updates/events.
+	 *
+	 * This interface is read-only.
+	 */
+	const struct panthor_fw_csg_output_iface *output;
+};
+
+/**
+ * struct panthor_fw_global_iface - Firmware global interface
+ */
+struct panthor_fw_global_iface {
+	/**
+	 * @lock: Lock protecting access to the panthor_fw_global_input_iface::req
+	 * field.
+	 *
+	 * Needed so we can update the req field concurrently from the interrupt
+	 * handler and the scheduler/FW management logic.
+	 *
+	 * TODO: Ideally we'd want to use a cmpxchg() to update the req, but FW
+	 * interface sections are mapped uncached/write-combined right now, and
+	 * using cmpxchg() on such mappings leads to SError faults. Revisit when
+	 * we have 'SHARED' GPU mappings hooked up.
+	 */
+	spinlock_t lock;
+
+	/**
+	 * @control: Command stream group slot control interface.
+	 *
+	 * Used to expose global FW properties.
+	 *
+	 * This interface is read-only.
+	 */
+	const struct panthor_fw_global_control_iface *control;
+
+	/**
+	 * @input: Global input interface.
+	 *
+	 * Used for host updates/events.
+	 */
+	struct panthor_fw_global_input_iface *input;
+
+	/**
+	 * @output: Global output interface.
+	 *
+	 * Used for FW updates/events.
+	 *
+	 * This interface is read-only.
+	 */
+	const struct panthor_fw_global_output_iface *output;
+};
+
+/**
+ * panthor_fw_toggle_reqs() - Toggle acknowledge bits to send an event to the FW
+ * @__iface: The interface to operate on.
+ * @__in_reg: Name of the register to update in the input section of the interface.
+ * @__out_reg: Name of the register to take as a reference in the output section of the
+ * interface.
+ * @__mask: Mask to apply to the update.
+ *
+ * The Host -> FW event/message passing was designed to be lockless, with each side of
+ * the channel having its writeable section. Events are signaled as a difference between
+ * the host and FW side in the req/ack registers (when a bit differs, there's an event
+ * pending, when they are the same, nothing needs attention).
+ *
+ * This helper allows one to update the req register based on the current value of the
+ * ack register managed by the FW. Toggling a specific bit will flag an event. In order
+ * for events to be re-evaluated, the interface doorbell needs to be rung.
+ *
+ * Concurrent accesses to the same req register is covered.
+ *
+ * Anything requiring atomic updates to multiple registers requires a dedicated lock.
+ */
+#define panthor_fw_toggle_reqs(__iface, __in_reg, __out_reg, __mask) \
+	do { \
+		u32 __cur_val, __new_val, __out_val; \
+		spin_lock(&(__iface)->lock); \
+		__cur_val = READ_ONCE((__iface)->input->__in_reg); \
+		__out_val = READ_ONCE((__iface)->output->__out_reg); \
+		__new_val = ((__out_val ^ (__mask)) & (__mask)) | (__cur_val & ~(__mask)); \
+		WRITE_ONCE((__iface)->input->__in_reg, __new_val); \
+		spin_unlock(&(__iface)->lock); \
+	} while (0)
+
+/**
+ * panthor_fw_update_reqs() - Update bits to reflect a configuration change
+ * @__iface: The interface to operate on.
+ * @__in_reg: Name of the register to update in the input section of the interface.
+ * @__val: Value to set.
+ * @__mask: Mask to apply to the update.
+ *
+ * Some configuration get passed through req registers that are also used to
+ * send events to the FW. Those req registers being updated from the interrupt
+ * handler, they require special helpers to update the configuration part as well.
+ *
+ * Concurrent accesses to the same req register is covered.
+ *
+ * Anything requiring atomic updates to multiple registers requires a dedicated lock.
+ */
+#define panthor_fw_update_reqs(__iface, __in_reg, __val, __mask) \
+	do { \
+		u32 __cur_val, __new_val; \
+		spin_lock(&(__iface)->lock); \
+		__cur_val = READ_ONCE((__iface)->input->__in_reg); \
+		__new_val = (__cur_val & ~(__mask)) | ((__val) & (__mask)); \
+		WRITE_ONCE((__iface)->input->__in_reg, __new_val); \
+		spin_unlock(&(__iface)->lock); \
+	} while (0)
+
+struct panthor_fw_global_iface *
+panthor_fw_get_glb_iface(struct panthor_device *ptdev);
+
+struct panthor_fw_csg_iface *
+panthor_fw_get_csg_iface(struct panthor_device *ptdev, u32 csg_slot);
+
+struct panthor_fw_cs_iface *
+panthor_fw_get_cs_iface(struct panthor_device *ptdev, u32 csg_slot, u32 cs_slot);
+
+int panthor_fw_csg_wait_acks(struct panthor_device *ptdev, u32 csg_id, u32 req_mask,
+			     u32 *acked, u32 timeout_ms);
+
+int panthor_fw_glb_wait_acks(struct panthor_device *ptdev, u32 req_mask, u32 *acked,
+			     u32 timeout_ms);
+
+void panthor_fw_ring_csg_doorbells(struct panthor_device *ptdev, u32 csg_slot);
+
+struct panthor_kernel_bo *
+panthor_fw_alloc_queue_iface_mem(struct panthor_device *ptdev,
+				 struct panthor_fw_ringbuf_input_iface **input,
+				 const struct panthor_fw_ringbuf_output_iface **output,
+				 u32 *input_fw_va, u32 *output_fw_va);
+struct panthor_kernel_bo *
+panthor_fw_alloc_suspend_buf_mem(struct panthor_device *ptdev, size_t size);
+
+struct panthor_vm *panthor_fw_vm(struct panthor_device *ptdev);
+
+void panthor_fw_pre_reset(struct panthor_device *ptdev, bool on_hang);
+int panthor_fw_post_reset(struct panthor_device *ptdev);
+
+static inline void panthor_fw_suspend(struct panthor_device *ptdev)
+{
+	panthor_fw_pre_reset(ptdev, false);
+}
+
+static inline int panthor_fw_resume(struct panthor_device *ptdev)
+{
+	return panthor_fw_post_reset(ptdev);
+}
+
+int panthor_fw_init(struct panthor_device *ptdev);
+void panthor_fw_unplug(struct panthor_device *ptdev);
+
+#endif
diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c
new file mode 100644
index 000000000000..38f560864879
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_gem.c
@@ -0,0 +1,232 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+/* Copyright 2023 Collabora ltd. */
+
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+
+#include <drm/panthor_drm.h>
+
+#include "panthor_device.h"
+#include "panthor_gem.h"
+#include "panthor_mmu.h"
+
+static void panthor_gem_free_object(struct drm_gem_object *obj)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
+	struct drm_gem_object *vm_root_gem = bo->exclusive_vm_root_gem;
+
+	drm_gem_free_mmap_offset(&bo->base.base);
+	mutex_destroy(&bo->gpuva_list_lock);
+	drm_gem_shmem_free(&bo->base);
+	drm_gem_object_put(vm_root_gem);
+}
+
+/**
+ * panthor_kernel_bo_destroy() - Destroy a kernel buffer object
+ * @bo: Kernel buffer object to destroy. If NULL or an ERR_PTR(), the destruction
+ * is skipped.
+ */
+void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo)
+{
+	struct panthor_vm *vm;
+	int ret;
+
+	if (IS_ERR_OR_NULL(bo))
+		return;
+
+	vm = bo->vm;
+	panthor_kernel_bo_vunmap(bo);
+
+	if (drm_WARN_ON(bo->obj->dev,
+			to_panthor_bo(bo->obj)->exclusive_vm_root_gem != panthor_vm_root_gem(vm)))
+		goto out_free_bo;
+
+	ret = panthor_vm_unmap_range(vm, bo->va_node.start,
+				     panthor_kernel_bo_size(bo));
+	if (ret)
+		goto out_free_bo;
+
+	panthor_vm_free_va(vm, &bo->va_node);
+	drm_gem_object_put(bo->obj);
+
+out_free_bo:
+	panthor_vm_put(vm);
+	kfree(bo);
+}
+
+/**
+ * panthor_kernel_bo_create() - Create and map a GEM object to a VM
+ * @ptdev: Device.
+ * @vm: VM to map the GEM to. If NULL, the kernel object is not GPU mapped.
+ * @size: Size of the buffer object.
+ * @bo_flags: Combination of drm_panthor_bo_flags flags.
+ * @vm_map_flags: Combination of drm_panthor_vm_bind_op_flags (only those
+ * that are related to map operations).
+ * @gpu_va: GPU address assigned when mapping to the VM.
+ * If gpu_va == PANTHOR_VM_KERNEL_AUTO_VA, the virtual address will be
+ * automatically allocated.
+ *
+ * Return: A valid pointer in case of success, an ERR_PTR() otherwise.
+ */
+struct panthor_kernel_bo *
+panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
+			 size_t size, u32 bo_flags, u32 vm_map_flags,
+			 u64 gpu_va)
+{
+	struct drm_gem_shmem_object *obj;
+	struct panthor_kernel_bo *kbo;
+	struct panthor_gem_object *bo;
+	int ret;
+
+	if (drm_WARN_ON(&ptdev->base, !vm))
+		return ERR_PTR(-EINVAL);
+
+	kbo = kzalloc(sizeof(*kbo), GFP_KERNEL);
+	if (!kbo)
+		return ERR_PTR(-ENOMEM);
+
+	obj = drm_gem_shmem_create(&ptdev->base, size);
+	if (IS_ERR(obj)) {
+		ret = PTR_ERR(obj);
+		goto err_free_bo;
+	}
+
+	bo = to_panthor_bo(&obj->base);
+	size = obj->base.size;
+	kbo->obj = &obj->base;
+	bo->flags = bo_flags;
+
+	ret = panthor_vm_alloc_va(vm, gpu_va, size, &kbo->va_node);
+	if (ret)
+		goto err_put_obj;
+
+	ret = panthor_vm_map_bo_range(vm, bo, 0, size, kbo->va_node.start, vm_map_flags);
+	if (ret)
+		goto err_free_va;
+
+	kbo->vm = panthor_vm_get(vm);
+	bo->exclusive_vm_root_gem = panthor_vm_root_gem(vm);
+	drm_gem_object_get(bo->exclusive_vm_root_gem);
+	bo->base.base.resv = bo->exclusive_vm_root_gem->resv;
+	return kbo;
+
+err_free_va:
+	panthor_vm_free_va(vm, &kbo->va_node);
+
+err_put_obj:
+	drm_gem_object_put(&obj->base);
+
+err_free_bo:
+	kfree(kbo);
+	return ERR_PTR(ret);
+}
+
+static int panthor_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
+
+	/* Don't allow mmap on objects that have the NO_MMAP flag set. */
+	if (bo->flags & DRM_PANTHOR_BO_NO_MMAP)
+		return -EINVAL;
+
+	return drm_gem_shmem_object_mmap(obj, vma);
+}
+
+static struct dma_buf *
+panthor_gem_prime_export(struct drm_gem_object *obj, int flags)
+{
+	/* We can't export GEMs that have an exclusive VM. */
+	if (to_panthor_bo(obj)->exclusive_vm_root_gem)
+		return ERR_PTR(-EINVAL);
+
+	return drm_gem_prime_export(obj, flags);
+}
+
+static const struct drm_gem_object_funcs panthor_gem_funcs = {
+	.free = panthor_gem_free_object,
+	.print_info = drm_gem_shmem_object_print_info,
+	.pin = drm_gem_shmem_object_pin,
+	.unpin = drm_gem_shmem_object_unpin,
+	.get_sg_table = drm_gem_shmem_object_get_sg_table,
+	.vmap = drm_gem_shmem_object_vmap,
+	.vunmap = drm_gem_shmem_object_vunmap,
+	.mmap = panthor_gem_mmap,
+	.export = panthor_gem_prime_export,
+	.vm_ops = &drm_gem_shmem_vm_ops,
+};
+
+/**
+ * panthor_gem_create_object - Implementation of driver->gem_create_object.
+ * @ddev: DRM device
+ * @size: Size in bytes of the memory the object will reference
+ *
+ * This lets the GEM helpers allocate object structs for us, and keep
+ * our BO stats correct.
+ */
+struct drm_gem_object *panthor_gem_create_object(struct drm_device *ddev, size_t size)
+{
+	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
+	struct panthor_gem_object *obj;
+
+	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
+	if (!obj)
+		return ERR_PTR(-ENOMEM);
+
+	obj->base.base.funcs = &panthor_gem_funcs;
+	obj->base.map_wc = !ptdev->coherent;
+	mutex_init(&obj->gpuva_list_lock);
+	drm_gem_gpuva_set_lock(&obj->base.base, &obj->gpuva_list_lock);
+
+	return &obj->base.base;
+}
+
+/**
+ * panthor_gem_create_with_handle() - Create a GEM object and attach it to a handle.
+ * @file: DRM file.
+ * @ddev: DRM device.
+ * @exclusive_vm: Exclusive VM. Not NULL if the GEM object can't be shared.
+ * @size: Size of the GEM object to allocate.
+ * @flags: Combination of drm_panthor_bo_flags flags.
+ * @handle: Pointer holding the handle pointing to the new GEM object.
+ *
+ * Return: Zero on success
+ */
+int
+panthor_gem_create_with_handle(struct drm_file *file,
+			       struct drm_device *ddev,
+			       struct panthor_vm *exclusive_vm,
+			       u64 *size, u32 flags, u32 *handle)
+{
+	int ret;
+	struct drm_gem_shmem_object *shmem;
+	struct panthor_gem_object *bo;
+
+	shmem = drm_gem_shmem_create(ddev, *size);
+	if (IS_ERR(shmem))
+		return PTR_ERR(shmem);
+
+	bo = to_panthor_bo(&shmem->base);
+	bo->flags = flags;
+
+	if (exclusive_vm) {
+		bo->exclusive_vm_root_gem = panthor_vm_root_gem(exclusive_vm);
+		drm_gem_object_get(bo->exclusive_vm_root_gem);
+		bo->base.base.resv = bo->exclusive_vm_root_gem->resv;
+	}
+
+	/*
+	 * Allocate an id of idr table where the obj is registered
+	 * and handle has the id what user can see.
+	 */
+	ret = drm_gem_handle_create(file, &shmem->base, handle);
+	if (!ret)
+		*size = bo->base.base.size;
+
+	/* drop reference from allocate - handle holds it now. */
+	drm_gem_object_put(&shmem->base);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/panthor/panthor_gem.h b/drivers/gpu/drm/panthor/panthor_gem.h
new file mode 100644
index 000000000000..e43021cf6d45
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_gem.h
@@ -0,0 +1,146 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+/* Copyright 2023 Collabora ltd. */
+
+#ifndef __PANTHOR_GEM_H__
+#define __PANTHOR_GEM_H__
+
+#include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_mm.h>
+
+#include <linux/iosys-map.h>
+#include <linux/rwsem.h>
+
+struct panthor_vm;
+
+/**
+ * struct panthor_gem_object - Driver specific GEM object.
+ */
+struct panthor_gem_object {
+	/** @base: Inherit from drm_gem_shmem_object. */
+	struct drm_gem_shmem_object base;
+
+	/**
+	 * @exclusive_vm_root_gem: Root GEM of the exclusive VM this GEM object
+	 * is attached to.
+	 *
+	 * If @exclusive_vm_root_gem != NULL, any attempt to bind the GEM to a
+	 * different VM will fail.
+	 *
+	 * All FW memory objects have this field set to the root GEM of the MCU
+	 * VM.
+	 */
+	struct drm_gem_object *exclusive_vm_root_gem;
+
+	/**
+	 * @gpuva_list_lock: Custom GPUVA lock.
+	 *
+	 * Used to protect insertion of drm_gpuva elements to the
+	 * drm_gem_object.gpuva.list list.
+	 *
+	 * We can't use the GEM resv for that, because drm_gpuva_link() is
+	 * called in a dma-signaling path, where we're not allowed to take
+	 * resv locks.
+	 */
+	struct mutex gpuva_list_lock;
+
+	/** @flags: Combination of drm_panthor_bo_flags flags. */
+	u32 flags;
+};
+
+/**
+ * struct panthor_kernel_bo - Kernel buffer object.
+ *
+ * These objects are only manipulated by the kernel driver and not
+ * directly exposed to the userspace. The GPU address of a kernel
+ * BO might be passed to userspace though.
+ */
+struct panthor_kernel_bo {
+	/**
+	 * @obj: The GEM object backing this kernel buffer object.
+	 */
+	struct drm_gem_object *obj;
+
+	/**
+	 * @vm: VM this private buffer is attached to.
+	 */
+	struct panthor_vm *vm;
+
+	/**
+	 * @va_node: VA space allocated to this GEM.
+	 */
+	struct drm_mm_node va_node;
+
+	/**
+	 * @kmap: Kernel CPU mapping of @gem.
+	 */
+	void *kmap;
+};
+
+static inline
+struct panthor_gem_object *to_panthor_bo(struct drm_gem_object *obj)
+{
+	return container_of(to_drm_gem_shmem_obj(obj), struct panthor_gem_object, base);
+}
+
+struct drm_gem_object *panthor_gem_create_object(struct drm_device *ddev, size_t size);
+
+struct drm_gem_object *
+panthor_gem_prime_import_sg_table(struct drm_device *ddev,
+				  struct dma_buf_attachment *attach,
+				  struct sg_table *sgt);
+
+int
+panthor_gem_create_with_handle(struct drm_file *file,
+			       struct drm_device *ddev,
+			       struct panthor_vm *exclusive_vm,
+			       u64 *size, u32 flags, uint32_t *handle);
+
+static inline u64
+panthor_kernel_bo_gpuva(struct panthor_kernel_bo *bo)
+{
+	return bo->va_node.start;
+}
+
+static inline size_t
+panthor_kernel_bo_size(struct panthor_kernel_bo *bo)
+{
+	return bo->obj->size;
+}
+
+static inline int
+panthor_kernel_bo_vmap(struct panthor_kernel_bo *bo)
+{
+	struct iosys_map map;
+	int ret;
+
+	if (bo->kmap)
+		return 0;
+
+	ret = drm_gem_vmap_unlocked(bo->obj, &map);
+	if (ret)
+		return ret;
+
+	bo->kmap = map.vaddr;
+	return 0;
+}
+
+static inline void
+panthor_kernel_bo_vunmap(struct panthor_kernel_bo *bo)
+{
+	if (bo->kmap) {
+		struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->kmap);
+
+		drm_gem_vunmap_unlocked(bo->obj, &map);
+		bo->kmap = NULL;
+	}
+}
+
+struct panthor_kernel_bo *
+panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
+			 size_t size, u32 bo_flags, u32 vm_map_flags,
+			 u64 gpu_va);
+
+void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo);
+
+#endif /* __PANTHOR_GEM_H__ */
diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c
new file mode 100644
index 000000000000..5251d8764e7d
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_gpu.c
@@ -0,0 +1,482 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
+/* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
+/* Copyright 2019 Collabora ltd. */
+
+#include <linux/bitfield.h>
+#include <linux/bitmap.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#include <drm/drm_drv.h>
+#include <drm/drm_managed.h>
+
+#include "panthor_device.h"
+#include "panthor_gpu.h"
+#include "panthor_regs.h"
+
+/**
+ * struct panthor_gpu - GPU block management data.
+ */
+struct panthor_gpu {
+	/** @irq: GPU irq. */
+	struct panthor_irq irq;
+
+	/** @reqs_lock: Lock protecting access to pending_reqs. */
+	spinlock_t reqs_lock;
+
+	/** @pending_reqs: Pending GPU requests. */
+	u32 pending_reqs;
+
+	/** @reqs_acked: GPU request wait queue. */
+	wait_queue_head_t reqs_acked;
+};
+
+/**
+ * struct panthor_model - GPU model description
+ */
+struct panthor_model {
+	/** @name: Model name. */
+	const char *name;
+
+	/** @arch_major: Major version number of architecture. */
+	u8 arch_major;
+
+	/** @product_major: Major version number of product. */
+	u8 product_major;
+};
+
+/**
+ * GPU_MODEL() - Define a GPU model. A GPU product can be uniquely identified
+ * by a combination of the major architecture version and the major product
+ * version.
+ * @_name: Name for the GPU model.
+ * @_arch_major: Architecture major.
+ * @_product_major: Product major.
+ */
+#define GPU_MODEL(_name, _arch_major, _product_major) \
+{\
+	.name = __stringify(_name),				\
+	.arch_major = _arch_major,				\
+	.product_major = _product_major,			\
+}
+
+static const struct panthor_model gpu_models[] = {
+	GPU_MODEL(g610, 10, 7),
+	{},
+};
+
+#define GPU_INTERRUPTS_MASK	\
+	(GPU_IRQ_FAULT | \
+	 GPU_IRQ_PROTM_FAULT | \
+	 GPU_IRQ_RESET_COMPLETED | \
+	 GPU_IRQ_CLEAN_CACHES_COMPLETED)
+
+static void panthor_gpu_init_info(struct panthor_device *ptdev)
+{
+	const struct panthor_model *model;
+	u32 arch_major, product_major;
+	u32 major, minor, status;
+	unsigned int i;
+
+	ptdev->gpu_info.gpu_id = gpu_read(ptdev, GPU_ID);
+	ptdev->gpu_info.csf_id = gpu_read(ptdev, GPU_CSF_ID);
+	ptdev->gpu_info.gpu_rev = gpu_read(ptdev, GPU_REVID);
+	ptdev->gpu_info.core_features = gpu_read(ptdev, GPU_CORE_FEATURES);
+	ptdev->gpu_info.l2_features = gpu_read(ptdev, GPU_L2_FEATURES);
+	ptdev->gpu_info.tiler_features = gpu_read(ptdev, GPU_TILER_FEATURES);
+	ptdev->gpu_info.mem_features = gpu_read(ptdev, GPU_MEM_FEATURES);
+	ptdev->gpu_info.mmu_features = gpu_read(ptdev, GPU_MMU_FEATURES);
+	ptdev->gpu_info.thread_features = gpu_read(ptdev, GPU_THREAD_FEATURES);
+	ptdev->gpu_info.max_threads = gpu_read(ptdev, GPU_THREAD_MAX_THREADS);
+	ptdev->gpu_info.thread_max_workgroup_size = gpu_read(ptdev, GPU_THREAD_MAX_WORKGROUP_SIZE);
+	ptdev->gpu_info.thread_max_barrier_size = gpu_read(ptdev, GPU_THREAD_MAX_BARRIER_SIZE);
+	ptdev->gpu_info.coherency_features = gpu_read(ptdev, GPU_COHERENCY_FEATURES);
+	for (i = 0; i < 4; i++)
+		ptdev->gpu_info.texture_features[i] = gpu_read(ptdev, GPU_TEXTURE_FEATURES(i));
+
+	ptdev->gpu_info.as_present = gpu_read(ptdev, GPU_AS_PRESENT);
+
+	ptdev->gpu_info.shader_present = gpu_read(ptdev, GPU_SHADER_PRESENT_LO);
+	ptdev->gpu_info.shader_present |= (u64)gpu_read(ptdev, GPU_SHADER_PRESENT_HI) << 32;
+
+	ptdev->gpu_info.tiler_present = gpu_read(ptdev, GPU_TILER_PRESENT_LO);
+	ptdev->gpu_info.tiler_present |= (u64)gpu_read(ptdev, GPU_TILER_PRESENT_HI) << 32;
+
+	ptdev->gpu_info.l2_present = gpu_read(ptdev, GPU_L2_PRESENT_LO);
+	ptdev->gpu_info.l2_present |= (u64)gpu_read(ptdev, GPU_L2_PRESENT_HI) << 32;
+
+	arch_major = GPU_ARCH_MAJOR(ptdev->gpu_info.gpu_id);
+	product_major = GPU_PROD_MAJOR(ptdev->gpu_info.gpu_id);
+	major = GPU_VER_MAJOR(ptdev->gpu_info.gpu_id);
+	minor = GPU_VER_MINOR(ptdev->gpu_info.gpu_id);
+	status = GPU_VER_STATUS(ptdev->gpu_info.gpu_id);
+
+	for (model = gpu_models; model->name; model++) {
+		if (model->arch_major == arch_major &&
+		    model->product_major == product_major)
+			break;
+	}
+
+	drm_info(&ptdev->base,
+		 "mali-%s id 0x%x major 0x%x minor 0x%x status 0x%x",
+		 model->name ?: "unknown", ptdev->gpu_info.gpu_id >> 16,
+		 major, minor, status);
+
+	drm_info(&ptdev->base,
+		 "Features: L2:%#x Tiler:%#x Mem:%#x MMU:%#x AS:%#x",
+		 ptdev->gpu_info.l2_features,
+		 ptdev->gpu_info.tiler_features,
+		 ptdev->gpu_info.mem_features,
+		 ptdev->gpu_info.mmu_features,
+		 ptdev->gpu_info.as_present);
+
+	drm_info(&ptdev->base,
+		 "shader_present=0x%0llx l2_present=0x%0llx tiler_present=0x%0llx",
+		 ptdev->gpu_info.shader_present, ptdev->gpu_info.l2_present,
+		 ptdev->gpu_info.tiler_present);
+}
+
+static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status)
+{
+	if (status & GPU_IRQ_FAULT) {
+		u32 fault_status = gpu_read(ptdev, GPU_FAULT_STATUS);
+		u64 address = ((u64)gpu_read(ptdev, GPU_FAULT_ADDR_HI) << 32) |
+			      gpu_read(ptdev, GPU_FAULT_ADDR_LO);
+
+		drm_warn(&ptdev->base, "GPU Fault 0x%08x (%s) at 0x%016llx\n",
+			 fault_status, panthor_exception_name(ptdev, fault_status & 0xFF),
+			 address);
+	}
+	if (status & GPU_IRQ_PROTM_FAULT)
+		drm_warn(&ptdev->base, "GPU Fault in protected mode\n");
+
+	spin_lock(&ptdev->gpu->reqs_lock);
+	if (status & ptdev->gpu->pending_reqs) {
+		ptdev->gpu->pending_reqs &= ~status;
+		wake_up_all(&ptdev->gpu->reqs_acked);
+	}
+	spin_unlock(&ptdev->gpu->reqs_lock);
+}
+PANTHOR_IRQ_HANDLER(gpu, GPU, panthor_gpu_irq_handler);
+
+/**
+ * panthor_gpu_unplug() - Called when the GPU is unplugged.
+ * @ptdev: Device to unplug.
+ */
+void panthor_gpu_unplug(struct panthor_device *ptdev)
+{
+	unsigned long flags;
+
+	/* Make sure the IRQ handler is not running after that point. */
+	panthor_gpu_irq_suspend(&ptdev->gpu->irq);
+
+	/* Wake-up all waiters. */
+	spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
+	ptdev->gpu->pending_reqs = 0;
+	wake_up_all(&ptdev->gpu->reqs_acked);
+	spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
+}
+
+/**
+ * panthor_gpu_init() - Initialize the GPU block
+ * @ptdev: Device.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_gpu_init(struct panthor_device *ptdev)
+{
+	struct panthor_gpu *gpu;
+	u32 pa_bits;
+	int ret, irq;
+
+	gpu = drmm_kzalloc(&ptdev->base, sizeof(*gpu), GFP_KERNEL);
+	if (!gpu)
+		return -ENOMEM;
+
+	spin_lock_init(&gpu->reqs_lock);
+	init_waitqueue_head(&gpu->reqs_acked);
+	ptdev->gpu = gpu;
+	panthor_gpu_init_info(ptdev);
+
+	dma_set_max_seg_size(ptdev->base.dev, UINT_MAX);
+	pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features);
+	ret = dma_set_mask_and_coherent(ptdev->base.dev, DMA_BIT_MASK(pa_bits));
+	if (ret)
+		return ret;
+
+	irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "gpu");
+	if (irq < 0)
+		return irq;
+
+	ret = panthor_request_gpu_irq(ptdev, &ptdev->gpu->irq, irq, GPU_INTERRUPTS_MASK);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+/**
+ * panthor_gpu_block_power_off() - Power-off a specific block of the GPU
+ * @ptdev: Device.
+ * @blk_name: Block name.
+ * @pwroff_reg: Power-off register for this block.
+ * @pwrtrans_reg: Power transition register for this block.
+ * @mask: Sub-elements to power-off.
+ * @timeout_us: Timeout in microseconds.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_gpu_block_power_off(struct panthor_device *ptdev,
+				const char *blk_name,
+				u32 pwroff_reg, u32 pwrtrans_reg,
+				u64 mask, u32 timeout_us)
+{
+	u32 val, i;
+	int ret;
+
+	for (i = 0; i < 2; i++) {
+		u32 mask32 = mask >> (i * 32);
+
+		if (!mask32)
+			continue;
+
+		ret = readl_relaxed_poll_timeout(ptdev->iomem + pwrtrans_reg + (i * 4),
+						 val, !(mask32 & val),
+						 100, timeout_us);
+		if (ret) {
+			drm_err(&ptdev->base, "timeout waiting on %s:%llx power transition",
+				blk_name, mask);
+			return ret;
+		}
+	}
+
+	if (mask & GENMASK(31, 0))
+		gpu_write(ptdev, pwroff_reg, mask);
+
+	if (mask >> 32)
+		gpu_write(ptdev, pwroff_reg + 4, mask >> 32);
+
+	for (i = 0; i < 2; i++) {
+		u32 mask32 = mask >> (i * 32);
+
+		if (!mask32)
+			continue;
+
+		ret = readl_relaxed_poll_timeout(ptdev->iomem + pwrtrans_reg + (i * 4),
+						 val, !(mask32 & val),
+						 100, timeout_us);
+		if (ret) {
+			drm_err(&ptdev->base, "timeout waiting on %s:%llx power transition",
+				blk_name, mask);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_gpu_block_power_on() - Power-on a specific block of the GPU
+ * @ptdev: Device.
+ * @blk_name: Block name.
+ * @pwron_reg: Power-on register for this block.
+ * @pwrtrans_reg: Power transition register for this block.
+ * @rdy_reg: Power transition ready register.
+ * @mask: Sub-elements to power-on.
+ * @timeout_us: Timeout in microseconds.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_gpu_block_power_on(struct panthor_device *ptdev,
+			       const char *blk_name,
+			       u32 pwron_reg, u32 pwrtrans_reg,
+			       u32 rdy_reg, u64 mask, u32 timeout_us)
+{
+	u32 val, i;
+	int ret;
+
+	for (i = 0; i < 2; i++) {
+		u32 mask32 = mask >> (i * 32);
+
+		if (!mask32)
+			continue;
+
+		ret = readl_relaxed_poll_timeout(ptdev->iomem + pwrtrans_reg + (i * 4),
+						 val, !(mask32 & val),
+						 100, timeout_us);
+		if (ret) {
+			drm_err(&ptdev->base, "timeout waiting on %s:%llx power transition",
+				blk_name, mask);
+			return ret;
+		}
+	}
+
+	if (mask & GENMASK(31, 0))
+		gpu_write(ptdev, pwron_reg, mask);
+
+	if (mask >> 32)
+		gpu_write(ptdev, pwron_reg + 4, mask >> 32);
+
+	for (i = 0; i < 2; i++) {
+		u32 mask32 = mask >> (i * 32);
+
+		if (!mask32)
+			continue;
+
+		ret = readl_relaxed_poll_timeout(ptdev->iomem + rdy_reg + (i * 4),
+						 val, (mask32 & val) == mask32,
+						 100, timeout_us);
+		if (ret) {
+			drm_err(&ptdev->base, "timeout waiting on %s:%llx readiness",
+				blk_name, mask);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_gpu_l2_power_on() - Power-on the L2-cache
+ * @ptdev: Device.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_gpu_l2_power_on(struct panthor_device *ptdev)
+{
+	if (ptdev->gpu_info.l2_present != 1) {
+		/*
+		 * Only support one core group now.
+		 * ~(l2_present - 1) unsets all bits in l2_present except
+		 * the bottom bit. (l2_present - 2) has all the bits in
+		 * the first core group set. AND them together to generate
+		 * a mask of cores in the first core group.
+		 */
+		u64 core_mask = ~(ptdev->gpu_info.l2_present - 1) &
+				(ptdev->gpu_info.l2_present - 2);
+		drm_info_once(&ptdev->base, "using only 1st core group (%lu cores from %lu)\n",
+			      hweight64(core_mask),
+			      hweight64(ptdev->gpu_info.shader_present));
+	}
+
+	return panthor_gpu_power_on(ptdev, L2, 1, 20000);
+}
+
+/**
+ * panthor_gpu_flush_caches() - Flush caches
+ * @ptdev: Device.
+ * @l2: L2 flush type.
+ * @lsc: LSC flush type.
+ * @other: Other flush type.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_gpu_flush_caches(struct panthor_device *ptdev,
+			     u32 l2, u32 lsc, u32 other)
+{
+	bool timedout = false;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
+	if (!drm_WARN_ON(&ptdev->base,
+			 ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) {
+		ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED;
+		gpu_write(ptdev, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other));
+	}
+	spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
+
+	if (!wait_event_timeout(ptdev->gpu->reqs_acked,
+				!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED),
+				msecs_to_jiffies(100))) {
+		spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
+		if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 &&
+		    !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED))
+			timedout = true;
+		else
+			ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED;
+		spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
+	}
+
+	if (timedout) {
+		drm_err(&ptdev->base, "Flush caches timeout");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_gpu_soft_reset() - Issue a soft-reset
+ * @ptdev: Device.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_gpu_soft_reset(struct panthor_device *ptdev)
+{
+	bool timedout = false;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
+	if (!drm_WARN_ON(&ptdev->base,
+			 ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED)) {
+		ptdev->gpu->pending_reqs |= GPU_IRQ_RESET_COMPLETED;
+		gpu_write(ptdev, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED);
+		gpu_write(ptdev, GPU_CMD, GPU_SOFT_RESET);
+	}
+	spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
+
+	if (!wait_event_timeout(ptdev->gpu->reqs_acked,
+				!(ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED),
+				msecs_to_jiffies(100))) {
+		spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
+		if ((ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED) != 0 &&
+		    !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_RESET_COMPLETED))
+			timedout = true;
+		else
+			ptdev->gpu->pending_reqs &= ~GPU_IRQ_RESET_COMPLETED;
+		spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
+	}
+
+	if (timedout) {
+		drm_err(&ptdev->base, "Soft reset timeout");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_gpu_suspend() - Suspend the GPU block.
+ * @ptdev: Device.
+ *
+ * Suspend the GPU irq. This should be called last in the suspend procedure,
+ * after all other blocks have been suspented.
+ */
+void panthor_gpu_suspend(struct panthor_device *ptdev)
+{
+	/*
+	 * It may be preferable to simply power down the L2, but for now just
+	 * soft-reset which will leave the L2 powered down.
+	 */
+	panthor_gpu_soft_reset(ptdev);
+	panthor_gpu_irq_suspend(&ptdev->gpu->irq);
+}
+
+/**
+ * panthor_gpu_resume() - Resume the GPU block.
+ * @ptdev: Device.
+ *
+ * Resume the IRQ handler and power-on the L2-cache.
+ * The FW takes care of powering the other blocks.
+ */
+void panthor_gpu_resume(struct panthor_device *ptdev)
+{
+	panthor_gpu_irq_resume(&ptdev->gpu->irq, GPU_INTERRUPTS_MASK);
+	panthor_gpu_l2_power_on(ptdev);
+}
diff --git a/drivers/gpu/drm/panthor/panthor_gpu.h b/drivers/gpu/drm/panthor/panthor_gpu.h
new file mode 100644
index 000000000000..bba7555dd3c6
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_gpu.h
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
+/* Copyright 2019 Collabora ltd. */
+
+#ifndef __PANTHOR_GPU_H__
+#define __PANTHOR_GPU_H__
+
+struct panthor_device;
+
+int panthor_gpu_init(struct panthor_device *ptdev);
+void panthor_gpu_unplug(struct panthor_device *ptdev);
+void panthor_gpu_suspend(struct panthor_device *ptdev);
+void panthor_gpu_resume(struct panthor_device *ptdev);
+
+int panthor_gpu_block_power_on(struct panthor_device *ptdev,
+			       const char *blk_name,
+			       u32 pwron_reg, u32 pwrtrans_reg,
+			       u32 rdy_reg, u64 mask, u32 timeout_us);
+int panthor_gpu_block_power_off(struct panthor_device *ptdev,
+				const char *blk_name,
+				u32 pwroff_reg, u32 pwrtrans_reg,
+				u64 mask, u32 timeout_us);
+
+/**
+ * panthor_gpu_power_on() - Power on the GPU block.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+#define panthor_gpu_power_on(ptdev, type, mask, timeout_us) \
+	panthor_gpu_block_power_on(ptdev, #type, \
+				  type ## _PWRON_LO, \
+				  type ## _PWRTRANS_LO, \
+				  type ## _READY_LO, \
+				  mask, timeout_us)
+
+/**
+ * panthor_gpu_power_off() - Power off the GPU block.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+#define panthor_gpu_power_off(ptdev, type, mask, timeout_us) \
+	panthor_gpu_block_power_off(ptdev, #type, \
+				   type ## _PWROFF_LO, \
+				   type ## _PWRTRANS_LO, \
+				   mask, timeout_us)
+
+int panthor_gpu_l2_power_on(struct panthor_device *ptdev);
+int panthor_gpu_flush_caches(struct panthor_device *ptdev,
+			     u32 l2, u32 lsc, u32 other);
+int panthor_gpu_soft_reset(struct panthor_device *ptdev);
+
+#endif
diff --git a/drivers/gpu/drm/panthor/panthor_heap.c b/drivers/gpu/drm/panthor/panthor_heap.c
new file mode 100644
index 000000000000..3796a9eb22af
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_heap.c
@@ -0,0 +1,605 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/* Copyright 2023 Collabora ltd. */
+
+#include <linux/iosys-map.h>
+#include <linux/rwsem.h>
+
+#include <drm/panthor_drm.h>
+
+#include "panthor_device.h"
+#include "panthor_gem.h"
+#include "panthor_heap.h"
+#include "panthor_mmu.h"
+#include "panthor_regs.h"
+
+/*
+ * The GPU heap context is an opaque structure used by the GPU to track the
+ * heap allocations. The driver should only touch it to initialize it (zero all
+ * fields). Because the CPU and GPU can both access this structure it is
+ * required to be GPU cache line aligned.
+ */
+#define HEAP_CONTEXT_SIZE	32
+
+/**
+ * struct panthor_heap_chunk_header - Heap chunk header
+ */
+struct panthor_heap_chunk_header {
+	/**
+	 * @next: Next heap chunk in the list.
+	 *
+	 * This is a GPU VA.
+	 */
+	u64 next;
+
+	/** @unknown: MBZ. */
+	u32 unknown[14];
+};
+
+/**
+ * struct panthor_heap_chunk - Structure used to keep track of allocated heap chunks.
+ */
+struct panthor_heap_chunk {
+	/** @node: Used to insert the heap chunk in panthor_heap::chunks. */
+	struct list_head node;
+
+	/** @bo: Buffer object backing the heap chunk. */
+	struct panthor_kernel_bo *bo;
+};
+
+/**
+ * struct panthor_heap - Structure used to manage tiler heap contexts.
+ */
+struct panthor_heap {
+	/** @chunks: List containing all heap chunks allocated so far. */
+	struct list_head chunks;
+
+	/** @lock: Lock protecting insertion in the chunks list. */
+	struct mutex lock;
+
+	/** @chunk_size: Size of each chunk. */
+	u32 chunk_size;
+
+	/** @max_chunks: Maximum number of chunks. */
+	u32 max_chunks;
+
+	/**
+	 * @target_in_flight: Number of in-flight render passes after which
+	 * we'd let the FW wait for fragment job to finish instead of allocating new chunks.
+	 */
+	u32 target_in_flight;
+
+	/** @chunk_count: Number of heap chunks currently allocated. */
+	u32 chunk_count;
+};
+
+#define MAX_HEAPS_PER_POOL    128
+
+/**
+ * struct panthor_heap_pool - Pool of heap contexts
+ *
+ * The pool is attached to a panthor_file and can't be shared across processes.
+ */
+struct panthor_heap_pool {
+	/** @refcount: Reference count. */
+	struct kref refcount;
+
+	/** @ptdev: Device. */
+	struct panthor_device *ptdev;
+
+	/** @vm: VM this pool is bound to. */
+	struct panthor_vm *vm;
+
+	/** @lock: Lock protecting access to @xa. */
+	struct rw_semaphore lock;
+
+	/** @xa: Array storing panthor_heap objects. */
+	struct xarray xa;
+
+	/** @gpu_contexts: Buffer object containing the GPU heap contexts. */
+	struct panthor_kernel_bo *gpu_contexts;
+};
+
+static int panthor_heap_ctx_stride(struct panthor_device *ptdev)
+{
+	u32 l2_features = ptdev->gpu_info.l2_features;
+	u32 gpu_cache_line_size = GPU_L2_FEATURES_LINE_SIZE(l2_features);
+
+	return ALIGN(HEAP_CONTEXT_SIZE, gpu_cache_line_size);
+}
+
+static int panthor_get_heap_ctx_offset(struct panthor_heap_pool *pool, int id)
+{
+	return panthor_heap_ctx_stride(pool->ptdev) * id;
+}
+
+static void *panthor_get_heap_ctx(struct panthor_heap_pool *pool, int id)
+{
+	return pool->gpu_contexts->kmap +
+	       panthor_get_heap_ctx_offset(pool, id);
+}
+
+static void panthor_free_heap_chunk(struct panthor_vm *vm,
+				    struct panthor_heap *heap,
+				    struct panthor_heap_chunk *chunk)
+{
+	mutex_lock(&heap->lock);
+	list_del(&chunk->node);
+	heap->chunk_count--;
+	mutex_unlock(&heap->lock);
+
+	panthor_kernel_bo_destroy(chunk->bo);
+	kfree(chunk);
+}
+
+static int panthor_alloc_heap_chunk(struct panthor_device *ptdev,
+				    struct panthor_vm *vm,
+				    struct panthor_heap *heap,
+				    bool initial_chunk)
+{
+	struct panthor_heap_chunk *chunk;
+	struct panthor_heap_chunk_header *hdr;
+	int ret;
+
+	chunk = kmalloc(sizeof(*chunk), GFP_KERNEL);
+	if (!chunk)
+		return -ENOMEM;
+
+	chunk->bo = panthor_kernel_bo_create(ptdev, vm, heap->chunk_size,
+					     DRM_PANTHOR_BO_NO_MMAP,
+					     DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC,
+					     PANTHOR_VM_KERNEL_AUTO_VA);
+	if (IS_ERR(chunk->bo)) {
+		ret = PTR_ERR(chunk->bo);
+		goto err_free_chunk;
+	}
+
+	ret = panthor_kernel_bo_vmap(chunk->bo);
+	if (ret)
+		goto err_destroy_bo;
+
+	hdr = chunk->bo->kmap;
+	memset(hdr, 0, sizeof(*hdr));
+
+	if (initial_chunk && !list_empty(&heap->chunks)) {
+		struct panthor_heap_chunk *prev_chunk;
+		u64 prev_gpuva;
+
+		prev_chunk = list_first_entry(&heap->chunks,
+					      struct panthor_heap_chunk,
+					      node);
+
+		prev_gpuva = panthor_kernel_bo_gpuva(prev_chunk->bo);
+		hdr->next = (prev_gpuva & GENMASK_ULL(63, 12)) |
+			    (heap->chunk_size >> 12);
+	}
+
+	panthor_kernel_bo_vunmap(chunk->bo);
+
+	mutex_lock(&heap->lock);
+	list_add(&chunk->node, &heap->chunks);
+	heap->chunk_count++;
+	mutex_unlock(&heap->lock);
+
+	return 0;
+
+err_destroy_bo:
+	panthor_kernel_bo_destroy(chunk->bo);
+
+err_free_chunk:
+	kfree(chunk);
+
+	return ret;
+}
+
+static void panthor_free_heap_chunks(struct panthor_vm *vm,
+				     struct panthor_heap *heap)
+{
+	struct panthor_heap_chunk *chunk, *tmp;
+
+	list_for_each_entry_safe(chunk, tmp, &heap->chunks, node)
+		panthor_free_heap_chunk(vm, heap, chunk);
+}
+
+static int panthor_alloc_heap_chunks(struct panthor_device *ptdev,
+				     struct panthor_vm *vm,
+				     struct panthor_heap *heap,
+				     u32 chunk_count)
+{
+	int ret;
+	u32 i;
+
+	for (i = 0; i < chunk_count; i++) {
+		ret = panthor_alloc_heap_chunk(ptdev, vm, heap, true);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int
+panthor_heap_destroy_locked(struct panthor_heap_pool *pool, u32 handle)
+{
+	struct panthor_heap *heap;
+
+	heap = xa_erase(&pool->xa, handle);
+	if (!heap)
+		return -EINVAL;
+
+	panthor_free_heap_chunks(pool->vm, heap);
+	mutex_destroy(&heap->lock);
+	kfree(heap);
+	return 0;
+}
+
+/**
+ * panthor_heap_destroy() - Destroy a heap context
+ * @pool: Pool this context belongs to.
+ * @handle: Handle returned by panthor_heap_create().
+ */
+int panthor_heap_destroy(struct panthor_heap_pool *pool, u32 handle)
+{
+	int ret;
+
+	down_write(&pool->lock);
+	ret = panthor_heap_destroy_locked(pool, handle);
+	up_write(&pool->lock);
+
+	return ret;
+}
+
+/**
+ * panthor_heap_create() - Create a heap context
+ * @pool: Pool to instantiate the heap context from.
+ * @initial_chunk_count: Number of chunk allocated at initialization time.
+ * Must be at least 1.
+ * @chunk_size: The size of each chunk. Must be page-aligned and lie in the
+ * [128k:8M] range.
+ * @max_chunks: Maximum number of chunks that can be allocated.
+ * @target_in_flight: Maximum number of in-flight render passes.
+ * @heap_ctx_gpu_va: Pointer holding the GPU address of the allocated heap
+ * context.
+ * @first_chunk_gpu_va: Pointer holding the GPU address of the first chunk
+ * assigned to the heap context.
+ *
+ * Return: a positive handle on success, a negative error otherwise.
+ */
+int panthor_heap_create(struct panthor_heap_pool *pool,
+			u32 initial_chunk_count,
+			u32 chunk_size,
+			u32 max_chunks,
+			u32 target_in_flight,
+			u64 *heap_ctx_gpu_va,
+			u64 *first_chunk_gpu_va)
+{
+	struct panthor_heap *heap;
+	struct panthor_heap_chunk *first_chunk;
+	struct panthor_vm *vm;
+	int ret = 0;
+	u32 id;
+
+	if (initial_chunk_count == 0)
+		return -EINVAL;
+
+	if (initial_chunk_count > max_chunks)
+		return -EINVAL;
+
+	if (!IS_ALIGNED(chunk_size, PAGE_SIZE) ||
+	    chunk_size < SZ_128K || chunk_size > SZ_8M)
+		return -EINVAL;
+
+	down_read(&pool->lock);
+	vm = panthor_vm_get(pool->vm);
+	up_read(&pool->lock);
+
+	/* The pool has been destroyed, we can't create a new heap. */
+	if (!vm)
+		return -EINVAL;
+
+	heap = kzalloc(sizeof(*heap), GFP_KERNEL);
+	if (!heap) {
+		ret = -ENOMEM;
+		goto err_put_vm;
+	}
+
+	mutex_init(&heap->lock);
+	INIT_LIST_HEAD(&heap->chunks);
+	heap->chunk_size = chunk_size;
+	heap->max_chunks = max_chunks;
+	heap->target_in_flight = target_in_flight;
+
+	ret = panthor_alloc_heap_chunks(pool->ptdev, vm, heap,
+					initial_chunk_count);
+	if (ret)
+		goto err_free_heap;
+
+	first_chunk = list_first_entry(&heap->chunks,
+				       struct panthor_heap_chunk,
+				       node);
+	*first_chunk_gpu_va = panthor_kernel_bo_gpuva(first_chunk->bo);
+
+	down_write(&pool->lock);
+	/* The pool has been destroyed, we can't create a new heap. */
+	if (!pool->vm) {
+		ret = -EINVAL;
+	} else {
+		ret = xa_alloc(&pool->xa, &id, heap,
+			       XA_LIMIT(0, MAX_HEAPS_PER_POOL - 1), GFP_KERNEL);
+		if (!ret) {
+			void *gpu_ctx = panthor_get_heap_ctx(pool, id);
+
+			memset(gpu_ctx, 0, panthor_heap_ctx_stride(pool->ptdev));
+			*heap_ctx_gpu_va = panthor_kernel_bo_gpuva(pool->gpu_contexts) +
+					   panthor_get_heap_ctx_offset(pool, id);
+		}
+	}
+	up_write(&pool->lock);
+
+	if (ret)
+		goto err_free_heap;
+
+	panthor_vm_put(vm);
+	return id;
+
+err_free_heap:
+	panthor_free_heap_chunks(pool->vm, heap);
+	mutex_destroy(&heap->lock);
+	kfree(heap);
+
+err_put_vm:
+	panthor_vm_put(vm);
+	return ret;
+}
+
+/**
+ * panthor_heap_return_chunk() - Return an unused heap chunk
+ * @pool: The pool this heap belongs to.
+ * @heap_gpu_va: The GPU address of the heap context.
+ * @chunk_gpu_va: The chunk VA to return.
+ *
+ * This function is used when a chunk allocated with panthor_heap_grow()
+ * couldn't be linked to the heap context through the FW interface because
+ * the group requesting the allocation was scheduled out in the meantime.
+ */
+int panthor_heap_return_chunk(struct panthor_heap_pool *pool,
+			      u64 heap_gpu_va,
+			      u64 chunk_gpu_va)
+{
+	u64 offset = heap_gpu_va - panthor_kernel_bo_gpuva(pool->gpu_contexts);
+	u32 heap_id = (u32)offset / panthor_heap_ctx_stride(pool->ptdev);
+	struct panthor_heap_chunk *chunk, *tmp, *removed = NULL;
+	struct panthor_heap *heap;
+	int ret;
+
+	if (offset > U32_MAX || heap_id >= MAX_HEAPS_PER_POOL)
+		return -EINVAL;
+
+	down_read(&pool->lock);
+	heap = xa_load(&pool->xa, heap_id);
+	if (!heap) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
+	chunk_gpu_va &= GENMASK_ULL(63, 12);
+
+	mutex_lock(&heap->lock);
+	list_for_each_entry_safe(chunk, tmp, &heap->chunks, node) {
+		if (panthor_kernel_bo_gpuva(chunk->bo) == chunk_gpu_va) {
+			removed = chunk;
+			list_del(&chunk->node);
+			heap->chunk_count--;
+			break;
+		}
+	}
+	mutex_unlock(&heap->lock);
+
+	if (removed) {
+		panthor_kernel_bo_destroy(chunk->bo);
+		kfree(chunk);
+		ret = 0;
+	} else {
+		ret = -EINVAL;
+	}
+
+out_unlock:
+	up_read(&pool->lock);
+	return ret;
+}
+
+/**
+ * panthor_heap_grow() - Make a heap context grow.
+ * @pool: The pool this heap belongs to.
+ * @heap_gpu_va: The GPU address of the heap context.
+ * @renderpasses_in_flight: Number of render passes currently in-flight.
+ * @pending_frag_count: Number of fragment jobs waiting for execution/completion.
+ * @new_chunk_gpu_va: Pointer used to return the chunk VA.
+ *
+ * Return:
+ * - 0 if a new heap was allocated
+ * - -ENOMEM if the tiler context reached the maximum number of chunks
+ *   or if too many render passes are in-flight
+ *   or if the allocation failed
+ * - -EINVAL if any of the arguments passed to panthor_heap_grow() is invalid
+ */
+int panthor_heap_grow(struct panthor_heap_pool *pool,
+		      u64 heap_gpu_va,
+		      u32 renderpasses_in_flight,
+		      u32 pending_frag_count,
+		      u64 *new_chunk_gpu_va)
+{
+	u64 offset = heap_gpu_va - panthor_kernel_bo_gpuva(pool->gpu_contexts);
+	u32 heap_id = (u32)offset / panthor_heap_ctx_stride(pool->ptdev);
+	struct panthor_heap_chunk *chunk;
+	struct panthor_heap *heap;
+	int ret;
+
+	if (offset > U32_MAX || heap_id >= MAX_HEAPS_PER_POOL)
+		return -EINVAL;
+
+	down_read(&pool->lock);
+	heap = xa_load(&pool->xa, heap_id);
+	if (!heap) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
+	/* If we reached the target in-flight render passes, or if we
+	 * reached the maximum number of chunks, let the FW figure another way to
+	 * find some memory (wait for render passes to finish, or call the exception
+	 * handler provided by the userspace driver, if any).
+	 */
+	if (renderpasses_in_flight > heap->target_in_flight ||
+	    heap->chunk_count >= heap->max_chunks) {
+		ret = -ENOMEM;
+		goto out_unlock;
+	}
+
+	/* FIXME: panthor_alloc_heap_chunk() triggers a kernel BO creation,
+	 * which goes through the blocking allocation path. Ultimately, we
+	 * want a non-blocking allocation, so we can immediately report to the
+	 * FW when the system is running out of memory. In that case, the FW
+	 * can call a user-provided exception handler, which might try to free
+	 * some tiler memory by issuing an intermediate fragment job. If the
+	 * exception handler can't do anything, it will flag the queue as
+	 * faulty so the job that triggered this tiler chunk allocation and all
+	 * further jobs in this queue fail immediately instead of having to
+	 * wait for the job timeout.
+	 */
+	ret = panthor_alloc_heap_chunk(pool->ptdev, pool->vm, heap, false);
+	if (ret)
+		goto out_unlock;
+
+	chunk = list_first_entry(&heap->chunks,
+				 struct panthor_heap_chunk,
+				 node);
+	*new_chunk_gpu_va = (panthor_kernel_bo_gpuva(chunk->bo) & GENMASK_ULL(63, 12)) |
+			    (heap->chunk_size >> 12);
+	ret = 0;
+
+out_unlock:
+	up_read(&pool->lock);
+	return ret;
+}
+
+static void panthor_heap_pool_release(struct kref *refcount)
+{
+	struct panthor_heap_pool *pool =
+		container_of(refcount, struct panthor_heap_pool, refcount);
+
+	xa_destroy(&pool->xa);
+	kfree(pool);
+}
+
+/**
+ * panthor_heap_pool_put() - Release a heap pool reference
+ * @pool: Pool to release the reference on. Can be NULL.
+ */
+void panthor_heap_pool_put(struct panthor_heap_pool *pool)
+{
+	if (pool)
+		kref_put(&pool->refcount, panthor_heap_pool_release);
+}
+
+/**
+ * panthor_heap_pool_get() - Get a heap pool reference
+ * @pool: Pool to get the reference on. Can be NULL.
+ *
+ * Return: @pool.
+ */
+struct panthor_heap_pool *
+panthor_heap_pool_get(struct panthor_heap_pool *pool)
+{
+	if (pool)
+		kref_get(&pool->refcount);
+
+	return pool;
+}
+
+/**
+ * panthor_heap_pool_create() - Create a heap pool
+ * @ptdev: Device.
+ * @vm: The VM this heap pool will be attached to.
+ *
+ * Heap pools might contain up to 128 heap contexts, and are per-VM.
+ *
+ * Return: A valid pointer on success, a negative error code otherwise.
+ */
+struct panthor_heap_pool *
+panthor_heap_pool_create(struct panthor_device *ptdev, struct panthor_vm *vm)
+{
+	size_t bosize = ALIGN(MAX_HEAPS_PER_POOL *
+			      panthor_heap_ctx_stride(ptdev),
+			      4096);
+	struct panthor_heap_pool *pool;
+	int ret = 0;
+
+	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
+	if (!pool)
+		return ERR_PTR(-ENOMEM);
+
+	/* We want a weak ref here: the heap pool belongs to the VM, so we're
+	 * sure that, as long as the heap pool exists, the VM exists too.
+	 */
+	pool->vm = vm;
+	pool->ptdev = ptdev;
+	init_rwsem(&pool->lock);
+	xa_init_flags(&pool->xa, XA_FLAGS_ALLOC);
+	kref_init(&pool->refcount);
+
+	pool->gpu_contexts = panthor_kernel_bo_create(ptdev, vm, bosize,
+						      DRM_PANTHOR_BO_NO_MMAP,
+						      DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC,
+						      PANTHOR_VM_KERNEL_AUTO_VA);
+	if (IS_ERR(pool->gpu_contexts)) {
+		ret = PTR_ERR(pool->gpu_contexts);
+		goto err_destroy_pool;
+	}
+
+	ret = panthor_kernel_bo_vmap(pool->gpu_contexts);
+	if (ret)
+		goto err_destroy_pool;
+
+	return pool;
+
+err_destroy_pool:
+	panthor_heap_pool_destroy(pool);
+	return ERR_PTR(ret);
+}
+
+/**
+ * panthor_heap_pool_destroy() - Destroy a heap pool.
+ * @pool: Pool to destroy.
+ *
+ * This function destroys all heap contexts and their resources. Thus
+ * preventing any use of the heap context or the chunk attached to them
+ * after that point.
+ *
+ * If the GPU still has access to some heap contexts, a fault should be
+ * triggered, which should flag the command stream groups using these
+ * context as faulty.
+ *
+ * The heap pool object is only released when all references to this pool
+ * are released.
+ */
+void panthor_heap_pool_destroy(struct panthor_heap_pool *pool)
+{
+	struct panthor_heap *heap;
+	unsigned long i;
+
+	if (!pool)
+		return;
+
+	down_write(&pool->lock);
+	xa_for_each(&pool->xa, i, heap)
+		drm_WARN_ON(&pool->ptdev->base, panthor_heap_destroy_locked(pool, i));
+
+	if (!IS_ERR_OR_NULL(pool->gpu_contexts))
+		panthor_kernel_bo_destroy(pool->gpu_contexts);
+
+	/* Reflects the fact the pool has been destroyed. */
+	pool->vm = NULL;
+	up_write(&pool->lock);
+
+	panthor_heap_pool_put(pool);
+}
diff --git a/drivers/gpu/drm/panthor/panthor_heap.h b/drivers/gpu/drm/panthor/panthor_heap.h
new file mode 100644
index 000000000000..25a5f2bba445
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_heap.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2023 Collabora ltd. */
+
+#ifndef __PANTHOR_HEAP_H__
+#define __PANTHOR_HEAP_H__
+
+#include <linux/types.h>
+
+struct panthor_device;
+struct panthor_heap_pool;
+struct panthor_vm;
+
+int panthor_heap_create(struct panthor_heap_pool *pool,
+			u32 initial_chunk_count,
+			u32 chunk_size,
+			u32 max_chunks,
+			u32 target_in_flight,
+			u64 *heap_ctx_gpu_va,
+			u64 *first_chunk_gpu_va);
+int panthor_heap_destroy(struct panthor_heap_pool *pool, u32 handle);
+
+struct panthor_heap_pool *
+panthor_heap_pool_create(struct panthor_device *ptdev, struct panthor_vm *vm);
+void panthor_heap_pool_destroy(struct panthor_heap_pool *pool);
+
+struct panthor_heap_pool *
+panthor_heap_pool_get(struct panthor_heap_pool *pool);
+void panthor_heap_pool_put(struct panthor_heap_pool *pool);
+
+int panthor_heap_grow(struct panthor_heap_pool *pool,
+		      u64 heap_gpu_va,
+		      u32 renderpasses_in_flight,
+		      u32 pending_frag_count,
+		      u64 *new_chunk_gpu_va);
+int panthor_heap_return_chunk(struct panthor_heap_pool *pool,
+			      u64 heap_gpu_va,
+			      u64 chunk_gpu_va);
+
+#endif
diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c
new file mode 100644
index 000000000000..fa0a002b1016
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_mmu.c
@@ -0,0 +1,2774 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+/* Copyright 2023 Collabora ltd. */
+
+#include <drm/drm_debugfs.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_exec.h>
+#include <drm/drm_gpuvm.h>
+#include <drm/drm_managed.h>
+#include <drm/gpu_scheduler.h>
+#include <drm/panthor_drm.h>
+
+#include <linux/atomic.h>
+#include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/io-pgtable.h>
+#include <linux/iommu.h>
+#include <linux/kmemleak.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/rwsem.h>
+#include <linux/sched.h>
+#include <linux/shmem_fs.h>
+#include <linux/sizes.h>
+
+#include "panthor_device.h"
+#include "panthor_gem.h"
+#include "panthor_heap.h"
+#include "panthor_mmu.h"
+#include "panthor_regs.h"
+#include "panthor_sched.h"
+
+#define MAX_AS_SLOTS			32
+
+struct panthor_vm;
+
+/**
+ * struct panthor_as_slot - Address space slot
+ */
+struct panthor_as_slot {
+	/** @vm: VM bound to this slot. NULL is no VM is bound. */
+	struct panthor_vm *vm;
+};
+
+/**
+ * struct panthor_mmu - MMU related data
+ */
+struct panthor_mmu {
+	/** @irq: The MMU irq. */
+	struct panthor_irq irq;
+
+	/** @as: Address space related fields.
+	 *
+	 * The GPU has a limited number of address spaces (AS) slots, forcing
+	 * us to re-assign them to re-assign slots on-demand.
+	 */
+	struct {
+		/** @slots_lock: Lock protecting access to all other AS fields. */
+		struct mutex slots_lock;
+
+		/** @alloc_mask: Bitmask encoding the allocated slots. */
+		unsigned long alloc_mask;
+
+		/** @faulty_mask: Bitmask encoding the faulty slots. */
+		unsigned long faulty_mask;
+
+		/** @slots: VMs currently bound to the AS slots. */
+		struct panthor_as_slot slots[MAX_AS_SLOTS];
+
+		/**
+		 * @lru_list: List of least recently used VMs.
+		 *
+		 * We use this list to pick a VM to evict when all slots are
+		 * used.
+		 *
+		 * There should be no more active VMs than there are AS slots,
+		 * so this LRU is just here to keep VMs bound until there's
+		 * a need to release a slot, thus avoid unnecessary TLB/cache
+		 * flushes.
+		 */
+		struct list_head lru_list;
+	} as;
+
+	/** @vm: VMs management fields */
+	struct {
+		/** @lock: Lock protecting access to list. */
+		struct mutex lock;
+
+		/** @list: List containing all VMs. */
+		struct list_head list;
+
+		/** @reset_in_progress: True if a reset is in progress. */
+		bool reset_in_progress;
+
+		/** @wq: Workqueue used for the VM_BIND queues. */
+		struct workqueue_struct *wq;
+	} vm;
+};
+
+/**
+ * struct panthor_vm_pool - VM pool object
+ */
+struct panthor_vm_pool {
+	/** @xa: Array used for VM handle tracking. */
+	struct xarray xa;
+};
+
+/**
+ * struct panthor_vma - GPU mapping object
+ *
+ * This is used to track GEM mappings in GPU space.
+ */
+struct panthor_vma {
+	/** @base: Inherits from drm_gpuva. */
+	struct drm_gpuva base;
+
+	/** @node: Used to implement deferred release of VMAs. */
+	struct list_head node;
+
+	/**
+	 * @flags: Combination of drm_panthor_vm_bind_op_flags.
+	 *
+	 * Only map related flags are accepted.
+	 */
+	u32 flags;
+};
+
+/**
+ * struct panthor_vm_op_ctx - VM operation context
+ *
+ * With VM operations potentially taking place in a dma-signaling path, we
+ * need to make sure everything that might require resource allocation is
+ * pre-allocated upfront. This is what this operation context is far.
+ *
+ * We also collect resources that have been freed, so we can release them
+ * asynchronously, and let the VM_BIND scheduler process the next VM_BIND
+ * request.
+ */
+struct panthor_vm_op_ctx {
+	/** @rsvd_page_tables: Pages reserved for the MMU page table update. */
+	struct {
+		/** @count: Number of pages reserved. */
+		u32 count;
+
+		/** @ptr: Point to the first unused page in the @pages table. */
+		u32 ptr;
+
+		/**
+		 * @page: Array of pages that can be used for an MMU page table update.
+		 *
+		 * After an VM operation, there might be free pages left in this array.
+		 * They should be returned to the pt_cache as part of the op_ctx cleanup.
+		 */
+		void **pages;
+	} rsvd_page_tables;
+
+	/**
+	 * @preallocated_vmas: Pre-allocated VMAs to handle the remap case.
+	 *
+	 * Partial unmap requests or map requests overlapping existing mappings will
+	 * trigger a remap call, which need to register up to three panthor_vma objects
+	 * (one for the new mapping, and two for the previous and next mappings).
+	 */
+	struct panthor_vma *preallocated_vmas[3];
+
+	/** @flags: Combination of drm_panthor_vm_bind_op_flags. */
+	u32 flags;
+
+	/** @va: Virtual range targeted by the VM operation. */
+	struct {
+		/** @addr: Start address. */
+		u64 addr;
+
+		/** @range: Range size. */
+		u64 range;
+	} va;
+
+	/**
+	 * @returned_vmas: List of panthor_vma objects returned after a VM operation.
+	 *
+	 * For unmap operations, this will contain all VMAs that were covered by the
+	 * specified VA range.
+	 *
+	 * For map operations, this will contain all VMAs that previously mapped to
+	 * the specified VA range.
+	 *
+	 * Those VMAs, and the resources they point to will be released as part of
+	 * the op_ctx cleanup operation.
+	 */
+	struct list_head returned_vmas;
+
+	/** @map: Fields specific to a map operation. */
+	struct {
+		/** @vm_bo: Buffer object to map. */
+		struct drm_gpuvm_bo *vm_bo;
+
+		/** @bo_offset: Offset in the buffer object. */
+		u64 bo_offset;
+
+		/**
+		 * @sgt: sg-table pointing to pages backing the GEM object.
+		 *
+		 * This is gathered at job creation time, such that we don't have
+		 * to allocate in ::run_job().
+		 */
+		struct sg_table *sgt;
+
+		/**
+		 * @new_vma: The new VMA object that will be inserted to the VA tree.
+		 */
+		struct panthor_vma *new_vma;
+	} map;
+};
+
+/**
+ * struct panthor_vm - VM object
+ *
+ * A VM is an object representing a GPU (or MCU) virtual address space.
+ * It embeds the MMU page table for this address space, a tree containing
+ * all the virtual mappings of GEM objects, and other things needed to manage
+ * the VM.
+ *
+ * Except for the MCU VM, which is managed by the kernel, all other VMs are
+ * created by userspace and mostly managed by userspace, using the
+ * %DRM_IOCTL_PANTHOR_VM_BIND ioctl.
+ *
+ * A portion of the virtual address space is reserved for kernel objects,
+ * like heap chunks, and userspace gets to decide how much of the virtual
+ * address space is left to the kernel (half of the virtual address space
+ * by default).
+ */
+struct panthor_vm {
+	/**
+	 * @base: Inherit from drm_gpuvm.
+	 *
+	 * We delegate all the VA management to the common drm_gpuvm framework
+	 * and only implement hooks to update the MMU page table.
+	 */
+	struct drm_gpuvm base;
+
+	/**
+	 * @sched: Scheduler used for asynchronous VM_BIND request.
+	 *
+	 * We use a 1:1 scheduler here.
+	 */
+	struct drm_gpu_scheduler sched;
+
+	/**
+	 * @entity: Scheduling entity representing the VM_BIND queue.
+	 *
+	 * There's currently one bind queue per VM. It doesn't make sense to
+	 * allow more given the VM operations are serialized anyway.
+	 */
+	struct drm_sched_entity entity;
+
+	/** @ptdev: Device. */
+	struct panthor_device *ptdev;
+
+	/** @memattr: Value to program to the AS_MEMATTR register. */
+	u64 memattr;
+
+	/** @pgtbl_ops: Page table operations. */
+	struct io_pgtable_ops *pgtbl_ops;
+
+	/** @root_page_table: Stores the root page table pointer. */
+	void *root_page_table;
+
+	/**
+	 * @op_lock: Lock used to serialize operations on a VM.
+	 *
+	 * The serialization of jobs queued to the VM_BIND queue is already
+	 * taken care of by drm_sched, but we need to serialize synchronous
+	 * and asynchronous VM_BIND request. This is what this lock is for.
+	 */
+	struct mutex op_lock;
+
+	/**
+	 * @op_ctx: The context attached to the currently executing VM operation.
+	 *
+	 * NULL when no operation is in progress.
+	 */
+	struct panthor_vm_op_ctx *op_ctx;
+
+	/**
+	 * @mm: Memory management object representing the auto-VA/kernel-VA.
+	 *
+	 * Used to auto-allocate VA space for kernel-managed objects (tiler
+	 * heaps, ...).
+	 *
+	 * For the MCU VM, this is managing the VA range that's used to map
+	 * all shared interfaces.
+	 *
+	 * For user VMs, the range is specified by userspace, and must not
+	 * exceed half of the VA space addressable.
+	 */
+	struct drm_mm mm;
+
+	/** @mm_lock: Lock protecting the @mm field. */
+	struct mutex mm_lock;
+
+	/** @kernel_auto_va: Automatic VA-range for kernel BOs. */
+	struct {
+		/** @start: Start of the automatic VA-range for kernel BOs. */
+		u64 start;
+
+		/** @size: Size of the automatic VA-range for kernel BOs. */
+		u64 end;
+	} kernel_auto_va;
+
+	/** @as: Address space related fields. */
+	struct {
+		/**
+		 * @id: ID of the address space this VM is bound to.
+		 *
+		 * A value of -1 means the VM is inactive/not bound.
+		 */
+		int id;
+
+		/** @active_cnt: Number of active users of this VM. */
+		refcount_t active_cnt;
+
+		/**
+		 * @lru_node: Used to instead the VM in the panthor_mmu::as::lru_list.
+		 *
+		 * Active VMs should not be inserted in the LRU list.
+		 */
+		struct list_head lru_node;
+	} as;
+
+	/**
+	 * @heaps: Tiler heap related fields.
+	 */
+	struct {
+		/**
+		 * @pool: The heap pool attached to this VM.
+		 *
+		 * Will stay NULL until someone creates a heap context on this VM.
+		 */
+		struct panthor_heap_pool *pool;
+
+		/** @lock: Lock used to protect access to @pool. */
+		struct mutex lock;
+	} heaps;
+
+	/** @node: Used to insert the VM in the panthor_mmu::vm::list. */
+	struct list_head node;
+
+	/** @for_mcu: True if this is the MCU VM. */
+	bool for_mcu;
+
+	/**
+	 * @destroyed: True if the VM was destroyed.
+	 *
+	 * No further bind requests should be queued to a destroyed VM.
+	 */
+	bool destroyed;
+
+	/**
+	 * @unusable: True if the VM has turned unusable because something
+	 * bad happened during an asynchronous request.
+	 *
+	 * We don't try to recover from such failures, because this implies
+	 * informing userspace about the specific operation that failed, and
+	 * hoping the userspace driver can replay things from there. This all
+	 * sounds very complicated for little gain.
+	 *
+	 * Instead, we should just flag the VM as unusable, and fail any
+	 * further request targeting this VM.
+	 *
+	 * We also provide a way to query a VM state, so userspace can destroy
+	 * it and create a new one.
+	 *
+	 * As an analogy, this would be mapped to a VK_ERROR_DEVICE_LOST
+	 * situation, where the logical device needs to be re-created.
+	 */
+	bool unusable;
+
+	/**
+	 * @unhandled_fault: Unhandled fault happened.
+	 *
+	 * This should be reported to the scheduler, and the queue/group be
+	 * flagged as faulty as a result.
+	 */
+	bool unhandled_fault;
+};
+
+/**
+ * struct panthor_vm_bind_job - VM bind job
+ */
+struct panthor_vm_bind_job {
+	/** @base: Inherit from drm_sched_job. */
+	struct drm_sched_job base;
+
+	/** @refcount: Reference count. */
+	struct kref refcount;
+
+	/** @cleanup_op_ctx_work: Work used to cleanup the VM operation context. */
+	struct work_struct cleanup_op_ctx_work;
+
+	/** @vm: VM targeted by the VM operation. */
+	struct panthor_vm *vm;
+
+	/** @ctx: Operation context. */
+	struct panthor_vm_op_ctx ctx;
+};
+
+/**
+ * @pt_cache: Cache used to allocate MMU page tables.
+ *
+ * The pre-allocation pattern forces us to over-allocate to plan for
+ * the worst case scenario, and return the pages we didn't use.
+ *
+ * Having a kmem_cache allows us to speed allocations.
+ */
+static struct kmem_cache *pt_cache;
+
+/**
+ * alloc_pt() - Custom page table allocator
+ * @cookie: Cookie passed at page table allocation time.
+ * @size: Size of the page table. This size should be fixed,
+ * and determined at creation time based on the granule size.
+ * @gfp: GFP flags.
+ *
+ * We want a custom allocator so we can use a cache for page table
+ * allocations and amortize the cost of the over-reservation that's
+ * done to allow asynchronous VM operations.
+ *
+ * Return: non-NULL on success, NULL if the allocation failed for any
+ * reason.
+ */
+static void *alloc_pt(void *cookie, size_t size, gfp_t gfp)
+{
+	struct panthor_vm *vm = cookie;
+	void *page;
+
+	/* Allocation of the root page table happening during init. */
+	if (unlikely(!vm->root_page_table)) {
+		struct page *p;
+
+		drm_WARN_ON(&vm->ptdev->base, vm->op_ctx);
+		p = alloc_pages_node(dev_to_node(vm->ptdev->base.dev),
+				     gfp | __GFP_ZERO, get_order(size));
+		page = p ? page_address(p) : NULL;
+		vm->root_page_table = page;
+		return page;
+	}
+
+	/* We're not supposed to have anything bigger than 4k here, because we picked a
+	 * 4k granule size at init time.
+	 */
+	if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
+		return NULL;
+
+	/* We must have some op_ctx attached to the VM and it must have at least one
+	 * free page.
+	 */
+	if (drm_WARN_ON(&vm->ptdev->base, !vm->op_ctx) ||
+	    drm_WARN_ON(&vm->ptdev->base,
+			vm->op_ctx->rsvd_page_tables.ptr >= vm->op_ctx->rsvd_page_tables.count))
+		return NULL;
+
+	page = vm->op_ctx->rsvd_page_tables.pages[vm->op_ctx->rsvd_page_tables.ptr++];
+	memset(page, 0, SZ_4K);
+
+	/* Page table entries don't use virtual addresses, which trips out
+	 * kmemleak. kmemleak_alloc_phys() might work, but physical addresses
+	 * are mixed with other fields, and I fear kmemleak won't detect that
+	 * either.
+	 *
+	 * Let's just ignore memory passed to the page-table driver for now.
+	 */
+	kmemleak_ignore(page);
+	return page;
+}
+
+/**
+ * @free_pt() - Custom page table free function
+ * @cookie: Cookie passed at page table allocation time.
+ * @data: Page table to free.
+ * @size: Size of the page table. This size should be fixed,
+ * and determined at creation time based on the granule size.
+ */
+static void free_pt(void *cookie, void *data, size_t size)
+{
+	struct panthor_vm *vm = cookie;
+
+	if (unlikely(vm->root_page_table == data)) {
+		free_pages((unsigned long)data, get_order(size));
+		vm->root_page_table = NULL;
+		return;
+	}
+
+	if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
+		return;
+
+	/* Return the page to the pt_cache. */
+	kmem_cache_free(pt_cache, data);
+}
+
+static int wait_ready(struct panthor_device *ptdev, u32 as_nr)
+{
+	int ret;
+	u32 val;
+
+	/* Wait for the MMU status to indicate there is no active command, in
+	 * case one is pending.
+	 */
+	ret = readl_relaxed_poll_timeout_atomic(ptdev->iomem + AS_STATUS(as_nr),
+						val, !(val & AS_STATUS_AS_ACTIVE),
+						10, 100000);
+
+	if (ret) {
+		panthor_device_schedule_reset(ptdev);
+		drm_err(&ptdev->base, "AS_ACTIVE bit stuck\n");
+	}
+
+	return ret;
+}
+
+static int write_cmd(struct panthor_device *ptdev, u32 as_nr, u32 cmd)
+{
+	int status;
+
+	/* write AS_COMMAND when MMU is ready to accept another command */
+	status = wait_ready(ptdev, as_nr);
+	if (!status)
+		gpu_write(ptdev, AS_COMMAND(as_nr), cmd);
+
+	return status;
+}
+
+static void lock_region(struct panthor_device *ptdev, u32 as_nr,
+			u64 region_start, u64 size)
+{
+	u8 region_width;
+	u64 region;
+	u64 region_end = region_start + size;
+
+	if (!size)
+		return;
+
+	/*
+	 * The locked region is a naturally aligned power of 2 block encoded as
+	 * log2 minus(1).
+	 * Calculate the desired start/end and look for the highest bit which
+	 * differs. The smallest naturally aligned block must include this bit
+	 * change, the desired region starts with this bit (and subsequent bits)
+	 * zeroed and ends with the bit (and subsequent bits) set to one.
+	 */
+	region_width = max(fls64(region_start ^ (region_end - 1)),
+			   const_ilog2(AS_LOCK_REGION_MIN_SIZE)) - 1;
+
+	/*
+	 * Mask off the low bits of region_start (which would be ignored by
+	 * the hardware anyway)
+	 */
+	region_start &= GENMASK_ULL(63, region_width);
+
+	region = region_width | region_start;
+
+	/* Lock the region that needs to be updated */
+	gpu_write(ptdev, AS_LOCKADDR_LO(as_nr), lower_32_bits(region));
+	gpu_write(ptdev, AS_LOCKADDR_HI(as_nr), upper_32_bits(region));
+	write_cmd(ptdev, as_nr, AS_COMMAND_LOCK);
+}
+
+static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr,
+				      u64 iova, u64 size, u32 op)
+{
+	lockdep_assert_held(&ptdev->mmu->as.slots_lock);
+
+	if (as_nr < 0)
+		return 0;
+
+	if (op != AS_COMMAND_UNLOCK)
+		lock_region(ptdev, as_nr, iova, size);
+
+	/* Run the MMU operation */
+	write_cmd(ptdev, as_nr, op);
+
+	/* Wait for the flush to complete */
+	return wait_ready(ptdev, as_nr);
+}
+
+static int mmu_hw_do_operation(struct panthor_vm *vm,
+			       u64 iova, u64 size, u32 op)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+	int ret;
+
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	ret = mmu_hw_do_operation_locked(ptdev, vm->as.id, iova, size, op);
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+	return ret;
+}
+
+static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr,
+				 u64 transtab, u64 transcfg, u64 memattr)
+{
+	int ret;
+
+	ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
+	if (ret)
+		return ret;
+
+	gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), lower_32_bits(transtab));
+	gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), upper_32_bits(transtab));
+
+	gpu_write(ptdev, AS_MEMATTR_LO(as_nr), lower_32_bits(memattr));
+	gpu_write(ptdev, AS_MEMATTR_HI(as_nr), upper_32_bits(memattr));
+
+	gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), lower_32_bits(transcfg));
+	gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), upper_32_bits(transcfg));
+
+	return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);
+}
+
+static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr)
+{
+	int ret;
+
+	ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
+	if (ret)
+		return ret;
+
+	gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), 0);
+	gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), 0);
+
+	gpu_write(ptdev, AS_MEMATTR_LO(as_nr), 0);
+	gpu_write(ptdev, AS_MEMATTR_HI(as_nr), 0);
+
+	gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED);
+	gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), 0);
+
+	return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);
+}
+
+static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value)
+{
+	/* Bits 16 to 31 mean REQ_COMPLETE. */
+	return value & GENMASK(15, 0);
+}
+
+static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as)
+{
+	return BIT(as);
+}
+
+/**
+ * panthor_vm_has_unhandled_faults() - Check if a VM has unhandled faults
+ * @vm: VM to check.
+ *
+ * Return: true if the VM has unhandled faults, false otherwise.
+ */
+bool panthor_vm_has_unhandled_faults(struct panthor_vm *vm)
+{
+	return vm->unhandled_fault;
+}
+
+/**
+ * panthor_vm_is_unusable() - Check if the VM is still usable
+ * @vm: VM to check.
+ *
+ * Return: true if the VM is unusable, false otherwise.
+ */
+bool panthor_vm_is_unusable(struct panthor_vm *vm)
+{
+	return vm->unusable;
+}
+
+static void panthor_vm_release_as_locked(struct panthor_vm *vm)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+
+	lockdep_assert_held(&ptdev->mmu->as.slots_lock);
+
+	if (drm_WARN_ON(&ptdev->base, vm->as.id < 0))
+		return;
+
+	ptdev->mmu->as.slots[vm->as.id].vm = NULL;
+	clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);
+	refcount_set(&vm->as.active_cnt, 0);
+	list_del_init(&vm->as.lru_node);
+	vm->as.id = -1;
+}
+
+/**
+ * panthor_vm_active() - Flag a VM as active
+ * @VM: VM to flag as active.
+ *
+ * Assigns an address space to a VM so it can be used by the GPU/MCU.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_active(struct panthor_vm *vm)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+	u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
+	struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;
+	int ret = 0, as, cookie;
+	u64 transtab, transcfg;
+
+	if (!drm_dev_enter(&ptdev->base, &cookie))
+		return -ENODEV;
+
+	if (refcount_inc_not_zero(&vm->as.active_cnt))
+		goto out_dev_exit;
+
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+
+	if (refcount_inc_not_zero(&vm->as.active_cnt))
+		goto out_unlock;
+
+	as = vm->as.id;
+	if (as >= 0) {
+		/* Unhandled pagefault on this AS, the MMU was disabled. We need to
+		 * re-enable the MMU after clearing+unmasking the AS interrupts.
+		 */
+		if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as))
+			goto out_enable_as;
+
+		goto out_make_active;
+	}
+
+	/* Check for a free AS */
+	if (vm->for_mcu) {
+		drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0));
+		as = 0;
+	} else {
+		as = ffz(ptdev->mmu->as.alloc_mask | BIT(0));
+	}
+
+	if (!(BIT(as) & ptdev->gpu_info.as_present)) {
+		struct panthor_vm *lru_vm;
+
+		lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list,
+						  struct panthor_vm,
+						  as.lru_node);
+		if (drm_WARN_ON(&ptdev->base, !lru_vm)) {
+			ret = -EBUSY;
+			goto out_unlock;
+		}
+
+		drm_WARN_ON(&ptdev->base, refcount_read(&lru_vm->as.active_cnt));
+		as = lru_vm->as.id;
+		panthor_vm_release_as_locked(lru_vm);
+	}
+
+	/* Assign the free or reclaimed AS to the FD */
+	vm->as.id = as;
+	set_bit(as, &ptdev->mmu->as.alloc_mask);
+	ptdev->mmu->as.slots[as].vm = vm;
+
+out_enable_as:
+	transtab = cfg->arm_lpae_s1_cfg.ttbr;
+	transcfg = AS_TRANSCFG_PTW_MEMATTR_WB |
+		   AS_TRANSCFG_PTW_RA |
+		   AS_TRANSCFG_ADRMODE_AARCH64_4K |
+		   AS_TRANSCFG_INA_BITS(55 - va_bits);
+	if (ptdev->coherent)
+		transcfg |= AS_TRANSCFG_PTW_SH_OS;
+
+	/* If the VM is re-activated, we clear the fault. */
+	vm->unhandled_fault = false;
+
+	/* Unhandled pagefault on this AS, clear the fault and re-enable interrupts
+	 * before enabling the AS.
+	 */
+	if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as)) {
+		gpu_write(ptdev, MMU_INT_CLEAR, panthor_mmu_as_fault_mask(ptdev, as));
+		ptdev->mmu->as.faulty_mask &= ~panthor_mmu_as_fault_mask(ptdev, as);
+		gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);
+	}
+
+	ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);
+
+out_make_active:
+	if (!ret) {
+		refcount_set(&vm->as.active_cnt, 1);
+		list_del_init(&vm->as.lru_node);
+	}
+
+out_unlock:
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+out_dev_exit:
+	drm_dev_exit(cookie);
+	return ret;
+}
+
+/**
+ * panthor_vm_idle() - Flag a VM idle
+ * @VM: VM to flag as idle.
+ *
+ * When we know the GPU is done with the VM (no more jobs to process),
+ * we can relinquish the AS slot attached to this VM, if any.
+ *
+ * We don't release the slot immediately, but instead place the VM in
+ * the LRU list, so it can be evicted if another VM needs an AS slot.
+ * This way, VMs keep attached to the AS they were given until we run
+ * out of free slot, limiting the number of MMU operations (TLB flush
+ * and other AS updates).
+ */
+void panthor_vm_idle(struct panthor_vm *vm)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+
+	if (!refcount_dec_and_mutex_lock(&vm->as.active_cnt, &ptdev->mmu->as.slots_lock))
+		return;
+
+	if (!drm_WARN_ON(&ptdev->base, vm->as.id == -1 || !list_empty(&vm->as.lru_node)))
+		list_add_tail(&vm->as.lru_node, &ptdev->mmu->as.lru_list);
+
+	refcount_set(&vm->as.active_cnt, 0);
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+}
+
+static void panthor_vm_stop(struct panthor_vm *vm)
+{
+	drm_sched_stop(&vm->sched, NULL);
+}
+
+static void panthor_vm_start(struct panthor_vm *vm)
+{
+	drm_sched_start(&vm->sched, true);
+}
+
+/**
+ * panthor_vm_as() - Get the AS slot attached to a VM
+ * @vm: VM to get the AS slot of.
+ *
+ * Return: -1 if the VM is not assigned an AS slot yet, >= 0 otherwise.
+ */
+int panthor_vm_as(struct panthor_vm *vm)
+{
+	return vm->as.id;
+}
+
+static size_t get_pgsize(u64 addr, size_t size, size_t *count)
+{
+	/*
+	 * io-pgtable only operates on multiple pages within a single table
+	 * entry, so we need to split at boundaries of the table size, i.e.
+	 * the next block size up. The distance from address A to the next
+	 * boundary of block size B is logically B - A % B, but in unsigned
+	 * two's complement where B is a power of two we get the equivalence
+	 * B - A % B == (B - A) % B == (n * B - A) % B, and choose n = 0 :)
+	 */
+	size_t blk_offset = -addr % SZ_2M;
+
+	if (blk_offset || size < SZ_2M) {
+		*count = min_not_zero(blk_offset, size) / SZ_4K;
+		return SZ_4K;
+	}
+	blk_offset = -addr % SZ_1G ?: SZ_1G;
+	*count = min(blk_offset, size) / SZ_2M;
+	return SZ_2M;
+}
+
+static int panthor_vm_flush_range(struct panthor_vm *vm, u64 iova, u64 size)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+	int ret = 0, cookie;
+
+	if (vm->as.id < 0)
+		return 0;
+
+	/* If the device is unplugged, we just silently skip the flush. */
+	if (!drm_dev_enter(&ptdev->base, &cookie))
+		return 0;
+
+	/* Flush the PTs only if we're already awake */
+	if (pm_runtime_active(ptdev->base.dev))
+		ret = mmu_hw_do_operation(vm, iova, size, AS_COMMAND_FLUSH_PT);
+
+	drm_dev_exit(cookie);
+	return ret;
+}
+
+static int panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, u64 size)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+	struct io_pgtable_ops *ops = vm->pgtbl_ops;
+	u64 offset = 0;
+
+	drm_dbg(&ptdev->base, "unmap: as=%d, iova=%llx, len=%llx", vm->as.id, iova, size);
+
+	while (offset < size) {
+		size_t unmapped_sz = 0, pgcount;
+		size_t pgsize = get_pgsize(iova + offset, size - offset, &pgcount);
+
+		unmapped_sz = ops->unmap_pages(ops, iova + offset, pgsize, pgcount, NULL);
+
+		if (drm_WARN_ON(&ptdev->base, unmapped_sz != pgsize * pgcount)) {
+			drm_err(&ptdev->base, "failed to unmap range %llx-%llx (requested range %llx-%llx)\n",
+				iova + offset + unmapped_sz,
+				iova + offset + pgsize * pgcount,
+				iova, iova + size);
+			panthor_vm_flush_range(vm, iova, offset + unmapped_sz);
+			return  -EINVAL;
+		}
+		offset += unmapped_sz;
+	}
+
+	return panthor_vm_flush_range(vm, iova, size);
+}
+
+static int
+panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot,
+		     struct sg_table *sgt, u64 offset, u64 size)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+	unsigned int count;
+	struct scatterlist *sgl;
+	struct io_pgtable_ops *ops = vm->pgtbl_ops;
+	u64 start_iova = iova;
+	int ret;
+
+	if (!size)
+		return 0;
+
+	for_each_sgtable_dma_sg(sgt, sgl, count) {
+		dma_addr_t paddr = sg_dma_address(sgl);
+		size_t len = sg_dma_len(sgl);
+
+		if (len <= offset) {
+			offset -= len;
+			continue;
+		}
+
+		paddr += offset;
+		len -= offset;
+		len = min_t(size_t, len, size);
+		size -= len;
+
+		drm_dbg(&ptdev->base, "map: as=%d, iova=%llx, paddr=%pad, len=%zx",
+			vm->as.id, iova, &paddr, len);
+
+		while (len) {
+			size_t pgcount, mapped = 0;
+			size_t pgsize = get_pgsize(iova | paddr, len, &pgcount);
+
+			ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot,
+					     GFP_KERNEL, &mapped);
+			iova += mapped;
+			paddr += mapped;
+			len -= mapped;
+
+			if (drm_WARN_ON(&ptdev->base, !ret && !mapped))
+				ret = -ENOMEM;
+
+			if (ret) {
+				/* If something failed, unmap what we've already mapped before
+				 * returning. The unmap call is not supposed to fail.
+				 */
+				drm_WARN_ON(&ptdev->base,
+					    panthor_vm_unmap_pages(vm, start_iova,
+								   iova - start_iova));
+				return ret;
+			}
+		}
+
+		if (!size)
+			break;
+	}
+
+	return panthor_vm_flush_range(vm, start_iova, iova - start_iova);
+}
+
+static int flags_to_prot(u32 flags)
+{
+	int prot = 0;
+
+	if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC)
+		prot |= IOMMU_NOEXEC;
+
+	if (!(flags & DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED))
+		prot |= IOMMU_CACHE;
+
+	if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_READONLY)
+		prot |= IOMMU_READ;
+	else
+		prot |= IOMMU_READ | IOMMU_WRITE;
+
+	return prot;
+}
+
+/**
+ * panthor_vm_alloc_va() - Allocate a region in the auto-va space
+ * @VM: VM to allocate a region on.
+ * @va: start of the VA range. Can be PANTHOR_VM_KERNEL_AUTO_VA if the user
+ * wants the VA to be automatically allocated from the auto-VA range.
+ * @size: size of the VA range.
+ * @va_node: drm_mm_node to initialize. Must be zero-initialized.
+ *
+ * Some GPU objects, like heap chunks, are fully managed by the kernel and
+ * need to be mapped to the userspace VM, in the region reserved for kernel
+ * objects.
+ *
+ * This function takes care of allocating a region in the kernel auto-VA space.
+ *
+ * Return: 0 on success, an error code otherwise.
+ */
+int
+panthor_vm_alloc_va(struct panthor_vm *vm, u64 va, u64 size,
+		    struct drm_mm_node *va_node)
+{
+	int ret;
+
+	if (!size || (size & ~PAGE_MASK))
+		return -EINVAL;
+
+	if (va != PANTHOR_VM_KERNEL_AUTO_VA && (va & ~PAGE_MASK))
+		return -EINVAL;
+
+	mutex_lock(&vm->mm_lock);
+	if (va != PANTHOR_VM_KERNEL_AUTO_VA) {
+		va_node->start = va;
+		va_node->size = size;
+		ret = drm_mm_reserve_node(&vm->mm, va_node);
+	} else {
+		ret = drm_mm_insert_node_in_range(&vm->mm, va_node, size,
+						  size >= SZ_2M ? SZ_2M : SZ_4K,
+						  0, vm->kernel_auto_va.start,
+						  vm->kernel_auto_va.end,
+						  DRM_MM_INSERT_BEST);
+	}
+	mutex_unlock(&vm->mm_lock);
+
+	return ret;
+}
+
+/**
+ * panthor_vm_free_va() - Free a region allocated with panthor_vm_alloc_va()
+ * @VM: VM to free the region on.
+ * @va_node: Memory node representing the region to free.
+ */
+void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *va_node)
+{
+	mutex_lock(&vm->mm_lock);
+	drm_mm_remove_node(va_node);
+	mutex_unlock(&vm->mm_lock);
+}
+
+static void panthor_vm_bo_put(struct drm_gpuvm_bo *vm_bo)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
+	struct drm_gpuvm *vm = vm_bo->vm;
+	bool unpin;
+
+	/* We must retain the GEM before calling drm_gpuvm_bo_put(),
+	 * otherwise the mutex might be destroyed while we hold it.
+	 * Same goes for the VM, since we take the VM resv lock.
+	 */
+	drm_gem_object_get(&bo->base.base);
+	drm_gpuvm_get(vm);
+
+	/* We take the resv lock to protect against concurrent accesses to the
+	 * gpuvm evicted/extobj lists that are modified in
+	 * drm_gpuvm_bo_destroy(), which is called if drm_gpuvm_bo_put()
+	 * releases sthe last vm_bo reference.
+	 * We take the BO GPUVA list lock to protect the vm_bo removal from the
+	 * GEM vm_bo list.
+	 */
+	dma_resv_lock(drm_gpuvm_resv(vm), NULL);
+	mutex_lock(&bo->gpuva_list_lock);
+	unpin = drm_gpuvm_bo_put(vm_bo);
+	mutex_unlock(&bo->gpuva_list_lock);
+	dma_resv_unlock(drm_gpuvm_resv(vm));
+
+	/* If the vm_bo object was destroyed, release the pin reference that
+	 * was hold by this object.
+	 */
+	if (unpin && !bo->base.base.import_attach)
+		drm_gem_shmem_unpin(&bo->base);
+
+	drm_gpuvm_put(vm);
+	drm_gem_object_put(&bo->base.base);
+}
+
+static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx,
+				      struct panthor_vm *vm)
+{
+	struct panthor_vma *vma, *tmp_vma;
+
+	u32 remaining_pt_count = op_ctx->rsvd_page_tables.count -
+				 op_ctx->rsvd_page_tables.ptr;
+
+	if (remaining_pt_count) {
+		kmem_cache_free_bulk(pt_cache, remaining_pt_count,
+				     op_ctx->rsvd_page_tables.pages +
+				     op_ctx->rsvd_page_tables.ptr);
+	}
+
+	kfree(op_ctx->rsvd_page_tables.pages);
+
+	if (op_ctx->map.vm_bo)
+		panthor_vm_bo_put(op_ctx->map.vm_bo);
+
+	for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++)
+		kfree(op_ctx->preallocated_vmas[i]);
+
+	list_for_each_entry_safe(vma, tmp_vma, &op_ctx->returned_vmas, node) {
+		list_del(&vma->node);
+		panthor_vm_bo_put(vma->base.vm_bo);
+		kfree(vma);
+	}
+}
+
+static struct panthor_vma *
+panthor_vm_op_ctx_get_vma(struct panthor_vm_op_ctx *op_ctx)
+{
+	for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++) {
+		struct panthor_vma *vma = op_ctx->preallocated_vmas[i];
+
+		if (vma) {
+			op_ctx->preallocated_vmas[i] = NULL;
+			return vma;
+		}
+	}
+
+	return NULL;
+}
+
+static int
+panthor_vm_op_ctx_prealloc_vmas(struct panthor_vm_op_ctx *op_ctx)
+{
+	u32 vma_count;
+
+	switch (op_ctx->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
+	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
+		/* One VMA for the new mapping, and two more VMAs for the remap case
+		 * which might contain both a prev and next VA.
+		 */
+		vma_count = 3;
+		break;
+
+	case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
+		/* Partial unmaps might trigger a remap with either a prev or a next VA,
+		 * but not both.
+		 */
+		vma_count = 1;
+		break;
+
+	default:
+		return 0;
+	}
+
+	for (u32 i = 0; i < vma_count; i++) {
+		struct panthor_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
+
+		if (!vma)
+			return -ENOMEM;
+
+		op_ctx->preallocated_vmas[i] = vma;
+	}
+
+	return 0;
+}
+
+#define PANTHOR_VM_BIND_OP_MAP_FLAGS \
+	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
+	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
+	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \
+	 DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
+
+static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
+					 struct panthor_vm *vm,
+					 struct panthor_gem_object *bo,
+					 u64 offset,
+					 u64 size, u64 va,
+					 u32 flags)
+{
+	struct drm_gpuvm_bo *preallocated_vm_bo;
+	struct sg_table *sgt = NULL;
+	u64 pt_count;
+	int ret;
+
+	if (!bo)
+		return -EINVAL;
+
+	if ((flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) ||
+	    (flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP)
+		return -EINVAL;
+
+	/* Make sure the VA and size are aligned and in-bounds. */
+	if (size > bo->base.base.size || offset > bo->base.base.size - size)
+		return -EINVAL;
+
+	/* If the BO has an exclusive VM attached, it can't be mapped to other VMs. */
+	if (bo->exclusive_vm_root_gem &&
+	    bo->exclusive_vm_root_gem != panthor_vm_root_gem(vm))
+		return -EINVAL;
+
+	memset(op_ctx, 0, sizeof(*op_ctx));
+	INIT_LIST_HEAD(&op_ctx->returned_vmas);
+	op_ctx->flags = flags;
+	op_ctx->va.range = size;
+	op_ctx->va.addr = va;
+
+	ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx);
+	if (ret)
+		goto err_cleanup;
+
+	if (!bo->base.base.import_attach) {
+		/* Pre-reserve the BO pages, so the map operation doesn't have to
+		 * allocate.
+		 */
+		ret = drm_gem_shmem_pin(&bo->base);
+		if (ret)
+			goto err_cleanup;
+	}
+
+	sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
+	if (IS_ERR(sgt)) {
+		if (!bo->base.base.import_attach)
+			drm_gem_shmem_unpin(&bo->base);
+
+		ret = PTR_ERR(sgt);
+		goto err_cleanup;
+	}
+
+	op_ctx->map.sgt = sgt;
+
+	preallocated_vm_bo = drm_gpuvm_bo_create(&vm->base, &bo->base.base);
+	if (!preallocated_vm_bo) {
+		if (!bo->base.base.import_attach)
+			drm_gem_shmem_unpin(&bo->base);
+
+		ret = -ENOMEM;
+		goto err_cleanup;
+	}
+
+	mutex_lock(&bo->gpuva_list_lock);
+	op_ctx->map.vm_bo = drm_gpuvm_bo_obtain_prealloc(preallocated_vm_bo);
+	mutex_unlock(&bo->gpuva_list_lock);
+
+	/* If the a vm_bo for this <VM,BO> combination exists, it already
+	 * retains a pin ref, and we can release the one we took earlier.
+	 *
+	 * If our pre-allocated vm_bo is picked, it now retains the pin ref,
+	 * which will be released in panthor_vm_bo_put().
+	 */
+	if (preallocated_vm_bo != op_ctx->map.vm_bo &&
+	    !bo->base.base.import_attach)
+		drm_gem_shmem_unpin(&bo->base);
+
+	op_ctx->map.bo_offset = offset;
+
+	/* L1, L2 and L3 page tables.
+	 * We could optimize L3 allocation by iterating over the sgt and merging
+	 * 2M contiguous blocks, but it's simpler to over-provision and return
+	 * the pages if they're not used.
+	 */
+	pt_count = ((ALIGN(va + size, 1ull << 39) - ALIGN_DOWN(va, 1ull << 39)) >> 39) +
+		   ((ALIGN(va + size, 1ull << 30) - ALIGN_DOWN(va, 1ull << 30)) >> 30) +
+		   ((ALIGN(va + size, 1ull << 21) - ALIGN_DOWN(va, 1ull << 21)) >> 21);
+
+	op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,
+						 sizeof(*op_ctx->rsvd_page_tables.pages),
+						 GFP_KERNEL);
+	if (!op_ctx->rsvd_page_tables.pages) {
+		ret = -ENOMEM;
+		goto err_cleanup;
+	}
+
+	ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
+				    op_ctx->rsvd_page_tables.pages);
+	op_ctx->rsvd_page_tables.count = ret;
+	if (ret != pt_count) {
+		ret = -ENOMEM;
+		goto err_cleanup;
+	}
+
+	/* Insert BO into the extobj list last, when we know nothing can fail. */
+	dma_resv_lock(panthor_vm_resv(vm), NULL);
+	drm_gpuvm_bo_extobj_add(op_ctx->map.vm_bo);
+	dma_resv_unlock(panthor_vm_resv(vm));
+
+	return 0;
+
+err_cleanup:
+	panthor_vm_cleanup_op_ctx(op_ctx, vm);
+	return ret;
+}
+
+static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx,
+					   struct panthor_vm *vm,
+					   u64 va, u64 size)
+{
+	u32 pt_count = 0;
+	int ret;
+
+	memset(op_ctx, 0, sizeof(*op_ctx));
+	INIT_LIST_HEAD(&op_ctx->returned_vmas);
+	op_ctx->va.range = size;
+	op_ctx->va.addr = va;
+	op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP;
+
+	/* Pre-allocate L3 page tables to account for the split-2M-block
+	 * situation on unmap.
+	 */
+	if (va != ALIGN(va, SZ_2M))
+		pt_count++;
+
+	if (va + size != ALIGN(va + size, SZ_2M) &&
+	    ALIGN(va + size, SZ_2M) != ALIGN(va, SZ_2M))
+		pt_count++;
+
+	ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx);
+	if (ret)
+		goto err_cleanup;
+
+	if (pt_count) {
+		op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,
+							 sizeof(*op_ctx->rsvd_page_tables.pages),
+							 GFP_KERNEL);
+		if (!op_ctx->rsvd_page_tables.pages) {
+			ret = -ENOMEM;
+			goto err_cleanup;
+		}
+
+		ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
+					    op_ctx->rsvd_page_tables.pages);
+		if (ret != pt_count) {
+			ret = -ENOMEM;
+			goto err_cleanup;
+		}
+		op_ctx->rsvd_page_tables.count = pt_count;
+	}
+
+	return 0;
+
+err_cleanup:
+	panthor_vm_cleanup_op_ctx(op_ctx, vm);
+	return ret;
+}
+
+static void panthor_vm_prepare_sync_only_op_ctx(struct panthor_vm_op_ctx *op_ctx,
+						struct panthor_vm *vm)
+{
+	memset(op_ctx, 0, sizeof(*op_ctx));
+	INIT_LIST_HEAD(&op_ctx->returned_vmas);
+	op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY;
+}
+
+/**
+ * panthor_vm_get_bo_for_va() - Get the GEM object mapped at a virtual address
+ * @vm: VM to look into.
+ * @va: Virtual address to search for.
+ * @bo_offset: Offset of the GEM object mapped at this virtual address.
+ * Only valid on success.
+ *
+ * The object returned by this function might no longer be mapped when the
+ * function returns. It's the caller responsibility to ensure there's no
+ * concurrent map/unmap operations making the returned value invalid, or
+ * make sure it doesn't matter if the object is no longer mapped.
+ *
+ * Return: A valid pointer on success, an ERR_PTR() otherwise.
+ */
+struct panthor_gem_object *
+panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset)
+{
+	struct panthor_gem_object *bo = ERR_PTR(-ENOENT);
+	struct drm_gpuva *gpuva;
+	struct panthor_vma *vma;
+
+	/* Take the VM lock to prevent concurrent map/unmap operations. */
+	mutex_lock(&vm->op_lock);
+	gpuva = drm_gpuva_find_first(&vm->base, va, 1);
+	vma = gpuva ? container_of(gpuva, struct panthor_vma, base) : NULL;
+	if (vma && vma->base.gem.obj) {
+		drm_gem_object_get(vma->base.gem.obj);
+		bo = to_panthor_bo(vma->base.gem.obj);
+		*bo_offset = vma->base.gem.offset + (va - vma->base.va.addr);
+	}
+	mutex_unlock(&vm->op_lock);
+
+	return bo;
+}
+
+#define PANTHOR_VM_MIN_KERNEL_VA_SIZE	SZ_256M
+
+static u64
+panthor_vm_create_get_user_va_range(const struct drm_panthor_vm_create *args,
+				    u64 full_va_range)
+{
+	u64 user_va_range;
+
+	/* Make sure we have a minimum amount of VA space for kernel objects. */
+	if (full_va_range < PANTHOR_VM_MIN_KERNEL_VA_SIZE)
+		return 0;
+
+	if (args->user_va_range) {
+		/* Use the user provided value if != 0. */
+		user_va_range = args->user_va_range;
+	} else if (TASK_SIZE_OF(current) < full_va_range) {
+		/* If the task VM size is smaller than the GPU VA range, pick this
+		 * as our default user VA range, so userspace can CPU/GPU map buffers
+		 * at the same address.
+		 */
+		user_va_range = TASK_SIZE_OF(current);
+	} else {
+		/* If the GPU VA range is smaller than the task VM size, we
+		 * just have to live with the fact we won't be able to map
+		 * all buffers at the same GPU/CPU address.
+		 *
+		 * If the GPU VA range is bigger than 4G (more than 32-bit of
+		 * VA), we split the range in two, and assign half of it to
+		 * the user and the other half to the kernel, if it's not, we
+		 * keep the kernel VA space as small as possible.
+		 */
+		user_va_range = full_va_range > SZ_4G ?
+				full_va_range / 2 :
+				full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE;
+	}
+
+	if (full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE < user_va_range)
+		user_va_range = full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE;
+
+	return user_va_range;
+}
+
+#define PANTHOR_VM_CREATE_FLAGS		0
+
+static int
+panthor_vm_create_check_args(const struct panthor_device *ptdev,
+			     const struct drm_panthor_vm_create *args,
+			     u64 *kernel_va_start, u64 *kernel_va_range)
+{
+	u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
+	u64 full_va_range = 1ull << va_bits;
+	u64 user_va_range;
+
+	if (args->flags & ~PANTHOR_VM_CREATE_FLAGS)
+		return -EINVAL;
+
+	user_va_range = panthor_vm_create_get_user_va_range(args, full_va_range);
+	if (!user_va_range || (args->user_va_range && args->user_va_range > user_va_range))
+		return -EINVAL;
+
+	/* Pick a kernel VA range that's a power of two, to have a clear split. */
+	*kernel_va_range = rounddown_pow_of_two(full_va_range - user_va_range);
+	*kernel_va_start = full_va_range - *kernel_va_range;
+	return 0;
+}
+
+/*
+ * Only 32 VMs per open file. If that becomes a limiting factor, we can
+ * increase this number.
+ */
+#define PANTHOR_MAX_VMS_PER_FILE	32
+
+/**
+ * panthor_vm_pool_create_vm() - Create a VM
+ * @pool: The VM to create this VM on.
+ * @kernel_va_start: Start of the region reserved for kernel objects.
+ * @kernel_va_range: Size of the region reserved for kernel objects.
+ *
+ * Return: a positive VM ID on success, a negative error code otherwise.
+ */
+int panthor_vm_pool_create_vm(struct panthor_device *ptdev,
+			      struct panthor_vm_pool *pool,
+			      struct drm_panthor_vm_create *args)
+{
+	u64 kernel_va_start, kernel_va_range;
+	struct panthor_vm *vm;
+	int ret;
+	u32 id;
+
+	ret = panthor_vm_create_check_args(ptdev, args, &kernel_va_start, &kernel_va_range);
+	if (ret)
+		return ret;
+
+	vm = panthor_vm_create(ptdev, false, kernel_va_start, kernel_va_range,
+			       kernel_va_start, kernel_va_range);
+	if (IS_ERR(vm))
+		return PTR_ERR(vm);
+
+	ret = xa_alloc(&pool->xa, &id, vm,
+		       XA_LIMIT(1, PANTHOR_MAX_VMS_PER_FILE), GFP_KERNEL);
+
+	if (ret) {
+		panthor_vm_put(vm);
+		return ret;
+	}
+
+	args->user_va_range = kernel_va_start;
+	return id;
+}
+
+static void panthor_vm_destroy(struct panthor_vm *vm)
+{
+	if (!vm)
+		return;
+
+	vm->destroyed = true;
+
+	mutex_lock(&vm->heaps.lock);
+	panthor_heap_pool_destroy(vm->heaps.pool);
+	vm->heaps.pool = NULL;
+	mutex_unlock(&vm->heaps.lock);
+
+	drm_WARN_ON(&vm->ptdev->base,
+		    panthor_vm_unmap_range(vm, vm->base.mm_start, vm->base.mm_range));
+	panthor_vm_put(vm);
+}
+
+/**
+ * panthor_vm_pool_destroy_vm() - Destroy a VM.
+ * @pool: VM pool.
+ * @handle: VM handle.
+ *
+ * This function doesn't free the VM object or its resources, it just kills
+ * all mappings, and makes sure nothing can be mapped after that point.
+ *
+ * If there was any active jobs at the time this function is called, these
+ * jobs should experience page faults and be killed as a result.
+ *
+ * The VM resources are freed when the last reference on the VM object is
+ * dropped.
+ */
+int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle)
+{
+	struct panthor_vm *vm;
+
+	vm = xa_erase(&pool->xa, handle);
+
+	panthor_vm_destroy(vm);
+
+	return vm ? 0 : -EINVAL;
+}
+
+/**
+ * panthor_vm_pool_get_vm() - Retrieve VM object bound to a VM handle
+ * @pool: VM pool to check.
+ * @handle: Handle of the VM to retrieve.
+ *
+ * Return: A valid pointer if the VM exists, NULL otherwise.
+ */
+struct panthor_vm *
+panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle)
+{
+	struct panthor_vm *vm;
+
+	vm = panthor_vm_get(xa_load(&pool->xa, handle));
+
+	return vm;
+}
+
+/**
+ * panthor_vm_pool_destroy() - Destroy a VM pool.
+ * @pfile: File.
+ *
+ * Destroy all VMs in the pool, and release the pool resources.
+ *
+ * Note that VMs can outlive the pool they were created from if other
+ * objects hold a reference to there VMs.
+ */
+void panthor_vm_pool_destroy(struct panthor_file *pfile)
+{
+	struct panthor_vm *vm;
+	unsigned long i;
+
+	if (!pfile->vms)
+		return;
+
+	xa_for_each(&pfile->vms->xa, i, vm)
+		panthor_vm_destroy(vm);
+
+	xa_destroy(&pfile->vms->xa);
+	kfree(pfile->vms);
+}
+
+/**
+ * panthor_vm_pool_create() - Create a VM pool
+ * @pfile: File.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_pool_create(struct panthor_file *pfile)
+{
+	pfile->vms = kzalloc(sizeof(*pfile->vms), GFP_KERNEL);
+	if (!pfile->vms)
+		return -ENOMEM;
+
+	xa_init_flags(&pfile->vms->xa, XA_FLAGS_ALLOC1);
+	return 0;
+}
+
+/* dummy TLB ops, the real TLB flush happens in panthor_vm_flush_range() */
+static void mmu_tlb_flush_all(void *cookie)
+{
+}
+
+static void mmu_tlb_flush_walk(unsigned long iova, size_t size, size_t granule, void *cookie)
+{
+}
+
+static const struct iommu_flush_ops mmu_tlb_ops = {
+	.tlb_flush_all = mmu_tlb_flush_all,
+	.tlb_flush_walk = mmu_tlb_flush_walk,
+};
+
+static const char *access_type_name(struct panthor_device *ptdev,
+				    u32 fault_status)
+{
+	switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
+	case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
+		return "ATOMIC";
+	case AS_FAULTSTATUS_ACCESS_TYPE_READ:
+		return "READ";
+	case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
+		return "WRITE";
+	case AS_FAULTSTATUS_ACCESS_TYPE_EX:
+		return "EXECUTE";
+	default:
+		drm_WARN_ON(&ptdev->base, 1);
+		return NULL;
+	}
+}
+
+static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
+{
+	bool has_unhandled_faults = false;
+
+	status = panthor_mmu_fault_mask(ptdev, status);
+	while (status) {
+		u32 as = ffs(status | (status >> 16)) - 1;
+		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
+		u32 new_int_mask;
+		u64 addr;
+		u32 fault_status;
+		u32 exception_type;
+		u32 access_type;
+		u32 source_id;
+
+		fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as));
+		addr = gpu_read(ptdev, AS_FAULTADDRESS_LO(as));
+		addr |= (u64)gpu_read(ptdev, AS_FAULTADDRESS_HI(as)) << 32;
+
+		/* decode the fault status */
+		exception_type = fault_status & 0xFF;
+		access_type = (fault_status >> 8) & 0x3;
+		source_id = (fault_status >> 16);
+
+		mutex_lock(&ptdev->mmu->as.slots_lock);
+
+		ptdev->mmu->as.faulty_mask |= mask;
+		new_int_mask =
+			panthor_mmu_fault_mask(ptdev, ~ptdev->mmu->as.faulty_mask);
+
+		/* terminal fault, print info about the fault */
+		drm_err(&ptdev->base,
+			"Unhandled Page fault in AS%d at VA 0x%016llX\n"
+			"raw fault status: 0x%X\n"
+			"decoded fault status: %s\n"
+			"exception type 0x%X: %s\n"
+			"access type 0x%X: %s\n"
+			"source id 0x%X\n",
+			as, addr,
+			fault_status,
+			(fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
+			exception_type, panthor_exception_name(ptdev, exception_type),
+			access_type, access_type_name(ptdev, fault_status),
+			source_id);
+
+		/* Ignore MMU interrupts on this AS until it's been
+		 * re-enabled.
+		 */
+		ptdev->mmu->irq.mask = new_int_mask;
+		gpu_write(ptdev, MMU_INT_MASK, new_int_mask);
+
+		if (ptdev->mmu->as.slots[as].vm)
+			ptdev->mmu->as.slots[as].vm->unhandled_fault = true;
+
+		/* Disable the MMU to kill jobs on this AS. */
+		panthor_mmu_as_disable(ptdev, as);
+		mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+		status &= ~mask;
+		has_unhandled_faults = true;
+	}
+
+	if (has_unhandled_faults)
+		panthor_sched_report_mmu_fault(ptdev);
+}
+PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);
+
+/**
+ * panthor_mmu_suspend() - Suspend the MMU logic
+ * @ptdev: Device.
+ *
+ * All we do here is de-assign the AS slots on all active VMs, so things
+ * get flushed to the main memory, and no further access to these VMs are
+ * possible.
+ *
+ * We also suspend the MMU IRQ.
+ */
+void panthor_mmu_suspend(struct panthor_device *ptdev)
+{
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
+		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
+
+		if (vm) {
+			drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
+			panthor_vm_release_as_locked(vm);
+		}
+	}
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
+}
+
+/**
+ * panthor_mmu_resume() - Resume the MMU logic
+ * @ptdev: Device.
+ *
+ * Resume the IRQ.
+ *
+ * We don't re-enable previously active VMs. We assume other parts of the
+ * driver will call panthor_vm_active() on the VMs they intend to use.
+ */
+void panthor_mmu_resume(struct panthor_device *ptdev)
+{
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	ptdev->mmu->as.alloc_mask = 0;
+	ptdev->mmu->as.faulty_mask = 0;
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
+}
+
+/**
+ * panthor_mmu_pre_reset() - Prepare for a reset
+ * @ptdev: Device.
+ *
+ * Suspend the IRQ, and make sure all VM_BIND queues are stopped, so we
+ * don't get asked to do a VM operation while the GPU is down.
+ *
+ * We don't cleanly shutdown the AS slots here, because the reset might
+ * come from an AS_ACTIVE_BIT stuck situation.
+ */
+void panthor_mmu_pre_reset(struct panthor_device *ptdev)
+{
+	struct panthor_vm *vm;
+
+	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
+
+	mutex_lock(&ptdev->mmu->vm.lock);
+	ptdev->mmu->vm.reset_in_progress = true;
+	list_for_each_entry(vm, &ptdev->mmu->vm.list, node)
+		panthor_vm_stop(vm);
+	mutex_unlock(&ptdev->mmu->vm.lock);
+}
+
+/**
+ * panthor_mmu_post_reset() - Restore things after a reset
+ * @ptdev: Device.
+ *
+ * Put the MMU logic back in action after a reset. That implies resuming the
+ * IRQ and re-enabling the VM_BIND queues.
+ */
+void panthor_mmu_post_reset(struct panthor_device *ptdev)
+{
+	struct panthor_vm *vm;
+
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+
+	/* Now that the reset is effective, we can assume that none of the
+	 * AS slots are setup, and clear the faulty flags too.
+	 */
+	ptdev->mmu->as.alloc_mask = 0;
+	ptdev->mmu->as.faulty_mask = 0;
+
+	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
+		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
+
+		if (vm)
+			panthor_vm_release_as_locked(vm);
+	}
+
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
+
+	/* Restart the VM_BIND queues. */
+	mutex_lock(&ptdev->mmu->vm.lock);
+	list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
+		panthor_vm_start(vm);
+	}
+	ptdev->mmu->vm.reset_in_progress = false;
+	mutex_unlock(&ptdev->mmu->vm.lock);
+}
+
+static void panthor_vm_free(struct drm_gpuvm *gpuvm)
+{
+	struct panthor_vm *vm = container_of(gpuvm, struct panthor_vm, base);
+	struct panthor_device *ptdev = vm->ptdev;
+
+	mutex_lock(&vm->heaps.lock);
+	if (drm_WARN_ON(&ptdev->base, vm->heaps.pool))
+		panthor_heap_pool_destroy(vm->heaps.pool);
+	mutex_unlock(&vm->heaps.lock);
+	mutex_destroy(&vm->heaps.lock);
+
+	mutex_lock(&ptdev->mmu->vm.lock);
+	list_del(&vm->node);
+	/* Restore the scheduler state so we can call drm_sched_entity_destroy()
+	 * and drm_sched_fini(). If get there, that means we have no job left
+	 * and no new jobs can be queued, so we can start the scheduler without
+	 * risking interfering with the reset.
+	 */
+	if (ptdev->mmu->vm.reset_in_progress)
+		panthor_vm_start(vm);
+	mutex_unlock(&ptdev->mmu->vm.lock);
+
+	drm_sched_entity_destroy(&vm->entity);
+	drm_sched_fini(&vm->sched);
+
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	if (vm->as.id >= 0) {
+		int cookie;
+
+		if (drm_dev_enter(&ptdev->base, &cookie)) {
+			panthor_mmu_as_disable(ptdev, vm->as.id);
+			drm_dev_exit(cookie);
+		}
+
+		ptdev->mmu->as.slots[vm->as.id].vm = NULL;
+		clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);
+		list_del(&vm->as.lru_node);
+	}
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+	free_io_pgtable_ops(vm->pgtbl_ops);
+
+	drm_mm_takedown(&vm->mm);
+	kfree(vm);
+}
+
+/**
+ * panthor_vm_put() - Release a reference on a VM
+ * @vm: VM to release the reference on. Can be NULL.
+ */
+void panthor_vm_put(struct panthor_vm *vm)
+{
+	drm_gpuvm_put(vm ? &vm->base : NULL);
+}
+
+/**
+ * panthor_vm_get() - Get a VM reference
+ * @vm: VM to get the reference on. Can be NULL.
+ *
+ * Return: @vm value.
+ */
+struct panthor_vm *panthor_vm_get(struct panthor_vm *vm)
+{
+	if (vm)
+		drm_gpuvm_get(&vm->base);
+
+	return vm;
+}
+
+/**
+ * panthor_vm_get_heap_pool() - Get the heap pool attached to a VM
+ * @vm: VM to query the heap pool on.
+ * @create: True if the heap pool should be created when it doesn't exist.
+ *
+ * Heap pools are per-VM. This function allows one to retrieve the heap pool
+ * attached to a VM.
+ *
+ * If no heap pool exists yet, and @create is true, we create one.
+ *
+ * The returned panthor_heap_pool should be released with panthor_heap_pool_put().
+ *
+ * Return: A valid pointer on success, an ERR_PTR() otherwise.
+ */
+struct panthor_heap_pool *panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create)
+{
+	struct panthor_heap_pool *pool;
+
+	mutex_lock(&vm->heaps.lock);
+	if (!vm->heaps.pool && create) {
+		if (vm->destroyed)
+			pool = ERR_PTR(-EINVAL);
+		else
+			pool = panthor_heap_pool_create(vm->ptdev, vm);
+
+		if (!IS_ERR(pool))
+			vm->heaps.pool = panthor_heap_pool_get(pool);
+	} else {
+		pool = panthor_heap_pool_get(vm->heaps.pool);
+		if (!pool)
+			pool = ERR_PTR(-ENOENT);
+	}
+	mutex_unlock(&vm->heaps.lock);
+
+	return pool;
+}
+
+static u64 mair_to_memattr(u64 mair)
+{
+	u64 memattr = 0;
+	u32 i;
+
+	for (i = 0; i < 8; i++) {
+		u8 in_attr = mair >> (8 * i), out_attr;
+		u8 outer = in_attr >> 4, inner = in_attr & 0xf;
+
+		/* For caching to be enabled, inner and outer caching policy
+		 * have to be both write-back, if one of them is write-through
+		 * or non-cacheable, we just choose non-cacheable. Device
+		 * memory is also translated to non-cacheable.
+		 */
+		if (!(outer & 3) || !(outer & 4) || !(inner & 4)) {
+			out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_NC |
+				   AS_MEMATTR_AARCH64_SH_MIDGARD_INNER |
+				   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(false, false);
+		} else {
+			/* Use SH_CPU_INNER mode so SH_IS, which is used when
+			 * IOMMU_CACHE is set, actually maps to the standard
+			 * definition of inner-shareable and not Mali's
+			 * internal-shareable mode.
+			 */
+			out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_WB |
+				   AS_MEMATTR_AARCH64_SH_CPU_INNER |
+				   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(inner & 1, inner & 2);
+		}
+
+		memattr |= (u64)out_attr << (8 * i);
+	}
+
+	return memattr;
+}
+
+static void panthor_vma_link(struct panthor_vm *vm,
+			     struct panthor_vma *vma,
+			     struct drm_gpuvm_bo *vm_bo)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
+
+	mutex_lock(&bo->gpuva_list_lock);
+	drm_gpuva_link(&vma->base, vm_bo);
+	drm_WARN_ON(&vm->ptdev->base, drm_gpuvm_bo_put(vm_bo));
+	mutex_unlock(&bo->gpuva_list_lock);
+}
+
+static void panthor_vma_unlink(struct panthor_vm *vm,
+			       struct panthor_vma *vma)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
+	struct drm_gpuvm_bo *vm_bo = drm_gpuvm_bo_get(vma->base.vm_bo);
+
+	mutex_lock(&bo->gpuva_list_lock);
+	drm_gpuva_unlink(&vma->base);
+	mutex_unlock(&bo->gpuva_list_lock);
+
+	/* drm_gpuva_unlink() release the vm_bo, but we manually retained it
+	 * when entering this function, so we can implement deferred VMA
+	 * destruction. Re-assign it here.
+	 */
+	vma->base.vm_bo = vm_bo;
+	list_add_tail(&vma->node, &vm->op_ctx->returned_vmas);
+}
+
+static void panthor_vma_init(struct panthor_vma *vma, u32 flags)
+{
+	INIT_LIST_HEAD(&vma->node);
+	vma->flags = flags;
+}
+
+#define PANTHOR_VM_MAP_FLAGS \
+	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
+	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
+	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED)
+
+static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv)
+{
+	struct panthor_vm *vm = priv;
+	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
+	struct panthor_vma *vma = panthor_vm_op_ctx_get_vma(op_ctx);
+	int ret;
+
+	if (!vma)
+		return -EINVAL;
+
+	panthor_vma_init(vma, op_ctx->flags & PANTHOR_VM_MAP_FLAGS);
+
+	ret = panthor_vm_map_pages(vm, op->map.va.addr, flags_to_prot(vma->flags),
+				   op_ctx->map.sgt, op->map.gem.offset,
+				   op->map.va.range);
+	if (ret)
+		return ret;
+
+	/* Ref owned by the mapping now, clear the obj field so we don't release the
+	 * pinning/obj ref behind GPUVA's back.
+	 */
+	drm_gpuva_map(&vm->base, &vma->base, &op->map);
+	panthor_vma_link(vm, vma, op_ctx->map.vm_bo);
+	op_ctx->map.vm_bo = NULL;
+	return 0;
+}
+
+static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
+				       void *priv)
+{
+	struct panthor_vma *unmap_vma = container_of(op->remap.unmap->va, struct panthor_vma, base);
+	struct panthor_vm *vm = priv;
+	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
+	struct panthor_vma *prev_vma = NULL, *next_vma = NULL;
+	u64 unmap_start, unmap_range;
+	int ret;
+
+	drm_gpuva_op_remap_to_unmap_range(&op->remap, &unmap_start, &unmap_range);
+	ret = panthor_vm_unmap_pages(vm, unmap_start, unmap_range);
+	if (ret)
+		return ret;
+
+	if (op->remap.prev) {
+		prev_vma = panthor_vm_op_ctx_get_vma(op_ctx);
+		panthor_vma_init(prev_vma, unmap_vma->flags);
+	}
+
+	if (op->remap.next) {
+		next_vma = panthor_vm_op_ctx_get_vma(op_ctx);
+		panthor_vma_init(next_vma, unmap_vma->flags);
+	}
+
+	drm_gpuva_remap(prev_vma ? &prev_vma->base : NULL,
+			next_vma ? &next_vma->base : NULL,
+			&op->remap);
+
+	if (prev_vma) {
+		/* panthor_vma_link() transfers the vm_bo ownership to
+		 * the VMA object. Since the vm_bo we're passing is still
+		 * owned by the old mapping which will be released when this
+		 * mapping is destroyed, we need to grab a ref here.
+		 */
+		panthor_vma_link(vm, prev_vma,
+				 drm_gpuvm_bo_get(op->remap.unmap->va->vm_bo));
+	}
+
+	if (next_vma) {
+		panthor_vma_link(vm, next_vma,
+				 drm_gpuvm_bo_get(op->remap.unmap->va->vm_bo));
+	}
+
+	panthor_vma_unlink(vm, unmap_vma);
+	return 0;
+}
+
+static int panthor_gpuva_sm_step_unmap(struct drm_gpuva_op *op,
+				       void *priv)
+{
+	struct panthor_vma *unmap_vma = container_of(op->unmap.va, struct panthor_vma, base);
+	struct panthor_vm *vm = priv;
+	int ret;
+
+	ret = panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
+				     unmap_vma->base.va.range);
+	if (drm_WARN_ON(&vm->ptdev->base, ret))
+		return ret;
+
+	drm_gpuva_unmap(&op->unmap);
+	panthor_vma_unlink(vm, unmap_vma);
+	return 0;
+}
+
+static const struct drm_gpuvm_ops panthor_gpuvm_ops = {
+	.vm_free = panthor_vm_free,
+	.sm_step_map = panthor_gpuva_sm_step_map,
+	.sm_step_remap = panthor_gpuva_sm_step_remap,
+	.sm_step_unmap = panthor_gpuva_sm_step_unmap,
+};
+
+/**
+ * panthor_vm_resv() - Get the dma_resv object attached to a VM.
+ * @vm: VM to get the dma_resv of.
+ *
+ * Return: A dma_resv object.
+ */
+struct dma_resv *panthor_vm_resv(struct panthor_vm *vm)
+{
+	return drm_gpuvm_resv(&vm->base);
+}
+
+struct drm_gem_object *panthor_vm_root_gem(struct panthor_vm *vm)
+{
+	if (!vm)
+		return NULL;
+
+	return vm->base.r_obj;
+}
+
+static int
+panthor_vm_exec_op(struct panthor_vm *vm, struct panthor_vm_op_ctx *op,
+		   bool flag_vm_unusable_on_failure)
+{
+	u32 op_type = op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK;
+	int ret;
+
+	if (op_type == DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY)
+		return 0;
+
+	mutex_lock(&vm->op_lock);
+	vm->op_ctx = op;
+	switch (op_type) {
+	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
+		if (vm->unusable) {
+			ret = -EINVAL;
+			break;
+		}
+
+		ret = drm_gpuvm_sm_map(&vm->base, vm, op->va.addr, op->va.range,
+				       op->map.vm_bo->obj, op->map.bo_offset);
+		break;
+
+	case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
+		ret = drm_gpuvm_sm_unmap(&vm->base, vm, op->va.addr, op->va.range);
+		break;
+
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	if (ret && flag_vm_unusable_on_failure)
+		vm->unusable = true;
+
+	vm->op_ctx = NULL;
+	mutex_unlock(&vm->op_lock);
+
+	return ret;
+}
+
+static struct dma_fence *
+panthor_vm_bind_run_job(struct drm_sched_job *sched_job)
+{
+	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
+	bool cookie;
+	int ret;
+
+	/* Not only we report an error whose result is propagated to the
+	 * drm_sched finished fence, but we also flag the VM as unusable, because
+	 * a failure in the async VM_BIND results in an inconsistent state. VM needs
+	 * to be destroyed and recreated.
+	 */
+	cookie = dma_fence_begin_signalling();
+	ret = panthor_vm_exec_op(job->vm, &job->ctx, true);
+	dma_fence_end_signalling(cookie);
+
+	return ret ? ERR_PTR(ret) : NULL;
+}
+
+static void panthor_vm_bind_job_release(struct kref *kref)
+{
+	struct panthor_vm_bind_job *job = container_of(kref, struct panthor_vm_bind_job, refcount);
+
+	if (job->base.s_fence)
+		drm_sched_job_cleanup(&job->base);
+
+	panthor_vm_cleanup_op_ctx(&job->ctx, job->vm);
+	panthor_vm_put(job->vm);
+	kfree(job);
+}
+
+/**
+ * panthor_vm_bind_job_put() - Release a VM_BIND job reference
+ * @sched_job: Job to release the reference on.
+ */
+void panthor_vm_bind_job_put(struct drm_sched_job *sched_job)
+{
+	struct panthor_vm_bind_job *job =
+		container_of(sched_job, struct panthor_vm_bind_job, base);
+
+	if (sched_job)
+		kref_put(&job->refcount, panthor_vm_bind_job_release);
+}
+
+static void
+panthor_vm_bind_free_job(struct drm_sched_job *sched_job)
+{
+	struct panthor_vm_bind_job *job =
+		container_of(sched_job, struct panthor_vm_bind_job, base);
+
+	drm_sched_job_cleanup(sched_job);
+
+	/* Do the heavy cleanups asynchronously, so we're out of the
+	 * dma-signaling path and can acquire dma-resv locks safely.
+	 */
+	queue_work(panthor_cleanup_wq, &job->cleanup_op_ctx_work);
+}
+
+static enum drm_gpu_sched_stat
+panthor_vm_bind_timedout_job(struct drm_sched_job *sched_job)
+{
+	WARN(1, "VM_BIND ops are synchronous for now, there should be no timeout!");
+	return DRM_GPU_SCHED_STAT_NOMINAL;
+}
+
+static const struct drm_sched_backend_ops panthor_vm_bind_ops = {
+	.run_job = panthor_vm_bind_run_job,
+	.free_job = panthor_vm_bind_free_job,
+	.timedout_job = panthor_vm_bind_timedout_job,
+};
+
+/**
+ * panthor_vm_create() - Create a VM
+ * @ptdev: Device.
+ * @for_mcu: True if this is the FW MCU VM.
+ * @kernel_va_start: Start of the range reserved for kernel BO mapping.
+ * @kernel_va_size: Size of the range reserved for kernel BO mapping.
+ * @auto_kernel_va_start: Start of the auto-VA kernel range.
+ * @auto_kernel_va_size: Size of the auto-VA kernel range.
+ *
+ * Return: A valid pointer on success, an ERR_PTR() otherwise.
+ */
+struct panthor_vm *
+panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
+		  u64 kernel_va_start, u64 kernel_va_size,
+		  u64 auto_kernel_va_start, u64 auto_kernel_va_size)
+{
+	u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
+	u32 pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features);
+	u64 full_va_range = 1ull << va_bits;
+	struct drm_gem_object *dummy_gem;
+	struct drm_gpu_scheduler *sched;
+	struct io_pgtable_cfg pgtbl_cfg;
+	u64 mair, min_va, va_range;
+	struct panthor_vm *vm;
+	int ret;
+
+	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
+	if (!vm)
+		return ERR_PTR(-ENOMEM);
+
+	/* We allocate a dummy GEM for the VM. */
+	dummy_gem = drm_gpuvm_resv_object_alloc(&ptdev->base);
+	if (!dummy_gem) {
+		ret = -ENOMEM;
+		goto err_free_vm;
+	}
+
+	mutex_init(&vm->heaps.lock);
+	vm->for_mcu = for_mcu;
+	vm->ptdev = ptdev;
+	mutex_init(&vm->op_lock);
+
+	if (for_mcu) {
+		/* CSF MCU is a cortex M7, and can only address 4G */
+		min_va = 0;
+		va_range = SZ_4G;
+	} else {
+		min_va = 0;
+		va_range = full_va_range;
+	}
+
+	mutex_init(&vm->mm_lock);
+	drm_mm_init(&vm->mm, kernel_va_start, kernel_va_size);
+	vm->kernel_auto_va.start = auto_kernel_va_start;
+	vm->kernel_auto_va.end = vm->kernel_auto_va.start + auto_kernel_va_size - 1;
+
+	INIT_LIST_HEAD(&vm->node);
+	INIT_LIST_HEAD(&vm->as.lru_node);
+	vm->as.id = -1;
+	refcount_set(&vm->as.active_cnt, 0);
+
+	pgtbl_cfg = (struct io_pgtable_cfg) {
+		.pgsize_bitmap	= SZ_4K | SZ_2M,
+		.ias		= va_bits,
+		.oas		= pa_bits,
+		.coherent_walk	= ptdev->coherent,
+		.tlb		= &mmu_tlb_ops,
+		.iommu_dev	= ptdev->base.dev,
+		.alloc		= alloc_pt,
+		.free		= free_pt,
+	};
+
+	vm->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1, &pgtbl_cfg, vm);
+	if (!vm->pgtbl_ops) {
+		ret = -EINVAL;
+		goto err_mm_takedown;
+	}
+
+	/* Bind operations are synchronous for now, no timeout needed. */
+	ret = drm_sched_init(&vm->sched, &panthor_vm_bind_ops, ptdev->mmu->vm.wq,
+			     1, 1, 0,
+			     MAX_SCHEDULE_TIMEOUT, NULL, NULL,
+			     "panthor-vm-bind", ptdev->base.dev);
+	if (ret)
+		goto err_free_io_pgtable;
+
+	sched = &vm->sched;
+	ret = drm_sched_entity_init(&vm->entity, 0, &sched, 1, NULL);
+	if (ret)
+		goto err_sched_fini;
+
+	mair = io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg.arm_lpae_s1_cfg.mair;
+	vm->memattr = mair_to_memattr(mair);
+
+	mutex_lock(&ptdev->mmu->vm.lock);
+	list_add_tail(&vm->node, &ptdev->mmu->vm.list);
+
+	/* If a reset is in progress, stop the scheduler. */
+	if (ptdev->mmu->vm.reset_in_progress)
+		panthor_vm_stop(vm);
+	mutex_unlock(&ptdev->mmu->vm.lock);
+
+	/* We intentionally leave the reserved range to zero, because we want kernel VMAs
+	 * to be handled the same way user VMAs are.
+	 */
+	drm_gpuvm_init(&vm->base, for_mcu ? "panthor-MCU-VM" : "panthor-GPU-VM",
+		       DRM_GPUVM_RESV_PROTECTED, &ptdev->base, dummy_gem,
+		       min_va, va_range, 0, 0, &panthor_gpuvm_ops);
+	drm_gem_object_put(dummy_gem);
+	return vm;
+
+err_sched_fini:
+	drm_sched_fini(&vm->sched);
+
+err_free_io_pgtable:
+	free_io_pgtable_ops(vm->pgtbl_ops);
+
+err_mm_takedown:
+	drm_mm_takedown(&vm->mm);
+	drm_gem_object_put(dummy_gem);
+
+err_free_vm:
+	kfree(vm);
+	return ERR_PTR(ret);
+}
+
+static int
+panthor_vm_bind_prepare_op_ctx(struct drm_file *file,
+			       struct panthor_vm *vm,
+			       const struct drm_panthor_vm_bind_op *op,
+			       struct panthor_vm_op_ctx *op_ctx)
+{
+	struct drm_gem_object *gem;
+	int ret;
+
+	/* Aligned on page size. */
+	if ((op->va | op->size) & ~PAGE_MASK)
+		return -EINVAL;
+
+	switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
+	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
+		gem = drm_gem_object_lookup(file, op->bo_handle);
+		ret = panthor_vm_prepare_map_op_ctx(op_ctx, vm,
+						    gem ? to_panthor_bo(gem) : NULL,
+						    op->bo_offset,
+						    op->size,
+						    op->va,
+						    op->flags);
+		drm_gem_object_put(gem);
+		return ret;
+
+	case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
+		if (op->flags & ~DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
+			return -EINVAL;
+
+		if (op->bo_handle || op->bo_offset)
+			return -EINVAL;
+
+		return panthor_vm_prepare_unmap_op_ctx(op_ctx, vm, op->va, op->size);
+
+	case DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY:
+		if (op->flags & ~DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
+			return -EINVAL;
+
+		if (op->bo_handle || op->bo_offset)
+			return -EINVAL;
+
+		if (op->va || op->size)
+			return -EINVAL;
+
+		if (!op->syncs.count)
+			return -EINVAL;
+
+		panthor_vm_prepare_sync_only_op_ctx(op_ctx, vm);
+		return 0;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static void panthor_vm_bind_job_cleanup_op_ctx_work(struct work_struct *work)
+{
+	struct panthor_vm_bind_job *job =
+		container_of(work, struct panthor_vm_bind_job, cleanup_op_ctx_work);
+
+	panthor_vm_bind_job_put(&job->base);
+}
+
+/**
+ * panthor_vm_bind_job_create() - Create a VM_BIND job
+ * @file: File.
+ * @vm: VM targeted by the VM_BIND job.
+ * @op: VM operation data.
+ *
+ * Return: A valid pointer on success, an ERR_PTR() otherwise.
+ */
+struct drm_sched_job *
+panthor_vm_bind_job_create(struct drm_file *file,
+			   struct panthor_vm *vm,
+			   const struct drm_panthor_vm_bind_op *op)
+{
+	struct panthor_vm_bind_job *job;
+	int ret;
+
+	if (!vm)
+		return ERR_PTR(-EINVAL);
+
+	if (vm->destroyed || vm->unusable)
+		return ERR_PTR(-EINVAL);
+
+	job = kzalloc(sizeof(*job), GFP_KERNEL);
+	if (!job)
+		return ERR_PTR(-ENOMEM);
+
+	ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &job->ctx);
+	if (ret) {
+		kfree(job);
+		return ERR_PTR(ret);
+	}
+
+	INIT_WORK(&job->cleanup_op_ctx_work, panthor_vm_bind_job_cleanup_op_ctx_work);
+	kref_init(&job->refcount);
+	job->vm = panthor_vm_get(vm);
+
+	ret = drm_sched_job_init(&job->base, &vm->entity, 1, vm);
+	if (ret)
+		goto err_put_job;
+
+	return &job->base;
+
+err_put_job:
+	panthor_vm_bind_job_put(&job->base);
+	return ERR_PTR(ret);
+}
+
+/**
+ * panthor_vm_bind_job_prepare_resvs() - Prepare VM_BIND job dma_resvs
+ * @exec: The locking/preparation context.
+ * @sched_job: The job to prepare resvs on.
+ *
+ * Locks and prepare the VM resv.
+ *
+ * If this is a map operation, locks and prepares the GEM resv.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec,
+				      struct drm_sched_job *sched_job)
+{
+	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
+	int ret;
+
+	/* Acquire the VM lock an reserve a slot for this VM bind job. */
+	ret = drm_gpuvm_prepare_vm(&job->vm->base, exec, 1);
+	if (ret)
+		return ret;
+
+	if (job->ctx.map.vm_bo) {
+		/* Lock/prepare the GEM being mapped. */
+		ret = drm_exec_prepare_obj(exec, job->ctx.map.vm_bo->obj, 1);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * panthor_vm_bind_job_update_resvs() - Update the resv objects touched by a job
+ * @exec: drm_exec context.
+ * @sched_job: Job to update the resvs on.
+ */
+void panthor_vm_bind_job_update_resvs(struct drm_exec *exec,
+				      struct drm_sched_job *sched_job)
+{
+	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
+
+	/* Explicit sync => we just register our job finished fence as bookkeep. */
+	drm_gpuvm_resv_add_fence(&job->vm->base, exec,
+				 &sched_job->s_fence->finished,
+				 DMA_RESV_USAGE_BOOKKEEP,
+				 DMA_RESV_USAGE_BOOKKEEP);
+}
+
+void panthor_vm_update_resvs(struct panthor_vm *vm, struct drm_exec *exec,
+			     struct dma_fence *fence,
+			     enum dma_resv_usage private_usage,
+			     enum dma_resv_usage extobj_usage)
+{
+	drm_gpuvm_resv_add_fence(&vm->base, exec, fence, private_usage, extobj_usage);
+}
+
+/**
+ * panthor_vm_bind_exec_sync_op() - Execute a VM_BIND operation synchronously.
+ * @file: File.
+ * @vm: VM targeted by the VM operation.
+ * @op: Data describing the VM operation.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_bind_exec_sync_op(struct drm_file *file,
+				 struct panthor_vm *vm,
+				 struct drm_panthor_vm_bind_op *op)
+{
+	struct panthor_vm_op_ctx op_ctx;
+	int ret;
+
+	/* No sync objects allowed on synchronous operations. */
+	if (op->syncs.count)
+		return -EINVAL;
+
+	if (!op->size)
+		return 0;
+
+	ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &op_ctx);
+	if (ret)
+		return ret;
+
+	ret = panthor_vm_exec_op(vm, &op_ctx, false);
+	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
+
+	return ret;
+}
+
+/**
+ * panthor_vm_map_bo_range() - Map a GEM object range to a VM
+ * @vm: VM to map the GEM to.
+ * @bo: GEM object to map.
+ * @offset: Offset in the GEM object.
+ * @size: Size to map.
+ * @va: Virtual address to map the object to.
+ * @flags: Combination of drm_panthor_vm_bind_op_flags flags.
+ * Only map-related flags are valid.
+ *
+ * Internal use only. For userspace requests, use
+ * panthor_vm_bind_exec_sync_op() instead.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,
+			    u64 offset, u64 size, u64 va, u32 flags)
+{
+	struct panthor_vm_op_ctx op_ctx;
+	int ret;
+
+	ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, offset, size, va, flags);
+	if (ret)
+		return ret;
+
+	ret = panthor_vm_exec_op(vm, &op_ctx, false);
+	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
+
+	return ret;
+}
+
+/**
+ * panthor_vm_unmap_range() - Unmap a portion of the VA space
+ * @vm: VM to unmap the region from.
+ * @va: Virtual address to unmap. Must be 4k aligned.
+ * @size: Size of the region to unmap. Must be 4k aligned.
+ *
+ * Internal use only. For userspace requests, use
+ * panthor_vm_bind_exec_sync_op() instead.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, u64 size)
+{
+	struct panthor_vm_op_ctx op_ctx;
+	int ret;
+
+	ret = panthor_vm_prepare_unmap_op_ctx(&op_ctx, vm, va, size);
+	if (ret)
+		return ret;
+
+	ret = panthor_vm_exec_op(vm, &op_ctx, false);
+	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
+
+	return ret;
+}
+
+/**
+ * panthor_vm_prepare_mapped_bos_resvs() - Prepare resvs on VM BOs.
+ * @exec: Locking/preparation context.
+ * @vm: VM targeted by the GPU job.
+ * @slot_count: Number of slots to reserve.
+ *
+ * GPU jobs assume all BOs bound to the VM at the time the job is submitted
+ * are available when the job is executed. In order to guarantee that, we
+ * need to reserve a slot on all BOs mapped to a VM and update this slot with
+ * the job fence after its submission.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, struct panthor_vm *vm,
+					u32 slot_count)
+{
+	int ret;
+
+	/* Acquire the VM lock and reserve a slot for this GPU job. */
+	ret = drm_gpuvm_prepare_vm(&vm->base, exec, slot_count);
+	if (ret)
+		return ret;
+
+	return drm_gpuvm_prepare_objects(&vm->base, exec, slot_count);
+}
+
+/**
+ * panthor_mmu_unplug() - Unplug the MMU logic
+ * @ptdev: Device.
+ *
+ * No access to the MMU regs should be done after this function is called.
+ * We suspend the IRQ and disable all VMs to guarantee that.
+ */
+void panthor_mmu_unplug(struct panthor_device *ptdev)
+{
+	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
+
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
+		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
+
+		if (vm) {
+			drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
+			panthor_vm_release_as_locked(vm);
+		}
+	}
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+}
+
+static void panthor_mmu_release_wq(struct drm_device *ddev, void *res)
+{
+	destroy_workqueue(res);
+}
+
+/**
+ * panthor_mmu_init() - Initialize the MMU logic.
+ * @ptdev: Device.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_mmu_init(struct panthor_device *ptdev)
+{
+	u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
+	struct panthor_mmu *mmu;
+	int ret, irq;
+
+	mmu = drmm_kzalloc(&ptdev->base, sizeof(*mmu), GFP_KERNEL);
+	if (!mmu)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&mmu->as.lru_list);
+
+	ret = drmm_mutex_init(&ptdev->base, &mmu->as.slots_lock);
+	if (ret)
+		return ret;
+
+	INIT_LIST_HEAD(&mmu->vm.list);
+	ret = drmm_mutex_init(&ptdev->base, &mmu->vm.lock);
+	if (ret)
+		return ret;
+
+	ptdev->mmu = mmu;
+
+	irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "mmu");
+	if (irq <= 0)
+		return -ENODEV;
+
+	ret = panthor_request_mmu_irq(ptdev, &mmu->irq, irq,
+				      panthor_mmu_fault_mask(ptdev, ~0));
+	if (ret)
+		return ret;
+
+	mmu->vm.wq = alloc_workqueue("panthor-vm-bind", WQ_UNBOUND, 0);
+	if (!mmu->vm.wq)
+		return -ENOMEM;
+
+	/* On 32-bit kernels, the VA space is limited by the io_pgtable_ops abstraction,
+	 * which passes iova as an unsigned long. Patch the mmu_features to reflect this
+	 * limitation.
+	 */
+	if (sizeof(unsigned long) * 8 < va_bits) {
+		ptdev->gpu_info.mmu_features &= ~GENMASK(7, 0);
+		ptdev->gpu_info.mmu_features |= sizeof(unsigned long) * 8;
+	}
+
+	return drmm_add_action_or_reset(&ptdev->base, panthor_mmu_release_wq, mmu->vm.wq);
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int show_vm_gpuvas(struct panthor_vm *vm, struct seq_file *m)
+{
+	int ret;
+
+	mutex_lock(&vm->op_lock);
+	ret = drm_debugfs_gpuva_info(m, &vm->base);
+	mutex_unlock(&vm->op_lock);
+
+	return ret;
+}
+
+static int show_each_vm(struct seq_file *m, void *arg)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *ddev = node->minor->dev;
+	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
+	int (*show)(struct panthor_vm *, struct seq_file *) = node->info_ent->data;
+	struct panthor_vm *vm;
+	int ret = 0;
+
+	mutex_lock(&ptdev->mmu->vm.lock);
+	list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
+		ret = show(vm, m);
+		if (ret < 0)
+			break;
+
+		seq_puts(m, "\n");
+	}
+	mutex_unlock(&ptdev->mmu->vm.lock);
+
+	return ret;
+}
+
+static struct drm_info_list panthor_mmu_debugfs_list[] = {
+	DRM_DEBUGFS_GPUVA_INFO(show_each_vm, show_vm_gpuvas),
+};
+
+/**
+ * panthor_mmu_debugfs_init() - Initialize MMU debugfs entries
+ * @minor: Minor.
+ */
+void panthor_mmu_debugfs_init(struct drm_minor *minor)
+{
+	drm_debugfs_create_files(panthor_mmu_debugfs_list,
+				 ARRAY_SIZE(panthor_mmu_debugfs_list),
+				 minor->debugfs_root, minor);
+}
+#endif /* CONFIG_DEBUG_FS */
+
+/**
+ * panthor_mmu_pt_cache_init() - Initialize the page table cache.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+int panthor_mmu_pt_cache_init(void)
+{
+	pt_cache = kmem_cache_create("panthor-mmu-pt", SZ_4K, SZ_4K, 0, NULL);
+	if (!pt_cache)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * panthor_mmu_pt_cache_fini() - Destroy the page table cache.
+ */
+void panthor_mmu_pt_cache_fini(void)
+{
+	kmem_cache_destroy(pt_cache);
+}
diff --git a/drivers/gpu/drm/panthor/panthor_mmu.h b/drivers/gpu/drm/panthor/panthor_mmu.h
new file mode 100644
index 000000000000..f3c1ed19f973
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_mmu.h
@@ -0,0 +1,102 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+/* Copyright 2023 Collabora ltd. */
+
+#ifndef __PANTHOR_MMU_H__
+#define __PANTHOR_MMU_H__
+
+#include <linux/dma-resv.h>
+
+struct drm_exec;
+struct drm_sched_job;
+struct panthor_gem_object;
+struct panthor_heap_pool;
+struct panthor_vm;
+struct panthor_vma;
+struct panthor_mmu;
+
+int panthor_mmu_init(struct panthor_device *ptdev);
+void panthor_mmu_unplug(struct panthor_device *ptdev);
+void panthor_mmu_pre_reset(struct panthor_device *ptdev);
+void panthor_mmu_post_reset(struct panthor_device *ptdev);
+void panthor_mmu_suspend(struct panthor_device *ptdev);
+void panthor_mmu_resume(struct panthor_device *ptdev);
+
+int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,
+			    u64 offset, u64 size, u64 va, u32 flags);
+int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, u64 size);
+struct panthor_gem_object *
+panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset);
+
+int panthor_vm_active(struct panthor_vm *vm);
+void panthor_vm_idle(struct panthor_vm *vm);
+int panthor_vm_as(struct panthor_vm *vm);
+
+struct panthor_heap_pool *
+panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create);
+
+struct panthor_vm *panthor_vm_get(struct panthor_vm *vm);
+void panthor_vm_put(struct panthor_vm *vm);
+struct panthor_vm *panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
+				     u64 kernel_va_start, u64 kernel_va_size,
+				     u64 kernel_auto_va_start,
+				     u64 kernel_auto_va_size);
+
+int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec,
+					struct panthor_vm *vm,
+					u32 slot_count);
+int panthor_vm_add_bos_resvs_deps_to_job(struct panthor_vm *vm,
+					 struct drm_sched_job *job);
+void panthor_vm_add_job_fence_to_bos_resvs(struct panthor_vm *vm,
+					   struct drm_sched_job *job);
+
+struct dma_resv *panthor_vm_resv(struct panthor_vm *vm);
+struct drm_gem_object *panthor_vm_root_gem(struct panthor_vm *vm);
+
+void panthor_vm_pool_destroy(struct panthor_file *pfile);
+int panthor_vm_pool_create(struct panthor_file *pfile);
+int panthor_vm_pool_create_vm(struct panthor_device *ptdev,
+			      struct panthor_vm_pool *pool,
+			      struct drm_panthor_vm_create *args);
+int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle);
+struct panthor_vm *panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle);
+
+bool panthor_vm_has_unhandled_faults(struct panthor_vm *vm);
+bool panthor_vm_is_unusable(struct panthor_vm *vm);
+
+/*
+ * PANTHOR_VM_KERNEL_AUTO_VA: Use this magic address when you want the GEM
+ * logic to auto-allocate the virtual address in the reserved kernel VA range.
+ */
+#define PANTHOR_VM_KERNEL_AUTO_VA		~0ull
+
+int panthor_vm_alloc_va(struct panthor_vm *vm, u64 va, u64 size,
+			struct drm_mm_node *va_node);
+void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *va_node);
+
+int panthor_vm_bind_exec_sync_op(struct drm_file *file,
+				 struct panthor_vm *vm,
+				 struct drm_panthor_vm_bind_op *op);
+
+struct drm_sched_job *
+panthor_vm_bind_job_create(struct drm_file *file,
+			   struct panthor_vm *vm,
+			   const struct drm_panthor_vm_bind_op *op);
+void panthor_vm_bind_job_put(struct drm_sched_job *job);
+int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec,
+				      struct drm_sched_job *job);
+void panthor_vm_bind_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *job);
+
+void panthor_vm_update_resvs(struct panthor_vm *vm, struct drm_exec *exec,
+			     struct dma_fence *fence,
+			     enum dma_resv_usage private_usage,
+			     enum dma_resv_usage extobj_usage);
+
+int panthor_mmu_pt_cache_init(void);
+void panthor_mmu_pt_cache_fini(void);
+
+#ifdef CONFIG_DEBUG_FS
+void panthor_mmu_debugfs_init(struct drm_minor *minor);
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/panthor/panthor_regs.h b/drivers/gpu/drm/panthor/panthor_regs.h
new file mode 100644
index 000000000000..b7b3b3add166
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_regs.h
@@ -0,0 +1,239 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+/* Copyright 2023 Collabora ltd. */
+/*
+ * Register definitions based on mali_kbase_gpu_regmap.h and
+ * mali_kbase_gpu_regmap_csf.h
+ * (C) COPYRIGHT 2010-2022 ARM Limited. All rights reserved.
+ */
+#ifndef __PANTHOR_REGS_H__
+#define __PANTHOR_REGS_H__
+
+#define GPU_ID						0x0
+#define   GPU_ARCH_MAJOR(x)				((x) >> 28)
+#define   GPU_ARCH_MINOR(x)				(((x) & GENMASK(27, 24)) >> 24)
+#define   GPU_ARCH_REV(x)				(((x) & GENMASK(23, 20)) >> 20)
+#define   GPU_PROD_MAJOR(x)				(((x) & GENMASK(19, 16)) >> 16)
+#define   GPU_VER_MAJOR(x)				(((x) & GENMASK(15, 12)) >> 12)
+#define   GPU_VER_MINOR(x)				(((x) & GENMASK(11, 4)) >> 4)
+#define   GPU_VER_STATUS(x)				((x) & GENMASK(3, 0))
+
+#define GPU_L2_FEATURES					0x4
+#define  GPU_L2_FEATURES_LINE_SIZE(x)			(1 << ((x) & GENMASK(7, 0)))
+
+#define GPU_CORE_FEATURES				0x8
+
+#define GPU_TILER_FEATURES				0xC
+#define GPU_MEM_FEATURES				0x10
+#define   GROUPS_L2_COHERENT				BIT(0)
+
+#define GPU_MMU_FEATURES				0x14
+#define  GPU_MMU_FEATURES_VA_BITS(x)			((x) & GENMASK(7, 0))
+#define  GPU_MMU_FEATURES_PA_BITS(x)			(((x) >> 8) & GENMASK(7, 0))
+#define GPU_AS_PRESENT					0x18
+#define GPU_CSF_ID					0x1C
+
+#define GPU_INT_RAWSTAT					0x20
+#define GPU_INT_CLEAR					0x24
+#define GPU_INT_MASK					0x28
+#define GPU_INT_STAT					0x2c
+#define   GPU_IRQ_FAULT					BIT(0)
+#define   GPU_IRQ_PROTM_FAULT				BIT(1)
+#define   GPU_IRQ_RESET_COMPLETED			BIT(8)
+#define   GPU_IRQ_POWER_CHANGED				BIT(9)
+#define   GPU_IRQ_POWER_CHANGED_ALL			BIT(10)
+#define   GPU_IRQ_CLEAN_CACHES_COMPLETED		BIT(17)
+#define   GPU_IRQ_DOORBELL_MIRROR			BIT(18)
+#define   GPU_IRQ_MCU_STATUS_CHANGED			BIT(19)
+#define GPU_CMD						0x30
+#define   GPU_CMD_DEF(type, payload)			((type) | ((payload) << 8))
+#define   GPU_SOFT_RESET				GPU_CMD_DEF(1, 1)
+#define   GPU_HARD_RESET				GPU_CMD_DEF(1, 2)
+#define   CACHE_CLEAN					BIT(0)
+#define   CACHE_INV					BIT(1)
+#define   GPU_FLUSH_CACHES(l2, lsc, oth)		\
+	  GPU_CMD_DEF(4, ((l2) << 0) | ((lsc) << 4) | ((oth) << 8))
+
+#define GPU_STATUS					0x34
+#define   GPU_STATUS_ACTIVE				BIT(0)
+#define   GPU_STATUS_PWR_ACTIVE				BIT(1)
+#define   GPU_STATUS_PAGE_FAULT				BIT(4)
+#define   GPU_STATUS_PROTM_ACTIVE			BIT(7)
+#define   GPU_STATUS_DBG_ENABLED			BIT(8)
+
+#define GPU_FAULT_STATUS				0x3C
+#define GPU_FAULT_ADDR_LO				0x40
+#define GPU_FAULT_ADDR_HI				0x44
+
+#define GPU_PWR_KEY					0x50
+#define  GPU_PWR_KEY_UNLOCK				0x2968A819
+#define GPU_PWR_OVERRIDE0				0x54
+#define GPU_PWR_OVERRIDE1				0x58
+
+#define GPU_TIMESTAMP_OFFSET_LO				0x88
+#define GPU_TIMESTAMP_OFFSET_HI				0x8C
+#define GPU_CYCLE_COUNT_LO				0x90
+#define GPU_CYCLE_COUNT_HI				0x94
+#define GPU_TIMESTAMP_LO				0x98
+#define GPU_TIMESTAMP_HI				0x9C
+
+#define GPU_THREAD_MAX_THREADS				0xA0
+#define GPU_THREAD_MAX_WORKGROUP_SIZE			0xA4
+#define GPU_THREAD_MAX_BARRIER_SIZE			0xA8
+#define GPU_THREAD_FEATURES				0xAC
+
+#define GPU_TEXTURE_FEATURES(n)				(0xB0 + ((n) * 4))
+
+#define GPU_SHADER_PRESENT_LO				0x100
+#define GPU_SHADER_PRESENT_HI				0x104
+#define GPU_TILER_PRESENT_LO				0x110
+#define GPU_TILER_PRESENT_HI				0x114
+#define GPU_L2_PRESENT_LO				0x120
+#define GPU_L2_PRESENT_HI				0x124
+
+#define SHADER_READY_LO					0x140
+#define SHADER_READY_HI					0x144
+#define TILER_READY_LO					0x150
+#define TILER_READY_HI					0x154
+#define L2_READY_LO					0x160
+#define L2_READY_HI					0x164
+
+#define SHADER_PWRON_LO					0x180
+#define SHADER_PWRON_HI					0x184
+#define TILER_PWRON_LO					0x190
+#define TILER_PWRON_HI					0x194
+#define L2_PWRON_LO					0x1A0
+#define L2_PWRON_HI					0x1A4
+
+#define SHADER_PWROFF_LO				0x1C0
+#define SHADER_PWROFF_HI				0x1C4
+#define TILER_PWROFF_LO					0x1D0
+#define TILER_PWROFF_HI					0x1D4
+#define L2_PWROFF_LO					0x1E0
+#define L2_PWROFF_HI					0x1E4
+
+#define SHADER_PWRTRANS_LO				0x200
+#define SHADER_PWRTRANS_HI				0x204
+#define TILER_PWRTRANS_LO				0x210
+#define TILER_PWRTRANS_HI				0x214
+#define L2_PWRTRANS_LO					0x220
+#define L2_PWRTRANS_HI					0x224
+
+#define SHADER_PWRACTIVE_LO				0x240
+#define SHADER_PWRACTIVE_HI				0x244
+#define TILER_PWRACTIVE_LO				0x250
+#define TILER_PWRACTIVE_HI				0x254
+#define L2_PWRACTIVE_LO					0x260
+#define L2_PWRACTIVE_HI					0x264
+
+#define GPU_REVID					0x280
+
+#define GPU_COHERENCY_FEATURES				0x300
+#define GPU_COHERENCY_PROT_BIT(name)			BIT(GPU_COHERENCY_  ## name)
+
+#define GPU_COHERENCY_PROTOCOL				0x304
+#define   GPU_COHERENCY_ACE				0
+#define   GPU_COHERENCY_ACE_LITE			1
+#define   GPU_COHERENCY_NONE				31
+
+#define MCU_CONTROL					0x700
+#define MCU_CONTROL_ENABLE				1
+#define MCU_CONTROL_AUTO				2
+#define MCU_CONTROL_DISABLE				0
+
+#define MCU_STATUS					0x704
+#define MCU_STATUS_DISABLED				0
+#define MCU_STATUS_ENABLED				1
+#define MCU_STATUS_HALT					2
+#define MCU_STATUS_FATAL				3
+
+/* Job Control regs */
+#define JOB_INT_RAWSTAT					0x1000
+#define JOB_INT_CLEAR					0x1004
+#define JOB_INT_MASK					0x1008
+#define JOB_INT_STAT					0x100c
+#define   JOB_INT_GLOBAL_IF				BIT(31)
+#define   JOB_INT_CSG_IF(x)				BIT(x)
+
+/* MMU regs */
+#define MMU_INT_RAWSTAT					0x2000
+#define MMU_INT_CLEAR					0x2004
+#define MMU_INT_MASK					0x2008
+#define MMU_INT_STAT					0x200c
+
+/* AS_COMMAND register commands */
+
+#define MMU_BASE					0x2400
+#define MMU_AS_SHIFT					6
+#define MMU_AS(as)					(MMU_BASE + ((as) << MMU_AS_SHIFT))
+
+#define AS_TRANSTAB_LO(as)				(MMU_AS(as) + 0x0)
+#define AS_TRANSTAB_HI(as)				(MMU_AS(as) + 0x4)
+#define AS_MEMATTR_LO(as)				(MMU_AS(as) + 0x8)
+#define AS_MEMATTR_HI(as)				(MMU_AS(as) + 0xC)
+#define   AS_MEMATTR_AARCH64_INNER_ALLOC_IMPL		(2 << 2)
+#define   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(w, r)	((3 << 2) | \
+							 ((w) ? BIT(0) : 0) | \
+							 ((r) ? BIT(1) : 0))
+#define   AS_MEMATTR_AARCH64_SH_MIDGARD_INNER		(0 << 4)
+#define   AS_MEMATTR_AARCH64_SH_CPU_INNER		(1 << 4)
+#define   AS_MEMATTR_AARCH64_SH_CPU_INNER_SHADER_COH	(2 << 4)
+#define   AS_MEMATTR_AARCH64_SHARED			(0 << 6)
+#define   AS_MEMATTR_AARCH64_INNER_OUTER_NC		(1 << 6)
+#define   AS_MEMATTR_AARCH64_INNER_OUTER_WB		(2 << 6)
+#define   AS_MEMATTR_AARCH64_FAULT			(3 << 6)
+#define AS_LOCKADDR_LO(as)				(MMU_AS(as) + 0x10)
+#define AS_LOCKADDR_HI(as)				(MMU_AS(as) + 0x14)
+#define AS_COMMAND(as)					(MMU_AS(as) + 0x18)
+#define   AS_COMMAND_NOP				0
+#define   AS_COMMAND_UPDATE				1
+#define   AS_COMMAND_LOCK				2
+#define   AS_COMMAND_UNLOCK				3
+#define   AS_COMMAND_FLUSH_PT				4
+#define   AS_COMMAND_FLUSH_MEM				5
+#define   AS_LOCK_REGION_MIN_SIZE			(1ULL << 15)
+#define AS_FAULTSTATUS(as)				(MMU_AS(as) + 0x1C)
+#define  AS_FAULTSTATUS_ACCESS_TYPE_MASK		(0x3 << 8)
+#define  AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC		(0x0 << 8)
+#define  AS_FAULTSTATUS_ACCESS_TYPE_EX			(0x1 << 8)
+#define  AS_FAULTSTATUS_ACCESS_TYPE_READ		(0x2 << 8)
+#define  AS_FAULTSTATUS_ACCESS_TYPE_WRITE		(0x3 << 8)
+#define AS_FAULTADDRESS_LO(as)				(MMU_AS(as) + 0x20)
+#define AS_FAULTADDRESS_HI(as)				(MMU_AS(as) + 0x24)
+#define AS_STATUS(as)					(MMU_AS(as) + 0x28)
+#define   AS_STATUS_AS_ACTIVE				BIT(0)
+#define AS_TRANSCFG_LO(as)				(MMU_AS(as) + 0x30)
+#define AS_TRANSCFG_HI(as)				(MMU_AS(as) + 0x34)
+#define   AS_TRANSCFG_ADRMODE_UNMAPPED			(1 << 0)
+#define   AS_TRANSCFG_ADRMODE_IDENTITY			(2 << 0)
+#define   AS_TRANSCFG_ADRMODE_AARCH64_4K		(6 << 0)
+#define   AS_TRANSCFG_ADRMODE_AARCH64_64K		(8 << 0)
+#define   AS_TRANSCFG_INA_BITS(x)			((x) << 6)
+#define   AS_TRANSCFG_OUTA_BITS(x)			((x) << 14)
+#define   AS_TRANSCFG_SL_CONCAT				BIT(22)
+#define   AS_TRANSCFG_PTW_MEMATTR_NC			(1 << 24)
+#define   AS_TRANSCFG_PTW_MEMATTR_WB			(2 << 24)
+#define   AS_TRANSCFG_PTW_SH_NS				(0 << 28)
+#define   AS_TRANSCFG_PTW_SH_OS				(2 << 28)
+#define   AS_TRANSCFG_PTW_SH_IS				(3 << 28)
+#define   AS_TRANSCFG_PTW_RA				BIT(30)
+#define   AS_TRANSCFG_DISABLE_HIER_AP			BIT(33)
+#define   AS_TRANSCFG_DISABLE_AF_FAULT			BIT(34)
+#define   AS_TRANSCFG_WXN				BIT(35)
+#define   AS_TRANSCFG_XREADABLE				BIT(36)
+#define AS_FAULTEXTRA_LO(as)				(MMU_AS(as) + 0x38)
+#define AS_FAULTEXTRA_HI(as)				(MMU_AS(as) + 0x3C)
+
+#define CSF_GPU_LATEST_FLUSH_ID				0x10000
+
+#define CSF_DOORBELL(i)					(0x80000 + ((i) * 0x10000))
+#define CSF_GLB_DOORBELL_ID				0
+
+#define gpu_write(dev, reg, data) \
+	writel(data, (dev)->iomem + (reg))
+
+#define gpu_read(dev, reg) \
+	readl((dev)->iomem + (reg))
+
+#endif
diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c
new file mode 100644
index 000000000000..79ffcbc41d78
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_sched.c
@@ -0,0 +1,3550 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/* Copyright 2023 Collabora ltd. */
+
+#include <drm/drm_drv.h>
+#include <drm/drm_exec.h>
+#include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_managed.h>
+#include <drm/gpu_scheduler.h>
+#include <drm/panthor_drm.h>
+
+#include <linux/build_bug.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma-resv.h>
+#include <linux/firmware.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/iosys-map.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#include "panthor_devfreq.h"
+#include "panthor_device.h"
+#include "panthor_fw.h"
+#include "panthor_gem.h"
+#include "panthor_gpu.h"
+#include "panthor_heap.h"
+#include "panthor_mmu.h"
+#include "panthor_regs.h"
+#include "panthor_sched.h"
+
+/**
+ * DOC: Scheduler
+ *
+ * Mali CSF hardware adopts a firmware-assisted scheduling model, where
+ * the firmware takes care of scheduling aspects, to some extent.
+ *
+ * The scheduling happens at the scheduling group level, each group
+ * contains 1 to N queues (N is FW/hardware dependent, and exposed
+ * through the firmware interface). Each queue is assigned a command
+ * stream ring buffer, which serves as a way to get jobs submitted to
+ * the GPU, among other things.
+ *
+ * The firmware can schedule a maximum of M groups (M is FW/hardware
+ * dependent, and exposed through the firmware interface). Passed
+ * this maximum number of groups, the kernel must take care of
+ * rotating the groups passed to the firmware so every group gets
+ * a chance to have his queues scheduled for execution.
+ *
+ * The current implementation only supports with kernel-mode queues.
+ * In other terms, userspace doesn't have access to the ring-buffer.
+ * Instead, userspace passes indirect command stream buffers that are
+ * called from the queue ring-buffer by the kernel using a pre-defined
+ * sequence of command stream instructions to ensure the userspace driver
+ * always gets consistent results (cache maintenance,
+ * synchronization, ...).
+ *
+ * We rely on the drm_gpu_scheduler framework to deal with job
+ * dependencies and submission. As any other driver dealing with a
+ * FW-scheduler, we use the 1:1 entity:scheduler mode, such that each
+ * entity has its own job scheduler. When a job is ready to be executed
+ * (all its dependencies are met), it is pushed to the appropriate
+ * queue ring-buffer, and the group is scheduled for execution if it
+ * wasn't already active.
+ *
+ * Kernel-side group scheduling is timeslice-based. When we have less
+ * groups than there are slots, the periodic tick is disabled and we
+ * just let the FW schedule the active groups. When there are more
+ * groups than slots, we let each group a chance to execute stuff for
+ * a given amount of time, and then re-evaluate and pick new groups
+ * to schedule. The group selection algorithm is based on
+ * priority+round-robin.
+ *
+ * Even though user-mode queues is out of the scope right now, the
+ * current design takes them into account by avoiding any guess on the
+ * group/queue state that would be based on information we wouldn't have
+ * if userspace was in charge of the ring-buffer. That's also one of the
+ * reason we don't do 'cooperative' scheduling (encoding FW group slot
+ * reservation as dma_fence that would be returned from the
+ * drm_gpu_scheduler::prepare_job() hook, and treating group rotation as
+ * a queue of waiters, ordered by job submission order). This approach
+ * would work for kernel-mode queues, but would make user-mode queues a
+ * lot more complicated to retrofit.
+ */
+
+#define JOB_TIMEOUT_MS				5000
+
+#define MIN_CS_PER_CSG				8
+
+#define MIN_CSGS				3
+#define MAX_CSG_PRIO				0xf
+
+struct panthor_group;
+
+/**
+ * struct panthor_csg_slot - Command stream group slot
+ *
+ * This represents a FW slot for a scheduling group.
+ */
+struct panthor_csg_slot {
+	/** @group: Scheduling group bound to this slot. */
+	struct panthor_group *group;
+
+	/** @priority: Group priority. */
+	u8 priority;
+
+	/**
+	 * @idle: True if the group bound to this slot is idle.
+	 *
+	 * A group is idle when it has nothing waiting for execution on
+	 * all its queues, or when queues are blocked waiting for something
+	 * to happen (synchronization object).
+	 */
+	bool idle;
+};
+
+/**
+ * enum panthor_csg_priority - Group priority
+ */
+enum panthor_csg_priority {
+	/** @PANTHOR_CSG_PRIORITY_LOW: Low priority group. */
+	PANTHOR_CSG_PRIORITY_LOW = 0,
+
+	/** @PANTHOR_CSG_PRIORITY_MEDIUM: Medium priority group. */
+	PANTHOR_CSG_PRIORITY_MEDIUM,
+
+	/** @PANTHOR_CSG_PRIORITY_HIGH: High priority group. */
+	PANTHOR_CSG_PRIORITY_HIGH,
+
+	/**
+	 * @PANTHOR_CSG_PRIORITY_RT: Real-time priority group.
+	 *
+	 * Real-time priority allows one to preempt scheduling of other
+	 * non-real-time groups. When such a group becomes executable,
+	 * it will evict the group with the lowest non-rt priority if
+	 * there's no free group slot available.
+	 *
+	 * Currently not exposed to userspace.
+	 */
+	PANTHOR_CSG_PRIORITY_RT,
+
+	/** @PANTHOR_CSG_PRIORITY_COUNT: Number of priority levels. */
+	PANTHOR_CSG_PRIORITY_COUNT,
+};
+
+/**
+ * struct panthor_scheduler - Object used to manage the scheduler
+ */
+struct panthor_scheduler {
+	/** @ptdev: Device. */
+	struct panthor_device *ptdev;
+
+	/**
+	 * @wq: Workqueue used by our internal scheduler logic and
+	 * drm_gpu_scheduler.
+	 *
+	 * Used for the scheduler tick, group update or other kind of FW
+	 * event processing that can't be handled in the threaded interrupt
+	 * path. Also passed to the drm_gpu_scheduler instances embedded
+	 * in panthor_queue.
+	 */
+	struct workqueue_struct *wq;
+
+	/**
+	 * @heap_alloc_wq: Workqueue used to schedule tiler_oom works.
+	 *
+	 * We have a queue dedicated to heap chunk allocation works to avoid
+	 * blocking the rest of the scheduler if the allocation tries to
+	 * reclaim memory.
+	 */
+	struct workqueue_struct *heap_alloc_wq;
+
+	/** @tick_work: Work executed on a scheduling tick. */
+	struct delayed_work tick_work;
+
+	/**
+	 * @sync_upd_work: Work used to process synchronization object updates.
+	 *
+	 * We use this work to unblock queues/groups that were waiting on a
+	 * synchronization object.
+	 */
+	struct work_struct sync_upd_work;
+
+	/**
+	 * @fw_events_work: Work used to process FW events outside the interrupt path.
+	 *
+	 * Even if the interrupt is threaded, we need any event processing
+	 * that require taking the panthor_scheduler::lock to be processed
+	 * outside the interrupt path so we don't block the tick logic when
+	 * it calls panthor_fw_{csg,wait}_wait_acks(). Since most of the
+	 * event processing requires taking this lock, we just delegate all
+	 * FW event processing to the scheduler workqueue.
+	 */
+	struct work_struct fw_events_work;
+
+	/**
+	 * @fw_events: Bitmask encoding pending FW events.
+	 */
+	atomic_t fw_events;
+
+	/**
+	 * @resched_target: When the next tick should occur.
+	 *
+	 * Expressed in jiffies.
+	 */
+	u64 resched_target;
+
+	/**
+	 * @last_tick: When the last tick occurred.
+	 *
+	 * Expressed in jiffies.
+	 */
+	u64 last_tick;
+
+	/** @tick_period: Tick period in jiffies. */
+	u64 tick_period;
+
+	/**
+	 * @lock: Lock protecting access to all the scheduler fields.
+	 *
+	 * Should be taken in the tick work, the irq handler, and anywhere the @groups
+	 * fields are touched.
+	 */
+	struct mutex lock;
+
+	/** @groups: Various lists used to classify groups. */
+	struct {
+		/**
+		 * @runnable: Runnable group lists.
+		 *
+		 * When a group has queues that want to execute something,
+		 * its panthor_group::run_node should be inserted here.
+		 *
+		 * One list per-priority.
+		 */
+		struct list_head runnable[PANTHOR_CSG_PRIORITY_COUNT];
+
+		/**
+		 * @idle: Idle group lists.
+		 *
+		 * When all queues of a group are idle (either because they
+		 * have nothing to execute, or because they are blocked), the
+		 * panthor_group::run_node field should be inserted here.
+		 *
+		 * One list per-priority.
+		 */
+		struct list_head idle[PANTHOR_CSG_PRIORITY_COUNT];
+
+		/**
+		 * @waiting: List of groups whose queues are blocked on a
+		 * synchronization object.
+		 *
+		 * Insert panthor_group::wait_node here when a group is waiting
+		 * for synchronization objects to be signaled.
+		 *
+		 * This list is evaluated in the @sync_upd_work work.
+		 */
+		struct list_head waiting;
+	} groups;
+
+	/**
+	 * @csg_slots: FW command stream group slots.
+	 */
+	struct panthor_csg_slot csg_slots[MAX_CSGS];
+
+	/** @csg_slot_count: Number of command stream group slots exposed by the FW. */
+	u32 csg_slot_count;
+
+	/** @cs_slot_count: Number of command stream slot per group slot exposed by the FW. */
+	u32 cs_slot_count;
+
+	/** @as_slot_count: Number of address space slots supported by the MMU. */
+	u32 as_slot_count;
+
+	/** @used_csg_slot_count: Number of command stream group slot currently used. */
+	u32 used_csg_slot_count;
+
+	/** @sb_slot_count: Number of scoreboard slots. */
+	u32 sb_slot_count;
+
+	/**
+	 * @might_have_idle_groups: True if an active group might have become idle.
+	 *
+	 * This will force a tick, so other runnable groups can be scheduled if one
+	 * or more active groups became idle.
+	 */
+	bool might_have_idle_groups;
+
+	/** @pm: Power management related fields. */
+	struct {
+		/** @has_ref: True if the scheduler owns a runtime PM reference. */
+		bool has_ref;
+	} pm;
+
+	/** @reset: Reset related fields. */
+	struct {
+		/** @lock: Lock protecting the other reset fields. */
+		struct mutex lock;
+
+		/**
+		 * @in_progress: True if a reset is in progress.
+		 *
+		 * Set to true in panthor_sched_pre_reset() and back to false in
+		 * panthor_sched_post_reset().
+		 */
+		atomic_t in_progress;
+
+		/**
+		 * @stopped_groups: List containing all groups that were stopped
+		 * before a reset.
+		 *
+		 * Insert panthor_group::run_node in the pre_reset path.
+		 */
+		struct list_head stopped_groups;
+	} reset;
+};
+
+/**
+ * struct panthor_syncobj_32b - 32-bit FW synchronization object
+ */
+struct panthor_syncobj_32b {
+	/** @seqno: Sequence number. */
+	u32 seqno;
+
+	/**
+	 * @status: Status.
+	 *
+	 * Not zero on failure.
+	 */
+	u32 status;
+};
+
+/**
+ * struct panthor_syncobj_64b - 64-bit FW synchronization object
+ */
+struct panthor_syncobj_64b {
+	/** @seqno: Sequence number. */
+	u64 seqno;
+
+	/**
+	 * @status: Status.
+	 *
+	 * Not zero on failure.
+	 */
+	u32 status;
+
+	/** @pad: MBZ. */
+	u32 pad;
+};
+
+/**
+ * struct panthor_queue - Execution queue
+ */
+struct panthor_queue {
+	/** @scheduler: DRM scheduler used for this queue. */
+	struct drm_gpu_scheduler scheduler;
+
+	/** @entity: DRM scheduling entity used for this queue. */
+	struct drm_sched_entity entity;
+
+	/**
+	 * @remaining_time: Time remaining before the job timeout expires.
+	 *
+	 * The job timeout is suspended when the queue is not scheduled by the
+	 * FW. Every time we suspend the timer, we need to save the remaining
+	 * time so we can restore it later on.
+	 */
+	unsigned long remaining_time;
+
+	/** @timeout_suspended: True if the job timeout was suspended. */
+	bool timeout_suspended;
+
+	/**
+	 * @doorbell_id: Doorbell assigned to this queue.
+	 *
+	 * Right now, all groups share the same doorbell, and the doorbell ID
+	 * is assigned to group_slot + 1 when the group is assigned a slot. But
+	 * we might decide to provide fine grained doorbell assignment at some
+	 * point, so don't have to wake up all queues in a group every time one
+	 * of them is updated.
+	 */
+	u8 doorbell_id;
+
+	/**
+	 * @priority: Priority of the queue inside the group.
+	 *
+	 * Must be less than 16 (Only 4 bits available).
+	 */
+	u8 priority;
+#define CSF_MAX_QUEUE_PRIO	GENMASK(3, 0)
+
+	/** @ringbuf: Command stream ring-buffer. */
+	struct panthor_kernel_bo *ringbuf;
+
+	/** @iface: Firmware interface. */
+	struct {
+		/** @mem: FW memory allocated for this interface. */
+		struct panthor_kernel_bo *mem;
+
+		/** @input: Input interface. */
+		struct panthor_fw_ringbuf_input_iface *input;
+
+		/** @output: Output interface. */
+		const struct panthor_fw_ringbuf_output_iface *output;
+
+		/** @input_fw_va: FW virtual address of the input interface buffer. */
+		u32 input_fw_va;
+
+		/** @output_fw_va: FW virtual address of the output interface buffer. */
+		u32 output_fw_va;
+	} iface;
+
+	/**
+	 * @syncwait: Stores information about the synchronization object this
+	 * queue is waiting on.
+	 */
+	struct {
+		/** @gpu_va: GPU address of the synchronization object. */
+		u64 gpu_va;
+
+		/** @ref: Reference value to compare against. */
+		u64 ref;
+
+		/** @gt: True if this is a greater-than test. */
+		bool gt;
+
+		/** @sync64: True if this is a 64-bit sync object. */
+		bool sync64;
+
+		/** @bo: Buffer object holding the synchronization object. */
+		struct drm_gem_object *obj;
+
+		/** @offset: Offset of the synchronization object inside @bo. */
+		u64 offset;
+
+		/**
+		 * @kmap: Kernel mapping of the buffer object holding the
+		 * synchronization object.
+		 */
+		void *kmap;
+	} syncwait;
+
+	/** @fence_ctx: Fence context fields. */
+	struct {
+		/** @lock: Used to protect access to all fences allocated by this context. */
+		spinlock_t lock;
+
+		/**
+		 * @id: Fence context ID.
+		 *
+		 * Allocated with dma_fence_context_alloc().
+		 */
+		u64 id;
+
+		/** @seqno: Sequence number of the last initialized fence. */
+		atomic64_t seqno;
+
+		/**
+		 * @in_flight_jobs: List containing all in-flight jobs.
+		 *
+		 * Used to keep track and signal panthor_job::done_fence when the
+		 * synchronization object attached to the queue is signaled.
+		 */
+		struct list_head in_flight_jobs;
+	} fence_ctx;
+};
+
+/**
+ * enum panthor_group_state - Scheduling group state.
+ */
+enum panthor_group_state {
+	/** @PANTHOR_CS_GROUP_CREATED: Group was created, but not scheduled yet. */
+	PANTHOR_CS_GROUP_CREATED,
+
+	/** @PANTHOR_CS_GROUP_ACTIVE: Group is currently scheduled. */
+	PANTHOR_CS_GROUP_ACTIVE,
+
+	/**
+	 * @PANTHOR_CS_GROUP_SUSPENDED: Group was scheduled at least once, but is
+	 * inactive/suspended right now.
+	 */
+	PANTHOR_CS_GROUP_SUSPENDED,
+
+	/**
+	 * @PANTHOR_CS_GROUP_TERMINATED: Group was terminated.
+	 *
+	 * Can no longer be scheduled. The only allowed action is a destruction.
+	 */
+	PANTHOR_CS_GROUP_TERMINATED,
+
+	/**
+	 * @PANTHOR_CS_GROUP_UNKNOWN_STATE: Group is an unknown state.
+	 *
+	 * The FW returned an inconsistent state. The group is flagged unusable
+	 * and can no longer be scheduled. The only allowed action is a
+	 * destruction.
+	 *
+	 * When that happens, we also schedule a FW reset, to start from a fresh
+	 * state.
+	 */
+	PANTHOR_CS_GROUP_UNKNOWN_STATE,
+};
+
+/**
+ * struct panthor_group - Scheduling group object
+ */
+struct panthor_group {
+	/** @refcount: Reference count */
+	struct kref refcount;
+
+	/** @ptdev: Device. */
+	struct panthor_device *ptdev;
+
+	/** @vm: VM bound to the group. */
+	struct panthor_vm *vm;
+
+	/** @compute_core_mask: Mask of shader cores that can be used for compute jobs. */
+	u64 compute_core_mask;
+
+	/** @fragment_core_mask: Mask of shader cores that can be used for fragment jobs. */
+	u64 fragment_core_mask;
+
+	/** @tiler_core_mask: Mask of tiler cores that can be used for tiler jobs. */
+	u64 tiler_core_mask;
+
+	/** @max_compute_cores: Maximum number of shader cores used for compute jobs. */
+	u8 max_compute_cores;
+
+	/** @max_fragment_cores: Maximum number of shader cores used for fragment jobs. */
+	u8 max_fragment_cores;
+
+	/** @max_tiler_cores: Maximum number of tiler cores used for tiler jobs. */
+	u8 max_tiler_cores;
+
+	/** @priority: Group priority (check panthor_csg_priority). */
+	u8 priority;
+
+	/** @blocked_queues: Bitmask reflecting the blocked queues. */
+	u32 blocked_queues;
+
+	/** @idle_queues: Bitmask reflecting the idle queues. */
+	u32 idle_queues;
+
+	/** @fatal_lock: Lock used to protect access to fatal fields. */
+	spinlock_t fatal_lock;
+
+	/** @fatal_queues: Bitmask reflecting the queues that hit a fatal exception. */
+	u32 fatal_queues;
+
+	/** @tiler_oom: Mask of queues that have a tiler OOM event to process. */
+	atomic_t tiler_oom;
+
+	/** @queue_count: Number of queues in this group. */
+	u32 queue_count;
+
+	/** @queues: Queues owned by this group. */
+	struct panthor_queue *queues[MAX_CS_PER_CSG];
+
+	/**
+	 * @csg_id: ID of the FW group slot.
+	 *
+	 * -1 when the group is not scheduled/active.
+	 */
+	int csg_id;
+
+	/**
+	 * @destroyed: True when the group has been destroyed.
+	 *
+	 * If a group is destroyed it becomes useless: no further jobs can be submitted
+	 * to its queues. We simply wait for all references to be dropped so we can
+	 * release the group object.
+	 */
+	bool destroyed;
+
+	/**
+	 * @timedout: True when a timeout occurred on any of the queues owned by
+	 * this group.
+	 *
+	 * Timeouts can be reported by drm_sched or by the FW. In any case, any
+	 * timeout situation is unrecoverable, and the group becomes useless.
+	 * We simply wait for all references to be dropped so we can release the
+	 * group object.
+	 */
+	bool timedout;
+
+	/**
+	 * @syncobjs: Pool of per-queue synchronization objects.
+	 *
+	 * One sync object per queue. The position of the sync object is
+	 * determined by the queue index.
+	 */
+	struct panthor_kernel_bo *syncobjs;
+
+	/** @state: Group state. */
+	enum panthor_group_state state;
+
+	/**
+	 * @suspend_buf: Suspend buffer.
+	 *
+	 * Stores the state of the group and its queues when a group is suspended.
+	 * Used at resume time to restore the group in its previous state.
+	 *
+	 * The size of the suspend buffer is exposed through the FW interface.
+	 */
+	struct panthor_kernel_bo *suspend_buf;
+
+	/**
+	 * @protm_suspend_buf: Protection mode suspend buffer.
+	 *
+	 * Stores the state of the group and its queues when a group that's in
+	 * protection mode is suspended.
+	 *
+	 * Used at resume time to restore the group in its previous state.
+	 *
+	 * The size of the protection mode suspend buffer is exposed through the
+	 * FW interface.
+	 */
+	struct panthor_kernel_bo *protm_suspend_buf;
+
+	/** @sync_upd_work: Work used to check/signal job fences. */
+	struct work_struct sync_upd_work;
+
+	/** @tiler_oom_work: Work used to process tiler OOM events happening on this group. */
+	struct work_struct tiler_oom_work;
+
+	/** @term_work: Work used to finish the group termination procedure. */
+	struct work_struct term_work;
+
+	/**
+	 * @release_work: Work used to release group resources.
+	 *
+	 * We need to postpone the group release to avoid a deadlock when
+	 * the last ref is released in the tick work.
+	 */
+	struct work_struct release_work;
+
+	/**
+	 * @run_node: Node used to insert the group in the
+	 * panthor_group::groups::{runnable,idle} and
+	 * panthor_group::reset.stopped_groups lists.
+	 */
+	struct list_head run_node;
+
+	/**
+	 * @wait_node: Node used to insert the group in the
+	 * panthor_group::groups::waiting list.
+	 */
+	struct list_head wait_node;
+};
+
+/**
+ * group_queue_work() - Queue a group work
+ * @group: Group to queue the work for.
+ * @wname: Work name.
+ *
+ * Grabs a ref and queue a work item to the scheduler workqueue. If
+ * the work was already queued, we release the reference we grabbed.
+ *
+ * Work callbacks must release the reference we grabbed here.
+ */
+#define group_queue_work(group, wname) \
+	do { \
+		group_get(group); \
+		if (!queue_work((group)->ptdev->scheduler->wq, &(group)->wname ## _work)) \
+			group_put(group); \
+	} while (0)
+
+/**
+ * sched_queue_work() - Queue a scheduler work.
+ * @sched: Scheduler object.
+ * @wname: Work name.
+ *
+ * Conditionally queues a scheduler work if no reset is pending/in-progress.
+ */
+#define sched_queue_work(sched, wname) \
+	do { \
+		if (!atomic_read(&(sched)->reset.in_progress) && \
+		    !panthor_device_reset_is_pending((sched)->ptdev)) \
+			queue_work((sched)->wq, &(sched)->wname ## _work); \
+	} while (0)
+
+/**
+ * sched_queue_delayed_work() - Queue a scheduler delayed work.
+ * @sched: Scheduler object.
+ * @wname: Work name.
+ * @delay: Work delay in jiffies.
+ *
+ * Conditionally queues a scheduler delayed work if no reset is
+ * pending/in-progress.
+ */
+#define sched_queue_delayed_work(sched, wname, delay) \
+	do { \
+		if (!atomic_read(&sched->reset.in_progress) && \
+		    !panthor_device_reset_is_pending((sched)->ptdev)) \
+			mod_delayed_work((sched)->wq, &(sched)->wname ## _work, delay); \
+	} while (0)
+
+/*
+ * We currently set the maximum of groups per file to an arbitrary low value.
+ * But this can be updated if we need more.
+ */
+#define MAX_GROUPS_PER_POOL 128
+
+/**
+ * struct panthor_group_pool - Group pool
+ *
+ * Each file get assigned a group pool.
+ */
+struct panthor_group_pool {
+	/** @xa: Xarray used to manage group handles. */
+	struct xarray xa;
+};
+
+/**
+ * struct panthor_job - Used to manage GPU job
+ */
+struct panthor_job {
+	/** @base: Inherit from drm_sched_job. */
+	struct drm_sched_job base;
+
+	/** @refcount: Reference count. */
+	struct kref refcount;
+
+	/** @group: Group of the queue this job will be pushed to. */
+	struct panthor_group *group;
+
+	/** @queue_idx: Index of the queue inside @group. */
+	u32 queue_idx;
+
+	/** @call_info: Information about the userspace command stream call. */
+	struct {
+		/** @start: GPU address of the userspace command stream. */
+		u64 start;
+
+		/** @size: Size of the userspace command stream. */
+		u32 size;
+
+		/**
+		 * @latest_flush: Flush ID at the time the userspace command
+		 * stream was built.
+		 *
+		 * Needed for the flush reduction mechanism.
+		 */
+		u32 latest_flush;
+	} call_info;
+
+	/** @ringbuf: Position of this job is in the ring buffer. */
+	struct {
+		/** @start: Start offset. */
+		u64 start;
+
+		/** @end: End offset. */
+		u64 end;
+	} ringbuf;
+
+	/**
+	 * @node: Used to insert the job in the panthor_queue::fence_ctx::in_flight_jobs
+	 * list.
+	 */
+	struct list_head node;
+
+	/** @done_fence: Fence signaled when the job is finished or cancelled. */
+	struct dma_fence *done_fence;
+};
+
+static void
+panthor_queue_put_syncwait_obj(struct panthor_queue *queue)
+{
+	if (queue->syncwait.kmap) {
+		struct iosys_map map = IOSYS_MAP_INIT_VADDR(queue->syncwait.kmap);
+
+		drm_gem_vunmap_unlocked(queue->syncwait.obj, &map);
+		queue->syncwait.kmap = NULL;
+	}
+
+	drm_gem_object_put(queue->syncwait.obj);
+	queue->syncwait.obj = NULL;
+}
+
+static void *
+panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue *queue)
+{
+	struct panthor_device *ptdev = group->ptdev;
+	struct panthor_gem_object *bo;
+	struct iosys_map map;
+	int ret;
+
+	if (queue->syncwait.kmap)
+		return queue->syncwait.kmap + queue->syncwait.offset;
+
+	bo = panthor_vm_get_bo_for_va(group->vm,
+				      queue->syncwait.gpu_va,
+				      &queue->syncwait.offset);
+	if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(bo)))
+		goto err_put_syncwait_obj;
+
+	queue->syncwait.obj = &bo->base.base;
+	ret = drm_gem_vmap_unlocked(queue->syncwait.obj, &map);
+	if (drm_WARN_ON(&ptdev->base, ret))
+		goto err_put_syncwait_obj;
+
+	queue->syncwait.kmap = map.vaddr;
+	if (drm_WARN_ON(&ptdev->base, !queue->syncwait.kmap))
+		goto err_put_syncwait_obj;
+
+	return queue->syncwait.kmap + queue->syncwait.offset;
+
+err_put_syncwait_obj:
+	panthor_queue_put_syncwait_obj(queue);
+	return NULL;
+}
+
+static void group_free_queue(struct panthor_group *group, struct panthor_queue *queue)
+{
+	if (IS_ERR_OR_NULL(queue))
+		return;
+
+	if (queue->entity.fence_context)
+		drm_sched_entity_destroy(&queue->entity);
+
+	if (queue->scheduler.ops)
+		drm_sched_fini(&queue->scheduler);
+
+	panthor_queue_put_syncwait_obj(queue);
+
+	panthor_kernel_bo_destroy(queue->ringbuf);
+	panthor_kernel_bo_destroy(queue->iface.mem);
+
+	kfree(queue);
+}
+
+static void group_release_work(struct work_struct *work)
+{
+	struct panthor_group *group = container_of(work,
+						   struct panthor_group,
+						   release_work);
+	u32 i;
+
+	for (i = 0; i < group->queue_count; i++)
+		group_free_queue(group, group->queues[i]);
+
+	panthor_kernel_bo_destroy(group->suspend_buf);
+	panthor_kernel_bo_destroy(group->protm_suspend_buf);
+	panthor_kernel_bo_destroy(group->syncobjs);
+
+	panthor_vm_put(group->vm);
+	kfree(group);
+}
+
+static void group_release(struct kref *kref)
+{
+	struct panthor_group *group = container_of(kref,
+						   struct panthor_group,
+						   refcount);
+	struct panthor_device *ptdev = group->ptdev;
+
+	drm_WARN_ON(&ptdev->base, group->csg_id >= 0);
+	drm_WARN_ON(&ptdev->base, !list_empty(&group->run_node));
+	drm_WARN_ON(&ptdev->base, !list_empty(&group->wait_node));
+
+	queue_work(panthor_cleanup_wq, &group->release_work);
+}
+
+static void group_put(struct panthor_group *group)
+{
+	if (group)
+		kref_put(&group->refcount, group_release);
+}
+
+static struct panthor_group *
+group_get(struct panthor_group *group)
+{
+	if (group)
+		kref_get(&group->refcount);
+
+	return group;
+}
+
+/**
+ * group_bind_locked() - Bind a group to a group slot
+ * @group: Group.
+ * @csg_id: Slot.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int
+group_bind_locked(struct panthor_group *group, u32 csg_id)
+{
+	struct panthor_device *ptdev = group->ptdev;
+	struct panthor_csg_slot *csg_slot;
+	int ret;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	if (drm_WARN_ON(&ptdev->base, group->csg_id != -1 || csg_id >= MAX_CSGS ||
+			ptdev->scheduler->csg_slots[csg_id].group))
+		return -EINVAL;
+
+	ret = panthor_vm_active(group->vm);
+	if (ret)
+		return ret;
+
+	csg_slot = &ptdev->scheduler->csg_slots[csg_id];
+	group_get(group);
+	group->csg_id = csg_id;
+
+	/* Dummy doorbell allocation: doorbell is assigned to the group and
+	 * all queues use the same doorbell.
+	 *
+	 * TODO: Implement LRU-based doorbell assignment, so the most often
+	 * updated queues get their own doorbell, thus avoiding useless checks
+	 * on queues belonging to the same group that are rarely updated.
+	 */
+	for (u32 i = 0; i < group->queue_count; i++)
+		group->queues[i]->doorbell_id = csg_id + 1;
+
+	csg_slot->group = group;
+
+	return 0;
+}
+
+/**
+ * group_unbind_locked() - Unbind a group from a slot.
+ * @group: Group to unbind.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int
+group_unbind_locked(struct panthor_group *group)
+{
+	struct panthor_device *ptdev = group->ptdev;
+	struct panthor_csg_slot *slot;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	if (drm_WARN_ON(&ptdev->base, group->csg_id < 0 || group->csg_id >= MAX_CSGS))
+		return -EINVAL;
+
+	if (drm_WARN_ON(&ptdev->base, group->state == PANTHOR_CS_GROUP_ACTIVE))
+		return -EINVAL;
+
+	slot = &ptdev->scheduler->csg_slots[group->csg_id];
+	panthor_vm_idle(group->vm);
+	group->csg_id = -1;
+
+	/* Tiler OOM events will be re-issued next time the group is scheduled. */
+	atomic_set(&group->tiler_oom, 0);
+	cancel_work(&group->tiler_oom_work);
+
+	for (u32 i = 0; i < group->queue_count; i++)
+		group->queues[i]->doorbell_id = -1;
+
+	slot->group = NULL;
+
+	group_put(group);
+	return 0;
+}
+
+/**
+ * cs_slot_prog_locked() - Program a queue slot
+ * @ptdev: Device.
+ * @csg_id: Group slot ID.
+ * @cs_id: Queue slot ID.
+ *
+ * Program a queue slot with the queue information so things can start being
+ * executed on this queue.
+ *
+ * The group slot must have a group bound to it already (group_bind_locked()).
+ */
+static void
+cs_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
+{
+	struct panthor_queue *queue = ptdev->scheduler->csg_slots[csg_id].group->queues[cs_id];
+	struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	queue->iface.input->extract = queue->iface.output->extract;
+	drm_WARN_ON(&ptdev->base, queue->iface.input->insert < queue->iface.input->extract);
+
+	cs_iface->input->ringbuf_base = panthor_kernel_bo_gpuva(queue->ringbuf);
+	cs_iface->input->ringbuf_size = panthor_kernel_bo_size(queue->ringbuf);
+	cs_iface->input->ringbuf_input = queue->iface.input_fw_va;
+	cs_iface->input->ringbuf_output = queue->iface.output_fw_va;
+	cs_iface->input->config = CS_CONFIG_PRIORITY(queue->priority) |
+				  CS_CONFIG_DOORBELL(queue->doorbell_id);
+	cs_iface->input->ack_irq_mask = ~0;
+	panthor_fw_update_reqs(cs_iface, req,
+			       CS_IDLE_SYNC_WAIT |
+			       CS_IDLE_EMPTY |
+			       CS_STATE_START |
+			       CS_EXTRACT_EVENT,
+			       CS_IDLE_SYNC_WAIT |
+			       CS_IDLE_EMPTY |
+			       CS_STATE_MASK |
+			       CS_EXTRACT_EVENT);
+	if (queue->iface.input->insert != queue->iface.input->extract && queue->timeout_suspended) {
+		drm_sched_resume_timeout(&queue->scheduler, queue->remaining_time);
+		queue->timeout_suspended = false;
+	}
+}
+
+/**
+ * cs_slot_reset_locked() - Reset a queue slot
+ * @ptdev: Device.
+ * @csg_id: Group slot.
+ * @cs_id: Queue slot.
+ *
+ * Change the queue slot state to STOP and suspend the queue timeout if
+ * the queue is not blocked.
+ *
+ * The group slot must have a group bound to it (group_bind_locked()).
+ */
+static int
+cs_slot_reset_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
+{
+	struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
+	struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group;
+	struct panthor_queue *queue = group->queues[cs_id];
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	panthor_fw_update_reqs(cs_iface, req,
+			       CS_STATE_STOP,
+			       CS_STATE_MASK);
+
+	/* If the queue is blocked, we want to keep the timeout running, so
+	 * we can detect unbounded waits and kill the group when that happens.
+	 */
+	if (!(group->blocked_queues & BIT(cs_id)) && !queue->timeout_suspended) {
+		queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler);
+		queue->timeout_suspended = true;
+		WARN_ON(queue->remaining_time > msecs_to_jiffies(JOB_TIMEOUT_MS));
+	}
+
+	return 0;
+}
+
+/**
+ * csg_slot_sync_priority_locked() - Synchronize the group slot priority
+ * @ptdev: Device.
+ * @csg_id: Group slot ID.
+ *
+ * Group slot priority update happens asynchronously. When we receive a
+ * %CSG_ENDPOINT_CONFIG, we know the update is effective, and can
+ * reflect it to our panthor_csg_slot object.
+ */
+static void
+csg_slot_sync_priority_locked(struct panthor_device *ptdev, u32 csg_id)
+{
+	struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
+	struct panthor_fw_csg_iface *csg_iface;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
+	csg_slot->priority = (csg_iface->input->endpoint_req & CSG_EP_REQ_PRIORITY_MASK) >> 28;
+}
+
+/**
+ * cs_slot_sync_queue_state_locked() - Synchronize the queue slot priority
+ * @ptdev: Device.
+ * @csg_id: Group slot.
+ * @cs_id: Queue slot.
+ *
+ * Queue state is updated on group suspend or STATUS_UPDATE event.
+ */
+static void
+cs_slot_sync_queue_state_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
+{
+	struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group;
+	struct panthor_queue *queue = group->queues[cs_id];
+	struct panthor_fw_cs_iface *cs_iface =
+		panthor_fw_get_cs_iface(group->ptdev, csg_id, cs_id);
+
+	u32 status_wait_cond;
+
+	switch (cs_iface->output->status_blocked_reason) {
+	case CS_STATUS_BLOCKED_REASON_UNBLOCKED:
+		if (queue->iface.input->insert == queue->iface.output->extract &&
+		    cs_iface->output->status_scoreboards == 0)
+			group->idle_queues |= BIT(cs_id);
+		break;
+
+	case CS_STATUS_BLOCKED_REASON_SYNC_WAIT:
+		if (list_empty(&group->wait_node)) {
+			list_move_tail(&group->wait_node,
+				       &group->ptdev->scheduler->groups.waiting);
+		}
+		group->blocked_queues |= BIT(cs_id);
+		queue->syncwait.gpu_va = cs_iface->output->status_wait_sync_ptr;
+		queue->syncwait.ref = cs_iface->output->status_wait_sync_value;
+		status_wait_cond = cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_COND_MASK;
+		queue->syncwait.gt = status_wait_cond == CS_STATUS_WAIT_SYNC_COND_GT;
+		if (cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_64B) {
+			u64 sync_val_hi = cs_iface->output->status_wait_sync_value_hi;
+
+			queue->syncwait.sync64 = true;
+			queue->syncwait.ref |= sync_val_hi << 32;
+		} else {
+			queue->syncwait.sync64 = false;
+		}
+		break;
+
+	default:
+		/* Other reasons are not blocking. Consider the queue as runnable
+		 * in those cases.
+		 */
+		break;
+	}
+}
+
+static void
+csg_slot_sync_queues_state_locked(struct panthor_device *ptdev, u32 csg_id)
+{
+	struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
+	struct panthor_group *group = csg_slot->group;
+	u32 i;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	group->idle_queues = 0;
+	group->blocked_queues = 0;
+
+	for (i = 0; i < group->queue_count; i++) {
+		if (group->queues[i])
+			cs_slot_sync_queue_state_locked(ptdev, csg_id, i);
+	}
+}
+
+static void
+csg_slot_sync_state_locked(struct panthor_device *ptdev, u32 csg_id)
+{
+	struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
+	struct panthor_fw_csg_iface *csg_iface;
+	struct panthor_group *group;
+	enum panthor_group_state new_state, old_state;
+	u32 csg_state;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
+	group = csg_slot->group;
+
+	if (!group)
+		return;
+
+	old_state = group->state;
+	csg_state = csg_iface->output->ack & CSG_STATE_MASK;
+	switch (csg_state) {
+	case CSG_STATE_START:
+	case CSG_STATE_RESUME:
+		new_state = PANTHOR_CS_GROUP_ACTIVE;
+		break;
+	case CSG_STATE_TERMINATE:
+		new_state = PANTHOR_CS_GROUP_TERMINATED;
+		break;
+	case CSG_STATE_SUSPEND:
+		new_state = PANTHOR_CS_GROUP_SUSPENDED;
+		break;
+	default:
+		/* The unknown state might be caused by a FW state corruption,
+		 * which means the group metadata can't be trusted anymore, and
+		 * the SUSPEND operation might propagate the corruption to the
+		 * suspend buffers. Flag the group state as unknown to make
+		 * sure it's unusable after that point.
+		 */
+		drm_err(&ptdev->base, "Invalid state on CSG %d (state=%d)",
+			csg_id, csg_state);
+		new_state = PANTHOR_CS_GROUP_UNKNOWN_STATE;
+		break;
+	}
+
+	if (old_state == new_state)
+		return;
+
+	/* The unknown state might be caused by a FW issue, reset the FW to
+	 * take a fresh start.
+	 */
+	if (new_state == PANTHOR_CS_GROUP_UNKNOWN_STATE)
+		panthor_device_schedule_reset(ptdev);
+
+	if (new_state == PANTHOR_CS_GROUP_SUSPENDED)
+		csg_slot_sync_queues_state_locked(ptdev, csg_id);
+
+	if (old_state == PANTHOR_CS_GROUP_ACTIVE) {
+		u32 i;
+
+		/* Reset the queue slots so we start from a clean
+		 * state when starting/resuming a new group on this
+		 * CSG slot. No wait needed here, and no ringbell
+		 * either, since the CS slot will only be re-used
+		 * on the next CSG start operation.
+		 */
+		for (i = 0; i < group->queue_count; i++) {
+			if (group->queues[i])
+				cs_slot_reset_locked(ptdev, csg_id, i);
+		}
+	}
+
+	group->state = new_state;
+}
+
+static int
+csg_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 priority)
+{
+	struct panthor_fw_csg_iface *csg_iface;
+	struct panthor_csg_slot *csg_slot;
+	struct panthor_group *group;
+	u32 queue_mask = 0, i;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	if (priority > MAX_CSG_PRIO)
+		return -EINVAL;
+
+	if (drm_WARN_ON(&ptdev->base, csg_id >= MAX_CSGS))
+		return -EINVAL;
+
+	csg_slot = &ptdev->scheduler->csg_slots[csg_id];
+	group = csg_slot->group;
+	if (!group || group->state == PANTHOR_CS_GROUP_ACTIVE)
+		return 0;
+
+	csg_iface = panthor_fw_get_csg_iface(group->ptdev, csg_id);
+
+	for (i = 0; i < group->queue_count; i++) {
+		if (group->queues[i]) {
+			cs_slot_prog_locked(ptdev, csg_id, i);
+			queue_mask |= BIT(i);
+		}
+	}
+
+	csg_iface->input->allow_compute = group->compute_core_mask;
+	csg_iface->input->allow_fragment = group->fragment_core_mask;
+	csg_iface->input->allow_other = group->tiler_core_mask;
+	csg_iface->input->endpoint_req = CSG_EP_REQ_COMPUTE(group->max_compute_cores) |
+					 CSG_EP_REQ_FRAGMENT(group->max_fragment_cores) |
+					 CSG_EP_REQ_TILER(group->max_tiler_cores) |
+					 CSG_EP_REQ_PRIORITY(priority);
+	csg_iface->input->config = panthor_vm_as(group->vm);
+
+	if (group->suspend_buf)
+		csg_iface->input->suspend_buf = panthor_kernel_bo_gpuva(group->suspend_buf);
+	else
+		csg_iface->input->suspend_buf = 0;
+
+	if (group->protm_suspend_buf) {
+		csg_iface->input->protm_suspend_buf =
+			panthor_kernel_bo_gpuva(group->protm_suspend_buf);
+	} else {
+		csg_iface->input->protm_suspend_buf = 0;
+	}
+
+	csg_iface->input->ack_irq_mask = ~0;
+	panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, queue_mask);
+	return 0;
+}
+
+static void
+cs_slot_process_fatal_event_locked(struct panthor_device *ptdev,
+				   u32 csg_id, u32 cs_id)
+{
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
+	struct panthor_group *group = csg_slot->group;
+	struct panthor_fw_cs_iface *cs_iface;
+	u32 fatal;
+	u64 info;
+
+	lockdep_assert_held(&sched->lock);
+
+	cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
+	fatal = cs_iface->output->fatal;
+	info = cs_iface->output->fatal_info;
+
+	if (group)
+		group->fatal_queues |= BIT(cs_id);
+
+	if (CS_EXCEPTION_TYPE(fatal) == DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE) {
+		/* If this exception is unrecoverable, queue a reset, and make
+		 * sure we stop scheduling groups until the reset has happened.
+		 */
+		panthor_device_schedule_reset(ptdev);
+		cancel_delayed_work(&sched->tick_work);
+	} else {
+		sched_queue_delayed_work(sched, tick, 0);
+	}
+
+	drm_warn(&ptdev->base,
+		 "CSG slot %d CS slot: %d\n"
+		 "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n"
+		 "CS_FATAL.EXCEPTION_DATA: 0x%x\n"
+		 "CS_FATAL_INFO.EXCEPTION_DATA: 0x%llx\n",
+		 csg_id, cs_id,
+		 (unsigned int)CS_EXCEPTION_TYPE(fatal),
+		 panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fatal)),
+		 (unsigned int)CS_EXCEPTION_DATA(fatal),
+		 info);
+}
+
+static void
+cs_slot_process_fault_event_locked(struct panthor_device *ptdev,
+				   u32 csg_id, u32 cs_id)
+{
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
+	struct panthor_group *group = csg_slot->group;
+	struct panthor_queue *queue = group && cs_id < group->queue_count ?
+				      group->queues[cs_id] : NULL;
+	struct panthor_fw_cs_iface *cs_iface;
+	u32 fault;
+	u64 info;
+
+	lockdep_assert_held(&sched->lock);
+
+	cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
+	fault = cs_iface->output->fault;
+	info = cs_iface->output->fault_info;
+
+	if (queue && CS_EXCEPTION_TYPE(fault) == DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT) {
+		u64 cs_extract = queue->iface.output->extract;
+		struct panthor_job *job;
+
+		spin_lock(&queue->fence_ctx.lock);
+		list_for_each_entry(job, &queue->fence_ctx.in_flight_jobs, node) {
+			if (cs_extract >= job->ringbuf.end)
+				continue;
+
+			if (cs_extract < job->ringbuf.start)
+				break;
+
+			dma_fence_set_error(job->done_fence, -EINVAL);
+		}
+		spin_unlock(&queue->fence_ctx.lock);
+	}
+
+	drm_warn(&ptdev->base,
+		 "CSG slot %d CS slot: %d\n"
+		 "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n"
+		 "CS_FAULT.EXCEPTION_DATA: 0x%x\n"
+		 "CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n",
+		 csg_id, cs_id,
+		 (unsigned int)CS_EXCEPTION_TYPE(fault),
+		 panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fault)),
+		 (unsigned int)CS_EXCEPTION_DATA(fault),
+		 info);
+}
+
+static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id)
+{
+	struct panthor_device *ptdev = group->ptdev;
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	u32 renderpasses_in_flight, pending_frag_count;
+	struct panthor_heap_pool *heaps = NULL;
+	u64 heap_address, new_chunk_va = 0;
+	u32 vt_start, vt_end, frag_end;
+	int ret, csg_id;
+
+	mutex_lock(&sched->lock);
+	csg_id = group->csg_id;
+	if (csg_id >= 0) {
+		struct panthor_fw_cs_iface *cs_iface;
+
+		cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
+		heaps = panthor_vm_get_heap_pool(group->vm, false);
+		heap_address = cs_iface->output->heap_address;
+		vt_start = cs_iface->output->heap_vt_start;
+		vt_end = cs_iface->output->heap_vt_end;
+		frag_end = cs_iface->output->heap_frag_end;
+		renderpasses_in_flight = vt_start - frag_end;
+		pending_frag_count = vt_end - frag_end;
+	}
+	mutex_unlock(&sched->lock);
+
+	/* The group got scheduled out, we stop here. We will get a new tiler OOM event
+	 * when it's scheduled again.
+	 */
+	if (unlikely(csg_id < 0))
+		return 0;
+
+	if (IS_ERR(heaps) || frag_end > vt_end || vt_end >= vt_start) {
+		ret = -EINVAL;
+	} else {
+		/* We do the allocation without holding the scheduler lock to avoid
+		 * blocking the scheduling.
+		 */
+		ret = panthor_heap_grow(heaps, heap_address,
+					renderpasses_in_flight,
+					pending_frag_count, &new_chunk_va);
+	}
+
+	/* If the heap context doesn't have memory for us, we want to let the
+	 * FW try to reclaim memory by waiting for fragment jobs to land or by
+	 * executing the tiler OOM exception handler, which is supposed to
+	 * implement incremental rendering.
+	 */
+	if (ret && ret != -ENOMEM) {
+		drm_warn(&ptdev->base, "Failed to extend the tiler heap\n");
+		group->fatal_queues |= BIT(cs_id);
+		sched_queue_delayed_work(sched, tick, 0);
+		goto out_put_heap_pool;
+	}
+
+	mutex_lock(&sched->lock);
+	csg_id = group->csg_id;
+	if (csg_id >= 0) {
+		struct panthor_fw_csg_iface *csg_iface;
+		struct panthor_fw_cs_iface *cs_iface;
+
+		csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
+		cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
+
+		cs_iface->input->heap_start = new_chunk_va;
+		cs_iface->input->heap_end = new_chunk_va;
+		panthor_fw_update_reqs(cs_iface, req, cs_iface->output->ack, CS_TILER_OOM);
+		panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, BIT(cs_id));
+		panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id));
+	}
+	mutex_unlock(&sched->lock);
+
+	/* We allocated a chunck, but couldn't link it to the heap
+	 * context because the group was scheduled out while we were
+	 * allocating memory. We need to return this chunk to the heap.
+	 */
+	if (unlikely(csg_id < 0 && new_chunk_va))
+		panthor_heap_return_chunk(heaps, heap_address, new_chunk_va);
+
+	ret = 0;
+
+out_put_heap_pool:
+	panthor_heap_pool_put(heaps);
+	return ret;
+}
+
+static void group_tiler_oom_work(struct work_struct *work)
+{
+	struct panthor_group *group =
+		container_of(work, struct panthor_group, tiler_oom_work);
+	u32 tiler_oom = atomic_xchg(&group->tiler_oom, 0);
+
+	while (tiler_oom) {
+		u32 cs_id = ffs(tiler_oom) - 1;
+
+		group_process_tiler_oom(group, cs_id);
+		tiler_oom &= ~BIT(cs_id);
+	}
+
+	group_put(group);
+}
+
+static void
+cs_slot_process_tiler_oom_event_locked(struct panthor_device *ptdev,
+				       u32 csg_id, u32 cs_id)
+{
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
+	struct panthor_group *group = csg_slot->group;
+
+	lockdep_assert_held(&sched->lock);
+
+	if (drm_WARN_ON(&ptdev->base, !group))
+		return;
+
+	atomic_or(BIT(cs_id), &group->tiler_oom);
+
+	/* We don't use group_queue_work() here because we want to queue the
+	 * work item to the heap_alloc_wq.
+	 */
+	group_get(group);
+	if (!queue_work(sched->heap_alloc_wq, &group->tiler_oom_work))
+		group_put(group);
+}
+
+static bool cs_slot_process_irq_locked(struct panthor_device *ptdev,
+				       u32 csg_id, u32 cs_id)
+{
+	struct panthor_fw_cs_iface *cs_iface;
+	u32 req, ack, events;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
+	req = cs_iface->input->req;
+	ack = cs_iface->output->ack;
+	events = (req ^ ack) & CS_EVT_MASK;
+
+	if (events & CS_FATAL)
+		cs_slot_process_fatal_event_locked(ptdev, csg_id, cs_id);
+
+	if (events & CS_FAULT)
+		cs_slot_process_fault_event_locked(ptdev, csg_id, cs_id);
+
+	if (events & CS_TILER_OOM)
+		cs_slot_process_tiler_oom_event_locked(ptdev, csg_id, cs_id);
+
+	/* We don't acknowledge the TILER_OOM event since its handling is
+	 * deferred to a separate work.
+	 */
+	panthor_fw_update_reqs(cs_iface, req, ack, CS_FATAL | CS_FAULT);
+
+	return (events & (CS_FAULT | CS_TILER_OOM)) != 0;
+}
+
+static void csg_slot_sync_idle_state_locked(struct panthor_device *ptdev, u32 csg_id)
+{
+	struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
+	struct panthor_fw_csg_iface *csg_iface;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
+	csg_slot->idle = csg_iface->output->status_state & CSG_STATUS_STATE_IS_IDLE;
+}
+
+static void csg_slot_process_idle_event_locked(struct panthor_device *ptdev, u32 csg_id)
+{
+	struct panthor_scheduler *sched = ptdev->scheduler;
+
+	lockdep_assert_held(&sched->lock);
+
+	sched->might_have_idle_groups = true;
+
+	/* Schedule a tick so we can evict idle groups and schedule non-idle
+	 * ones. This will also update runtime PM and devfreq busy/idle states,
+	 * so the device can lower its frequency or get suspended.
+	 */
+	sched_queue_delayed_work(sched, tick, 0);
+}
+
+static void csg_slot_sync_update_locked(struct panthor_device *ptdev,
+					u32 csg_id)
+{
+	struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
+	struct panthor_group *group = csg_slot->group;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	if (group)
+		group_queue_work(group, sync_upd);
+
+	sched_queue_work(ptdev->scheduler, sync_upd);
+}
+
+static void
+csg_slot_process_progress_timer_event_locked(struct panthor_device *ptdev, u32 csg_id)
+{
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
+	struct panthor_group *group = csg_slot->group;
+
+	lockdep_assert_held(&sched->lock);
+
+	drm_warn(&ptdev->base, "CSG slot %d progress timeout\n", csg_id);
+
+	group = csg_slot->group;
+	if (!drm_WARN_ON(&ptdev->base, !group))
+		group->timedout = true;
+
+	sched_queue_delayed_work(sched, tick, 0);
+}
+
+static void sched_process_csg_irq_locked(struct panthor_device *ptdev, u32 csg_id)
+{
+	u32 req, ack, cs_irq_req, cs_irq_ack, cs_irqs, csg_events;
+	struct panthor_fw_csg_iface *csg_iface;
+	u32 ring_cs_db_mask = 0;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	if (drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count))
+		return;
+
+	csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
+	req = READ_ONCE(csg_iface->input->req);
+	ack = READ_ONCE(csg_iface->output->ack);
+	cs_irq_req = READ_ONCE(csg_iface->output->cs_irq_req);
+	cs_irq_ack = READ_ONCE(csg_iface->input->cs_irq_ack);
+	csg_events = (req ^ ack) & CSG_EVT_MASK;
+
+	/* There may not be any pending CSG/CS interrupts to process */
+	if (req == ack && cs_irq_req == cs_irq_ack)
+		return;
+
+	/* Immediately set IRQ_ACK bits to be same as the IRQ_REQ bits before
+	 * examining the CS_ACK & CS_REQ bits. This would ensure that Host
+	 * doesn't miss an interrupt for the CS in the race scenario where
+	 * whilst Host is servicing an interrupt for the CS, firmware sends
+	 * another interrupt for that CS.
+	 */
+	csg_iface->input->cs_irq_ack = cs_irq_req;
+
+	panthor_fw_update_reqs(csg_iface, req, ack,
+			       CSG_SYNC_UPDATE |
+			       CSG_IDLE |
+			       CSG_PROGRESS_TIMER_EVENT);
+
+	if (csg_events & CSG_IDLE)
+		csg_slot_process_idle_event_locked(ptdev, csg_id);
+
+	if (csg_events & CSG_PROGRESS_TIMER_EVENT)
+		csg_slot_process_progress_timer_event_locked(ptdev, csg_id);
+
+	cs_irqs = cs_irq_req ^ cs_irq_ack;
+	while (cs_irqs) {
+		u32 cs_id = ffs(cs_irqs) - 1;
+
+		if (cs_slot_process_irq_locked(ptdev, csg_id, cs_id))
+			ring_cs_db_mask |= BIT(cs_id);
+
+		cs_irqs &= ~BIT(cs_id);
+	}
+
+	if (csg_events & CSG_SYNC_UPDATE)
+		csg_slot_sync_update_locked(ptdev, csg_id);
+
+	if (ring_cs_db_mask)
+		panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, ring_cs_db_mask);
+
+	panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id));
+}
+
+static void sched_process_idle_event_locked(struct panthor_device *ptdev)
+{
+	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	/* Acknowledge the idle event and schedule a tick. */
+	panthor_fw_update_reqs(glb_iface, req, glb_iface->output->ack, GLB_IDLE);
+	sched_queue_delayed_work(ptdev->scheduler, tick, 0);
+}
+
+/**
+ * sched_process_global_irq_locked() - Process the scheduling part of a global IRQ
+ * @ptdev: Device.
+ */
+static void sched_process_global_irq_locked(struct panthor_device *ptdev)
+{
+	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
+	u32 req, ack, evts;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	req = READ_ONCE(glb_iface->input->req);
+	ack = READ_ONCE(glb_iface->output->ack);
+	evts = (req ^ ack) & GLB_EVT_MASK;
+
+	if (evts & GLB_IDLE)
+		sched_process_idle_event_locked(ptdev);
+}
+
+static void process_fw_events_work(struct work_struct *work)
+{
+	struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler,
+						      fw_events_work);
+	u32 events = atomic_xchg(&sched->fw_events, 0);
+	struct panthor_device *ptdev = sched->ptdev;
+
+	mutex_lock(&sched->lock);
+
+	if (events & JOB_INT_GLOBAL_IF) {
+		sched_process_global_irq_locked(ptdev);
+		events &= ~JOB_INT_GLOBAL_IF;
+	}
+
+	while (events) {
+		u32 csg_id = ffs(events) - 1;
+
+		sched_process_csg_irq_locked(ptdev, csg_id);
+		events &= ~BIT(csg_id);
+	}
+
+	mutex_unlock(&sched->lock);
+}
+
+/**
+ * panthor_sched_report_fw_events() - Report FW events to the scheduler.
+ */
+void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events)
+{
+	if (!ptdev->scheduler)
+		return;
+
+	atomic_or(events, &ptdev->scheduler->fw_events);
+	sched_queue_work(ptdev->scheduler, fw_events);
+}
+
+static const char *fence_get_driver_name(struct dma_fence *fence)
+{
+	return "panthor";
+}
+
+static const char *queue_fence_get_timeline_name(struct dma_fence *fence)
+{
+	return "queue-fence";
+}
+
+static const struct dma_fence_ops panthor_queue_fence_ops = {
+	.get_driver_name = fence_get_driver_name,
+	.get_timeline_name = queue_fence_get_timeline_name,
+};
+
+struct panthor_csg_slots_upd_ctx {
+	u32 update_mask;
+	u32 timedout_mask;
+	struct {
+		u32 value;
+		u32 mask;
+	} requests[MAX_CSGS];
+};
+
+static void csgs_upd_ctx_init(struct panthor_csg_slots_upd_ctx *ctx)
+{
+	memset(ctx, 0, sizeof(*ctx));
+}
+
+static void csgs_upd_ctx_queue_reqs(struct panthor_device *ptdev,
+				    struct panthor_csg_slots_upd_ctx *ctx,
+				    u32 csg_id, u32 value, u32 mask)
+{
+	if (drm_WARN_ON(&ptdev->base, !mask) ||
+	    drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count))
+		return;
+
+	ctx->requests[csg_id].value = (ctx->requests[csg_id].value & ~mask) | (value & mask);
+	ctx->requests[csg_id].mask |= mask;
+	ctx->update_mask |= BIT(csg_id);
+}
+
+static int csgs_upd_ctx_apply_locked(struct panthor_device *ptdev,
+				     struct panthor_csg_slots_upd_ctx *ctx)
+{
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	u32 update_slots = ctx->update_mask;
+
+	lockdep_assert_held(&sched->lock);
+
+	if (!ctx->update_mask)
+		return 0;
+
+	while (update_slots) {
+		struct panthor_fw_csg_iface *csg_iface;
+		u32 csg_id = ffs(update_slots) - 1;
+
+		update_slots &= ~BIT(csg_id);
+		csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
+		panthor_fw_update_reqs(csg_iface, req,
+				       ctx->requests[csg_id].value,
+				       ctx->requests[csg_id].mask);
+	}
+
+	panthor_fw_ring_csg_doorbells(ptdev, ctx->update_mask);
+
+	update_slots = ctx->update_mask;
+	while (update_slots) {
+		struct panthor_fw_csg_iface *csg_iface;
+		u32 csg_id = ffs(update_slots) - 1;
+		u32 req_mask = ctx->requests[csg_id].mask, acked;
+		int ret;
+
+		update_slots &= ~BIT(csg_id);
+		csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
+
+		ret = panthor_fw_csg_wait_acks(ptdev, csg_id, req_mask, &acked, 100);
+
+		if (acked & CSG_ENDPOINT_CONFIG)
+			csg_slot_sync_priority_locked(ptdev, csg_id);
+
+		if (acked & CSG_STATE_MASK)
+			csg_slot_sync_state_locked(ptdev, csg_id);
+
+		if (acked & CSG_STATUS_UPDATE) {
+			csg_slot_sync_queues_state_locked(ptdev, csg_id);
+			csg_slot_sync_idle_state_locked(ptdev, csg_id);
+		}
+
+		if (ret && acked != req_mask &&
+		    ((csg_iface->input->req ^ csg_iface->output->ack) & req_mask) != 0) {
+			drm_err(&ptdev->base, "CSG %d update request timedout", csg_id);
+			ctx->timedout_mask |= BIT(csg_id);
+		}
+	}
+
+	if (ctx->timedout_mask)
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+struct panthor_sched_tick_ctx {
+	struct list_head old_groups[PANTHOR_CSG_PRIORITY_COUNT];
+	struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT];
+	u32 idle_group_count;
+	u32 group_count;
+	enum panthor_csg_priority min_priority;
+	struct panthor_vm *vms[MAX_CS_PER_CSG];
+	u32 as_count;
+	bool immediate_tick;
+	u32 csg_upd_failed_mask;
+};
+
+static bool
+tick_ctx_is_full(const struct panthor_scheduler *sched,
+		 const struct panthor_sched_tick_ctx *ctx)
+{
+	return ctx->group_count == sched->csg_slot_count;
+}
+
+static bool
+group_is_idle(struct panthor_group *group)
+{
+	struct panthor_device *ptdev = group->ptdev;
+	u32 inactive_queues;
+
+	if (group->csg_id >= 0)
+		return ptdev->scheduler->csg_slots[group->csg_id].idle;
+
+	inactive_queues = group->idle_queues | group->blocked_queues;
+	return hweight32(inactive_queues) == group->queue_count;
+}
+
+static bool
+group_can_run(struct panthor_group *group)
+{
+	return group->state != PANTHOR_CS_GROUP_TERMINATED &&
+	       group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE &&
+	       !group->destroyed && group->fatal_queues == 0 &&
+	       !group->timedout;
+}
+
+static void
+tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched,
+			       struct panthor_sched_tick_ctx *ctx,
+			       struct list_head *queue,
+			       bool skip_idle_groups,
+			       bool owned_by_tick_ctx)
+{
+	struct panthor_group *group, *tmp;
+
+	if (tick_ctx_is_full(sched, ctx))
+		return;
+
+	list_for_each_entry_safe(group, tmp, queue, run_node) {
+		u32 i;
+
+		if (!group_can_run(group))
+			continue;
+
+		if (skip_idle_groups && group_is_idle(group))
+			continue;
+
+		for (i = 0; i < ctx->as_count; i++) {
+			if (ctx->vms[i] == group->vm)
+				break;
+		}
+
+		if (i == ctx->as_count && ctx->as_count == sched->as_slot_count)
+			continue;
+
+		if (!owned_by_tick_ctx)
+			group_get(group);
+
+		list_move_tail(&group->run_node, &ctx->groups[group->priority]);
+		ctx->group_count++;
+		if (group_is_idle(group))
+			ctx->idle_group_count++;
+
+		if (i == ctx->as_count)
+			ctx->vms[ctx->as_count++] = group->vm;
+
+		if (ctx->min_priority > group->priority)
+			ctx->min_priority = group->priority;
+
+		if (tick_ctx_is_full(sched, ctx))
+			return;
+	}
+}
+
+static void
+tick_ctx_insert_old_group(struct panthor_scheduler *sched,
+			  struct panthor_sched_tick_ctx *ctx,
+			  struct panthor_group *group,
+			  bool full_tick)
+{
+	struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id];
+	struct panthor_group *other_group;
+
+	if (!full_tick) {
+		list_add_tail(&group->run_node, &ctx->old_groups[group->priority]);
+		return;
+	}
+
+	/* Rotate to make sure groups with lower CSG slot
+	 * priorities have a chance to get a higher CSG slot
+	 * priority next time they get picked. This priority
+	 * has an impact on resource request ordering, so it's
+	 * important to make sure we don't let one group starve
+	 * all other groups with the same group priority.
+	 */
+	list_for_each_entry(other_group,
+			    &ctx->old_groups[csg_slot->group->priority],
+			    run_node) {
+		struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id];
+
+		if (other_csg_slot->priority > csg_slot->priority) {
+			list_add_tail(&csg_slot->group->run_node, &other_group->run_node);
+			return;
+		}
+	}
+
+	list_add_tail(&group->run_node, &ctx->old_groups[group->priority]);
+}
+
+static void
+tick_ctx_init(struct panthor_scheduler *sched,
+	      struct panthor_sched_tick_ctx *ctx,
+	      bool full_tick)
+{
+	struct panthor_device *ptdev = sched->ptdev;
+	struct panthor_csg_slots_upd_ctx upd_ctx;
+	int ret;
+	u32 i;
+
+	memset(ctx, 0, sizeof(*ctx));
+	csgs_upd_ctx_init(&upd_ctx);
+
+	ctx->min_priority = PANTHOR_CSG_PRIORITY_COUNT;
+	for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) {
+		INIT_LIST_HEAD(&ctx->groups[i]);
+		INIT_LIST_HEAD(&ctx->old_groups[i]);
+	}
+
+	for (i = 0; i < sched->csg_slot_count; i++) {
+		struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];
+		struct panthor_group *group = csg_slot->group;
+		struct panthor_fw_csg_iface *csg_iface;
+
+		if (!group)
+			continue;
+
+		csg_iface = panthor_fw_get_csg_iface(ptdev, i);
+		group_get(group);
+
+		/* If there was unhandled faults on the VM, force processing of
+		 * CSG IRQs, so we can flag the faulty queue.
+		 */
+		if (panthor_vm_has_unhandled_faults(group->vm)) {
+			sched_process_csg_irq_locked(ptdev, i);
+
+			/* No fatal fault reported, flag all queues as faulty. */
+			if (!group->fatal_queues)
+				group->fatal_queues |= GENMASK(group->queue_count - 1, 0);
+		}
+
+		tick_ctx_insert_old_group(sched, ctx, group, full_tick);
+		csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i,
+					csg_iface->output->ack ^ CSG_STATUS_UPDATE,
+					CSG_STATUS_UPDATE);
+	}
+
+	ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
+	if (ret) {
+		panthor_device_schedule_reset(ptdev);
+		ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;
+	}
+}
+
+#define NUM_INSTRS_PER_SLOT		16
+
+static void
+group_term_post_processing(struct panthor_group *group)
+{
+	struct panthor_job *job, *tmp;
+	LIST_HEAD(faulty_jobs);
+	bool cookie;
+	u32 i = 0;
+
+	if (drm_WARN_ON(&group->ptdev->base, group_can_run(group)))
+		return;
+
+	cookie = dma_fence_begin_signalling();
+	for (i = 0; i < group->queue_count; i++) {
+		struct panthor_queue *queue = group->queues[i];
+		struct panthor_syncobj_64b *syncobj;
+		int err;
+
+		if (group->fatal_queues & BIT(i))
+			err = -EINVAL;
+		else if (group->timedout)
+			err = -ETIMEDOUT;
+		else
+			err = -ECANCELED;
+
+		if (!queue)
+			continue;
+
+		spin_lock(&queue->fence_ctx.lock);
+		list_for_each_entry_safe(job, tmp, &queue->fence_ctx.in_flight_jobs, node) {
+			list_move_tail(&job->node, &faulty_jobs);
+			dma_fence_set_error(job->done_fence, err);
+			dma_fence_signal_locked(job->done_fence);
+		}
+		spin_unlock(&queue->fence_ctx.lock);
+
+		/* Manually update the syncobj seqno to unblock waiters. */
+		syncobj = group->syncobjs->kmap + (i * sizeof(*syncobj));
+		syncobj->status = ~0;
+		syncobj->seqno = atomic64_read(&queue->fence_ctx.seqno);
+		sched_queue_work(group->ptdev->scheduler, sync_upd);
+	}
+	dma_fence_end_signalling(cookie);
+
+	list_for_each_entry_safe(job, tmp, &faulty_jobs, node) {
+		list_del_init(&job->node);
+		panthor_job_put(&job->base);
+	}
+}
+
+static void group_term_work(struct work_struct *work)
+{
+	struct panthor_group *group =
+		container_of(work, struct panthor_group, term_work);
+
+	group_term_post_processing(group);
+	group_put(group);
+}
+
+static void
+tick_ctx_cleanup(struct panthor_scheduler *sched,
+		 struct panthor_sched_tick_ctx *ctx)
+{
+	struct panthor_group *group, *tmp;
+	u32 i;
+
+	for (i = 0; i < ARRAY_SIZE(ctx->old_groups); i++) {
+		list_for_each_entry_safe(group, tmp, &ctx->old_groups[i], run_node) {
+			/* If everything went fine, we should only have groups
+			 * to be terminated in the old_groups lists.
+			 */
+			drm_WARN_ON(&group->ptdev->base, !ctx->csg_upd_failed_mask &&
+				    group_can_run(group));
+
+			if (!group_can_run(group)) {
+				list_del_init(&group->run_node);
+				list_del_init(&group->wait_node);
+				group_queue_work(group, term);
+			} else if (group->csg_id >= 0) {
+				list_del_init(&group->run_node);
+			} else {
+				list_move(&group->run_node,
+					  group_is_idle(group) ?
+					  &sched->groups.idle[group->priority] :
+					  &sched->groups.runnable[group->priority]);
+			}
+			group_put(group);
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) {
+		/* If everything went fine, the groups to schedule lists should
+		 * be empty.
+		 */
+		drm_WARN_ON(&group->ptdev->base,
+			    !ctx->csg_upd_failed_mask && !list_empty(&ctx->groups[i]));
+
+		list_for_each_entry_safe(group, tmp, &ctx->groups[i], run_node) {
+			if (group->csg_id >= 0) {
+				list_del_init(&group->run_node);
+			} else {
+				list_move(&group->run_node,
+					  group_is_idle(group) ?
+					  &sched->groups.idle[group->priority] :
+					  &sched->groups.runnable[group->priority]);
+			}
+			group_put(group);
+		}
+	}
+}
+
+static void
+tick_ctx_apply(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *ctx)
+{
+	struct panthor_group *group, *tmp;
+	struct panthor_device *ptdev = sched->ptdev;
+	struct panthor_csg_slot *csg_slot;
+	int prio, new_csg_prio = MAX_CSG_PRIO, i;
+	u32 free_csg_slots = 0;
+	struct panthor_csg_slots_upd_ctx upd_ctx;
+	int ret;
+
+	csgs_upd_ctx_init(&upd_ctx);
+
+	for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
+		/* Suspend or terminate evicted groups. */
+		list_for_each_entry(group, &ctx->old_groups[prio], run_node) {
+			bool term = !group_can_run(group);
+			int csg_id = group->csg_id;
+
+			if (drm_WARN_ON(&ptdev->base, csg_id < 0))
+				continue;
+
+			csg_slot = &sched->csg_slots[csg_id];
+			csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
+						term ? CSG_STATE_TERMINATE : CSG_STATE_SUSPEND,
+						CSG_STATE_MASK);
+		}
+
+		/* Update priorities on already running groups. */
+		list_for_each_entry(group, &ctx->groups[prio], run_node) {
+			struct panthor_fw_csg_iface *csg_iface;
+			int csg_id = group->csg_id;
+
+			if (csg_id < 0) {
+				new_csg_prio--;
+				continue;
+			}
+
+			csg_slot = &sched->csg_slots[csg_id];
+			csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
+			if (csg_slot->priority == new_csg_prio) {
+				new_csg_prio--;
+				continue;
+			}
+
+			panthor_fw_update_reqs(csg_iface, endpoint_req,
+					       CSG_EP_REQ_PRIORITY(new_csg_prio),
+					       CSG_EP_REQ_PRIORITY_MASK);
+			csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
+						csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG,
+						CSG_ENDPOINT_CONFIG);
+			new_csg_prio--;
+		}
+	}
+
+	ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
+	if (ret) {
+		panthor_device_schedule_reset(ptdev);
+		ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;
+		return;
+	}
+
+	/* Unbind evicted groups. */
+	for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
+		list_for_each_entry(group, &ctx->old_groups[prio], run_node) {
+			/* This group is gone. Process interrupts to clear
+			 * any pending interrupts before we start the new
+			 * group.
+			 */
+			if (group->csg_id >= 0)
+				sched_process_csg_irq_locked(ptdev, group->csg_id);
+
+			group_unbind_locked(group);
+		}
+	}
+
+	for (i = 0; i < sched->csg_slot_count; i++) {
+		if (!sched->csg_slots[i].group)
+			free_csg_slots |= BIT(i);
+	}
+
+	csgs_upd_ctx_init(&upd_ctx);
+	new_csg_prio = MAX_CSG_PRIO;
+
+	/* Start new groups. */
+	for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
+		list_for_each_entry(group, &ctx->groups[prio], run_node) {
+			int csg_id = group->csg_id;
+			struct panthor_fw_csg_iface *csg_iface;
+
+			if (csg_id >= 0) {
+				new_csg_prio--;
+				continue;
+			}
+
+			csg_id = ffs(free_csg_slots) - 1;
+			if (drm_WARN_ON(&ptdev->base, csg_id < 0))
+				break;
+
+			csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
+			csg_slot = &sched->csg_slots[csg_id];
+			group_bind_locked(group, csg_id);
+			csg_slot_prog_locked(ptdev, csg_id, new_csg_prio--);
+			csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
+						group->state == PANTHOR_CS_GROUP_SUSPENDED ?
+						CSG_STATE_RESUME : CSG_STATE_START,
+						CSG_STATE_MASK);
+			csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
+						csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG,
+						CSG_ENDPOINT_CONFIG);
+			free_csg_slots &= ~BIT(csg_id);
+		}
+	}
+
+	ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
+	if (ret) {
+		panthor_device_schedule_reset(ptdev);
+		ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;
+		return;
+	}
+
+	for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
+		list_for_each_entry_safe(group, tmp, &ctx->groups[prio], run_node) {
+			list_del_init(&group->run_node);
+
+			/* If the group has been destroyed while we were
+			 * scheduling, ask for an immediate tick to
+			 * re-evaluate as soon as possible and get rid of
+			 * this dangling group.
+			 */
+			if (group->destroyed)
+				ctx->immediate_tick = true;
+			group_put(group);
+		}
+
+		/* Return evicted groups to the idle or run queues. Groups
+		 * that can no longer be run (because they've been destroyed
+		 * or experienced an unrecoverable error) will be scheduled
+		 * for destruction in tick_ctx_cleanup().
+		 */
+		list_for_each_entry_safe(group, tmp, &ctx->old_groups[prio], run_node) {
+			if (!group_can_run(group))
+				continue;
+
+			if (group_is_idle(group))
+				list_move_tail(&group->run_node, &sched->groups.idle[prio]);
+			else
+				list_move_tail(&group->run_node, &sched->groups.runnable[prio]);
+			group_put(group);
+		}
+	}
+
+	sched->used_csg_slot_count = ctx->group_count;
+	sched->might_have_idle_groups = ctx->idle_group_count > 0;
+}
+
+static u64
+tick_ctx_update_resched_target(struct panthor_scheduler *sched,
+			       const struct panthor_sched_tick_ctx *ctx)
+{
+	/* We had space left, no need to reschedule until some external event happens. */
+	if (!tick_ctx_is_full(sched, ctx))
+		goto no_tick;
+
+	/* If idle groups were scheduled, no need to wake up until some external
+	 * event happens (group unblocked, new job submitted, ...).
+	 */
+	if (ctx->idle_group_count)
+		goto no_tick;
+
+	if (drm_WARN_ON(&sched->ptdev->base, ctx->min_priority >= PANTHOR_CSG_PRIORITY_COUNT))
+		goto no_tick;
+
+	/* If there are groups of the same priority waiting, we need to
+	 * keep the scheduler ticking, otherwise, we'll just wait for
+	 * new groups with higher priority to be queued.
+	 */
+	if (!list_empty(&sched->groups.runnable[ctx->min_priority])) {
+		u64 resched_target = sched->last_tick + sched->tick_period;
+
+		if (time_before64(sched->resched_target, sched->last_tick) ||
+		    time_before64(resched_target, sched->resched_target))
+			sched->resched_target = resched_target;
+
+		return sched->resched_target - sched->last_tick;
+	}
+
+no_tick:
+	sched->resched_target = U64_MAX;
+	return U64_MAX;
+}
+
+static void tick_work(struct work_struct *work)
+{
+	struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler,
+						      tick_work.work);
+	struct panthor_device *ptdev = sched->ptdev;
+	struct panthor_sched_tick_ctx ctx;
+	u64 remaining_jiffies = 0, resched_delay;
+	u64 now = get_jiffies_64();
+	int prio, ret, cookie;
+
+	if (!drm_dev_enter(&ptdev->base, &cookie))
+		return;
+
+	ret = pm_runtime_resume_and_get(ptdev->base.dev);
+	if (drm_WARN_ON(&ptdev->base, ret))
+		goto out_dev_exit;
+
+	if (time_before64(now, sched->resched_target))
+		remaining_jiffies = sched->resched_target - now;
+
+	mutex_lock(&sched->lock);
+	if (panthor_device_reset_is_pending(sched->ptdev))
+		goto out_unlock;
+
+	tick_ctx_init(sched, &ctx, remaining_jiffies != 0);
+	if (ctx.csg_upd_failed_mask)
+		goto out_cleanup_ctx;
+
+	if (remaining_jiffies) {
+		/* Scheduling forced in the middle of a tick. Only RT groups
+		 * can preempt non-RT ones. Currently running RT groups can't be
+		 * preempted.
+		 */
+		for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
+		     prio >= 0 && !tick_ctx_is_full(sched, &ctx);
+		     prio--) {
+			tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio],
+						       true, true);
+			if (prio == PANTHOR_CSG_PRIORITY_RT) {
+				tick_ctx_pick_groups_from_list(sched, &ctx,
+							       &sched->groups.runnable[prio],
+							       true, false);
+			}
+		}
+	}
+
+	/* First pick non-idle groups */
+	for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
+	     prio >= 0 && !tick_ctx_is_full(sched, &ctx);
+	     prio--) {
+		tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio],
+					       true, false);
+		tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true);
+	}
+
+	/* If we have free CSG slots left, pick idle groups */
+	for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
+	     prio >= 0 && !tick_ctx_is_full(sched, &ctx);
+	     prio--) {
+		/* Check the old_group queue first to avoid reprogramming the slots */
+		tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], false, true);
+		tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.idle[prio],
+					       false, false);
+	}
+
+	tick_ctx_apply(sched, &ctx);
+	if (ctx.csg_upd_failed_mask)
+		goto out_cleanup_ctx;
+
+	if (ctx.idle_group_count == ctx.group_count) {
+		panthor_devfreq_record_idle(sched->ptdev);
+		if (sched->pm.has_ref) {
+			pm_runtime_put_autosuspend(ptdev->base.dev);
+			sched->pm.has_ref = false;
+		}
+	} else {
+		panthor_devfreq_record_busy(sched->ptdev);
+		if (!sched->pm.has_ref) {
+			pm_runtime_get(ptdev->base.dev);
+			sched->pm.has_ref = true;
+		}
+	}
+
+	sched->last_tick = now;
+	resched_delay = tick_ctx_update_resched_target(sched, &ctx);
+	if (ctx.immediate_tick)
+		resched_delay = 0;
+
+	if (resched_delay != U64_MAX)
+		sched_queue_delayed_work(sched, tick, resched_delay);
+
+out_cleanup_ctx:
+	tick_ctx_cleanup(sched, &ctx);
+
+out_unlock:
+	mutex_unlock(&sched->lock);
+	pm_runtime_mark_last_busy(ptdev->base.dev);
+	pm_runtime_put_autosuspend(ptdev->base.dev);
+
+out_dev_exit:
+	drm_dev_exit(cookie);
+}
+
+static int panthor_queue_eval_syncwait(struct panthor_group *group, u8 queue_idx)
+{
+	struct panthor_queue *queue = group->queues[queue_idx];
+	union {
+		struct panthor_syncobj_64b sync64;
+		struct panthor_syncobj_32b sync32;
+	} *syncobj;
+	bool result;
+	u64 value;
+
+	syncobj = panthor_queue_get_syncwait_obj(group, queue);
+	if (!syncobj)
+		return -EINVAL;
+
+	value = queue->syncwait.sync64 ?
+		syncobj->sync64.seqno :
+		syncobj->sync32.seqno;
+
+	if (queue->syncwait.gt)
+		result = value > queue->syncwait.ref;
+	else
+		result = value <= queue->syncwait.ref;
+
+	if (result)
+		panthor_queue_put_syncwait_obj(queue);
+
+	return result;
+}
+
+static void sync_upd_work(struct work_struct *work)
+{
+	struct panthor_scheduler *sched = container_of(work,
+						      struct panthor_scheduler,
+						      sync_upd_work);
+	struct panthor_group *group, *tmp;
+	bool immediate_tick = false;
+
+	mutex_lock(&sched->lock);
+	list_for_each_entry_safe(group, tmp, &sched->groups.waiting, wait_node) {
+		u32 tested_queues = group->blocked_queues;
+		u32 unblocked_queues = 0;
+
+		while (tested_queues) {
+			u32 cs_id = ffs(tested_queues) - 1;
+			int ret;
+
+			ret = panthor_queue_eval_syncwait(group, cs_id);
+			drm_WARN_ON(&group->ptdev->base, ret < 0);
+			if (ret)
+				unblocked_queues |= BIT(cs_id);
+
+			tested_queues &= ~BIT(cs_id);
+		}
+
+		if (unblocked_queues) {
+			group->blocked_queues &= ~unblocked_queues;
+
+			if (group->csg_id < 0) {
+				list_move(&group->run_node,
+					  &sched->groups.runnable[group->priority]);
+				if (group->priority == PANTHOR_CSG_PRIORITY_RT)
+					immediate_tick = true;
+			}
+		}
+
+		if (!group->blocked_queues)
+			list_del_init(&group->wait_node);
+	}
+	mutex_unlock(&sched->lock);
+
+	if (immediate_tick)
+		sched_queue_delayed_work(sched, tick, 0);
+}
+
+static void group_schedule_locked(struct panthor_group *group, u32 queue_mask)
+{
+	struct panthor_device *ptdev = group->ptdev;
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	struct list_head *queue = &sched->groups.runnable[group->priority];
+	u64 delay_jiffies = 0;
+	bool was_idle;
+	u64 now;
+
+	if (!group_can_run(group))
+		return;
+
+	/* All updated queues are blocked, no need to wake up the scheduler. */
+	if ((queue_mask & group->blocked_queues) == queue_mask)
+		return;
+
+	was_idle = group_is_idle(group);
+	group->idle_queues &= ~queue_mask;
+
+	/* Don't mess up with the lists if we're in a middle of a reset. */
+	if (atomic_read(&sched->reset.in_progress))
+		return;
+
+	if (was_idle && !group_is_idle(group))
+		list_move_tail(&group->run_node, queue);
+
+	/* RT groups are preemptive. */
+	if (group->priority == PANTHOR_CSG_PRIORITY_RT) {
+		sched_queue_delayed_work(sched, tick, 0);
+		return;
+	}
+
+	/* Some groups might be idle, force an immediate tick to
+	 * re-evaluate.
+	 */
+	if (sched->might_have_idle_groups) {
+		sched_queue_delayed_work(sched, tick, 0);
+		return;
+	}
+
+	/* Scheduler is ticking, nothing to do. */
+	if (sched->resched_target != U64_MAX) {
+		/* If there are free slots, force immediating ticking. */
+		if (sched->used_csg_slot_count < sched->csg_slot_count)
+			sched_queue_delayed_work(sched, tick, 0);
+
+		return;
+	}
+
+	/* Scheduler tick was off, recalculate the resched_target based on the
+	 * last tick event, and queue the scheduler work.
+	 */
+	now = get_jiffies_64();
+	sched->resched_target = sched->last_tick + sched->tick_period;
+	if (sched->used_csg_slot_count == sched->csg_slot_count &&
+	    time_before64(now, sched->resched_target))
+		delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX);
+
+	sched_queue_delayed_work(sched, tick, delay_jiffies);
+}
+
+static void queue_stop(struct panthor_queue *queue,
+		       struct panthor_job *bad_job)
+{
+	drm_sched_stop(&queue->scheduler, bad_job ? &bad_job->base : NULL);
+}
+
+static void queue_start(struct panthor_queue *queue)
+{
+	struct panthor_job *job;
+
+	/* Re-assign the parent fences. */
+	list_for_each_entry(job, &queue->scheduler.pending_list, base.list)
+		job->base.s_fence->parent = dma_fence_get(job->done_fence);
+
+	drm_sched_start(&queue->scheduler, true);
+}
+
+static void panthor_group_stop(struct panthor_group *group)
+{
+	struct panthor_scheduler *sched = group->ptdev->scheduler;
+
+	lockdep_assert_held(&sched->reset.lock);
+
+	for (u32 i = 0; i < group->queue_count; i++)
+		queue_stop(group->queues[i], NULL);
+
+	group_get(group);
+	list_move_tail(&group->run_node, &sched->reset.stopped_groups);
+}
+
+static void panthor_group_start(struct panthor_group *group)
+{
+	struct panthor_scheduler *sched = group->ptdev->scheduler;
+
+	lockdep_assert_held(&group->ptdev->scheduler->reset.lock);
+
+	for (u32 i = 0; i < group->queue_count; i++)
+		queue_start(group->queues[i]);
+
+	if (group_can_run(group)) {
+		list_move_tail(&group->run_node,
+			       group_is_idle(group) ?
+			       &sched->groups.idle[group->priority] :
+			       &sched->groups.runnable[group->priority]);
+	} else {
+		list_del_init(&group->run_node);
+		list_del_init(&group->wait_node);
+		group_queue_work(group, term);
+	}
+
+	group_put(group);
+}
+
+static void panthor_sched_immediate_tick(struct panthor_device *ptdev)
+{
+	struct panthor_scheduler *sched = ptdev->scheduler;
+
+	sched_queue_delayed_work(sched, tick, 0);
+}
+
+/**
+ * panthor_sched_report_mmu_fault() - Report MMU faults to the scheduler.
+ */
+void panthor_sched_report_mmu_fault(struct panthor_device *ptdev)
+{
+	/* Force a tick to immediately kill faulty groups. */
+	if (ptdev->scheduler)
+		panthor_sched_immediate_tick(ptdev);
+}
+
+void panthor_sched_resume(struct panthor_device *ptdev)
+{
+	/* Force a tick to re-evaluate after a resume. */
+	panthor_sched_immediate_tick(ptdev);
+}
+
+void panthor_sched_suspend(struct panthor_device *ptdev)
+{
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	struct panthor_csg_slots_upd_ctx upd_ctx;
+	struct panthor_group *group;
+	u32 suspended_slots;
+	u32 i;
+
+	mutex_lock(&sched->lock);
+	csgs_upd_ctx_init(&upd_ctx);
+	for (i = 0; i < sched->csg_slot_count; i++) {
+		struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];
+
+		if (csg_slot->group) {
+			csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i,
+						group_can_run(csg_slot->group) ?
+						CSG_STATE_SUSPEND : CSG_STATE_TERMINATE,
+						CSG_STATE_MASK);
+		}
+	}
+
+	suspended_slots = upd_ctx.update_mask;
+
+	csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
+	suspended_slots &= ~upd_ctx.timedout_mask;
+
+	if (upd_ctx.timedout_mask) {
+		u32 slot_mask = upd_ctx.timedout_mask;
+
+		drm_err(&ptdev->base, "CSG suspend failed, escalating to termination");
+		csgs_upd_ctx_init(&upd_ctx);
+		while (slot_mask) {
+			u32 csg_id = ffs(slot_mask) - 1;
+
+			csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
+						CSG_STATE_TERMINATE,
+						CSG_STATE_MASK);
+			slot_mask &= ~BIT(csg_id);
+		}
+
+		csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
+
+		slot_mask = upd_ctx.timedout_mask;
+		while (slot_mask) {
+			u32 csg_id = ffs(slot_mask) - 1;
+			struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
+
+			/* Terminate command timedout, but the soft-reset will
+			 * automatically terminate all active groups, so let's
+			 * force the state to halted here.
+			 */
+			if (csg_slot->group->state != PANTHOR_CS_GROUP_TERMINATED)
+				csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED;
+			slot_mask &= ~BIT(csg_id);
+		}
+	}
+
+	/* Flush L2 and LSC caches to make sure suspend state is up-to-date.
+	 * If the flush fails, flag all queues for termination.
+	 */
+	if (suspended_slots) {
+		bool flush_caches_failed = false;
+		u32 slot_mask = suspended_slots;
+
+		if (panthor_gpu_flush_caches(ptdev, CACHE_CLEAN, CACHE_CLEAN, 0))
+			flush_caches_failed = true;
+
+		while (slot_mask) {
+			u32 csg_id = ffs(slot_mask) - 1;
+			struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
+
+			if (flush_caches_failed)
+				csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED;
+			else
+				csg_slot_sync_update_locked(ptdev, csg_id);
+
+			slot_mask &= ~BIT(csg_id);
+		}
+	}
+
+	for (i = 0; i < sched->csg_slot_count; i++) {
+		struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];
+
+		group = csg_slot->group;
+		if (!group)
+			continue;
+
+		group_get(group);
+
+		if (group->csg_id >= 0)
+			sched_process_csg_irq_locked(ptdev, group->csg_id);
+
+		group_unbind_locked(group);
+
+		drm_WARN_ON(&group->ptdev->base, !list_empty(&group->run_node));
+
+		if (group_can_run(group)) {
+			list_add(&group->run_node,
+				 &sched->groups.idle[group->priority]);
+		} else {
+			/* We don't bother stopping the scheduler if the group is
+			 * faulty, the group termination work will finish the job.
+			 */
+			list_del_init(&group->wait_node);
+			group_queue_work(group, term);
+		}
+		group_put(group);
+	}
+	mutex_unlock(&sched->lock);
+}
+
+void panthor_sched_pre_reset(struct panthor_device *ptdev)
+{
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	struct panthor_group *group, *group_tmp;
+	u32 i;
+
+	mutex_lock(&sched->reset.lock);
+	atomic_set(&sched->reset.in_progress, true);
+
+	/* Cancel all scheduler works. Once this is done, these works can't be
+	 * scheduled again until the reset operation is complete.
+	 */
+	cancel_work_sync(&sched->sync_upd_work);
+	cancel_delayed_work_sync(&sched->tick_work);
+
+	panthor_sched_suspend(ptdev);
+
+	/* Stop all groups that might still accept jobs, so we don't get passed
+	 * new jobs while we're resetting.
+	 */
+	for (i = 0; i < ARRAY_SIZE(sched->groups.runnable); i++) {
+		/* All groups should be in the idle lists. */
+		drm_WARN_ON(&ptdev->base, !list_empty(&sched->groups.runnable[i]));
+		list_for_each_entry_safe(group, group_tmp, &sched->groups.runnable[i], run_node)
+			panthor_group_stop(group);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sched->groups.idle); i++) {
+		list_for_each_entry_safe(group, group_tmp, &sched->groups.idle[i], run_node)
+			panthor_group_stop(group);
+	}
+
+	mutex_unlock(&sched->reset.lock);
+}
+
+void panthor_sched_post_reset(struct panthor_device *ptdev, bool reset_failed)
+{
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	struct panthor_group *group, *group_tmp;
+
+	mutex_lock(&sched->reset.lock);
+
+	list_for_each_entry_safe(group, group_tmp, &sched->reset.stopped_groups, run_node) {
+		/* Consider all previously running group as terminated if the
+		 * reset failed.
+		 */
+		if (reset_failed)
+			group->state = PANTHOR_CS_GROUP_TERMINATED;
+
+		panthor_group_start(group);
+	}
+
+	/* We're done resetting the GPU, clear the reset.in_progress bit so we can
+	 * kick the scheduler.
+	 */
+	atomic_set(&sched->reset.in_progress, false);
+	mutex_unlock(&sched->reset.lock);
+
+	/* No need to queue a tick and update syncs if the reset failed. */
+	if (!reset_failed) {
+		sched_queue_delayed_work(sched, tick, 0);
+		sched_queue_work(sched, sync_upd);
+	}
+}
+
+static void group_sync_upd_work(struct work_struct *work)
+{
+	struct panthor_group *group =
+		container_of(work, struct panthor_group, sync_upd_work);
+	struct panthor_job *job, *job_tmp;
+	LIST_HEAD(done_jobs);
+	u32 queue_idx;
+	bool cookie;
+
+	cookie = dma_fence_begin_signalling();
+	for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) {
+		struct panthor_queue *queue = group->queues[queue_idx];
+		struct panthor_syncobj_64b *syncobj;
+
+		if (!queue)
+			continue;
+
+		syncobj = group->syncobjs->kmap + (queue_idx * sizeof(*syncobj));
+
+		spin_lock(&queue->fence_ctx.lock);
+		list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) {
+			if (!job->call_info.size)
+				continue;
+
+			if (syncobj->seqno < job->done_fence->seqno)
+				break;
+
+			list_move_tail(&job->node, &done_jobs);
+			dma_fence_signal_locked(job->done_fence);
+		}
+		spin_unlock(&queue->fence_ctx.lock);
+	}
+	dma_fence_end_signalling(cookie);
+
+	list_for_each_entry_safe(job, job_tmp, &done_jobs, node) {
+		list_del_init(&job->node);
+		panthor_job_put(&job->base);
+	}
+
+	group_put(group);
+}
+
+static struct dma_fence *
+queue_run_job(struct drm_sched_job *sched_job)
+{
+	struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
+	struct panthor_group *group = job->group;
+	struct panthor_queue *queue = group->queues[job->queue_idx];
+	struct panthor_device *ptdev = group->ptdev;
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	u32 ringbuf_size = panthor_kernel_bo_size(queue->ringbuf);
+	u32 ringbuf_insert = queue->iface.input->insert & (ringbuf_size - 1);
+	u64 addr_reg = ptdev->csif_info.cs_reg_count -
+		       ptdev->csif_info.unpreserved_cs_reg_count;
+	u64 val_reg = addr_reg + 2;
+	u64 sync_addr = panthor_kernel_bo_gpuva(group->syncobjs) +
+			job->queue_idx * sizeof(struct panthor_syncobj_64b);
+	u32 waitall_mask = GENMASK(sched->sb_slot_count - 1, 0);
+	struct dma_fence *done_fence;
+	int ret;
+
+	u64 call_instrs[NUM_INSTRS_PER_SLOT] = {
+		/* MOV32 rX+2, cs.latest_flush */
+		(2ull << 56) | (val_reg << 48) | job->call_info.latest_flush,
+
+		/* FLUSH_CACHE2.clean_inv_all.no_wait.signal(0) rX+2 */
+		(36ull << 56) | (0ull << 48) | (val_reg << 40) | (0 << 16) | 0x233,
+
+		/* MOV48 rX:rX+1, cs.start */
+		(1ull << 56) | (addr_reg << 48) | job->call_info.start,
+
+		/* MOV32 rX+2, cs.size */
+		(2ull << 56) | (val_reg << 48) | job->call_info.size,
+
+		/* WAIT(0) => waits for FLUSH_CACHE2 instruction */
+		(3ull << 56) | (1 << 16),
+
+		/* CALL rX:rX+1, rX+2 */
+		(32ull << 56) | (addr_reg << 40) | (val_reg << 32),
+
+		/* MOV48 rX:rX+1, sync_addr */
+		(1ull << 56) | (addr_reg << 48) | sync_addr,
+
+		/* MOV48 rX+2, #1 */
+		(1ull << 56) | (val_reg << 48) | 1,
+
+		/* WAIT(all) */
+		(3ull << 56) | (waitall_mask << 16),
+
+		/* SYNC_ADD64.system_scope.propage_err.nowait rX:rX+1, rX+2*/
+		(51ull << 56) | (0ull << 48) | (addr_reg << 40) | (val_reg << 32) | (0 << 16) | 1,
+
+		/* ERROR_BARRIER, so we can recover from faults at job
+		 * boundaries.
+		 */
+		(47ull << 56),
+	};
+
+	/* Need to be cacheline aligned to please the prefetcher. */
+	static_assert(sizeof(call_instrs) % 64 == 0,
+		      "call_instrs is not aligned on a cacheline");
+
+	/* Stream size is zero, nothing to do => return a NULL fence and let
+	 * drm_sched signal the parent.
+	 */
+	if (!job->call_info.size)
+		return NULL;
+
+	ret = pm_runtime_resume_and_get(ptdev->base.dev);
+	if (drm_WARN_ON(&ptdev->base, ret))
+		return ERR_PTR(ret);
+
+	mutex_lock(&sched->lock);
+	if (!group_can_run(group)) {
+		done_fence = ERR_PTR(-ECANCELED);
+		goto out_unlock;
+	}
+
+	dma_fence_init(job->done_fence,
+		       &panthor_queue_fence_ops,
+		       &queue->fence_ctx.lock,
+		       queue->fence_ctx.id,
+		       atomic64_inc_return(&queue->fence_ctx.seqno));
+
+	memcpy(queue->ringbuf->kmap + ringbuf_insert,
+	       call_instrs, sizeof(call_instrs));
+
+	panthor_job_get(&job->base);
+	spin_lock(&queue->fence_ctx.lock);
+	list_add_tail(&job->node, &queue->fence_ctx.in_flight_jobs);
+	spin_unlock(&queue->fence_ctx.lock);
+
+	job->ringbuf.start = queue->iface.input->insert;
+	job->ringbuf.end = job->ringbuf.start + sizeof(call_instrs);
+
+	/* Make sure the ring buffer is updated before the INSERT
+	 * register.
+	 */
+	wmb();
+
+	queue->iface.input->extract = queue->iface.output->extract;
+	queue->iface.input->insert = job->ringbuf.end;
+
+	if (group->csg_id < 0) {
+		/* If the queue is blocked, we want to keep the timeout running, so we
+		 * can detect unbounded waits and kill the group when that happens.
+		 * Otherwise, we suspend the timeout so the time we spend waiting for
+		 * a CSG slot is not counted.
+		 */
+		if (!(group->blocked_queues & BIT(job->queue_idx)) &&
+		    !queue->timeout_suspended) {
+			queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler);
+			queue->timeout_suspended = true;
+		}
+
+		group_schedule_locked(group, BIT(job->queue_idx));
+	} else {
+		gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1);
+		if (!sched->pm.has_ref &&
+		    !(group->blocked_queues & BIT(job->queue_idx))) {
+			pm_runtime_get(ptdev->base.dev);
+			sched->pm.has_ref = true;
+		}
+	}
+
+	done_fence = dma_fence_get(job->done_fence);
+
+out_unlock:
+	mutex_unlock(&sched->lock);
+	pm_runtime_mark_last_busy(ptdev->base.dev);
+	pm_runtime_put_autosuspend(ptdev->base.dev);
+
+	return done_fence;
+}
+
+static enum drm_gpu_sched_stat
+queue_timedout_job(struct drm_sched_job *sched_job)
+{
+	struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
+	struct panthor_group *group = job->group;
+	struct panthor_device *ptdev = group->ptdev;
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	struct panthor_queue *queue = group->queues[job->queue_idx];
+
+	drm_warn(&ptdev->base, "job timeout\n");
+
+	drm_WARN_ON(&ptdev->base, atomic_read(&sched->reset.in_progress));
+
+	queue_stop(queue, job);
+
+	mutex_lock(&sched->lock);
+	group->timedout = true;
+	if (group->csg_id >= 0) {
+		sched_queue_delayed_work(ptdev->scheduler, tick, 0);
+	} else {
+		/* Remove from the run queues, so the scheduler can't
+		 * pick the group on the next tick.
+		 */
+		list_del_init(&group->run_node);
+		list_del_init(&group->wait_node);
+
+		group_queue_work(group, term);
+	}
+	mutex_unlock(&sched->lock);
+
+	queue_start(queue);
+
+	return DRM_GPU_SCHED_STAT_NOMINAL;
+}
+
+static void queue_free_job(struct drm_sched_job *sched_job)
+{
+	drm_sched_job_cleanup(sched_job);
+	panthor_job_put(sched_job);
+}
+
+static const struct drm_sched_backend_ops panthor_queue_sched_ops = {
+	.run_job = queue_run_job,
+	.timedout_job = queue_timedout_job,
+	.free_job = queue_free_job,
+};
+
+static struct panthor_queue *
+group_create_queue(struct panthor_group *group,
+		   const struct drm_panthor_queue_create *args)
+{
+	struct drm_gpu_scheduler *drm_sched;
+	struct panthor_queue *queue;
+	int ret;
+
+	if (args->pad[0] || args->pad[1] || args->pad[2])
+		return ERR_PTR(-EINVAL);
+
+	if (args->ringbuf_size < SZ_4K || args->ringbuf_size > SZ_64K ||
+	    !is_power_of_2(args->ringbuf_size))
+		return ERR_PTR(-EINVAL);
+
+	if (args->priority > CSF_MAX_QUEUE_PRIO)
+		return ERR_PTR(-EINVAL);
+
+	queue = kzalloc(sizeof(*queue), GFP_KERNEL);
+	if (!queue)
+		return ERR_PTR(-ENOMEM);
+
+	queue->fence_ctx.id = dma_fence_context_alloc(1);
+	spin_lock_init(&queue->fence_ctx.lock);
+	INIT_LIST_HEAD(&queue->fence_ctx.in_flight_jobs);
+
+	queue->priority = args->priority;
+
+	queue->ringbuf = panthor_kernel_bo_create(group->ptdev, group->vm,
+						  args->ringbuf_size,
+						  DRM_PANTHOR_BO_NO_MMAP,
+						  DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |
+						  DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,
+						  PANTHOR_VM_KERNEL_AUTO_VA);
+	if (IS_ERR(queue->ringbuf)) {
+		ret = PTR_ERR(queue->ringbuf);
+		goto err_free_queue;
+	}
+
+	ret = panthor_kernel_bo_vmap(queue->ringbuf);
+	if (ret)
+		goto err_free_queue;
+
+	queue->iface.mem = panthor_fw_alloc_queue_iface_mem(group->ptdev,
+							    &queue->iface.input,
+							    &queue->iface.output,
+							    &queue->iface.input_fw_va,
+							    &queue->iface.output_fw_va);
+	if (IS_ERR(queue->iface.mem)) {
+		ret = PTR_ERR(queue->iface.mem);
+		goto err_free_queue;
+	}
+
+	ret = drm_sched_init(&queue->scheduler, &panthor_queue_sched_ops,
+			     group->ptdev->scheduler->wq, 1,
+			     args->ringbuf_size / (NUM_INSTRS_PER_SLOT * sizeof(u64)),
+			     0, msecs_to_jiffies(JOB_TIMEOUT_MS),
+			     group->ptdev->reset.wq,
+			     NULL, "panthor-queue", group->ptdev->base.dev);
+	if (ret)
+		goto err_free_queue;
+
+	drm_sched = &queue->scheduler;
+	ret = drm_sched_entity_init(&queue->entity, 0, &drm_sched, 1, NULL);
+
+	return queue;
+
+err_free_queue:
+	group_free_queue(group, queue);
+	return ERR_PTR(ret);
+}
+
+#define MAX_GROUPS_PER_POOL		128
+
+int panthor_group_create(struct panthor_file *pfile,
+			 const struct drm_panthor_group_create *group_args,
+			 const struct drm_panthor_queue_create *queue_args)
+{
+	struct panthor_device *ptdev = pfile->ptdev;
+	struct panthor_group_pool *gpool = pfile->groups;
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0);
+	struct panthor_group *group = NULL;
+	u32 gid, i, suspend_size;
+	int ret;
+
+	if (group_args->pad)
+		return -EINVAL;
+
+	if (group_args->priority > PANTHOR_CSG_PRIORITY_HIGH)
+		return -EINVAL;
+
+	if ((group_args->compute_core_mask & ~ptdev->gpu_info.shader_present) ||
+	    (group_args->fragment_core_mask & ~ptdev->gpu_info.shader_present) ||
+	    (group_args->tiler_core_mask & ~ptdev->gpu_info.tiler_present))
+		return -EINVAL;
+
+	if (hweight64(group_args->compute_core_mask) < group_args->max_compute_cores ||
+	    hweight64(group_args->fragment_core_mask) < group_args->max_fragment_cores ||
+	    hweight64(group_args->tiler_core_mask) < group_args->max_tiler_cores)
+		return -EINVAL;
+
+	group = kzalloc(sizeof(*group), GFP_KERNEL);
+	if (!group)
+		return -ENOMEM;
+
+	spin_lock_init(&group->fatal_lock);
+	kref_init(&group->refcount);
+	group->state = PANTHOR_CS_GROUP_CREATED;
+	group->csg_id = -1;
+
+	group->ptdev = ptdev;
+	group->max_compute_cores = group_args->max_compute_cores;
+	group->compute_core_mask = group_args->compute_core_mask;
+	group->max_fragment_cores = group_args->max_fragment_cores;
+	group->fragment_core_mask = group_args->fragment_core_mask;
+	group->max_tiler_cores = group_args->max_tiler_cores;
+	group->tiler_core_mask = group_args->tiler_core_mask;
+	group->priority = group_args->priority;
+
+	INIT_LIST_HEAD(&group->wait_node);
+	INIT_LIST_HEAD(&group->run_node);
+	INIT_WORK(&group->term_work, group_term_work);
+	INIT_WORK(&group->sync_upd_work, group_sync_upd_work);
+	INIT_WORK(&group->tiler_oom_work, group_tiler_oom_work);
+	INIT_WORK(&group->release_work, group_release_work);
+
+	group->vm = panthor_vm_pool_get_vm(pfile->vms, group_args->vm_id);
+	if (!group->vm) {
+		ret = -EINVAL;
+		goto err_put_group;
+	}
+
+	suspend_size = csg_iface->control->suspend_size;
+	group->suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size);
+	if (IS_ERR(group->suspend_buf)) {
+		ret = PTR_ERR(group->suspend_buf);
+		group->suspend_buf = NULL;
+		goto err_put_group;
+	}
+
+	suspend_size = csg_iface->control->protm_suspend_size;
+	group->protm_suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size);
+	if (IS_ERR(group->protm_suspend_buf)) {
+		ret = PTR_ERR(group->protm_suspend_buf);
+		group->protm_suspend_buf = NULL;
+		goto err_put_group;
+	}
+
+	group->syncobjs = panthor_kernel_bo_create(ptdev, group->vm,
+						   group_args->queues.count *
+						   sizeof(struct panthor_syncobj_64b),
+						   DRM_PANTHOR_BO_NO_MMAP,
+						   DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |
+						   DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,
+						   PANTHOR_VM_KERNEL_AUTO_VA);
+	if (IS_ERR(group->syncobjs)) {
+		ret = PTR_ERR(group->syncobjs);
+		goto err_put_group;
+	}
+
+	ret = panthor_kernel_bo_vmap(group->syncobjs);
+	if (ret)
+		goto err_put_group;
+
+	memset(group->syncobjs->kmap, 0,
+	       group_args->queues.count * sizeof(struct panthor_syncobj_64b));
+
+	for (i = 0; i < group_args->queues.count; i++) {
+		group->queues[i] = group_create_queue(group, &queue_args[i]);
+		if (IS_ERR(group->queues[i])) {
+			ret = PTR_ERR(group->queues[i]);
+			group->queues[i] = NULL;
+			goto err_put_group;
+		}
+
+		group->queue_count++;
+	}
+
+	group->idle_queues = GENMASK(group->queue_count - 1, 0);
+
+	ret = xa_alloc(&gpool->xa, &gid, group, XA_LIMIT(1, MAX_GROUPS_PER_POOL), GFP_KERNEL);
+	if (ret)
+		goto err_put_group;
+
+	mutex_lock(&sched->reset.lock);
+	if (atomic_read(&sched->reset.in_progress)) {
+		panthor_group_stop(group);
+	} else {
+		mutex_lock(&sched->lock);
+		list_add_tail(&group->run_node,
+			      &sched->groups.idle[group->priority]);
+		mutex_unlock(&sched->lock);
+	}
+	mutex_unlock(&sched->reset.lock);
+
+	return gid;
+
+err_put_group:
+	group_put(group);
+	return ret;
+}
+
+int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle)
+{
+	struct panthor_group_pool *gpool = pfile->groups;
+	struct panthor_device *ptdev = pfile->ptdev;
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	struct panthor_group *group;
+
+	group = xa_erase(&gpool->xa, group_handle);
+	if (!group)
+		return -EINVAL;
+
+	for (u32 i = 0; i < group->queue_count; i++) {
+		if (group->queues[i])
+			drm_sched_entity_destroy(&group->queues[i]->entity);
+	}
+
+	mutex_lock(&sched->reset.lock);
+	mutex_lock(&sched->lock);
+	group->destroyed = true;
+	if (group->csg_id >= 0) {
+		sched_queue_delayed_work(sched, tick, 0);
+	} else if (!atomic_read(&sched->reset.in_progress)) {
+		/* Remove from the run queues, so the scheduler can't
+		 * pick the group on the next tick.
+		 */
+		list_del_init(&group->run_node);
+		list_del_init(&group->wait_node);
+		group_queue_work(group, term);
+	}
+	mutex_unlock(&sched->lock);
+	mutex_unlock(&sched->reset.lock);
+
+	group_put(group);
+	return 0;
+}
+
+int panthor_group_get_state(struct panthor_file *pfile,
+			    struct drm_panthor_group_get_state *get_state)
+{
+	struct panthor_group_pool *gpool = pfile->groups;
+	struct panthor_device *ptdev = pfile->ptdev;
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	struct panthor_group *group;
+
+	if (get_state->pad)
+		return -EINVAL;
+
+	group = group_get(xa_load(&gpool->xa, get_state->group_handle));
+	if (!group)
+		return -EINVAL;
+
+	memset(get_state, 0, sizeof(*get_state));
+
+	mutex_lock(&sched->lock);
+	if (group->timedout)
+		get_state->state |= DRM_PANTHOR_GROUP_STATE_TIMEDOUT;
+	if (group->fatal_queues) {
+		get_state->state |= DRM_PANTHOR_GROUP_STATE_FATAL_FAULT;
+		get_state->fatal_queues = group->fatal_queues;
+	}
+	mutex_unlock(&sched->lock);
+
+	group_put(group);
+	return 0;
+}
+
+int panthor_group_pool_create(struct panthor_file *pfile)
+{
+	struct panthor_group_pool *gpool;
+
+	gpool = kzalloc(sizeof(*gpool), GFP_KERNEL);
+	if (!gpool)
+		return -ENOMEM;
+
+	xa_init_flags(&gpool->xa, XA_FLAGS_ALLOC1);
+	pfile->groups = gpool;
+	return 0;
+}
+
+void panthor_group_pool_destroy(struct panthor_file *pfile)
+{
+	struct panthor_group_pool *gpool = pfile->groups;
+	struct panthor_group *group;
+	unsigned long i;
+
+	if (IS_ERR_OR_NULL(gpool))
+		return;
+
+	xa_for_each(&gpool->xa, i, group)
+		panthor_group_destroy(pfile, i);
+
+	xa_destroy(&gpool->xa);
+	kfree(gpool);
+	pfile->groups = NULL;
+}
+
+static void job_release(struct kref *ref)
+{
+	struct panthor_job *job = container_of(ref, struct panthor_job, refcount);
+
+	drm_WARN_ON(&job->group->ptdev->base, !list_empty(&job->node));
+
+	if (job->base.s_fence)
+		drm_sched_job_cleanup(&job->base);
+
+	if (job->done_fence && job->done_fence->ops)
+		dma_fence_put(job->done_fence);
+	else
+		dma_fence_free(job->done_fence);
+
+	group_put(job->group);
+
+	kfree(job);
+}
+
+struct drm_sched_job *panthor_job_get(struct drm_sched_job *sched_job)
+{
+	if (sched_job) {
+		struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
+
+		kref_get(&job->refcount);
+	}
+
+	return sched_job;
+}
+
+void panthor_job_put(struct drm_sched_job *sched_job)
+{
+	struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
+
+	if (sched_job)
+		kref_put(&job->refcount, job_release);
+}
+
+struct panthor_vm *panthor_job_vm(struct drm_sched_job *sched_job)
+{
+	struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
+
+	return job->group->vm;
+}
+
+struct drm_sched_job *
+panthor_job_create(struct panthor_file *pfile,
+		   u16 group_handle,
+		   const struct drm_panthor_queue_submit *qsubmit)
+{
+	struct panthor_group_pool *gpool = pfile->groups;
+	struct panthor_job *job;
+	int ret;
+
+	if (qsubmit->pad)
+		return ERR_PTR(-EINVAL);
+
+	/* If stream_addr is zero, so stream_size should be. */
+	if ((qsubmit->stream_size == 0) != (qsubmit->stream_addr == 0))
+		return ERR_PTR(-EINVAL);
+
+	/* Make sure the address is aligned on 64-byte (cacheline) and the size is
+	 * aligned on 8-byte (instruction size).
+	 */
+	if ((qsubmit->stream_addr & 63) || (qsubmit->stream_size & 7))
+		return ERR_PTR(-EINVAL);
+
+	/* bits 24:30 must be zero. */
+	if (qsubmit->latest_flush & GENMASK(30, 24))
+		return ERR_PTR(-EINVAL);
+
+	job = kzalloc(sizeof(*job), GFP_KERNEL);
+	if (!job)
+		return ERR_PTR(-ENOMEM);
+
+	kref_init(&job->refcount);
+	job->queue_idx = qsubmit->queue_index;
+	job->call_info.size = qsubmit->stream_size;
+	job->call_info.start = qsubmit->stream_addr;
+	job->call_info.latest_flush = qsubmit->latest_flush;
+	INIT_LIST_HEAD(&job->node);
+
+	job->group = group_get(xa_load(&gpool->xa, group_handle));
+	if (!job->group) {
+		ret = -EINVAL;
+		goto err_put_job;
+	}
+
+	if (job->queue_idx >= job->group->queue_count ||
+	    !job->group->queues[job->queue_idx]) {
+		ret = -EINVAL;
+		goto err_put_job;
+	}
+
+	job->done_fence = kzalloc(sizeof(*job->done_fence), GFP_KERNEL);
+	if (!job->done_fence) {
+		ret = -ENOMEM;
+		goto err_put_job;
+	}
+
+	ret = drm_sched_job_init(&job->base,
+				 &job->group->queues[job->queue_idx]->entity,
+				 1, job->group);
+	if (ret)
+		goto err_put_job;
+
+	return &job->base;
+
+err_put_job:
+	panthor_job_put(&job->base);
+	return ERR_PTR(ret);
+}
+
+void panthor_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *sched_job)
+{
+	struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
+
+	/* Still not sure why we want USAGE_WRITE for external objects, since I
+	 * was assuming this would be handled through explicit syncs being imported
+	 * to external BOs with DMA_BUF_IOCTL_IMPORT_SYNC_FILE, but other drivers
+	 * seem to pass DMA_RESV_USAGE_WRITE, so there must be a good reason.
+	 */
+	panthor_vm_update_resvs(job->group->vm, exec, &sched_job->s_fence->finished,
+				DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_WRITE);
+}
+
+void panthor_sched_unplug(struct panthor_device *ptdev)
+{
+	struct panthor_scheduler *sched = ptdev->scheduler;
+
+	cancel_delayed_work_sync(&sched->tick_work);
+
+	mutex_lock(&sched->lock);
+	if (sched->pm.has_ref) {
+		pm_runtime_put(ptdev->base.dev);
+		sched->pm.has_ref = false;
+	}
+	mutex_unlock(&sched->lock);
+}
+
+static void panthor_sched_fini(struct drm_device *ddev, void *res)
+{
+	struct panthor_scheduler *sched = res;
+	int prio;
+
+	if (!sched || !sched->csg_slot_count)
+		return;
+
+	cancel_delayed_work_sync(&sched->tick_work);
+
+	if (sched->wq)
+		destroy_workqueue(sched->wq);
+
+	if (sched->heap_alloc_wq)
+		destroy_workqueue(sched->heap_alloc_wq);
+
+	for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
+		drm_WARN_ON(ddev, !list_empty(&sched->groups.runnable[prio]));
+		drm_WARN_ON(ddev, !list_empty(&sched->groups.idle[prio]));
+	}
+
+	drm_WARN_ON(ddev, !list_empty(&sched->groups.waiting));
+}
+
+int panthor_sched_init(struct panthor_device *ptdev)
+{
+	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
+	struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0);
+	struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, 0, 0);
+	struct panthor_scheduler *sched;
+	u32 gpu_as_count, num_groups;
+	int prio, ret;
+
+	sched = drmm_kzalloc(&ptdev->base, sizeof(*sched), GFP_KERNEL);
+	if (!sched)
+		return -ENOMEM;
+
+	/* The highest bit in JOB_INT_* is reserved for globabl IRQs. That
+	 * leaves 31 bits for CSG IRQs, hence the MAX_CSGS clamp here.
+	 */
+	num_groups = min_t(u32, MAX_CSGS, glb_iface->control->group_num);
+
+	/* The FW-side scheduler might deadlock if two groups with the same
+	 * priority try to access a set of resources that overlaps, with part
+	 * of the resources being allocated to one group and the other part to
+	 * the other group, both groups waiting for the remaining resources to
+	 * be allocated. To avoid that, it is recommended to assign each CSG a
+	 * different priority. In theory we could allow several groups to have
+	 * the same CSG priority if they don't request the same resources, but
+	 * that makes the scheduling logic more complicated, so let's clamp
+	 * the number of CSG slots to MAX_CSG_PRIO + 1 for now.
+	 */
+	num_groups = min_t(u32, MAX_CSG_PRIO + 1, num_groups);
+
+	/* We need at least one AS for the MCU and one for the GPU contexts. */
+	gpu_as_count = hweight32(ptdev->gpu_info.as_present & GENMASK(31, 1));
+	if (!gpu_as_count) {
+		drm_err(&ptdev->base, "Not enough AS (%d, expected at least 2)",
+			gpu_as_count + 1);
+		return -EINVAL;
+	}
+
+	sched->ptdev = ptdev;
+	sched->sb_slot_count = CS_FEATURES_SCOREBOARDS(cs_iface->control->features);
+	sched->csg_slot_count = num_groups;
+	sched->cs_slot_count = csg_iface->control->stream_num;
+	sched->as_slot_count = gpu_as_count;
+	ptdev->csif_info.csg_slot_count = sched->csg_slot_count;
+	ptdev->csif_info.cs_slot_count = sched->cs_slot_count;
+	ptdev->csif_info.scoreboard_slot_count = sched->sb_slot_count;
+
+	sched->last_tick = 0;
+	sched->resched_target = U64_MAX;
+	sched->tick_period = msecs_to_jiffies(10);
+	INIT_DELAYED_WORK(&sched->tick_work, tick_work);
+	INIT_WORK(&sched->sync_upd_work, sync_upd_work);
+	INIT_WORK(&sched->fw_events_work, process_fw_events_work);
+
+	ret = drmm_mutex_init(&ptdev->base, &sched->lock);
+	if (ret)
+		return ret;
+
+	for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
+		INIT_LIST_HEAD(&sched->groups.runnable[prio]);
+		INIT_LIST_HEAD(&sched->groups.idle[prio]);
+	}
+	INIT_LIST_HEAD(&sched->groups.waiting);
+
+	ret = drmm_mutex_init(&ptdev->base, &sched->reset.lock);
+	if (ret)
+		return ret;
+
+	INIT_LIST_HEAD(&sched->reset.stopped_groups);
+
+	/* sched->heap_alloc_wq will be used for heap chunk allocation on
+	 * tiler OOM events, which means we can't use the same workqueue for
+	 * the scheduler because works queued by the scheduler are in
+	 * the dma-signalling path. Allocate a dedicated heap_alloc_wq to
+	 * work around this limitation.
+	 *
+	 * FIXME: Ultimately, what we need is a failable/non-blocking GEM
+	 * allocation path that we can call when a heap OOM is reported. The
+	 * FW is smart enough to fall back on other methods if the kernel can't
+	 * allocate memory, and fail the tiling job if none of these
+	 * countermeasures worked.
+	 *
+	 * Set WQ_MEM_RECLAIM on sched->wq to unblock the situation when the
+	 * system is running out of memory.
+	 */
+	sched->heap_alloc_wq = alloc_workqueue("panthor-heap-alloc", WQ_UNBOUND, 0);
+	sched->wq = alloc_workqueue("panthor-csf-sched", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
+	if (!sched->wq || !sched->heap_alloc_wq) {
+		panthor_sched_fini(&ptdev->base, sched);
+		drm_err(&ptdev->base, "Failed to allocate the workqueues");
+		return -ENOMEM;
+	}
+
+	ret = drmm_add_action_or_reset(&ptdev->base, panthor_sched_fini, sched);
+	if (ret)
+		return ret;
+
+	ptdev->scheduler = sched;
+	return 0;
+}
diff --git a/drivers/gpu/drm/panthor/panthor_sched.h b/drivers/gpu/drm/panthor/panthor_sched.h
new file mode 100644
index 000000000000..3a30d2328b30
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_sched.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2023 Collabora ltd. */
+
+#ifndef __PANTHOR_SCHED_H__
+#define __PANTHOR_SCHED_H__
+
+struct drm_exec;
+struct dma_fence;
+struct drm_file;
+struct drm_gem_object;
+struct drm_sched_job;
+struct drm_panthor_group_create;
+struct drm_panthor_queue_create;
+struct drm_panthor_group_get_state;
+struct drm_panthor_queue_submit;
+struct panthor_device;
+struct panthor_file;
+struct panthor_group_pool;
+struct panthor_job;
+
+int panthor_group_create(struct panthor_file *pfile,
+			 const struct drm_panthor_group_create *group_args,
+			 const struct drm_panthor_queue_create *queue_args);
+int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle);
+int panthor_group_get_state(struct panthor_file *pfile,
+			    struct drm_panthor_group_get_state *get_state);
+
+struct drm_sched_job *
+panthor_job_create(struct panthor_file *pfile,
+		   u16 group_handle,
+		   const struct drm_panthor_queue_submit *qsubmit);
+struct drm_sched_job *panthor_job_get(struct drm_sched_job *job);
+struct panthor_vm *panthor_job_vm(struct drm_sched_job *sched_job);
+void panthor_job_put(struct drm_sched_job *job);
+void panthor_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *job);
+
+int panthor_group_pool_create(struct panthor_file *pfile);
+void panthor_group_pool_destroy(struct panthor_file *pfile);
+
+int panthor_sched_init(struct panthor_device *ptdev);
+void panthor_sched_unplug(struct panthor_device *ptdev);
+void panthor_sched_pre_reset(struct panthor_device *ptdev);
+void panthor_sched_post_reset(struct panthor_device *ptdev, bool reset_failed);
+void panthor_sched_suspend(struct panthor_device *ptdev);
+void panthor_sched_resume(struct panthor_device *ptdev);
+
+void panthor_sched_report_mmu_fault(struct panthor_device *ptdev);
+void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events);
+
+#endif
diff --git a/drivers/gpu/drm/qxl/qxl_object.c b/drivers/gpu/drm/qxl/qxl_object.c
index 1e46b0a6e478..5893e27a7ae5 100644
--- a/drivers/gpu/drm/qxl/qxl_object.c
+++ b/drivers/gpu/drm/qxl/qxl_object.c
@@ -29,9 +29,6 @@
 #include "qxl_drv.h"
 #include "qxl_object.h"
 
-static int __qxl_bo_pin(struct qxl_bo *bo);
-static void __qxl_bo_unpin(struct qxl_bo *bo);
-
 static void qxl_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 {
 	struct qxl_bo *bo;
@@ -167,13 +164,9 @@ int qxl_bo_vmap_locked(struct qxl_bo *bo, struct iosys_map *map)
 		goto out;
 	}
 
-	r = __qxl_bo_pin(bo);
-	if (r)
-		return r;
-
 	r = ttm_bo_vmap(&bo->tbo, &bo->map);
 	if (r) {
-		__qxl_bo_unpin(bo);
+		qxl_bo_unpin_locked(bo);
 		return r;
 	}
 	bo->map_count = 1;
@@ -246,7 +239,6 @@ void qxl_bo_vunmap_locked(struct qxl_bo *bo)
 		return;
 	bo->kptr = NULL;
 	ttm_bo_vunmap(&bo->tbo, &bo->map);
-	__qxl_bo_unpin(bo);
 }
 
 int qxl_bo_vunmap(struct qxl_bo *bo)
@@ -290,12 +282,14 @@ struct qxl_bo *qxl_bo_ref(struct qxl_bo *bo)
 	return bo;
 }
 
-static int __qxl_bo_pin(struct qxl_bo *bo)
+int qxl_bo_pin_locked(struct qxl_bo *bo)
 {
 	struct ttm_operation_ctx ctx = { false, false };
 	struct drm_device *ddev = bo->tbo.base.dev;
 	int r;
 
+	dma_resv_assert_held(bo->tbo.base.resv);
+
 	if (bo->tbo.pin_count) {
 		ttm_bo_pin(&bo->tbo);
 		return 0;
@@ -309,14 +303,16 @@ static int __qxl_bo_pin(struct qxl_bo *bo)
 	return r;
 }
 
-static void __qxl_bo_unpin(struct qxl_bo *bo)
+void qxl_bo_unpin_locked(struct qxl_bo *bo)
 {
+	dma_resv_assert_held(bo->tbo.base.resv);
+
 	ttm_bo_unpin(&bo->tbo);
 }
 
 /*
  * Reserve the BO before pinning the object.  If the BO was reserved
- * beforehand, use the internal version directly __qxl_bo_pin.
+ * beforehand, use the internal version directly qxl_bo_pin_locked.
  *
  */
 int qxl_bo_pin(struct qxl_bo *bo)
@@ -327,14 +323,14 @@ int qxl_bo_pin(struct qxl_bo *bo)
 	if (r)
 		return r;
 
-	r = __qxl_bo_pin(bo);
+	r = qxl_bo_pin_locked(bo);
 	qxl_bo_unreserve(bo);
 	return r;
 }
 
 /*
  * Reserve the BO before pinning the object.  If the BO was reserved
- * beforehand, use the internal version directly __qxl_bo_unpin.
+ * beforehand, use the internal version directly qxl_bo_unpin_locked.
  *
  */
 int qxl_bo_unpin(struct qxl_bo *bo)
@@ -345,7 +341,7 @@ int qxl_bo_unpin(struct qxl_bo *bo)
 	if (r)
 		return r;
 
-	__qxl_bo_unpin(bo);
+	qxl_bo_unpin_locked(bo);
 	qxl_bo_unreserve(bo);
 	return 0;
 }
diff --git a/drivers/gpu/drm/qxl/qxl_object.h b/drivers/gpu/drm/qxl/qxl_object.h
index 53392cb90eec..1cf5bc759101 100644
--- a/drivers/gpu/drm/qxl/qxl_object.h
+++ b/drivers/gpu/drm/qxl/qxl_object.h
@@ -67,6 +67,8 @@ void *qxl_bo_kmap_atomic_page(struct qxl_device *qdev, struct qxl_bo *bo, int pa
 void qxl_bo_kunmap_atomic_page(struct qxl_device *qdev, struct qxl_bo *bo, void *map);
 extern struct qxl_bo *qxl_bo_ref(struct qxl_bo *bo);
 extern void qxl_bo_unref(struct qxl_bo **bo);
+extern int qxl_bo_pin_locked(struct qxl_bo *bo);
+extern void qxl_bo_unpin_locked(struct qxl_bo *bo);
 extern int qxl_bo_pin(struct qxl_bo *bo);
 extern int qxl_bo_unpin(struct qxl_bo *bo);
 extern void qxl_ttm_placement_from_domain(struct qxl_bo *qbo, u32 domain);
diff --git a/drivers/gpu/drm/qxl/qxl_prime.c b/drivers/gpu/drm/qxl/qxl_prime.c
index 9169c26357d3..19bf551a7b31 100644
--- a/drivers/gpu/drm/qxl/qxl_prime.c
+++ b/drivers/gpu/drm/qxl/qxl_prime.c
@@ -32,14 +32,14 @@ int qxl_gem_prime_pin(struct drm_gem_object *obj)
 {
 	struct qxl_bo *bo = gem_to_qxl_bo(obj);
 
-	return qxl_bo_pin(bo);
+	return qxl_bo_pin_locked(bo);
 }
 
 void qxl_gem_prime_unpin(struct drm_gem_object *obj)
 {
 	struct qxl_bo *bo = gem_to_qxl_bo(obj);
 
-	qxl_bo_unpin(bo);
+	qxl_bo_unpin_locked(bo);
 }
 
 struct sg_table *qxl_gem_prime_get_sg_table(struct drm_gem_object *obj)
diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c
index 9febc8b73f09..368d26da0d6a 100644
--- a/drivers/gpu/drm/qxl/qxl_release.c
+++ b/drivers/gpu/drm/qxl/qxl_release.c
@@ -58,56 +58,16 @@ static long qxl_fence_wait(struct dma_fence *fence, bool intr,
 			   signed long timeout)
 {
 	struct qxl_device *qdev;
-	struct qxl_release *release;
-	int count = 0, sc = 0;
-	bool have_drawable_releases;
 	unsigned long cur, end = jiffies + timeout;
 
 	qdev = container_of(fence->lock, struct qxl_device, release_lock);
-	release = container_of(fence, struct qxl_release, base);
-	have_drawable_releases = release->type == QXL_RELEASE_DRAWABLE;
-
-retry:
-	sc++;
-
-	if (dma_fence_is_signaled(fence))
-		goto signaled;
-
-	qxl_io_notify_oom(qdev);
-
-	for (count = 0; count < 11; count++) {
-		if (!qxl_queue_garbage_collect(qdev, true))
-			break;
-
-		if (dma_fence_is_signaled(fence))
-			goto signaled;
-	}
-
-	if (dma_fence_is_signaled(fence))
-		goto signaled;
 
-	if (have_drawable_releases || sc < 4) {
-		if (sc > 2)
-			/* back off */
-			usleep_range(500, 1000);
-
-		if (time_after(jiffies, end))
-			return 0;
-
-		if (have_drawable_releases && sc > 300) {
-			DMA_FENCE_WARN(fence,
-				       "failed to wait on release %llu after spincount %d\n",
-				       fence->context & ~0xf0000000, sc);
-			goto signaled;
-		}
-		goto retry;
-	}
-	/*
-	 * yeah, original sync_obj_wait gave up after 3 spins when
-	 * have_drawable_releases is not set.
-	 */
+	if (!wait_event_timeout(qdev->release_event,
+				(dma_fence_is_signaled(fence) ||
+				 (qxl_io_notify_oom(qdev), 0)),
+				timeout))
+		return 0;
 
-signaled:
 	cur = jiffies;
 	if (time_after(cur, end))
 		return 0;
diff --git a/drivers/gpu/drm/radeon/pptable.h b/drivers/gpu/drm/radeon/pptable.h
index 94947229888b..b7f22597ee95 100644
--- a/drivers/gpu/drm/radeon/pptable.h
+++ b/drivers/gpu/drm/radeon/pptable.h
@@ -424,7 +424,7 @@ typedef struct _ATOM_PPLIB_SUMO_CLOCK_INFO{
 typedef struct _ATOM_PPLIB_STATE_V2
 {
       //number of valid dpm levels in this state; Driver uses it to calculate the whole 
-      //size of the state: sizeof(ATOM_PPLIB_STATE_V2) + (ucNumDPMLevels - 1) * sizeof(UCHAR)
+      //size of the state: struct_size(ATOM_PPLIB_STATE_V2, clockInfoIndex, ucNumDPMLevels)
       UCHAR ucNumDPMLevels;
       
       //a index to the array of nonClockInfos
@@ -432,14 +432,14 @@ typedef struct _ATOM_PPLIB_STATE_V2
       /**
       * Driver will read the first ucNumDPMLevels in this array
       */
-      UCHAR clockInfoIndex[1];
+      UCHAR clockInfoIndex[] __counted_by(ucNumDPMLevels);
 } ATOM_PPLIB_STATE_V2;
 
 typedef struct _StateArray{
     //how many states we have 
     UCHAR ucNumEntries;
     
-    ATOM_PPLIB_STATE_V2 states[1];
+    ATOM_PPLIB_STATE_V2 states[] __counted_by(ucNumEntries);
 }StateArray;
 
 
@@ -450,7 +450,7 @@ typedef struct _ClockInfoArray{
     //sizeof(ATOM_PPLIB_CLOCK_INFO)
     UCHAR ucEntrySize;
     
-    UCHAR clockInfo[1];
+    UCHAR clockInfo[] __counted_by(ucNumEntries);
 }ClockInfoArray;
 
 typedef struct _NonClockInfoArray{
@@ -460,7 +460,7 @@ typedef struct _NonClockInfoArray{
     //sizeof(ATOM_PPLIB_NONCLOCK_INFO)
     UCHAR ucEntrySize;
     
-    ATOM_PPLIB_NONCLOCK_INFO nonClockInfo[1];
+    ATOM_PPLIB_NONCLOCK_INFO nonClockInfo[] __counted_by(ucNumEntries);
 }NonClockInfoArray;
 
 typedef struct _ATOM_PPLIB_Clock_Voltage_Dependency_Record
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index 86b8b770af19..0b1e19345f43 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -26,6 +26,7 @@
  *          Jerome Glisse
  */
 
+#include <linux/debugfs.h>
 #include <linux/firmware.h>
 #include <linux/module.h>
 #include <linux/pci.h>
diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c
index 25201b9a5aae..1620f534f55f 100644
--- a/drivers/gpu/drm/radeon/r300.c
+++ b/drivers/gpu/drm/radeon/r300.c
@@ -26,6 +26,7 @@
  *          Jerome Glisse
  */
 
+#include <linux/debugfs.h>
 #include <linux/pci.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
diff --git a/drivers/gpu/drm/radeon/r420.c b/drivers/gpu/drm/radeon/r420.c
index eae8a6389f5e..a979662eaa73 100644
--- a/drivers/gpu/drm/radeon/r420.c
+++ b/drivers/gpu/drm/radeon/r420.c
@@ -26,6 +26,7 @@
  *          Jerome Glisse
  */
 
+#include <linux/debugfs.h>
 #include <linux/pci.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index b5e97d95a19f..087d41e370fd 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -26,11 +26,12 @@
  *          Jerome Glisse
  */
 
+#include <linux/debugfs.h>
 #include <linux/firmware.h>
 #include <linux/module.h>
 #include <linux/pci.h>
-#include <linux/slab.h>
 #include <linux/seq_file.h>
+#include <linux/slab.h>
 
 #include <drm/drm_device.h>
 #include <drm/drm_vblank.h>
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 3e5ff17e3caf..0999c8eaae94 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -132,7 +132,6 @@ extern int radeon_cik_support;
 /* RADEON_IB_POOL_SIZE must be a power of 2 */
 #define RADEON_IB_POOL_SIZE			16
 #define RADEON_DEBUGFS_MAX_COMPONENTS		32
-#define RADEONFB_CONN_LIMIT			4
 #define RADEON_BIOS_NUM_SCRATCH			8
 
 /* internal ring indices */
diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c
index bb1f0a3371ab..10793a433bf5 100644
--- a/drivers/gpu/drm/radeon/radeon_atombios.c
+++ b/drivers/gpu/drm/radeon/radeon_atombios.c
@@ -923,8 +923,12 @@ bool radeon_get_atom_connector_info_from_supported_devices_table(struct
 		max_device = ATOM_MAX_SUPPORTED_DEVICE_INFO;
 
 	for (i = 0; i < max_device; i++) {
-		ATOM_CONNECTOR_INFO_I2C ci =
-		    supported_devices->info.asConnInfo[i];
+		ATOM_CONNECTOR_INFO_I2C ci;
+
+		if (frev > 1)
+			ci = supported_devices->info_2d1.asConnInfo[i];
+		else
+			ci = supported_devices->info.asConnInfo[i];
 
 		bios_connectors[i].valid = false;
 
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index efd18c8d84c8..5f1d24d3120c 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -683,7 +683,7 @@ static void radeon_crtc_init(struct drm_device *dev, int index)
 	struct radeon_device *rdev = dev->dev_private;
 	struct radeon_crtc *radeon_crtc;
 
-	radeon_crtc = kzalloc(sizeof(struct radeon_crtc) + (RADEONFB_CONN_LIMIT * sizeof(struct drm_connector *)), GFP_KERNEL);
+	radeon_crtc = kzalloc(sizeof(*radeon_crtc), GFP_KERNEL);
 	if (radeon_crtc == NULL)
 		return;
 
@@ -709,12 +709,6 @@ static void radeon_crtc_init(struct drm_device *dev, int index)
 	dev->mode_config.cursor_width = radeon_crtc->max_cursor_width;
 	dev->mode_config.cursor_height = radeon_crtc->max_cursor_height;
 
-#if 0
-	radeon_crtc->mode_set.crtc = &radeon_crtc->base;
-	radeon_crtc->mode_set.connectors = (struct drm_connector **)(radeon_crtc + 1);
-	radeon_crtc->mode_set.num_connectors = 0;
-#endif
-
 	if (rdev->is_atom_bios && (ASIC_IS_AVIVO(rdev) || radeon_r4xx_atom))
 		radeon_atombios_init_crtc(dev, radeon_crtc);
 	else
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index 9ebe4a0b9a6c..4fb780d96f32 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -30,6 +30,7 @@
  */
 
 #include <linux/atomic.h>
+#include <linux/debugfs.h>
 #include <linux/firmware.h>
 #include <linux/kref.h>
 #include <linux/sched/signal.h>
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index 3fec3acdaf28..2ef201a072f1 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -26,6 +26,7 @@
  *          Jerome Glisse
  */
 
+#include <linux/debugfs.h>
 #include <linux/iosys-map.h>
 #include <linux/pci.h>
 
diff --git a/drivers/gpu/drm/radeon/radeon_ib.c b/drivers/gpu/drm/radeon/radeon_ib.c
index fb9ecf5dbe2b..63d914f3414d 100644
--- a/drivers/gpu/drm/radeon/radeon_ib.c
+++ b/drivers/gpu/drm/radeon/radeon_ib.c
@@ -27,6 +27,8 @@
  *          Christian König
  */
 
+#include <linux/debugfs.h>
+
 #include <drm/drm_file.h>
 
 #include "radeon.h"
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 4482c8c5f5ce..2d9d9f46f243 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -21,6 +21,7 @@
  *          Alex Deucher <alexdeucher@gmail.com>
  */
 
+#include <linux/debugfs.h>
 #include <linux/hwmon-sysfs.h>
 #include <linux/hwmon.h>
 #include <linux/pci.h>
diff --git a/drivers/gpu/drm/radeon/radeon_prime.c b/drivers/gpu/drm/radeon/radeon_prime.c
index b3cfc99f4d7e..a77881f035e7 100644
--- a/drivers/gpu/drm/radeon/radeon_prime.c
+++ b/drivers/gpu/drm/radeon/radeon_prime.c
@@ -73,32 +73,21 @@ int radeon_gem_prime_pin(struct drm_gem_object *obj)
 	struct radeon_bo *bo = gem_to_radeon_bo(obj);
 	int ret = 0;
 
-	ret = radeon_bo_reserve(bo, false);
-	if (unlikely(ret != 0))
-		return ret;
-
 	/* pin buffer into GTT */
 	ret = radeon_bo_pin(bo, RADEON_GEM_DOMAIN_GTT, NULL);
 	if (likely(ret == 0))
 		bo->prime_shared_count++;
 
-	radeon_bo_unreserve(bo);
 	return ret;
 }
 
 void radeon_gem_prime_unpin(struct drm_gem_object *obj)
 {
 	struct radeon_bo *bo = gem_to_radeon_bo(obj);
-	int ret = 0;
-
-	ret = radeon_bo_reserve(bo, false);
-	if (unlikely(ret != 0))
-		return;
 
 	radeon_bo_unpin(bo);
 	if (bo->prime_shared_count)
 		bo->prime_shared_count--;
-	radeon_bo_unreserve(bo);
 }
 
 
diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c
index 38048593bb4a..8d1d458286a8 100644
--- a/drivers/gpu/drm/radeon/radeon_ring.c
+++ b/drivers/gpu/drm/radeon/radeon_ring.c
@@ -27,6 +27,8 @@
  *          Christian König
  */
 
+#include <linux/debugfs.h>
+
 #include <drm/drm_device.h>
 #include <drm/drm_file.h>
 
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 2078b0000e22..5c65b6dfb99a 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -30,6 +30,7 @@
  *    Dave Airlie
  */
 
+#include <linux/debugfs.h>
 #include <linux/dma-mapping.h>
 #include <linux/pagemap.h>
 #include <linux/pci.h>
diff --git a/drivers/gpu/drm/radeon/rs400.c b/drivers/gpu/drm/radeon/rs400.c
index d7f552d441ab..d4d1501e6576 100644
--- a/drivers/gpu/drm/radeon/rs400.c
+++ b/drivers/gpu/drm/radeon/rs400.c
@@ -26,6 +26,7 @@
  *          Jerome Glisse
  */
 
+#include <linux/debugfs.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 
diff --git a/drivers/gpu/drm/radeon/rv515.c b/drivers/gpu/drm/radeon/rv515.c
index 79709d26d983..bbc6ccabf788 100644
--- a/drivers/gpu/drm/radeon/rv515.c
+++ b/drivers/gpu/drm/radeon/rv515.c
@@ -26,6 +26,7 @@
  *          Jerome Glisse
  */
 
+#include <linux/debugfs.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 
diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c
index 0ae6331d6430..8643ff2eec46 100644
--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c
+++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c
@@ -66,9 +66,6 @@ void rzg2l_du_vsp_disable(struct rzg2l_du_crtc *crtc)
 void rzg2l_du_vsp_atomic_flush(struct rzg2l_du_crtc *crtc)
 {
 	struct vsp1_du_atomic_pipe_config cfg = { { 0, } };
-	struct rzg2l_du_crtc_state *state;
-
-	state = to_rzg2l_crtc_state(crtc->crtc.state);
 
 	vsp1_du_atomic_flush(crtc->vsp->vsp, crtc->vsp_pipe, &cfg);
 }
diff --git a/drivers/gpu/drm/rockchip/cdn-dp-core.c b/drivers/gpu/drm/rockchip/cdn-dp-core.c
index a855c45ae7f3..bd7aa891b839 100644
--- a/drivers/gpu/drm/rockchip/cdn-dp-core.c
+++ b/drivers/gpu/drm/rockchip/cdn-dp-core.c
@@ -262,20 +262,21 @@ static const struct drm_connector_funcs cdn_dp_atomic_connector_funcs = {
 static int cdn_dp_connector_get_modes(struct drm_connector *connector)
 {
 	struct cdn_dp_device *dp = connector_to_dp(connector);
-	struct edid *edid;
 	int ret = 0;
 
 	mutex_lock(&dp->lock);
-	edid = dp->edid;
-	if (edid) {
+
+	if (dp->drm_edid) {
+		/* FIXME: get rid of drm_edid_raw() */
+		const struct edid *edid = drm_edid_raw(dp->drm_edid);
+
 		DRM_DEV_DEBUG_KMS(dp->dev, "got edid: width[%d] x height[%d]\n",
 				  edid->width_cm, edid->height_cm);
 
-		dp->sink_has_audio = drm_detect_monitor_audio(edid);
-
-		drm_connector_update_edid_property(connector, edid);
-		ret = drm_add_edid_modes(connector, edid);
 	}
+
+	ret = drm_edid_connector_add_modes(connector);
+
 	mutex_unlock(&dp->lock);
 
 	return ret;
@@ -380,9 +381,13 @@ static int cdn_dp_get_sink_capability(struct cdn_dp_device *dp)
 		return ret;
 	}
 
-	kfree(dp->edid);
-	dp->edid = drm_do_get_edid(&dp->connector,
-				   cdn_dp_get_edid_block, dp);
+	drm_edid_free(dp->drm_edid);
+	dp->drm_edid = drm_edid_read_custom(&dp->connector,
+					    cdn_dp_get_edid_block, dp);
+	drm_edid_connector_update(&dp->connector, dp->drm_edid);
+
+	dp->sink_has_audio = dp->connector.display_info.has_audio;
+
 	return 0;
 }
 
@@ -488,8 +493,8 @@ static int cdn_dp_disable(struct cdn_dp_device *dp)
 	dp->max_lanes = 0;
 	dp->max_rate = 0;
 	if (!dp->connected) {
-		kfree(dp->edid);
-		dp->edid = NULL;
+		drm_edid_free(dp->drm_edid);
+		dp->drm_edid = NULL;
 	}
 
 	return 0;
@@ -1131,8 +1136,8 @@ static void cdn_dp_unbind(struct device *dev, struct device *master, void *data)
 	pm_runtime_disable(dev);
 	if (dp->fw_loaded)
 		release_firmware(dp->fw);
-	kfree(dp->edid);
-	dp->edid = NULL;
+	drm_edid_free(dp->drm_edid);
+	dp->drm_edid = NULL;
 }
 
 static const struct component_ops cdn_dp_component_ops = {
@@ -1259,7 +1264,6 @@ struct platform_driver cdn_dp_driver = {
 	.shutdown = cdn_dp_shutdown,
 	.driver = {
 		   .name = "cdn-dp",
-		   .owner = THIS_MODULE,
 		   .of_match_table = cdn_dp_dt_ids,
 		   .pm = &cdn_dp_pm_ops,
 	},
diff --git a/drivers/gpu/drm/rockchip/cdn-dp-core.h b/drivers/gpu/drm/rockchip/cdn-dp-core.h
index 5b2fed1f5f55..8e6e95d269da 100644
--- a/drivers/gpu/drm/rockchip/cdn-dp-core.h
+++ b/drivers/gpu/drm/rockchip/cdn-dp-core.h
@@ -70,7 +70,7 @@ struct cdn_dp_device {
 	struct drm_display_mode mode;
 	struct platform_device *audio_pdev;
 	struct work_struct event_work;
-	struct edid *edid;
+	const struct drm_edid *drm_edid;
 
 	struct mutex lock;
 	bool connected;
diff --git a/drivers/gpu/drm/rockchip/inno_hdmi.c b/drivers/gpu/drm/rockchip/inno_hdmi.c
index 1d2261643743..3df2cfcf9998 100644
--- a/drivers/gpu/drm/rockchip/inno_hdmi.c
+++ b/drivers/gpu/drm/rockchip/inno_hdmi.c
@@ -606,18 +606,16 @@ inno_hdmi_connector_detect(struct drm_connector *connector, bool force)
 static int inno_hdmi_connector_get_modes(struct drm_connector *connector)
 {
 	struct inno_hdmi *hdmi = connector_to_inno_hdmi(connector);
-	struct edid *edid;
+	const struct drm_edid *drm_edid;
 	int ret = 0;
 
 	if (!hdmi->ddc)
 		return 0;
 
-	edid = drm_get_edid(connector, hdmi->ddc);
-	if (edid) {
-		drm_connector_update_edid_property(connector, edid);
-		ret = drm_add_edid_modes(connector, edid);
-		kfree(edid);
-	}
+	drm_edid = drm_edid_read_ddc(connector, hdmi->ddc);
+	drm_edid_connector_update(connector, drm_edid);
+	ret = drm_edid_connector_add_modes(connector);
+	drm_edid_free(drm_edid);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/rockchip/rk3066_hdmi.c b/drivers/gpu/drm/rockchip/rk3066_hdmi.c
index 95cd1b49eda8..784de990da1b 100644
--- a/drivers/gpu/drm/rockchip/rk3066_hdmi.c
+++ b/drivers/gpu/drm/rockchip/rk3066_hdmi.c
@@ -466,18 +466,16 @@ rk3066_hdmi_connector_detect(struct drm_connector *connector, bool force)
 static int rk3066_hdmi_connector_get_modes(struct drm_connector *connector)
 {
 	struct rk3066_hdmi *hdmi = connector_to_rk3066_hdmi(connector);
-	struct edid *edid;
+	const struct drm_edid *drm_edid;
 	int ret = 0;
 
 	if (!hdmi->ddc)
 		return 0;
 
-	edid = drm_get_edid(connector, hdmi->ddc);
-	if (edid) {
-		drm_connector_update_edid_property(connector, edid);
-		ret = drm_add_edid_modes(connector, edid);
-		kfree(edid);
-	}
+	drm_edid = drm_edid_read_ddc(connector, hdmi->ddc);
+	drm_edid_connector_update(connector, drm_edid);
+	ret = drm_edid_connector_add_modes(connector);
+	drm_edid_free(drm_edid);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
index fdd768bbd487..62ebbdb16253 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
@@ -706,6 +706,8 @@ static void vop2_setup_scale(struct vop2 *vop2, const struct vop2_win *win,
 	const struct drm_format_info *info;
 	u16 hor_scl_mode, ver_scl_mode;
 	u16 hscl_filter_mode, vscl_filter_mode;
+	uint16_t cbcr_src_w = src_w;
+	uint16_t cbcr_src_h = src_h;
 	u8 gt2 = 0;
 	u8 gt4 = 0;
 	u32 val;
@@ -763,27 +765,27 @@ static void vop2_setup_scale(struct vop2 *vop2, const struct vop2_win *win,
 	vop2_win_write(win, VOP2_WIN_YRGB_VSCL_FILTER_MODE, vscl_filter_mode);
 
 	if (info->is_yuv) {
-		src_w /= info->hsub;
-		src_h /= info->vsub;
+		cbcr_src_w /= info->hsub;
+		cbcr_src_h /= info->vsub;
 
 		gt4 = 0;
 		gt2 = 0;
 
-		if (src_h >= (4 * dst_h)) {
+		if (cbcr_src_h >= (4 * dst_h)) {
 			gt4 = 1;
-			src_h >>= 2;
-		} else if (src_h >= (2 * dst_h)) {
+			cbcr_src_h >>= 2;
+		} else if (cbcr_src_h >= (2 * dst_h)) {
 			gt2 = 1;
-			src_h >>= 1;
+			cbcr_src_h >>= 1;
 		}
 
-		hor_scl_mode = scl_get_scl_mode(src_w, dst_w);
-		ver_scl_mode = scl_get_scl_mode(src_h, dst_h);
+		hor_scl_mode = scl_get_scl_mode(cbcr_src_w, dst_w);
+		ver_scl_mode = scl_get_scl_mode(cbcr_src_h, dst_h);
 
-		val = vop2_scale_factor(src_w, dst_w);
+		val = vop2_scale_factor(cbcr_src_w, dst_w);
 		vop2_win_write(win, VOP2_WIN_SCALE_CBCR_X, val);
 
-		val = vop2_scale_factor(src_h, dst_h);
+		val = vop2_scale_factor(cbcr_src_h, dst_h);
 		vop2_win_write(win, VOP2_WIN_SCALE_CBCR_Y, val);
 
 		vop2_win_write(win, VOP2_WIN_VSD_CBCR_GT4, gt4);
diff --git a/drivers/gpu/drm/rockchip/rockchip_lvds.c b/drivers/gpu/drm/rockchip/rockchip_lvds.c
index 77b76cff1adb..9a01aa450741 100644
--- a/drivers/gpu/drm/rockchip/rockchip_lvds.c
+++ b/drivers/gpu/drm/rockchip/rockchip_lvds.c
@@ -17,7 +17,6 @@
 #include <linux/regmap.h>
 #include <linux/reset.h>
 
-#include <drm/display/drm_dp_helper.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_bridge_connector.h>
diff --git a/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h b/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h
index f8ed093b7356..c75302ca3427 100644
--- a/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h
+++ b/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h
@@ -48,7 +48,7 @@ DECLARE_EVENT_CLASS(drm_sched_job,
 			   __entry->entity = entity;
 			   __entry->id = sched_job->id;
 			   __entry->fence = &sched_job->s_fence->finished;
-			   __assign_str(name, sched_job->sched->name);
+			   __assign_str(name);
 			   __entry->job_count = spsc_queue_count(&entity->job_queue);
 			   __entry->hw_job_count = atomic_read(
 				   &sched_job->sched->credit_count);
@@ -94,7 +94,7 @@ TRACE_EVENT(drm_sched_job_wait_dep,
 			     ),
 
 	    TP_fast_assign(
-			   __assign_str(name, sched_job->sched->name);
+			   __assign_str(name);
 			   __entry->id = sched_job->id;
 			   __entry->fence = fence;
 			   __entry->ctx = fence->context;
diff --git a/drivers/gpu/drm/sti/sti_drv.c b/drivers/gpu/drm/sti/sti_drv.c
index 4bab93c4fefd..1799c12babf5 100644
--- a/drivers/gpu/drm/sti/sti_drv.c
+++ b/drivers/gpu/drm/sti/sti_drv.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/component.h>
+#include <linux/debugfs.h>
 #include <linux/dma-mapping.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
index 2d1880c61b50..245b34adca5a 100644
--- a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
+++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
@@ -214,20 +214,24 @@ sun4i_hdmi_connector_mode_valid(struct drm_connector *connector,
 static int sun4i_hdmi_get_modes(struct drm_connector *connector)
 {
 	struct sun4i_hdmi *hdmi = drm_connector_to_sun4i_hdmi(connector);
-	struct edid *edid;
+	const struct drm_edid *drm_edid;
 	int ret;
 
-	edid = drm_get_edid(connector, hdmi->ddc_i2c ?: hdmi->i2c);
-	if (!edid)
+	drm_edid = drm_edid_read_ddc(connector, hdmi->ddc_i2c ?: hdmi->i2c);
+
+	drm_edid_connector_update(connector, drm_edid);
+	cec_s_phys_addr(hdmi->cec_adap,
+			connector->display_info.source_physical_address, false);
+
+	if (!drm_edid)
 		return 0;
 
 	DRM_DEBUG_DRIVER("Monitor is %s monitor\n",
 			 connector->display_info.is_hdmi ? "an HDMI" : "a DVI");
 
-	drm_connector_update_edid_property(connector, edid);
-	cec_s_phys_addr_from_edid(hdmi->cec_adap, edid);
-	ret = drm_add_edid_modes(connector, edid);
-	kfree(edid);
+
+	ret = drm_edid_connector_add_modes(connector);
+	drm_edid_free(drm_edid);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/tegra/Kconfig b/drivers/gpu/drm/tegra/Kconfig
index 84e7e6bc3a0c..782f51d3044a 100644
--- a/drivers/gpu/drm/tegra/Kconfig
+++ b/drivers/gpu/drm/tegra/Kconfig
@@ -8,7 +8,7 @@ config DRM_TEGRA
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HDMI_HELPER
 	select DRM_DISPLAY_HELPER
-	select DRM_DP_AUX_BUS
+	select DRM_DISPLAY_DP_AUX_BUS
 	select DRM_KMS_HELPER
 	select DRM_MIPI_DSI
 	select DRM_PANEL
diff --git a/drivers/gpu/drm/tests/drm_buddy_test.c b/drivers/gpu/drm/tests/drm_buddy_test.c
index e48863a44556..dd8fb9f8341a 100644
--- a/drivers/gpu/drm/tests/drm_buddy_test.c
+++ b/drivers/gpu/drm/tests/drm_buddy_test.c
@@ -23,9 +23,11 @@ static inline u64 get_size(int order, u64 chunk_size)
 
 static void drm_test_buddy_alloc_range_bias(struct kunit *test)
 {
-	u32 mm_size, ps, bias_size, bias_start, bias_end, bias_rem;
+	u32 mm_size, size, ps, bias_size, bias_start, bias_end, bias_rem;
 	DRM_RND_STATE(prng, random_seed);
 	unsigned int i, count, *order;
+	struct drm_buddy_block *block;
+	unsigned long flags;
 	struct drm_buddy mm;
 	LIST_HEAD(allocated);
 
@@ -103,7 +105,7 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test)
 							      DRM_BUDDY_RANGE_ALLOCATION),
 				       "buddy_alloc i failed with bias(%x-%x), size=%u, ps=%u\n",
 				       bias_start, bias_end, bias_size, bias_size);
-		drm_buddy_free_list(&mm, &tmp);
+		drm_buddy_free_list(&mm, &tmp, 0);
 
 		/* single page with internal round_up */
 		KUNIT_ASSERT_FALSE_MSG(test,
@@ -113,7 +115,7 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test)
 							      DRM_BUDDY_RANGE_ALLOCATION),
 				       "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
 				       bias_start, bias_end, ps, bias_size);
-		drm_buddy_free_list(&mm, &tmp);
+		drm_buddy_free_list(&mm, &tmp, 0);
 
 		/* random size within */
 		size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
@@ -153,14 +155,14 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test)
 			 * unallocated, and ideally not always on the bias
 			 * boundaries.
 			 */
-			drm_buddy_free_list(&mm, &tmp);
+			drm_buddy_free_list(&mm, &tmp, 0);
 		} else {
 			list_splice_tail(&tmp, &allocated);
 		}
 	}
 
 	kfree(order);
-	drm_buddy_free_list(&mm, &allocated);
+	drm_buddy_free_list(&mm, &allocated, 0);
 	drm_buddy_fini(&mm);
 
 	/*
@@ -220,7 +222,181 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test)
 			      "buddy_alloc passed with bias(%x-%x), size=%u\n",
 			      bias_start, bias_end, ps);
 
-	drm_buddy_free_list(&mm, &allocated);
+	drm_buddy_free_list(&mm, &allocated, 0);
+	drm_buddy_fini(&mm);
+
+	/*
+	 * Allocate cleared blocks in the bias range when the DRM buddy's clear avail is
+	 * zero. This will validate the bias range allocation in scenarios like system boot
+	 * when no cleared blocks are available and exercise the fallback path too. The resulting
+	 * blocks should always be dirty.
+	 */
+
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, ps),
+			       "buddy_init failed\n");
+
+	bias_start = round_up(prandom_u32_state(&prng) % (mm_size - ps), ps);
+	bias_end = round_up(bias_start + prandom_u32_state(&prng) % (mm_size - bias_start), ps);
+	bias_end = max(bias_end, bias_start + ps);
+	bias_rem = bias_end - bias_start;
+
+	flags = DRM_BUDDY_CLEAR_ALLOCATION | DRM_BUDDY_RANGE_ALLOCATION;
+	size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps);
+
+	KUNIT_ASSERT_FALSE_MSG(test,
+			       drm_buddy_alloc_blocks(&mm, bias_start,
+						      bias_end, size, ps,
+						      &allocated,
+						      flags),
+			       "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n",
+			       bias_start, bias_end, size, ps);
+
+	list_for_each_entry(block, &allocated, link)
+		KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), false);
+
+	drm_buddy_free_list(&mm, &allocated, 0);
+	drm_buddy_fini(&mm);
+}
+
+static void drm_test_buddy_alloc_clear(struct kunit *test)
+{
+	unsigned long n_pages, total, i = 0;
+	DRM_RND_STATE(prng, random_seed);
+	const unsigned long ps = SZ_4K;
+	struct drm_buddy_block *block;
+	const int max_order = 12;
+	LIST_HEAD(allocated);
+	struct drm_buddy mm;
+	unsigned int order;
+	u32 mm_size, size;
+	LIST_HEAD(dirty);
+	LIST_HEAD(clean);
+
+	mm_size = SZ_4K << max_order;
+	KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps));
+
+	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+
+	/*
+	 * Idea is to allocate and free some random portion of the address space,
+	 * returning those pages as non-dirty and randomly alternate between
+	 * requesting dirty and non-dirty pages (not going over the limit
+	 * we freed as non-dirty), putting that into two separate lists.
+	 * Loop over both lists at the end checking that the dirty list
+	 * is indeed all dirty pages and vice versa. Free it all again,
+	 * keeping the dirty/clear status.
+	 */
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+							    5 * ps, ps, &allocated,
+							    DRM_BUDDY_TOPDOWN_ALLOCATION),
+				"buddy_alloc hit an error size=%lu\n", 5 * ps);
+	drm_buddy_free_list(&mm, &allocated, DRM_BUDDY_CLEARED);
+
+	n_pages = 10;
+	do {
+		unsigned long flags;
+		struct list_head *list;
+		int slot = i % 2;
+
+		if (slot == 0) {
+			list = &dirty;
+			flags = 0;
+		} else {
+			list = &clean;
+			flags = DRM_BUDDY_CLEAR_ALLOCATION;
+		}
+
+		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+								    ps, ps, list,
+								    flags),
+					"buddy_alloc hit an error size=%lu\n", ps);
+	} while (++i < n_pages);
+
+	list_for_each_entry(block, &clean, link)
+		KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), true);
+
+	list_for_each_entry(block, &dirty, link)
+		KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), false);
+
+	drm_buddy_free_list(&mm, &clean, DRM_BUDDY_CLEARED);
+
+	/*
+	 * Trying to go over the clear limit for some allocation.
+	 * The allocation should never fail with reasonable page-size.
+	 */
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+							    10 * ps, ps, &clean,
+							    DRM_BUDDY_CLEAR_ALLOCATION),
+				"buddy_alloc hit an error size=%lu\n", 10 * ps);
+
+	drm_buddy_free_list(&mm, &clean, DRM_BUDDY_CLEARED);
+	drm_buddy_free_list(&mm, &dirty, 0);
+	drm_buddy_fini(&mm);
+
+	KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps));
+
+	/*
+	 * Create a new mm. Intentionally fragment the address space by creating
+	 * two alternating lists. Free both lists, one as dirty the other as clean.
+	 * Try to allocate double the previous size with matching min_page_size. The
+	 * allocation should never fail as it calls the force_merge. Also check that
+	 * the page is always dirty after force_merge. Free the page as dirty, then
+	 * repeat the whole thing, increment the order until we hit the max_order.
+	 */
+
+	i = 0;
+	n_pages = mm_size / ps;
+	do {
+		struct list_head *list;
+		int slot = i % 2;
+
+		if (slot == 0)
+			list = &dirty;
+		else
+			list = &clean;
+
+		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+								    ps, ps, list, 0),
+					"buddy_alloc hit an error size=%lu\n", ps);
+	} while (++i < n_pages);
+
+	drm_buddy_free_list(&mm, &clean, DRM_BUDDY_CLEARED);
+	drm_buddy_free_list(&mm, &dirty, 0);
+
+	order = 1;
+	do {
+		size = SZ_4K << order;
+
+		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+								    size, size, &allocated,
+								    DRM_BUDDY_CLEAR_ALLOCATION),
+					"buddy_alloc hit an error size=%u\n", size);
+		total = 0;
+		list_for_each_entry(block, &allocated, link) {
+			if (size != mm_size)
+				KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), false);
+			total += drm_buddy_block_size(&mm, block);
+		}
+		KUNIT_EXPECT_EQ(test, total, size);
+
+		drm_buddy_free_list(&mm, &allocated, 0);
+	} while (++order <= max_order);
+
+	drm_buddy_fini(&mm);
+
+	/*
+	 * Create a new mm with a non power-of-two size. Allocate a random size, free as
+	 * cleared and then call fini. This will ensure the multi-root force merge during
+	 * fini.
+	 */
+	mm_size = 12 * SZ_4K;
+	size = max(round_up(prandom_u32_state(&prng) % mm_size, ps), ps);
+	KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps));
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
+							    size, ps, &allocated,
+							    DRM_BUDDY_TOPDOWN_ALLOCATION),
+				"buddy_alloc hit an error size=%u\n", size);
+	drm_buddy_free_list(&mm, &allocated, DRM_BUDDY_CLEARED);
 	drm_buddy_fini(&mm);
 }
 
@@ -269,7 +445,7 @@ static void drm_test_buddy_alloc_contiguous(struct kunit *test)
 							   DRM_BUDDY_CONTIGUOUS_ALLOCATION),
 			       "buddy_alloc didn't error size=%lu\n", 3 * ps);
 
-	drm_buddy_free_list(&mm, &middle);
+	drm_buddy_free_list(&mm, &middle, 0);
 	KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
 							   3 * ps, ps, &allocated,
 							   DRM_BUDDY_CONTIGUOUS_ALLOCATION),
@@ -279,7 +455,7 @@ static void drm_test_buddy_alloc_contiguous(struct kunit *test)
 							   DRM_BUDDY_CONTIGUOUS_ALLOCATION),
 			       "buddy_alloc didn't error size=%lu\n", 2 * ps);
 
-	drm_buddy_free_list(&mm, &right);
+	drm_buddy_free_list(&mm, &right, 0);
 	KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
 							   3 * ps, ps, &allocated,
 							   DRM_BUDDY_CONTIGUOUS_ALLOCATION),
@@ -294,7 +470,7 @@ static void drm_test_buddy_alloc_contiguous(struct kunit *test)
 							    DRM_BUDDY_CONTIGUOUS_ALLOCATION),
 			       "buddy_alloc hit an error size=%lu\n", 2 * ps);
 
-	drm_buddy_free_list(&mm, &left);
+	drm_buddy_free_list(&mm, &left, 0);
 	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size,
 							    3 * ps, ps, &allocated,
 							    DRM_BUDDY_CONTIGUOUS_ALLOCATION),
@@ -306,7 +482,7 @@ static void drm_test_buddy_alloc_contiguous(struct kunit *test)
 
 	KUNIT_ASSERT_EQ(test, total, ps * 2 + ps * 3);
 
-	drm_buddy_free_list(&mm, &allocated);
+	drm_buddy_free_list(&mm, &allocated, 0);
 	drm_buddy_fini(&mm);
 }
 
@@ -329,8 +505,8 @@ static void drm_test_buddy_alloc_pathological(struct kunit *test)
 	 * Eventually we will have a fully 50% fragmented mm.
 	 */
 
-	mm_size = PAGE_SIZE << max_order;
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, PAGE_SIZE),
+	mm_size = SZ_4K << max_order;
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K),
 			       "buddy_init failed\n");
 
 	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
@@ -344,7 +520,7 @@ static void drm_test_buddy_alloc_pathological(struct kunit *test)
 		}
 
 		for (order = top; order--;) {
-			size = get_size(order, PAGE_SIZE);
+			size = get_size(order, mm.chunk_size);
 			KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start,
 									    mm_size, size, size,
 										&tmp, flags),
@@ -358,7 +534,7 @@ static void drm_test_buddy_alloc_pathological(struct kunit *test)
 		}
 
 		/* There should be one final page for this sub-allocation */
-		size = get_size(0, PAGE_SIZE);
+		size = get_size(0, mm.chunk_size);
 		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
 								    size, size, &tmp, flags),
 							   "buddy_alloc hit -ENOMEM for hole\n");
@@ -368,18 +544,18 @@ static void drm_test_buddy_alloc_pathological(struct kunit *test)
 
 		list_move_tail(&block->link, &holes);
 
-		size = get_size(top, PAGE_SIZE);
+		size = get_size(top, mm.chunk_size);
 		KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
 								   size, size, &tmp, flags),
 							  "buddy_alloc unexpectedly succeeded at top-order %d/%d, it should be full!",
 							  top, max_order);
 	}
 
-	drm_buddy_free_list(&mm, &holes);
+	drm_buddy_free_list(&mm, &holes, 0);
 
 	/* Nothing larger than blocks of chunk_size now available */
 	for (order = 1; order <= max_order; order++) {
-		size = get_size(order, PAGE_SIZE);
+		size = get_size(order, mm.chunk_size);
 		KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
 								   size, size, &tmp, flags),
 							  "buddy_alloc unexpectedly succeeded at order %d, it should be full!",
@@ -387,7 +563,7 @@ static void drm_test_buddy_alloc_pathological(struct kunit *test)
 	}
 
 	list_splice_tail(&holes, &blocks);
-	drm_buddy_free_list(&mm, &blocks);
+	drm_buddy_free_list(&mm, &blocks, 0);
 	drm_buddy_fini(&mm);
 }
 
@@ -408,14 +584,14 @@ static void drm_test_buddy_alloc_pessimistic(struct kunit *test)
 	 * page left.
 	 */
 
-	mm_size = PAGE_SIZE << max_order;
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, PAGE_SIZE),
+	mm_size = SZ_4K << max_order;
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K),
 			       "buddy_init failed\n");
 
 	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
 
 	for (order = 0; order < max_order; order++) {
-		size = get_size(order, PAGE_SIZE);
+		size = get_size(order, mm.chunk_size);
 		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
 								    size, size, &tmp, flags),
 							   "buddy_alloc hit -ENOMEM with order=%d\n",
@@ -428,7 +604,7 @@ static void drm_test_buddy_alloc_pessimistic(struct kunit *test)
 	}
 
 	/* And now the last remaining block available */
-	size = get_size(0, PAGE_SIZE);
+	size = get_size(0, mm.chunk_size);
 	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
 							    size, size, &tmp, flags),
 						   "buddy_alloc hit -ENOMEM on final alloc\n");
@@ -440,7 +616,7 @@ static void drm_test_buddy_alloc_pessimistic(struct kunit *test)
 
 	/* Should be completely full! */
 	for (order = max_order; order--;) {
-		size = get_size(order, PAGE_SIZE);
+		size = get_size(order, mm.chunk_size);
 		KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
 								   size, size, &tmp, flags),
 							  "buddy_alloc unexpectedly succeeded, it should be full!");
@@ -456,7 +632,7 @@ static void drm_test_buddy_alloc_pessimistic(struct kunit *test)
 		list_del(&block->link);
 		drm_buddy_free_block(&mm, block);
 
-		size = get_size(order, PAGE_SIZE);
+		size = get_size(order, mm.chunk_size);
 		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
 								    size, size, &tmp, flags),
 							   "buddy_alloc hit -ENOMEM with order=%d\n",
@@ -471,7 +647,7 @@ static void drm_test_buddy_alloc_pessimistic(struct kunit *test)
 	}
 
 	/* To confirm, now the whole mm should be available */
-	size = get_size(max_order, PAGE_SIZE);
+	size = get_size(max_order, mm.chunk_size);
 	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
 							    size, size, &tmp, flags),
 						   "buddy_alloc (realloc) hit -ENOMEM with order=%d\n",
@@ -482,7 +658,7 @@ static void drm_test_buddy_alloc_pessimistic(struct kunit *test)
 
 	list_del(&block->link);
 	drm_buddy_free_block(&mm, block);
-	drm_buddy_free_list(&mm, &blocks);
+	drm_buddy_free_list(&mm, &blocks, 0);
 	drm_buddy_fini(&mm);
 }
 
@@ -502,15 +678,15 @@ static void drm_test_buddy_alloc_optimistic(struct kunit *test)
 	 * try to allocate them all.
 	 */
 
-	mm_size = PAGE_SIZE * ((1 << (max_order + 1)) - 1);
+	mm_size = SZ_4K * ((1 << (max_order + 1)) - 1);
 
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, PAGE_SIZE),
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K),
 			       "buddy_init failed\n");
 
 	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
 
 	for (order = 0; order <= max_order; order++) {
-		size = get_size(order, PAGE_SIZE);
+		size = get_size(order, mm.chunk_size);
 		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
 								    size, size, &tmp, flags),
 							   "buddy_alloc hit -ENOMEM with order=%d\n",
@@ -523,12 +699,12 @@ static void drm_test_buddy_alloc_optimistic(struct kunit *test)
 	}
 
 	/* Should be completely full! */
-	size = get_size(0, PAGE_SIZE);
+	size = get_size(0, mm.chunk_size);
 	KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
 							   size, size, &tmp, flags),
 						  "buddy_alloc unexpectedly succeeded, it should be full!");
 
-	drm_buddy_free_list(&mm, &blocks);
+	drm_buddy_free_list(&mm, &blocks, 0);
 	drm_buddy_fini(&mm);
 }
 
@@ -540,7 +716,7 @@ static void drm_test_buddy_alloc_limit(struct kunit *test)
 	LIST_HEAD(allocated);
 	struct drm_buddy mm;
 
-	KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, size, PAGE_SIZE));
+	KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, size, SZ_4K));
 
 	KUNIT_EXPECT_EQ_MSG(test, mm.max_order, DRM_BUDDY_MAX_ORDER,
 			    "mm.max_order(%d) != %d\n", mm.max_order,
@@ -548,7 +724,7 @@ static void drm_test_buddy_alloc_limit(struct kunit *test)
 
 	size = mm.chunk_size << mm.max_order;
 	KUNIT_EXPECT_FALSE(test, drm_buddy_alloc_blocks(&mm, start, size, size,
-							PAGE_SIZE, &allocated, flags));
+							mm.chunk_size, &allocated, flags));
 
 	block = list_first_entry_or_null(&allocated, struct drm_buddy_block, link);
 	KUNIT_EXPECT_TRUE(test, block);
@@ -558,12 +734,12 @@ static void drm_test_buddy_alloc_limit(struct kunit *test)
 						drm_buddy_block_order(block), mm.max_order);
 
 	KUNIT_EXPECT_EQ_MSG(test, drm_buddy_block_size(&mm, block),
-			    BIT_ULL(mm.max_order) * PAGE_SIZE,
+			    BIT_ULL(mm.max_order) * mm.chunk_size,
 						"block size(%llu) != %llu\n",
 						drm_buddy_block_size(&mm, block),
-						BIT_ULL(mm.max_order) * PAGE_SIZE);
+						BIT_ULL(mm.max_order) * mm.chunk_size);
 
-	drm_buddy_free_list(&mm, &allocated);
+	drm_buddy_free_list(&mm, &allocated, 0);
 	drm_buddy_fini(&mm);
 }
 
@@ -584,6 +760,7 @@ static struct kunit_case drm_buddy_tests[] = {
 	KUNIT_CASE(drm_test_buddy_alloc_pessimistic),
 	KUNIT_CASE(drm_test_buddy_alloc_pathological),
 	KUNIT_CASE(drm_test_buddy_alloc_contiguous),
+	KUNIT_CASE(drm_test_buddy_alloc_clear),
 	KUNIT_CASE(drm_test_buddy_alloc_range_bias),
 	{}
 };
diff --git a/drivers/gpu/drm/tidss/tidss_kms.c b/drivers/gpu/drm/tidss/tidss_kms.c
index a0e494c806a9..f371518f8697 100644
--- a/drivers/gpu/drm/tidss/tidss_kms.c
+++ b/drivers/gpu/drm/tidss/tidss_kms.c
@@ -135,8 +135,7 @@ static int tidss_dispc_modeset_init(struct tidss_device *tidss)
 			dev_dbg(dev, "no panel/bridge for port %d\n", i);
 			continue;
 		} else if (ret) {
-			dev_dbg(dev, "port %d probe returned %d\n", i, ret);
-			return ret;
+			return dev_err_probe(dev, ret, "port %d probe failed\n", i);
 		}
 
 		if (panel) {
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_panel.c b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
index 9aefd010acde..68093d6b6b16 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_panel.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
@@ -6,7 +6,6 @@
 
 #include <linux/backlight.h>
 #include <linux/gpio/consumer.h>
-#include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
 
 #include <video/display_timing.h>
@@ -308,7 +307,6 @@ static int panel_probe(struct platform_device *pdev)
 	struct backlight_device *backlight;
 	struct panel_module *panel_mod;
 	struct tilcdc_module *mod;
-	struct pinctrl *pinctrl;
 	int ret;
 
 	/* bail out early if no DT data: */
@@ -342,10 +340,6 @@ static int panel_probe(struct platform_device *pdev)
 
 	tilcdc_module_init(mod, "panel", &panel_module_ops);
 
-	pinctrl = devm_pinctrl_get_select_default(&pdev->dev);
-	if (IS_ERR(pinctrl))
-		dev_warn(&pdev->dev, "pins are not configured\n");
-
 	panel_mod->timings = of_get_display_timings(node);
 	if (!panel_mod->timings) {
 		dev_err(&pdev->dev, "could not get panel timings\n");
diff --git a/drivers/gpu/drm/tiny/simpledrm.c b/drivers/gpu/drm/tiny/simpledrm.c
index 7ce1c4617675..1d8fa07572c5 100644
--- a/drivers/gpu/drm/tiny/simpledrm.c
+++ b/drivers/gpu/drm/tiny/simpledrm.c
@@ -25,6 +25,7 @@
 #include <drm/drm_gem_shmem_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_panic.h>
 #include <drm/drm_probe_helper.h>
 
 #define DRIVER_NAME	"simpledrm"
@@ -671,11 +672,26 @@ static void simpledrm_primary_plane_helper_atomic_disable(struct drm_plane *plan
 	drm_dev_exit(idx);
 }
 
+static int simpledrm_primary_plane_helper_get_scanout_buffer(struct drm_plane *plane,
+							     struct drm_scanout_buffer *sb)
+{
+	struct simpledrm_device *sdev = simpledrm_device_of_dev(plane->dev);
+
+	sb->width = sdev->mode.hdisplay;
+	sb->height = sdev->mode.vdisplay;
+	sb->format = sdev->format;
+	sb->pitch[0] = sdev->pitch;
+	sb->map[0] = sdev->screen_base;
+
+	return 0;
+}
+
 static const struct drm_plane_helper_funcs simpledrm_primary_plane_helper_funcs = {
 	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
 	.atomic_check = simpledrm_primary_plane_helper_atomic_check,
 	.atomic_update = simpledrm_primary_plane_helper_atomic_update,
 	.atomic_disable = simpledrm_primary_plane_helper_atomic_disable,
+	.get_scanout_buffer = simpledrm_primary_plane_helper_get_scanout_buffer,
 };
 
 static const struct drm_plane_funcs simpledrm_primary_plane_funcs = {
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 96a724e8f3ff..6396dece0db1 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -402,7 +402,6 @@ void ttm_bo_put(struct ttm_buffer_object *bo)
 EXPORT_SYMBOL(ttm_bo_put);
 
 static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo,
-				     struct ttm_resource **mem,
 				     struct ttm_operation_ctx *ctx,
 				     struct ttm_place *hop)
 {
@@ -469,7 +468,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo,
 		if (ret != -EMULTIHOP)
 			break;
 
-		ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop);
+		ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop);
 	} while (!ret);
 
 	if (ret) {
@@ -698,7 +697,6 @@ EXPORT_SYMBOL(ttm_bo_unpin);
  */
 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 				 struct ttm_resource_manager *man,
-				 struct ttm_resource *mem,
 				 bool no_wait_gpu)
 {
 	struct dma_fence *fence;
@@ -724,64 +722,36 @@ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 	return ret;
 }
 
-/*
- * Repeatedly evict memory from the LRU for @mem_type until we create enough
- * space, or we've evicted everything and there isn't enough space.
- */
-static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
-				  const struct ttm_place *place,
-				  struct ttm_resource **mem,
-				  struct ttm_operation_ctx *ctx)
-{
-	struct ttm_device *bdev = bo->bdev;
-	struct ttm_resource_manager *man;
-	struct ww_acquire_ctx *ticket;
-	int ret;
-
-	man = ttm_manager_type(bdev, place->mem_type);
-	ticket = dma_resv_locking_ctx(bo->base.resv);
-	do {
-		ret = ttm_resource_alloc(bo, place, mem);
-		if (likely(!ret))
-			break;
-		if (unlikely(ret != -ENOSPC))
-			return ret;
-		ret = ttm_mem_evict_first(bdev, man, place, ctx,
-					  ticket);
-		if (unlikely(ret != 0))
-			return ret;
-	} while (1);
-
-	return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
-}
-
 /**
- * ttm_bo_mem_space
+ * ttm_bo_alloc_resource - Allocate backing store for a BO
  *
- * @bo: Pointer to a struct ttm_buffer_object. the data of which
- * we want to allocate space for.
- * @placement: Proposed new placement for the buffer object.
- * @mem: A struct ttm_resource.
+ * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for
+ * @placement: Proposed new placement for the buffer object
  * @ctx: if and how to sleep, lock buffers and alloc memory
+ * @force_space: If we should evict buffers to force space
+ * @res: The resulting struct ttm_resource.
  *
- * Allocate memory space for the buffer object pointed to by @bo, using
- * the placement flags in @placement, potentially evicting other idle buffer objects.
- * This function may sleep while waiting for space to become available.
+ * Allocates a resource for the buffer object pointed to by @bo, using the
+ * placement flags in @placement, potentially evicting other buffer objects when
+ * @force_space is true.
+ * This function may sleep while waiting for resources to become available.
  * Returns:
- * -EBUSY: No space available (only if no_wait == 1).
+ * -EBUSY: No space available (only if no_wait == true).
  * -ENOSPC: Could not allocate space for the buffer object, either due to
  * fragmentation or concurrent allocators.
  * -ERESTARTSYS: An interruptible sleep was interrupted by a signal.
  */
-int ttm_bo_mem_space(struct ttm_buffer_object *bo,
-			struct ttm_placement *placement,
-			struct ttm_resource **mem,
-			struct ttm_operation_ctx *ctx)
+static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo,
+				 struct ttm_placement *placement,
+				 struct ttm_operation_ctx *ctx,
+				 bool force_space,
+				 struct ttm_resource **res)
 {
 	struct ttm_device *bdev = bo->bdev;
-	bool type_found = false;
+	struct ww_acquire_ctx *ticket;
 	int i, ret;
 
+	ticket = dma_resv_locking_ctx(bo->base.resv);
 	ret = dma_resv_reserve_fences(bo->base.resv, 1);
 	if (unlikely(ret))
 		return ret;
@@ -790,98 +760,73 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 		const struct ttm_place *place = &placement->placement[i];
 		struct ttm_resource_manager *man;
 
-		if (place->flags & TTM_PL_FLAG_FALLBACK)
-			continue;
-
 		man = ttm_manager_type(bdev, place->mem_type);
 		if (!man || !ttm_resource_manager_used(man))
 			continue;
 
-		type_found = true;
-		ret = ttm_resource_alloc(bo, place, mem);
-		if (ret == -ENOSPC)
+		if (place->flags & (force_space ? TTM_PL_FLAG_DESIRED :
+				    TTM_PL_FLAG_FALLBACK))
+			continue;
+
+		do {
+			ret = ttm_resource_alloc(bo, place, res);
+			if (unlikely(ret && ret != -ENOSPC))
+				return ret;
+			if (likely(!ret) || !force_space)
+				break;
+
+			ret = ttm_mem_evict_first(bdev, man, place, ctx,
+						  ticket);
+			if (unlikely(ret == -EBUSY))
+				break;
+			if (unlikely(ret))
+				return ret;
+		} while (1);
+		if (ret)
 			continue;
-		if (unlikely(ret))
-			goto error;
 
-		ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
+		ret = ttm_bo_add_move_fence(bo, man, ctx->no_wait_gpu);
 		if (unlikely(ret)) {
-			ttm_resource_free(bo, mem);
+			ttm_resource_free(bo, res);
 			if (ret == -EBUSY)
 				continue;
 
-			goto error;
+			return ret;
 		}
 		return 0;
 	}
 
-	for (i = 0; i < placement->num_placement; ++i) {
-		const struct ttm_place *place = &placement->placement[i];
-		struct ttm_resource_manager *man;
-
-		if (place->flags & TTM_PL_FLAG_DESIRED)
-			continue;
-
-		man = ttm_manager_type(bdev, place->mem_type);
-		if (!man || !ttm_resource_manager_used(man))
-			continue;
-
-		type_found = true;
-		ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
-		if (likely(!ret))
-			return 0;
-
-		if (ret && ret != -EBUSY)
-			goto error;
-	}
-
-	ret = -ENOSPC;
-	if (!type_found) {
-		pr_err(TTM_PFX "No compatible memory type found\n");
-		ret = -EINVAL;
-	}
-
-error:
-	return ret;
+	return -ENOSPC;
 }
-EXPORT_SYMBOL(ttm_bo_mem_space);
 
-static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
-			      struct ttm_placement *placement,
-			      struct ttm_operation_ctx *ctx)
+/*
+ * ttm_bo_mem_space - Wrapper around ttm_bo_alloc_resource
+ *
+ * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for
+ * @placement: Proposed new placement for the buffer object
+ * @res: The resulting struct ttm_resource.
+ * @ctx: if and how to sleep, lock buffers and alloc memory
+ *
+ * Tries both idle allocation and forcefully eviction of buffers. See
+ * ttm_bo_alloc_resource for details.
+ */
+int ttm_bo_mem_space(struct ttm_buffer_object *bo,
+		     struct ttm_placement *placement,
+		     struct ttm_resource **res,
+		     struct ttm_operation_ctx *ctx)
 {
-	struct ttm_resource *mem;
-	struct ttm_place hop;
+	bool force_space = false;
 	int ret;
 
-	dma_resv_assert_held(bo->base.resv);
+	do {
+		ret = ttm_bo_alloc_resource(bo, placement, ctx,
+					    force_space, res);
+		force_space = !force_space;
+	} while (ret == -ENOSPC && force_space);
 
-	/*
-	 * Determine where to move the buffer.
-	 *
-	 * If driver determines move is going to need
-	 * an extra step then it will return -EMULTIHOP
-	 * and the buffer will be moved to the temporary
-	 * stop and the driver will be called to make
-	 * the second hop.
-	 */
-	ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
-	if (ret)
-		return ret;
-bounce:
-	ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop);
-	if (ret == -EMULTIHOP) {
-		ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop);
-		if (ret)
-			goto out;
-		/* try and move to final place now. */
-		goto bounce;
-	}
-out:
-	if (ret)
-		ttm_resource_free(bo, &mem);
 	return ret;
 }
+EXPORT_SYMBOL(ttm_bo_mem_space);
 
 /**
  * ttm_bo_validate
@@ -902,6 +847,9 @@ int ttm_bo_validate(struct ttm_buffer_object *bo,
 		    struct ttm_placement *placement,
 		    struct ttm_operation_ctx *ctx)
 {
+	struct ttm_resource *res;
+	struct ttm_place hop;
+	bool force_space;
 	int ret;
 
 	dma_resv_assert_held(bo->base.resv);
@@ -912,20 +860,53 @@ int ttm_bo_validate(struct ttm_buffer_object *bo,
 	if (!placement->num_placement)
 		return ttm_bo_pipeline_gutting(bo);
 
-	/* Check whether we need to move buffer. */
-	if (bo->resource && ttm_resource_compatible(bo->resource, placement))
-		return 0;
+	force_space = false;
+	do {
+		/* Check whether we need to move buffer. */
+		if (bo->resource &&
+		    ttm_resource_compatible(bo->resource, placement,
+					    force_space))
+			return 0;
 
-	/* Moving of pinned BOs is forbidden */
-	if (bo->pin_count)
-		return -EINVAL;
+		/* Moving of pinned BOs is forbidden */
+		if (bo->pin_count)
+			return -EINVAL;
+
+		/*
+		 * Determine where to move the buffer.
+		 *
+		 * If driver determines move is going to need
+		 * an extra step then it will return -EMULTIHOP
+		 * and the buffer will be moved to the temporary
+		 * stop and the driver will be called to make
+		 * the second hop.
+		 */
+		ret = ttm_bo_alloc_resource(bo, placement, ctx, force_space,
+					    &res);
+		force_space = !force_space;
+		if (ret == -ENOSPC)
+			continue;
+		if (ret)
+			return ret;
+
+bounce:
+		ret = ttm_bo_handle_move_mem(bo, res, false, ctx, &hop);
+		if (ret == -EMULTIHOP) {
+			ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop);
+			/* try and move to final place now. */
+			if (!ret)
+				goto bounce;
+		}
+		if (ret) {
+			ttm_resource_free(bo, &res);
+			return ret;
+		}
+
+	} while (ret && force_space);
 
-	ret = ttm_bo_move_buffer(bo, placement, ctx);
 	/* For backward compatibility with userspace */
 	if (ret == -ENOSPC)
 		return -ENOMEM;
-	if (ret)
-		return ret;
 
 	/*
 	 * We might need to add a TTM.
diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
index 76027960054f..434cf0258000 100644
--- a/drivers/gpu/drm/ttm/ttm_device.c
+++ b/drivers/gpu/drm/ttm/ttm_device.c
@@ -27,6 +27,7 @@
 
 #define pr_fmt(fmt) "[TTM DEVICE] " fmt
 
+#include <linux/debugfs.h>
 #include <linux/mm.h>
 
 #include <drm/ttm/ttm_bo.h>
diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
index 112438d965ff..6e1fd6985ffc 100644
--- a/drivers/gpu/drm/ttm/ttm_pool.c
+++ b/drivers/gpu/drm/ttm/ttm_pool.c
@@ -288,17 +288,23 @@ static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool,
 						  enum ttm_caching caching,
 						  unsigned int order)
 {
-	if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE)
+	if (pool->use_dma_alloc)
 		return &pool->caching[caching].orders[order];
 
 #ifdef CONFIG_X86
 	switch (caching) {
 	case ttm_write_combined:
+		if (pool->nid != NUMA_NO_NODE)
+			return &pool->caching[caching].orders[order];
+
 		if (pool->use_dma32)
 			return &global_dma32_write_combined[order];
 
 		return &global_write_combined[order];
 	case ttm_uncached:
+		if (pool->nid != NUMA_NO_NODE)
+			return &pool->caching[caching].orders[order];
+
 		if (pool->use_dma32)
 			return &global_dma32_uncached[order];
 
@@ -566,11 +572,17 @@ void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
 	pool->use_dma_alloc = use_dma_alloc;
 	pool->use_dma32 = use_dma32;
 
-	if (use_dma_alloc || nid != NUMA_NO_NODE) {
-		for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
-			for (j = 0; j < NR_PAGE_ORDERS; ++j)
-				ttm_pool_type_init(&pool->caching[i].orders[j],
-						   pool, i, j);
+	for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) {
+		for (j = 0; j < NR_PAGE_ORDERS; ++j) {
+			struct ttm_pool_type *pt;
+
+			/* Initialize only pool types which are actually used */
+			pt = ttm_pool_select_type(pool, i, j);
+			if (pt != &pool->caching[i].orders[j])
+				continue;
+
+			ttm_pool_type_init(pt, pool, i, j);
+		}
 	}
 }
 EXPORT_SYMBOL(ttm_pool_init);
@@ -599,10 +611,16 @@ void ttm_pool_fini(struct ttm_pool *pool)
 {
 	unsigned int i, j;
 
-	if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE) {
-		for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
-			for (j = 0; j < NR_PAGE_ORDERS; ++j)
-				ttm_pool_type_fini(&pool->caching[i].orders[j]);
+	for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) {
+		for (j = 0; j < NR_PAGE_ORDERS; ++j) {
+			struct ttm_pool_type *pt;
+
+			pt = ttm_pool_select_type(pool, i, j);
+			if (pt != &pool->caching[i].orders[j])
+				continue;
+
+			ttm_pool_type_fini(pt);
+		}
 	}
 
 	/* We removed the pool types from the LRU, but we need to also make sure
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index fb14f7716cf8..4a66b851b67d 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -22,8 +22,9 @@
  * Authors: Christian König
  */
 
-#include <linux/iosys-map.h>
+#include <linux/debugfs.h>
 #include <linux/io-mapping.h>
+#include <linux/iosys-map.h>
 #include <linux/scatterlist.h>
 
 #include <drm/ttm/ttm_bo.h>
@@ -105,6 +106,7 @@ static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
 		pos->first = res;
 		pos->last = res;
 	} else {
+		WARN_ON(pos->first->bo->base.resv != res->bo->base.resv);
 		ttm_lru_bulk_move_pos_tail(pos, res);
 	}
 }
@@ -295,11 +297,13 @@ bool ttm_resource_intersects(struct ttm_device *bdev,
  *
  * @res: the resource to check
  * @placement: the placement to check against
+ * @evicting: true if the caller is doing evictions
  *
  * Returns true if the placement is compatible.
  */
 bool ttm_resource_compatible(struct ttm_resource *res,
-			     struct ttm_placement *placement)
+			     struct ttm_placement *placement,
+			     bool evicting)
 {
 	struct ttm_buffer_object *bo = res->bo;
 	struct ttm_device *bdev = bo->bdev;
@@ -315,14 +319,20 @@ bool ttm_resource_compatible(struct ttm_resource *res,
 		if (res->mem_type != place->mem_type)
 			continue;
 
+		if (place->flags & (evicting ? TTM_PL_FLAG_DESIRED :
+				    TTM_PL_FLAG_FALLBACK))
+			continue;
+
+		if (place->flags & TTM_PL_FLAG_CONTIGUOUS &&
+		    !(res->placement & TTM_PL_FLAG_CONTIGUOUS))
+			continue;
+
 		man = ttm_manager_type(bdev, res->mem_type);
 		if (man->func->compatible &&
 		    !man->func->compatible(man, res, place, bo->base.size))
 			continue;
 
-		if ((!(place->flags & TTM_PL_FLAG_CONTIGUOUS) ||
-		     (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
-			return true;
+		return true;
 	}
 	return false;
 }
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index 578a7c37f00b..7b00ddf0ce49 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -32,10 +32,11 @@
 #define pr_fmt(fmt) "[TTM] " fmt
 
 #include <linux/cc_platform.h>
-#include <linux/sched.h>
-#include <linux/shmem_fs.h>
+#include <linux/debugfs.h>
 #include <linux/file.h>
 #include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/shmem_fs.h>
 #include <drm/drm_cache.h>
 #include <drm/drm_device.h>
 #include <drm/drm_util.h>
@@ -92,7 +93,7 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc)
 	 */
 	if (bdev->pool.use_dma_alloc && cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
 		page_flags |= TTM_TT_FLAG_DECRYPTED;
-		drm_info(ddev, "TT memory decryption enabled.");
+		drm_info_once(ddev, "TT memory decryption enabled.");
 	}
 
 	bo->ttm = bdev->funcs->ttm_tt_create(bo, page_flags);
diff --git a/drivers/gpu/drm/v3d/v3d_bo.c b/drivers/gpu/drm/v3d/v3d_bo.c
index a07ede668cc1..a165cbcdd27b 100644
--- a/drivers/gpu/drm/v3d/v3d_bo.c
+++ b/drivers/gpu/drm/v3d/v3d_bo.c
@@ -21,6 +21,7 @@
 
 #include <linux/dma-buf.h>
 #include <linux/pfn_t.h>
+#include <linux/vmalloc.h>
 
 #include "v3d_drv.h"
 #include "uapi/drm/v3d_drm.h"
diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
index 3debf37e7d9b..28b7ddce7747 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.c
+++ b/drivers/gpu/drm/v3d/v3d_drv.c
@@ -115,14 +115,13 @@ v3d_open(struct drm_device *dev, struct drm_file *file)
 	v3d_priv->v3d = v3d;
 
 	for (i = 0; i < V3D_MAX_QUEUES; i++) {
-		v3d_priv->enabled_ns[i] = 0;
-		v3d_priv->start_ns[i] = 0;
-		v3d_priv->jobs_sent[i] = 0;
-
 		sched = &v3d->queue[i].sched;
 		drm_sched_entity_init(&v3d_priv->sched_entity[i],
 				      DRM_SCHED_PRIORITY_NORMAL, &sched,
 				      1, NULL);
+
+		memset(&v3d_priv->stats[i], 0, sizeof(v3d_priv->stats[i]));
+		seqcount_init(&v3d_priv->stats[i].lock);
 	}
 
 	v3d_perfmon_open_file(v3d_priv);
@@ -144,6 +143,20 @@ v3d_postclose(struct drm_device *dev, struct drm_file *file)
 	kfree(v3d_priv);
 }
 
+void v3d_get_stats(const struct v3d_stats *stats, u64 timestamp,
+		   u64 *active_runtime, u64 *jobs_completed)
+{
+	unsigned int seq;
+
+	do {
+		seq = read_seqcount_begin(&stats->lock);
+		*active_runtime = stats->enabled_ns;
+		if (stats->start_ns)
+			*active_runtime += timestamp - stats->start_ns;
+		*jobs_completed = stats->jobs_completed;
+	} while (read_seqcount_retry(&stats->lock, seq));
+}
+
 static void v3d_show_fdinfo(struct drm_printer *p, struct drm_file *file)
 {
 	struct v3d_file_priv *file_priv = file->driver_priv;
@@ -151,20 +164,22 @@ static void v3d_show_fdinfo(struct drm_printer *p, struct drm_file *file)
 	enum v3d_queue queue;
 
 	for (queue = 0; queue < V3D_MAX_QUEUES; queue++) {
+		struct v3d_stats *stats = &file_priv->stats[queue];
+		u64 active_runtime, jobs_completed;
+
+		v3d_get_stats(stats, timestamp, &active_runtime, &jobs_completed);
+
 		/* Note that, in case of a GPU reset, the time spent during an
 		 * attempt of executing the job is not computed in the runtime.
 		 */
 		drm_printf(p, "drm-engine-%s: \t%llu ns\n",
-			   v3d_queue_to_string(queue),
-			   file_priv->start_ns[queue] ? file_priv->enabled_ns[queue]
-						      + timestamp - file_priv->start_ns[queue]
-						      : file_priv->enabled_ns[queue]);
+			   v3d_queue_to_string(queue), active_runtime);
 
 		/* Note that we only count jobs that completed. Therefore, jobs
 		 * that were resubmitted due to a GPU reset are not computed.
 		 */
 		drm_printf(p, "v3d-jobs-%s: \t%llu jobs\n",
-			   v3d_queue_to_string(queue), file_priv->jobs_sent[queue]);
+			   v3d_queue_to_string(queue), jobs_completed);
 	}
 }
 
diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
index 1950c723dde1..a2c516fe6d79 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.h
+++ b/drivers/gpu/drm/v3d/v3d_drv.h
@@ -36,15 +36,27 @@ static inline char *v3d_queue_to_string(enum v3d_queue queue)
 	return "UNKNOWN";
 }
 
+struct v3d_stats {
+	u64 start_ns;
+	u64 enabled_ns;
+	u64 jobs_completed;
+
+	/*
+	 * This seqcount is used to protect the access to the GPU stats
+	 * variables. It must be used as, while we are reading the stats,
+	 * IRQs can happen and the stats can be updated.
+	 */
+	seqcount_t lock;
+};
+
 struct v3d_queue_state {
 	struct drm_gpu_scheduler sched;
 
 	u64 fence_context;
 	u64 emit_seqno;
 
-	u64 start_ns;
-	u64 enabled_ns;
-	u64 jobs_sent;
+	/* Stores the GPU stats for this queue in the global context. */
+	struct v3d_stats stats;
 };
 
 /* Performance monitor object. The perform lifetime is controlled by userspace
@@ -188,11 +200,8 @@ struct v3d_file_priv {
 
 	struct drm_sched_entity sched_entity[V3D_MAX_QUEUES];
 
-	u64 start_ns[V3D_MAX_QUEUES];
-
-	u64 enabled_ns[V3D_MAX_QUEUES];
-
-	u64 jobs_sent[V3D_MAX_QUEUES];
+	/* Stores the GPU stats for a specific queue for this fd. */
+	struct v3d_stats stats[V3D_MAX_QUEUES];
 };
 
 struct v3d_bo {
@@ -508,6 +517,10 @@ struct drm_gem_object *v3d_prime_import_sg_table(struct drm_device *dev,
 /* v3d_debugfs.c */
 void v3d_debugfs_init(struct drm_minor *minor);
 
+/* v3d_drv.c */
+void v3d_get_stats(const struct v3d_stats *stats, u64 timestamp,
+		   u64 *active_runtime, u64 *jobs_completed);
+
 /* v3d_fence.c */
 extern const struct dma_fence_ops v3d_fence_ops;
 struct dma_fence *v3d_fence_create(struct v3d_dev *v3d, enum v3d_queue queue);
@@ -543,6 +556,7 @@ void v3d_mmu_insert_ptes(struct v3d_bo *bo);
 void v3d_mmu_remove_ptes(struct v3d_bo *bo);
 
 /* v3d_sched.c */
+void v3d_job_update_stats(struct v3d_job *job, enum v3d_queue queue);
 int v3d_sched_init(struct v3d_dev *v3d);
 void v3d_sched_fini(struct v3d_dev *v3d);
 
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index afc565078c78..da8faf3b9011 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -247,10 +247,11 @@ v3d_gem_init(struct drm_device *dev)
 	int ret, i;
 
 	for (i = 0; i < V3D_MAX_QUEUES; i++) {
-		v3d->queue[i].fence_context = dma_fence_context_alloc(1);
-		v3d->queue[i].start_ns = 0;
-		v3d->queue[i].enabled_ns = 0;
-		v3d->queue[i].jobs_sent = 0;
+		struct v3d_queue_state *queue = &v3d->queue[i];
+
+		queue->fence_context = dma_fence_context_alloc(1);
+		memset(&queue->stats, 0, sizeof(queue->stats));
+		seqcount_init(&queue->stats.lock);
 	}
 
 	spin_lock_init(&v3d->mm_lock);
diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c
index 2e04f6cb661e..d469bda52c1a 100644
--- a/drivers/gpu/drm/v3d/v3d_irq.c
+++ b/drivers/gpu/drm/v3d/v3d_irq.c
@@ -102,19 +102,8 @@ v3d_irq(int irq, void *arg)
 	if (intsts & V3D_INT_FLDONE) {
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->bin_job->base.irq_fence);
-		struct v3d_file_priv *file = v3d->bin_job->base.file->driver_priv;
-		u64 runtime = local_clock() - file->start_ns[V3D_BIN];
-
-		file->enabled_ns[V3D_BIN] += local_clock() - file->start_ns[V3D_BIN];
-		file->jobs_sent[V3D_BIN]++;
-		v3d->queue[V3D_BIN].jobs_sent++;
-
-		file->start_ns[V3D_BIN] = 0;
-		v3d->queue[V3D_BIN].start_ns = 0;
-
-		file->enabled_ns[V3D_BIN] += runtime;
-		v3d->queue[V3D_BIN].enabled_ns += runtime;
 
+		v3d_job_update_stats(&v3d->bin_job->base, V3D_BIN);
 		trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
 		status = IRQ_HANDLED;
@@ -123,19 +112,8 @@ v3d_irq(int irq, void *arg)
 	if (intsts & V3D_INT_FRDONE) {
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->render_job->base.irq_fence);
-		struct v3d_file_priv *file = v3d->render_job->base.file->driver_priv;
-		u64 runtime = local_clock() - file->start_ns[V3D_RENDER];
-
-		file->enabled_ns[V3D_RENDER] += local_clock() - file->start_ns[V3D_RENDER];
-		file->jobs_sent[V3D_RENDER]++;
-		v3d->queue[V3D_RENDER].jobs_sent++;
-
-		file->start_ns[V3D_RENDER] = 0;
-		v3d->queue[V3D_RENDER].start_ns = 0;
-
-		file->enabled_ns[V3D_RENDER] += runtime;
-		v3d->queue[V3D_RENDER].enabled_ns += runtime;
 
+		v3d_job_update_stats(&v3d->render_job->base, V3D_RENDER);
 		trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
 		status = IRQ_HANDLED;
@@ -144,19 +122,8 @@ v3d_irq(int irq, void *arg)
 	if (intsts & V3D_INT_CSDDONE(v3d->ver)) {
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->csd_job->base.irq_fence);
-		struct v3d_file_priv *file = v3d->csd_job->base.file->driver_priv;
-		u64 runtime = local_clock() - file->start_ns[V3D_CSD];
-
-		file->enabled_ns[V3D_CSD] += local_clock() - file->start_ns[V3D_CSD];
-		file->jobs_sent[V3D_CSD]++;
-		v3d->queue[V3D_CSD].jobs_sent++;
-
-		file->start_ns[V3D_CSD] = 0;
-		v3d->queue[V3D_CSD].start_ns = 0;
-
-		file->enabled_ns[V3D_CSD] += runtime;
-		v3d->queue[V3D_CSD].enabled_ns += runtime;
 
+		v3d_job_update_stats(&v3d->csd_job->base, V3D_CSD);
 		trace_v3d_csd_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
 		status = IRQ_HANDLED;
@@ -192,19 +159,8 @@ v3d_hub_irq(int irq, void *arg)
 	if (intsts & V3D_HUB_INT_TFUC) {
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->tfu_job->base.irq_fence);
-		struct v3d_file_priv *file = v3d->tfu_job->base.file->driver_priv;
-		u64 runtime = local_clock() - file->start_ns[V3D_TFU];
-
-		file->enabled_ns[V3D_TFU] += local_clock() - file->start_ns[V3D_TFU];
-		file->jobs_sent[V3D_TFU]++;
-		v3d->queue[V3D_TFU].jobs_sent++;
-
-		file->start_ns[V3D_TFU] = 0;
-		v3d->queue[V3D_TFU].start_ns = 0;
-
-		file->enabled_ns[V3D_TFU] += runtime;
-		v3d->queue[V3D_TFU].enabled_ns += runtime;
 
+		v3d_job_update_stats(&v3d->tfu_job->base, V3D_TFU);
 		trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
 		status = IRQ_HANDLED;
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index 54015ad765c7..7cd8c335cd9b 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -105,11 +105,51 @@ v3d_switch_perfmon(struct v3d_dev *v3d, struct v3d_job *job)
 		v3d_perfmon_start(v3d, job->perfmon);
 }
 
+static void
+v3d_job_start_stats(struct v3d_job *job, enum v3d_queue queue)
+{
+	struct v3d_dev *v3d = job->v3d;
+	struct v3d_file_priv *file = job->file->driver_priv;
+	struct v3d_stats *global_stats = &v3d->queue[queue].stats;
+	struct v3d_stats *local_stats = &file->stats[queue];
+	u64 now = local_clock();
+
+	write_seqcount_begin(&local_stats->lock);
+	local_stats->start_ns = now;
+	write_seqcount_end(&local_stats->lock);
+
+	write_seqcount_begin(&global_stats->lock);
+	global_stats->start_ns = now;
+	write_seqcount_end(&global_stats->lock);
+}
+
+static void
+v3d_stats_update(struct v3d_stats *stats, u64 now)
+{
+	write_seqcount_begin(&stats->lock);
+	stats->enabled_ns += now - stats->start_ns;
+	stats->jobs_completed++;
+	stats->start_ns = 0;
+	write_seqcount_end(&stats->lock);
+}
+
+void
+v3d_job_update_stats(struct v3d_job *job, enum v3d_queue queue)
+{
+	struct v3d_dev *v3d = job->v3d;
+	struct v3d_file_priv *file = job->file->driver_priv;
+	struct v3d_stats *global_stats = &v3d->queue[queue].stats;
+	struct v3d_stats *local_stats = &file->stats[queue];
+	u64 now = local_clock();
+
+	v3d_stats_update(local_stats, now);
+	v3d_stats_update(global_stats, now);
+}
+
 static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_bin_job *job = to_bin_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
-	struct v3d_file_priv *file = job->base.file->driver_priv;
 	struct drm_device *dev = &v3d->drm;
 	struct dma_fence *fence;
 	unsigned long irqflags;
@@ -141,9 +181,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 	trace_v3d_submit_cl(dev, false, to_v3d_fence(fence)->seqno,
 			    job->start, job->end);
 
-	file->start_ns[V3D_BIN] = local_clock();
-	v3d->queue[V3D_BIN].start_ns = file->start_ns[V3D_BIN];
-
+	v3d_job_start_stats(&job->base, V3D_BIN);
 	v3d_switch_perfmon(v3d, &job->base);
 
 	/* Set the current and end address of the control list.
@@ -168,7 +206,6 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_render_job *job = to_render_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
-	struct v3d_file_priv *file = job->base.file->driver_priv;
 	struct drm_device *dev = &v3d->drm;
 	struct dma_fence *fence;
 
@@ -196,9 +233,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 	trace_v3d_submit_cl(dev, true, to_v3d_fence(fence)->seqno,
 			    job->start, job->end);
 
-	file->start_ns[V3D_RENDER] = local_clock();
-	v3d->queue[V3D_RENDER].start_ns = file->start_ns[V3D_RENDER];
-
+	v3d_job_start_stats(&job->base, V3D_RENDER);
 	v3d_switch_perfmon(v3d, &job->base);
 
 	/* XXX: Set the QCFG */
@@ -217,7 +252,6 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_tfu_job *job = to_tfu_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
-	struct v3d_file_priv *file = job->base.file->driver_priv;
 	struct drm_device *dev = &v3d->drm;
 	struct dma_fence *fence;
 
@@ -232,8 +266,7 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
 
 	trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
 
-	file->start_ns[V3D_TFU] = local_clock();
-	v3d->queue[V3D_TFU].start_ns = file->start_ns[V3D_TFU];
+	v3d_job_start_stats(&job->base, V3D_TFU);
 
 	V3D_WRITE(V3D_TFU_IIA(v3d->ver), job->args.iia);
 	V3D_WRITE(V3D_TFU_IIS(v3d->ver), job->args.iis);
@@ -260,7 +293,6 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_csd_job *job = to_csd_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
-	struct v3d_file_priv *file = job->base.file->driver_priv;
 	struct drm_device *dev = &v3d->drm;
 	struct dma_fence *fence;
 	int i, csd_cfg0_reg, csd_cfg_reg_count;
@@ -279,9 +311,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
 
 	trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);
 
-	file->start_ns[V3D_CSD] = local_clock();
-	v3d->queue[V3D_CSD].start_ns = file->start_ns[V3D_CSD];
-
+	v3d_job_start_stats(&job->base, V3D_CSD);
 	v3d_switch_perfmon(v3d, &job->base);
 
 	csd_cfg0_reg = V3D_CSD_QUEUED_CFG0(v3d->ver);
@@ -530,8 +560,6 @@ v3d_cpu_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_cpu_job *job = to_cpu_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
-	struct v3d_file_priv *file = job->base.file->driver_priv;
-	u64 runtime;
 
 	v3d->cpu_job = job;
 
@@ -540,25 +568,13 @@ v3d_cpu_job_run(struct drm_sched_job *sched_job)
 		return NULL;
 	}
 
-	file->start_ns[V3D_CPU] = local_clock();
-	v3d->queue[V3D_CPU].start_ns = file->start_ns[V3D_CPU];
-
+	v3d_job_start_stats(&job->base, V3D_CPU);
 	trace_v3d_cpu_job_begin(&v3d->drm, job->job_type);
 
 	cpu_job_function[job->job_type](job);
 
 	trace_v3d_cpu_job_end(&v3d->drm, job->job_type);
-
-	runtime = local_clock() - file->start_ns[V3D_CPU];
-
-	file->enabled_ns[V3D_CPU] += runtime;
-	v3d->queue[V3D_CPU].enabled_ns += runtime;
-
-	file->jobs_sent[V3D_CPU]++;
-	v3d->queue[V3D_CPU].jobs_sent++;
-
-	file->start_ns[V3D_CPU] = 0;
-	v3d->queue[V3D_CPU].start_ns = 0;
+	v3d_job_update_stats(&job->base, V3D_CPU);
 
 	return NULL;
 }
@@ -568,24 +584,12 @@ v3d_cache_clean_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_job *job = to_v3d_job(sched_job);
 	struct v3d_dev *v3d = job->v3d;
-	struct v3d_file_priv *file = job->file->driver_priv;
-	u64 runtime;
 
-	file->start_ns[V3D_CACHE_CLEAN] = local_clock();
-	v3d->queue[V3D_CACHE_CLEAN].start_ns = file->start_ns[V3D_CACHE_CLEAN];
+	v3d_job_start_stats(job, V3D_CACHE_CLEAN);
 
 	v3d_clean_caches(v3d);
 
-	runtime = local_clock() - file->start_ns[V3D_CACHE_CLEAN];
-
-	file->enabled_ns[V3D_CACHE_CLEAN] += runtime;
-	v3d->queue[V3D_CACHE_CLEAN].enabled_ns += runtime;
-
-	file->jobs_sent[V3D_CACHE_CLEAN]++;
-	v3d->queue[V3D_CACHE_CLEAN].jobs_sent++;
-
-	file->start_ns[V3D_CACHE_CLEAN] = 0;
-	v3d->queue[V3D_CACHE_CLEAN].start_ns = 0;
+	v3d_job_update_stats(job, V3D_CACHE_CLEAN);
 
 	return NULL;
 }
diff --git a/drivers/gpu/drm/v3d/v3d_sysfs.c b/drivers/gpu/drm/v3d/v3d_sysfs.c
index d106845ba890..d610e355964f 100644
--- a/drivers/gpu/drm/v3d/v3d_sysfs.c
+++ b/drivers/gpu/drm/v3d/v3d_sysfs.c
@@ -15,16 +15,15 @@ gpu_stats_show(struct device *dev, struct device_attribute *attr, char *buf)
 	struct v3d_dev *v3d = to_v3d_dev(drm);
 	enum v3d_queue queue;
 	u64 timestamp = local_clock();
-	u64 active_runtime;
 	ssize_t len = 0;
 
 	len += sysfs_emit(buf, "queue\ttimestamp\tjobs\truntime\n");
 
 	for (queue = 0; queue < V3D_MAX_QUEUES; queue++) {
-		if (v3d->queue[queue].start_ns)
-			active_runtime = timestamp - v3d->queue[queue].start_ns;
-		else
-			active_runtime = 0;
+		struct v3d_stats *stats = &v3d->queue[queue].stats;
+		u64 active_runtime, jobs_completed;
+
+		v3d_get_stats(stats, timestamp, &active_runtime, &jobs_completed);
 
 		/* Each line will display the queue name, timestamp, the number
 		 * of jobs sent to that queue and the runtime, as can be seem here:
@@ -38,9 +37,7 @@ gpu_stats_show(struct device *dev, struct device_attribute *attr, char *buf)
 		 */
 		len += sysfs_emit_at(buf, len, "%s\t%llu\t%llu\t%llu\n",
 				     v3d_queue_to_string(queue),
-				     timestamp,
-				     v3d->queue[queue].jobs_sent,
-				     v3d->queue[queue].enabled_ns + active_runtime);
+				     timestamp, jobs_completed, active_runtime);
 	}
 
 	return len;
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index ab61e96e7e14..08e29fa82563 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -5,6 +5,7 @@
 #ifndef _VC4_DRV_H_
 #define _VC4_DRV_H_
 
+#include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/of.h>
 #include <linux/refcount.h>
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index d8751ea20303..d30f8e8e8967 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -412,15 +412,14 @@ static void vc4_hdmi_handle_hotplug(struct vc4_hdmi *vc4_hdmi,
 				    enum drm_connector_status status)
 {
 	struct drm_connector *connector = &vc4_hdmi->connector;
-	struct edid *edid;
+	const struct drm_edid *drm_edid;
 	int ret;
 
 	/*
-	 * NOTE: This function should really be called with
-	 * vc4_hdmi->mutex held, but doing so results in reentrancy
-	 * issues since cec_s_phys_addr_from_edid might call
-	 * .adap_enable, which leads to that funtion being called with
-	 * our mutex held.
+	 * NOTE: This function should really be called with vc4_hdmi->mutex
+	 * held, but doing so results in reentrancy issues since
+	 * cec_s_phys_addr() might call .adap_enable, which leads to that
+	 * funtion being called with our mutex held.
 	 *
 	 * A similar situation occurs with vc4_hdmi_reset_link() that
 	 * will call into our KMS hooks if the scrambling was enabled.
@@ -435,12 +434,16 @@ static void vc4_hdmi_handle_hotplug(struct vc4_hdmi *vc4_hdmi,
 		return;
 	}
 
-	edid = drm_get_edid(connector, vc4_hdmi->ddc);
-	if (!edid)
+	drm_edid = drm_edid_read_ddc(connector, vc4_hdmi->ddc);
+
+	drm_edid_connector_update(connector, drm_edid);
+	cec_s_phys_addr(vc4_hdmi->cec_adap,
+			connector->display_info.source_physical_address, false);
+
+	if (!drm_edid)
 		return;
 
-	cec_s_phys_addr_from_edid(vc4_hdmi->cec_adap, edid);
-	kfree(edid);
+	drm_edid_free(drm_edid);
 
 	for (;;) {
 		ret = vc4_hdmi_reset_link(connector, ctx);
@@ -492,28 +495,29 @@ static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
 {
 	struct vc4_hdmi *vc4_hdmi = connector_to_vc4_hdmi(connector);
 	struct vc4_dev *vc4 = to_vc4_dev(connector->dev);
+	const struct drm_edid *drm_edid;
 	int ret = 0;
-	struct edid *edid;
 
 	/*
-	 * NOTE: This function should really take vc4_hdmi->mutex, but
-	 * doing so results in reentrancy issues since
-	 * cec_s_phys_addr_from_edid might call .adap_enable, which
-	 * leads to that funtion being called with our mutex held.
+	 * NOTE: This function should really take vc4_hdmi->mutex, but doing so
+	 * results in reentrancy issues since cec_s_phys_addr() might call
+	 * .adap_enable, which leads to that funtion being called with our mutex
+	 * held.
 	 *
 	 * Concurrency isn't an issue at the moment since we don't share
 	 * any state with any of the other frameworks so we can ignore
 	 * the lock for now.
 	 */
 
-	edid = drm_get_edid(connector, vc4_hdmi->ddc);
-	cec_s_phys_addr_from_edid(vc4_hdmi->cec_adap, edid);
-	if (!edid)
+	drm_edid = drm_edid_read_ddc(connector, vc4_hdmi->ddc);
+	drm_edid_connector_update(connector, drm_edid);
+	cec_s_phys_addr(vc4_hdmi->cec_adap,
+			connector->display_info.source_physical_address, false);
+	if (!drm_edid)
 		return 0;
 
-	drm_connector_update_edid_property(connector, edid);
-	ret = drm_add_edid_modes(connector, edid);
-	kfree(edid);
+	ret = drm_edid_connector_add_modes(connector);
+	drm_edid_free(drm_edid);
 
 	if (!vc4->hvs->vc5_hdmi_enable_hdmi_20) {
 		struct drm_device *drm = connector->dev;
@@ -2740,6 +2744,8 @@ static int vc4_hdmi_audio_init(struct vc4_hdmi *vc4_hdmi)
 		index = 1;
 
 	addr = of_get_address(dev->of_node, index, NULL, NULL);
+	if (!addr)
+		return -EINVAL;
 
 	vc4_hdmi->audio.dma_data.addr = be32_to_cpup(addr) + mai_data->offset;
 	vc4_hdmi->audio.dma_data.addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c
index 9539aa28937f..188e126383c2 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.c
@@ -154,7 +154,6 @@ static struct virtio_driver virtio_gpu_driver = {
 	.feature_table = features,
 	.feature_table_size = ARRAY_SIZE(features),
 	.driver.name = KBUILD_MODNAME,
-	.driver.owner = THIS_MODULE,
 	.id_table = id_table,
 	.probe = virtio_gpu_probe,
 	.remove = virtio_gpu_remove,
diff --git a/drivers/gpu/drm/virtio/virtgpu_trace.h b/drivers/gpu/drm/virtio/virtgpu_trace.h
index 031bc77689d5..227bf0ae7ed5 100644
--- a/drivers/gpu/drm/virtio/virtgpu_trace.h
+++ b/drivers/gpu/drm/virtio/virtgpu_trace.h
@@ -25,7 +25,7 @@ DECLARE_EVENT_CLASS(virtio_gpu_cmd,
 	TP_fast_assign(
 		       __entry->dev = vq->vdev->index;
 		       __entry->vq = vq->index;
-		       __assign_str(name, vq->name);
+		       __assign_str(name);
 		       __entry->type = le32_to_cpu(hdr->type);
 		       __entry->flags = le32_to_cpu(hdr->flags);
 		       __entry->fence_id = le64_to_cpu(hdr->fence_id);
diff --git a/drivers/gpu/drm/vkms/vkms_crtc.c b/drivers/gpu/drm/vkms/vkms_crtc.c
index 61e500b8c9da..40b4d084e3ce 100644
--- a/drivers/gpu/drm/vkms/vkms_crtc.c
+++ b/drivers/gpu/drm/vkms/vkms_crtc.c
@@ -61,9 +61,7 @@ static enum hrtimer_restart vkms_vblank_simulate(struct hrtimer *timer)
 
 static int vkms_enable_vblank(struct drm_crtc *crtc)
 {
-	struct drm_device *dev = crtc->dev;
-	unsigned int pipe = drm_crtc_index(crtc);
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
 	struct vkms_output *out = drm_crtc_to_vkms_output(crtc);
 
 	drm_calc_timestamping_constants(crtc, &crtc->mode);
@@ -88,10 +86,9 @@ static bool vkms_get_vblank_timestamp(struct drm_crtc *crtc,
 				      bool in_vblank_irq)
 {
 	struct drm_device *dev = crtc->dev;
-	unsigned int pipe = crtc->index;
 	struct vkms_device *vkmsdev = drm_device_to_vkms_device(dev);
 	struct vkms_output *output = &vkmsdev->output;
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
 
 	if (!READ_ONCE(vblank->enabled)) {
 		*vblank_time = ktime_get();
diff --git a/drivers/gpu/drm/vmwgfx/Makefile b/drivers/gpu/drm/vmwgfx/Makefile
index e94479d9cd5b..46a4ab688a7f 100644
--- a/drivers/gpu/drm/vmwgfx/Makefile
+++ b/drivers/gpu/drm/vmwgfx/Makefile
@@ -10,6 +10,6 @@ vmwgfx-y := vmwgfx_execbuf.o vmwgfx_gmr.o vmwgfx_kms.o vmwgfx_drv.o \
 	    vmwgfx_simple_resource.o vmwgfx_va.o vmwgfx_blit.o \
 	    vmwgfx_validation.o vmwgfx_page_dirty.o vmwgfx_streamoutput.o \
 	    vmwgfx_devcaps.o ttm_object.o vmwgfx_system_manager.o \
-	    vmwgfx_gem.o
+	    vmwgfx_gem.o vmwgfx_vkms.o
 
 obj-$(CONFIG_DRM_VMWGFX) := vmwgfx.o
diff --git a/drivers/gpu/drm/vmwgfx/ttm_object.c b/drivers/gpu/drm/vmwgfx/ttm_object.c
index 6806c05e57f6..3353e97687d1 100644
--- a/drivers/gpu/drm/vmwgfx/ttm_object.c
+++ b/drivers/gpu/drm/vmwgfx/ttm_object.c
@@ -87,14 +87,11 @@ struct ttm_object_file {
  *
  * @object_lock: lock that protects idr.
  *
- * @object_count: Per device object count.
- *
  * This is the per-device data structure needed for ttm object management.
  */
 
 struct ttm_object_device {
 	spinlock_t object_lock;
-	atomic_t object_count;
 	struct dma_buf_ops ops;
 	void (*dmabuf_release)(struct dma_buf *dma_buf);
 	struct idr idr;
@@ -431,7 +428,6 @@ ttm_object_device_init(const struct dma_buf_ops *ops)
 		return NULL;
 
 	spin_lock_init(&tdev->object_lock);
-	atomic_set(&tdev->object_count, 0);
 
 	/*
 	 * Our base is at VMWGFX_NUM_MOB + 1 because we want to create
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c b/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c
index ae2de914eb89..2731f6ded1c2 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c
@@ -54,6 +54,7 @@
 #include "vmwgfx_drv.h"
 #include "vmwgfx_binding.h"
 #include "device_include/svga3d_reg.h"
+#include <linux/vmalloc.h>
 
 #define VMW_BINDING_RT_BIT     0
 #define VMW_BINDING_PS_BIT     1
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c b/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c
index c52c7bf1485b..717d624e9a05 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c
@@ -456,8 +456,10 @@ int vmw_bo_cpu_blit(struct ttm_buffer_object *dst,
 		.no_wait_gpu = false
 	};
 	u32 j, initial_line = dst_offset / dst_stride;
-	struct vmw_bo_blit_line_data d;
+	struct vmw_bo_blit_line_data d = {0};
 	int ret = 0;
+	struct page **dst_pages = NULL;
+	struct page **src_pages = NULL;
 
 	/* Buffer objects need to be either pinned or reserved: */
 	if (!(dst->pin_count))
@@ -477,12 +479,35 @@ int vmw_bo_cpu_blit(struct ttm_buffer_object *dst,
 			return ret;
 	}
 
+	if (!src->ttm->pages && src->ttm->sg) {
+		src_pages = kvmalloc_array(src->ttm->num_pages,
+					   sizeof(struct page *), GFP_KERNEL);
+		if (!src_pages)
+			return -ENOMEM;
+		ret = drm_prime_sg_to_page_array(src->ttm->sg, src_pages,
+						 src->ttm->num_pages);
+		if (ret)
+			goto out;
+	}
+	if (!dst->ttm->pages && dst->ttm->sg) {
+		dst_pages = kvmalloc_array(dst->ttm->num_pages,
+					   sizeof(struct page *), GFP_KERNEL);
+		if (!dst_pages) {
+			ret = -ENOMEM;
+			goto out;
+		}
+		ret = drm_prime_sg_to_page_array(dst->ttm->sg, dst_pages,
+						 dst->ttm->num_pages);
+		if (ret)
+			goto out;
+	}
+
 	d.mapped_dst = 0;
 	d.mapped_src = 0;
 	d.dst_addr = NULL;
 	d.src_addr = NULL;
-	d.dst_pages = dst->ttm->pages;
-	d.src_pages = src->ttm->pages;
+	d.dst_pages = dst->ttm->pages ? dst->ttm->pages : dst_pages;
+	d.src_pages = src->ttm->pages ? src->ttm->pages : src_pages;
 	d.dst_num_pages = PFN_UP(dst->resource->size);
 	d.src_num_pages = PFN_UP(src->resource->size);
 	d.dst_prot = ttm_io_prot(dst, dst->resource, PAGE_KERNEL);
@@ -504,6 +529,10 @@ out:
 		kunmap_atomic(d.src_addr);
 	if (d.dst_addr)
 		kunmap_atomic(d.dst_addr);
+	if (src_pages)
+		kvfree(src_pages);
+	if (dst_pages)
+		kvfree(dst_pages);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c
index bfd41ce3c8f4..00144632c600 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c
@@ -204,6 +204,7 @@ int vmw_bo_pin_in_start_of_vram(struct vmw_private *dev_priv,
 			     VMW_BO_DOMAIN_VRAM,
 			     VMW_BO_DOMAIN_VRAM);
 	buf->places[0].lpfn = PFN_UP(bo->resource->size);
+	buf->busy_places[0].lpfn = PFN_UP(bo->resource->size);
 	ret = ttm_bo_validate(bo, &buf->placement, &ctx);
 
 	/* For some reason we didn't end up at the start of vram */
@@ -377,7 +378,8 @@ static int vmw_bo_init(struct vmw_private *dev_priv,
 {
 	struct ttm_operation_ctx ctx = {
 		.interruptible = params->bo_type != ttm_bo_type_kernel,
-		.no_wait_gpu = false
+		.no_wait_gpu = false,
+		.resv = params->resv,
 	};
 	struct ttm_device *bdev = &dev_priv->bdev;
 	struct drm_device *vdev = &dev_priv->drm;
@@ -394,8 +396,8 @@ static int vmw_bo_init(struct vmw_private *dev_priv,
 
 	vmw_bo_placement_set(vmw_bo, params->domain, params->busy_domain);
 	ret = ttm_bo_init_reserved(bdev, &vmw_bo->tbo, params->bo_type,
-				   &vmw_bo->placement, 0, &ctx, NULL,
-				   NULL, destroy);
+				   &vmw_bo->placement, 0, &ctx,
+				   params->sg, params->resv, destroy);
 	if (unlikely(ret))
 		return ret;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.h b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.h
index 0d496dc9c6af..f349642e6190 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.h
@@ -55,6 +55,8 @@ struct vmw_bo_params {
 	enum ttm_bo_type bo_type;
 	size_t size;
 	bool pin;
+	struct dma_resv *resv;
+	struct sg_table *sg;
 };
 
 /**
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c
index 195ff8792e5a..dd4ca6a9c690 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c
@@ -31,6 +31,7 @@
 #include <drm/ttm/ttm_placement.h>
 
 #include <linux/sched/signal.h>
+#include <linux/vmalloc.h>
 
 bool vmw_supports_3d(struct vmw_private *dev_priv)
 {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c b/drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c
index 829df395c2ed..6e6beff9e262 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_devcaps.c
@@ -25,6 +25,7 @@
  *
  **************************************************************************/
 
+#include <linux/vmalloc.h>
 #include "vmwgfx_devcaps.h"
 
 #include "vmwgfx_drv.h"
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 0a304706e013..8f1730aeacc9 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -32,6 +32,7 @@
 #include "vmwgfx_binding.h"
 #include "vmwgfx_devcaps.h"
 #include "vmwgfx_mksstat.h"
+#include "vmwgfx_vkms.h"
 #include "ttm_object.h"
 
 #include <drm/drm_aperture.h>
@@ -53,6 +54,7 @@
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/version.h>
+#include <linux/vmalloc.h>
 
 #define VMWGFX_DRIVER_DESC "Linux drm driver for VMware graphics devices"
 
@@ -911,6 +913,8 @@ static int vmw_driver_load(struct vmw_private *dev_priv, u32 pci_id)
 			     "Please switch to a supported graphics device to avoid problems.");
 	}
 
+	vmw_vkms_init(dev_priv);
+
 	ret = vmw_dma_select_mode(dev_priv);
 	if (unlikely(ret != 0)) {
 		drm_info(&dev_priv->drm,
@@ -1196,6 +1200,7 @@ static void vmw_driver_unload(struct drm_device *dev)
 
 	vmw_svga_disable(dev_priv);
 
+	vmw_vkms_cleanup(dev_priv);
 	vmw_kms_close(dev_priv);
 	vmw_overlay_close(dev_priv);
 
@@ -1628,6 +1633,7 @@ static const struct drm_driver driver = {
 
 	.prime_fd_to_handle = vmw_prime_fd_to_handle,
 	.prime_handle_to_fd = vmw_prime_handle_to_fd,
+	.gem_prime_import_sg_table = vmw_prime_import_sg_table,
 
 	.fops = &vmwgfx_driver_fops,
 	.name = VMWGFX_DRIVER_NAME,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 12efecc17df6..4ecaea0026fc 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -117,25 +117,8 @@ struct vmwgfx_hash_item {
 	unsigned long key;
 };
 
-
-/**
- * struct vmw_validate_buffer - Carries validation info about buffers.
- *
- * @base: Validation info for TTM.
- * @hash: Hash entry for quick lookup of the TTM buffer object.
- *
- * This structure contains also driver private validation info
- * on top of the info needed by TTM.
- */
-struct vmw_validate_buffer {
-	struct ttm_validate_buffer base;
-	struct vmwgfx_hash_item hash;
-	bool validate_as_mob;
-};
-
 struct vmw_res_func;
 
-
 /**
  * struct vmw-resource - base class for hardware resources
  *
@@ -445,15 +428,6 @@ struct vmw_sw_context{
 struct vmw_legacy_display;
 struct vmw_overlay;
 
-struct vmw_vga_topology_state {
-	uint32_t width;
-	uint32_t height;
-	uint32_t primary;
-	uint32_t pos_x;
-	uint32_t pos_y;
-};
-
-
 /*
  * struct vmw_otable - Guest Memory OBject table metadata
  *
@@ -501,7 +475,6 @@ struct vmw_private {
 	struct drm_device drm;
 	struct ttm_device bdev;
 
-	struct drm_vma_offset_manager vma_manager;
 	u32 pci_id;
 	resource_size_t io_start;
 	resource_size_t vram_start;
@@ -642,6 +615,9 @@ struct vmw_private {
 
 	uint32 *devcaps;
 
+	bool vkms_enabled;
+	struct workqueue_struct *crc_workq;
+
 	/*
 	 * mksGuestStat instance-descriptor and pid arrays
 	 */
@@ -836,6 +812,7 @@ void vmw_resource_mob_attach(struct vmw_resource *res);
 void vmw_resource_mob_detach(struct vmw_resource *res);
 void vmw_resource_dirty_update(struct vmw_resource *res, pgoff_t start,
 			       pgoff_t end);
+int vmw_resource_clean(struct vmw_resource *res);
 int vmw_resources_clean(struct vmw_bo *vbo, pgoff_t start,
 			pgoff_t end, pgoff_t *num_prefault);
 
@@ -1130,6 +1107,9 @@ extern int vmw_prime_handle_to_fd(struct drm_device *dev,
 				  struct drm_file *file_priv,
 				  uint32_t handle, uint32_t flags,
 				  int *prime_fd);
+struct drm_gem_object *vmw_prime_import_sg_table(struct drm_device *dev,
+						 struct dma_buf_attachment *attach,
+						 struct sg_table *table);
 
 /*
  * MemoryOBject management -  vmwgfx_mob.c
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index cc3086e649eb..2e52d73eba48 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -35,6 +35,7 @@
 
 #include <linux/sync_file.h>
 #include <linux/hashtable.h>
+#include <linux/vmalloc.h>
 
 /*
  * Helper macro to get dx_ctx_node if available otherwise print an error
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
index 2a0cda324703..5efc6a766f64 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
@@ -991,7 +991,7 @@ static int vmw_event_fence_action_create(struct drm_file *file_priv,
 	}
 
 	event->event.base.type = DRM_VMW_EVENT_FENCE_SIGNALED;
-	event->event.base.length = sizeof(*event);
+	event->event.base.length = sizeof(event->event);
 	event->event.user_data = user_data;
 
 	ret = drm_event_reserve_init(dev, file_priv, &event->base, &event->event.base);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_gem.c b/drivers/gpu/drm/vmwgfx/vmwgfx_gem.c
index 12787bb9c111..07185c108218 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_gem.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_gem.c
@@ -30,6 +30,8 @@
 #include "drm/drm_prime.h"
 #include "drm/drm_gem_ttm_helper.h"
 
+#include <linux/debugfs.h>
+
 static void vmw_gem_object_free(struct drm_gem_object *gobj)
 {
 	struct ttm_buffer_object *bo = drm_gem_ttm_of_gem(gobj);
@@ -48,33 +50,20 @@ static void vmw_gem_object_close(struct drm_gem_object *obj,
 {
 }
 
-static int vmw_gem_pin_private(struct drm_gem_object *obj, bool do_pin)
+static int vmw_gem_object_pin(struct drm_gem_object *obj)
 {
-	struct ttm_buffer_object *bo = drm_gem_ttm_of_gem(obj);
 	struct vmw_bo *vbo = to_vmw_bo(obj);
-	int ret;
-
-	ret = ttm_bo_reserve(bo, false, false, NULL);
-	if (unlikely(ret != 0))
-		goto err;
-
-	vmw_bo_pin_reserved(vbo, do_pin);
-
-	ttm_bo_unreserve(bo);
-
-err:
-	return ret;
-}
 
+	vmw_bo_pin_reserved(vbo, true);
 
-static int vmw_gem_object_pin(struct drm_gem_object *obj)
-{
-	return vmw_gem_pin_private(obj, true);
+	return 0;
 }
 
 static void vmw_gem_object_unpin(struct drm_gem_object *obj)
 {
-	vmw_gem_pin_private(obj, false);
+	struct vmw_bo *vbo = to_vmw_bo(obj);
+
+	vmw_bo_pin_reserved(vbo, false);
 }
 
 static struct sg_table *vmw_gem_object_get_sg_table(struct drm_gem_object *obj)
@@ -149,6 +138,38 @@ out_no_bo:
 	return ret;
 }
 
+struct drm_gem_object *vmw_prime_import_sg_table(struct drm_device *dev,
+						 struct dma_buf_attachment *attach,
+						 struct sg_table *table)
+{
+	int ret;
+	struct vmw_private *dev_priv = vmw_priv(dev);
+	struct drm_gem_object *gem = NULL;
+	struct vmw_bo *vbo;
+	struct vmw_bo_params params = {
+		.domain = (dev_priv->has_mob) ? VMW_BO_DOMAIN_SYS : VMW_BO_DOMAIN_VRAM,
+		.busy_domain = VMW_BO_DOMAIN_SYS,
+		.bo_type = ttm_bo_type_sg,
+		.size = attach->dmabuf->size,
+		.pin = false,
+		.resv = attach->dmabuf->resv,
+		.sg = table,
+
+	};
+
+	dma_resv_lock(params.resv, NULL);
+
+	ret = vmw_bo_create(dev_priv, &params, &vbo);
+	if (ret != 0)
+		goto out_no_bo;
+
+	vbo->tbo.base.funcs = &vmw_gem_object_funcs;
+
+	gem = &vbo->tbo.base;
+out_no_bo:
+	dma_resv_unlock(params.resv);
+	return gem;
+}
 
 int vmw_gem_object_create_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *filp)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
index a1da5678c731..835d1eed8dd9 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
@@ -31,6 +31,7 @@
 
 #include <drm/vmwgfx_drm.h>
 #include <linux/pci.h>
+#include <linux/vmalloc.h>
 
 int vmw_getparam_ioctl(struct drm_device *dev, void *data,
 		       struct drm_file *file_priv)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index cd4925346ed4..13b2820cae51 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -27,6 +27,7 @@
 #include "vmwgfx_kms.h"
 
 #include "vmwgfx_bo.h"
+#include "vmwgfx_vkms.h"
 #include "vmw_surface_cache.h"
 
 #include <drm/drm_atomic.h>
@@ -37,9 +38,16 @@
 #include <drm/drm_sysfs.h>
 #include <drm/drm_edid.h>
 
+void vmw_du_init(struct vmw_display_unit *du)
+{
+	vmw_vkms_crtc_init(&du->crtc);
+}
+
 void vmw_du_cleanup(struct vmw_display_unit *du)
 {
 	struct vmw_private *dev_priv = vmw_priv(du->primary.dev);
+
+	vmw_vkms_crtc_cleanup(&du->crtc);
 	drm_plane_cleanup(&du->primary);
 	if (vmw_cmd_supported(dev_priv))
 		drm_plane_cleanup(&du->cursor.base);
@@ -775,7 +783,6 @@ vmw_du_cursor_plane_atomic_update(struct drm_plane *plane,
 	hotspot_y = du->hotspot_y + new_state->hotspot_y;
 
 	du->cursor_surface = vps->surf;
-	du->cursor_bo = vps->bo;
 
 	if (!vps->surf && !vps->bo) {
 		vmw_cursor_update_position(dev_priv, false, 0, 0);
@@ -858,15 +865,6 @@ int vmw_du_primary_plane_atomic_check(struct drm_plane *plane,
 						  DRM_PLANE_NO_SCALING,
 						  DRM_PLANE_NO_SCALING,
 						  false, true);
-
-	if (!ret && new_fb) {
-		struct drm_crtc *crtc = new_state->crtc;
-		struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
-
-		vmw_connector_state_to_vcs(du->connector.state);
-	}
-
-
 	return ret;
 }
 
@@ -933,6 +931,7 @@ int vmw_du_cursor_plane_atomic_check(struct drm_plane *plane,
 int vmw_du_crtc_atomic_check(struct drm_crtc *crtc,
 			     struct drm_atomic_state *state)
 {
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
 	struct drm_crtc_state *new_state = drm_atomic_get_new_crtc_state(state,
 									 crtc);
 	struct vmw_display_unit *du = vmw_crtc_to_du(new_state->crtc);
@@ -940,9 +939,13 @@ int vmw_du_crtc_atomic_check(struct drm_crtc *crtc,
 	bool has_primary = new_state->plane_mask &
 			   drm_plane_mask(crtc->primary);
 
-	/* We always want to have an active plane with an active CRTC */
-	if (has_primary != new_state->enable)
-		return -EINVAL;
+	/*
+	 * This is fine in general, but broken userspace might expect
+	 * some actual rendering so give a clue as why it's blank.
+	 */
+	if (new_state->enable && !has_primary)
+		drm_dbg_driver(&vmw->drm,
+			       "CRTC without a primary plane will be blank.\n");
 
 
 	if (new_state->connector_mask != connector_mask &&
@@ -965,15 +968,9 @@ int vmw_du_crtc_atomic_check(struct drm_crtc *crtc,
 void vmw_du_crtc_atomic_begin(struct drm_crtc *crtc,
 			      struct drm_atomic_state *state)
 {
+	vmw_vkms_crtc_atomic_begin(crtc, state);
 }
 
-
-void vmw_du_crtc_atomic_flush(struct drm_crtc *crtc,
-			      struct drm_atomic_state *state)
-{
-}
-
-
 /**
  * vmw_du_crtc_duplicate_state - duplicate crtc state
  * @crtc: DRM crtc
@@ -1361,7 +1358,6 @@ static int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv,
 
 	drm_helper_mode_fill_fb_struct(dev, &vfbs->base.base, mode_cmd);
 	vfbs->surface = vmw_surface_reference(surface);
-	vfbs->base.user_handle = mode_cmd->handles[0];
 	vfbs->is_bo_proxy = is_bo_proxy;
 
 	*out = &vfbs->base;
@@ -1529,7 +1525,6 @@ static int vmw_kms_new_framebuffer_bo(struct vmw_private *dev_priv,
 	drm_helper_mode_fill_fb_struct(dev, &vfbd->base.base, mode_cmd);
 	vfbd->base.bo = true;
 	vfbd->buffer = vmw_bo_reference(bo);
-	vfbd->base.user_handle = mode_cmd->handles[0];
 	*out = &vfbd->base;
 
 	ret = drm_framebuffer_init(dev, &vfbd->base.base,
@@ -2040,6 +2035,29 @@ vmw_kms_create_hotplug_mode_update_property(struct vmw_private *dev_priv)
 					  "hotplug_mode_update", 0, 1);
 }
 
+static void
+vmw_atomic_commit_tail(struct drm_atomic_state *old_state)
+{
+	struct vmw_private *vmw = vmw_priv(old_state->dev);
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state;
+	int i;
+
+	drm_atomic_helper_commit_tail(old_state);
+
+	if (vmw->vkms_enabled) {
+		for_each_old_crtc_in_state(old_state, crtc, old_crtc_state, i) {
+			struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+			(void)old_crtc_state;
+			flush_work(&du->vkms.crc_generator_work);
+		}
+	}
+}
+
+static const struct drm_mode_config_helper_funcs vmw_mode_config_helpers = {
+	.atomic_commit_tail = vmw_atomic_commit_tail,
+};
+
 int vmw_kms_init(struct vmw_private *dev_priv)
 {
 	struct drm_device *dev = &dev_priv->drm;
@@ -2059,6 +2077,7 @@ int vmw_kms_init(struct vmw_private *dev_priv)
 	dev->mode_config.max_width = dev_priv->texture_max_width;
 	dev->mode_config.max_height = dev_priv->texture_max_height;
 	dev->mode_config.preferred_depth = dev_priv->assume_16bpp ? 16 : 32;
+	dev->mode_config.helper_private = &vmw_mode_config_helpers;
 
 	drm_mode_create_suggested_offset_properties(dev);
 	vmw_kms_create_hotplug_mode_update_property(dev_priv);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index a94947b588e8..bf24f2f0dcfc 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -217,21 +217,11 @@ struct vmw_kms_dirty {
 struct vmw_framebuffer {
 	struct drm_framebuffer base;
 	bool bo;
-	uint32_t user_handle;
-};
-
-/*
- * Clip rectangle
- */
-struct vmw_clip_rect {
-	int x1, x2, y1, y2;
 };
 
 struct vmw_framebuffer_surface {
 	struct vmw_framebuffer base;
 	struct vmw_surface *surface;
-	struct vmw_bo *buffer;
-	struct list_head head;
 	bool is_bo_proxy;  /* true if this is proxy surface for DMA buf */
 };
 
@@ -243,10 +233,10 @@ struct vmw_framebuffer_bo {
 
 
 static const uint32_t __maybe_unused vmw_primary_plane_formats[] = {
-	DRM_FORMAT_XRGB1555,
-	DRM_FORMAT_RGB565,
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB1555,
 };
 
 static const uint32_t __maybe_unused vmw_cursor_plane_formats[] = {
@@ -359,7 +349,6 @@ struct vmw_display_unit {
 	struct vmw_cursor_plane cursor;
 
 	struct vmw_surface *cursor_surface;
-	struct vmw_bo *cursor_bo;
 	size_t cursor_age;
 
 	int cursor_x;
@@ -387,11 +376,25 @@ struct vmw_display_unit {
 	bool is_implicit;
 	int set_gui_x;
 	int set_gui_y;
-};
 
-struct vmw_validation_ctx {
-	struct vmw_resource *res;
-	struct vmw_bo *buf;
+	struct {
+		struct work_struct crc_generator_work;
+		struct hrtimer timer;
+		ktime_t period_ns;
+
+		/* protects concurrent access to the vblank handler */
+		atomic_t atomic_lock;
+		/* protected by @atomic_lock */
+		bool crc_enabled;
+		struct vmw_surface *surface;
+
+		/* protects concurrent access to the crc worker */
+		spinlock_t crc_state_lock;
+		/* protected by @crc_state_lock */
+		bool crc_pending;
+		u64 frame_start;
+		u64 frame_end;
+	} vkms;
 };
 
 #define vmw_crtc_to_du(x) \
@@ -403,6 +406,7 @@ struct vmw_validation_ctx {
 /*
  * Shared display unit functions - vmwgfx_kms.c
  */
+void vmw_du_init(struct vmw_display_unit *du);
 void vmw_du_cleanup(struct vmw_display_unit *du);
 void vmw_du_crtc_save(struct drm_crtc *crtc);
 void vmw_du_crtc_restore(struct drm_crtc *crtc);
@@ -489,8 +493,6 @@ int vmw_du_crtc_atomic_check(struct drm_crtc *crtc,
 			     struct drm_atomic_state *state);
 void vmw_du_crtc_atomic_begin(struct drm_crtc *crtc,
 			      struct drm_atomic_state *state);
-void vmw_du_crtc_atomic_flush(struct drm_crtc *crtc,
-			      struct drm_atomic_state *state);
 void vmw_du_crtc_reset(struct drm_crtc *crtc);
 struct drm_crtc_state *vmw_du_crtc_duplicate_state(struct drm_crtc *crtc);
 void vmw_du_crtc_destroy_state(struct drm_crtc *crtc,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index c4db4aecca6c..5befc2719a49 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -27,6 +27,7 @@
 
 #include "vmwgfx_bo.h"
 #include "vmwgfx_kms.h"
+#include "vmwgfx_vkms.h"
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
@@ -241,33 +242,6 @@ static void vmw_ldu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 {
 }
 
-/**
- * vmw_ldu_crtc_atomic_enable - Noop
- *
- * @crtc: CRTC associated with the new screen
- * @state: Unused
- *
- * This is called after a mode set has been completed.  Here's
- * usually a good place to call vmw_ldu_add_active/vmw_ldu_del_active
- * but since for LDU the display plane is closely tied to the
- * CRTC, it makes more sense to do those at plane update time.
- */
-static void vmw_ldu_crtc_atomic_enable(struct drm_crtc *crtc,
-				       struct drm_atomic_state *state)
-{
-}
-
-/**
- * vmw_ldu_crtc_atomic_disable - Turns off CRTC
- *
- * @crtc: CRTC to be turned off
- * @state: Unused
- */
-static void vmw_ldu_crtc_atomic_disable(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
-{
-}
-
 static const struct drm_crtc_funcs vmw_legacy_crtc_funcs = {
 	.gamma_set = vmw_du_crtc_gamma_set,
 	.destroy = vmw_ldu_crtc_destroy,
@@ -276,6 +250,9 @@ static const struct drm_crtc_funcs vmw_legacy_crtc_funcs = {
 	.atomic_destroy_state = vmw_du_crtc_destroy_state,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
+	.enable_vblank          = vmw_vkms_enable_vblank,
+	.disable_vblank         = vmw_vkms_disable_vblank,
+	.get_vblank_timestamp   = vmw_vkms_get_vblank_timestamp,
 };
 
 
@@ -418,9 +395,9 @@ static const struct drm_crtc_helper_funcs vmw_ldu_crtc_helper_funcs = {
 	.mode_set_nofb = vmw_ldu_crtc_mode_set_nofb,
 	.atomic_check = vmw_du_crtc_atomic_check,
 	.atomic_begin = vmw_du_crtc_atomic_begin,
-	.atomic_flush = vmw_du_crtc_atomic_flush,
-	.atomic_enable = vmw_ldu_crtc_atomic_enable,
-	.atomic_disable = vmw_ldu_crtc_atomic_disable,
+	.atomic_flush = vmw_vkms_crtc_atomic_flush,
+	.atomic_enable = vmw_vkms_crtc_atomic_enable,
+	.atomic_disable = vmw_vkms_crtc_atomic_disable,
 };
 
 
@@ -541,6 +518,8 @@ static int vmw_ldu_init(struct vmw_private *dev_priv, unsigned unit)
 			 dev_priv->implicit_placement_property,
 			 1);
 
+	vmw_du_init(&ldu->base);
+
 	return 0;
 
 err_free_unregister:
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_prime.c b/drivers/gpu/drm/vmwgfx/vmwgfx_prime.c
index 2d72a5ee7c0c..c99cad444991 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_prime.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_prime.c
@@ -75,8 +75,12 @@ int vmw_prime_fd_to_handle(struct drm_device *dev,
 			   int fd, u32 *handle)
 {
 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
+	int ret = ttm_prime_fd_to_handle(tfile, fd, handle);
 
-	return ttm_prime_fd_to_handle(tfile, fd, handle);
+	if (ret)
+		ret = drm_gem_prime_fd_to_handle(dev, file_priv, fd, handle);
+
+	return ret;
 }
 
 int vmw_prime_handle_to_fd(struct drm_device *dev,
@@ -85,5 +89,12 @@ int vmw_prime_handle_to_fd(struct drm_device *dev,
 			   int *prime_fd)
 {
 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
-	return ttm_prime_handle_to_fd(tfile, handle, flags, prime_fd);
+	int ret;
+
+	if (handle > VMWGFX_NUM_MOB)
+		ret = ttm_prime_handle_to_fd(tfile, handle, flags, prime_fd);
+	else
+		ret = drm_gem_prime_handle_to_fd(dev, file_priv, handle, flags, prime_fd);
+
+	return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index ca300c7427d2..848dba09981b 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -1064,6 +1064,22 @@ void vmw_resource_dirty_update(struct vmw_resource *res, pgoff_t start,
 					   end << PAGE_SHIFT);
 }
 
+int vmw_resource_clean(struct vmw_resource *res)
+{
+	int ret = 0;
+
+	if (res->res_dirty) {
+		if (!res->func->clean)
+			return -EINVAL;
+
+		ret = res->func->clean(res);
+		if (ret)
+			return ret;
+		res->res_dirty = false;
+	}
+	return ret;
+}
+
 /**
  * vmw_resources_clean - Clean resources intersecting a mob range
  * @vbo: The mob buffer object
@@ -1080,6 +1096,7 @@ int vmw_resources_clean(struct vmw_bo *vbo, pgoff_t start,
 	unsigned long res_start = start << PAGE_SHIFT;
 	unsigned long res_end = end << PAGE_SHIFT;
 	unsigned long last_cleaned = 0;
+	int ret;
 
 	/*
 	 * Find the resource with lowest backup_offset that intersects the
@@ -1106,18 +1123,9 @@ int vmw_resources_clean(struct vmw_bo *vbo, pgoff_t start,
 	 * intersecting the range.
 	 */
 	while (found) {
-		if (found->res_dirty) {
-			int ret;
-
-			if (!found->func->clean)
-				return -EINVAL;
-
-			ret = found->func->clean(found);
-			if (ret)
-				return ret;
-
-			found->res_dirty = false;
-		}
+		ret = vmw_resource_clean(found);
+		if (ret)
+			return ret;
 		last_cleaned = found->guest_memory_offset + found->guest_memory_size;
 		cur = rb_next(&found->mob_node);
 		if (!cur)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
index 30c3ad27b662..df0039a8ef29 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
@@ -27,11 +27,13 @@
 
 #include "vmwgfx_bo.h"
 #include "vmwgfx_kms.h"
+#include "vmwgfx_vkms.h"
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
+#include <drm/drm_vblank.h>
 
 #define vmw_crtc_to_sou(x) \
 	container_of(x, struct vmw_screen_object_unit, base.crtc)
@@ -89,7 +91,6 @@ struct vmw_kms_sou_define_gmrfb {
 struct vmw_screen_object_unit {
 	struct vmw_display_unit base;
 
-	unsigned long buffer_size; /**< Size of allocated buffer */
 	struct vmw_bo *buffer; /**< Backing store buffer */
 
 	bool defined;
@@ -240,7 +241,6 @@ static void vmw_sou_crtc_mode_set_nofb(struct drm_crtc *crtc)
 		int x, y;
 
 		sou->buffer = vps->bo;
-		sou->buffer_size = vps->bo_size;
 
 		conn_state = sou->base.connector.state;
 		vmw_conn_state = vmw_connector_state_to_vcs(conn_state);
@@ -255,7 +255,6 @@ static void vmw_sou_crtc_mode_set_nofb(struct drm_crtc *crtc)
 
 	} else {
 		sou->buffer = NULL;
-		sou->buffer_size = 0;
 	}
 }
 
@@ -271,19 +270,6 @@ static void vmw_sou_crtc_helper_prepare(struct drm_crtc *crtc)
 }
 
 /**
- * vmw_sou_crtc_atomic_enable - Noop
- *
- * @crtc: CRTC associated with the new screen
- * @state: Unused
- *
- * This is called after a mode set has been completed.
- */
-static void vmw_sou_crtc_atomic_enable(struct drm_crtc *crtc,
-				       struct drm_atomic_state *state)
-{
-}
-
-/**
  * vmw_sou_crtc_atomic_disable - Turns off CRTC
  *
  * @crtc: CRTC to be turned off
@@ -305,6 +291,9 @@ static void vmw_sou_crtc_atomic_disable(struct drm_crtc *crtc,
 	sou = vmw_crtc_to_sou(crtc);
 	dev_priv = vmw_priv(crtc->dev);
 
+	if (dev_priv->vkms_enabled)
+		drm_crtc_vblank_off(crtc);
+
 	if (sou->defined) {
 		ret = vmw_sou_fifo_destroy(dev_priv, sou);
 		if (ret)
@@ -320,6 +309,9 @@ static const struct drm_crtc_funcs vmw_screen_object_crtc_funcs = {
 	.atomic_destroy_state = vmw_du_crtc_destroy_state,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
+	.enable_vblank          = vmw_vkms_enable_vblank,
+	.disable_vblank         = vmw_vkms_disable_vblank,
+	.get_vblank_timestamp   = vmw_vkms_get_vblank_timestamp,
 };
 
 /*
@@ -797,8 +789,8 @@ static const struct drm_crtc_helper_funcs vmw_sou_crtc_helper_funcs = {
 	.mode_set_nofb = vmw_sou_crtc_mode_set_nofb,
 	.atomic_check = vmw_du_crtc_atomic_check,
 	.atomic_begin = vmw_du_crtc_atomic_begin,
-	.atomic_flush = vmw_du_crtc_atomic_flush,
-	.atomic_enable = vmw_sou_crtc_atomic_enable,
+	.atomic_flush = vmw_vkms_crtc_atomic_flush,
+	.atomic_enable = vmw_vkms_crtc_atomic_enable,
 	.atomic_disable = vmw_sou_crtc_atomic_disable,
 };
 
@@ -908,6 +900,9 @@ static int vmw_sou_init(struct vmw_private *dev_priv, unsigned unit)
 				   dev->mode_config.suggested_x_property, 0);
 	drm_object_attach_property(&connector->base,
 				   dev->mode_config.suggested_y_property, 0);
+
+	vmw_du_init(&sou->base);
+
 	return 0;
 
 err_free_unregister:
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index 3c8414a13dba..2041c4d48daa 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -27,12 +27,14 @@
 
 #include "vmwgfx_bo.h"
 #include "vmwgfx_kms.h"
+#include "vmwgfx_vkms.h"
 #include "vmw_surface_cache.h"
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
+#include <drm/drm_vblank.h>
 
 #define vmw_crtc_to_stdu(x) \
 	container_of(x, struct vmw_screen_target_display_unit, base.crtc)
@@ -407,16 +409,6 @@ static void vmw_stdu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 			  crtc->x, crtc->y);
 }
 
-
-static void vmw_stdu_crtc_helper_prepare(struct drm_crtc *crtc)
-{
-}
-
-static void vmw_stdu_crtc_atomic_enable(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
-{
-}
-
 static void vmw_stdu_crtc_atomic_disable(struct drm_crtc *crtc,
 					 struct drm_atomic_state *state)
 {
@@ -424,7 +416,6 @@ static void vmw_stdu_crtc_atomic_disable(struct drm_crtc *crtc,
 	struct vmw_screen_target_display_unit *stdu;
 	int ret;
 
-
 	if (!crtc) {
 		DRM_ERROR("CRTC is NULL\n");
 		return;
@@ -433,6 +424,9 @@ static void vmw_stdu_crtc_atomic_disable(struct drm_crtc *crtc,
 	stdu     = vmw_crtc_to_stdu(crtc);
 	dev_priv = vmw_priv(crtc->dev);
 
+	if (dev_priv->vkms_enabled)
+		drm_crtc_vblank_off(crtc);
+
 	if (stdu->defined) {
 		ret = vmw_stdu_bind_st(dev_priv, stdu, NULL);
 		if (ret)
@@ -770,7 +764,6 @@ out_unref:
 	return ret;
 }
 
-
 /*
  *  Screen Target CRTC dispatch table
  */
@@ -782,6 +775,12 @@ static const struct drm_crtc_funcs vmw_stdu_crtc_funcs = {
 	.atomic_destroy_state = vmw_du_crtc_destroy_state,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
+	.enable_vblank		= vmw_vkms_enable_vblank,
+	.disable_vblank		= vmw_vkms_disable_vblank,
+	.get_vblank_timestamp	= vmw_vkms_get_vblank_timestamp,
+	.get_crc_sources	= vmw_vkms_get_crc_sources,
+	.set_crc_source		= vmw_vkms_set_crc_source,
+	.verify_crc_source	= vmw_vkms_verify_crc_source,
 };
 
 
@@ -1413,6 +1412,17 @@ vmw_stdu_primary_plane_atomic_update(struct drm_plane *plane,
 		vmw_fence_obj_unreference(&fence);
 }
 
+static void
+vmw_stdu_crtc_atomic_flush(struct drm_crtc *crtc,
+			   struct drm_atomic_state *state)
+{
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
+	struct vmw_screen_target_display_unit *stdu = vmw_crtc_to_stdu(crtc);
+
+	if (vmw->vkms_enabled)
+		vmw_vkms_set_crc_surface(crtc, stdu->display_srf);
+	vmw_vkms_crtc_atomic_flush(crtc, state);
+}
 
 static const struct drm_plane_funcs vmw_stdu_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
@@ -1453,12 +1463,11 @@ drm_plane_helper_funcs vmw_stdu_primary_plane_helper_funcs = {
 };
 
 static const struct drm_crtc_helper_funcs vmw_stdu_crtc_helper_funcs = {
-	.prepare = vmw_stdu_crtc_helper_prepare,
 	.mode_set_nofb = vmw_stdu_crtc_mode_set_nofb,
 	.atomic_check = vmw_du_crtc_atomic_check,
 	.atomic_begin = vmw_du_crtc_atomic_begin,
-	.atomic_flush = vmw_du_crtc_atomic_flush,
-	.atomic_enable = vmw_stdu_crtc_atomic_enable,
+	.atomic_flush = vmw_stdu_crtc_atomic_flush,
+	.atomic_enable = vmw_vkms_crtc_atomic_enable,
 	.atomic_disable = vmw_stdu_crtc_atomic_disable,
 };
 
@@ -1575,6 +1584,9 @@ static int vmw_stdu_init(struct vmw_private *dev_priv, unsigned unit)
 				   dev->mode_config.suggested_x_property, 0);
 	drm_object_attach_property(&connector->base,
 				   dev->mode_config.suggested_y_property, 0);
+
+	vmw_du_init(&stdu->base);
+
 	return 0;
 
 err_free_unregister:
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c
index 4d23d0a70bcb..621d98b376bb 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c
@@ -188,13 +188,18 @@ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)
 	switch (dev_priv->map_mode) {
 	case vmw_dma_map_bind:
 	case vmw_dma_map_populate:
-		vsgt->sgt = &vmw_tt->sgt;
-		ret = sg_alloc_table_from_pages_segment(
-			&vmw_tt->sgt, vsgt->pages, vsgt->num_pages, 0,
-			(unsigned long)vsgt->num_pages << PAGE_SHIFT,
-			dma_get_max_seg_size(dev_priv->drm.dev), GFP_KERNEL);
-		if (ret)
-			goto out_sg_alloc_fail;
+		if (vmw_tt->dma_ttm.page_flags  & TTM_TT_FLAG_EXTERNAL) {
+			vsgt->sgt = vmw_tt->dma_ttm.sg;
+		} else {
+			vsgt->sgt = &vmw_tt->sgt;
+			ret = sg_alloc_table_from_pages_segment(&vmw_tt->sgt,
+				vsgt->pages, vsgt->num_pages, 0,
+				(unsigned long)vsgt->num_pages << PAGE_SHIFT,
+				dma_get_max_seg_size(dev_priv->drm.dev),
+				GFP_KERNEL);
+			if (ret)
+				goto out_sg_alloc_fail;
+		}
 
 		ret = vmw_ttm_map_for_dma(vmw_tt);
 		if (unlikely(ret != 0))
@@ -209,8 +214,9 @@ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)
 	return 0;
 
 out_map_fail:
-	sg_free_table(vmw_tt->vsgt.sgt);
-	vmw_tt->vsgt.sgt = NULL;
+	drm_warn(&dev_priv->drm, "VSG table map failed!");
+	sg_free_table(vsgt->sgt);
+	vsgt->sgt = NULL;
 out_sg_alloc_fail:
 	return ret;
 }
@@ -356,15 +362,17 @@ static void vmw_ttm_destroy(struct ttm_device *bdev, struct ttm_tt *ttm)
 static int vmw_ttm_populate(struct ttm_device *bdev,
 			    struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
 {
-	int ret;
+	bool external = (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) != 0;
 
-	/* TODO: maybe completely drop this ? */
 	if (ttm_tt_is_populated(ttm))
 		return 0;
 
-	ret = ttm_pool_alloc(&bdev->pool, ttm, ctx);
+	if (external && ttm->sg)
+		return  drm_prime_sg_to_dma_addr_array(ttm->sg,
+						       ttm->dma_address,
+						       ttm->num_pages);
 
-	return ret;
+	return ttm_pool_alloc(&bdev->pool, ttm, ctx);
 }
 
 static void vmw_ttm_unpopulate(struct ttm_device *bdev,
@@ -372,6 +380,10 @@ static void vmw_ttm_unpopulate(struct ttm_device *bdev,
 {
 	struct vmw_ttm_tt *vmw_tt = container_of(ttm, struct vmw_ttm_tt,
 						 dma_ttm);
+	bool external = (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) != 0;
+
+	if (external)
+		return;
 
 	vmw_ttm_unbind(bdev, ttm);
 
@@ -390,6 +402,7 @@ static struct ttm_tt *vmw_ttm_tt_create(struct ttm_buffer_object *bo,
 {
 	struct vmw_ttm_tt *vmw_be;
 	int ret;
+	bool external = bo->type == ttm_bo_type_sg;
 
 	vmw_be = kzalloc(sizeof(*vmw_be), GFP_KERNEL);
 	if (!vmw_be)
@@ -398,7 +411,10 @@ static struct ttm_tt *vmw_ttm_tt_create(struct ttm_buffer_object *bo,
 	vmw_be->dev_priv = vmw_priv_from_ttm(bo->bdev);
 	vmw_be->mob = NULL;
 
-	if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent)
+	if (external)
+		page_flags |= TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_EXTERNAL_MAPPABLE;
+
+	if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent || external)
 		ret = ttm_sg_tt_init(&vmw_be->dma_ttm, bo, page_flags,
 				     ttm_cached);
 	else
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
deleted file mode 100644
index 90097d04b45f..000000000000
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
+++ /dev/null
@@ -1,110 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0 OR MIT
-/**************************************************************************
- *
- * Copyright 2009-2011 VMware, Inc., Palo Alto, CA., USA
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sub license, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
- * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
- * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
- * USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- **************************************************************************/
-
-#include "vmwgfx_drv.h"
-
-static int vmw_bo_vm_lookup(struct ttm_device *bdev,
-				   struct drm_file *filp,
-				   unsigned long offset,
-				   unsigned long pages,
-				   struct ttm_buffer_object **p_bo)
-{
-	struct vmw_private *dev_priv = container_of(bdev, struct vmw_private, bdev);
-	struct drm_device *drm = &dev_priv->drm;
-	struct drm_vma_offset_node *node;
-	int ret;
-
-	*p_bo = NULL;
-
-	drm_vma_offset_lock_lookup(bdev->vma_manager);
-
-	node = drm_vma_offset_lookup_locked(bdev->vma_manager, offset, pages);
-	if (likely(node)) {
-		*p_bo = container_of(node, struct ttm_buffer_object,
-				  base.vma_node);
-		*p_bo = ttm_bo_get_unless_zero(*p_bo);
-	}
-
-	drm_vma_offset_unlock_lookup(bdev->vma_manager);
-
-	if (!*p_bo) {
-		drm_err(drm, "Could not find buffer object to map\n");
-		return -EINVAL;
-	}
-
-	if (!drm_vma_node_is_allowed(node, filp)) {
-		ret = -EACCES;
-		goto out_no_access;
-	}
-
-	return 0;
-out_no_access:
-	ttm_bo_put(*p_bo);
-	return ret;
-}
-
-int vmw_mmap(struct file *filp, struct vm_area_struct *vma)
-{
-	static const struct vm_operations_struct vmw_vm_ops = {
-		.pfn_mkwrite = vmw_bo_vm_mkwrite,
-		.page_mkwrite = vmw_bo_vm_mkwrite,
-		.fault = vmw_bo_vm_fault,
-		.open = ttm_bo_vm_open,
-		.close = ttm_bo_vm_close,
-	};
-	struct drm_file *file_priv = filp->private_data;
-	struct vmw_private *dev_priv = vmw_priv(file_priv->minor->dev);
-	struct ttm_device *bdev = &dev_priv->bdev;
-	struct ttm_buffer_object *bo;
-	int ret;
-
-	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET_START))
-		return -EINVAL;
-
-	ret = vmw_bo_vm_lookup(bdev, file_priv, vma->vm_pgoff, vma_pages(vma), &bo);
-	if (unlikely(ret != 0))
-		return ret;
-
-	ret = ttm_bo_mmap_obj(vma, bo);
-	if (unlikely(ret != 0))
-		goto out_unref;
-
-	vma->vm_ops = &vmw_vm_ops;
-
-	/* Use VM_PFNMAP rather than VM_MIXEDMAP if not a COW mapping */
-	if (!is_cow_mapping(vma->vm_flags))
-		vm_flags_mod(vma, VM_PFNMAP, VM_MIXEDMAP);
-
-	ttm_bo_put(bo); /* release extra ref taken by ttm_bo_mmap_obj() */
-
-	return 0;
-
-out_unref:
-	ttm_bo_put(bo);
-	return ret;
-}
-
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c
index aaacbdcbd742..e7625b3f71e0 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c
@@ -32,9 +32,6 @@
 
 #include <linux/slab.h>
 
-
-#define VMWGFX_VALIDATION_MEM_GRAN (16*PAGE_SIZE)
-
 /**
  * struct vmw_validation_bo_node - Buffer object validation metadata.
  * @base: Metadata used for TTM reservation- and validation.
@@ -112,20 +109,10 @@ void *vmw_validation_mem_alloc(struct vmw_validation_context *ctx,
 		return NULL;
 
 	if (ctx->mem_size_left < size) {
-		struct page *page;
-
-		if (ctx->vm && ctx->vm_size_left < PAGE_SIZE) {
-			ctx->vm_size_left += VMWGFX_VALIDATION_MEM_GRAN;
-			ctx->total_mem += VMWGFX_VALIDATION_MEM_GRAN;
-		}
-
-		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+		struct page *page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 		if (!page)
 			return NULL;
 
-		if (ctx->vm)
-			ctx->vm_size_left -= PAGE_SIZE;
-
 		list_add_tail(&page->lru, &ctx->page_list);
 		ctx->page_address = page_address(page);
 		ctx->mem_size_left = PAGE_SIZE;
@@ -155,10 +142,6 @@ static void vmw_validation_mem_free(struct vmw_validation_context *ctx)
 	}
 
 	ctx->mem_size_left = 0;
-	if (ctx->vm && ctx->total_mem) {
-		ctx->total_mem = 0;
-		ctx->vm_size_left = 0;
-	}
 }
 
 /**
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h
index 240ee0c4ebfd..353d837907d8 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h
@@ -52,10 +52,6 @@
  * buffer objects
  * @mem_size_left: Free memory left in the last page in @page_list
  * @page_address: Kernel virtual address of the last page in @page_list
- * @vm: A pointer to the memory reservation interface or NULL if no
- * memory reservation is needed.
- * @vm_size_left: Amount of reserved memory that so far has not been allocated.
- * @total_mem: Amount of reserved memory.
  */
 struct vmw_validation_context {
 	struct vmw_sw_context *sw_context;
@@ -68,9 +64,6 @@ struct vmw_validation_context {
 	unsigned int merge_dups;
 	unsigned int mem_size_left;
 	u8 *page_address;
-	struct vmw_validation_mem *vm;
-	size_t vm_size_left;
-	size_t total_mem;
 };
 
 struct vmw_bo;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.c
new file mode 100644
index 000000000000..7e93a45948f7
--- /dev/null
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.c
@@ -0,0 +1,632 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/**************************************************************************
+ *
+ * Copyright (c) 2024 Broadcom. All Rights Reserved. The term
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+#include "vmwgfx_vkms.h"
+
+#include "vmwgfx_bo.h"
+#include "vmwgfx_drv.h"
+#include "vmwgfx_kms.h"
+#include "vmwgfx_vkms.h"
+
+#include "vmw_surface_cache.h"
+
+#include <drm/drm_crtc.h>
+#include <drm/drm_debugfs_crc.h>
+#include <drm/drm_print.h>
+#include <drm/drm_vblank.h>
+
+#include <linux/crc32.h>
+#include <linux/delay.h>
+
+#define GUESTINFO_VBLANK  "guestinfo.vmwgfx.vkms_enable"
+
+static int
+vmw_surface_sync(struct vmw_private *vmw,
+		 struct vmw_surface *surf)
+{
+	int ret;
+	struct vmw_fence_obj *fence = NULL;
+	struct vmw_bo *bo = surf->res.guest_memory_bo;
+
+	vmw_resource_clean(&surf->res);
+
+	ret = ttm_bo_reserve(&bo->tbo, false, false, NULL);
+	if (ret != 0) {
+		drm_warn(&vmw->drm, "%s: failed reserve\n", __func__);
+		goto done;
+	}
+
+	ret = vmw_execbuf_fence_commands(NULL, vmw, &fence, NULL);
+	if (ret != 0) {
+		drm_warn(&vmw->drm, "%s: failed execbuf\n", __func__);
+		ttm_bo_unreserve(&bo->tbo);
+		goto done;
+	}
+
+	dma_fence_wait(&fence->base, false);
+	dma_fence_put(&fence->base);
+
+	ttm_bo_unreserve(&bo->tbo);
+done:
+	return ret;
+}
+
+static int
+compute_crc(struct drm_crtc *crtc,
+	    struct vmw_surface *surf,
+	    u32 *crc)
+{
+	u8 *mapped_surface;
+	struct vmw_bo *bo = surf->res.guest_memory_bo;
+	const struct SVGA3dSurfaceDesc *desc =
+		vmw_surface_get_desc(surf->metadata.format);
+	u32 row_pitch_bytes;
+	SVGA3dSize blocks;
+	u32 y;
+
+	*crc = 0;
+
+	vmw_surface_get_size_in_blocks(desc, &surf->metadata.base_size, &blocks);
+	row_pitch_bytes = blocks.width * desc->pitchBytesPerBlock;
+	WARN_ON(!bo);
+	mapped_surface = vmw_bo_map_and_cache(bo);
+
+	for (y = 0; y < blocks.height; y++) {
+		*crc = crc32_le(*crc, mapped_surface, row_pitch_bytes);
+		mapped_surface += row_pitch_bytes;
+	}
+
+	vmw_bo_unmap(bo);
+
+	return 0;
+}
+
+static void
+crc_generate_worker(struct work_struct *work)
+{
+	struct vmw_display_unit *du =
+		container_of(work, struct vmw_display_unit, vkms.crc_generator_work);
+	struct drm_crtc *crtc = &du->crtc;
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
+	bool crc_pending;
+	u64 frame_start, frame_end;
+	u32 crc32 = 0;
+	struct vmw_surface *surf = 0;
+	int ret;
+
+	spin_lock_irq(&du->vkms.crc_state_lock);
+	crc_pending = du->vkms.crc_pending;
+	spin_unlock_irq(&du->vkms.crc_state_lock);
+
+	/*
+	 * We raced with the vblank hrtimer and previous work already computed
+	 * the crc, nothing to do.
+	 */
+	if (!crc_pending)
+		return;
+
+	spin_lock_irq(&du->vkms.crc_state_lock);
+	surf = du->vkms.surface;
+	spin_unlock_irq(&du->vkms.crc_state_lock);
+
+	if (vmw_surface_sync(vmw, surf)) {
+		drm_warn(crtc->dev, "CRC worker wasn't able to sync the crc surface!\n");
+		return;
+	}
+
+	ret = compute_crc(crtc, surf, &crc32);
+	if (ret)
+		return;
+
+	spin_lock_irq(&du->vkms.crc_state_lock);
+	frame_start = du->vkms.frame_start;
+	frame_end = du->vkms.frame_end;
+	crc_pending = du->vkms.crc_pending;
+	du->vkms.frame_start = 0;
+	du->vkms.frame_end = 0;
+	du->vkms.crc_pending = false;
+	spin_unlock_irq(&du->vkms.crc_state_lock);
+
+	/*
+	 * The worker can fall behind the vblank hrtimer, make sure we catch up.
+	 */
+	while (frame_start <= frame_end)
+		drm_crtc_add_crc_entry(crtc, true, frame_start++, &crc32);
+}
+
+static enum hrtimer_restart
+vmw_vkms_vblank_simulate(struct hrtimer *timer)
+{
+	struct vmw_display_unit *du = container_of(timer, struct vmw_display_unit, vkms.timer);
+	struct drm_crtc *crtc = &du->crtc;
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
+	struct vmw_surface *surf = NULL;
+	u64 ret_overrun;
+	bool locked, ret;
+
+	ret_overrun = hrtimer_forward_now(&du->vkms.timer,
+					  du->vkms.period_ns);
+	if (ret_overrun != 1)
+		drm_dbg_driver(crtc->dev, "vblank timer missed %lld frames.\n",
+			       ret_overrun - 1);
+
+	locked = vmw_vkms_vblank_trylock(crtc);
+	ret = drm_crtc_handle_vblank(crtc);
+	WARN_ON(!ret);
+	if (!locked)
+		return HRTIMER_RESTART;
+	surf = du->vkms.surface;
+	vmw_vkms_unlock(crtc);
+
+	if (du->vkms.crc_enabled && surf) {
+		u64 frame = drm_crtc_accurate_vblank_count(crtc);
+
+		spin_lock(&du->vkms.crc_state_lock);
+		if (!du->vkms.crc_pending)
+			du->vkms.frame_start = frame;
+		else
+			drm_dbg_driver(crtc->dev,
+				       "crc worker falling behind, frame_start: %llu, frame_end: %llu\n",
+				       du->vkms.frame_start, frame);
+		du->vkms.frame_end = frame;
+		du->vkms.crc_pending = true;
+		spin_unlock(&du->vkms.crc_state_lock);
+
+		ret = queue_work(vmw->crc_workq, &du->vkms.crc_generator_work);
+		if (!ret)
+			drm_dbg_driver(crtc->dev, "Composer worker already queued\n");
+	}
+
+	return HRTIMER_RESTART;
+}
+
+void
+vmw_vkms_init(struct vmw_private *vmw)
+{
+	char buffer[64];
+	const size_t max_buf_len = sizeof(buffer) - 1;
+	size_t buf_len = max_buf_len;
+	int ret;
+
+	vmw->vkms_enabled = false;
+
+	ret = vmw_host_get_guestinfo(GUESTINFO_VBLANK, buffer, &buf_len);
+	if (ret || buf_len > max_buf_len)
+		return;
+	buffer[buf_len] = '\0';
+
+	ret = kstrtobool(buffer, &vmw->vkms_enabled);
+	if (!ret && vmw->vkms_enabled) {
+		ret = drm_vblank_init(&vmw->drm, VMWGFX_NUM_DISPLAY_UNITS);
+		vmw->vkms_enabled = (ret == 0);
+	}
+
+	vmw->crc_workq = alloc_ordered_workqueue("vmwgfx_crc_generator", 0);
+	if (!vmw->crc_workq) {
+		drm_warn(&vmw->drm, "crc workqueue allocation failed. Disabling vkms.");
+		vmw->vkms_enabled = false;
+	}
+	if (vmw->vkms_enabled)
+		drm_info(&vmw->drm, "VKMS enabled\n");
+}
+
+void
+vmw_vkms_cleanup(struct vmw_private *vmw)
+{
+	destroy_workqueue(vmw->crc_workq);
+}
+
+bool
+vmw_vkms_get_vblank_timestamp(struct drm_crtc *crtc,
+			      int *max_error,
+			      ktime_t *vblank_time,
+			      bool in_vblank_irq)
+{
+	struct drm_device *dev = crtc->dev;
+	struct vmw_private *vmw = vmw_priv(dev);
+	unsigned int pipe = crtc->index;
+	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+
+	if (!vmw->vkms_enabled)
+		return false;
+
+	if (!READ_ONCE(vblank->enabled)) {
+		*vblank_time = ktime_get();
+		return true;
+	}
+
+	*vblank_time = READ_ONCE(du->vkms.timer.node.expires);
+
+	if (WARN_ON(*vblank_time == vblank->time))
+		return true;
+
+	/*
+	 * To prevent races we roll the hrtimer forward before we do any
+	 * interrupt processing - this is how real hw works (the interrupt is
+	 * only generated after all the vblank registers are updated) and what
+	 * the vblank core expects. Therefore we need to always correct the
+	 * timestampe by one frame.
+	 */
+	*vblank_time -= du->vkms.period_ns;
+
+	return true;
+}
+
+int
+vmw_vkms_enable_vblank(struct drm_crtc *crtc)
+{
+	struct drm_device *dev = crtc->dev;
+	struct vmw_private *vmw = vmw_priv(dev);
+	unsigned int pipe = drm_crtc_index(crtc);
+	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+
+	if (!vmw->vkms_enabled)
+		return -EINVAL;
+
+	drm_calc_timestamping_constants(crtc, &crtc->mode);
+
+	hrtimer_init(&du->vkms.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	du->vkms.timer.function = &vmw_vkms_vblank_simulate;
+	du->vkms.period_ns = ktime_set(0, vblank->framedur_ns);
+	hrtimer_start(&du->vkms.timer, du->vkms.period_ns, HRTIMER_MODE_REL);
+
+	return 0;
+}
+
+void
+vmw_vkms_disable_vblank(struct drm_crtc *crtc)
+{
+	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
+
+	if (!vmw->vkms_enabled)
+		return;
+
+	hrtimer_cancel(&du->vkms.timer);
+	du->vkms.surface = NULL;
+	du->vkms.period_ns = ktime_set(0, 0);
+}
+
+enum vmw_vkms_lock_state {
+	VMW_VKMS_LOCK_UNLOCKED     = 0,
+	VMW_VKMS_LOCK_MODESET      = 1,
+	VMW_VKMS_LOCK_VBLANK       = 2
+};
+
+void
+vmw_vkms_crtc_init(struct drm_crtc *crtc)
+{
+	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+
+	atomic_set(&du->vkms.atomic_lock, VMW_VKMS_LOCK_UNLOCKED);
+	spin_lock_init(&du->vkms.crc_state_lock);
+
+	INIT_WORK(&du->vkms.crc_generator_work, crc_generate_worker);
+	du->vkms.surface = NULL;
+}
+
+void
+vmw_vkms_crtc_cleanup(struct drm_crtc *crtc)
+{
+	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+
+	WARN_ON(work_pending(&du->vkms.crc_generator_work));
+	hrtimer_cancel(&du->vkms.timer);
+}
+
+void
+vmw_vkms_crtc_atomic_begin(struct drm_crtc *crtc,
+			   struct drm_atomic_state *state)
+{
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
+
+	if (vmw->vkms_enabled)
+		vmw_vkms_modeset_lock(crtc);
+}
+
+void
+vmw_vkms_crtc_atomic_flush(struct drm_crtc *crtc,
+			   struct drm_atomic_state *state)
+{
+	unsigned long flags;
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
+
+	if (!vmw->vkms_enabled)
+		return;
+
+	if (crtc->state->event) {
+		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+
+		if (drm_crtc_vblank_get(crtc) != 0)
+			drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		else
+			drm_crtc_arm_vblank_event(crtc, crtc->state->event);
+
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+
+		crtc->state->event = NULL;
+	}
+
+	vmw_vkms_unlock(crtc);
+}
+
+void
+vmw_vkms_crtc_atomic_enable(struct drm_crtc *crtc,
+			    struct drm_atomic_state *state)
+{
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
+
+	if (vmw->vkms_enabled)
+		drm_crtc_vblank_on(crtc);
+}
+
+void
+vmw_vkms_crtc_atomic_disable(struct drm_crtc *crtc,
+			     struct drm_atomic_state *state)
+{
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
+
+	if (vmw->vkms_enabled)
+		drm_crtc_vblank_off(crtc);
+}
+
+static bool
+is_crc_supported(struct drm_crtc *crtc)
+{
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
+
+	if (!vmw->vkms_enabled)
+		return false;
+
+	if (vmw->active_display_unit != vmw_du_screen_target)
+		return false;
+
+	return true;
+}
+
+static const char * const pipe_crc_sources[] = {"auto"};
+
+static int
+crc_parse_source(const char *src_name,
+		 bool *enabled)
+{
+	int ret = 0;
+
+	if (!src_name) {
+		*enabled = false;
+	} else if (strcmp(src_name, "auto") == 0) {
+		*enabled = true;
+	} else {
+		*enabled = false;
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+const char *const *
+vmw_vkms_get_crc_sources(struct drm_crtc *crtc,
+			 size_t *count)
+{
+	*count = 0;
+	if (!is_crc_supported(crtc))
+		return NULL;
+
+	*count = ARRAY_SIZE(pipe_crc_sources);
+	return pipe_crc_sources;
+}
+
+int
+vmw_vkms_verify_crc_source(struct drm_crtc *crtc,
+			   const char *src_name,
+			   size_t *values_cnt)
+{
+	bool enabled;
+
+	if (!is_crc_supported(crtc))
+		return -EINVAL;
+
+	if (crc_parse_source(src_name, &enabled) < 0) {
+		drm_dbg_driver(crtc->dev, "unknown source '%s'\n", src_name);
+		return -EINVAL;
+	}
+
+	*values_cnt = 1;
+
+	return 0;
+}
+
+int
+vmw_vkms_set_crc_source(struct drm_crtc *crtc,
+			const char *src_name)
+{
+	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+	bool enabled, prev_enabled, locked;
+	int ret;
+
+	if (!is_crc_supported(crtc))
+		return -EINVAL;
+
+	ret = crc_parse_source(src_name, &enabled);
+
+	if (enabled)
+		drm_crtc_vblank_get(crtc);
+
+	locked = vmw_vkms_modeset_lock_relaxed(crtc);
+	prev_enabled = du->vkms.crc_enabled;
+	du->vkms.crc_enabled = enabled;
+	if (locked)
+		vmw_vkms_unlock(crtc);
+
+	if (prev_enabled)
+		drm_crtc_vblank_put(crtc);
+
+	return ret;
+}
+
+void
+vmw_vkms_set_crc_surface(struct drm_crtc *crtc,
+			 struct vmw_surface *surf)
+{
+	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
+
+	if (vmw->vkms_enabled) {
+		WARN_ON(atomic_read(&du->vkms.atomic_lock) != VMW_VKMS_LOCK_MODESET);
+		du->vkms.surface = surf;
+	}
+}
+
+/**
+ * vmw_vkms_lock_max_wait_ns - Return the max wait for the vkms lock
+ * @du: The vmw_display_unit from which to grab the vblank timings
+ *
+ * Returns the maximum wait time used to acquire the vkms lock. By
+ * default uses a time of a single frame and in case where vblank
+ * was not initialized for the display unit 1/60th of a second.
+ */
+static inline u64
+vmw_vkms_lock_max_wait_ns(struct vmw_display_unit *du)
+{
+	s64 nsecs = ktime_to_ns(du->vkms.period_ns);
+
+	return  (nsecs > 0) ? nsecs : 16666666;
+}
+
+/**
+ * vmw_vkms_modeset_lock - Protects access to crtc during modeset
+ * @crtc: The crtc to lock for vkms
+ *
+ * This function prevents the VKMS timers/callbacks from being called
+ * while a modeset operation is in process. We don't want the callbacks
+ * e.g. the vblank simulator to be trying to access incomplete state
+ * so we need to make sure they execute only when the modeset has
+ * finished.
+ *
+ * Normally this would have been done with a spinlock but locking the
+ * entire atomic modeset with vmwgfx is impossible because kms prepare
+ * executes non-atomic ops (e.g. vmw_validation_prepare holds a mutex to
+ * guard various bits of state). Which means that we need to synchronize
+ * atomic context (the vblank handler) with the non-atomic entirity
+ * of kms - so use an atomic_t to track which part of vkms has access
+ * to the basic vkms state.
+ */
+void
+vmw_vkms_modeset_lock(struct drm_crtc *crtc)
+{
+	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+	const u64 nsecs_delay = 10;
+	const u64 MAX_NSECS_DELAY = vmw_vkms_lock_max_wait_ns(du);
+	u64 total_delay = 0;
+	int ret;
+
+	do {
+		ret = atomic_cmpxchg(&du->vkms.atomic_lock,
+				     VMW_VKMS_LOCK_UNLOCKED,
+				     VMW_VKMS_LOCK_MODESET);
+		if (ret == VMW_VKMS_LOCK_UNLOCKED || total_delay >= MAX_NSECS_DELAY)
+			break;
+		ndelay(nsecs_delay);
+		total_delay += nsecs_delay;
+	} while (1);
+
+	if (total_delay >= MAX_NSECS_DELAY) {
+		drm_warn(crtc->dev, "VKMS lock expired! total_delay = %lld, ret = %d, cur = %d\n",
+			 total_delay, ret, atomic_read(&du->vkms.atomic_lock));
+	}
+}
+
+/**
+ * vmw_vkms_modeset_lock_relaxed - Protects access to crtc during modeset
+ * @crtc: The crtc to lock for vkms
+ *
+ * Much like vmw_vkms_modeset_lock except that when the crtc is currently
+ * in a modeset it will return immediately.
+ *
+ * Returns true if actually locked vkms to modeset or false otherwise.
+ */
+bool
+vmw_vkms_modeset_lock_relaxed(struct drm_crtc *crtc)
+{
+	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+	const u64 nsecs_delay = 10;
+	const u64 MAX_NSECS_DELAY = vmw_vkms_lock_max_wait_ns(du);
+	u64 total_delay = 0;
+	int ret;
+
+	do {
+		ret = atomic_cmpxchg(&du->vkms.atomic_lock,
+				     VMW_VKMS_LOCK_UNLOCKED,
+				     VMW_VKMS_LOCK_MODESET);
+		if (ret == VMW_VKMS_LOCK_UNLOCKED ||
+		    ret == VMW_VKMS_LOCK_MODESET ||
+		    total_delay >= MAX_NSECS_DELAY)
+			break;
+		ndelay(nsecs_delay);
+		total_delay += nsecs_delay;
+	} while (1);
+
+	if (total_delay >= MAX_NSECS_DELAY) {
+		drm_warn(crtc->dev, "VKMS relaxed lock expired!\n");
+		return false;
+	}
+
+	return ret == VMW_VKMS_LOCK_UNLOCKED;
+}
+
+/**
+ * vmw_vkms_vblank_trylock - Protects access to crtc during vblank
+ * @crtc: The crtc to lock for vkms
+ *
+ * Tries to lock vkms for vblank, returns immediately.
+ *
+ * Returns true if locked vkms to vblank or false otherwise.
+ */
+bool
+vmw_vkms_vblank_trylock(struct drm_crtc *crtc)
+{
+	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+	u32 ret;
+
+	ret = atomic_cmpxchg(&du->vkms.atomic_lock,
+			     VMW_VKMS_LOCK_UNLOCKED,
+			     VMW_VKMS_LOCK_VBLANK);
+
+	return ret == VMW_VKMS_LOCK_UNLOCKED;
+}
+
+void
+vmw_vkms_unlock(struct drm_crtc *crtc)
+{
+	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
+
+	/* Release flag; mark it as unlocked. */
+	atomic_set(&du->vkms.atomic_lock, VMW_VKMS_LOCK_UNLOCKED);
+}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.h
new file mode 100644
index 000000000000..69ddd33a8444
--- /dev/null
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.h
@@ -0,0 +1,75 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/**************************************************************************
+ *
+ * Copyright (c) 2024 Broadcom. All Rights Reserved. The term
+ * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+
+#ifndef VMWGFX_VKMS_H_
+#define VMWGFX_VKMS_H_
+
+#include <linux/hrtimer_types.h>
+#include <linux/types.h>
+
+struct drm_atomic_state;
+struct drm_crtc;
+struct vmw_private;
+struct vmw_surface;
+
+void vmw_vkms_init(struct vmw_private *vmw);
+void vmw_vkms_cleanup(struct vmw_private *vmw);
+
+void vmw_vkms_modeset_lock(struct drm_crtc *crtc);
+bool vmw_vkms_modeset_lock_relaxed(struct drm_crtc *crtc);
+bool vmw_vkms_vblank_trylock(struct drm_crtc *crtc);
+void vmw_vkms_unlock(struct drm_crtc *crtc);
+
+bool vmw_vkms_get_vblank_timestamp(struct drm_crtc *crtc,
+				   int *max_error,
+				   ktime_t *vblank_time,
+				   bool in_vblank_irq);
+int vmw_vkms_enable_vblank(struct drm_crtc *crtc);
+void vmw_vkms_disable_vblank(struct drm_crtc *crtc);
+
+void vmw_vkms_crtc_init(struct drm_crtc *crtc);
+void vmw_vkms_crtc_cleanup(struct drm_crtc *crtc);
+void  vmw_vkms_crtc_atomic_begin(struct drm_crtc *crtc,
+				 struct drm_atomic_state *state);
+void vmw_vkms_crtc_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *state);
+void vmw_vkms_crtc_atomic_enable(struct drm_crtc *crtc,
+				 struct drm_atomic_state *state);
+void vmw_vkms_crtc_atomic_disable(struct drm_crtc *crtc,
+				  struct drm_atomic_state *state);
+
+const char *const *vmw_vkms_get_crc_sources(struct drm_crtc *crtc,
+					    size_t *count);
+int vmw_vkms_verify_crc_source(struct drm_crtc *crtc,
+			       const char *src_name,
+			       size_t *values_cnt);
+int vmw_vkms_set_crc_source(struct drm_crtc *crtc,
+			    const char *src_name);
+void vmw_vkms_set_crc_surface(struct drm_crtc *crtc,
+			      struct vmw_surface *surf);
+
+#endif
diff --git a/drivers/gpu/drm/xe/Kconfig b/drivers/gpu/drm/xe/Kconfig
index 1a556d087e63..63f1e2d1649f 100644
--- a/drivers/gpu/drm/xe/Kconfig
+++ b/drivers/gpu/drm/xe/Kconfig
@@ -26,6 +26,7 @@ config DRM_XE
 	select INPUT if ACPI
 	select ACPI_VIDEO if X86 && ACPI
 	select ACPI_BUTTON if ACPI
+	select X86_PLATFORM_DEVICES if X86 && ACPI
 	select ACPI_WMI if X86 && ACPI
 	select SYNC_FILE
 	select IOSF_MBI
@@ -41,6 +42,7 @@ config DRM_XE
 	select MMU_NOTIFIER
 	select WANT_DEV_COREDUMP
 	select AUXILIARY_BUS
+	select HMM_MIRROR
 	help
 	  Experimental driver for Intel Xe series GPUs
 
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
index c29a850859ad..b165bbf52aef 100644
--- a/drivers/gpu/drm/xe/Makefile
+++ b/drivers/gpu/drm/xe/Makefile
@@ -32,7 +32,7 @@ endif
 # Enable -Werror in CI and development
 subdir-ccflags-$(CONFIG_DRM_XE_WERROR) += -Werror
 
-subdir-ccflags-y += -I$(obj) -I$(srctree)/$(src)
+subdir-ccflags-y += -I$(obj) -I$(src)
 
 # generated sources
 hostprogs := xe_gen_wa_oob
@@ -43,12 +43,13 @@ quiet_cmd_wa_oob = GEN     $(notdir $(generated_oob))
       cmd_wa_oob = mkdir -p $(@D); $^ $(generated_oob)
 
 $(obj)/generated/%_wa_oob.c $(obj)/generated/%_wa_oob.h: $(obj)/xe_gen_wa_oob \
-		 $(srctree)/$(src)/xe_wa_oob.rules
+		 $(src)/xe_wa_oob.rules
 	$(call cmd,wa_oob)
 
 uses_generated_oob := \
 	$(obj)/xe_gsc.o \
 	$(obj)/xe_guc.o \
+	$(obj)/xe_guc_ads.o \
 	$(obj)/xe_migrate.o \
 	$(obj)/xe_ring_ops.o \
 	$(obj)/xe_vm.o \
@@ -97,6 +98,8 @@ xe-y += xe_bb.o \
 	xe_guc_db_mgr.o \
 	xe_guc_debugfs.o \
 	xe_guc_hwconfig.o \
+	xe_guc_id_mgr.o \
+	xe_guc_klv_helpers.o \
 	xe_guc_log.o \
 	xe_guc_pc.o \
 	xe_guc_submit.o \
@@ -145,6 +148,8 @@ xe-y += xe_bb.o \
 	xe_wa.o \
 	xe_wopcm.o
 
+xe-$(CONFIG_HMM_MIRROR) += xe_hmm.o
+
 # graphics hardware monitoring (HWMON) support
 xe-$(CONFIG_HWMON) += xe_hwmon.o
 
@@ -155,9 +160,14 @@ xe-y += \
 	xe_sriov.o
 
 xe-$(CONFIG_PCI_IOV) += \
+	xe_gt_sriov_pf.o \
+	xe_gt_sriov_pf_config.o \
+	xe_gt_sriov_pf_control.o \
+	xe_gt_sriov_pf_policy.o \
 	xe_lmtt.o \
 	xe_lmtt_2l.o \
-	xe_lmtt_ml.o
+	xe_lmtt_ml.o \
+	xe_sriov_pf.o
 
 # include helpers for tests even when XE is built-in
 ifdef CONFIG_DRM_XE_KUNIT_TEST
@@ -166,15 +176,12 @@ endif
 
 # i915 Display compat #defines and #includes
 subdir-ccflags-$(CONFIG_DRM_XE_DISPLAY) += \
-	-I$(srctree)/$(src)/display/ext \
-	-I$(srctree)/$(src)/compat-i915-headers \
+	-I$(src)/display/ext \
+	-I$(src)/compat-i915-headers \
 	-I$(srctree)/drivers/gpu/drm/i915/display/ \
 	-Ddrm_i915_gem_object=xe_bo \
 	-Ddrm_i915_private=xe_device
 
-CFLAGS_i915-display/intel_fbdev.o = -Wno-override-init
-CFLAGS_i915-display/intel_display_device.o = -Wno-override-init
-
 # Rule to build SOC code shared with i915
 $(obj)/i915-soc/%.o: $(srctree)/drivers/gpu/drm/i915/soc/%.c FORCE
 	$(call cmd,force_checksrc)
@@ -257,6 +264,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/intel_global_state.o \
 	i915-display/intel_gmbus.o \
 	i915-display/intel_hdcp.o \
+	i915-display/intel_hdcp_gsc_message.o \
 	i915-display/intel_hdmi.o \
 	i915-display/intel_hotplug.o \
 	i915-display/intel_hotplug_irq.o \
@@ -278,6 +286,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/intel_vdsc.o \
 	i915-display/intel_vga.o \
 	i915-display/intel_vrr.o \
+	i915-display/intel_dmc_wl.o \
 	i915-display/intel_wm.o \
 	i915-display/skl_scaler.o \
 	i915-display/skl_universal_plane.o \
@@ -310,7 +319,7 @@ ifneq ($(CONFIG_DRM_XE_DISPLAY),y)
 endif
 
 always-$(CONFIG_DRM_XE_WERROR) += \
-	$(patsubst %.h,%.hdrtest, $(shell cd $(srctree)/$(src) && find * -name '*.h' $(hdrtest_find_args)))
+	$(patsubst %.h,%.hdrtest, $(shell cd $(src) && find * -name '*.h' $(hdrtest_find_args)))
 
 quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@)
       cmd_hdrtest = $(CC) -DHDRTEST $(filter-out $(CFLAGS_GCOV), $(c_flags)) -S -o /dev/null -x c /dev/null -include $<; touch $@
diff --git a/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h
index 5496a5890847..c1ad09b36453 100644
--- a/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h
@@ -3,8 +3,8 @@
  * Copyright © 2023 Intel Corporation
  */
 
-#ifndef _GUC_ACTIONS_PF_ABI_H
-#define _GUC_ACTIONS_PF_ABI_H
+#ifndef _ABI_GUC_ACTIONS_SRIOV_ABI_H
+#define _ABI_GUC_ACTIONS_SRIOV_ABI_H
 
 #include "guc_communication_ctb_abi.h"
 
@@ -171,4 +171,200 @@
 #define VF2GUC_RELAY_TO_PF_REQUEST_MSG_n_RELAY_DATAx	GUC_HXG_REQUEST_MSG_n_DATAn
 #define VF2GUC_RELAY_TO_PF_REQUEST_MSG_NUM_RELAY_DATA	GUC_RELAY_MSG_MAX_LEN
 
+/**
+ * DOC: GUC2PF_VF_STATE_NOTIFY
+ *
+ * The GUC2PF_VF_STATE_NOTIFY message is used by the GuC to notify PF about change
+ * of the VF state.
+ *
+ * This G2H message is sent as `CTB HXG Message`_.
+ *
+ *  +---+-------+--------------------------------------------------------------+
+ *  |   | Bits  | Description                                                  |
+ *  +===+=======+==============================================================+
+ *  | 0 |    31 | ORIGIN = GUC_HXG_ORIGIN_GUC_                                 |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 30:28 | TYPE = GUC_HXG_TYPE_EVENT_                                   |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 27:16 | DATA0 = MBZ                                                  |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   |  15:0 | ACTION = _`GUC_ACTION_GUC2PF_VF_STATE_NOTIFY` = 0x5106       |
+ *  +---+-------+--------------------------------------------------------------+
+ *  | 1 |  31:0 | DATA1 = **VFID** - VF identifier                             |
+ *  +---+-------+--------------------------------------------------------------+
+ *  | 2 |  31:0 | DATA2 = **EVENT** - notification event:                      |
+ *  |   |       |                                                              |
+ *  |   |       |   - _`GUC_PF_NOTIFY_VF_ENABLE` = 1 (only if VFID = 0)        |
+ *  |   |       |   - _`GUC_PF_NOTIFY_VF_FLR` = 1                              |
+ *  |   |       |   - _`GUC_PF_NOTIFY_VF_FLR_DONE` = 2                         |
+ *  |   |       |   - _`GUC_PF_NOTIFY_VF_PAUSE_DONE` = 3                       |
+ *  |   |       |   - _`GUC_PF_NOTIFY_VF_FIXUP_DONE` = 4                       |
+ *  +---+-------+--------------------------------------------------------------+
+ */
+#define GUC_ACTION_GUC2PF_VF_STATE_NOTIFY		0x5106u
+
+#define GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_LEN		(GUC_HXG_EVENT_MSG_MIN_LEN + 2u)
+#define GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_0_MBZ		GUC_HXG_EVENT_MSG_0_DATA0
+#define GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_1_VFID		GUC_HXG_EVENT_MSG_n_DATAn
+#define GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_2_EVENT	GUC_HXG_EVENT_MSG_n_DATAn
+#define   GUC_PF_NOTIFY_VF_ENABLE			1u
+#define   GUC_PF_NOTIFY_VF_FLR				1u
+#define   GUC_PF_NOTIFY_VF_FLR_DONE			2u
+#define   GUC_PF_NOTIFY_VF_PAUSE_DONE			3u
+#define   GUC_PF_NOTIFY_VF_FIXUP_DONE			4u
+
+/**
+ * DOC: PF2GUC_UPDATE_VGT_POLICY
+ *
+ * This message is used by the PF to set `GuC VGT Policy KLVs`_.
+ *
+ * This message must be sent as `CTB HXG Message`_.
+ *
+ *  +---+-------+--------------------------------------------------------------+
+ *  |   | Bits  | Description                                                  |
+ *  +===+=======+==============================================================+
+ *  | 0 |    31 | ORIGIN = GUC_HXG_ORIGIN_HOST_                                |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_                                 |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 27:16 | MBZ                                                          |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   |  15:0 | ACTION = _`GUC_ACTION_PF2GUC_UPDATE_VGT_POLICY` = 0x5502     |
+ *  +---+-------+--------------------------------------------------------------+
+ *  | 1 |  31:0 | **CFG_ADDR_LO** - dword aligned GGTT offset that             |
+ *  |   |       | represents the start of `GuC VGT Policy KLVs`_ list.         |
+ *  +---+-------+--------------------------------------------------------------+
+ *  | 2 |  31:0 | **CFG_ADDR_HI** - upper 32 bits of above offset.             |
+ *  +---+-------+--------------------------------------------------------------+
+ *  | 3 |  31:0 | **CFG_SIZE** - size (in dwords) of the config buffer         |
+ *  +---+-------+--------------------------------------------------------------+
+ *
+ *  +---+-------+--------------------------------------------------------------+
+ *  |   | Bits  | Description                                                  |
+ *  +===+=======+==============================================================+
+ *  | 0 |    31 | ORIGIN = GUC_HXG_ORIGIN_GUC_                                 |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_                        |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   |  27:0 | **COUNT** - number of KLVs successfully applied              |
+ *  +---+-------+--------------------------------------------------------------+
+ */
+#define GUC_ACTION_PF2GUC_UPDATE_VGT_POLICY			0x5502u
+
+#define PF2GUC_UPDATE_VGT_POLICY_REQUEST_MSG_LEN		(GUC_HXG_REQUEST_MSG_MIN_LEN + 3u)
+#define PF2GUC_UPDATE_VGT_POLICY_REQUEST_MSG_0_MBZ		GUC_HXG_REQUEST_MSG_0_DATA0
+#define PF2GUC_UPDATE_VGT_POLICY_REQUEST_MSG_1_CFG_ADDR_LO	GUC_HXG_REQUEST_MSG_n_DATAn
+#define PF2GUC_UPDATE_VGT_POLICY_REQUEST_MSG_2_CFG_ADDR_HI	GUC_HXG_REQUEST_MSG_n_DATAn
+#define PF2GUC_UPDATE_VGT_POLICY_REQUEST_MSG_3_CFG_SIZE		GUC_HXG_REQUEST_MSG_n_DATAn
+
+#define PF2GUC_UPDATE_VGT_POLICY_RESPONSE_MSG_LEN		GUC_HXG_RESPONSE_MSG_MIN_LEN
+#define PF2GUC_UPDATE_VGT_POLICY_RESPONSE_MSG_0_COUNT		GUC_HXG_RESPONSE_MSG_0_DATA0
+
+/**
+ * DOC: PF2GUC_UPDATE_VF_CFG
+ *
+ * The `PF2GUC_UPDATE_VF_CFG`_ message is used by PF to provision single VF in GuC.
+ *
+ * This message must be sent as `CTB HXG Message`_.
+ *
+ *  +---+-------+--------------------------------------------------------------+
+ *  |   | Bits  | Description                                                  |
+ *  +===+=======+==============================================================+
+ *  | 0 |    31 | ORIGIN = GUC_HXG_ORIGIN_HOST_                                |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_                                 |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 27:16 | MBZ                                                          |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   |  15:0 | ACTION = _`GUC_ACTION_PF2GUC_UPDATE_VF_CFG` = 0x5503         |
+ *  +---+-------+--------------------------------------------------------------+
+ *  | 1 |  31:0 | **VFID** - identifier of the VF that the KLV                 |
+ *  |   |       | configurations are being applied to                          |
+ *  +---+-------+--------------------------------------------------------------+
+ *  | 2 |  31:0 | **CFG_ADDR_LO** - dword aligned GGTT offset that represents  |
+ *  |   |       | the start of a list of virtualization related KLV configs    |
+ *  |   |       | that are to be applied to the VF.                            |
+ *  |   |       | If this parameter is zero, the list is not parsed.           |
+ *  |   |       | If full configs address parameter is zero and configs_size is|
+ *  |   |       | zero associated VF config shall be reset to its default state|
+ *  +---+-------+--------------------------------------------------------------+
+ *  | 3 |  31:0 | **CFG_ADDR_HI** - upper 32 bits of configs address.          |
+ *  +---+-------+--------------------------------------------------------------+
+ *  | 4 |  31:0 | **CFG_SIZE** - size (in dwords) of the config buffer         |
+ *  +---+-------+--------------------------------------------------------------+
+ *
+ *  +---+-------+--------------------------------------------------------------+
+ *  |   | Bits  | Description                                                  |
+ *  +===+=======+==============================================================+
+ *  | 0 |    31 | ORIGIN = GUC_HXG_ORIGIN_GUC_                                 |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_                        |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   |  27:0 | **COUNT** - number of KLVs successfully applied              |
+ *  +---+-------+--------------------------------------------------------------+
+ */
+#define GUC_ACTION_PF2GUC_UPDATE_VF_CFG			0x5503u
+
+#define PF2GUC_UPDATE_VF_CFG_REQUEST_MSG_LEN		(GUC_HXG_REQUEST_MSG_MIN_LEN + 4u)
+#define PF2GUC_UPDATE_VF_CFG_REQUEST_MSG_0_MBZ		GUC_HXG_REQUEST_MSG_0_DATA0
+#define PF2GUC_UPDATE_VF_CFG_REQUEST_MSG_1_VFID		GUC_HXG_REQUEST_MSG_n_DATAn
+#define PF2GUC_UPDATE_VF_CFG_REQUEST_MSG_2_CFG_ADDR_LO	GUC_HXG_REQUEST_MSG_n_DATAn
+#define PF2GUC_UPDATE_VF_CFG_REQUEST_MSG_3_CFG_ADDR_HI	GUC_HXG_REQUEST_MSG_n_DATAn
+#define PF2GUC_UPDATE_VF_CFG_REQUEST_MSG_4_CFG_SIZE	GUC_HXG_REQUEST_MSG_n_DATAn
+
+#define PF2GUC_UPDATE_VF_CFG_RESPONSE_MSG_LEN		GUC_HXG_RESPONSE_MSG_MIN_LEN
+#define PF2GUC_UPDATE_VF_CFG_RESPONSE_MSG_0_COUNT	GUC_HXG_RESPONSE_MSG_0_DATA0
+
+/**
+ * DOC: PF2GUC_VF_CONTROL
+ *
+ * The PF2GUC_VF_CONTROL message is used by the PF to trigger VF state change
+ * maintained by the GuC.
+ *
+ * This H2G message must be sent as `CTB HXG Message`_.
+ *
+ *  +---+-------+--------------------------------------------------------------+
+ *  |   | Bits  | Description                                                  |
+ *  +===+=======+==============================================================+
+ *  | 0 |    31 | ORIGIN = GUC_HXG_ORIGIN_HOST_                                |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_                                 |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 27:16 | DATA0 = MBZ                                                  |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   |  15:0 | ACTION = _`GUC_ACTION_PF2GUC_VF_CONTROL_CMD` = 0x5506        |
+ *  +---+-------+--------------------------------------------------------------+
+ *  | 1 |  31:0 | DATA1 = **VFID** - VF identifier                             |
+ *  +---+-------+--------------------------------------------------------------+
+ *  | 2 |  31:0 | DATA2 = **COMMAND** - control command:                       |
+ *  |   |       |                                                              |
+ *  |   |       |   - _`GUC_PF_TRIGGER_VF_PAUSE` = 1                           |
+ *  |   |       |   - _`GUC_PF_TRIGGER_VF_RESUME` = 2                          |
+ *  |   |       |   - _`GUC_PF_TRIGGER_VF_STOP` = 3                            |
+ *  |   |       |   - _`GUC_PF_TRIGGER_VF_FLR_START` = 4                       |
+ *  |   |       |   - _`GUC_PF_TRIGGER_VF_FLR_FINISH` = 5                      |
+ *  +---+-------+--------------------------------------------------------------+
+ *
+ *  +---+-------+--------------------------------------------------------------+
+ *  |   | Bits  | Description                                                  |
+ *  +===+=======+==============================================================+
+ *  | 0 |    31 | ORIGIN = GUC_HXG_ORIGIN_GUC_                                 |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_                        |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   |  27:0 | DATA0 = MBZ                                                  |
+ *  +---+-------+--------------------------------------------------------------+
+ */
+#define GUC_ACTION_PF2GUC_VF_CONTROL			0x5506u
+
+#define PF2GUC_VF_CONTROL_REQUEST_MSG_LEN		(GUC_HXG_EVENT_MSG_MIN_LEN + 2u)
+#define PF2GUC_VF_CONTROL_REQUEST_MSG_0_MBZ		GUC_HXG_EVENT_MSG_0_DATA0
+#define PF2GUC_VF_CONTROL_REQUEST_MSG_1_VFID		GUC_HXG_EVENT_MSG_n_DATAn
+#define PF2GUC_VF_CONTROL_REQUEST_MSG_2_COMMAND		GUC_HXG_EVENT_MSG_n_DATAn
+#define   GUC_PF_TRIGGER_VF_PAUSE			1u
+#define   GUC_PF_TRIGGER_VF_RESUME			2u
+#define   GUC_PF_TRIGGER_VF_STOP			3u
+#define   GUC_PF_TRIGGER_VF_FLR_START			4u
+#define   GUC_PF_TRIGGER_VF_FLR_FINISH			5u
+
 #endif
diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
index 0400bc0fccdc..511cf974d585 100644
--- a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
@@ -319,4 +319,14 @@ enum  {
 #define GUC_KLV_VF_CFG_BEGIN_CONTEXT_ID_KEY	0x8a0b
 #define GUC_KLV_VF_CFG_BEGIN_CONTEXT_ID_LEN	1u
 
+/*
+ * Workaround keys:
+ */
+enum xe_guc_klv_ids {
+	GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED				= 0x9002,
+	GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING				= 0x9005,
+	GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE				= 0x9007,
+	GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE			= 0x9008,
+};
+
 #endif
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h
index 420eba0e4be0..cd4632276141 100644
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h
+++ b/drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h
@@ -19,14 +19,12 @@
 #include "xe_bo.h"
 #include "xe_pm.h"
 #include "xe_step.h"
-#include "i915_gem.h"
 #include "i915_gem_stolen.h"
 #include "i915_gpu_error.h"
 #include "i915_reg_defs.h"
 #include "i915_utils.h"
 #include "intel_gt_types.h"
 #include "intel_step.h"
-#include "intel_uc_fw.h"
 #include "intel_uncore.h"
 #include "intel_runtime_pm.h"
 #include <linux/pm_runtime.h>
@@ -41,12 +39,8 @@ static inline struct drm_i915_private *kdev_to_i915(struct device *kdev)
 	return dev_get_drvdata(kdev);
 }
 
-
-#define INTEL_JASPERLAKE 0
-#define INTEL_ELKHARTLAKE 0
 #define IS_PLATFORM(xe, x) ((xe)->info.platform == x)
 #define INTEL_INFO(dev_priv)	(&((dev_priv)->info))
-#define INTEL_DEVID(dev_priv)	((dev_priv)->info.devid)
 #define IS_I830(dev_priv)	(dev_priv && 0)
 #define IS_I845G(dev_priv)	(dev_priv && 0)
 #define IS_I85X(dev_priv)	(dev_priv && 0)
@@ -84,12 +78,12 @@ static inline struct drm_i915_private *kdev_to_i915(struct device *kdev)
 #define IS_ROCKETLAKE(dev_priv)	IS_PLATFORM(dev_priv, XE_ROCKETLAKE)
 #define IS_DG1(dev_priv)        IS_PLATFORM(dev_priv, XE_DG1)
 #define IS_ALDERLAKE_S(dev_priv) IS_PLATFORM(dev_priv, XE_ALDERLAKE_S)
-#define IS_ALDERLAKE_P(dev_priv) IS_PLATFORM(dev_priv, XE_ALDERLAKE_P)
-#define IS_XEHPSDV(dev_priv) (dev_priv && 0)
+#define IS_ALDERLAKE_P(dev_priv) (IS_PLATFORM(dev_priv, XE_ALDERLAKE_P) || \
+				  IS_PLATFORM(dev_priv, XE_ALDERLAKE_N))
 #define IS_DG2(dev_priv)	IS_PLATFORM(dev_priv, XE_DG2)
-#define IS_PONTEVECCHIO(dev_priv) IS_PLATFORM(dev_priv, XE_PVC)
 #define IS_METEORLAKE(dev_priv) IS_PLATFORM(dev_priv, XE_METEORLAKE)
 #define IS_LUNARLAKE(dev_priv) IS_PLATFORM(dev_priv, XE_LUNARLAKE)
+#define IS_BATTLEMAGE(dev_priv)  IS_PLATFORM(dev_priv, XE_BATTLEMAGE)
 
 #define IS_HASWELL_ULT(dev_priv) (dev_priv && 0)
 #define IS_BROADWELL_ULT(dev_priv) (dev_priv && 0)
@@ -97,19 +91,12 @@ static inline struct drm_i915_private *kdev_to_i915(struct device *kdev)
 
 #define IP_VER(ver, rel)                ((ver) << 8 | (rel))
 
-#define INTEL_DISPLAY_ENABLED(xe) (HAS_DISPLAY((xe)) && !intel_opregion_headless_sku((xe)))
-
-#define IS_GRAPHICS_VER(xe, first, last) \
-	((xe)->info.graphics_verx100 >= first * 100 && \
-	 (xe)->info.graphics_verx100 <= (last*100 + 99))
 #define IS_MOBILE(xe) (xe && 0)
-#define HAS_LLC(xe) (!IS_DGFX((xe)))
 
 #define HAS_GMD_ID(xe) GRAPHICS_VERx100(xe) >= 1270
 
 /* Workarounds not handled yet */
 #define IS_DISPLAY_STEP(xe, first, last) ({u8 __step = (xe)->info.step.display; first <= __step && __step <= last; })
-#define IS_GRAPHICS_STEP(xe, first, last) ({u8 __step = (xe)->info.step.graphics; first <= __step && __step <= last; })
 
 #define IS_LP(xe) (0)
 #define IS_GEN9_LP(xe) (0)
@@ -126,27 +113,6 @@ static inline struct drm_i915_private *kdev_to_i915(struct device *kdev)
 #define IS_KABYLAKE_ULT(xe) (xe && 0)
 #define IS_SKYLAKE_ULT(xe) (xe && 0)
 
-#define IS_DG1_GRAPHICS_STEP(xe, first, last) (IS_DG1(xe) && IS_GRAPHICS_STEP(xe, first, last))
-#define IS_DG2_GRAPHICS_STEP(xe, variant, first, last) \
-	((xe)->info.subplatform == XE_SUBPLATFORM_DG2_ ## variant && \
-	 IS_GRAPHICS_STEP(xe, first, last))
-#define IS_XEHPSDV_GRAPHICS_STEP(xe, first, last) (IS_XEHPSDV(xe) && IS_GRAPHICS_STEP(xe, first, last))
-
-/* XXX: No basedie stepping support yet */
-#define IS_PVC_BD_STEP(xe, first, last) (!WARN_ON(1) && IS_PONTEVECCHIO(xe))
-
-#define IS_TIGERLAKE_DISPLAY_STEP(xe, first, last) (IS_TIGERLAKE(xe) && IS_DISPLAY_STEP(xe, first, last))
-#define IS_ROCKETLAKE_DISPLAY_STEP(xe, first, last) (IS_ROCKETLAKE(xe) && IS_DISPLAY_STEP(xe, first, last))
-#define IS_DG1_DISPLAY_STEP(xe, first, last) (IS_DG1(xe) && IS_DISPLAY_STEP(xe, first, last))
-#define IS_DG2_DISPLAY_STEP(xe, first, last) (IS_DG2(xe) && IS_DISPLAY_STEP(xe, first, last))
-#define IS_ADLP_DISPLAY_STEP(xe, first, last) (IS_ALDERLAKE_P(xe) && IS_DISPLAY_STEP(xe, first, last))
-#define IS_ADLS_DISPLAY_STEP(xe, first, last) (IS_ALDERLAKE_S(xe) && IS_DISPLAY_STEP(xe, first, last))
-#define IS_JSL_EHL_DISPLAY_STEP(xe, first, last) (IS_JSL_EHL(xe) && IS_DISPLAY_STEP(xe, first, last))
-#define IS_MTL_DISPLAY_STEP(xe, first, last) (IS_METEORLAKE(xe) && IS_DISPLAY_STEP(xe, first, last))
-
-/* FIXME: Add subplatform here */
-#define IS_MTL_GRAPHICS_STEP(xe, sub, first, last) (IS_METEORLAKE(xe) && IS_DISPLAY_STEP(xe, first, last))
-
 #define IS_DG2_G10(xe) ((xe)->info.subplatform == XE_SUBPLATFORM_DG2_G10)
 #define IS_DG2_G11(xe) ((xe)->info.subplatform == XE_SUBPLATFORM_DG2_G11)
 #define IS_DG2_G12(xe) ((xe)->info.subplatform == XE_SUBPLATFORM_DG2_G12)
@@ -154,30 +120,31 @@ static inline struct drm_i915_private *kdev_to_i915(struct device *kdev)
 #define IS_ICL_WITH_PORT_F(xe) (xe && 0)
 #define HAS_FLAT_CCS(xe) (xe_device_has_flat_ccs(xe))
 #define to_intel_bo(x) gem_to_xe_bo((x))
-#define mkwrite_device_info(xe) (INTEL_INFO(xe))
 
 #define HAS_128_BYTE_Y_TILING(xe) (xe || 1)
 
-#define intel_has_gpu_reset(a) (a && 0)
-
 #include "intel_wakeref.h"
 
 static inline intel_wakeref_t intel_runtime_pm_get(struct xe_runtime_pm *pm)
 {
 	struct xe_device *xe = container_of(pm, struct xe_device, runtime_pm);
 
-	if (xe_pm_runtime_get(xe) < 0) {
-		xe_pm_runtime_put(xe);
-		return 0;
-	}
-	return 1;
+	return xe_pm_runtime_resume_and_get(xe);
 }
 
 static inline intel_wakeref_t intel_runtime_pm_get_if_in_use(struct xe_runtime_pm *pm)
 {
 	struct xe_device *xe = container_of(pm, struct xe_device, runtime_pm);
 
-	return xe_pm_runtime_get_if_active(xe);
+	return xe_pm_runtime_get_if_in_use(xe);
+}
+
+static inline intel_wakeref_t intel_runtime_pm_get_noresume(struct xe_runtime_pm *pm)
+{
+	struct xe_device *xe = container_of(pm, struct xe_device, runtime_pm);
+
+	xe_pm_runtime_get_noresume(xe);
+	return true;
 }
 
 static inline void intel_runtime_pm_put_unchecked(struct xe_runtime_pm *pm)
@@ -217,7 +184,6 @@ struct i915_sched_attr {
 #define RUNTIME_INFO(xe)		(&(xe)->info.i915_runtime)
 
 #define FORCEWAKE_ALL XE_FORCEWAKE_ALL
-#define HPD_STORM_DEFAULT_THRESHOLD 50
 
 #ifdef CONFIG_ARM64
 /*
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_fixed.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_fixed.h
deleted file mode 100644
index 12c671fd5235..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_fixed.h
+++ /dev/null
@@ -1,6 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include "../../i915/i915_fixed.h"
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_gem.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_gem.h
deleted file mode 100644
index 06b723a479c5..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_gem.h
+++ /dev/null
@@ -1,9 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef __I915_GEM_H__
-#define __I915_GEM_H__
-#define GEM_BUG_ON
-#endif
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_gem_stolen.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_gem_stolen.h
index bd233007c1b7..cb6c7598824b 100644
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_gem_stolen.h
+++ b/drivers/gpu/drm/xe/compat-i915-headers/i915_gem_stolen.h
@@ -17,10 +17,15 @@ static inline int i915_gem_stolen_insert_node_in_range(struct xe_device *xe,
 {
 	struct xe_bo *bo;
 	int err;
-	u32 flags = XE_BO_CREATE_PINNED_BIT | XE_BO_CREATE_STOLEN_BIT;
+	u32 flags = XE_BO_FLAG_PINNED | XE_BO_FLAG_STOLEN;
 
-	if (align)
+	if (start < SZ_4K)
+		start = SZ_4K;
+
+	if (align) {
 		size = ALIGN(size, align);
+		start = ALIGN(start, align);
+	}
 
 	bo = xe_bo_create_locked_range(xe, xe_device_get_root_tile(xe),
 				       NULL, size, start, end,
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_vgpu.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_vgpu.h
index 80b024d435dc..4931c7198f13 100644
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_vgpu.h
+++ b/drivers/gpu/drm/xe/compat-i915-headers/i915_vgpu.h
@@ -9,36 +9,10 @@
 #include <linux/types.h>
 
 struct drm_i915_private;
-struct i915_ggtt;
 
-static inline void intel_vgpu_detect(struct drm_i915_private *i915)
-{
-}
 static inline bool intel_vgpu_active(struct drm_i915_private *i915)
 {
 	return false;
 }
-static inline void intel_vgpu_register(struct drm_i915_private *i915)
-{
-}
-static inline bool intel_vgpu_has_full_ppgtt(struct drm_i915_private *i915)
-{
-	return false;
-}
-static inline bool intel_vgpu_has_hwsp_emulation(struct drm_i915_private *i915)
-{
-	return false;
-}
-static inline bool intel_vgpu_has_huge_gtt(struct drm_i915_private *i915)
-{
-	return false;
-}
-static inline int intel_vgt_balloon(struct i915_ggtt *ggtt)
-{
-	return 0;
-}
-static inline void intel_vgt_deballoon(struct i915_ggtt *ggtt)
-{
-}
 
 #endif /* _I915_VGPU_H_ */
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_uc_fw.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_uc_fw.h
deleted file mode 100644
index 009745328992..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/intel_uc_fw.h
+++ /dev/null
@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef _INTEL_UC_FW_H_
-#define _INTEL_UC_FW_H_
-
-#define INTEL_UC_FIRMWARE_URL "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git"
-
-#endif
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h
index cd26ddc0f69e..ef79793caa72 100644
--- a/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h
+++ b/drivers/gpu/drm/xe/compat-i915-headers/intel_uncore.h
@@ -25,15 +25,15 @@ static inline u32 intel_uncore_read(struct intel_uncore *uncore,
 	return xe_mmio_read32(__compat_uncore_to_gt(uncore), reg);
 }
 
-static inline u32 intel_uncore_read8(struct intel_uncore *uncore,
-				     i915_reg_t i915_reg)
+static inline u8 intel_uncore_read8(struct intel_uncore *uncore,
+				    i915_reg_t i915_reg)
 {
 	struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg));
 
 	return xe_mmio_read8(__compat_uncore_to_gt(uncore), reg);
 }
 
-static inline u32 intel_uncore_read16(struct intel_uncore *uncore,
+static inline u16 intel_uncore_read16(struct intel_uncore *uncore,
 				      i915_reg_t i915_reg)
 {
 	struct xe_reg reg = XE_REG(i915_mmio_reg_offset(i915_reg));
diff --git a/drivers/gpu/drm/xe/display/intel_fb_bo.c b/drivers/gpu/drm/xe/display/intel_fb_bo.c
index b21da7b745a5..e18521acc516 100644
--- a/drivers/gpu/drm/xe/display/intel_fb_bo.c
+++ b/drivers/gpu/drm/xe/display/intel_fb_bo.c
@@ -11,7 +11,7 @@
 
 void intel_fb_bo_framebuffer_fini(struct xe_bo *bo)
 {
-	if (bo->flags & XE_BO_CREATE_PINNED_BIT) {
+	if (bo->flags & XE_BO_FLAG_PINNED) {
 		/* Unpin our kernel fb first */
 		xe_bo_lock(bo, false);
 		xe_bo_unpin(bo);
@@ -31,23 +31,27 @@ int intel_fb_bo_framebuffer_init(struct intel_framebuffer *intel_fb,
 
 	ret = ttm_bo_reserve(&bo->ttm, true, false, NULL);
 	if (ret)
-		return ret;
+		goto err;
 
-	if (!(bo->flags & XE_BO_SCANOUT_BIT)) {
+	if (!(bo->flags & XE_BO_FLAG_SCANOUT)) {
 		/*
-		 * XE_BO_SCANOUT_BIT should ideally be set at creation, or is
+		 * XE_BO_FLAG_SCANOUT should ideally be set at creation, or is
 		 * automatically set when creating FB. We cannot change caching
 		 * mode when the boect is VM_BINDed, so we can only set
 		 * coherency with display when unbound.
 		 */
 		if (XE_IOCTL_DBG(i915, !list_empty(&bo->ttm.base.gpuva.list))) {
 			ttm_bo_unreserve(&bo->ttm);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto err;
 		}
-		bo->flags |= XE_BO_SCANOUT_BIT;
+		bo->flags |= XE_BO_FLAG_SCANOUT;
 	}
 	ttm_bo_unreserve(&bo->ttm);
+	return 0;
 
+err:
+	xe_bo_put(bo);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c
index 51ae3561fd0d..9e4bcfdbc7e5 100644
--- a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c
+++ b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c
@@ -42,9 +42,9 @@ struct drm_framebuffer *intel_fbdev_fb_alloc(struct drm_fb_helper *helper,
 	if (!IS_DGFX(dev_priv)) {
 		obj = xe_bo_create_pin_map(dev_priv, xe_device_get_root_tile(dev_priv),
 					   NULL, size,
-					   ttm_bo_type_kernel, XE_BO_SCANOUT_BIT |
-					   XE_BO_CREATE_STOLEN_BIT |
-					   XE_BO_CREATE_PINNED_BIT);
+					   ttm_bo_type_kernel, XE_BO_FLAG_SCANOUT |
+					   XE_BO_FLAG_STOLEN |
+					   XE_BO_FLAG_PINNED);
 		if (!IS_ERR(obj))
 			drm_info(&dev_priv->drm, "Allocated fbdev into stolen\n");
 		else
@@ -52,9 +52,9 @@ struct drm_framebuffer *intel_fbdev_fb_alloc(struct drm_fb_helper *helper,
 	}
 	if (IS_ERR(obj)) {
 		obj = xe_bo_create_pin_map(dev_priv, xe_device_get_root_tile(dev_priv), NULL, size,
-					  ttm_bo_type_kernel, XE_BO_SCANOUT_BIT |
-					  XE_BO_CREATE_VRAM_IF_DGFX(xe_device_get_root_tile(dev_priv)) |
-					  XE_BO_CREATE_PINNED_BIT);
+					  ttm_bo_type_kernel, XE_BO_FLAG_SCANOUT |
+					  XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(dev_priv)) |
+					  XE_BO_FLAG_PINNED);
 	}
 
 	if (IS_ERR(obj)) {
@@ -81,8 +81,8 @@ int intel_fbdev_fb_fill_info(struct drm_i915_private *i915, struct fb_info *info
 {
 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
 
-	if (!(obj->flags & XE_BO_CREATE_SYSTEM_BIT)) {
-		if (obj->flags & XE_BO_CREATE_STOLEN_BIT)
+	if (!(obj->flags & XE_BO_FLAG_SYSTEM)) {
+		if (obj->flags & XE_BO_FLAG_STOLEN)
 			info->fix.smem_start = xe_ttm_stolen_io_offset(obj, 0);
 		else
 			info->fix.smem_start =
diff --git a/drivers/gpu/drm/xe/display/xe_display.c b/drivers/gpu/drm/xe/display/xe_display.c
index 6ec375c1c4b6..0de0566e5b39 100644
--- a/drivers/gpu/drm/xe/display/xe_display.c
+++ b/drivers/gpu/drm/xe/display/xe_display.c
@@ -51,14 +51,6 @@ bool xe_display_driver_probe_defer(struct pci_dev *pdev)
 	return intel_display_driver_probe_defer(pdev);
 }
 
-static void xe_display_last_close(struct drm_device *dev)
-{
-	struct xe_device *xe = to_xe_device(dev);
-
-	if (xe->info.enable_display)
-		intel_fbdev_restore_mode(to_xe_device(dev));
-}
-
 /**
  * xe_display_driver_set_hooks - Add driver flags and hooks for display
  * @driver: DRM device driver
@@ -73,7 +65,6 @@ void xe_display_driver_set_hooks(struct drm_driver *driver)
 		return;
 
 	driver->driver_features |= DRIVER_MODESET | DRIVER_ATOMIC;
-	driver->lastclose = xe_display_last_close;
 }
 
 static void unset_display_features(struct xe_device *xe)
@@ -101,8 +92,6 @@ static void display_destroy(struct drm_device *dev, void *dummy)
  */
 int xe_display_create(struct xe_device *xe)
 {
-	int err;
-
 	spin_lock_init(&xe->display.fb_tracking.lock);
 
 	xe->display.hotplug.dp_wq = alloc_ordered_workqueue("xe-dp", 0);
@@ -110,11 +99,7 @@ int xe_display_create(struct xe_device *xe)
 	drmm_mutex_init(&xe->drm, &xe->sb_lock);
 	xe->enabled_irq_mask = ~0;
 
-	err = drmm_add_action_or_reset(&xe->drm, display_destroy, NULL);
-	if (err)
-		return err;
-
-	return 0;
+	return drmm_add_action_or_reset(&xe->drm, display_destroy, NULL);
 }
 
 static void xe_display_fini_nommio(struct drm_device *dev, void *dummy)
@@ -218,9 +203,7 @@ void xe_display_fini(struct xe_device *xe)
 	if (!xe->info.enable_display)
 		return;
 
-	/* poll work can call into fbdev, hence clean that up afterwards */
 	intel_hpd_poll_fini(xe);
-	intel_fbdev_fini(xe);
 
 	intel_hdcp_component_fini(xe);
 	intel_audio_deinit(xe);
diff --git a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c
index 27c2fb1c002a..44c9fd2143cc 100644
--- a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c
+++ b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c
@@ -45,8 +45,8 @@ bool intel_dsb_buffer_create(struct intel_crtc *crtc, struct intel_dsb_buffer *d
 	obj = xe_bo_create_pin_map(i915, xe_device_get_root_tile(i915),
 				   NULL, PAGE_ALIGN(size),
 				   ttm_bo_type_kernel,
-				   XE_BO_CREATE_VRAM_IF_DGFX(xe_device_get_root_tile(i915)) |
-				   XE_BO_CREATE_GGTT_BIT);
+				   XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(i915)) |
+				   XE_BO_FLAG_GGTT);
 	if (IS_ERR(obj)) {
 		kfree(vma);
 		return false;
diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c
index 722c84a56607..3e1ae37c4c8b 100644
--- a/drivers/gpu/drm/xe/display/xe_fb_pin.c
+++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c
@@ -10,6 +10,7 @@
 #include "intel_fb_pin.h"
 #include "xe_ggtt.h"
 #include "xe_gt.h"
+#include "xe_pm.h"
 
 #include <drm/ttm/ttm_bo.h>
 
@@ -30,7 +31,7 @@ write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_
 
 		for (row = 0; row < height; row++) {
 			u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,
-							      xe->pat.idx[XE_CACHE_WB]);
+							      xe->pat.idx[XE_CACHE_NONE]);
 
 			iosys_map_wr(map, *dpt_ofs, u64, pte);
 			*dpt_ofs += 8;
@@ -62,7 +63,7 @@ write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs,
 		for (column = 0; column < width; column++) {
 			iosys_map_wr(map, *dpt_ofs, u64,
 				     pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,
-				     xe->pat.idx[XE_CACHE_WB]));
+				     xe->pat.idx[XE_CACHE_NONE]));
 
 			*dpt_ofs += 8;
 			src_idx++;
@@ -99,18 +100,21 @@ static int __xe_pin_fb_vma_dpt(struct intel_framebuffer *fb,
 	if (IS_DGFX(xe))
 		dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size,
 					   ttm_bo_type_kernel,
-					   XE_BO_CREATE_VRAM0_BIT |
-					   XE_BO_CREATE_GGTT_BIT);
+					   XE_BO_FLAG_VRAM0 |
+					   XE_BO_FLAG_GGTT |
+					   XE_BO_FLAG_PAGETABLE);
 	else
 		dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size,
 					   ttm_bo_type_kernel,
-					   XE_BO_CREATE_STOLEN_BIT |
-					   XE_BO_CREATE_GGTT_BIT);
+					   XE_BO_FLAG_STOLEN |
+					   XE_BO_FLAG_GGTT |
+					   XE_BO_FLAG_PAGETABLE);
 	if (IS_ERR(dpt))
 		dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size,
 					   ttm_bo_type_kernel,
-					   XE_BO_CREATE_SYSTEM_BIT |
-					   XE_BO_CREATE_GGTT_BIT);
+					   XE_BO_FLAG_SYSTEM |
+					   XE_BO_FLAG_GGTT |
+					   XE_BO_FLAG_PAGETABLE);
 	if (IS_ERR(dpt))
 		return PTR_ERR(dpt);
 
@@ -119,7 +123,7 @@ static int __xe_pin_fb_vma_dpt(struct intel_framebuffer *fb,
 
 		for (x = 0; x < size / XE_PAGE_SIZE; x++) {
 			u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x * XE_PAGE_SIZE,
-							      xe->pat.idx[XE_CACHE_WB]);
+							      xe->pat.idx[XE_CACHE_NONE]);
 
 			iosys_map_wr(&dpt->vmap, x * 8, u64, pte);
 		}
@@ -165,7 +169,7 @@ write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo
 
 		for (row = 0; row < height; row++) {
 			u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,
-							      xe->pat.idx[XE_CACHE_WB]);
+							      xe->pat.idx[XE_CACHE_NONE]);
 
 			xe_ggtt_set_pte(ggtt, *ggtt_ofs, pte);
 			*ggtt_ofs += XE_PAGE_SIZE;
@@ -190,7 +194,7 @@ static int __xe_pin_fb_vma_ggtt(struct intel_framebuffer *fb,
 	/* TODO: Consider sharing framebuffer mapping?
 	 * embed i915_vma inside intel_framebuffer
 	 */
-	xe_device_mem_access_get(tile_to_xe(ggtt->tile));
+	xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
 	ret = mutex_lock_interruptible(&ggtt->lock);
 	if (ret)
 		goto out;
@@ -211,7 +215,7 @@ static int __xe_pin_fb_vma_ggtt(struct intel_framebuffer *fb,
 
 		for (x = 0; x < size; x += XE_PAGE_SIZE) {
 			u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x,
-							      xe->pat.idx[XE_CACHE_WB]);
+							      xe->pat.idx[XE_CACHE_NONE]);
 
 			xe_ggtt_set_pte(ggtt, vma->node.start + x, pte);
 		}
@@ -238,11 +242,10 @@ static int __xe_pin_fb_vma_ggtt(struct intel_framebuffer *fb,
 					   rot_info->plane[i].dst_stride);
 	}
 
-	xe_ggtt_invalidate(ggtt);
 out_unlock:
 	mutex_unlock(&ggtt->lock);
 out:
-	xe_device_mem_access_put(tile_to_xe(ggtt->tile));
+	xe_pm_runtime_put(tile_to_xe(ggtt->tile));
 	return ret;
 }
 
@@ -260,7 +263,7 @@ static struct i915_vma *__xe_pin_fb_vma(struct intel_framebuffer *fb,
 
 	if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) &&
 	    intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 &&
-	    !(bo->flags & XE_BO_NEEDS_CPU_ACCESS)) {
+	    !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) {
 		struct xe_tile *tile = xe_device_get_root_tile(xe);
 
 		/*
@@ -321,7 +324,7 @@ static void __xe_unpin_fb_vma(struct i915_vma *vma)
 		xe_bo_unpin_map_no_vm(vma->dpt);
 	else if (!drm_mm_node_allocated(&vma->bo->ggtt_node) ||
 		 vma->bo->ggtt_node.start != vma->node.start)
-		xe_ggtt_remove_node(ggtt, &vma->node);
+		xe_ggtt_remove_node(ggtt, &vma->node, false);
 
 	ttm_bo_reserve(&vma->bo->ttm, false, false, NULL);
 	ttm_bo_unpin(&vma->bo->ttm);
@@ -353,7 +356,7 @@ int intel_plane_pin_fb(struct intel_plane_state *plane_state)
 	struct i915_vma *vma;
 
 	/* We reject creating !SCANOUT fb's, so this is weird.. */
-	drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_SCANOUT_BIT));
+	drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT));
 
 	vma = __xe_pin_fb_vma(to_intel_framebuffer(fb), &plane_state->view.gtt);
 	if (IS_ERR(vma))
@@ -381,4 +384,4 @@ struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb)
 void intel_dpt_destroy(struct i915_address_space *vm)
 {
 	return;
-}
\ No newline at end of file
+}
diff --git a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c
index 0f11a39333e2..d46f87a039f2 100644
--- a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c
+++ b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c
@@ -3,32 +3,250 @@
  * Copyright 2023, Intel Corporation.
  */
 
-#include "i915_drv.h"
+#include <drm/drm_print.h>
+#include <drm/i915_hdcp_interface.h>
+#include <linux/delay.h>
+
+#include "abi/gsc_command_header_abi.h"
 #include "intel_hdcp_gsc.h"
+#include "intel_hdcp_gsc_message.h"
+#include "xe_bo.h"
+#include "xe_device.h"
+#include "xe_device_types.h"
+#include "xe_gsc_proxy.h"
+#include "xe_gsc_submit.h"
+#include "xe_gt.h"
+#include "xe_map.h"
+#include "xe_pm.h"
+#include "xe_uc_fw.h"
+
+#define HECI_MEADDRESS_HDCP 18
+
+struct intel_hdcp_gsc_message {
+	struct xe_bo *hdcp_bo;
+	u64 hdcp_cmd_in;
+	u64 hdcp_cmd_out;
+};
 
-bool intel_hdcp_gsc_cs_required(struct drm_i915_private *i915)
+#define HDCP_GSC_HEADER_SIZE sizeof(struct intel_gsc_mtl_header)
+
+bool intel_hdcp_gsc_cs_required(struct xe_device *xe)
 {
-	return true;
+	return DISPLAY_VER(xe) >= 14;
 }
 
-bool intel_hdcp_gsc_check_status(struct drm_i915_private *i915)
+bool intel_hdcp_gsc_check_status(struct xe_device *xe)
 {
-	return false;
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_gt *gt = tile->media_gt;
+	bool ret = true;
+
+	if (!xe_uc_fw_is_enabled(&gt->uc.gsc.fw))
+		return false;
+
+	xe_pm_runtime_get(xe);
+	if (xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC)) {
+		drm_dbg_kms(&xe->drm,
+			    "failed to get forcewake to check proxy status\n");
+		ret = false;
+		goto out;
+	}
+
+	if (!xe_gsc_proxy_init_done(&gt->uc.gsc))
+		ret = false;
+
+	xe_force_wake_put(gt_to_fw(gt), XE_FW_GSC);
+out:
+	xe_pm_runtime_put(xe);
+	return ret;
 }
 
-int intel_hdcp_gsc_init(struct drm_i915_private *i915)
+/*This function helps allocate memory for the command that we will send to gsc cs */
+static int intel_hdcp_gsc_initialize_message(struct xe_device *xe,
+					     struct intel_hdcp_gsc_message *hdcp_message)
 {
-	drm_info(&i915->drm, "HDCP support not yet implemented\n");
-	return -ENODEV;
+	struct xe_bo *bo = NULL;
+	u64 cmd_in, cmd_out;
+	int ret = 0;
+
+	/* allocate object of two page for HDCP command memory and store it */
+	bo = xe_bo_create_pin_map(xe, xe_device_get_root_tile(xe), NULL, PAGE_SIZE * 2,
+				  ttm_bo_type_kernel,
+				  XE_BO_FLAG_SYSTEM |
+				  XE_BO_FLAG_GGTT);
+
+	if (IS_ERR(bo)) {
+		drm_err(&xe->drm, "Failed to allocate bo for HDCP streaming command!\n");
+		ret = PTR_ERR(bo);
+		goto out;
+	}
+
+	cmd_in = xe_bo_ggtt_addr(bo);
+	cmd_out = cmd_in + PAGE_SIZE;
+	xe_map_memset(xe, &bo->vmap, 0, 0, bo->size);
+
+	hdcp_message->hdcp_bo = bo;
+	hdcp_message->hdcp_cmd_in = cmd_in;
+	hdcp_message->hdcp_cmd_out = cmd_out;
+out:
+	return ret;
 }
 
-void intel_hdcp_gsc_fini(struct drm_i915_private *i915)
+static int intel_hdcp_gsc_hdcp2_init(struct xe_device *xe)
 {
+	struct intel_hdcp_gsc_message *hdcp_message;
+	int ret;
+
+	hdcp_message = kzalloc(sizeof(*hdcp_message), GFP_KERNEL);
+
+	if (!hdcp_message)
+		return -ENOMEM;
+
+	/*
+	 * NOTE: No need to lock the comp mutex here as it is already
+	 * going to be taken before this function called
+	 */
+	ret = intel_hdcp_gsc_initialize_message(xe, hdcp_message);
+	if (ret) {
+		drm_err(&xe->drm, "Could not initialize hdcp_message\n");
+		kfree(hdcp_message);
+		return ret;
+	}
+
+	xe->display.hdcp.hdcp_message = hdcp_message;
+	return ret;
 }
 
-ssize_t intel_hdcp_gsc_msg_send(struct drm_i915_private *i915, u8 *msg_in,
+static const struct i915_hdcp_ops gsc_hdcp_ops = {
+	.initiate_hdcp2_session = intel_hdcp_gsc_initiate_session,
+	.verify_receiver_cert_prepare_km =
+				intel_hdcp_gsc_verify_receiver_cert_prepare_km,
+	.verify_hprime = intel_hdcp_gsc_verify_hprime,
+	.store_pairing_info = intel_hdcp_gsc_store_pairing_info,
+	.initiate_locality_check = intel_hdcp_gsc_initiate_locality_check,
+	.verify_lprime = intel_hdcp_gsc_verify_lprime,
+	.get_session_key = intel_hdcp_gsc_get_session_key,
+	.repeater_check_flow_prepare_ack =
+				intel_hdcp_gsc_repeater_check_flow_prepare_ack,
+	.verify_mprime = intel_hdcp_gsc_verify_mprime,
+	.enable_hdcp_authentication = intel_hdcp_gsc_enable_authentication,
+	.close_hdcp_session = intel_hdcp_gsc_close_session,
+};
+
+int intel_hdcp_gsc_init(struct xe_device *xe)
+{
+	struct i915_hdcp_arbiter *data;
+	int ret;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	mutex_lock(&xe->display.hdcp.hdcp_mutex);
+	xe->display.hdcp.arbiter = data;
+	xe->display.hdcp.arbiter->hdcp_dev = xe->drm.dev;
+	xe->display.hdcp.arbiter->ops = &gsc_hdcp_ops;
+	ret = intel_hdcp_gsc_hdcp2_init(xe);
+	if (ret)
+		kfree(data);
+
+	mutex_unlock(&xe->display.hdcp.hdcp_mutex);
+
+	return ret;
+}
+
+void intel_hdcp_gsc_fini(struct xe_device *xe)
+{
+	struct intel_hdcp_gsc_message *hdcp_message =
+					xe->display.hdcp.hdcp_message;
+
+	if (!hdcp_message)
+		return;
+
+	xe_bo_unpin_map_no_vm(hdcp_message->hdcp_bo);
+	kfree(hdcp_message);
+}
+
+static int xe_gsc_send_sync(struct xe_device *xe,
+			    struct intel_hdcp_gsc_message *hdcp_message,
+			    u32 msg_size_in, u32 msg_size_out,
+			    u32 addr_out_off)
+{
+	struct xe_gt *gt = hdcp_message->hdcp_bo->tile->media_gt;
+	struct iosys_map *map = &hdcp_message->hdcp_bo->vmap;
+	struct xe_gsc *gsc = &gt->uc.gsc;
+	int ret;
+
+	ret = xe_gsc_pkt_submit_kernel(gsc, hdcp_message->hdcp_cmd_in, msg_size_in,
+				       hdcp_message->hdcp_cmd_out, msg_size_out);
+	if (ret) {
+		drm_err(&xe->drm, "failed to send gsc HDCP msg (%d)\n", ret);
+		return ret;
+	}
+
+	if (xe_gsc_check_and_update_pending(xe, map, 0, map, addr_out_off))
+		return -EAGAIN;
+
+	ret = xe_gsc_read_out_header(xe, map, addr_out_off,
+				     sizeof(struct hdcp_cmd_header), NULL);
+
+	return ret;
+}
+
+ssize_t intel_hdcp_gsc_msg_send(struct xe_device *xe, u8 *msg_in,
 				size_t msg_in_len, u8 *msg_out,
 				size_t msg_out_len)
 {
-	return -ENODEV;
+	const size_t max_msg_size = PAGE_SIZE - HDCP_GSC_HEADER_SIZE;
+	struct intel_hdcp_gsc_message *hdcp_message;
+	u64 host_session_id;
+	u32 msg_size_in, msg_size_out;
+	u32 addr_out_off, addr_in_wr_off = 0;
+	int ret, tries = 0;
+
+	if (msg_in_len > max_msg_size || msg_out_len > max_msg_size) {
+		ret = -ENOSPC;
+		goto out;
+	}
+
+	msg_size_in = msg_in_len + HDCP_GSC_HEADER_SIZE;
+	msg_size_out = msg_out_len + HDCP_GSC_HEADER_SIZE;
+	hdcp_message = xe->display.hdcp.hdcp_message;
+	addr_out_off = PAGE_SIZE;
+
+	host_session_id = xe_gsc_create_host_session_id();
+	xe_pm_runtime_get_noresume(xe);
+	addr_in_wr_off = xe_gsc_emit_header(xe, &hdcp_message->hdcp_bo->vmap,
+					    addr_in_wr_off, HECI_MEADDRESS_HDCP,
+					    host_session_id, msg_in_len);
+	xe_map_memcpy_to(xe, &hdcp_message->hdcp_bo->vmap, addr_in_wr_off,
+			 msg_in, msg_in_len);
+	/*
+	 * Keep sending request in case the pending bit is set no need to add
+	 * message handle as we are using same address hence loc. of header is
+	 * same and it will contain the message handle. we will send the message
+	 * 20 times each message 50 ms apart
+	 */
+	do {
+		ret = xe_gsc_send_sync(xe, hdcp_message, msg_size_in, msg_size_out,
+				       addr_out_off);
+
+		/* Only try again if gsc says so */
+		if (ret != -EAGAIN)
+			break;
+
+		msleep(50);
+
+	} while (++tries < 20);
+
+	if (ret)
+		goto out;
+
+	xe_map_memcpy_from(xe, msg_out, &hdcp_message->hdcp_bo->vmap,
+			   addr_out_off + HDCP_GSC_HEADER_SIZE,
+			   msg_out_len);
+
+out:
+	xe_pm_runtime_put(xe);
+	return ret;
 }
diff --git a/drivers/gpu/drm/xe/display/xe_plane_initial.c b/drivers/gpu/drm/xe/display/xe_plane_initial.c
index 866d1dd6eeb4..9693c56d386b 100644
--- a/drivers/gpu/drm/xe/display/xe_plane_initial.c
+++ b/drivers/gpu/drm/xe/display/xe_plane_initial.c
@@ -6,6 +6,7 @@
 /* for ioread64 */
 #include <linux/io-64-nonatomic-lo-hi.h>
 
+#include "regs/xe_gtt_defs.h"
 #include "xe_ggtt.h"
 
 #include "i915_drv.h"
@@ -62,7 +63,7 @@ initial_plane_bo(struct xe_device *xe,
 	if (plane_config->size == 0)
 		return NULL;
 
-	flags = XE_BO_CREATE_PINNED_BIT | XE_BO_SCANOUT_BIT | XE_BO_CREATE_GGTT_BIT;
+	flags = XE_BO_FLAG_PINNED | XE_BO_FLAG_SCANOUT | XE_BO_FLAG_GGTT;
 
 	base = round_down(plane_config->base, page_size);
 	if (IS_DGFX(xe)) {
@@ -79,7 +80,7 @@ initial_plane_bo(struct xe_device *xe,
 		}
 
 		phys_base = pte & ~(page_size - 1);
-		flags |= XE_BO_CREATE_VRAM0_BIT;
+		flags |= XE_BO_FLAG_VRAM0;
 
 		/*
 		 * We don't currently expect this to ever be placed in the
@@ -101,7 +102,7 @@ initial_plane_bo(struct xe_device *xe,
 		if (!stolen)
 			return NULL;
 		phys_base = base;
-		flags |= XE_BO_CREATE_STOLEN_BIT;
+		flags |= XE_BO_FLAG_STOLEN;
 
 		/*
 		 * If the FB is too big, just don't use it since fbdev is not very
diff --git a/drivers/gpu/drm/xe/instructions/xe_gfx_state_commands.h b/drivers/gpu/drm/xe/instructions/xe_gfx_state_commands.h
new file mode 100644
index 000000000000..dca62af5a5d5
--- /dev/null
+++ b/drivers/gpu/drm/xe/instructions/xe_gfx_state_commands.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#ifndef _XE_GFX_STATE_COMMANDS_H_
+#define _XE_GFX_STATE_COMMANDS_H_
+
+#include "instructions/xe_instr_defs.h"
+
+#define GFX_STATE_OPCODE			REG_GENMASK(28, 26)
+
+#define GFX_STATE_CMD(opcode) \
+	(XE_INSTR_GFX_STATE | REG_FIELD_PREP(GFX_STATE_OPCODE, opcode))
+
+#define STATE_WRITE_INLINE			GFX_STATE_CMD(0x0)
+
+#endif
diff --git a/drivers/gpu/drm/xe/instructions/xe_gfxpipe_commands.h b/drivers/gpu/drm/xe/instructions/xe_gfxpipe_commands.h
index 8e6dd061f2ae..31d28a67ef6a 100644
--- a/drivers/gpu/drm/xe/instructions/xe_gfxpipe_commands.h
+++ b/drivers/gpu/drm/xe/instructions/xe_gfxpipe_commands.h
@@ -47,6 +47,8 @@
 #define GPGPU_CSR_BASE_ADDRESS			GFXPIPE_COMMON_CMD(0x1, 0x4)
 #define STATE_COMPUTE_MODE			GFXPIPE_COMMON_CMD(0x1, 0x5)
 #define CMD_3DSTATE_BTD				GFXPIPE_COMMON_CMD(0x1, 0x6)
+#define STATE_SYSTEM_MEM_FENCE_ADDRESS		GFXPIPE_COMMON_CMD(0x1, 0x9)
+#define STATE_CONTEXT_DATA_BASE_ADDRESS		GFXPIPE_COMMON_CMD(0x1, 0xB)
 
 #define CMD_3DSTATE_VF_STATISTICS		GFXPIPE_SINGLE_DW_CMD(0x0, 0xB)
 
@@ -71,6 +73,7 @@
 #define CMD_3DSTATE_WM				GFXPIPE_3D_CMD(0x0, 0x14)
 #define CMD_3DSTATE_CONSTANT_VS			GFXPIPE_3D_CMD(0x0, 0x15)
 #define CMD_3DSTATE_CONSTANT_GS			GFXPIPE_3D_CMD(0x0, 0x16)
+#define CMD_3DSTATE_CONSTANT_PS			GFXPIPE_3D_CMD(0x0, 0x17)
 #define CMD_3DSTATE_SAMPLE_MASK			GFXPIPE_3D_CMD(0x0, 0x18)
 #define CMD_3DSTATE_CONSTANT_HS			GFXPIPE_3D_CMD(0x0, 0x19)
 #define CMD_3DSTATE_CONSTANT_DS			GFXPIPE_3D_CMD(0x0, 0x1A)
diff --git a/drivers/gpu/drm/xe/instructions/xe_instr_defs.h b/drivers/gpu/drm/xe/instructions/xe_instr_defs.h
index 04179b2a48e1..fd2ce7ace510 100644
--- a/drivers/gpu/drm/xe/instructions/xe_instr_defs.h
+++ b/drivers/gpu/drm/xe/instructions/xe_instr_defs.h
@@ -17,6 +17,7 @@
 #define   XE_INSTR_MI			REG_FIELD_PREP(XE_INSTR_CMD_TYPE, 0x0)
 #define   XE_INSTR_GSC			REG_FIELD_PREP(XE_INSTR_CMD_TYPE, 0x2)
 #define   XE_INSTR_GFXPIPE		REG_FIELD_PREP(XE_INSTR_CMD_TYPE, 0x3)
+#define   XE_INSTR_GFX_STATE		REG_FIELD_PREP(XE_INSTR_CMD_TYPE, 0x4)
 
 /*
  * Most (but not all) instructions have a "length" field in the instruction
diff --git a/drivers/gpu/drm/xe/regs/xe_engine_regs.h b/drivers/gpu/drm/xe/regs/xe_engine_regs.h
index deddc8be48c0..af71b87d8030 100644
--- a/drivers/gpu/drm/xe/regs/xe_engine_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_engine_regs.h
@@ -104,9 +104,6 @@
 #define FF_SLICE_CS_CHICKEN1(base)		XE_REG((base) + 0xe0, XE_REG_OPTION_MASKED)
 #define   FFSC_PERCTX_PREEMPT_CTRL		REG_BIT(14)
 
-#define FF_SLICE_CS_CHICKEN2(base)		XE_REG((base) + 0xe4, XE_REG_OPTION_MASKED)
-#define   PERF_FIX_BALANCING_CFE_DISABLE	REG_BIT(15)
-
 #define CS_DEBUG_MODE1(base)			XE_REG((base) + 0xec, XE_REG_OPTION_MASKED)
 #define   FF_DOP_CLOCK_GATE_DISABLE		REG_BIT(1)
 #define   REPLAY_MODE_GRANULARITY		REG_BIT(0)
diff --git a/drivers/gpu/drm/xe/regs/xe_gsc_regs.h b/drivers/gpu/drm/xe/regs/xe_gsc_regs.h
index 9886ec9cb08e..e2a925be137c 100644
--- a/drivers/gpu/drm/xe/regs/xe_gsc_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_gsc_regs.h
@@ -38,4 +38,11 @@
 #define HECI_H_GS1(base)	XE_REG((base) + 0xc4c)
 #define   HECI_H_GS1_ER_PREP	REG_BIT(0)
 
+#define GSCI_TIMER_STATUS				XE_REG(0x11ca28)
+#define   GSCI_TIMER_STATUS_VALUE			REG_GENMASK(1, 0)
+#define   GSCI_TIMER_STATUS_RESET_IN_PROGRESS		0
+#define   GSCI_TIMER_STATUS_TIMER_EXPIRED		1
+#define   GSCI_TIMER_STATUS_RESET_COMPLETE		2
+#define   GSCI_TIMER_STATUS_OUT_OF_RESET		3
+
 #endif
diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
index 15ac2d284d48..94445810ccc9 100644
--- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
@@ -69,10 +69,14 @@
 
 #define XEHP_TILE_ADDR_RANGE(_idx)		XE_REG_MCR(0x4900 + (_idx) * 4)
 #define XEHP_FLAT_CCS_BASE_ADDR			XE_REG_MCR(0x4910)
+#define XEHP_FLAT_CCS_PTR			REG_GENMASK(31, 8)
 
 #define WM_CHICKEN3				XE_REG_MCR(0x5588, XE_REG_OPTION_MASKED)
 #define   HIZ_PLANE_COMPRESSION_DIS		REG_BIT(10)
 
+#define CHICKEN_RASTER_1			XE_REG_MCR(0x6204, XE_REG_OPTION_MASKED)
+#define   DIS_SF_ROUND_NEAREST_EVEN		REG_BIT(8)
+
 #define CHICKEN_RASTER_2			XE_REG_MCR(0x6208, XE_REG_OPTION_MASKED)
 #define   TBIMR_FAST_CLIP			REG_BIT(5)
 
@@ -97,7 +101,8 @@
 #define CACHE_MODE_1				XE_REG(0x7004, XE_REG_OPTION_MASKED)
 #define   MSAA_OPTIMIZATION_REDUC_DISABLE	REG_BIT(11)
 
-#define COMMON_SLICE_CHICKEN1			XE_REG(0x7010)
+#define COMMON_SLICE_CHICKEN1			XE_REG(0x7010, XE_REG_OPTION_MASKED)
+#define   DISABLE_BOTTOM_CLIP_RECTANGLE_TEST	REG_BIT(14)
 
 #define HIZ_CHICKEN					XE_REG(0x7018, XE_REG_OPTION_MASKED)
 #define   DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE	REG_BIT(14)
@@ -141,6 +146,10 @@
 
 #define XE2_FLAT_CCS_BASE_RANGE_LOWER		XE_REG_MCR(0x8800)
 #define   XE2_FLAT_CCS_ENABLE			REG_BIT(0)
+#define XE2_FLAT_CCS_BASE_LOWER_ADDR_MASK	REG_GENMASK(31, 6)
+
+#define XE2_FLAT_CCS_BASE_RANGE_UPPER		XE_REG_MCR(0x8804)
+#define XE2_FLAT_CCS_BASE_UPPER_ADDR_MASK	REG_GENMASK(7, 0)
 
 #define GSCPSMI_BASE				XE_REG(0x880c)
 
@@ -156,7 +165,10 @@
 #define	MIRROR_FUSE3				XE_REG(0x9118)
 #define   XE2_NODE_ENABLE_MASK			REG_GENMASK(31, 16)
 #define   L3BANK_PAIR_COUNT			4
+#define   XEHPC_GT_L3_MODE_MASK			REG_GENMASK(7, 4)
+#define   XE2_GT_L3_MODE_MASK			REG_GENMASK(7, 4)
 #define   L3BANK_MASK				REG_GENMASK(3, 0)
+#define   XELP_GT_L3_MODE_MASK			REG_GENMASK(7, 0)
 /* on Xe_HP the same fuses indicates mslices instead of L3 banks */
 #define   MAX_MSLICES				4
 #define   MEML3_EN_MASK				REG_GENMASK(3, 0)
@@ -271,6 +283,10 @@
 #define FORCEWAKE_GT				XE_REG(0xa188)
 
 #define PG_ENABLE				XE_REG(0xa210)
+#define   VD2_MFXVDENC_POWERGATE_ENABLE		REG_BIT(8)
+#define   VD2_HCP_POWERGATE_ENABLE		REG_BIT(7)
+#define   VD0_MFXVDENC_POWERGATE_ENABLE		REG_BIT(4)
+#define   VD0_HCP_POWERGATE_ENABLE		REG_BIT(3)
 
 #define CTC_MODE				XE_REG(0xa26c)
 #define   CTC_SHIFT_PARAMETER_MASK		REG_GENMASK(2, 1)
@@ -349,6 +365,7 @@
 #define   THREAD_EX_ARB_MODE_RR_AFTER_DEP	REG_FIELD_PREP(THREAD_EX_ARB_MODE, 0x2)
 
 #define ROW_CHICKEN3				XE_REG_MCR(0xe49c, XE_REG_OPTION_MASKED)
+#define   XE2_EUPEND_CHK_FLUSH_DIS		REG_BIT(14)
 #define   DIS_FIX_EOT1_FLUSH			REG_BIT(9)
 
 #define TDL_TSL_CHICKEN				XE_REG_MCR(0xe4c4, XE_REG_OPTION_MASKED)
@@ -364,17 +381,22 @@
 #define   DISABLE_EARLY_READ			REG_BIT(14)
 #define   ENABLE_LARGE_GRF_MODE			REG_BIT(12)
 #define   PUSH_CONST_DEREF_HOLD_DIS		REG_BIT(8)
+#define   DISABLE_TDL_SVHS_GATING		REG_BIT(1)
 #define   DISABLE_DOP_GATING			REG_BIT(0)
 
 #define RT_CTRL					XE_REG_MCR(0xe530)
 #define   DIS_NULL_QUERY			REG_BIT(10)
 
+#define EU_SYSTOLIC_LIC_THROTTLE_CTL_WITH_LOCK	XE_REG_MCR(0xe534)
+#define   EU_SYSTOLIC_LIC_THROTTLE_CTL_LOCK_BIT	REG_BIT(31)
+
 #define XEHP_HDC_CHICKEN0					XE_REG_MCR(0xe5f0, XE_REG_OPTION_MASKED)
 #define   LSC_L1_FLUSH_CTL_3D_DATAPORT_FLUSH_EVENTS_MASK	REG_GENMASK(13, 11)
 #define   DIS_ATOMIC_CHAINING_TYPED_WRITES	REG_BIT(3)
 
 #define LSC_CHICKEN_BIT_0			XE_REG_MCR(0xe7c8)
 #define   DISABLE_D8_D16_COASLESCE		REG_BIT(30)
+#define   WR_REQ_CHAINING_DIS			REG_BIT(26)
 #define   TGM_WRITE_EOM_FORCE			REG_BIT(17)
 #define   FORCE_1_SUB_MESSAGE_PER_FRAGMENT	REG_BIT(15)
 #define   SEQUENTIAL_ACCESS_UPGRADE_DISABLE	REG_BIT(13)
@@ -439,7 +461,13 @@
 #define GT_PERF_STATUS				XE_REG(0x1381b4)
 #define   VOLTAGE_MASK				REG_GENMASK(10, 0)
 
-#define GT_INTR_DW(x)				XE_REG(0x190018 + ((x) * 4))
+/*
+ * Note: Interrupt registers 1900xx are VF accessible only until version 12.50.
+ *       On newer platforms, VFs are using memory-based interrupts instead.
+ *       However, for simplicity we keep this XE_REG_OPTION_VF tag intact.
+ */
+
+#define GT_INTR_DW(x)				XE_REG(0x190018 + ((x) * 4), XE_REG_OPTION_VF)
 #define   INTR_GSC				REG_BIT(31)
 #define   INTR_GUC				REG_BIT(25)
 #define   INTR_MGUC				REG_BIT(24)
@@ -450,16 +478,16 @@
 #define   INTR_VECS(x)				REG_BIT(31 - (x))
 #define   INTR_VCS(x)				REG_BIT(x)
 
-#define RENDER_COPY_INTR_ENABLE			XE_REG(0x190030)
-#define VCS_VECS_INTR_ENABLE			XE_REG(0x190034)
-#define GUC_SG_INTR_ENABLE			XE_REG(0x190038)
+#define RENDER_COPY_INTR_ENABLE			XE_REG(0x190030, XE_REG_OPTION_VF)
+#define VCS_VECS_INTR_ENABLE			XE_REG(0x190034, XE_REG_OPTION_VF)
+#define GUC_SG_INTR_ENABLE			XE_REG(0x190038, XE_REG_OPTION_VF)
 #define   ENGINE1_MASK				REG_GENMASK(31, 16)
 #define   ENGINE0_MASK				REG_GENMASK(15, 0)
-#define GPM_WGBOXPERF_INTR_ENABLE		XE_REG(0x19003c)
-#define GUNIT_GSC_INTR_ENABLE			XE_REG(0x190044)
-#define CCS_RSVD_INTR_ENABLE			XE_REG(0x190048)
+#define GPM_WGBOXPERF_INTR_ENABLE		XE_REG(0x19003c, XE_REG_OPTION_VF)
+#define GUNIT_GSC_INTR_ENABLE			XE_REG(0x190044, XE_REG_OPTION_VF)
+#define CCS_RSVD_INTR_ENABLE			XE_REG(0x190048, XE_REG_OPTION_VF)
 
-#define INTR_IDENTITY_REG(x)			XE_REG(0x190060 + ((x) * 4))
+#define INTR_IDENTITY_REG(x)			XE_REG(0x190060 + ((x) * 4), XE_REG_OPTION_VF)
 #define   INTR_DATA_VALID			REG_BIT(31)
 #define   INTR_ENGINE_INSTANCE(x)		REG_FIELD_GET(GENMASK(25, 20), x)
 #define   INTR_ENGINE_CLASS(x)			REG_FIELD_GET(GENMASK(18, 16), x)
@@ -468,16 +496,16 @@
 #define   OTHER_GSC_HECI2_INSTANCE		3
 #define   OTHER_GSC_INSTANCE			6
 
-#define IIR_REG_SELECTOR(x)			XE_REG(0x190070 + ((x) * 4))
-#define RCS0_RSVD_INTR_MASK			XE_REG(0x190090)
-#define BCS_RSVD_INTR_MASK			XE_REG(0x1900a0)
-#define VCS0_VCS1_INTR_MASK			XE_REG(0x1900a8)
-#define VCS2_VCS3_INTR_MASK			XE_REG(0x1900ac)
-#define VECS0_VECS1_INTR_MASK			XE_REG(0x1900d0)
+#define IIR_REG_SELECTOR(x)			XE_REG(0x190070 + ((x) * 4), XE_REG_OPTION_VF)
+#define RCS0_RSVD_INTR_MASK			XE_REG(0x190090, XE_REG_OPTION_VF)
+#define BCS_RSVD_INTR_MASK			XE_REG(0x1900a0, XE_REG_OPTION_VF)
+#define VCS0_VCS1_INTR_MASK			XE_REG(0x1900a8, XE_REG_OPTION_VF)
+#define VCS2_VCS3_INTR_MASK			XE_REG(0x1900ac, XE_REG_OPTION_VF)
+#define VECS0_VECS1_INTR_MASK			XE_REG(0x1900d0, XE_REG_OPTION_VF)
 #define HECI2_RSVD_INTR_MASK			XE_REG(0x1900e4)
-#define GUC_SG_INTR_MASK			XE_REG(0x1900e8)
-#define GPM_WGBOXPERF_INTR_MASK			XE_REG(0x1900ec)
-#define GUNIT_GSC_INTR_MASK			XE_REG(0x1900f4)
+#define GUC_SG_INTR_MASK			XE_REG(0x1900e8, XE_REG_OPTION_VF)
+#define GPM_WGBOXPERF_INTR_MASK			XE_REG(0x1900ec, XE_REG_OPTION_VF)
+#define GUNIT_GSC_INTR_MASK			XE_REG(0x1900f4, XE_REG_OPTION_VF)
 #define CCS0_CCS1_INTR_MASK			XE_REG(0x190100)
 #define CCS2_CCS3_INTR_MASK			XE_REG(0x190104)
 #define XEHPC_BCS1_BCS2_INTR_MASK		XE_REG(0x190110)
@@ -486,6 +514,7 @@
 #define XEHPC_BCS7_BCS8_INTR_MASK		XE_REG(0x19011c)
 #define   GT_WAIT_SEMAPHORE_INTERRUPT		REG_BIT(11)
 #define   GT_CONTEXT_SWITCH_INTERRUPT		REG_BIT(8)
+#define   GSC_ER_COMPLETE			REG_BIT(5)
 #define   GT_RENDER_PIPECTL_NOTIFY_INTERRUPT	REG_BIT(4)
 #define   GT_CS_MASTER_ERROR_INTERRUPT		REG_BIT(3)
 #define   GT_RENDER_USER_INTERRUPT		REG_BIT(0)
diff --git a/drivers/gpu/drm/xe/regs/xe_gtt_defs.h b/drivers/gpu/drm/xe/regs/xe_gtt_defs.h
new file mode 100644
index 000000000000..4389e5a76f89
--- /dev/null
+++ b/drivers/gpu/drm/xe/regs/xe_gtt_defs.h
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#ifndef _XE_GTT_DEFS_H_
+#define _XE_GTT_DEFS_H_
+
+#define XELPG_GGTT_PTE_PAT0	BIT_ULL(52)
+#define XELPG_GGTT_PTE_PAT1	BIT_ULL(53)
+
+#define GGTT_PTE_VFID		GENMASK_ULL(11, 2)
+
+#define GUC_GGTT_TOP		0xFEE00000
+
+#define XELPG_PPGTT_PTE_PAT3		BIT_ULL(62)
+#define XE2_PPGTT_PTE_PAT4		BIT_ULL(61)
+#define XE_PPGTT_PDE_PDPE_PAT2		BIT_ULL(12)
+#define XE_PPGTT_PTE_PAT2		BIT_ULL(7)
+#define XE_PPGTT_PTE_PAT1		BIT_ULL(4)
+#define XE_PPGTT_PTE_PAT0		BIT_ULL(3)
+
+#define XE_PDE_PS_2M			BIT_ULL(7)
+#define XE_PDPE_PS_1G			BIT_ULL(7)
+#define XE_PDE_IPS_64K			BIT_ULL(11)
+
+#define XE_GGTT_PTE_DM			BIT_ULL(1)
+#define XE_USM_PPGTT_PTE_AE		BIT_ULL(10)
+#define XE_PPGTT_PTE_DM			BIT_ULL(11)
+#define XE_PDE_64K			BIT_ULL(6)
+#define XE_PTE_PS64			BIT_ULL(8)
+#define XE_PTE_NULL			BIT_ULL(9)
+
+#define XE_PAGE_PRESENT			BIT_ULL(0)
+#define XE_PAGE_RW			BIT_ULL(1)
+
+#endif
diff --git a/drivers/gpu/drm/xe/regs/xe_guc_regs.h b/drivers/gpu/drm/xe/regs/xe_guc_regs.h
index 92320bbc9d3d..11682e675e0f 100644
--- a/drivers/gpu/drm/xe/regs/xe_guc_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_guc_regs.h
@@ -100,16 +100,23 @@
 #define GT_PM_CONFIG				XE_REG(0x13816c)
 #define   GT_DOORBELL_ENABLE			REG_BIT(0)
 
-#define GUC_HOST_INTERRUPT			XE_REG(0x1901f0)
+#define GUC_HOST_INTERRUPT			XE_REG(0x1901f0, XE_REG_OPTION_VF)
 
-#define VF_SW_FLAG(n)				XE_REG(0x190240 + (n) * 4)
+#define VF_SW_FLAG(n)				XE_REG(0x190240 + (n) * 4, XE_REG_OPTION_VF)
 #define VF_SW_FLAG_COUNT			4
 
-#define MED_GUC_HOST_INTERRUPT			XE_REG(0x190304)
+#define MED_GUC_HOST_INTERRUPT			XE_REG(0x190304, XE_REG_OPTION_VF)
 
-#define MED_VF_SW_FLAG(n)			XE_REG(0x190310 + (n) * 4)
+#define MED_VF_SW_FLAG(n)			XE_REG(0x190310 + (n) * 4, XE_REG_OPTION_VF)
 #define MED_VF_SW_FLAG_COUNT			4
 
+#define GUC_TLB_INV_CR				XE_REG(0xcee8)
+#define   GUC_TLB_INV_CR_INVALIDATE		REG_BIT(0)
+#define PVC_GUC_TLB_INV_DESC0			XE_REG(0xcf7c)
+#define   PVC_GUC_TLB_INV_DESC0_VALID		REG_BIT(0)
+#define PVC_GUC_TLB_INV_DESC1			XE_REG(0xcf80)
+#define   PVC_GUC_TLB_INV_DESC1_INVALIDATE	REG_BIT(6)
+
 /* GuC Interrupt Vector */
 #define GUC_INTR_GUC2HOST			REG_BIT(15)
 #define GUC_INTR_EXEC_ERROR			REG_BIT(14)
diff --git a/drivers/gpu/drm/xe/regs/xe_reg_defs.h b/drivers/gpu/drm/xe/regs/xe_reg_defs.h
index c50e7650c09a..23f7dc5bbe99 100644
--- a/drivers/gpu/drm/xe/regs/xe_reg_defs.h
+++ b/drivers/gpu/drm/xe/regs/xe_reg_defs.h
@@ -6,6 +6,8 @@
 #ifndef _XE_REG_DEFS_H_
 #define _XE_REG_DEFS_H_
 
+#include <linux/build_bug.h>
+
 #include "compat-i915-headers/i915_reg_defs.h"
 
 /**
@@ -36,6 +38,10 @@ struct xe_reg {
 			 */
 			u32 mcr:1;
 			/**
+			 * @vf: register is accessible from the Virtual Function.
+			 */
+			u32 vf:1;
+			/**
 			 * @ext: access MMIO extension space for current register.
 			 */
 			u32 ext:1;
@@ -44,6 +50,7 @@ struct xe_reg {
 		u32 raw;
 	};
 };
+static_assert(sizeof(struct xe_reg) == sizeof(u32));
 
 /**
  * struct xe_reg_mcr - MCR register definition
@@ -76,6 +83,13 @@ struct xe_reg_mcr {
 #define XE_REG_OPTION_MASKED		.masked = 1
 
 /**
+ * XE_REG_OPTION_VF - Register is "VF" accessible.
+ *
+ * To be used with XE_REG() and XE_REG_INITIALIZER().
+ */
+#define XE_REG_OPTION_VF		.vf = 1
+
+/**
  * XE_REG_INITIALIZER - Initializer for xe_reg_t.
  * @r_: Register offset
  * @...: Additional options like access mode. See struct xe_reg for available
@@ -117,4 +131,9 @@ struct xe_reg_mcr {
 				 .__reg = XE_REG_INITIALIZER(r_,  ##__VA_ARGS__, .mcr = 1)	\
 				 })
 
+static inline bool xe_reg_is_valid(struct xe_reg r)
+{
+	return r.addr;
+}
+
 #endif
diff --git a/drivers/gpu/drm/xe/regs/xe_regs.h b/drivers/gpu/drm/xe/regs/xe_regs.h
index 2c214bb9b671..722fb6dbb72e 100644
--- a/drivers/gpu/drm/xe/regs/xe_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_regs.h
@@ -57,7 +57,7 @@
 #define   DG1_MSTR_IRQ				REG_BIT(31)
 #define   DG1_MSTR_TILE(t)			REG_BIT(t)
 
-#define GFX_MSTR_IRQ				XE_REG(0x190010)
+#define GFX_MSTR_IRQ				XE_REG(0x190010, XE_REG_OPTION_VF)
 #define   MASTER_IRQ				REG_BIT(31)
 #define   GU_MISC_IRQ				REG_BIT(29)
 #define   DISPLAY_IRQ				REG_BIT(16)
diff --git a/drivers/gpu/drm/xe/regs/xe_sriov_regs.h b/drivers/gpu/drm/xe/regs/xe_sriov_regs.h
index 58a4e0fad1e1..617ddb84b7fa 100644
--- a/drivers/gpu/drm/xe/regs/xe_sriov_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_sriov_regs.h
@@ -14,4 +14,7 @@
 #define   LMEM_EN			REG_BIT(31)
 #define   LMTT_DIR_PTR			REG_GENMASK(30, 0) /* in multiples of 64KB */
 
+#define VF_CAP_REG			XE_REG(0x1901f8, XE_REG_OPTION_VF)
+#define   VF_CAP			REG_BIT(0)
+
 #endif
diff --git a/drivers/gpu/drm/xe/tests/Makefile b/drivers/gpu/drm/xe/tests/Makefile
index 9d1d88af8b2f..8cf2367449d8 100644
--- a/drivers/gpu/drm/xe/tests/Makefile
+++ b/drivers/gpu/drm/xe/tests/Makefile
@@ -1,7 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 
 # "live" kunit tests
-obj-$(CONFIG_DRM_XE_KUNIT_TEST) += \
+obj-$(CONFIG_DRM_XE_KUNIT_TEST) += xe_live_test.o
+xe_live_test-y = xe_live_test_mod.o \
 	xe_bo_test.o \
 	xe_dma_buf_test.o \
 	xe_migrate_test.o \
diff --git a/drivers/gpu/drm/xe/tests/xe_bo.c b/drivers/gpu/drm/xe/tests/xe_bo.c
index 3436fd9cf2b2..9f3c02826464 100644
--- a/drivers/gpu/drm/xe/tests/xe_bo.c
+++ b/drivers/gpu/drm/xe/tests/xe_bo.c
@@ -116,7 +116,7 @@ static void ccs_test_run_tile(struct xe_device *xe, struct xe_tile *tile,
 	int ret;
 
 	/* TODO: Sanity check */
-	unsigned int bo_flags = XE_BO_CREATE_VRAM_IF_DGFX(tile);
+	unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile);
 
 	if (IS_DGFX(xe))
 		kunit_info(test, "Testing vram id %u\n", tile->id);
@@ -163,7 +163,7 @@ static int ccs_test_run_device(struct xe_device *xe)
 		return 0;
 	}
 
-	xe_device_mem_access_get(xe);
+	xe_pm_runtime_get(xe);
 
 	for_each_tile(tile, xe, id) {
 		/* For igfx run only for primary tile */
@@ -172,7 +172,7 @@ static int ccs_test_run_device(struct xe_device *xe)
 		ccs_test_run_tile(xe, tile, test);
 	}
 
-	xe_device_mem_access_put(xe);
+	xe_pm_runtime_put(xe);
 
 	return 0;
 }
@@ -186,7 +186,7 @@ EXPORT_SYMBOL_IF_KUNIT(xe_ccs_migrate_kunit);
 static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struct kunit *test)
 {
 	struct xe_bo *bo, *external;
-	unsigned int bo_flags = XE_BO_CREATE_VRAM_IF_DGFX(tile);
+	unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile);
 	struct xe_vm *vm = xe_migrate_get_vm(xe_device_get_root_tile(xe)->migrate);
 	struct xe_gt *__gt;
 	int err, i, id;
@@ -335,12 +335,12 @@ static int evict_test_run_device(struct xe_device *xe)
 		return 0;
 	}
 
-	xe_device_mem_access_get(xe);
+	xe_pm_runtime_get(xe);
 
 	for_each_tile(tile, xe, id)
 		evict_test_run_tile(xe, tile, test);
 
-	xe_device_mem_access_put(xe);
+	xe_pm_runtime_put(xe);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/xe/tests/xe_bo_test.c b/drivers/gpu/drm/xe/tests/xe_bo_test.c
index f408f17f2164..a324cde77db8 100644
--- a/drivers/gpu/drm/xe/tests/xe_bo_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_bo_test.c
@@ -19,8 +19,3 @@ static struct kunit_suite xe_bo_test_suite = {
 };
 
 kunit_test_suite(xe_bo_test_suite);
-
-MODULE_AUTHOR("Intel Corporation");
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("xe_bo kunit test");
-MODULE_IMPORT_NS(EXPORTED_FOR_KUNIT_TESTING);
diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf.c b/drivers/gpu/drm/xe/tests/xe_dma_buf.c
index 9f6d571d7fa9..e7f9b531c465 100644
--- a/drivers/gpu/drm/xe/tests/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/tests/xe_dma_buf.c
@@ -12,6 +12,7 @@
 #include "tests/xe_pci_test.h"
 
 #include "xe_pci.h"
+#include "xe_pm.h"
 
 static bool p2p_enabled(struct dma_buf_test_params *params)
 {
@@ -36,14 +37,14 @@ static void check_residency(struct kunit *test, struct xe_bo *exported,
 	xe_bo_assert_held(imported);
 
 	mem_type = XE_PL_VRAM0;
-	if (!(params->mem_mask & XE_BO_CREATE_VRAM0_BIT))
+	if (!(params->mem_mask & XE_BO_FLAG_VRAM0))
 		/* No VRAM allowed */
 		mem_type = XE_PL_TT;
 	else if (params->force_different_devices && !p2p_enabled(params))
 		/* No P2P */
 		mem_type = XE_PL_TT;
 	else if (params->force_different_devices && !is_dynamic(params) &&
-		 (params->mem_mask & XE_BO_CREATE_SYSTEM_BIT))
+		 (params->mem_mask & XE_BO_FLAG_SYSTEM))
 		/* Pin migrated to TT */
 		mem_type = XE_PL_TT;
 
@@ -93,7 +94,7 @@ static void check_residency(struct kunit *test, struct xe_bo *exported,
 	 * possible, saving a migration step as the transfer is just
 	 * likely as fast from system memory.
 	 */
-	if (params->mem_mask & XE_BO_CREATE_SYSTEM_BIT)
+	if (params->mem_mask & XE_BO_FLAG_SYSTEM)
 		KUNIT_EXPECT_TRUE(test, xe_bo_is_mem_type(exported, XE_PL_TT));
 	else
 		KUNIT_EXPECT_TRUE(test, xe_bo_is_mem_type(exported, mem_type));
@@ -115,17 +116,17 @@ static void xe_test_dmabuf_import_same_driver(struct xe_device *xe)
 
 	/* No VRAM on this device? */
 	if (!ttm_manager_type(&xe->ttm, XE_PL_VRAM0) &&
-	    (params->mem_mask & XE_BO_CREATE_VRAM0_BIT))
+	    (params->mem_mask & XE_BO_FLAG_VRAM0))
 		return;
 
 	size = PAGE_SIZE;
-	if ((params->mem_mask & XE_BO_CREATE_VRAM0_BIT) &&
+	if ((params->mem_mask & XE_BO_FLAG_VRAM0) &&
 	    xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
 		size = SZ_64K;
 
 	kunit_info(test, "running %s\n", __func__);
 	bo = xe_bo_create_user(xe, NULL, NULL, size, DRM_XE_GEM_CPU_CACHING_WC,
-			       ttm_bo_type_device, XE_BO_CREATE_USER_BIT | params->mem_mask);
+			       ttm_bo_type_device, params->mem_mask);
 	if (IS_ERR(bo)) {
 		KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n",
 			   PTR_ERR(bo));
@@ -148,7 +149,7 @@ static void xe_test_dmabuf_import_same_driver(struct xe_device *xe)
 		 */
 		if (params->force_different_devices &&
 		    !p2p_enabled(params) &&
-		    !(params->mem_mask & XE_BO_CREATE_SYSTEM_BIT)) {
+		    !(params->mem_mask & XE_BO_FLAG_SYSTEM)) {
 			KUNIT_FAIL(test,
 				   "xe_gem_prime_import() succeeded when it shouldn't have\n");
 		} else {
@@ -161,7 +162,7 @@ static void xe_test_dmabuf_import_same_driver(struct xe_device *xe)
 			/* Pinning in VRAM is not allowed. */
 			if (!is_dynamic(params) &&
 			    params->force_different_devices &&
-			    !(params->mem_mask & XE_BO_CREATE_SYSTEM_BIT))
+			    !(params->mem_mask & XE_BO_FLAG_SYSTEM))
 				KUNIT_EXPECT_EQ(test, err, -EINVAL);
 			/* Otherwise only expect interrupts or success. */
 			else if (err && err != -EINTR && err != -ERESTARTSYS)
@@ -180,7 +181,7 @@ static void xe_test_dmabuf_import_same_driver(struct xe_device *xe)
 			   PTR_ERR(import));
 	} else if (!params->force_different_devices ||
 		   p2p_enabled(params) ||
-		   (params->mem_mask & XE_BO_CREATE_SYSTEM_BIT)) {
+		   (params->mem_mask & XE_BO_FLAG_SYSTEM)) {
 		/* Shouldn't fail if we can reuse same bo, use p2p or use system */
 		KUNIT_FAIL(test, "dynamic p2p attachment failed with err=%ld\n",
 			   PTR_ERR(import));
@@ -203,52 +204,52 @@ static const struct dma_buf_attach_ops nop2p_attach_ops = {
  * gem object.
  */
 static const struct dma_buf_test_params test_params[] = {
-	{.mem_mask = XE_BO_CREATE_VRAM0_BIT,
+	{.mem_mask = XE_BO_FLAG_VRAM0,
 	 .attach_ops = &xe_dma_buf_attach_ops},
-	{.mem_mask = XE_BO_CREATE_VRAM0_BIT,
+	{.mem_mask = XE_BO_FLAG_VRAM0,
 	 .attach_ops = &xe_dma_buf_attach_ops,
 	 .force_different_devices = true},
 
-	{.mem_mask = XE_BO_CREATE_VRAM0_BIT,
+	{.mem_mask = XE_BO_FLAG_VRAM0,
 	 .attach_ops = &nop2p_attach_ops},
-	{.mem_mask = XE_BO_CREATE_VRAM0_BIT,
+	{.mem_mask = XE_BO_FLAG_VRAM0,
 	 .attach_ops = &nop2p_attach_ops,
 	 .force_different_devices = true},
 
-	{.mem_mask = XE_BO_CREATE_VRAM0_BIT},
-	{.mem_mask = XE_BO_CREATE_VRAM0_BIT,
+	{.mem_mask = XE_BO_FLAG_VRAM0},
+	{.mem_mask = XE_BO_FLAG_VRAM0,
 	 .force_different_devices = true},
 
-	{.mem_mask = XE_BO_CREATE_SYSTEM_BIT,
+	{.mem_mask = XE_BO_FLAG_SYSTEM,
 	 .attach_ops = &xe_dma_buf_attach_ops},
-	{.mem_mask = XE_BO_CREATE_SYSTEM_BIT,
+	{.mem_mask = XE_BO_FLAG_SYSTEM,
 	 .attach_ops = &xe_dma_buf_attach_ops,
 	 .force_different_devices = true},
 
-	{.mem_mask = XE_BO_CREATE_SYSTEM_BIT,
+	{.mem_mask = XE_BO_FLAG_SYSTEM,
 	 .attach_ops = &nop2p_attach_ops},
-	{.mem_mask = XE_BO_CREATE_SYSTEM_BIT,
+	{.mem_mask = XE_BO_FLAG_SYSTEM,
 	 .attach_ops = &nop2p_attach_ops,
 	 .force_different_devices = true},
 
-	{.mem_mask = XE_BO_CREATE_SYSTEM_BIT},
-	{.mem_mask = XE_BO_CREATE_SYSTEM_BIT,
+	{.mem_mask = XE_BO_FLAG_SYSTEM},
+	{.mem_mask = XE_BO_FLAG_SYSTEM,
 	 .force_different_devices = true},
 
-	{.mem_mask = XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_VRAM0_BIT,
+	{.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0,
 	 .attach_ops = &xe_dma_buf_attach_ops},
-	{.mem_mask = XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_VRAM0_BIT,
+	{.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0,
 	 .attach_ops = &xe_dma_buf_attach_ops,
 	 .force_different_devices = true},
 
-	{.mem_mask = XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_VRAM0_BIT,
+	{.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0,
 	 .attach_ops = &nop2p_attach_ops},
-	{.mem_mask = XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_VRAM0_BIT,
+	{.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0,
 	 .attach_ops = &nop2p_attach_ops,
 	 .force_different_devices = true},
 
-	{.mem_mask = XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_VRAM0_BIT},
-	{.mem_mask = XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_VRAM0_BIT,
+	{.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0},
+	{.mem_mask = XE_BO_FLAG_SYSTEM | XE_BO_FLAG_VRAM0,
 	 .force_different_devices = true},
 
 	{}
@@ -259,6 +260,7 @@ static int dma_buf_run_device(struct xe_device *xe)
 	const struct dma_buf_test_params *params;
 	struct kunit *test = xe_cur_kunit();
 
+	xe_pm_runtime_get(xe);
 	for (params = test_params; params->mem_mask; ++params) {
 		struct dma_buf_test_params p = *params;
 
@@ -266,6 +268,7 @@ static int dma_buf_run_device(struct xe_device *xe)
 		test->priv = &p;
 		xe_test_dmabuf_import_same_driver(xe);
 	}
+	xe_pm_runtime_put(xe);
 
 	/* A non-zero return would halt iteration over driver devices */
 	return 0;
diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf_test.c b/drivers/gpu/drm/xe/tests/xe_dma_buf_test.c
index 9f5a9cda8c0f..99cdb718b6c6 100644
--- a/drivers/gpu/drm/xe/tests/xe_dma_buf_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_dma_buf_test.c
@@ -18,8 +18,3 @@ static struct kunit_suite xe_dma_buf_test_suite = {
 };
 
 kunit_test_suite(xe_dma_buf_test_suite);
-
-MODULE_AUTHOR("Intel Corporation");
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("xe_dma_buf kunit test");
-MODULE_IMPORT_NS(EXPORTED_FOR_KUNIT_TESTING);
diff --git a/drivers/gpu/drm/xe/tests/xe_guc_id_mgr_test.c b/drivers/gpu/drm/xe/tests/xe_guc_id_mgr_test.c
new file mode 100644
index 000000000000..ee30a1939eb0
--- /dev/null
+++ b/drivers/gpu/drm/xe/tests/xe_guc_id_mgr_test.c
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-2.0 AND MIT
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#include <kunit/test.h>
+
+#include "xe_device.h"
+#include "xe_kunit_helpers.h"
+
+static int guc_id_mgr_test_init(struct kunit *test)
+{
+	struct xe_guc_id_mgr *idm;
+
+	xe_kunit_helper_xe_device_test_init(test);
+	idm = &xe_device_get_gt(test->priv, 0)->uc.guc.submission_state.idm;
+
+	mutex_init(idm_mutex(idm));
+	test->priv = idm;
+	return 0;
+}
+
+static void bad_init(struct kunit *test)
+{
+	struct xe_guc_id_mgr *idm = test->priv;
+
+	KUNIT_EXPECT_EQ(test, -EINVAL, xe_guc_id_mgr_init(idm, 0));
+	KUNIT_EXPECT_EQ(test, -ERANGE, xe_guc_id_mgr_init(idm, GUC_ID_MAX + 1));
+}
+
+static void no_init(struct kunit *test)
+{
+	struct xe_guc_id_mgr *idm = test->priv;
+
+	mutex_lock(idm_mutex(idm));
+	KUNIT_EXPECT_EQ(test, -ENODATA, xe_guc_id_mgr_reserve_locked(idm, 0));
+	mutex_unlock(idm_mutex(idm));
+
+	KUNIT_EXPECT_EQ(test, -ENODATA, xe_guc_id_mgr_reserve(idm, 1, 1));
+}
+
+static void init_fini(struct kunit *test)
+{
+	struct xe_guc_id_mgr *idm = test->priv;
+
+	KUNIT_ASSERT_EQ(test, 0, xe_guc_id_mgr_init(idm, -1));
+	KUNIT_EXPECT_NOT_NULL(test, idm->bitmap);
+	KUNIT_EXPECT_EQ(test, idm->total, GUC_ID_MAX);
+	__fini_idm(NULL, idm);
+	KUNIT_EXPECT_NULL(test, idm->bitmap);
+	KUNIT_EXPECT_EQ(test, idm->total, 0);
+}
+
+static void check_used(struct kunit *test)
+{
+	struct xe_guc_id_mgr *idm = test->priv;
+	unsigned int n;
+
+	KUNIT_ASSERT_EQ(test, 0, xe_guc_id_mgr_init(idm, 2));
+
+	mutex_lock(idm_mutex(idm));
+
+	for (n = 0; n < idm->total; n++) {
+		kunit_info(test, "n=%u", n);
+		KUNIT_EXPECT_EQ(test, idm->used, n);
+		KUNIT_EXPECT_GE(test, idm_reserve_chunk_locked(idm, 1, 0), 0);
+		KUNIT_EXPECT_EQ(test, idm->used, n + 1);
+	}
+	KUNIT_EXPECT_EQ(test, idm->used, idm->total);
+	idm_release_chunk_locked(idm, 0, idm->used);
+	KUNIT_EXPECT_EQ(test, idm->used, 0);
+
+	mutex_unlock(idm_mutex(idm));
+}
+
+static void check_quota(struct kunit *test)
+{
+	struct xe_guc_id_mgr *idm = test->priv;
+	unsigned int n;
+
+	KUNIT_ASSERT_EQ(test, 0, xe_guc_id_mgr_init(idm, 2));
+
+	mutex_lock(idm_mutex(idm));
+
+	for (n = 0; n < idm->total - 1; n++) {
+		kunit_info(test, "n=%u", n);
+		KUNIT_EXPECT_EQ(test, idm_reserve_chunk_locked(idm, 1, idm->total), -EDQUOT);
+		KUNIT_EXPECT_EQ(test, idm_reserve_chunk_locked(idm, 1, idm->total - n), -EDQUOT);
+		KUNIT_EXPECT_EQ(test, idm_reserve_chunk_locked(idm, idm->total - n, 1), -EDQUOT);
+		KUNIT_EXPECT_GE(test, idm_reserve_chunk_locked(idm, 1, 1), 0);
+	}
+	KUNIT_EXPECT_LE(test, 0, idm_reserve_chunk_locked(idm, 1, 0));
+	KUNIT_EXPECT_EQ(test, idm->used, idm->total);
+	idm_release_chunk_locked(idm, 0, idm->total);
+	KUNIT_EXPECT_EQ(test, idm->used, 0);
+
+	mutex_unlock(idm_mutex(idm));
+}
+
+static void check_all(struct kunit *test)
+{
+	struct xe_guc_id_mgr *idm = test->priv;
+	unsigned int n;
+
+	KUNIT_ASSERT_EQ(test, 0, xe_guc_id_mgr_init(idm, -1));
+
+	mutex_lock(idm_mutex(idm));
+
+	for (n = 0; n < idm->total; n++)
+		KUNIT_EXPECT_LE(test, 0, idm_reserve_chunk_locked(idm, 1, 0));
+	KUNIT_EXPECT_EQ(test, idm->used, idm->total);
+	for (n = 0; n < idm->total; n++)
+		idm_release_chunk_locked(idm, n, 1);
+
+	mutex_unlock(idm_mutex(idm));
+}
+
+static struct kunit_case guc_id_mgr_test_cases[] = {
+	KUNIT_CASE(bad_init),
+	KUNIT_CASE(no_init),
+	KUNIT_CASE(init_fini),
+	KUNIT_CASE(check_used),
+	KUNIT_CASE(check_quota),
+	KUNIT_CASE_SLOW(check_all),
+	{}
+};
+
+static struct kunit_suite guc_id_mgr_suite = {
+	.name = "guc_idm",
+	.test_cases = guc_id_mgr_test_cases,
+
+	.init = guc_id_mgr_test_init,
+	.exit = NULL,
+};
+
+kunit_test_suites(&guc_id_mgr_suite);
diff --git a/drivers/gpu/drm/xe/tests/xe_live_test_mod.c b/drivers/gpu/drm/xe/tests/xe_live_test_mod.c
new file mode 100644
index 000000000000..eb1ea99a5a8b
--- /dev/null
+++ b/drivers/gpu/drm/xe/tests/xe_live_test_mod.c
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+#include <linux/module.h>
+
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("xe live kunit tests");
+MODULE_IMPORT_NS(EXPORTED_FOR_KUNIT_TESTING);
diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c
index c347e2c29f81..977d5f4e4490 100644
--- a/drivers/gpu/drm/xe/tests/xe_migrate.c
+++ b/drivers/gpu/drm/xe/tests/xe_migrate.c
@@ -10,6 +10,7 @@
 #include "tests/xe_pci_test.h"
 
 #include "xe_pci.h"
+#include "xe_pm.h"
 
 static bool sanity_fence_failed(struct xe_device *xe, struct dma_fence *fence,
 				const char *str, struct kunit *test)
@@ -112,7 +113,7 @@ static void test_copy(struct xe_migrate *m, struct xe_bo *bo,
 						   bo->size,
 						   ttm_bo_type_kernel,
 						   region |
-						   XE_BO_NEEDS_CPU_ACCESS);
+						   XE_BO_FLAG_NEEDS_CPU_ACCESS);
 	if (IS_ERR(remote)) {
 		KUNIT_FAIL(test, "Failed to allocate remote bo for %s: %pe\n",
 			   str, remote);
@@ -190,7 +191,7 @@ out_unlock:
 static void test_copy_sysmem(struct xe_migrate *m, struct xe_bo *bo,
 			     struct kunit *test)
 {
-	test_copy(m, bo, test, XE_BO_CREATE_SYSTEM_BIT);
+	test_copy(m, bo, test, XE_BO_FLAG_SYSTEM);
 }
 
 static void test_copy_vram(struct xe_migrate *m, struct xe_bo *bo,
@@ -202,9 +203,9 @@ static void test_copy_vram(struct xe_migrate *m, struct xe_bo *bo,
 		return;
 
 	if (bo->ttm.resource->mem_type == XE_PL_VRAM0)
-		region = XE_BO_CREATE_VRAM1_BIT;
+		region = XE_BO_FLAG_VRAM1;
 	else
-		region = XE_BO_CREATE_VRAM0_BIT;
+		region = XE_BO_FLAG_VRAM0;
 	test_copy(m, bo, test, region);
 }
 
@@ -280,8 +281,8 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test)
 
 	big = xe_bo_create_pin_map(xe, tile, m->q->vm, SZ_4M,
 				   ttm_bo_type_kernel,
-				   XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-				   XE_BO_CREATE_PINNED_BIT);
+				   XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+				   XE_BO_FLAG_PINNED);
 	if (IS_ERR(big)) {
 		KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big));
 		goto vunmap;
@@ -289,8 +290,8 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test)
 
 	pt = xe_bo_create_pin_map(xe, tile, m->q->vm, XE_PAGE_SIZE,
 				  ttm_bo_type_kernel,
-				  XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-				  XE_BO_CREATE_PINNED_BIT);
+				  XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+				  XE_BO_FLAG_PINNED);
 	if (IS_ERR(pt)) {
 		KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
 			   PTR_ERR(pt));
@@ -300,8 +301,8 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test)
 	tiny = xe_bo_create_pin_map(xe, tile, m->q->vm,
 				    2 * SZ_4K,
 				    ttm_bo_type_kernel,
-				    XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-				    XE_BO_CREATE_PINNED_BIT);
+				    XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+				    XE_BO_FLAG_PINNED);
 	if (IS_ERR(tiny)) {
 		KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
 			   PTR_ERR(pt));
@@ -423,17 +424,19 @@ static int migrate_test_run_device(struct xe_device *xe)
 	struct xe_tile *tile;
 	int id;
 
+	xe_pm_runtime_get(xe);
+
 	for_each_tile(tile, xe, id) {
 		struct xe_migrate *m = tile->migrate;
 
 		kunit_info(test, "Testing tile id %d.\n", id);
 		xe_vm_lock(m->q->vm, true);
-		xe_device_mem_access_get(xe);
 		xe_migrate_sanity_test(m, test);
-		xe_device_mem_access_put(xe);
 		xe_vm_unlock(m->q->vm);
 	}
 
+	xe_pm_runtime_put(xe);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/tests/xe_migrate_test.c b/drivers/gpu/drm/xe/tests/xe_migrate_test.c
index cf0c173b945f..eb0d8963419c 100644
--- a/drivers/gpu/drm/xe/tests/xe_migrate_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_migrate_test.c
@@ -18,8 +18,3 @@ static struct kunit_suite xe_migrate_test_suite = {
 };
 
 kunit_test_suite(xe_migrate_test_suite);
-
-MODULE_AUTHOR("Intel Corporation");
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("xe_migrate kunit test");
-MODULE_IMPORT_NS(EXPORTED_FOR_KUNIT_TESTING);
diff --git a/drivers/gpu/drm/xe/tests/xe_mocs.c b/drivers/gpu/drm/xe/tests/xe_mocs.c
index df5c36b70ab4..1b8617075b37 100644
--- a/drivers/gpu/drm/xe/tests/xe_mocs.c
+++ b/drivers/gpu/drm/xe/tests/xe_mocs.c
@@ -10,10 +10,11 @@
 #include "tests/xe_pci_test.h"
 #include "tests/xe_test.h"
 
-#include "xe_pci.h"
+#include "xe_device.h"
 #include "xe_gt.h"
 #include "xe_mocs.h"
-#include "xe_device.h"
+#include "xe_pci.h"
+#include "xe_pm.h"
 
 struct live_mocs {
 	struct xe_mocs_info table;
@@ -28,6 +29,8 @@ static int live_mocs_init(struct live_mocs *arg, struct xe_gt *gt)
 
 	flags = get_mocs_settings(gt_to_xe(gt), &arg->table);
 
+	kunit_info(test, "gt %d", gt->info.id);
+	kunit_info(test, "gt type %d", gt->info.type);
 	kunit_info(test, "table size %d", arg->table.size);
 	kunit_info(test, "table uc_index %d", arg->table.uc_index);
 	kunit_info(test, "table n_entries %d", arg->table.n_entries);
@@ -38,69 +41,72 @@ static int live_mocs_init(struct live_mocs *arg, struct xe_gt *gt)
 static void read_l3cc_table(struct xe_gt *gt,
 			    const struct xe_mocs_info *info)
 {
+	struct kunit *test = xe_cur_kunit();
+	u32 l3cc, l3cc_expected;
 	unsigned int i;
-	u32 l3cc;
 	u32 reg_val;
 	u32 ret;
 
-	struct kunit *test = xe_cur_kunit();
-
-	xe_device_mem_access_get(gt_to_xe(gt));
 	ret = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 	KUNIT_ASSERT_EQ_MSG(test, ret, 0, "Forcewake Failed.\n");
-	mocs_dbg(&gt_to_xe(gt)->drm, "L3CC entries:%d\n", info->n_entries);
-	for (i = 0;
-	     i < (info->n_entries + 1) / 2 ?
-	     (l3cc = l3cc_combine(get_entry_l3cc(info, 2 * i),
-				  get_entry_l3cc(info, 2 * i + 1))), 1 : 0;
-	     i++) {
-		if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250)
-			reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i));
-		else
-			reg_val = xe_mmio_read32(gt, XELP_LNCFCMOCS(i));
-		mocs_dbg(&gt_to_xe(gt)->drm, "%d 0x%x 0x%x 0x%x\n", i,
-			 XELP_LNCFCMOCS(i).addr, reg_val, l3cc);
-		if (reg_val != l3cc)
-			KUNIT_FAIL(test, "l3cc reg 0x%x has incorrect val.\n",
-				   XELP_LNCFCMOCS(i).addr);
+
+	for (i = 0; i < info->n_entries; i++) {
+		if (!(i & 1)) {
+			if (regs_are_mcr(gt))
+				reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i >> 1));
+			else
+				reg_val = xe_mmio_read32(gt, XELP_LNCFCMOCS(i >> 1));
+
+			mocs_dbg(gt, "reg_val=0x%x\n", reg_val);
+		} else {
+			/* Just re-use value read on previous iteration */
+			reg_val >>= 16;
+		}
+
+		l3cc_expected = get_entry_l3cc(info, i);
+		l3cc = reg_val & 0xffff;
+
+		mocs_dbg(gt, "[%u] expected=0x%x actual=0x%x\n",
+			 i, l3cc_expected, l3cc);
+
+		KUNIT_EXPECT_EQ_MSG(test, l3cc_expected, l3cc,
+				    "l3cc idx=%u has incorrect val.\n", i);
 	}
 	xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
-	xe_device_mem_access_put(gt_to_xe(gt));
 }
 
 static void read_mocs_table(struct xe_gt *gt,
 			    const struct xe_mocs_info *info)
 {
-	struct xe_device *xe = gt_to_xe(gt);
-
+	struct kunit *test = xe_cur_kunit();
+	u32 mocs, mocs_expected;
 	unsigned int i;
-	u32 mocs;
 	u32 reg_val;
 	u32 ret;
 
-	struct kunit *test = xe_cur_kunit();
+	KUNIT_EXPECT_TRUE_MSG(test, info->unused_entries_index,
+			      "Unused entries index should have been defined\n");
 
-	xe_device_mem_access_get(gt_to_xe(gt));
 	ret = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 	KUNIT_ASSERT_EQ_MSG(test, ret, 0, "Forcewake Failed.\n");
-	mocs_dbg(&gt_to_xe(gt)->drm, "Global MOCS entries:%d\n", info->n_entries);
-	drm_WARN_ONCE(&xe->drm, !info->unused_entries_index,
-		      "Unused entries index should have been defined\n");
-	for (i = 0;
-	     i < info->n_entries ? (mocs = get_entry_control(info, i)), 1 : 0;
-	     i++) {
-		if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250)
+
+	for (i = 0; i < info->n_entries; i++) {
+		if (regs_are_mcr(gt))
 			reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i));
 		else
 			reg_val = xe_mmio_read32(gt, XELP_GLOBAL_MOCS(i));
-		mocs_dbg(&gt_to_xe(gt)->drm, "%d 0x%x 0x%x 0x%x\n", i,
-			 XELP_GLOBAL_MOCS(i).addr, reg_val, mocs);
-		if (reg_val != mocs)
-			KUNIT_FAIL(test, "mocs reg 0x%x has incorrect val.\n",
-				   XELP_GLOBAL_MOCS(i).addr);
+
+		mocs_expected = get_entry_control(info, i);
+		mocs = reg_val;
+
+		mocs_dbg(gt, "[%u] expected=0x%x actual=0x%x\n",
+			 i, mocs_expected, mocs);
+
+		KUNIT_EXPECT_EQ_MSG(test, mocs_expected, mocs,
+				    "mocs reg 0x%x has incorrect val.\n", i);
 	}
+
 	xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
-	xe_device_mem_access_put(gt_to_xe(gt));
 }
 
 static int mocs_kernel_test_run_device(struct xe_device *xe)
@@ -113,6 +119,8 @@ static int mocs_kernel_test_run_device(struct xe_device *xe)
 	unsigned int flags;
 	int id;
 
+	xe_pm_runtime_get(xe);
+
 	for_each_gt(gt, xe, id) {
 		flags = live_mocs_init(&mocs, gt);
 		if (flags & HAS_GLOBAL_MOCS)
@@ -120,6 +128,9 @@ static int mocs_kernel_test_run_device(struct xe_device *xe)
 		if (flags & HAS_LNCF_MOCS)
 			read_l3cc_table(gt, &mocs.table);
 	}
+
+	xe_pm_runtime_put(xe);
+
 	return 0;
 }
 
@@ -139,6 +150,8 @@ static int mocs_reset_test_run_device(struct xe_device *xe)
 	int id;
 	struct kunit *test = xe_cur_kunit();
 
+	xe_pm_runtime_get(xe);
+
 	for_each_gt(gt, xe, id) {
 		flags = live_mocs_init(&mocs, gt);
 		kunit_info(test, "mocs_reset_test before reset\n");
@@ -156,6 +169,9 @@ static int mocs_reset_test_run_device(struct xe_device *xe)
 		if (flags & HAS_LNCF_MOCS)
 			read_l3cc_table(gt, &mocs.table);
 	}
+
+	xe_pm_runtime_put(xe);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/tests/xe_mocs_test.c b/drivers/gpu/drm/xe/tests/xe_mocs_test.c
index ee40f31e1e12..6315886b659e 100644
--- a/drivers/gpu/drm/xe/tests/xe_mocs_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_mocs_test.c
@@ -19,8 +19,3 @@ static struct kunit_suite xe_mocs_test_suite = {
 };
 
 kunit_test_suite(xe_mocs_test_suite);
-
-MODULE_AUTHOR("Intel Corporation");
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("xe_mocs kunit test");
-MODULE_IMPORT_NS(EXPORTED_FOR_KUNIT_TESTING);
diff --git a/drivers/gpu/drm/xe/tests/xe_wa_test.c b/drivers/gpu/drm/xe/tests/xe_wa_test.c
index 44570d888355..9d0c715142b9 100644
--- a/drivers/gpu/drm/xe/tests/xe_wa_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_wa_test.c
@@ -71,6 +71,7 @@ static const struct platform_test_case cases[] = {
 	SUBPLATFORM_CASE(DG2, G12, A1),
 	GMDID_CASE(METEORLAKE, 1270, A0, 1300, A0),
 	GMDID_CASE(METEORLAKE, 1271, A0, 1300, A0),
+	GMDID_CASE(METEORLAKE, 1274, A0, 1300, A0),
 	GMDID_CASE(LUNARLAKE, 2004, A0, 2000, A0),
 	GMDID_CASE(LUNARLAKE, 2004, B0, 2000, A0),
 };
diff --git a/drivers/gpu/drm/xe/xe_bb.c b/drivers/gpu/drm/xe/xe_bb.c
index 7c124475c428..541361caff3b 100644
--- a/drivers/gpu/drm/xe/xe_bb.c
+++ b/drivers/gpu/drm/xe/xe_bb.c
@@ -86,7 +86,8 @@ struct xe_sched_job *xe_bb_create_migration_job(struct xe_exec_queue *q,
 	};
 
 	xe_gt_assert(q->gt, second_idx <= bb->len);
-	xe_gt_assert(q->gt, q->vm->flags & XE_VM_FLAG_MIGRATION);
+	xe_gt_assert(q->gt, xe_sched_job_is_migration(q));
+	xe_gt_assert(q->gt, q->width == 1);
 
 	return __xe_bb_create_job(q, bb, addr);
 }
@@ -96,7 +97,8 @@ struct xe_sched_job *xe_bb_create_job(struct xe_exec_queue *q,
 {
 	u64 addr = xe_sa_bo_gpu_addr(bb->bo);
 
-	xe_gt_assert(q->gt, !(q->vm && q->vm->flags & XE_VM_FLAG_MIGRATION));
+	xe_gt_assert(q->gt, !xe_sched_job_is_migration(q));
+	xe_gt_assert(q->gt, q->width == 1);
 	return __xe_bb_create_job(q, bb, &addr);
 }
 
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 9c0837b6fdfc..bc1f794e3e61 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -22,6 +22,7 @@
 #include "xe_gt.h"
 #include "xe_map.h"
 #include "xe_migrate.h"
+#include "xe_pm.h"
 #include "xe_preempt_fence.h"
 #include "xe_res_cursor.h"
 #include "xe_trace.h"
@@ -111,7 +112,7 @@ bool xe_bo_is_stolen_devmem(struct xe_bo *bo)
 
 static bool xe_bo_is_user(struct xe_bo *bo)
 {
-	return bo->flags & XE_BO_CREATE_USER_BIT;
+	return bo->flags & XE_BO_FLAG_USER;
 }
 
 static struct xe_migrate *
@@ -137,7 +138,7 @@ static struct xe_mem_region *res_to_mem_region(struct ttm_resource *res)
 static void try_add_system(struct xe_device *xe, struct xe_bo *bo,
 			   u32 bo_flags, u32 *c)
 {
-	if (bo_flags & XE_BO_CREATE_SYSTEM_BIT) {
+	if (bo_flags & XE_BO_FLAG_SYSTEM) {
 		xe_assert(xe, *c < ARRAY_SIZE(bo->placements));
 
 		bo->placements[*c] = (struct ttm_place) {
@@ -164,12 +165,12 @@ static void add_vram(struct xe_device *xe, struct xe_bo *bo,
 	 * For eviction / restore on suspend / resume objects
 	 * pinned in VRAM must be contiguous
 	 */
-	if (bo_flags & (XE_BO_CREATE_PINNED_BIT |
-			XE_BO_CREATE_GGTT_BIT))
+	if (bo_flags & (XE_BO_FLAG_PINNED |
+			XE_BO_FLAG_GGTT))
 		place.flags |= TTM_PL_FLAG_CONTIGUOUS;
 
 	if (io_size < vram->usable_size) {
-		if (bo_flags & XE_BO_NEEDS_CPU_ACCESS) {
+		if (bo_flags & XE_BO_FLAG_NEEDS_CPU_ACCESS) {
 			place.fpfn = 0;
 			place.lpfn = io_size >> PAGE_SHIFT;
 		} else {
@@ -183,22 +184,22 @@ static void add_vram(struct xe_device *xe, struct xe_bo *bo,
 static void try_add_vram(struct xe_device *xe, struct xe_bo *bo,
 			 u32 bo_flags, u32 *c)
 {
-	if (bo_flags & XE_BO_CREATE_VRAM0_BIT)
+	if (bo_flags & XE_BO_FLAG_VRAM0)
 		add_vram(xe, bo, bo->placements, bo_flags, XE_PL_VRAM0, c);
-	if (bo_flags & XE_BO_CREATE_VRAM1_BIT)
+	if (bo_flags & XE_BO_FLAG_VRAM1)
 		add_vram(xe, bo, bo->placements, bo_flags, XE_PL_VRAM1, c);
 }
 
 static void try_add_stolen(struct xe_device *xe, struct xe_bo *bo,
 			   u32 bo_flags, u32 *c)
 {
-	if (bo_flags & XE_BO_CREATE_STOLEN_BIT) {
+	if (bo_flags & XE_BO_FLAG_STOLEN) {
 		xe_assert(xe, *c < ARRAY_SIZE(bo->placements));
 
 		bo->placements[*c] = (struct ttm_place) {
 			.mem_type = XE_PL_STOLEN,
-			.flags = bo_flags & (XE_BO_CREATE_PINNED_BIT |
-					     XE_BO_CREATE_GGTT_BIT) ?
+			.flags = bo_flags & (XE_BO_FLAG_PINNED |
+					     XE_BO_FLAG_GGTT) ?
 				TTM_PL_FLAG_CONTIGUOUS : 0,
 		};
 		*c += 1;
@@ -339,7 +340,7 @@ static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo,
 		break;
 	}
 
-	WARN_ON((bo->flags & XE_BO_CREATE_USER_BIT) && !bo->cpu_caching);
+	WARN_ON((bo->flags & XE_BO_FLAG_USER) && !bo->cpu_caching);
 
 	/*
 	 * Display scanout is always non-coherent with the CPU cache.
@@ -347,8 +348,8 @@ static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo,
 	 * For Xe_LPG and beyond, PPGTT PTE lookups are also non-coherent and
 	 * require a CPU:WC mapping.
 	 */
-	if ((!bo->cpu_caching && bo->flags & XE_BO_SCANOUT_BIT) ||
-	    (xe->info.graphics_verx100 >= 1270 && bo->flags & XE_BO_PAGETABLE))
+	if ((!bo->cpu_caching && bo->flags & XE_BO_FLAG_SCANOUT) ||
+	    (xe->info.graphics_verx100 >= 1270 && bo->flags & XE_BO_FLAG_PAGETABLE))
 		caching = ttm_write_combined;
 
 	err = ttm_tt_init(&tt->ttm, &bo->ttm, page_flags, caching, extra_pages);
@@ -715,7 +716,7 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
 
 	xe_assert(xe, migrate);
 	trace_xe_bo_move(bo, new_mem->mem_type, old_mem_type, move_lacks_source);
-	xe_device_mem_access_get(xe);
+	xe_pm_runtime_get_noresume(xe);
 
 	if (xe_bo_is_pinned(bo) && !xe_bo_is_user(bo)) {
 		/*
@@ -739,7 +740,7 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
 
 				if (XE_WARN_ON(new_mem->start == XE_BO_INVALID_OFFSET)) {
 					ret = -EINVAL;
-					xe_device_mem_access_put(xe);
+					xe_pm_runtime_put(xe);
 					goto out;
 				}
 
@@ -757,7 +758,7 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
 						new_mem, handle_system_ccs);
 		if (IS_ERR(fence)) {
 			ret = PTR_ERR(fence);
-			xe_device_mem_access_put(xe);
+			xe_pm_runtime_put(xe);
 			goto out;
 		}
 		if (!move_lacks_source) {
@@ -782,7 +783,7 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
 		dma_fence_put(fence);
 	}
 
-	xe_device_mem_access_put(xe);
+	xe_pm_runtime_put(xe);
 
 out:
 	return ret;
@@ -794,7 +795,6 @@ out:
  * @bo: The buffer object to move.
  *
  * On successful completion, the object memory will be moved to sytem memory.
- * This function blocks until the object has been fully moved.
  *
  * This is needed to for special handling of pinned VRAM object during
  * suspend-resume.
@@ -851,9 +851,6 @@ int xe_bo_evict_pinned(struct xe_bo *bo)
 	if (ret)
 		goto err_res_free;
 
-	dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL,
-			      false, MAX_SCHEDULE_TIMEOUT);
-
 	return 0;
 
 err_res_free:
@@ -866,7 +863,6 @@ err_res_free:
  * @bo: The buffer object to move.
  *
  * On successful completion, the object memory will be moved back to VRAM.
- * This function blocks until the object has been fully moved.
  *
  * This is needed to for special handling of pinned VRAM object during
  * suspend-resume.
@@ -908,9 +904,6 @@ int xe_bo_restore_pinned(struct xe_bo *bo)
 	if (ret)
 		goto err_res_free;
 
-	dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL,
-			      false, MAX_SCHEDULE_TIMEOUT);
-
 	return 0;
 
 err_res_free:
@@ -1110,12 +1103,12 @@ static vm_fault_t xe_gem_fault(struct vm_fault *vmf)
 	struct drm_device *ddev = tbo->base.dev;
 	struct xe_device *xe = to_xe_device(ddev);
 	struct xe_bo *bo = ttm_to_xe_bo(tbo);
-	bool needs_rpm = bo->flags & XE_BO_CREATE_VRAM_MASK;
+	bool needs_rpm = bo->flags & XE_BO_FLAG_VRAM_MASK;
 	vm_fault_t ret;
 	int idx;
 
 	if (needs_rpm)
-		xe_device_mem_access_get(xe);
+		xe_pm_runtime_get(xe);
 
 	ret = ttm_bo_vm_reserve(tbo, vmf);
 	if (ret)
@@ -1146,7 +1139,7 @@ static vm_fault_t xe_gem_fault(struct vm_fault *vmf)
 	dma_resv_unlock(tbo->base.resv);
 out:
 	if (needs_rpm)
-		xe_device_mem_access_put(xe);
+		xe_pm_runtime_put(xe);
 
 	return ret;
 }
@@ -1223,18 +1216,19 @@ struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo,
 		return ERR_PTR(-EINVAL);
 	}
 
-	if (flags & (XE_BO_CREATE_VRAM_MASK | XE_BO_CREATE_STOLEN_BIT) &&
-	    !(flags & XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT) &&
-	    xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) {
+	if (flags & (XE_BO_FLAG_VRAM_MASK | XE_BO_FLAG_STOLEN) &&
+	    !(flags & XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE) &&
+	    ((xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) ||
+	     (flags & XE_BO_NEEDS_64K))) {
 		aligned_size = ALIGN(size, SZ_64K);
 		if (type != ttm_bo_type_device)
 			size = ALIGN(size, SZ_64K);
-		flags |= XE_BO_INTERNAL_64K;
+		flags |= XE_BO_FLAG_INTERNAL_64K;
 		alignment = SZ_64K >> PAGE_SHIFT;
 
 	} else {
 		aligned_size = ALIGN(size, SZ_4K);
-		flags &= ~XE_BO_INTERNAL_64K;
+		flags &= ~XE_BO_FLAG_INTERNAL_64K;
 		alignment = SZ_4K >> PAGE_SHIFT;
 	}
 
@@ -1263,11 +1257,11 @@ struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo,
 	drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size);
 
 	if (resv) {
-		ctx.allow_res_evict = !(flags & XE_BO_CREATE_NO_RESV_EVICT);
+		ctx.allow_res_evict = !(flags & XE_BO_FLAG_NO_RESV_EVICT);
 		ctx.resv = resv;
 	}
 
-	if (!(flags & XE_BO_FIXED_PLACEMENT_BIT)) {
+	if (!(flags & XE_BO_FLAG_FIXED_PLACEMENT)) {
 		err = __xe_bo_placement_for_flags(xe, bo, bo->flags);
 		if (WARN_ON(err)) {
 			xe_ttm_bo_destroy(&bo->ttm);
@@ -1277,7 +1271,7 @@ struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo,
 
 	/* Defer populating type_sg bos */
 	placement = (type == ttm_bo_type_sg ||
-		     bo->flags & XE_BO_DEFER_BACKING) ? &sys_placement :
+		     bo->flags & XE_BO_FLAG_DEFER_BACKING) ? &sys_placement :
 		&bo->placement;
 	err = ttm_bo_init_reserved(&xe->ttm, &bo->ttm, type,
 				   placement, alignment,
@@ -1332,21 +1326,21 @@ static int __xe_bo_fixed_placement(struct xe_device *xe,
 {
 	struct ttm_place *place = bo->placements;
 
-	if (flags & (XE_BO_CREATE_USER_BIT|XE_BO_CREATE_SYSTEM_BIT))
+	if (flags & (XE_BO_FLAG_USER | XE_BO_FLAG_SYSTEM))
 		return -EINVAL;
 
 	place->flags = TTM_PL_FLAG_CONTIGUOUS;
 	place->fpfn = start >> PAGE_SHIFT;
 	place->lpfn = end >> PAGE_SHIFT;
 
-	switch (flags & (XE_BO_CREATE_STOLEN_BIT | XE_BO_CREATE_VRAM_MASK)) {
-	case XE_BO_CREATE_VRAM0_BIT:
+	switch (flags & (XE_BO_FLAG_STOLEN | XE_BO_FLAG_VRAM_MASK)) {
+	case XE_BO_FLAG_VRAM0:
 		place->mem_type = XE_PL_VRAM0;
 		break;
-	case XE_BO_CREATE_VRAM1_BIT:
+	case XE_BO_FLAG_VRAM1:
 		place->mem_type = XE_PL_VRAM1;
 		break;
-	case XE_BO_CREATE_STOLEN_BIT:
+	case XE_BO_FLAG_STOLEN:
 		place->mem_type = XE_PL_STOLEN;
 		break;
 
@@ -1380,7 +1374,7 @@ __xe_bo_create_locked(struct xe_device *xe,
 		if (IS_ERR(bo))
 			return bo;
 
-		flags |= XE_BO_FIXED_PLACEMENT_BIT;
+		flags |= XE_BO_FLAG_FIXED_PLACEMENT;
 		err = __xe_bo_fixed_placement(xe, bo, flags, start, end, size);
 		if (err) {
 			xe_bo_free(bo);
@@ -1390,7 +1384,7 @@ __xe_bo_create_locked(struct xe_device *xe,
 
 	bo = ___xe_bo_create_locked(xe, bo, tile, vm ? xe_vm_resv(vm) : NULL,
 				    vm && !xe_vm_in_fault_mode(vm) &&
-				    flags & XE_BO_CREATE_USER_BIT ?
+				    flags & XE_BO_FLAG_USER ?
 				    &vm->lru_bulk_move : NULL, size,
 				    cpu_caching, type, flags);
 	if (IS_ERR(bo))
@@ -1407,13 +1401,13 @@ __xe_bo_create_locked(struct xe_device *xe,
 		xe_vm_get(vm);
 	bo->vm = vm;
 
-	if (bo->flags & XE_BO_CREATE_GGTT_BIT) {
-		if (!tile && flags & XE_BO_CREATE_STOLEN_BIT)
+	if (bo->flags & XE_BO_FLAG_GGTT) {
+		if (!tile && flags & XE_BO_FLAG_STOLEN)
 			tile = xe_device_get_root_tile(xe);
 
 		xe_assert(xe, tile);
 
-		if (flags & XE_BO_FIXED_PLACEMENT_BIT) {
+		if (flags & XE_BO_FLAG_FIXED_PLACEMENT) {
 			err = xe_ggtt_insert_bo_at(tile->mem.ggtt, bo,
 						   start + bo->size, U64_MAX);
 		} else {
@@ -1456,7 +1450,7 @@ struct xe_bo *xe_bo_create_user(struct xe_device *xe, struct xe_tile *tile,
 {
 	struct xe_bo *bo = __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL,
 						 cpu_caching, type,
-						 flags | XE_BO_CREATE_USER_BIT);
+						 flags | XE_BO_FLAG_USER);
 	if (!IS_ERR(bo))
 		xe_bo_unlock_vm_held(bo);
 
@@ -1485,12 +1479,12 @@ struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_tile *tile
 	u64 start = offset == ~0ull ? 0 : offset;
 	u64 end = offset == ~0ull ? offset : start + size;
 
-	if (flags & XE_BO_CREATE_STOLEN_BIT &&
+	if (flags & XE_BO_FLAG_STOLEN &&
 	    xe_ttm_stolen_cpu_access_needs_ggtt(xe))
-		flags |= XE_BO_CREATE_GGTT_BIT;
+		flags |= XE_BO_FLAG_GGTT;
 
 	bo = xe_bo_create_locked_range(xe, tile, vm, size, start, end, type,
-				       flags | XE_BO_NEEDS_CPU_ACCESS);
+				       flags | XE_BO_FLAG_NEEDS_CPU_ACCESS);
 	if (IS_ERR(bo))
 		return bo;
 
@@ -1587,13 +1581,15 @@ struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_til
 int xe_managed_bo_reinit_in_vram(struct xe_device *xe, struct xe_tile *tile, struct xe_bo **src)
 {
 	struct xe_bo *bo;
+	u32 dst_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT;
+
+	dst_flags |= (*src)->flags & XE_BO_FLAG_GGTT_INVALIDATE;
 
 	xe_assert(xe, IS_DGFX(xe));
 	xe_assert(xe, !(*src)->vmap.is_iomem);
 
-	bo = xe_managed_bo_create_from_data(xe, tile, (*src)->vmap.vaddr, (*src)->size,
-					    XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-					    XE_BO_CREATE_GGTT_BIT);
+	bo = xe_managed_bo_create_from_data(xe, tile, (*src)->vmap.vaddr,
+					    (*src)->size, dst_flags);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
@@ -1668,8 +1664,8 @@ int xe_bo_pin(struct xe_bo *bo)
 	xe_assert(xe, !xe_bo_is_user(bo));
 
 	/* Pinned object must be in GGTT or have pinned flag */
-	xe_assert(xe, bo->flags & (XE_BO_CREATE_PINNED_BIT |
-				   XE_BO_CREATE_GGTT_BIT));
+	xe_assert(xe, bo->flags & (XE_BO_FLAG_PINNED |
+				   XE_BO_FLAG_GGTT));
 
 	/*
 	 * No reason we can't support pinning imported dma-bufs we just don't
@@ -1690,7 +1686,7 @@ int xe_bo_pin(struct xe_bo *bo)
 	 * during suspend / resume (force restore to same physical address).
 	 */
 	if (IS_DGFX(xe) && !(IS_ENABLED(CONFIG_DRM_XE_DEBUG) &&
-	    bo->flags & XE_BO_INTERNAL_TEST)) {
+	    bo->flags & XE_BO_FLAG_INTERNAL_TEST)) {
 		struct ttm_place *place = &(bo->placements[0]);
 
 		if (mem_type_is_vram(place->mem_type)) {
@@ -1758,7 +1754,7 @@ void xe_bo_unpin(struct xe_bo *bo)
 	xe_assert(xe, xe_bo_is_pinned(bo));
 
 	if (IS_DGFX(xe) && !(IS_ENABLED(CONFIG_DRM_XE_DEBUG) &&
-	    bo->flags & XE_BO_INTERNAL_TEST)) {
+	    bo->flags & XE_BO_FLAG_INTERNAL_TEST)) {
 		struct ttm_place *place = &(bo->placements[0]);
 
 		if (mem_type_is_vram(place->mem_type)) {
@@ -1861,7 +1857,7 @@ int xe_bo_vmap(struct xe_bo *bo)
 
 	xe_bo_assert_held(bo);
 
-	if (!(bo->flags & XE_BO_NEEDS_CPU_ACCESS))
+	if (!(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS))
 		return -EINVAL;
 
 	if (!iosys_map_is_null(&bo->vmap))
@@ -1943,29 +1939,29 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data,
 
 	bo_flags = 0;
 	if (args->flags & DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING)
-		bo_flags |= XE_BO_DEFER_BACKING;
+		bo_flags |= XE_BO_FLAG_DEFER_BACKING;
 
 	if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT)
-		bo_flags |= XE_BO_SCANOUT_BIT;
+		bo_flags |= XE_BO_FLAG_SCANOUT;
 
-	bo_flags |= args->placement << (ffs(XE_BO_CREATE_SYSTEM_BIT) - 1);
+	bo_flags |= args->placement << (ffs(XE_BO_FLAG_SYSTEM) - 1);
 
 	if (args->flags & DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM) {
-		if (XE_IOCTL_DBG(xe, !(bo_flags & XE_BO_CREATE_VRAM_MASK)))
+		if (XE_IOCTL_DBG(xe, !(bo_flags & XE_BO_FLAG_VRAM_MASK)))
 			return -EINVAL;
 
-		bo_flags |= XE_BO_NEEDS_CPU_ACCESS;
+		bo_flags |= XE_BO_FLAG_NEEDS_CPU_ACCESS;
 	}
 
 	if (XE_IOCTL_DBG(xe, !args->cpu_caching ||
 			 args->cpu_caching > DRM_XE_GEM_CPU_CACHING_WC))
 		return -EINVAL;
 
-	if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_CREATE_VRAM_MASK &&
+	if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_VRAM_MASK &&
 			 args->cpu_caching != DRM_XE_GEM_CPU_CACHING_WC))
 		return -EINVAL;
 
-	if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_SCANOUT_BIT &&
+	if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_SCANOUT &&
 			 args->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB))
 		return -EINVAL;
 
@@ -2206,6 +2202,9 @@ bool xe_bo_needs_ccs_pages(struct xe_bo *bo)
 {
 	struct xe_device *xe = xe_bo_device(bo);
 
+	if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe))
+		return false;
+
 	if (!xe_device_has_flat_ccs(xe) || bo->ttm.type != ttm_bo_type_device)
 		return false;
 
@@ -2214,7 +2213,7 @@ bool xe_bo_needs_ccs_pages(struct xe_bo *bo)
 	 * can't be used since there's no CCS storage associated with
 	 * non-VRAM addresses.
 	 */
-	if (IS_DGFX(xe) && (bo->flags & XE_BO_CREATE_SYSTEM_BIT))
+	if (IS_DGFX(xe) && (bo->flags & XE_BO_FLAG_SYSTEM))
 		return false;
 
 	return true;
@@ -2283,9 +2282,9 @@ int xe_bo_dumb_create(struct drm_file *file_priv,
 	bo = xe_bo_create_user(xe, NULL, NULL, args->size,
 			       DRM_XE_GEM_CPU_CACHING_WC,
 			       ttm_bo_type_device,
-			       XE_BO_CREATE_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
-			       XE_BO_CREATE_USER_BIT | XE_BO_SCANOUT_BIT |
-			       XE_BO_NEEDS_CPU_ACCESS);
+			       XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
+			       XE_BO_FLAG_SCANOUT |
+			       XE_BO_FLAG_NEEDS_CPU_ACCESS);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index c59ad15961ce..a885b14bf595 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -13,48 +13,34 @@
 #include "xe_vm_types.h"
 #include "xe_vm.h"
 
-/**
- * xe_vm_assert_held(vm) - Assert that the vm's reservation object is held.
- * @vm: The vm
- */
-#define xe_vm_assert_held(vm) dma_resv_assert_held(xe_vm_resv(vm))
-
-
-
 #define XE_DEFAULT_GTT_SIZE_MB          3072ULL /* 3GB by default */
 
-#define XE_BO_CREATE_USER_BIT		BIT(0)
+#define XE_BO_FLAG_USER		BIT(0)
 /* The bits below need to be contiguous, or things break */
-#define XE_BO_CREATE_SYSTEM_BIT		BIT(1)
-#define XE_BO_CREATE_VRAM0_BIT		BIT(2)
-#define XE_BO_CREATE_VRAM1_BIT		BIT(3)
-#define XE_BO_CREATE_VRAM_MASK		(XE_BO_CREATE_VRAM0_BIT | \
-					 XE_BO_CREATE_VRAM1_BIT)
+#define XE_BO_FLAG_SYSTEM		BIT(1)
+#define XE_BO_FLAG_VRAM0		BIT(2)
+#define XE_BO_FLAG_VRAM1		BIT(3)
+#define XE_BO_FLAG_VRAM_MASK		(XE_BO_FLAG_VRAM0 | XE_BO_FLAG_VRAM1)
 /* -- */
-#define XE_BO_CREATE_STOLEN_BIT		BIT(4)
-#define XE_BO_CREATE_VRAM_IF_DGFX(tile) \
-	(IS_DGFX(tile_to_xe(tile)) ? XE_BO_CREATE_VRAM0_BIT << (tile)->id : \
-	 XE_BO_CREATE_SYSTEM_BIT)
-#define XE_BO_CREATE_GGTT_BIT		BIT(5)
-#define XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT BIT(6)
-#define XE_BO_CREATE_PINNED_BIT		BIT(7)
-#define XE_BO_CREATE_NO_RESV_EVICT	BIT(8)
-#define XE_BO_DEFER_BACKING		BIT(9)
-#define XE_BO_SCANOUT_BIT		BIT(10)
-#define XE_BO_FIXED_PLACEMENT_BIT	BIT(11)
-#define XE_BO_PAGETABLE			BIT(12)
-#define XE_BO_NEEDS_CPU_ACCESS		BIT(13)
-#define XE_BO_NEEDS_UC			BIT(14)
+#define XE_BO_FLAG_STOLEN		BIT(4)
+#define XE_BO_FLAG_VRAM_IF_DGFX(tile)	(IS_DGFX(tile_to_xe(tile)) ? \
+					 XE_BO_FLAG_VRAM0 << (tile)->id : \
+					 XE_BO_FLAG_SYSTEM)
+#define XE_BO_FLAG_GGTT			BIT(5)
+#define XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE BIT(6)
+#define XE_BO_FLAG_PINNED		BIT(7)
+#define XE_BO_FLAG_NO_RESV_EVICT	BIT(8)
+#define XE_BO_FLAG_DEFER_BACKING	BIT(9)
+#define XE_BO_FLAG_SCANOUT		BIT(10)
+#define XE_BO_FLAG_FIXED_PLACEMENT	BIT(11)
+#define XE_BO_FLAG_PAGETABLE		BIT(12)
+#define XE_BO_FLAG_NEEDS_CPU_ACCESS	BIT(13)
+#define XE_BO_FLAG_NEEDS_UC		BIT(14)
+#define XE_BO_NEEDS_64K			BIT(15)
+#define XE_BO_FLAG_GGTT_INVALIDATE	BIT(16)
 /* this one is trigger internally only */
-#define XE_BO_INTERNAL_TEST		BIT(30)
-#define XE_BO_INTERNAL_64K		BIT(31)
-
-#define XELPG_PPGTT_PTE_PAT3		BIT_ULL(62)
-#define XE2_PPGTT_PTE_PAT4		BIT_ULL(61)
-#define XE_PPGTT_PDE_PDPE_PAT2		BIT_ULL(12)
-#define XE_PPGTT_PTE_PAT2		BIT_ULL(7)
-#define XE_PPGTT_PTE_PAT1		BIT_ULL(4)
-#define XE_PPGTT_PTE_PAT0		BIT_ULL(3)
+#define XE_BO_FLAG_INTERNAL_TEST	BIT(30)
+#define XE_BO_FLAG_INTERNAL_64K		BIT(31)
 
 #define XE_PTE_SHIFT			12
 #define XE_PAGE_SIZE			(1 << XE_PTE_SHIFT)
@@ -68,20 +54,6 @@
 #define XE_64K_PTE_MASK			(XE_64K_PAGE_SIZE - 1)
 #define XE_64K_PDE_MASK			(XE_PDE_MASK >> 4)
 
-#define XE_PDE_PS_2M			BIT_ULL(7)
-#define XE_PDPE_PS_1G			BIT_ULL(7)
-#define XE_PDE_IPS_64K			BIT_ULL(11)
-
-#define XE_GGTT_PTE_DM			BIT_ULL(1)
-#define XE_USM_PPGTT_PTE_AE		BIT_ULL(10)
-#define XE_PPGTT_PTE_DM			BIT_ULL(11)
-#define XE_PDE_64K			BIT_ULL(6)
-#define XE_PTE_PS64			BIT_ULL(8)
-#define XE_PTE_NULL			BIT_ULL(9)
-
-#define XE_PAGE_PRESENT			BIT_ULL(0)
-#define XE_PAGE_RW			BIT_ULL(1)
-
 #define XE_PL_SYSTEM		TTM_PL_SYSTEM
 #define XE_PL_TT		TTM_PL_TT
 #define XE_PL_VRAM0		TTM_PL_VRAM
diff --git a/drivers/gpu/drm/xe/xe_bo_evict.c b/drivers/gpu/drm/xe/xe_bo_evict.c
index 7a264a9ca06e..541b49007d73 100644
--- a/drivers/gpu/drm/xe/xe_bo_evict.c
+++ b/drivers/gpu/drm/xe/xe_bo_evict.c
@@ -146,7 +146,7 @@ int xe_bo_restore_kernel(struct xe_device *xe)
 			return ret;
 		}
 
-		if (bo->flags & XE_BO_CREATE_GGTT_BIT) {
+		if (bo->flags & XE_BO_FLAG_GGTT) {
 			struct xe_tile *tile = bo->tile;
 
 			mutex_lock(&tile->mem.ggtt->lock);
@@ -220,7 +220,7 @@ int xe_bo_restore_user(struct xe_device *xe)
 	list_splice_tail(&still_in_list, &xe->pinned.external_vram);
 	spin_unlock(&xe->pinned.lock);
 
-	/* Wait for validate to complete */
+	/* Wait for restore to complete */
 	for_each_tile(tile, xe, id)
 		xe_tile_migrate_wait(tile);
 
diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c
index 01db5b27bec5..0b7aebaae843 100644
--- a/drivers/gpu/drm/xe/xe_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_debugfs.c
@@ -5,6 +5,7 @@
 
 #include "xe_debugfs.h"
 
+#include <linux/debugfs.h>
 #include <linux/string_helpers.h>
 
 #include <drm/drm_debugfs.h>
@@ -12,6 +13,8 @@
 #include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_gt_debugfs.h"
+#include "xe_pm.h"
+#include "xe_sriov.h"
 #include "xe_step.h"
 
 #ifdef CONFIG_DRM_XE_DEBUG
@@ -37,6 +40,8 @@ static int info(struct seq_file *m, void *data)
 	struct xe_gt *gt;
 	u8 id;
 
+	xe_pm_runtime_get(xe);
+
 	drm_printf(&p, "graphics_verx100 %d\n", xe->info.graphics_verx100);
 	drm_printf(&p, "media_verx100 %d\n", xe->info.media_verx100);
 	drm_printf(&p, "stepping G:%s M:%s D:%s B:%s\n",
@@ -63,11 +68,22 @@ static int info(struct seq_file *m, void *data)
 			   gt->info.engine_mask);
 	}
 
+	xe_pm_runtime_put(xe);
+	return 0;
+}
+
+static int sriov_info(struct seq_file *m, void *data)
+{
+	struct xe_device *xe = node_to_xe(m->private);
+	struct drm_printer p = drm_seq_file_printer(m);
+
+	xe_sriov_print_info(xe, &p);
 	return 0;
 }
 
 static const struct drm_info_list debugfs_list[] = {
 	{"info", info, 0},
+	{ .name = "sriov_info", .show = sriov_info, },
 };
 
 static int forcewake_open(struct inode *inode, struct file *file)
@@ -76,8 +92,7 @@ static int forcewake_open(struct inode *inode, struct file *file)
 	struct xe_gt *gt;
 	u8 id;
 
-	xe_device_mem_access_get(xe);
-
+	xe_pm_runtime_get(xe);
 	for_each_gt(gt, xe, id)
 		XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL));
 
@@ -92,8 +107,7 @@ static int forcewake_release(struct inode *inode, struct file *file)
 
 	for_each_gt(gt, xe, id)
 		XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
-
-	xe_device_mem_access_put(xe);
+	xe_pm_runtime_put(xe);
 
 	return 0;
 }
@@ -127,7 +141,7 @@ void xe_debugfs_register(struct xe_device *xe)
 		if (man) {
 			char name[16];
 
-			sprintf(name, "vram%d_mm", mem_type - XE_PL_VRAM0);
+			snprintf(name, sizeof(name), "vram%d_mm", mem_type - XE_PL_VRAM0);
 			ttm_resource_manager_create_debugfs(man, root, name);
 		}
 	}
diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c
index 68d3d623a05b..3d7980232be1 100644
--- a/drivers/gpu/drm/xe/xe_devcoredump.c
+++ b/drivers/gpu/drm/xe/xe_devcoredump.c
@@ -9,10 +9,13 @@
 #include <linux/devcoredump.h>
 #include <generated/utsrelease.h>
 
+#include <drm/drm_managed.h>
+
 #include "xe_device.h"
 #include "xe_exec_queue.h"
 #include "xe_force_wake.h"
 #include "xe_gt.h"
+#include "xe_gt_printk.h"
 #include "xe_guc_ct.h"
 #include "xe_guc_submit.h"
 #include "xe_hw_engine.h"
@@ -64,9 +67,11 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work)
 {
 	struct xe_devcoredump_snapshot *ss = container_of(work, typeof(*ss), work);
 
-	xe_force_wake_get(gt_to_fw(ss->gt), XE_FORCEWAKE_ALL);
-	if (ss->vm)
-		xe_vm_snapshot_capture_delayed(ss->vm);
+	/* keep going if fw fails as we still want to save the memory and SW data */
+	if (xe_force_wake_get(gt_to_fw(ss->gt), XE_FORCEWAKE_ALL))
+		xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n");
+	xe_vm_snapshot_capture_delayed(ss->vm);
+	xe_guc_exec_queue_snapshot_capture_delayed(ss->ge);
 	xe_force_wake_put(gt_to_fw(ss->gt), XE_FORCEWAKE_ALL);
 }
 
@@ -74,17 +79,19 @@ static ssize_t xe_devcoredump_read(char *buffer, loff_t offset,
 				   size_t count, void *data, size_t datalen)
 {
 	struct xe_devcoredump *coredump = data;
-	struct xe_device *xe = coredump_to_xe(coredump);
-	struct xe_devcoredump_snapshot *ss = &coredump->snapshot;
+	struct xe_device *xe;
+	struct xe_devcoredump_snapshot *ss;
 	struct drm_printer p;
 	struct drm_print_iterator iter;
 	struct timespec64 ts;
 	int i;
 
-	/* Our device is gone already... */
-	if (!data || !coredump_to_xe(coredump))
+	if (!coredump)
 		return -ENODEV;
 
+	xe = coredump_to_xe(coredump);
+	ss = &coredump->snapshot;
+
 	/* Ensure delayed work is captured before continuing */
 	flush_work(&ss->work);
 
@@ -117,10 +124,8 @@ static ssize_t xe_devcoredump_read(char *buffer, loff_t offset,
 		if (coredump->snapshot.hwe[i])
 			xe_hw_engine_snapshot_print(coredump->snapshot.hwe[i],
 						    &p);
-	if (coredump->snapshot.vm) {
-		drm_printf(&p, "\n**** VM state ****\n");
-		xe_vm_snapshot_print(coredump->snapshot.vm, &p);
-	}
+	drm_printf(&p, "\n**** VM state ****\n");
+	xe_vm_snapshot_print(coredump->snapshot.vm, &p);
 
 	return count - iter.remain;
 }
@@ -180,10 +185,12 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump,
 		}
 	}
 
-	xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
+	/* keep going if fw fails as we still want to save the memory and SW data */
+	if (xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL))
+		xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n");
 
 	coredump->snapshot.ct = xe_guc_ct_snapshot_capture(&guc->ct, true);
-	coredump->snapshot.ge = xe_guc_exec_queue_snapshot_capture(job);
+	coredump->snapshot.ge = xe_guc_exec_queue_snapshot_capture(q);
 	coredump->snapshot.job = xe_sched_job_snapshot_capture(job);
 	coredump->snapshot.vm = xe_vm_snapshot_capture(q->vm);
 
@@ -196,8 +203,7 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump,
 		coredump->snapshot.hwe[id] = xe_hw_engine_snapshot_capture(hwe);
 	}
 
-	if (ss->vm)
-		queue_work(system_unbound_wq, &ss->work);
+	queue_work(system_unbound_wq, &ss->work);
 
 	xe_force_wake_put(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
 	dma_fence_end_signalling(cookie);
@@ -231,5 +237,14 @@ void xe_devcoredump(struct xe_sched_job *job)
 	dev_coredumpm(xe->drm.dev, THIS_MODULE, coredump, 0, GFP_KERNEL,
 		      xe_devcoredump_read, xe_devcoredump_free);
 }
-#endif
 
+static void xe_driver_devcoredump_fini(struct drm_device *drm, void *arg)
+{
+	dev_coredump_put(drm->dev);
+}
+
+int xe_devcoredump_init(struct xe_device *xe)
+{
+	return drmm_add_action_or_reset(&xe->drm, xe_driver_devcoredump_fini, xe);
+}
+#endif
diff --git a/drivers/gpu/drm/xe/xe_devcoredump.h b/drivers/gpu/drm/xe/xe_devcoredump.h
index df8671f0b5eb..e2fa65ce0932 100644
--- a/drivers/gpu/drm/xe/xe_devcoredump.h
+++ b/drivers/gpu/drm/xe/xe_devcoredump.h
@@ -11,10 +11,16 @@ struct xe_sched_job;
 
 #ifdef CONFIG_DEV_COREDUMP
 void xe_devcoredump(struct xe_sched_job *job);
+int xe_devcoredump_init(struct xe_device *xe);
 #else
 static inline void xe_devcoredump(struct xe_sched_job *job)
 {
 }
+
+static inline int xe_devcoredump_init(struct xe_device *xe)
+{
+	return 0;
+}
 #endif
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index d32ff3857e65..5ef9b50a20d0 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -9,6 +9,7 @@
 
 #include <drm/drm_aperture.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_client.h>
 #include <drm/drm_gem_ttm_helper.h>
 #include <drm/drm_ioctl.h>
 #include <drm/drm_managed.h>
@@ -20,6 +21,7 @@
 #include "regs/xe_regs.h"
 #include "xe_bo.h"
 #include "xe_debugfs.h"
+#include "xe_devcoredump.h"
 #include "xe_dma_buf.h"
 #include "xe_drm_client.h"
 #include "xe_drv.h"
@@ -45,12 +47,6 @@
 #include "xe_vm.h"
 #include "xe_wait_user_fence.h"
 
-#ifdef CONFIG_LOCKDEP
-struct lockdep_map xe_device_mem_access_lockdep_map = {
-	.name = "xe_device_mem_access_lockdep_map"
-};
-#endif
-
 static int xe_file_open(struct drm_device *dev, struct drm_file *file)
 {
 	struct xe_device *xe = to_xe_device(dev);
@@ -136,15 +132,48 @@ static const struct drm_ioctl_desc xe_ioctls[] = {
 			  DRM_RENDER_ALLOW),
 };
 
+static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct drm_file *file_priv = file->private_data;
+	struct xe_device *xe = to_xe_device(file_priv->minor->dev);
+	long ret;
+
+	ret = xe_pm_runtime_get_ioctl(xe);
+	if (ret >= 0)
+		ret = drm_ioctl(file, cmd, arg);
+	xe_pm_runtime_put(xe);
+
+	return ret;
+}
+
+#ifdef CONFIG_COMPAT
+static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct drm_file *file_priv = file->private_data;
+	struct xe_device *xe = to_xe_device(file_priv->minor->dev);
+	long ret;
+
+	ret = xe_pm_runtime_get_ioctl(xe);
+	if (ret >= 0)
+		ret = drm_compat_ioctl(file, cmd, arg);
+	xe_pm_runtime_put(xe);
+
+	return ret;
+}
+#else
+/* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */
+#define xe_drm_compat_ioctl NULL
+#endif
+
 static const struct file_operations xe_driver_fops = {
 	.owner = THIS_MODULE,
 	.open = drm_open,
 	.release = drm_release_noglobal,
-	.unlocked_ioctl = drm_ioctl,
+	.unlocked_ioctl = xe_drm_ioctl,
 	.mmap = drm_gem_mmap,
 	.poll = drm_poll,
 	.read = drm_read,
-	.compat_ioctl = drm_compat_ioctl,
+	.compat_ioctl = xe_drm_compat_ioctl,
 	.llseek = noop_llseek,
 #ifdef CONFIG_PROC_FS
 	.show_fdinfo = drm_show_fdinfo,
@@ -389,8 +418,70 @@ mask_err:
 	return err;
 }
 
-/*
- * Initialize MMIO resources that don't require any knowledge about tile count.
+static bool verify_lmem_ready(struct xe_gt *gt)
+{
+	u32 val = xe_mmio_read32(gt, GU_CNTL) & LMEM_INIT;
+
+	return !!val;
+}
+
+static int wait_for_lmem_ready(struct xe_device *xe)
+{
+	struct xe_gt *gt = xe_root_mmio_gt(xe);
+	unsigned long timeout, start;
+
+	if (!IS_DGFX(xe))
+		return 0;
+
+	if (IS_SRIOV_VF(xe))
+		return 0;
+
+	if (verify_lmem_ready(gt))
+		return 0;
+
+	drm_dbg(&xe->drm, "Waiting for lmem initialization\n");
+
+	start = jiffies;
+	timeout = start + msecs_to_jiffies(60 * 1000); /* 60 sec! */
+
+	do {
+		if (signal_pending(current))
+			return -EINTR;
+
+		/*
+		 * The boot firmware initializes local memory and
+		 * assesses its health. If memory training fails,
+		 * the punit will have been instructed to keep the GT powered
+		 * down.we won't be able to communicate with it
+		 *
+		 * If the status check is done before punit updates the register,
+		 * it can lead to the system being unusable.
+		 * use a timeout and defer the probe to prevent this.
+		 */
+		if (time_after(jiffies, timeout)) {
+			drm_dbg(&xe->drm, "lmem not initialized by firmware\n");
+			return -EPROBE_DEFER;
+		}
+
+		msleep(20);
+
+	} while (!verify_lmem_ready(gt));
+
+	drm_dbg(&xe->drm, "lmem ready after %ums",
+		jiffies_to_msecs(jiffies - start));
+
+	return 0;
+}
+
+/**
+ * xe_device_probe_early: Device early probe
+ * @xe: xe device instance
+ *
+ * Initialize MMIO resources that don't require any
+ * knowledge about tile count. Also initialize pcode and
+ * check vram initialization on root tile.
+ *
+ * Return: 0 on success, error code on failure
  */
 int xe_device_probe_early(struct xe_device *xe)
 {
@@ -400,7 +491,13 @@ int xe_device_probe_early(struct xe_device *xe)
 	if (err)
 		return err;
 
-	err = xe_mmio_root_tile_init(xe);
+	xe_sriov_probe_early(xe);
+
+	err = xe_pcode_probe_early(xe);
+	if (err)
+		return err;
+
+	err = wait_for_lmem_ready(xe);
 	if (err)
 		return err;
 
@@ -478,15 +575,15 @@ int xe_device_probe(struct xe_device *xe)
 			return err;
 	}
 
+	err = xe_devcoredump_init(xe);
+	if (err)
+		return err;
 	err = drmm_add_action_or_reset(&xe->drm, xe_driver_flr_fini, xe);
 	if (err)
 		return err;
 
-	for_each_gt(gt, xe, id) {
-		err = xe_pcode_probe(gt);
-		if (err)
-			return err;
-	}
+	for_each_gt(gt, xe, id)
+		xe_pcode_init(gt);
 
 	err = xe_display_init_noirq(xe);
 	if (err)
@@ -553,11 +650,7 @@ int xe_device_probe(struct xe_device *xe)
 
 	xe_hwmon_register(xe);
 
-	err = drmm_add_action_or_reset(&xe->drm, xe_device_sanitize, xe);
-	if (err)
-		return err;
-
-	return 0;
+	return drmm_add_action_or_reset(&xe->drm, xe_device_sanitize, xe);
 
 err_fini_display:
 	xe_display_driver_remove(xe);
@@ -621,87 +714,20 @@ u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)
 		DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0;
 }
 
-bool xe_device_mem_access_ongoing(struct xe_device *xe)
-{
-	if (xe_pm_read_callback_task(xe) != NULL)
-		return true;
-
-	return atomic_read(&xe->mem_access.ref);
-}
-
+/**
+ * xe_device_assert_mem_access - Inspect the current runtime_pm state.
+ * @xe: xe device instance
+ *
+ * To be used before any kind of memory access. It will splat a debug warning
+ * if the device is currently sleeping. But it doesn't guarantee in any way
+ * that the device is going to remain awake. Xe PM runtime get and put
+ * functions might be added to the outer bound of the memory access, while
+ * this check is intended for inner usage to splat some warning if the worst
+ * case has just happened.
+ */
 void xe_device_assert_mem_access(struct xe_device *xe)
 {
-	XE_WARN_ON(!xe_device_mem_access_ongoing(xe));
-}
-
-bool xe_device_mem_access_get_if_ongoing(struct xe_device *xe)
-{
-	bool active;
-
-	if (xe_pm_read_callback_task(xe) == current)
-		return true;
-
-	active = xe_pm_runtime_get_if_active(xe);
-	if (active) {
-		int ref = atomic_inc_return(&xe->mem_access.ref);
-
-		xe_assert(xe, ref != S32_MAX);
-	}
-
-	return active;
-}
-
-void xe_device_mem_access_get(struct xe_device *xe)
-{
-	int ref;
-
-	/*
-	 * This looks racy, but should be fine since the pm_callback_task only
-	 * transitions from NULL -> current (and back to NULL again), during the
-	 * runtime_resume() or runtime_suspend() callbacks, for which there can
-	 * only be a single one running for our device. We only need to prevent
-	 * recursively calling the runtime_get or runtime_put from those
-	 * callbacks, as well as preventing triggering any access_ongoing
-	 * asserts.
-	 */
-	if (xe_pm_read_callback_task(xe) == current)
-		return;
-
-	/*
-	 * Since the resume here is synchronous it can be quite easy to deadlock
-	 * if we are not careful. Also in practice it might be quite timing
-	 * sensitive to ever see the 0 -> 1 transition with the callers locks
-	 * held, so deadlocks might exist but are hard for lockdep to ever see.
-	 * With this in mind, help lockdep learn about the potentially scary
-	 * stuff that can happen inside the runtime_resume callback by acquiring
-	 * a dummy lock (it doesn't protect anything and gets compiled out on
-	 * non-debug builds).  Lockdep then only needs to see the
-	 * mem_access_lockdep_map -> runtime_resume callback once, and then can
-	 * hopefully validate all the (callers_locks) -> mem_access_lockdep_map.
-	 * For example if the (callers_locks) are ever grabbed in the
-	 * runtime_resume callback, lockdep should give us a nice splat.
-	 */
-	lock_map_acquire(&xe_device_mem_access_lockdep_map);
-	lock_map_release(&xe_device_mem_access_lockdep_map);
-
-	xe_pm_runtime_get(xe);
-	ref = atomic_inc_return(&xe->mem_access.ref);
-
-	xe_assert(xe, ref != S32_MAX);
-
-}
-
-void xe_device_mem_access_put(struct xe_device *xe)
-{
-	int ref;
-
-	if (xe_pm_read_callback_task(xe) == current)
-		return;
-
-	ref = atomic_dec_return(&xe->mem_access.ref);
-	xe_pm_runtime_put(xe);
-
-	xe_assert(xe, ref >= 0);
+	xe_assert(xe, !xe_pm_runtime_suspended(xe));
 }
 
 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p)
diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h
index d413bc2c6be5..36d4434ebccc 100644
--- a/drivers/gpu/drm/xe/xe_device.h
+++ b/drivers/gpu/drm/xe/xe_device.h
@@ -16,10 +16,6 @@ struct xe_file;
 #include "xe_force_wake.h"
 #include "xe_macros.h"
 
-#ifdef CONFIG_LOCKDEP
-extern struct lockdep_map xe_device_mem_access_lockdep_map;
-#endif
-
 static inline struct xe_device *to_xe_device(const struct drm_device *dev)
 {
 	return container_of(dev, struct xe_device, drm);
@@ -137,12 +133,7 @@ static inline struct xe_force_wake *gt_to_fw(struct xe_gt *gt)
 	return &gt->mmio.fw;
 }
 
-void xe_device_mem_access_get(struct xe_device *xe);
-bool xe_device_mem_access_get_if_ongoing(struct xe_device *xe);
-void xe_device_mem_access_put(struct xe_device *xe);
-
 void xe_device_assert_mem_access(struct xe_device *xe);
-bool xe_device_mem_access_ongoing(struct xe_device *xe);
 
 static inline bool xe_device_in_fault_mode(struct xe_device *xe)
 {
diff --git a/drivers/gpu/drm/xe/xe_device_sysfs.c b/drivers/gpu/drm/xe/xe_device_sysfs.c
index 99113a5a2b84..21677b8cd977 100644
--- a/drivers/gpu/drm/xe/xe_device_sysfs.c
+++ b/drivers/gpu/drm/xe/xe_device_sysfs.c
@@ -35,7 +35,9 @@ vram_d3cold_threshold_show(struct device *dev,
 	if (!xe)
 		return -EINVAL;
 
+	xe_pm_runtime_get(xe);
 	ret = sysfs_emit(buf, "%d\n", xe->d3cold.vram_threshold);
+	xe_pm_runtime_put(xe);
 
 	return ret;
 }
@@ -58,7 +60,9 @@ vram_d3cold_threshold_store(struct device *dev, struct device_attribute *attr,
 
 	drm_dbg(&xe->drm, "vram_d3cold_threshold: %u\n", vram_d3cold_threshold);
 
+	xe_pm_runtime_get(xe);
 	ret = xe_pm_set_vram_threshold(xe, vram_d3cold_threshold);
+	xe_pm_runtime_put(xe);
 
 	return ret ?: count;
 }
@@ -72,18 +76,14 @@ static void xe_device_sysfs_fini(struct drm_device *drm, void *arg)
 	sysfs_remove_file(&xe->drm.dev->kobj, &dev_attr_vram_d3cold_threshold.attr);
 }
 
-void xe_device_sysfs_init(struct xe_device *xe)
+int xe_device_sysfs_init(struct xe_device *xe)
 {
 	struct device *dev = xe->drm.dev;
 	int ret;
 
 	ret = sysfs_create_file(&dev->kobj, &dev_attr_vram_d3cold_threshold.attr);
-	if (ret) {
-		drm_warn(&xe->drm, "Failed to create sysfs file\n");
-		return;
-	}
-
-	ret = drmm_add_action_or_reset(&xe->drm, xe_device_sysfs_fini, xe);
 	if (ret)
-		drm_warn(&xe->drm, "Failed to add sysfs fini drm action\n");
+		return ret;
+
+	return drmm_add_action_or_reset(&xe->drm, xe_device_sysfs_fini, xe);
 }
diff --git a/drivers/gpu/drm/xe/xe_device_sysfs.h b/drivers/gpu/drm/xe/xe_device_sysfs.h
index 38b240684bee..f9e83d8bd2c7 100644
--- a/drivers/gpu/drm/xe/xe_device_sysfs.h
+++ b/drivers/gpu/drm/xe/xe_device_sysfs.h
@@ -8,6 +8,6 @@
 
 struct xe_device;
 
-void xe_device_sysfs_init(struct xe_device *xe);
+int xe_device_sysfs_init(struct xe_device *xe);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
index 8e3a222b41cf..2e62450d86e1 100644
--- a/drivers/gpu/drm/xe/xe_device_types.h
+++ b/drivers/gpu/drm/xe/xe_device_types.h
@@ -321,6 +321,10 @@ struct xe_device {
 	struct {
 		/** @sriov.__mode: SR-IOV mode (Don't access directly!) */
 		enum xe_sriov_mode __mode;
+
+		/** @sriov.pf: PF specific data */
+		struct xe_device_pf pf;
+
 		/** @sriov.wq: workqueue used by the virtualization workers */
 		struct workqueue_struct *wq;
 	} sriov;
@@ -380,9 +384,6 @@ struct xe_device {
 	 * triggering additional actions when they occur.
 	 */
 	struct {
-		/** @mem_access.ref: ref count of memory accesses */
-		atomic_t ref;
-
 		/**
 		 * @mem_access.vram_userfault: Encapsulate vram_userfault
 		 * related stuff
@@ -500,20 +501,9 @@ struct xe_device {
 	/* For pcode */
 	struct mutex sb_lock;
 
-	/* Should be in struct intel_display */
-	u32 skl_preferred_vco_freq, max_dotclk_freq, hti_state;
-	u8 snps_phy_failed_calibration;
-	struct drm_atomic_state *modeset_restore_state;
-	struct list_head global_obj_list;
-
-	union {
-		/* only to allow build, not used functionally */
-		u32 irq_mask;
-		u32 de_irq_mask[I915_MAX_PIPES];
-	};
-	u32 pipestat_irq_mask[I915_MAX_PIPES];
+	/* only to allow build, not used functionally */
+	u32 irq_mask;
 
-	bool display_irqs_enabled;
 	u32 enabled_irq_mask;
 
 	struct intel_uncore {
@@ -525,11 +515,7 @@ struct xe_device {
 		unsigned int hpll_freq;
 		unsigned int czclk_freq;
 		unsigned int fsb_freq, mem_freq, is_ddr3;
-		u8 vblank_enabled;
 	};
-	struct {
-		const char *dmc_firmware_path;
-	} params;
 
 	void *pxp;
 #endif
diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c
index da2627ed6ae7..68f309f5e981 100644
--- a/drivers/gpu/drm/xe/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/xe_dma_buf.c
@@ -16,6 +16,7 @@
 #include "tests/xe_test.h"
 #include "xe_bo.h"
 #include "xe_device.h"
+#include "xe_pm.h"
 #include "xe_ttm_vram_mgr.h"
 #include "xe_vm.h"
 
@@ -33,7 +34,7 @@ static int xe_dma_buf_attach(struct dma_buf *dmabuf,
 	if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT))
 		return -EOPNOTSUPP;
 
-	xe_device_mem_access_get(to_xe_device(obj->dev));
+	xe_pm_runtime_get(to_xe_device(obj->dev));
 	return 0;
 }
 
@@ -42,7 +43,7 @@ static void xe_dma_buf_detach(struct dma_buf *dmabuf,
 {
 	struct drm_gem_object *obj = attach->dmabuf->priv;
 
-	xe_device_mem_access_put(to_xe_device(obj->dev));
+	xe_pm_runtime_put(to_xe_device(obj->dev));
 }
 
 static int xe_dma_buf_pin(struct dma_buf_attachment *attach)
@@ -216,7 +217,7 @@ xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage,
 	dma_resv_lock(resv, NULL);
 	bo = ___xe_bo_create_locked(xe, storage, NULL, resv, NULL, dma_buf->size,
 				    0, /* Will require 1way or 2way for vm_bind */
-				    ttm_bo_type_sg, XE_BO_CREATE_SYSTEM_BIT);
+				    ttm_bo_type_sg, XE_BO_FLAG_SYSTEM);
 	if (IS_ERR(bo)) {
 		ret = PTR_ERR(bo);
 		goto error;
diff --git a/drivers/gpu/drm/xe/xe_drm_client.c b/drivers/gpu/drm/xe/xe_drm_client.c
index 87c10bd7958b..08f0b7c95901 100644
--- a/drivers/gpu/drm/xe/xe_drm_client.c
+++ b/drivers/gpu/drm/xe/xe_drm_client.c
@@ -78,7 +78,7 @@ void xe_drm_client_add_bo(struct xe_drm_client *client,
 
 	spin_lock(&client->bos_lock);
 	bo->client = xe_drm_client_get(client);
-	list_add_tail_rcu(&bo->client_link, &client->bos_list);
+	list_add_tail(&bo->client_link, &client->bos_list);
 	spin_unlock(&client->bos_lock);
 }
 
@@ -96,7 +96,7 @@ void xe_drm_client_remove_bo(struct xe_bo *bo)
 	struct xe_drm_client *client = bo->client;
 
 	spin_lock(&client->bos_lock);
-	list_del_rcu(&bo->client_link);
+	list_del(&bo->client_link);
 	spin_unlock(&client->bos_lock);
 
 	xe_drm_client_put(client);
@@ -154,8 +154,8 @@ static void show_meminfo(struct drm_printer *p, struct drm_file *file)
 
 	/* Internal objects. */
 	spin_lock(&client->bos_lock);
-	list_for_each_entry_rcu(bo, &client->bos_list, client_link) {
-		if (!bo || !kref_get_unless_zero(&bo->ttm.base.refcount))
+	list_for_each_entry(bo, &client->bos_list, client_link) {
+		if (!kref_get_unless_zero(&bo->ttm.base.refcount))
 			continue;
 		bo_meminfo(bo, stats);
 		xe_bo_put(bo);
diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c
index cc5e0f75de3c..97eeb973e897 100644
--- a/drivers/gpu/drm/xe/xe_exec.c
+++ b/drivers/gpu/drm/xe/xe_exec.c
@@ -216,7 +216,7 @@ retry:
 				goto err_unlock_list;
 			}
 			for (i = 0; i < num_syncs; i++)
-				xe_sync_entry_signal(&syncs[i], NULL, fence);
+				xe_sync_entry_signal(&syncs[i], fence);
 			xe_exec_queue_last_fence_set(q, vm, fence);
 			dma_fence_put(fence);
 		}
@@ -294,9 +294,10 @@ retry:
 		drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished,
 					 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_WRITE);
 
-	for (i = 0; i < num_syncs; i++)
-		xe_sync_entry_signal(&syncs[i], job,
-				     &job->drm.s_fence->finished);
+	for (i = 0; i < num_syncs; i++) {
+		xe_sync_entry_signal(&syncs[i], &job->drm.s_fence->finished);
+		xe_sched_job_init_user_fence(job, &syncs[i]);
+	}
 
 	if (xe_exec_queue_is_lr(q))
 		q->ring_ops->emit_job(job);
@@ -320,10 +321,7 @@ err_put_job:
 err_exec:
 	drm_exec_fini(exec);
 err_unlock_list:
-	if (write_locked)
-		up_write(&vm->lock);
-	else
-		up_read(&vm->lock);
+	up_read(&vm->lock);
 	if (err == -EAGAIN && !skip_retry)
 		goto retry;
 err_syncs:
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c
index ead25d5e723e..395de93579fa 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.c
+++ b/drivers/gpu/drm/xe/xe_exec_queue.c
@@ -31,7 +31,14 @@ enum xe_exec_queue_sched_prop {
 };
 
 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
-				      u64 extensions, int ext_number, bool create);
+				      u64 extensions, int ext_number);
+
+static void __xe_exec_queue_free(struct xe_exec_queue *q)
+{
+	if (q->vm)
+		xe_vm_put(q->vm);
+	kfree(q);
+}
 
 static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
 						   struct xe_vm *vm,
@@ -74,21 +81,21 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
 	else
 		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL;
 
+	if (vm)
+		q->vm = xe_vm_get(vm);
+
 	if (extensions) {
 		/*
 		 * may set q->usm, must come before xe_lrc_init(),
 		 * may overwrite q->sched_props, must come before q->ops->init()
 		 */
-		err = exec_queue_user_extensions(xe, q, extensions, 0, true);
+		err = exec_queue_user_extensions(xe, q, extensions, 0);
 		if (err) {
-			kfree(q);
+			__xe_exec_queue_free(q);
 			return ERR_PTR(err);
 		}
 	}
 
-	if (vm)
-		q->vm = xe_vm_get(vm);
-
 	if (xe_exec_queue_is_parallel(q)) {
 		q->parallel.composite_fence_ctx = dma_fence_context_alloc(1);
 		q->parallel.composite_fence_seqno = XE_FENCE_INITIAL_SEQNO;
@@ -97,13 +104,6 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
 	return q;
 }
 
-static void __xe_exec_queue_free(struct xe_exec_queue *q)
-{
-	if (q->vm)
-		xe_vm_put(q->vm);
-	kfree(q);
-}
-
 static int __xe_exec_queue_init(struct xe_exec_queue *q)
 {
 	struct xe_device *xe = gt_to_xe(q->gt);
@@ -128,7 +128,7 @@ static int __xe_exec_queue_init(struct xe_exec_queue *q)
 	 * already grabbed the rpm ref outside any sensitive locks.
 	 */
 	if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM || !q->vm))
-		drm_WARN_ON(&xe->drm, !xe_device_mem_access_get_if_ongoing(xe));
+		xe_pm_runtime_get_noresume(xe);
 
 	return 0;
 
@@ -217,7 +217,7 @@ void xe_exec_queue_fini(struct xe_exec_queue *q)
 	for (i = 0; i < q->width; ++i)
 		xe_lrc_finish(q->lrc + i);
 	if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM || !q->vm))
-		xe_device_mem_access_put(gt_to_xe(q->gt));
+		xe_pm_runtime_put(gt_to_xe(q->gt));
 	__xe_exec_queue_free(q);
 }
 
@@ -225,22 +225,22 @@ void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance)
 {
 	switch (q->class) {
 	case XE_ENGINE_CLASS_RENDER:
-		sprintf(q->name, "rcs%d", instance);
+		snprintf(q->name, sizeof(q->name), "rcs%d", instance);
 		break;
 	case XE_ENGINE_CLASS_VIDEO_DECODE:
-		sprintf(q->name, "vcs%d", instance);
+		snprintf(q->name, sizeof(q->name), "vcs%d", instance);
 		break;
 	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
-		sprintf(q->name, "vecs%d", instance);
+		snprintf(q->name, sizeof(q->name), "vecs%d", instance);
 		break;
 	case XE_ENGINE_CLASS_COPY:
-		sprintf(q->name, "bcs%d", instance);
+		snprintf(q->name, sizeof(q->name), "bcs%d", instance);
 		break;
 	case XE_ENGINE_CLASS_COMPUTE:
-		sprintf(q->name, "ccs%d", instance);
+		snprintf(q->name, sizeof(q->name), "ccs%d", instance);
 		break;
 	case XE_ENGINE_CLASS_OTHER:
-		sprintf(q->name, "gsccs%d", instance);
+		snprintf(q->name, sizeof(q->name), "gsccs%d", instance);
 		break;
 	default:
 		XE_WARN_ON(q->class);
@@ -268,7 +268,7 @@ xe_exec_queue_device_get_max_priority(struct xe_device *xe)
 }
 
 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q,
-				   u64 value, bool create)
+				   u64 value)
 {
 	if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH))
 		return -EINVAL;
@@ -276,9 +276,6 @@ static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q
 	if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe)))
 		return -EPERM;
 
-	if (!create)
-		return q->ops->set_priority(q, value);
-
 	q->sched_props.priority = value;
 	return 0;
 }
@@ -336,7 +333,7 @@ xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass,
 }
 
 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q,
-				    u64 value, bool create)
+				    u64 value)
 {
 	u32 min = 0, max = 0;
 
@@ -347,16 +344,13 @@ static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *
 	    !xe_hw_engine_timeout_in_range(value, min, max))
 		return -EINVAL;
 
-	if (!create)
-		return q->ops->set_timeslice(q, value);
-
 	q->sched_props.timeslice_us = value;
 	return 0;
 }
 
 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe,
 					     struct xe_exec_queue *q,
-					     u64 value, bool create);
+					     u64 value);
 
 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = {
 	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority,
@@ -365,8 +359,7 @@ static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = {
 
 static int exec_queue_user_ext_set_property(struct xe_device *xe,
 					    struct xe_exec_queue *q,
-					    u64 extension,
-					    bool create)
+					    u64 extension)
 {
 	u64 __user *address = u64_to_user_ptr(extension);
 	struct drm_xe_ext_set_property ext;
@@ -388,21 +381,20 @@ static int exec_queue_user_ext_set_property(struct xe_device *xe,
 	if (!exec_queue_set_property_funcs[idx])
 		return -EINVAL;
 
-	return exec_queue_set_property_funcs[idx](xe, q, ext.value,  create);
+	return exec_queue_set_property_funcs[idx](xe, q, ext.value);
 }
 
 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe,
 					       struct xe_exec_queue *q,
-					       u64 extension,
-					       bool create);
+					       u64 extension);
 
-static const xe_exec_queue_set_property_fn exec_queue_user_extension_funcs[] = {
+static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = {
 	[DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property,
 };
 
 #define MAX_USER_EXTENSIONS	16
 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
-				      u64 extensions, int ext_number, bool create)
+				      u64 extensions, int ext_number)
 {
 	u64 __user *address = u64_to_user_ptr(extensions);
 	struct drm_xe_user_extension ext;
@@ -423,13 +415,13 @@ static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue
 
 	idx = array_index_nospec(ext.name,
 				 ARRAY_SIZE(exec_queue_user_extension_funcs));
-	err = exec_queue_user_extension_funcs[idx](xe, q, extensions, create);
+	err = exec_queue_user_extension_funcs[idx](xe, q, extensions);
 	if (XE_IOCTL_DBG(xe, err))
 		return err;
 
 	if (ext.next_extension)
 		return exec_queue_user_extensions(xe, q, ext.next_extension,
-					      ++ext_number, create);
+						  ++ext_number);
 
 	return 0;
 }
@@ -597,7 +589,7 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
 				return -EINVAL;
 
 			/* The migration vm doesn't hold rpm ref */
-			xe_device_mem_access_get(xe);
+			xe_pm_runtime_get_noresume(xe);
 
 			flags = EXEC_QUEUE_FLAG_VM | (id ? EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD : 0);
 
@@ -606,7 +598,7 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
 						   args->width, hwe, flags,
 						   args->extensions);
 
-			xe_device_mem_access_put(xe); /* now held by engine */
+			xe_pm_runtime_put(xe); /* now held by engine */
 
 			xe_vm_put(migrate_vm);
 			if (IS_ERR(new)) {
diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h
index 462b33195032..ee78d497d838 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue_types.h
+++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h
@@ -76,14 +76,12 @@ struct xe_exec_queue {
 #define EXEC_QUEUE_FLAG_KERNEL			BIT(1)
 /* kernel engine only destroyed at driver unload */
 #define EXEC_QUEUE_FLAG_PERMANENT		BIT(2)
-/* queue keeps running pending jobs after destroy ioctl */
-#define EXEC_QUEUE_FLAG_PERSISTENT		BIT(3)
 /* for VM jobs. Caller needs to hold rpm ref when creating queue with this flag */
-#define EXEC_QUEUE_FLAG_VM			BIT(4)
+#define EXEC_QUEUE_FLAG_VM			BIT(3)
 /* child of VM queue for multi-tile VM jobs */
-#define EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD	BIT(5)
+#define EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD	BIT(4)
 /* kernel exec_queue only, set priority to highest level */
-#define EXEC_QUEUE_FLAG_HIGH_PRIORITY		BIT(6)
+#define EXEC_QUEUE_FLAG_HIGH_PRIORITY		BIT(5)
 
 	/**
 	 * @flags: flags for this exec queue, should statically setup aside from ban
diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c
index ab96edb058d6..0d541f55b4fc 100644
--- a/drivers/gpu/drm/xe/xe_ggtt.c
+++ b/drivers/gpu/drm/xe/xe_ggtt.c
@@ -5,12 +5,14 @@
 
 #include "xe_ggtt.h"
 
+#include <linux/io-64-nonatomic-lo-hi.h>
 #include <linux/sizes.h>
 
 #include <drm/drm_managed.h>
 #include <drm/i915_drm.h>
 
 #include "regs/xe_gt_regs.h"
+#include "regs/xe_gtt_defs.h"
 #include "regs/xe_regs.h"
 #include "xe_assert.h"
 #include "xe_bo.h"
@@ -19,16 +21,10 @@
 #include "xe_gt_printk.h"
 #include "xe_gt_tlb_invalidation.h"
 #include "xe_map.h"
-#include "xe_mmio.h"
+#include "xe_pm.h"
 #include "xe_sriov.h"
 #include "xe_wopcm.h"
 
-#define XELPG_GGTT_PTE_PAT0	BIT_ULL(52)
-#define XELPG_GGTT_PTE_PAT1	BIT_ULL(53)
-
-/* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */
-#define GUC_GGTT_TOP	0xFEE00000
-
 static u64 xelp_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
 				   u16 pat_index)
 {
@@ -200,20 +196,20 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt)
 	return drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt);
 }
 
+static void xe_ggtt_invalidate(struct xe_ggtt *ggtt);
+
 static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt)
 {
 	struct drm_mm_node *hole;
 	u64 start, end;
 
 	/* Display may have allocated inside ggtt, so be careful with clearing here */
-	xe_device_mem_access_get(tile_to_xe(ggtt->tile));
 	mutex_lock(&ggtt->lock);
 	drm_mm_for_each_hole(hole, &ggtt->mm, start, end)
 		xe_ggtt_clear(ggtt, start, end - start);
 
 	xe_ggtt_invalidate(ggtt);
 	mutex_unlock(&ggtt->lock);
-	xe_device_mem_access_put(tile_to_xe(ggtt->tile));
 }
 
 int xe_ggtt_init(struct xe_ggtt *ggtt)
@@ -227,11 +223,11 @@ int xe_ggtt_init(struct xe_ggtt *ggtt)
 	 * scratch entires, rather keep the scratch page in system memory on
 	 * platforms where 64K pages are needed for VRAM.
 	 */
-	flags = XE_BO_CREATE_PINNED_BIT;
+	flags = XE_BO_FLAG_PINNED;
 	if (ggtt->flags & XE_GGTT_FLAGS_64K)
-		flags |= XE_BO_CREATE_SYSTEM_BIT;
+		flags |= XE_BO_FLAG_SYSTEM;
 	else
-		flags |= XE_BO_CREATE_VRAM_IF_DGFX(ggtt->tile);
+		flags |= XE_BO_FLAG_VRAM_IF_DGFX(ggtt->tile);
 
 	ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile, XE_PAGE_SIZE, flags);
 	if (IS_ERR(ggtt->scratch)) {
@@ -249,51 +245,19 @@ err:
 	return err;
 }
 
-#define GUC_TLB_INV_CR				XE_REG(0xcee8)
-#define   GUC_TLB_INV_CR_INVALIDATE		REG_BIT(0)
-#define PVC_GUC_TLB_INV_DESC0			XE_REG(0xcf7c)
-#define   PVC_GUC_TLB_INV_DESC0_VALID		REG_BIT(0)
-#define PVC_GUC_TLB_INV_DESC1			XE_REG(0xcf80)
-#define   PVC_GUC_TLB_INV_DESC1_INVALIDATE	REG_BIT(6)
-
 static void ggtt_invalidate_gt_tlb(struct xe_gt *gt)
 {
+	int err;
+
 	if (!gt)
 		return;
 
-	/*
-	 * Invalidation can happen when there's no in-flight work keeping the
-	 * GT awake.  We need to explicitly grab forcewake to ensure the GT
-	 * and GuC are accessible.
-	 */
-	xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-
-	/* TODO: vfunc for GuC vs. non-GuC */
-
-	if (gt->uc.guc.submission_state.enabled) {
-		int seqno;
-
-		seqno = xe_gt_tlb_invalidation_guc(gt);
-		xe_gt_assert(gt, seqno > 0);
-		if (seqno > 0)
-			xe_gt_tlb_invalidation_wait(gt, seqno);
-	} else if (xe_device_uc_enabled(gt_to_xe(gt))) {
-		struct xe_device *xe = gt_to_xe(gt);
-
-		if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) {
-			xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC1,
-					PVC_GUC_TLB_INV_DESC1_INVALIDATE);
-			xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC0,
-					PVC_GUC_TLB_INV_DESC0_VALID);
-		} else
-			xe_mmio_write32(gt, GUC_TLB_INV_CR,
-					GUC_TLB_INV_CR_INVALIDATE);
-	}
-
-	xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
+	err = xe_gt_tlb_invalidation_ggtt(gt);
+	if (err)
+		drm_warn(&gt_to_xe(gt)->drm, "xe_gt_tlb_invalidation_ggtt error=%d", err);
 }
 
-void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
+static void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
 {
 	/* Each GT in a tile has its own TLB to cache GGTT lookups */
 	ggtt_invalidate_gt_tlb(ggtt->tile->primary_gt);
@@ -410,7 +374,7 @@ int xe_ggtt_insert_special_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
 
 void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
 {
-	u16 cache_mode = bo->flags & XE_BO_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
+	u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode];
 	u64 start = bo->ggtt_node.start;
 	u64 offset, pte;
@@ -419,8 +383,6 @@ void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
 		pte = ggtt->pt_ops->pte_encode_bo(bo, offset, pat_index);
 		xe_ggtt_set_pte(ggtt, start + offset, pte);
 	}
-
-	xe_ggtt_invalidate(ggtt);
 }
 
 static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
@@ -442,14 +404,17 @@ static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
 	if (err)
 		return err;
 
-	xe_device_mem_access_get(tile_to_xe(ggtt->tile));
+	xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
 	mutex_lock(&ggtt->lock);
 	err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node, bo->size,
 					  alignment, 0, start, end, 0);
 	if (!err)
 		xe_ggtt_map_bo(ggtt, bo);
 	mutex_unlock(&ggtt->lock);
-	xe_device_mem_access_put(tile_to_xe(ggtt->tile));
+
+	if (!err && bo->flags & XE_BO_FLAG_GGTT_INVALIDATE)
+		xe_ggtt_invalidate(ggtt);
+	xe_pm_runtime_put(tile_to_xe(ggtt->tile));
 
 	return err;
 }
@@ -465,19 +430,21 @@ int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
 	return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX);
 }
 
-void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node)
+void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
+			 bool invalidate)
 {
-	xe_device_mem_access_get(tile_to_xe(ggtt->tile));
-	mutex_lock(&ggtt->lock);
+	xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
 
+	mutex_lock(&ggtt->lock);
 	xe_ggtt_clear(ggtt, node->start, node->size);
 	drm_mm_remove_node(node);
 	node->size = 0;
+	mutex_unlock(&ggtt->lock);
 
-	xe_ggtt_invalidate(ggtt);
+	if (invalidate)
+		xe_ggtt_invalidate(ggtt);
 
-	mutex_unlock(&ggtt->lock);
-	xe_device_mem_access_put(tile_to_xe(ggtt->tile));
+	xe_pm_runtime_put(tile_to_xe(ggtt->tile));
 }
 
 void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
@@ -488,8 +455,53 @@ void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
 	/* This BO is not currently in the GGTT */
 	xe_tile_assert(ggtt->tile, bo->ggtt_node.size == bo->size);
 
-	xe_ggtt_remove_node(ggtt, &bo->ggtt_node);
+	xe_ggtt_remove_node(ggtt, &bo->ggtt_node,
+			    bo->flags & XE_BO_FLAG_GGTT_INVALIDATE);
+}
+
+#ifdef CONFIG_PCI_IOV
+static u64 xe_encode_vfid_pte(u16 vfid)
+{
+	return FIELD_PREP(GGTT_PTE_VFID, vfid) | XE_PAGE_PRESENT;
+}
+
+static void xe_ggtt_assign_locked(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
+{
+	u64 start = node->start;
+	u64 size = node->size;
+	u64 end = start + size - 1;
+	u64 pte = xe_encode_vfid_pte(vfid);
+
+	lockdep_assert_held(&ggtt->lock);
+
+	if (!drm_mm_node_allocated(node))
+		return;
+
+	while (start < end) {
+		xe_ggtt_set_pte(ggtt, start, pte);
+		start += XE_PAGE_SIZE;
+	}
+
+	xe_ggtt_invalidate(ggtt);
+}
+
+/**
+ * xe_ggtt_assign - assign a GGTT region to the VF
+ * @ggtt: the &xe_ggtt where the node belongs
+ * @node: the &drm_mm_node to update
+ * @vfid: the VF identifier
+ *
+ * This function is used by the PF driver to assign a GGTT region to the VF.
+ * In addition to PTE's VFID bits 11:2 also PRESENT bit 0 is set as on some
+ * platforms VFs can't modify that either.
+ */
+void xe_ggtt_assign(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
+{
+	mutex_lock(&ggtt->lock);
+	xe_ggtt_assign_locked(ggtt, node, vfid);
+	mutex_unlock(&ggtt->lock);
 }
+#endif
 
 int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p)
 {
diff --git a/drivers/gpu/drm/xe/xe_ggtt.h b/drivers/gpu/drm/xe/xe_ggtt.h
index 42705e1338e1..4a41a1762358 100644
--- a/drivers/gpu/drm/xe/xe_ggtt.h
+++ b/drivers/gpu/drm/xe/xe_ggtt.h
@@ -11,7 +11,6 @@
 struct drm_printer;
 
 void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte);
-void xe_ggtt_invalidate(struct xe_ggtt *ggtt);
 int xe_ggtt_init_early(struct xe_ggtt *ggtt);
 int xe_ggtt_init(struct xe_ggtt *ggtt);
 void xe_ggtt_printk(struct xe_ggtt *ggtt, const char *prefix);
@@ -24,7 +23,8 @@ int xe_ggtt_insert_special_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
 int xe_ggtt_insert_special_node_locked(struct xe_ggtt *ggtt,
 				       struct drm_mm_node *node,
 				       u32 size, u32 align, u32 mm_flags);
-void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node);
+void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
+			 bool invalidate);
 void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo);
 int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo);
 int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
@@ -33,4 +33,8 @@ void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo);
 
 int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p);
 
+#ifdef CONFIG_PCI_IOV
+void xe_ggtt_assign(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid);
+#endif
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_gsc.c b/drivers/gpu/drm/xe/xe_gsc.c
index a61994292c43..60202b903687 100644
--- a/drivers/gpu/drm/xe/xe_gsc.c
+++ b/drivers/gpu/drm/xe/xe_gsc.c
@@ -17,15 +17,18 @@
 #include "xe_gsc_proxy.h"
 #include "xe_gsc_submit.h"
 #include "xe_gt.h"
+#include "xe_gt_mcr.h"
 #include "xe_gt_printk.h"
 #include "xe_huc.h"
 #include "xe_map.h"
 #include "xe_mmio.h"
+#include "xe_pm.h"
 #include "xe_sched_job.h"
 #include "xe_uc_fw.h"
 #include "xe_wa.h"
 #include "instructions/xe_gsc_commands.h"
 #include "regs/xe_gsc_regs.h"
+#include "regs/xe_gt_regs.h"
 
 static struct xe_gt *
 gsc_to_gt(struct xe_gsc *gsc)
@@ -127,8 +130,8 @@ static int query_compatibility_version(struct xe_gsc *gsc)
 
 	bo = xe_bo_create_pin_map(xe, tile, NULL, GSC_VER_PKT_SZ * 2,
 				  ttm_bo_type_kernel,
-				  XE_BO_CREATE_SYSTEM_BIT |
-				  XE_BO_CREATE_GGTT_BIT);
+				  XE_BO_FLAG_SYSTEM |
+				  XE_BO_FLAG_GGTT);
 	if (IS_ERR(bo)) {
 		xe_gt_err(gt, "failed to allocate bo for GSC version query\n");
 		return PTR_ERR(bo);
@@ -250,9 +253,30 @@ static int gsc_upload(struct xe_gsc *gsc)
 static int gsc_upload_and_init(struct xe_gsc *gsc)
 {
 	struct xe_gt *gt = gsc_to_gt(gsc);
+	struct xe_tile *tile = gt_to_tile(gt);
 	int ret;
 
+	if (XE_WA(gt, 14018094691)) {
+		ret = xe_force_wake_get(gt_to_fw(tile->primary_gt), XE_FORCEWAKE_ALL);
+
+		/*
+		 * If the forcewake fails we want to keep going, because the worst
+		 * case outcome in failing to apply the WA is that PXP won't work,
+		 * which is not fatal. We still throw a warning so the issue is
+		 * seen if it happens.
+		 */
+		xe_gt_WARN_ON(tile->primary_gt, ret);
+
+		xe_gt_mcr_multicast_write(tile->primary_gt,
+					  EU_SYSTOLIC_LIC_THROTTLE_CTL_WITH_LOCK,
+					  EU_SYSTOLIC_LIC_THROTTLE_CTL_LOCK_BIT);
+	}
+
 	ret = gsc_upload(gsc);
+
+	if (XE_WA(gt, 14018094691))
+		xe_force_wake_put(gt_to_fw(tile->primary_gt), XE_FORCEWAKE_ALL);
+
 	if (ret)
 		return ret;
 
@@ -272,6 +296,44 @@ static int gsc_upload_and_init(struct xe_gsc *gsc)
 	return 0;
 }
 
+static int gsc_er_complete(struct xe_gt *gt)
+{
+	u32 er_status;
+
+	if (!gsc_fw_is_loaded(gt))
+		return 0;
+
+	/*
+	 * Starting on Xe2, the GSCCS engine reset is a 2-step process. When the
+	 * driver or the GuC hit the GDRST register, the CS is immediately reset
+	 * and a success is reported, but the GSC shim keeps resetting in the
+	 * background. While the shim reset is ongoing, the CS is able to accept
+	 * new context submission, but any commands that require the shim will
+	 * be stalled until the reset is completed. This means that we can keep
+	 * submitting to the GSCCS as long as we make sure that the preemption
+	 * timeout is big enough to cover any delay introduced by the reset.
+	 * When the shim reset completes, a specific CS interrupt is triggered,
+	 * in response to which we need to check the GSCI_TIMER_STATUS register
+	 * to see if the reset was successful or not.
+	 * Note that the GSCI_TIMER_STATUS register is not power save/restored,
+	 * so it gets reset on MC6 entry. However, a reset failure stops MC6,
+	 * so in that scenario we're always guaranteed to find the correct
+	 * value.
+	 */
+	er_status = xe_mmio_read32(gt, GSCI_TIMER_STATUS) & GSCI_TIMER_STATUS_VALUE;
+
+	if (er_status == GSCI_TIMER_STATUS_TIMER_EXPIRED) {
+		/*
+		 * XXX: we should trigger an FLR here, but we don't have support
+		 * for that yet.
+		 */
+		xe_gt_err(gt, "GSC ER timed out!\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
 static void gsc_work(struct work_struct *work)
 {
 	struct xe_gsc *gsc = container_of(work, typeof(*gsc), work);
@@ -285,8 +347,14 @@ static void gsc_work(struct work_struct *work)
 	gsc->work_actions = 0;
 	spin_unlock_irq(&gsc->lock);
 
-	xe_device_mem_access_get(xe);
-	xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);
+	xe_pm_runtime_get(xe);
+	xe_gt_WARN_ON(gt, xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC));
+
+	if (actions & GSC_ACTION_ER_COMPLETE) {
+		ret = gsc_er_complete(gt);
+		if (ret)
+			goto out;
+	}
 
 	if (actions & GSC_ACTION_FW_LOAD) {
 		ret = gsc_upload_and_init(gsc);
@@ -299,8 +367,26 @@ static void gsc_work(struct work_struct *work)
 	if (actions & GSC_ACTION_SW_PROXY)
 		xe_gsc_proxy_request_handler(gsc);
 
+out:
 	xe_force_wake_put(gt_to_fw(gt), XE_FW_GSC);
-	xe_device_mem_access_put(xe);
+	xe_pm_runtime_put(xe);
+}
+
+void xe_gsc_hwe_irq_handler(struct xe_hw_engine *hwe, u16 intr_vec)
+{
+	struct xe_gt *gt = hwe->gt;
+	struct xe_gsc *gsc = &gt->uc.gsc;
+
+	if (unlikely(!intr_vec))
+		return;
+
+	if (intr_vec & GSC_ER_COMPLETE) {
+		spin_lock(&gsc->lock);
+		gsc->work_actions |= GSC_ACTION_ER_COMPLETE;
+		spin_unlock(&gsc->lock);
+
+		queue_work(gsc->wq, &gsc->work);
+	}
 }
 
 int xe_gsc_init(struct xe_gsc *gsc)
@@ -382,8 +468,8 @@ int xe_gsc_init_post_hwconfig(struct xe_gsc *gsc)
 
 	bo = xe_bo_create_pin_map(xe, tile, NULL, SZ_4M,
 				  ttm_bo_type_kernel,
-				  XE_BO_CREATE_STOLEN_BIT |
-				  XE_BO_CREATE_GGTT_BIT);
+				  XE_BO_FLAG_STOLEN |
+				  XE_BO_FLAG_GGTT);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
diff --git a/drivers/gpu/drm/xe/xe_gsc.h b/drivers/gpu/drm/xe/xe_gsc.h
index c6fb32e3fd79..dd16e9b8b894 100644
--- a/drivers/gpu/drm/xe/xe_gsc.h
+++ b/drivers/gpu/drm/xe/xe_gsc.h
@@ -9,12 +9,14 @@
 #include "xe_gsc_types.h"
 
 struct xe_gt;
+struct xe_hw_engine;
 
 int xe_gsc_init(struct xe_gsc *gsc);
 int xe_gsc_init_post_hwconfig(struct xe_gsc *gsc);
 void xe_gsc_wait_for_worker_completion(struct xe_gsc *gsc);
 void xe_gsc_load_start(struct xe_gsc *gsc);
 void xe_gsc_remove(struct xe_gsc *gsc);
+void xe_gsc_hwe_irq_handler(struct xe_hw_engine *hwe, u16 intr_vec);
 
 void xe_gsc_wa_14015076503(struct xe_gt *gt, bool prep);
 
diff --git a/drivers/gpu/drm/xe/xe_gsc_proxy.c b/drivers/gpu/drm/xe/xe_gsc_proxy.c
index 309ef80e3b95..1b908d238bd1 100644
--- a/drivers/gpu/drm/xe/xe_gsc_proxy.c
+++ b/drivers/gpu/drm/xe/xe_gsc_proxy.c
@@ -66,7 +66,7 @@ static inline struct xe_device *kdev_to_xe(struct device *kdev)
 	return dev_get_drvdata(kdev);
 }
 
-static bool gsc_proxy_init_done(struct xe_gsc *gsc)
+bool xe_gsc_proxy_init_done(struct xe_gsc *gsc)
 {
 	struct xe_gt *gt = gsc_to_gt(gsc);
 	u32 fwsts1 = xe_mmio_read32(gt, HECI_FWSTS1(MTL_GSC_HECI1_BASE));
@@ -403,7 +403,6 @@ static int proxy_channel_alloc(struct xe_gsc *gsc)
 	struct xe_device *xe = gt_to_xe(gt);
 	struct xe_bo *bo;
 	void *csme;
-	int err;
 
 	csme = kzalloc(GSC_PROXY_CHANNEL_SIZE, GFP_KERNEL);
 	if (!csme)
@@ -411,8 +410,8 @@ static int proxy_channel_alloc(struct xe_gsc *gsc)
 
 	bo = xe_bo_create_pin_map(xe, tile, NULL, GSC_PROXY_CHANNEL_SIZE,
 				  ttm_bo_type_kernel,
-				  XE_BO_CREATE_SYSTEM_BIT |
-				  XE_BO_CREATE_GGTT_BIT);
+				  XE_BO_FLAG_SYSTEM |
+				  XE_BO_FLAG_GGTT);
 	if (IS_ERR(bo)) {
 		kfree(csme);
 		return PTR_ERR(bo);
@@ -424,11 +423,7 @@ static int proxy_channel_alloc(struct xe_gsc *gsc)
 	gsc->proxy.to_csme = csme;
 	gsc->proxy.from_csme = csme + GSC_PROXY_BUFFER_SIZE;
 
-	err = drmm_add_action_or_reset(&xe->drm, proxy_channel_free, gsc);
-	if (err)
-		return err;
-
-	return 0;
+	return drmm_add_action_or_reset(&xe->drm, proxy_channel_free, gsc);
 }
 
 /**
@@ -528,7 +523,7 @@ int xe_gsc_proxy_start(struct xe_gsc *gsc)
 	if (err)
 		return err;
 
-	if (!gsc_proxy_init_done(gsc)) {
+	if (!xe_gsc_proxy_init_done(gsc)) {
 		xe_gt_err(gsc_to_gt(gsc), "GSC FW reports proxy init not completed\n");
 		return -EIO;
 	}
diff --git a/drivers/gpu/drm/xe/xe_gsc_proxy.h b/drivers/gpu/drm/xe/xe_gsc_proxy.h
index 908f9441f093..c511ade6b863 100644
--- a/drivers/gpu/drm/xe/xe_gsc_proxy.h
+++ b/drivers/gpu/drm/xe/xe_gsc_proxy.h
@@ -11,6 +11,7 @@
 struct xe_gsc;
 
 int xe_gsc_proxy_init(struct xe_gsc *gsc);
+bool xe_gsc_proxy_init_done(struct xe_gsc *gsc);
 void xe_gsc_proxy_remove(struct xe_gsc *gsc);
 int xe_gsc_proxy_start(struct xe_gsc *gsc);
 
diff --git a/drivers/gpu/drm/xe/xe_gsc_submit.c b/drivers/gpu/drm/xe/xe_gsc_submit.c
index 348994b271be..d34d03248843 100644
--- a/drivers/gpu/drm/xe/xe_gsc_submit.c
+++ b/drivers/gpu/drm/xe/xe_gsc_submit.c
@@ -41,6 +41,21 @@ gsc_to_gt(struct xe_gsc *gsc)
 }
 
 /**
+ * xe_gsc_create_host_session_id - Creates a random 64 bit host_session id with
+ * bits 56-63 masked.
+ *
+ * Returns: random host_session_id which can be used to send messages to gsc cs
+ */
+u64 xe_gsc_create_host_session_id(void)
+{
+	u64 host_session_id;
+
+	get_random_bytes(&host_session_id, sizeof(u64));
+	host_session_id &= ~HOST_SESSION_CLIENT_MASK;
+	return host_session_id;
+}
+
+/**
  * xe_gsc_emit_header - write the MTL GSC header in memory
  * @xe: the Xe device
  * @map: the iosys map to write to
diff --git a/drivers/gpu/drm/xe/xe_gsc_submit.h b/drivers/gpu/drm/xe/xe_gsc_submit.h
index 1939855031a6..1416b5745a4c 100644
--- a/drivers/gpu/drm/xe/xe_gsc_submit.h
+++ b/drivers/gpu/drm/xe/xe_gsc_submit.h
@@ -28,4 +28,5 @@ int xe_gsc_read_out_header(struct xe_device *xe,
 int xe_gsc_pkt_submit_kernel(struct xe_gsc *gsc, u64 addr_in, u32 size_in,
 			     u64 addr_out, u32 size_out);
 
+u64 xe_gsc_create_host_session_id(void);
 #endif
diff --git a/drivers/gpu/drm/xe/xe_gsc_types.h b/drivers/gpu/drm/xe/xe_gsc_types.h
index 138d8cc0f19c..5926de20214c 100644
--- a/drivers/gpu/drm/xe/xe_gsc_types.h
+++ b/drivers/gpu/drm/xe/xe_gsc_types.h
@@ -47,6 +47,7 @@ struct xe_gsc {
 	u32 work_actions;
 #define GSC_ACTION_FW_LOAD BIT(0)
 #define GSC_ACTION_SW_PROXY BIT(1)
+#define GSC_ACTION_ER_COMPLETE BIT(2)
 
 	/** @proxy: sub-structure containing the SW proxy-related variables */
 	struct {
diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c
index a0afe1ba6dd5..491d0413de15 100644
--- a/drivers/gpu/drm/xe/xe_gt.c
+++ b/drivers/gpu/drm/xe/xe_gt.c
@@ -29,6 +29,7 @@
 #include "xe_gt_mcr.h"
 #include "xe_gt_pagefault.h"
 #include "xe_gt_printk.h"
+#include "xe_gt_sriov_pf.h"
 #include "xe_gt_sysfs.h"
 #include "xe_gt_tlb_invalidation.h"
 #include "xe_gt_topology.h"
@@ -43,6 +44,7 @@
 #include "xe_migrate.h"
 #include "xe_mmio.h"
 #include "xe_pat.h"
+#include "xe_pm.h"
 #include "xe_mocs.h"
 #include "xe_reg_sr.h"
 #include "xe_ring_ops.h"
@@ -310,6 +312,12 @@ int xe_gt_init_early(struct xe_gt *gt)
 {
 	int err;
 
+	if (IS_SRIOV_PF(gt_to_xe(gt))) {
+		err = xe_gt_sriov_pf_init_early(gt);
+		if (err)
+			return err;
+	}
+
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 	if (err)
 		return err;
@@ -346,7 +354,6 @@ static int gt_fw_domain_init(struct xe_gt *gt)
 {
 	int err, i;
 
-	xe_device_mem_access_get(gt_to_xe(gt));
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 	if (err)
 		goto err_hw_fence_irq;
@@ -359,7 +366,9 @@ static int gt_fw_domain_init(struct xe_gt *gt)
 			xe_lmtt_init(&gt_to_tile(gt)->sriov.pf.lmtt);
 	}
 
-	xe_gt_idle_sysfs_init(&gt->gtidle);
+	err = xe_gt_idle_sysfs_init(&gt->gtidle);
+	if (err)
+		goto err_force_wake;
 
 	/* Enable per hw engine IRQs */
 	xe_irq_enable_hwe(gt);
@@ -373,12 +382,12 @@ static int gt_fw_domain_init(struct xe_gt *gt)
 
 	err = xe_hw_engine_class_sysfs_init(gt);
 	if (err)
-		drm_warn(&gt_to_xe(gt)->drm,
-			 "failed to register engines sysfs directory, err: %d\n",
-			 err);
+		goto err_force_wake;
 
 	/* Initialize CCS mode sysfs after early initialization of HW engines */
-	xe_gt_ccs_mode_sysfs_init(gt);
+	err = xe_gt_ccs_mode_sysfs_init(gt);
+	if (err)
+		goto err_force_wake;
 
 	/*
 	 * Stash hardware-reported version.  Since this register does not exist
@@ -388,7 +397,6 @@ static int gt_fw_domain_init(struct xe_gt *gt)
 
 	err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
 	XE_WARN_ON(err);
-	xe_device_mem_access_put(gt_to_xe(gt));
 
 	return 0;
 
@@ -398,7 +406,6 @@ err_force_wake:
 err_hw_fence_irq:
 	for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
 		xe_hw_fence_irq_finish(&gt->fence_irq[i]);
-	xe_device_mem_access_put(gt_to_xe(gt));
 
 	return err;
 }
@@ -407,7 +414,6 @@ static int all_fw_domain_init(struct xe_gt *gt)
 {
 	int err, i;
 
-	xe_device_mem_access_get(gt_to_xe(gt));
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	if (err)
 		goto err_hw_fence_irq;
@@ -473,7 +479,6 @@ static int all_fw_domain_init(struct xe_gt *gt)
 
 	err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	XE_WARN_ON(err);
-	xe_device_mem_access_put(gt_to_xe(gt));
 
 	return 0;
 
@@ -482,7 +487,6 @@ err_force_wake:
 err_hw_fence_irq:
 	for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
 		xe_hw_fence_irq_finish(&gt->fence_irq[i]);
-	xe_device_mem_access_put(gt_to_xe(gt));
 
 	return err;
 }
@@ -495,7 +499,6 @@ int xe_gt_init_hwconfig(struct xe_gt *gt)
 {
 	int err;
 
-	xe_device_mem_access_get(gt_to_xe(gt));
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 	if (err)
 		goto out;
@@ -518,8 +521,6 @@ int xe_gt_init_hwconfig(struct xe_gt *gt)
 out_fw:
 	xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
 out:
-	xe_device_mem_access_put(gt_to_xe(gt));
-
 	return err;
 }
 
@@ -545,13 +546,17 @@ int xe_gt_init(struct xe_gt *gt)
 
 	xe_mocs_init_early(gt);
 
-	xe_gt_sysfs_init(gt);
+	err = xe_gt_sysfs_init(gt);
+	if (err)
+		return err;
 
 	err = gt_fw_domain_init(gt);
 	if (err)
 		return err;
 
-	xe_gt_freq_init(gt);
+	err = xe_gt_freq_init(gt);
+	if (err)
+		return err;
 
 	xe_force_wake_init_engines(gt, gt_to_fw(gt));
 
@@ -559,11 +564,7 @@ int xe_gt_init(struct xe_gt *gt)
 	if (err)
 		return err;
 
-	err = drmm_add_action_or_reset(&gt_to_xe(gt)->drm, gt_fini, gt);
-	if (err)
-		return err;
-
-	return 0;
+	return drmm_add_action_or_reset(&gt_to_xe(gt)->drm, gt_fini, gt);
 }
 
 static int do_gt_reset(struct xe_gt *gt)
@@ -643,9 +644,9 @@ static int gt_reset(struct xe_gt *gt)
 		goto err_fail;
 	}
 
+	xe_pm_runtime_get(gt_to_xe(gt));
 	xe_gt_sanitize(gt);
 
-	xe_device_mem_access_get(gt_to_xe(gt));
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	if (err)
 		goto err_msg;
@@ -669,8 +670,8 @@ static int gt_reset(struct xe_gt *gt)
 		goto err_out;
 
 	err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	xe_device_mem_access_put(gt_to_xe(gt));
 	XE_WARN_ON(err);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
 	xe_gt_info(gt, "reset done\n");
 
@@ -680,7 +681,7 @@ err_out:
 	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
 err_msg:
 	XE_WARN_ON(xe_uc_start(&gt->uc));
-	xe_device_mem_access_put(gt_to_xe(gt));
+	xe_pm_runtime_put(gt_to_xe(gt));
 err_fail:
 	xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err));
 
@@ -710,22 +711,20 @@ void xe_gt_reset_async(struct xe_gt *gt)
 
 void xe_gt_suspend_prepare(struct xe_gt *gt)
 {
-	xe_device_mem_access_get(gt_to_xe(gt));
 	XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL));
 
 	xe_uc_stop_prepare(&gt->uc);
 
 	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
-	xe_device_mem_access_put(gt_to_xe(gt));
 }
 
 int xe_gt_suspend(struct xe_gt *gt)
 {
 	int err;
 
+	xe_gt_dbg(gt, "suspending\n");
 	xe_gt_sanitize(gt);
 
-	xe_device_mem_access_get(gt_to_xe(gt));
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	if (err)
 		goto err_msg;
@@ -735,15 +734,13 @@ int xe_gt_suspend(struct xe_gt *gt)
 		goto err_force_wake;
 
 	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
-	xe_device_mem_access_put(gt_to_xe(gt));
-	xe_gt_info(gt, "suspended\n");
+	xe_gt_dbg(gt, "suspended\n");
 
 	return 0;
 
 err_force_wake:
 	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
 err_msg:
-	xe_device_mem_access_put(gt_to_xe(gt));
 	xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err));
 
 	return err;
@@ -753,7 +750,7 @@ int xe_gt_resume(struct xe_gt *gt)
 {
 	int err;
 
-	xe_device_mem_access_get(gt_to_xe(gt));
+	xe_gt_dbg(gt, "resuming\n");
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	if (err)
 		goto err_msg;
@@ -763,15 +760,13 @@ int xe_gt_resume(struct xe_gt *gt)
 		goto err_force_wake;
 
 	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
-	xe_device_mem_access_put(gt_to_xe(gt));
-	xe_gt_info(gt, "resumed\n");
+	xe_gt_dbg(gt, "resumed\n");
 
 	return 0;
 
 err_force_wake:
 	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
 err_msg:
-	xe_device_mem_access_put(gt_to_xe(gt));
 	xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err));
 
 	return err;
diff --git a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c
index 529fc286cd06..396aeb5b9924 100644
--- a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c
+++ b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c
@@ -167,25 +167,20 @@ static void xe_gt_ccs_mode_sysfs_fini(struct drm_device *drm, void *arg)
  * and it is expected that there are no open drm clients while doing so.
  * The number of available compute slices is exposed to user through a per-gt
  * 'num_cslices' sysfs interface.
+ *
+ * Returns: Returns error value for failure and 0 for success.
  */
-void xe_gt_ccs_mode_sysfs_init(struct xe_gt *gt)
+int xe_gt_ccs_mode_sysfs_init(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 	int err;
 
 	if (!xe_gt_ccs_mode_enabled(gt))
-		return;
+		return 0;
 
 	err = sysfs_create_files(gt->sysfs, gt_ccs_mode_attrs);
-	if (err) {
-		drm_warn(&xe->drm, "Sysfs creation for ccs_mode failed err: %d\n", err);
-		return;
-	}
+	if (err)
+		return err;
 
-	err = drmm_add_action_or_reset(&xe->drm, xe_gt_ccs_mode_sysfs_fini, gt);
-	if (err) {
-		sysfs_remove_files(gt->sysfs, gt_ccs_mode_attrs);
-		drm_warn(&xe->drm, "%s: drmm_add_action_or_reset failed, err: %d\n",
-			 __func__, err);
-	}
+	return drmm_add_action_or_reset(&xe->drm, xe_gt_ccs_mode_sysfs_fini, gt);
 }
diff --git a/drivers/gpu/drm/xe/xe_gt_ccs_mode.h b/drivers/gpu/drm/xe/xe_gt_ccs_mode.h
index f39975aaaab0..f8779852cf0d 100644
--- a/drivers/gpu/drm/xe/xe_gt_ccs_mode.h
+++ b/drivers/gpu/drm/xe/xe_gt_ccs_mode.h
@@ -12,7 +12,7 @@
 #include "xe_platform_types.h"
 
 void xe_gt_apply_ccs_mode(struct xe_gt *gt);
-void xe_gt_ccs_mode_sysfs_init(struct xe_gt *gt);
+int xe_gt_ccs_mode_sysfs_init(struct xe_gt *gt);
 
 static inline bool xe_gt_ccs_mode_enabled(const struct xe_gt *gt)
 {
diff --git a/drivers/gpu/drm/xe/xe_gt_clock.c b/drivers/gpu/drm/xe/xe_gt_clock.c
index 937054e31d72..c7bca20f6b65 100644
--- a/drivers/gpu/drm/xe/xe_gt_clock.c
+++ b/drivers/gpu/drm/xe/xe_gt_clock.c
@@ -78,8 +78,3 @@ int xe_gt_clock_init(struct xe_gt *gt)
 	gt->info.reference_clock = freq;
 	return 0;
 }
-
-u64 xe_gt_clock_cycles_to_ns(const struct xe_gt *gt, u64 count)
-{
-	return DIV_ROUND_CLOSEST_ULL(count * NSEC_PER_SEC, gt->info.reference_clock);
-}
diff --git a/drivers/gpu/drm/xe/xe_gt_clock.h b/drivers/gpu/drm/xe/xe_gt_clock.h
index aa162722f859..44fa0371b973 100644
--- a/drivers/gpu/drm/xe/xe_gt_clock.h
+++ b/drivers/gpu/drm/xe/xe_gt_clock.h
@@ -11,5 +11,5 @@
 struct xe_gt;
 
 int xe_gt_clock_init(struct xe_gt *gt);
-u64 xe_gt_clock_cycles_to_ns(const struct xe_gt *gt, u64 count);
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c
index c4b67cf09f8f..8cf0b2625efc 100644
--- a/drivers/gpu/drm/xe/xe_gt_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c
@@ -5,6 +5,8 @@
 
 #include "xe_gt_debugfs.h"
 
+#include <linux/debugfs.h>
+
 #include <drm/drm_debugfs.h>
 #include <drm/drm_managed.h>
 
@@ -18,193 +20,246 @@
 #include "xe_lrc.h"
 #include "xe_macros.h"
 #include "xe_pat.h"
+#include "xe_pm.h"
 #include "xe_reg_sr.h"
 #include "xe_reg_whitelist.h"
 #include "xe_uc_debugfs.h"
 #include "xe_wa.h"
 
-static struct xe_gt *node_to_gt(struct drm_info_node *node)
+/**
+ * xe_gt_debugfs_simple_show - A show callback for struct drm_info_list
+ * @m: the &seq_file
+ * @data: data used by the drm debugfs helpers
+ *
+ * This callback can be used in struct drm_info_list to describe debugfs
+ * files that are &xe_gt specific.
+ *
+ * It is assumed that those debugfs files will be created on directory entry
+ * which struct dentry d_inode->i_private points to &xe_gt.
+ *
+ * This function assumes that &m->private will be set to the &struct
+ * drm_info_node corresponding to the instance of the info on a given &struct
+ * drm_minor (see struct drm_info_list.show for details).
+ *
+ * This function also assumes that struct drm_info_list.data will point to the
+ * function code that will actually print a file content::
+ *
+ *   int (*print)(struct xe_gt *, struct drm_printer *)
+ *
+ * Example::
+ *
+ *    int foo(struct xe_gt *gt, struct drm_printer *p)
+ *    {
+ *        drm_printf(p, "GT%u\n", gt->info.id);
+ *        return 0;
+ *    }
+ *
+ *    static const struct drm_info_list bar[] = {
+ *        { name = "foo", .show = xe_gt_debugfs_simple_show, .data = foo },
+ *    };
+ *
+ *    dir = debugfs_create_dir("gt", parent);
+ *    dir->d_inode->i_private = gt;
+ *    drm_debugfs_create_files(bar, ARRAY_SIZE(bar), dir, minor);
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_debugfs_simple_show(struct seq_file *m, void *data)
 {
-	return node->info_ent->data;
+	struct drm_printer p = drm_seq_file_printer(m);
+	struct drm_info_node *node = m->private;
+	struct dentry *parent = node->dent->d_parent;
+	struct xe_gt *gt = parent->d_inode->i_private;
+	int (*print)(struct xe_gt *, struct drm_printer *) = node->info_ent->data;
+
+	if (WARN_ON(!print))
+		return -EINVAL;
+
+	return print(gt, &p);
 }
 
-static int hw_engines(struct seq_file *m, void *data)
+static int hw_engines(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct xe_gt *gt = node_to_gt(m->private);
 	struct xe_device *xe = gt_to_xe(gt);
-	struct drm_printer p = drm_seq_file_printer(m);
 	struct xe_hw_engine *hwe;
 	enum xe_hw_engine_id id;
 	int err;
 
-	xe_device_mem_access_get(xe);
+	xe_pm_runtime_get(xe);
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	if (err) {
-		xe_device_mem_access_put(xe);
+		xe_pm_runtime_put(xe);
 		return err;
 	}
 
 	for_each_hw_engine(hwe, gt, id)
-		xe_hw_engine_print(hwe, &p);
+		xe_hw_engine_print(hwe, p);
 
 	err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	xe_device_mem_access_put(xe);
+	xe_pm_runtime_put(xe);
 	if (err)
 		return err;
 
 	return 0;
 }
 
-static int force_reset(struct seq_file *m, void *data)
+static int force_reset(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct xe_gt *gt = node_to_gt(m->private);
-
+	xe_pm_runtime_get(gt_to_xe(gt));
 	xe_gt_reset_async(gt);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
 	return 0;
 }
 
-static int sa_info(struct seq_file *m, void *data)
+static int sa_info(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct xe_tile *tile = gt_to_tile(node_to_gt(m->private));
-	struct drm_printer p = drm_seq_file_printer(m);
+	struct xe_tile *tile = gt_to_tile(gt);
 
-	drm_suballoc_dump_debug_info(&tile->mem.kernel_bb_pool->base, &p,
+	xe_pm_runtime_get(gt_to_xe(gt));
+	drm_suballoc_dump_debug_info(&tile->mem.kernel_bb_pool->base, p,
 				     tile->mem.kernel_bb_pool->gpu_addr);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
 	return 0;
 }
 
-static int topology(struct seq_file *m, void *data)
+static int topology(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct xe_gt *gt = node_to_gt(m->private);
-	struct drm_printer p = drm_seq_file_printer(m);
-
-	xe_gt_topology_dump(gt, &p);
+	xe_pm_runtime_get(gt_to_xe(gt));
+	xe_gt_topology_dump(gt, p);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
 	return 0;
 }
 
-static int steering(struct seq_file *m, void *data)
+static int steering(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct xe_gt *gt = node_to_gt(m->private);
-	struct drm_printer p = drm_seq_file_printer(m);
-
-	xe_gt_mcr_steering_dump(gt, &p);
+	xe_pm_runtime_get(gt_to_xe(gt));
+	xe_gt_mcr_steering_dump(gt, p);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
 	return 0;
 }
 
-static int ggtt(struct seq_file *m, void *data)
+static int ggtt(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct xe_gt *gt = node_to_gt(m->private);
-	struct drm_printer p = drm_seq_file_printer(m);
+	int ret;
+
+	xe_pm_runtime_get(gt_to_xe(gt));
+	ret = xe_ggtt_dump(gt_to_tile(gt)->mem.ggtt, p);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
-	return xe_ggtt_dump(gt_to_tile(gt)->mem.ggtt, &p);
+	return ret;
 }
 
-static int register_save_restore(struct seq_file *m, void *data)
+static int register_save_restore(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct xe_gt *gt = node_to_gt(m->private);
-	struct drm_printer p = drm_seq_file_printer(m);
 	struct xe_hw_engine *hwe;
 	enum xe_hw_engine_id id;
 
-	xe_reg_sr_dump(&gt->reg_sr, &p);
-	drm_printf(&p, "\n");
+	xe_pm_runtime_get(gt_to_xe(gt));
 
-	drm_printf(&p, "Engine\n");
+	xe_reg_sr_dump(&gt->reg_sr, p);
+	drm_printf(p, "\n");
+
+	drm_printf(p, "Engine\n");
 	for_each_hw_engine(hwe, gt, id)
-		xe_reg_sr_dump(&hwe->reg_sr, &p);
-	drm_printf(&p, "\n");
+		xe_reg_sr_dump(&hwe->reg_sr, p);
+	drm_printf(p, "\n");
 
-	drm_printf(&p, "LRC\n");
+	drm_printf(p, "LRC\n");
 	for_each_hw_engine(hwe, gt, id)
-		xe_reg_sr_dump(&hwe->reg_lrc, &p);
-	drm_printf(&p, "\n");
+		xe_reg_sr_dump(&hwe->reg_lrc, p);
+	drm_printf(p, "\n");
 
-	drm_printf(&p, "Whitelist\n");
+	drm_printf(p, "Whitelist\n");
 	for_each_hw_engine(hwe, gt, id)
-		xe_reg_whitelist_dump(&hwe->reg_whitelist, &p);
+		xe_reg_whitelist_dump(&hwe->reg_whitelist, p);
+
+	xe_pm_runtime_put(gt_to_xe(gt));
 
 	return 0;
 }
 
-static int workarounds(struct seq_file *m, void *data)
+static int workarounds(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct xe_gt *gt = node_to_gt(m->private);
-	struct drm_printer p = drm_seq_file_printer(m);
-
-	xe_wa_dump(gt, &p);
+	xe_pm_runtime_get(gt_to_xe(gt));
+	xe_wa_dump(gt, p);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
 	return 0;
 }
 
-static int pat(struct seq_file *m, void *data)
+static int pat(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct xe_gt *gt = node_to_gt(m->private);
-	struct drm_printer p = drm_seq_file_printer(m);
-
-	xe_pat_dump(gt, &p);
+	xe_pm_runtime_get(gt_to_xe(gt));
+	xe_pat_dump(gt, p);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
 	return 0;
 }
 
-static int rcs_default_lrc(struct seq_file *m, void *data)
+static int rcs_default_lrc(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct drm_printer p = drm_seq_file_printer(m);
+	xe_pm_runtime_get(gt_to_xe(gt));
+	xe_lrc_dump_default(p, gt, XE_ENGINE_CLASS_RENDER);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
-	xe_lrc_dump_default(&p, node_to_gt(m->private), XE_ENGINE_CLASS_RENDER);
 	return 0;
 }
 
-static int ccs_default_lrc(struct seq_file *m, void *data)
+static int ccs_default_lrc(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct drm_printer p = drm_seq_file_printer(m);
+	xe_pm_runtime_get(gt_to_xe(gt));
+	xe_lrc_dump_default(p, gt, XE_ENGINE_CLASS_COMPUTE);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
-	xe_lrc_dump_default(&p, node_to_gt(m->private), XE_ENGINE_CLASS_COMPUTE);
 	return 0;
 }
 
-static int bcs_default_lrc(struct seq_file *m, void *data)
+static int bcs_default_lrc(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct drm_printer p = drm_seq_file_printer(m);
+	xe_pm_runtime_get(gt_to_xe(gt));
+	xe_lrc_dump_default(p, gt, XE_ENGINE_CLASS_COPY);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
-	xe_lrc_dump_default(&p, node_to_gt(m->private), XE_ENGINE_CLASS_COPY);
 	return 0;
 }
 
-static int vcs_default_lrc(struct seq_file *m, void *data)
+static int vcs_default_lrc(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct drm_printer p = drm_seq_file_printer(m);
+	xe_pm_runtime_get(gt_to_xe(gt));
+	xe_lrc_dump_default(p, gt, XE_ENGINE_CLASS_VIDEO_DECODE);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
-	xe_lrc_dump_default(&p, node_to_gt(m->private), XE_ENGINE_CLASS_VIDEO_DECODE);
 	return 0;
 }
 
-static int vecs_default_lrc(struct seq_file *m, void *data)
+static int vecs_default_lrc(struct xe_gt *gt, struct drm_printer *p)
 {
-	struct drm_printer p = drm_seq_file_printer(m);
+	xe_pm_runtime_get(gt_to_xe(gt));
+	xe_lrc_dump_default(p, gt, XE_ENGINE_CLASS_VIDEO_ENHANCE);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
-	xe_lrc_dump_default(&p, node_to_gt(m->private), XE_ENGINE_CLASS_VIDEO_ENHANCE);
 	return 0;
 }
 
 static const struct drm_info_list debugfs_list[] = {
-	{"hw_engines", hw_engines, 0},
-	{"force_reset", force_reset, 0},
-	{"sa_info", sa_info, 0},
-	{"topology", topology, 0},
-	{"steering", steering, 0},
-	{"ggtt", ggtt, 0},
-	{"register-save-restore", register_save_restore, 0},
-	{"workarounds", workarounds, 0},
-	{"pat", pat, 0},
-	{"default_lrc_rcs", rcs_default_lrc},
-	{"default_lrc_ccs", ccs_default_lrc},
-	{"default_lrc_bcs", bcs_default_lrc},
-	{"default_lrc_vcs", vcs_default_lrc},
-	{"default_lrc_vecs", vecs_default_lrc},
+	{"hw_engines", .show = xe_gt_debugfs_simple_show, .data = hw_engines},
+	{"force_reset", .show = xe_gt_debugfs_simple_show, .data = force_reset},
+	{"sa_info", .show = xe_gt_debugfs_simple_show, .data = sa_info},
+	{"topology", .show = xe_gt_debugfs_simple_show, .data = topology},
+	{"steering", .show = xe_gt_debugfs_simple_show, .data = steering},
+	{"ggtt", .show = xe_gt_debugfs_simple_show, .data = ggtt},
+	{"register-save-restore", .show = xe_gt_debugfs_simple_show, .data = register_save_restore},
+	{"workarounds", .show = xe_gt_debugfs_simple_show, .data = workarounds},
+	{"pat", .show = xe_gt_debugfs_simple_show, .data = pat},
+	{"default_lrc_rcs", .show = xe_gt_debugfs_simple_show, .data = rcs_default_lrc},
+	{"default_lrc_ccs", .show = xe_gt_debugfs_simple_show, .data = ccs_default_lrc},
+	{"default_lrc_bcs", .show = xe_gt_debugfs_simple_show, .data = bcs_default_lrc},
+	{"default_lrc_vcs", .show = xe_gt_debugfs_simple_show, .data = vcs_default_lrc},
+	{"default_lrc_vecs", .show = xe_gt_debugfs_simple_show, .data = vecs_default_lrc},
 };
 
 void xe_gt_debugfs_register(struct xe_gt *gt)
@@ -212,13 +267,11 @@ void xe_gt_debugfs_register(struct xe_gt *gt)
 	struct xe_device *xe = gt_to_xe(gt);
 	struct drm_minor *minor = gt_to_xe(gt)->drm.primary;
 	struct dentry *root;
-	struct drm_info_list *local;
 	char name[8];
-	int i;
 
 	xe_gt_assert(gt, minor->debugfs_root);
 
-	sprintf(name, "gt%d", gt->info.id);
+	snprintf(name, sizeof(name), "gt%d", gt->info.id);
 	root = debugfs_create_dir(name, minor->debugfs_root);
 	if (IS_ERR(root)) {
 		drm_warn(&xe->drm, "Create GT directory failed");
@@ -226,22 +279,13 @@ void xe_gt_debugfs_register(struct xe_gt *gt)
 	}
 
 	/*
-	 * Allocate local copy as we need to pass in the GT to the debugfs
-	 * entry and drm_debugfs_create_files just references the drm_info_list
-	 * passed in (e.g. can't define this on the stack).
+	 * Store the xe_gt pointer as private data of the gt/ directory node
+	 * so other GT specific attributes under that directory may refer to
+	 * it by looking at its parent node private data.
 	 */
-#define DEBUGFS_SIZE	(ARRAY_SIZE(debugfs_list) * sizeof(struct drm_info_list))
-	local = drmm_kmalloc(&xe->drm, DEBUGFS_SIZE, GFP_KERNEL);
-	if (!local)
-		return;
-
-	memcpy(local, debugfs_list, DEBUGFS_SIZE);
-#undef DEBUGFS_SIZE
-
-	for (i = 0; i < ARRAY_SIZE(debugfs_list); ++i)
-		local[i].data = gt;
+	root->d_inode->i_private = gt;
 
-	drm_debugfs_create_files(local,
+	drm_debugfs_create_files(debugfs_list,
 				 ARRAY_SIZE(debugfs_list),
 				 root, minor);
 
diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.h b/drivers/gpu/drm/xe/xe_gt_debugfs.h
index 5a329f118a57..05a6cc93c78c 100644
--- a/drivers/gpu/drm/xe/xe_gt_debugfs.h
+++ b/drivers/gpu/drm/xe/xe_gt_debugfs.h
@@ -6,8 +6,10 @@
 #ifndef _XE_GT_DEBUGFS_H_
 #define _XE_GT_DEBUGFS_H_
 
+struct seq_file;
 struct xe_gt;
 
 void xe_gt_debugfs_register(struct xe_gt *gt);
+int xe_gt_debugfs_simple_show(struct seq_file *m, void *data);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_gt_freq.c b/drivers/gpu/drm/xe/xe_gt_freq.c
index e5b0f4ecdbe8..855de40e40ea 100644
--- a/drivers/gpu/drm/xe/xe_gt_freq.c
+++ b/drivers/gpu/drm/xe/xe_gt_freq.c
@@ -15,6 +15,7 @@
 #include "xe_gt_sysfs.h"
 #include "xe_gt_throttle_sysfs.h"
 #include "xe_guc_pc.h"
+#include "xe_pm.h"
 
 /**
  * DOC: Xe GT Frequency Management
@@ -49,12 +50,23 @@ dev_to_pc(struct device *dev)
 	return &kobj_to_gt(dev->kobj.parent)->uc.guc.pc;
 }
 
+static struct xe_device *
+dev_to_xe(struct device *dev)
+{
+	return gt_to_xe(kobj_to_gt(dev->kobj.parent));
+}
+
 static ssize_t act_freq_show(struct device *dev,
 			     struct device_attribute *attr, char *buf)
 {
 	struct xe_guc_pc *pc = dev_to_pc(dev);
+	u32 freq;
 
-	return sysfs_emit(buf, "%d\n", xe_guc_pc_get_act_freq(pc));
+	xe_pm_runtime_get(dev_to_xe(dev));
+	freq = xe_guc_pc_get_act_freq(pc);
+	xe_pm_runtime_put(dev_to_xe(dev));
+
+	return sysfs_emit(buf, "%d\n", freq);
 }
 static DEVICE_ATTR_RO(act_freq);
 
@@ -65,7 +77,9 @@ static ssize_t cur_freq_show(struct device *dev,
 	u32 freq;
 	ssize_t ret;
 
+	xe_pm_runtime_get(dev_to_xe(dev));
 	ret = xe_guc_pc_get_cur_freq(pc, &freq);
+	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 
@@ -77,8 +91,13 @@ static ssize_t rp0_freq_show(struct device *dev,
 			     struct device_attribute *attr, char *buf)
 {
 	struct xe_guc_pc *pc = dev_to_pc(dev);
+	u32 freq;
 
-	return sysfs_emit(buf, "%d\n", xe_guc_pc_get_rp0_freq(pc));
+	xe_pm_runtime_get(dev_to_xe(dev));
+	freq = xe_guc_pc_get_rp0_freq(pc);
+	xe_pm_runtime_put(dev_to_xe(dev));
+
+	return sysfs_emit(buf, "%d\n", freq);
 }
 static DEVICE_ATTR_RO(rp0_freq);
 
@@ -86,8 +105,13 @@ static ssize_t rpe_freq_show(struct device *dev,
 			     struct device_attribute *attr, char *buf)
 {
 	struct xe_guc_pc *pc = dev_to_pc(dev);
+	u32 freq;
+
+	xe_pm_runtime_get(dev_to_xe(dev));
+	freq = xe_guc_pc_get_rpe_freq(pc);
+	xe_pm_runtime_put(dev_to_xe(dev));
 
-	return sysfs_emit(buf, "%d\n", xe_guc_pc_get_rpe_freq(pc));
+	return sysfs_emit(buf, "%d\n", freq);
 }
 static DEVICE_ATTR_RO(rpe_freq);
 
@@ -107,7 +131,9 @@ static ssize_t min_freq_show(struct device *dev,
 	u32 freq;
 	ssize_t ret;
 
+	xe_pm_runtime_get(dev_to_xe(dev));
 	ret = xe_guc_pc_get_min_freq(pc, &freq);
+	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 
@@ -125,7 +151,9 @@ static ssize_t min_freq_store(struct device *dev, struct device_attribute *attr,
 	if (ret)
 		return ret;
 
+	xe_pm_runtime_get(dev_to_xe(dev));
 	ret = xe_guc_pc_set_min_freq(pc, freq);
+	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 
@@ -140,7 +168,9 @@ static ssize_t max_freq_show(struct device *dev,
 	u32 freq;
 	ssize_t ret;
 
+	xe_pm_runtime_get(dev_to_xe(dev));
 	ret = xe_guc_pc_get_max_freq(pc, &freq);
+	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 
@@ -158,7 +188,9 @@ static ssize_t max_freq_store(struct device *dev, struct device_attribute *attr,
 	if (ret)
 		return ret;
 
+	xe_pm_runtime_get(dev_to_xe(dev));
 	ret = xe_guc_pc_set_max_freq(pc, freq);
+	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 
@@ -190,33 +222,28 @@ static void freq_fini(struct drm_device *drm, void *arg)
  * @gt: Xe GT object
  *
  * It needs to be initialized after GT Sysfs and GuC PC components are ready.
+ *
+ * Returns: Returns error value for failure and 0 for success.
  */
-void xe_gt_freq_init(struct xe_gt *gt)
+int xe_gt_freq_init(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 	int err;
 
 	if (xe->info.skip_guc_pc)
-		return;
+		return 0;
 
 	gt->freq = kobject_create_and_add("freq0", gt->sysfs);
-	if (!gt->freq) {
-		drm_warn(&xe->drm, "failed to add freq0 directory to %s\n",
-			 kobject_name(gt->sysfs));
-		return;
-	}
+	if (!gt->freq)
+		return -ENOMEM;
 
 	err = drmm_add_action_or_reset(&xe->drm, freq_fini, gt->freq);
-	if (err) {
-		drm_warn(&xe->drm, "%s: drmm_add_action_or_reset failed, err: %d\n",
-			 __func__, err);
-		return;
-	}
+	if (err)
+		return err;
 
 	err = sysfs_create_files(gt->freq, freq_attrs);
 	if (err)
-		drm_warn(&xe->drm,  "failed to add freq attrs to %s, err: %d\n",
-			 kobject_name(gt->freq), err);
+		return err;
 
-	xe_gt_throttle_sysfs_init(gt);
+	return xe_gt_throttle_sysfs_init(gt);
 }
diff --git a/drivers/gpu/drm/xe/xe_gt_freq.h b/drivers/gpu/drm/xe/xe_gt_freq.h
index f3fe3c90491a..b7fddbe7b9b6 100644
--- a/drivers/gpu/drm/xe/xe_gt_freq.h
+++ b/drivers/gpu/drm/xe/xe_gt_freq.h
@@ -8,6 +8,6 @@
 
 struct xe_gt;
 
-void xe_gt_freq_init(struct xe_gt *gt);
+int xe_gt_freq_init(struct xe_gt *gt);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c
index 9fcae65b6469..8fc0f3f6ecc5 100644
--- a/drivers/gpu/drm/xe/xe_gt_idle.c
+++ b/drivers/gpu/drm/xe/xe_gt_idle.c
@@ -12,6 +12,7 @@
 #include "xe_guc_pc.h"
 #include "regs/xe_gt_regs.h"
 #include "xe_mmio.h"
+#include "xe_pm.h"
 
 /**
  * DOC: Xe GT Idle
@@ -40,6 +41,15 @@ static struct xe_guc_pc *gtidle_to_pc(struct xe_gt_idle *gtidle)
 	return &gtidle_to_gt(gtidle)->uc.guc.pc;
 }
 
+static struct xe_device *
+pc_to_xe(struct xe_guc_pc *pc)
+{
+	struct xe_guc *guc = container_of(pc, struct xe_guc, pc);
+	struct xe_gt *gt = container_of(guc, struct xe_gt, uc.guc);
+
+	return gt_to_xe(gt);
+}
+
 static const char *gt_idle_state_to_string(enum xe_gt_idle_state state)
 {
 	switch (state) {
@@ -86,8 +96,14 @@ static ssize_t name_show(struct device *dev,
 			 struct device_attribute *attr, char *buff)
 {
 	struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
+	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
+	ssize_t ret;
+
+	xe_pm_runtime_get(pc_to_xe(pc));
+	ret = sysfs_emit(buff, "%s\n", gtidle->name);
+	xe_pm_runtime_put(pc_to_xe(pc));
 
-	return sysfs_emit(buff, "%s\n", gtidle->name);
+	return ret;
 }
 static DEVICE_ATTR_RO(name);
 
@@ -98,7 +114,9 @@ static ssize_t idle_status_show(struct device *dev,
 	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
 	enum xe_gt_idle_state state;
 
+	xe_pm_runtime_get(pc_to_xe(pc));
 	state = gtidle->idle_status(pc);
+	xe_pm_runtime_put(pc_to_xe(pc));
 
 	return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state));
 }
@@ -111,7 +129,10 @@ static ssize_t idle_residency_ms_show(struct device *dev,
 	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
 	u64 residency;
 
+	xe_pm_runtime_get(pc_to_xe(pc));
 	residency = gtidle->idle_residency(pc);
+	xe_pm_runtime_put(pc_to_xe(pc));
+
 	return sysfs_emit(buff, "%llu\n", get_residency_ms(gtidle, residency));
 }
 static DEVICE_ATTR_RO(idle_residency_ms);
@@ -131,7 +152,7 @@ static void gt_idle_sysfs_fini(struct drm_device *drm, void *arg)
 	kobject_put(kobj);
 }
 
-void xe_gt_idle_sysfs_init(struct xe_gt_idle *gtidle)
+int xe_gt_idle_sysfs_init(struct xe_gt_idle *gtidle)
 {
 	struct xe_gt *gt = gtidle_to_gt(gtidle);
 	struct xe_device *xe = gt_to_xe(gt);
@@ -139,16 +160,14 @@ void xe_gt_idle_sysfs_init(struct xe_gt_idle *gtidle)
 	int err;
 
 	kobj = kobject_create_and_add("gtidle", gt->sysfs);
-	if (!kobj) {
-		drm_warn(&xe->drm, "%s failed, err: %d\n", __func__, -ENOMEM);
-		return;
-	}
+	if (!kobj)
+		return -ENOMEM;
 
 	if (xe_gt_is_media_type(gt)) {
-		sprintf(gtidle->name, "gt%d-mc", gt->info.id);
+		snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-mc", gt->info.id);
 		gtidle->idle_residency = xe_guc_pc_mc6_residency;
 	} else {
-		sprintf(gtidle->name, "gt%d-rc", gt->info.id);
+		snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-rc", gt->info.id);
 		gtidle->idle_residency = xe_guc_pc_rc6_residency;
 	}
 
@@ -159,14 +178,10 @@ void xe_gt_idle_sysfs_init(struct xe_gt_idle *gtidle)
 	err = sysfs_create_files(kobj, gt_idle_attrs);
 	if (err) {
 		kobject_put(kobj);
-		drm_warn(&xe->drm, "failed to register gtidle sysfs, err: %d\n", err);
-		return;
+		return err;
 	}
 
-	err = drmm_add_action_or_reset(&xe->drm, gt_idle_sysfs_fini, kobj);
-	if (err)
-		drm_warn(&xe->drm, "%s: drmm_add_action_or_reset failed, err: %d\n",
-			 __func__, err);
+	return drmm_add_action_or_reset(&xe->drm, gt_idle_sysfs_fini, kobj);
 }
 
 void xe_gt_idle_enable_c6(struct xe_gt *gt)
diff --git a/drivers/gpu/drm/xe/xe_gt_idle.h b/drivers/gpu/drm/xe/xe_gt_idle.h
index 69280fd16b03..75bd99659b1b 100644
--- a/drivers/gpu/drm/xe/xe_gt_idle.h
+++ b/drivers/gpu/drm/xe/xe_gt_idle.h
@@ -10,7 +10,7 @@
 
 struct xe_gt;
 
-void xe_gt_idle_sysfs_init(struct xe_gt_idle *gtidle);
+int xe_gt_idle_sysfs_init(struct xe_gt_idle *gtidle);
 void xe_gt_idle_enable_c6(struct xe_gt *gt);
 void xe_gt_idle_disable_c6(struct xe_gt *gt);
 
diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.c b/drivers/gpu/drm/xe/xe_gt_mcr.c
index a7ab9ba645f9..577bd7043740 100644
--- a/drivers/gpu/drm/xe/xe_gt_mcr.c
+++ b/drivers/gpu/drm/xe/xe_gt_mcr.c
@@ -6,6 +6,7 @@
 #include "xe_gt_mcr.h"
 
 #include "regs/xe_gt_regs.h"
+#include "xe_assert.h"
 #include "xe_gt.h"
 #include "xe_gt_topology.h"
 #include "xe_gt_types.h"
@@ -294,14 +295,40 @@ static void init_steering_mslice(struct xe_gt *gt)
 	gt->steering[LNCF].instance_target = 0;		/* unused */
 }
 
-static void init_steering_dss(struct xe_gt *gt)
+static unsigned int dss_per_group(struct xe_gt *gt)
 {
-	unsigned int dss = min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0),
-			       xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0));
-	unsigned int dss_per_grp = gt_to_xe(gt)->info.platform == XE_PVC ? 8 : 4;
+	if (gt_to_xe(gt)->info.platform == XE_PVC)
+		return 8;
+	else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250)
+		return 4;
+	else
+		return 6;
+}
 
-	gt->steering[DSS].group_target = dss / dss_per_grp;
-	gt->steering[DSS].instance_target = dss % dss_per_grp;
+/**
+ * xe_gt_mcr_get_dss_steering - Get the group/instance steering for a DSS
+ * @gt: GT structure
+ * @dss: DSS ID to obtain steering for
+ * @group: pointer to storage for steering group ID
+ * @instance: pointer to storage for steering instance ID
+ */
+void xe_gt_mcr_get_dss_steering(struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance)
+{
+	int dss_per_grp = dss_per_group(gt);
+
+	xe_gt_assert(gt, dss < XE_MAX_DSS_FUSE_BITS);
+
+	*group = dss / dss_per_grp;
+	*instance = dss % dss_per_grp;
+}
+
+static void init_steering_dss(struct xe_gt *gt)
+{
+	xe_gt_mcr_get_dss_steering(gt,
+				   min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0),
+				       xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0)),
+				   &gt->steering[DSS].group_target,
+				   &gt->steering[DSS].instance_target);
 }
 
 static void init_steering_oaddrm(struct xe_gt *gt)
diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.h b/drivers/gpu/drm/xe/xe_gt_mcr.h
index 27ca1bc880a0..a7f4ab1aa584 100644
--- a/drivers/gpu/drm/xe/xe_gt_mcr.h
+++ b/drivers/gpu/drm/xe/xe_gt_mcr.h
@@ -7,6 +7,7 @@
 #define _XE_GT_MCR_H_
 
 #include "regs/xe_reg_defs.h"
+#include "xe_gt_topology.h"
 
 struct drm_printer;
 struct xe_gt;
@@ -25,5 +26,18 @@ void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr mcr_reg,
 			       u32 value);
 
 void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p);
+void xe_gt_mcr_get_dss_steering(struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance);
+
+/*
+ * Loop over each DSS and determine the group and instance IDs that
+ * should be used to steer MCR accesses toward this DSS.
+ * @dss: DSS ID to obtain steering for
+ * @gt: GT structure
+ * @group: steering group ID, data type: u16
+ * @instance: steering instance ID, data type: u16
+ */
+#define for_each_dss_steering(dss, gt, group, instance) \
+	for_each_dss((dss), (gt)) \
+		for_each_if((xe_gt_mcr_get_dss_steering((gt), (dss), &(group), &(instance)), true))
 
 #endif /* _XE_GT_MCR_H_ */
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c
new file mode 100644
index 000000000000..791dcdd767e2
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#include <drm/drm_managed.h>
+
+#include "xe_gt_sriov_pf.h"
+#include "xe_gt_sriov_pf_helpers.h"
+
+/*
+ * VF's metadata is maintained in the flexible array where:
+ *   - entry [0] contains metadata for the PF (only if applicable),
+ *   - entries [1..n] contain metadata for VF1..VFn::
+ *
+ *       <--------------------------- 1 + total_vfs ----------->
+ *      +-------+-------+-------+-----------------------+-------+
+ *      |   0   |   1   |   2   |                       |   n   |
+ *      +-------+-------+-------+-----------------------+-------+
+ *      |  PF   |  VF1  |  VF2  |      ...     ...      |  VFn  |
+ *      +-------+-------+-------+-----------------------+-------+
+ */
+static int pf_alloc_metadata(struct xe_gt *gt)
+{
+	unsigned int num_vfs = xe_gt_sriov_pf_get_totalvfs(gt);
+
+	gt->sriov.pf.vfs = drmm_kcalloc(&gt_to_xe(gt)->drm, 1 + num_vfs,
+					sizeof(*gt->sriov.pf.vfs), GFP_KERNEL);
+	if (!gt->sriov.pf.vfs)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * xe_gt_sriov_pf_init_early - Prepare SR-IOV PF data structures on PF.
+ * @gt: the &xe_gt to initialize
+ *
+ * Early initialization of the PF data.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_init_early(struct xe_gt *gt)
+{
+	int err;
+
+	err = pf_alloc_metadata(gt);
+	if (err)
+		return err;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf.h
new file mode 100644
index 000000000000..05142ffc4319
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#ifndef _XE_GT_SRIOV_PF_H_
+#define _XE_GT_SRIOV_PF_H_
+
+struct xe_gt;
+
+#ifdef CONFIG_PCI_IOV
+int xe_gt_sriov_pf_init_early(struct xe_gt *gt);
+#else
+static inline int xe_gt_sriov_pf_init_early(struct xe_gt *gt)
+{
+	return 0;
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
new file mode 100644
index 000000000000..79116ad58620
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
@@ -0,0 +1,1977 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#include <linux/string_choices.h>
+#include <linux/wordpart.h>
+
+#include "abi/guc_actions_sriov_abi.h"
+#include "abi/guc_klvs_abi.h"
+
+#include "regs/xe_guc_regs.h"
+
+#include "xe_bo.h"
+#include "xe_device.h"
+#include "xe_ggtt.h"
+#include "xe_gt.h"
+#include "xe_gt_sriov_pf_config.h"
+#include "xe_gt_sriov_pf_helpers.h"
+#include "xe_gt_sriov_pf_policy.h"
+#include "xe_gt_sriov_printk.h"
+#include "xe_guc.h"
+#include "xe_guc_ct.h"
+#include "xe_guc_db_mgr.h"
+#include "xe_guc_fwif.h"
+#include "xe_guc_id_mgr.h"
+#include "xe_guc_klv_helpers.h"
+#include "xe_guc_submit.h"
+#include "xe_lmtt.h"
+#include "xe_map.h"
+#include "xe_sriov.h"
+#include "xe_ttm_vram_mgr.h"
+#include "xe_wopcm.h"
+
+/*
+ * Return: number of KLVs that were successfully parsed and saved,
+ *         negative error code on failure.
+ */
+static int guc_action_update_vf_cfg(struct xe_guc *guc, u32 vfid,
+				    u64 addr, u32 size)
+{
+	u32 request[] = {
+		GUC_ACTION_PF2GUC_UPDATE_VF_CFG,
+		vfid,
+		lower_32_bits(addr),
+		upper_32_bits(addr),
+		size,
+	};
+
+	return xe_guc_ct_send_block(&guc->ct, request, ARRAY_SIZE(request));
+}
+
+/*
+ * Return: 0 on success, negative error code on failure.
+ */
+static int pf_send_vf_cfg_reset(struct xe_gt *gt, u32 vfid)
+{
+	struct xe_guc *guc = &gt->uc.guc;
+	int ret;
+
+	ret = guc_action_update_vf_cfg(guc, vfid, 0, 0);
+
+	return ret <= 0 ? ret : -EPROTO;
+}
+
+/*
+ * Return: number of KLVs that were successfully parsed and saved,
+ *         negative error code on failure.
+ */
+static int pf_send_vf_cfg_klvs(struct xe_gt *gt, u32 vfid, const u32 *klvs, u32 num_dwords)
+{
+	const u32 bytes = num_dwords * sizeof(u32);
+	struct xe_tile *tile = gt_to_tile(gt);
+	struct xe_device *xe = tile_to_xe(tile);
+	struct xe_guc *guc = &gt->uc.guc;
+	struct xe_bo *bo;
+	int ret;
+
+	bo = xe_bo_create_pin_map(xe, tile, NULL,
+				  ALIGN(bytes, PAGE_SIZE),
+				  ttm_bo_type_kernel,
+				  XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+				  XE_BO_FLAG_GGTT |
+				  XE_BO_FLAG_GGTT_INVALIDATE);
+	if (IS_ERR(bo))
+		return PTR_ERR(bo);
+
+	xe_map_memcpy_to(xe, &bo->vmap, 0, klvs, bytes);
+
+	ret = guc_action_update_vf_cfg(guc, vfid, xe_bo_ggtt_addr(bo), num_dwords);
+
+	xe_bo_unpin_map_no_vm(bo);
+
+	return ret;
+}
+
+/*
+ * Return: 0 on success, -ENOKEY if some KLVs were not updated, -EPROTO if reply was malformed,
+ *         negative error code on failure.
+ */
+static int pf_push_vf_cfg_klvs(struct xe_gt *gt, unsigned int vfid, u32 num_klvs,
+			       const u32 *klvs, u32 num_dwords)
+{
+	int ret;
+
+	xe_gt_assert(gt, num_klvs == xe_guc_klv_count(klvs, num_dwords));
+
+	ret = pf_send_vf_cfg_klvs(gt, vfid, klvs, num_dwords);
+
+	if (ret != num_klvs) {
+		int err = ret < 0 ? ret : ret < num_klvs ? -ENOKEY : -EPROTO;
+		struct drm_printer p = xe_gt_info_printer(gt);
+		char name[8];
+
+		xe_gt_sriov_notice(gt, "Failed to push %s %u config KLV%s (%pe)\n",
+				   xe_sriov_function_name(vfid, name, sizeof(name)),
+				   num_klvs, str_plural(num_klvs), ERR_PTR(err));
+		xe_guc_klv_print(klvs, num_dwords, &p);
+		return err;
+	}
+
+	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) {
+		struct drm_printer p = xe_gt_info_printer(gt);
+
+		xe_guc_klv_print(klvs, num_dwords, &p);
+	}
+
+	return 0;
+}
+
+static int pf_push_vf_cfg_u32(struct xe_gt *gt, unsigned int vfid, u16 key, u32 value)
+{
+	u32 klv[] = {
+		FIELD_PREP(GUC_KLV_0_KEY, key) | FIELD_PREP(GUC_KLV_0_LEN, 1),
+		value,
+	};
+
+	return pf_push_vf_cfg_klvs(gt, vfid, 1, klv, ARRAY_SIZE(klv));
+}
+
+static int pf_push_vf_cfg_u64(struct xe_gt *gt, unsigned int vfid, u16 key, u64 value)
+{
+	u32 klv[] = {
+		FIELD_PREP(GUC_KLV_0_KEY, key) | FIELD_PREP(GUC_KLV_0_LEN, 2),
+		lower_32_bits(value),
+		upper_32_bits(value),
+	};
+
+	return pf_push_vf_cfg_klvs(gt, vfid, 1, klv, ARRAY_SIZE(klv));
+}
+
+static int pf_push_vf_cfg_ggtt(struct xe_gt *gt, unsigned int vfid, u64 start, u64 size)
+{
+	u32 klvs[] = {
+		PREP_GUC_KLV_TAG(VF_CFG_GGTT_START),
+		lower_32_bits(start),
+		upper_32_bits(start),
+		PREP_GUC_KLV_TAG(VF_CFG_GGTT_SIZE),
+		lower_32_bits(size),
+		upper_32_bits(size),
+	};
+
+	return pf_push_vf_cfg_klvs(gt, vfid, 2, klvs, ARRAY_SIZE(klvs));
+}
+
+static int pf_push_vf_cfg_ctxs(struct xe_gt *gt, unsigned int vfid, u32 begin, u32 num)
+{
+	u32 klvs[] = {
+		PREP_GUC_KLV_TAG(VF_CFG_BEGIN_CONTEXT_ID),
+		begin,
+		PREP_GUC_KLV_TAG(VF_CFG_NUM_CONTEXTS),
+		num,
+	};
+
+	return pf_push_vf_cfg_klvs(gt, vfid, 2, klvs, ARRAY_SIZE(klvs));
+}
+
+static int pf_push_vf_cfg_dbs(struct xe_gt *gt, unsigned int vfid, u32 begin, u32 num)
+{
+	u32 klvs[] = {
+		PREP_GUC_KLV_TAG(VF_CFG_BEGIN_DOORBELL_ID),
+		begin,
+		PREP_GUC_KLV_TAG(VF_CFG_NUM_DOORBELLS),
+		num,
+	};
+
+	return pf_push_vf_cfg_klvs(gt, vfid, 2, klvs, ARRAY_SIZE(klvs));
+}
+
+static int pf_push_vf_cfg_exec_quantum(struct xe_gt *gt, unsigned int vfid, u32 exec_quantum)
+{
+	return pf_push_vf_cfg_u32(gt, vfid, GUC_KLV_VF_CFG_EXEC_QUANTUM_KEY, exec_quantum);
+}
+
+static int pf_push_vf_cfg_preempt_timeout(struct xe_gt *gt, unsigned int vfid, u32 preempt_timeout)
+{
+	return pf_push_vf_cfg_u32(gt, vfid, GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_KEY, preempt_timeout);
+}
+
+static int pf_push_vf_cfg_lmem(struct xe_gt *gt, unsigned int vfid, u64 size)
+{
+	return pf_push_vf_cfg_u64(gt, vfid, GUC_KLV_VF_CFG_LMEM_SIZE_KEY, size);
+}
+
+static struct xe_gt_sriov_config *pf_pick_vf_config(struct xe_gt *gt, unsigned int vfid)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	return &gt->sriov.pf.vfs[vfid].config;
+}
+
+/* Return: number of configuration dwords written */
+static u32 encode_config_ggtt(u32 *cfg, const struct xe_gt_sriov_config *config)
+{
+	u32 n = 0;
+
+	if (drm_mm_node_allocated(&config->ggtt_region)) {
+		cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_GGTT_START);
+		cfg[n++] = lower_32_bits(config->ggtt_region.start);
+		cfg[n++] = upper_32_bits(config->ggtt_region.start);
+
+		cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_GGTT_SIZE);
+		cfg[n++] = lower_32_bits(config->ggtt_region.size);
+		cfg[n++] = upper_32_bits(config->ggtt_region.size);
+	}
+
+	return n;
+}
+
+/* Return: number of configuration dwords written */
+static u32 encode_config(u32 *cfg, const struct xe_gt_sriov_config *config)
+{
+	u32 n = 0;
+
+	n += encode_config_ggtt(cfg, config);
+
+	cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_BEGIN_CONTEXT_ID);
+	cfg[n++] = config->begin_ctx;
+
+	cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_NUM_CONTEXTS);
+	cfg[n++] = config->num_ctxs;
+
+	cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_BEGIN_DOORBELL_ID);
+	cfg[n++] = config->begin_db;
+
+	cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_NUM_DOORBELLS);
+	cfg[n++] = config->num_dbs;
+
+	if (config->lmem_obj) {
+		cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_LMEM_SIZE);
+		cfg[n++] = lower_32_bits(config->lmem_obj->size);
+		cfg[n++] = upper_32_bits(config->lmem_obj->size);
+	}
+
+	cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_EXEC_QUANTUM);
+	cfg[n++] = config->exec_quantum;
+
+	cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_PREEMPT_TIMEOUT);
+	cfg[n++] = config->preempt_timeout;
+
+	return n;
+}
+
+static int pf_push_full_vf_config(struct xe_gt *gt, unsigned int vfid)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+	u32 max_cfg_dwords = SZ_4K / sizeof(u32);
+	u32 num_dwords;
+	int num_klvs;
+	u32 *cfg;
+	int err;
+
+	cfg = kcalloc(max_cfg_dwords, sizeof(u32), GFP_KERNEL);
+	if (!cfg)
+		return -ENOMEM;
+
+	num_dwords = encode_config(cfg, config);
+	xe_gt_assert(gt, num_dwords <= max_cfg_dwords);
+
+	if (xe_gt_is_media_type(gt)) {
+		struct xe_gt *primary = gt->tile->primary_gt;
+		struct xe_gt_sriov_config *other = pf_pick_vf_config(primary, vfid);
+
+		/* media-GT will never include a GGTT config */
+		xe_gt_assert(gt, !encode_config_ggtt(cfg + num_dwords, config));
+
+		/* the GGTT config must be taken from the primary-GT instead */
+		num_dwords += encode_config_ggtt(cfg + num_dwords, other);
+	}
+	xe_gt_assert(gt, num_dwords <= max_cfg_dwords);
+
+	num_klvs = xe_guc_klv_count(cfg, num_dwords);
+	err = pf_push_vf_cfg_klvs(gt, vfid, num_klvs, cfg, num_dwords);
+
+	kfree(cfg);
+	return err;
+}
+
+static u64 pf_get_ggtt_alignment(struct xe_gt *gt)
+{
+	struct xe_device *xe = gt_to_xe(gt);
+
+	return IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K;
+}
+
+static u64 pf_get_min_spare_ggtt(struct xe_gt *gt)
+{
+	/* XXX: preliminary */
+	return IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV) ?
+		pf_get_ggtt_alignment(gt) : SZ_64M;
+}
+
+static u64 pf_get_spare_ggtt(struct xe_gt *gt)
+{
+	u64 spare;
+
+	xe_gt_assert(gt, !xe_gt_is_media_type(gt));
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	spare = gt->sriov.pf.spare.ggtt_size;
+	spare = max_t(u64, spare, pf_get_min_spare_ggtt(gt));
+
+	return spare;
+}
+
+static int pf_set_spare_ggtt(struct xe_gt *gt, u64 size)
+{
+	xe_gt_assert(gt, !xe_gt_is_media_type(gt));
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	if (size && size < pf_get_min_spare_ggtt(gt))
+		return -EINVAL;
+
+	size = round_up(size, pf_get_ggtt_alignment(gt));
+	gt->sriov.pf.spare.ggtt_size = size;
+
+	return 0;
+}
+
+static int pf_distribute_config_ggtt(struct xe_tile *tile, unsigned int vfid, u64 start, u64 size)
+{
+	int err, err2 = 0;
+
+	err = pf_push_vf_cfg_ggtt(tile->primary_gt, vfid, start, size);
+
+	if (tile->media_gt && !err)
+		err2 = pf_push_vf_cfg_ggtt(tile->media_gt, vfid, start, size);
+
+	return err ?: err2;
+}
+
+static void pf_release_ggtt(struct xe_tile *tile, struct drm_mm_node *node)
+{
+	struct xe_ggtt *ggtt = tile->mem.ggtt;
+
+	if (drm_mm_node_allocated(node)) {
+		/*
+		 * explicit GGTT PTE assignment to the PF using xe_ggtt_assign()
+		 * is redundant, as PTE will be implicitly re-assigned to PF by
+		 * the xe_ggtt_clear() called by below xe_ggtt_remove_node().
+		 */
+		xe_ggtt_remove_node(ggtt, node, false);
+	}
+}
+
+static void pf_release_vf_config_ggtt(struct xe_gt *gt, struct xe_gt_sriov_config *config)
+{
+	pf_release_ggtt(gt_to_tile(gt), &config->ggtt_region);
+}
+
+static int pf_provision_vf_ggtt(struct xe_gt *gt, unsigned int vfid, u64 size)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+	struct drm_mm_node *node = &config->ggtt_region;
+	struct xe_tile *tile = gt_to_tile(gt);
+	struct xe_ggtt *ggtt = tile->mem.ggtt;
+	u64 alignment = pf_get_ggtt_alignment(gt);
+	int err;
+
+	xe_gt_assert(gt, vfid);
+	xe_gt_assert(gt, !xe_gt_is_media_type(gt));
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+
+	size = round_up(size, alignment);
+
+	if (drm_mm_node_allocated(node)) {
+		err = pf_distribute_config_ggtt(tile, vfid, 0, 0);
+		if (unlikely(err))
+			return err;
+
+		pf_release_ggtt(tile, node);
+	}
+	xe_gt_assert(gt, !drm_mm_node_allocated(node));
+
+	if (!size)
+		return 0;
+
+	err = xe_ggtt_insert_special_node(ggtt, node, size, alignment);
+	if (unlikely(err))
+		return err;
+
+	xe_ggtt_assign(ggtt, node, vfid);
+	xe_gt_sriov_dbg_verbose(gt, "VF%u assigned GGTT %llx-%llx\n",
+				vfid, node->start, node->start + node->size - 1);
+
+	err = pf_distribute_config_ggtt(gt->tile, vfid, node->start, node->size);
+	if (unlikely(err))
+		return err;
+
+	return 0;
+}
+
+static u64 pf_get_vf_config_ggtt(struct xe_gt *gt, unsigned int vfid)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+	struct drm_mm_node *node = &config->ggtt_region;
+
+	xe_gt_assert(gt, !xe_gt_is_media_type(gt));
+	return drm_mm_node_allocated(node) ? node->size : 0;
+}
+
+/**
+ * xe_gt_sriov_pf_config_get_ggtt - Query size of GGTT address space of the VF.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ *
+ * This function can only be called on PF.
+ *
+ * Return: size of the VF's assigned (or PF's spare) GGTT address space.
+ */
+u64 xe_gt_sriov_pf_config_get_ggtt(struct xe_gt *gt, unsigned int vfid)
+{
+	u64 size;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	if (vfid)
+		size = pf_get_vf_config_ggtt(gt_to_tile(gt)->primary_gt, vfid);
+	else
+		size = pf_get_spare_ggtt(gt_to_tile(gt)->primary_gt);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return size;
+}
+
+static int pf_config_set_u64_done(struct xe_gt *gt, unsigned int vfid, u64 value,
+				  u64 actual, const char *what, int err)
+{
+	char size[10];
+	char name[8];
+
+	xe_sriov_function_name(vfid, name, sizeof(name));
+
+	if (unlikely(err)) {
+		string_get_size(value, 1, STRING_UNITS_2, size, sizeof(size));
+		xe_gt_sriov_notice(gt, "Failed to provision %s with %llu (%s) %s (%pe)\n",
+				   name, value, size, what, ERR_PTR(err));
+		string_get_size(actual, 1, STRING_UNITS_2, size, sizeof(size));
+		xe_gt_sriov_info(gt, "%s provisioning remains at %llu (%s) %s\n",
+				 name, actual, size, what);
+		return err;
+	}
+
+	/* the actual value may have changed during provisioning */
+	string_get_size(actual, 1, STRING_UNITS_2, size, sizeof(size));
+	xe_gt_sriov_info(gt, "%s provisioned with %llu (%s) %s\n",
+			 name, actual, size, what);
+	return 0;
+}
+
+/**
+ * xe_gt_sriov_pf_config_set_ggtt - Provision VF with GGTT space.
+ * @gt: the &xe_gt (can't be media)
+ * @vfid: the VF identifier
+ * @size: requested GGTT size
+ *
+ * If &vfid represents PF, then function will change PF's spare GGTT config.
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_ggtt(struct xe_gt *gt, unsigned int vfid, u64 size)
+{
+	int err;
+
+	xe_gt_assert(gt, !xe_gt_is_media_type(gt));
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	if (vfid)
+		err = pf_provision_vf_ggtt(gt, vfid, size);
+	else
+		err = pf_set_spare_ggtt(gt, size);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_config_set_u64_done(gt, vfid, size,
+				      xe_gt_sriov_pf_config_get_ggtt(gt, vfid),
+				      vfid ? "GGTT" : "spare GGTT", err);
+}
+
+static int pf_config_bulk_set_u64_done(struct xe_gt *gt, unsigned int first, unsigned int num_vfs,
+				       u64 value, u64 (*get)(struct xe_gt*, unsigned int),
+				       const char *what, unsigned int last, int err)
+{
+	char size[10];
+
+	xe_gt_assert(gt, first);
+	xe_gt_assert(gt, num_vfs);
+	xe_gt_assert(gt, first <= last);
+
+	if (num_vfs == 1)
+		return pf_config_set_u64_done(gt, first, value, get(gt, first), what, err);
+
+	if (unlikely(err)) {
+		xe_gt_sriov_notice(gt, "Failed to bulk provision VF%u..VF%u with %s\n",
+				   first, first + num_vfs - 1, what);
+		if (last > first)
+			pf_config_bulk_set_u64_done(gt, first, last - first, value,
+						    get, what, last, 0);
+		return pf_config_set_u64_done(gt, last, value, get(gt, last), what, err);
+	}
+
+	/* pick actual value from first VF - bulk provisioning shall be equal across all VFs */
+	value = get(gt, first);
+	string_get_size(value, 1, STRING_UNITS_2, size, sizeof(size));
+	xe_gt_sriov_info(gt, "VF%u..VF%u provisioned with %llu (%s) %s\n",
+			 first, first + num_vfs - 1, value, size, what);
+	return 0;
+}
+
+/**
+ * xe_gt_sriov_pf_config_bulk_set_ggtt - Provision many VFs with GGTT.
+ * @gt: the &xe_gt (can't be media)
+ * @vfid: starting VF identifier (can't be 0)
+ * @num_vfs: number of VFs to provision
+ * @size: requested GGTT size
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_bulk_set_ggtt(struct xe_gt *gt, unsigned int vfid,
+					unsigned int num_vfs, u64 size)
+{
+	unsigned int n;
+	int err = 0;
+
+	xe_gt_assert(gt, vfid);
+	xe_gt_assert(gt, !xe_gt_is_media_type(gt));
+
+	if (!num_vfs)
+		return 0;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	for (n = vfid; n < vfid + num_vfs; n++) {
+		err = pf_provision_vf_ggtt(gt, n, size);
+		if (err)
+			break;
+	}
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_config_bulk_set_u64_done(gt, vfid, num_vfs, size,
+					   xe_gt_sriov_pf_config_get_ggtt,
+					   "GGTT", n, err);
+}
+
+/* Return: size of the largest continuous GGTT region */
+static u64 pf_get_max_ggtt(struct xe_gt *gt)
+{
+	struct xe_ggtt *ggtt = gt_to_tile(gt)->mem.ggtt;
+	const struct drm_mm *mm = &ggtt->mm;
+	const struct drm_mm_node *entry;
+	u64 alignment = pf_get_ggtt_alignment(gt);
+	u64 spare = pf_get_spare_ggtt(gt);
+	u64 hole_min_start = xe_wopcm_size(gt_to_xe(gt));
+	u64 hole_start, hole_end, hole_size;
+	u64 max_hole = 0;
+
+	mutex_lock(&ggtt->lock);
+
+	drm_mm_for_each_hole(entry, mm, hole_start, hole_end) {
+		hole_start = max(hole_start, hole_min_start);
+		hole_start = ALIGN(hole_start, alignment);
+		hole_end = ALIGN_DOWN(hole_end, alignment);
+		if (hole_start >= hole_end)
+			continue;
+		hole_size = hole_end - hole_start;
+		xe_gt_sriov_dbg_verbose(gt, "HOLE start %llx size %lluK\n",
+					hole_start, hole_size / SZ_1K);
+		spare -= min3(spare, hole_size, max_hole);
+		max_hole = max(max_hole, hole_size);
+	}
+
+	mutex_unlock(&ggtt->lock);
+
+	xe_gt_sriov_dbg_verbose(gt, "HOLE max %lluK reserved %lluK\n",
+				max_hole / SZ_1K, spare / SZ_1K);
+	return max_hole > spare ? max_hole - spare : 0;
+}
+
+static u64 pf_estimate_fair_ggtt(struct xe_gt *gt, unsigned int num_vfs)
+{
+	u64 available = pf_get_max_ggtt(gt);
+	u64 alignment = pf_get_ggtt_alignment(gt);
+	u64 fair;
+
+	/*
+	 * To simplify the logic we only look at single largest GGTT region
+	 * as that will be always the best fit for 1 VF case, and most likely
+	 * will also nicely cover other cases where VFs are provisioned on the
+	 * fresh and idle PF driver, without any stale GGTT allocations spread
+	 * in the middle of the full GGTT range.
+	 */
+
+	fair = div_u64(available, num_vfs);
+	fair = ALIGN_DOWN(fair, alignment);
+	xe_gt_sriov_dbg_verbose(gt, "GGTT available(%lluK) fair(%u x %lluK)\n",
+				available / SZ_1K, num_vfs, fair / SZ_1K);
+	return fair;
+}
+
+/**
+ * xe_gt_sriov_pf_config_set_fair_ggtt - Provision many VFs with fair GGTT.
+ * @gt: the &xe_gt (can't be media)
+ * @vfid: starting VF identifier (can't be 0)
+ * @num_vfs: number of VFs to provision
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_fair_ggtt(struct xe_gt *gt, unsigned int vfid,
+					unsigned int num_vfs)
+{
+	u64 fair;
+
+	xe_gt_assert(gt, vfid);
+	xe_gt_assert(gt, num_vfs);
+	xe_gt_assert(gt, !xe_gt_is_media_type(gt));
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	fair = pf_estimate_fair_ggtt(gt, num_vfs);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	if (!fair)
+		return -ENOSPC;
+
+	return xe_gt_sriov_pf_config_bulk_set_ggtt(gt, vfid, num_vfs, fair);
+}
+
+static u32 pf_get_min_spare_ctxs(struct xe_gt *gt)
+{
+	/* XXX: preliminary */
+	return IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV) ?
+		hweight64(gt->info.engine_mask) : SZ_256;
+}
+
+static u32 pf_get_spare_ctxs(struct xe_gt *gt)
+{
+	u32 spare;
+
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	spare = gt->sriov.pf.spare.num_ctxs;
+	spare = max_t(u32, spare, pf_get_min_spare_ctxs(gt));
+
+	return spare;
+}
+
+static int pf_set_spare_ctxs(struct xe_gt *gt, u32 spare)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	if (spare > GUC_ID_MAX)
+		return -EINVAL;
+
+	if (spare && spare < pf_get_min_spare_ctxs(gt))
+		return -EINVAL;
+
+	gt->sriov.pf.spare.num_ctxs = spare;
+
+	return 0;
+}
+
+/* Return: start ID or negative error code on failure */
+static int pf_reserve_ctxs(struct xe_gt *gt, u32 num)
+{
+	struct xe_guc_id_mgr *idm = &gt->uc.guc.submission_state.idm;
+	unsigned int spare = pf_get_spare_ctxs(gt);
+
+	return xe_guc_id_mgr_reserve(idm, num, spare);
+}
+
+static void pf_release_ctxs(struct xe_gt *gt, u32 start, u32 num)
+{
+	struct xe_guc_id_mgr *idm = &gt->uc.guc.submission_state.idm;
+
+	if (num)
+		xe_guc_id_mgr_release(idm, start, num);
+}
+
+static void pf_release_config_ctxs(struct xe_gt *gt, struct xe_gt_sriov_config *config)
+{
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	pf_release_ctxs(gt, config->begin_ctx, config->num_ctxs);
+	config->begin_ctx = 0;
+	config->num_ctxs = 0;
+}
+
+static int pf_provision_vf_ctxs(struct xe_gt *gt, unsigned int vfid, u32 num_ctxs)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+	int ret;
+
+	xe_gt_assert(gt, vfid);
+
+	if (num_ctxs > GUC_ID_MAX)
+		return -EINVAL;
+
+	if (config->num_ctxs) {
+		ret = pf_push_vf_cfg_ctxs(gt, vfid, 0, 0);
+		if (unlikely(ret))
+			return ret;
+
+		pf_release_config_ctxs(gt, config);
+	}
+
+	if (!num_ctxs)
+		return 0;
+
+	ret = pf_reserve_ctxs(gt, num_ctxs);
+	if (unlikely(ret < 0))
+		return ret;
+
+	config->begin_ctx = ret;
+	config->num_ctxs = num_ctxs;
+
+	ret = pf_push_vf_cfg_ctxs(gt, vfid, config->begin_ctx, config->num_ctxs);
+	if (unlikely(ret)) {
+		pf_release_config_ctxs(gt, config);
+		return ret;
+	}
+
+	xe_gt_sriov_dbg_verbose(gt, "VF%u contexts %u-%u\n",
+				vfid, config->begin_ctx, config->begin_ctx + config->num_ctxs - 1);
+	return 0;
+}
+
+static u32 pf_get_vf_config_ctxs(struct xe_gt *gt, unsigned int vfid)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+
+	return config->num_ctxs;
+}
+
+/**
+ * xe_gt_sriov_pf_config_get_ctxs - Get VF's GuC contexts IDs quota.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ *
+ * This function can only be called on PF.
+ * If &vfid represents a PF then number of PF's spare GuC context IDs is returned.
+ *
+ * Return: VF's quota (or PF's spare).
+ */
+u32 xe_gt_sriov_pf_config_get_ctxs(struct xe_gt *gt, unsigned int vfid)
+{
+	u32 num_ctxs;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	if (vfid)
+		num_ctxs = pf_get_vf_config_ctxs(gt, vfid);
+	else
+		num_ctxs = pf_get_spare_ctxs(gt);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return num_ctxs;
+}
+
+static const char *no_unit(u32 unused)
+{
+	return "";
+}
+
+static const char *spare_unit(u32 unused)
+{
+	return " spare";
+}
+
+static int pf_config_set_u32_done(struct xe_gt *gt, unsigned int vfid, u32 value, u32 actual,
+				  const char *what, const char *(*unit)(u32), int err)
+{
+	char name[8];
+
+	xe_sriov_function_name(vfid, name, sizeof(name));
+
+	if (unlikely(err)) {
+		xe_gt_sriov_notice(gt, "Failed to provision %s with %u%s %s (%pe)\n",
+				   name, value, unit(value), what, ERR_PTR(err));
+		xe_gt_sriov_info(gt, "%s provisioning remains at %u%s %s\n",
+				 name, actual, unit(actual), what);
+		return err;
+	}
+
+	/* the actual value may have changed during provisioning */
+	xe_gt_sriov_info(gt, "%s provisioned with %u%s %s\n",
+			 name, actual, unit(actual), what);
+	return 0;
+}
+
+/**
+ * xe_gt_sriov_pf_config_set_ctxs - Configure GuC contexts IDs quota for the VF.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ * @num_ctxs: requested number of GuC contexts IDs (0 to release)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_ctxs(struct xe_gt *gt, unsigned int vfid, u32 num_ctxs)
+{
+	int err;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	if (vfid)
+		err = pf_provision_vf_ctxs(gt, vfid, num_ctxs);
+	else
+		err = pf_set_spare_ctxs(gt, num_ctxs);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_config_set_u32_done(gt, vfid, num_ctxs,
+				      xe_gt_sriov_pf_config_get_ctxs(gt, vfid),
+				      "GuC context IDs", vfid ? no_unit : spare_unit, err);
+}
+
+static int pf_config_bulk_set_u32_done(struct xe_gt *gt, unsigned int first, unsigned int num_vfs,
+				       u32 value, u32 (*get)(struct xe_gt*, unsigned int),
+				       const char *what, const char *(*unit)(u32),
+				       unsigned int last, int err)
+{
+	xe_gt_assert(gt, first);
+	xe_gt_assert(gt, num_vfs);
+	xe_gt_assert(gt, first <= last);
+
+	if (num_vfs == 1)
+		return pf_config_set_u32_done(gt, first, value, get(gt, first), what, unit, err);
+
+	if (unlikely(err)) {
+		xe_gt_sriov_notice(gt, "Failed to bulk provision VF%u..VF%u with %s\n",
+				   first, first + num_vfs - 1, what);
+		if (last > first)
+			pf_config_bulk_set_u32_done(gt, first, last - first, value,
+						    get, what, unit, last, 0);
+		return pf_config_set_u32_done(gt, last, value, get(gt, last), what, unit, err);
+	}
+
+	/* pick actual value from first VF - bulk provisioning shall be equal across all VFs */
+	value = get(gt, first);
+	xe_gt_sriov_info(gt, "VF%u..VF%u provisioned with %u%s %s\n",
+			 first, first + num_vfs - 1, value, unit(value), what);
+	return 0;
+}
+
+/**
+ * xe_gt_sriov_pf_config_bulk_set_ctxs - Provision many VFs with GuC context IDs.
+ * @gt: the &xe_gt
+ * @vfid: starting VF identifier
+ * @num_vfs: number of VFs to provision
+ * @num_ctxs: requested number of GuC contexts IDs (0 to release)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_bulk_set_ctxs(struct xe_gt *gt, unsigned int vfid,
+					unsigned int num_vfs, u32 num_ctxs)
+{
+	unsigned int n;
+	int err = 0;
+
+	xe_gt_assert(gt, vfid);
+
+	if (!num_vfs)
+		return 0;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	for (n = vfid; n < vfid + num_vfs; n++) {
+		err = pf_provision_vf_ctxs(gt, n, num_ctxs);
+		if (err)
+			break;
+	}
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_config_bulk_set_u32_done(gt, vfid, num_vfs, num_ctxs,
+					   xe_gt_sriov_pf_config_get_ctxs,
+					   "GuC context IDs", no_unit, n, err);
+}
+
+static u32 pf_estimate_fair_ctxs(struct xe_gt *gt, unsigned int num_vfs)
+{
+	struct xe_guc_id_mgr *idm = &gt->uc.guc.submission_state.idm;
+	u32 spare = pf_get_spare_ctxs(gt);
+	u32 fair = (idm->total - spare) / num_vfs;
+	int ret;
+
+	for (; fair; --fair) {
+		ret = xe_guc_id_mgr_reserve(idm, fair * num_vfs, spare);
+		if (ret < 0)
+			continue;
+		xe_guc_id_mgr_release(idm, ret, fair * num_vfs);
+		break;
+	}
+
+	xe_gt_sriov_dbg_verbose(gt, "contexts fair(%u x %u)\n", num_vfs, fair);
+	return fair;
+}
+
+/**
+ * xe_gt_sriov_pf_config_set_fair_ctxs - Provision many VFs with fair GuC context IDs.
+ * @gt: the &xe_gt
+ * @vfid: starting VF identifier (can't be 0)
+ * @num_vfs: number of VFs to provision (can't be 0)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_fair_ctxs(struct xe_gt *gt, unsigned int vfid,
+					unsigned int num_vfs)
+{
+	u32 fair;
+
+	xe_gt_assert(gt, vfid);
+	xe_gt_assert(gt, num_vfs);
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	fair = pf_estimate_fair_ctxs(gt, num_vfs);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	if (!fair)
+		return -ENOSPC;
+
+	return xe_gt_sriov_pf_config_bulk_set_ctxs(gt, vfid, num_vfs, fair);
+}
+
+static u32 pf_get_min_spare_dbs(struct xe_gt *gt)
+{
+	/* XXX: preliminary, we don't use doorbells yet! */
+	return IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV) ? 1 : 0;
+}
+
+static u32 pf_get_spare_dbs(struct xe_gt *gt)
+{
+	u32 spare;
+
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	spare = gt->sriov.pf.spare.num_dbs;
+	spare = max_t(u32, spare, pf_get_min_spare_dbs(gt));
+
+	return spare;
+}
+
+static int pf_set_spare_dbs(struct xe_gt *gt, u32 spare)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	if (spare > GUC_NUM_DOORBELLS)
+		return -EINVAL;
+
+	if (spare && spare < pf_get_min_spare_dbs(gt))
+		return -EINVAL;
+
+	gt->sriov.pf.spare.num_dbs = spare;
+	return 0;
+}
+
+/* Return: start ID or negative error code on failure */
+static int pf_reserve_dbs(struct xe_gt *gt, u32 num)
+{
+	struct xe_guc_db_mgr *dbm = &gt->uc.guc.dbm;
+	unsigned int spare = pf_get_spare_dbs(gt);
+
+	return xe_guc_db_mgr_reserve_range(dbm, num, spare);
+}
+
+static void pf_release_dbs(struct xe_gt *gt, u32 start, u32 num)
+{
+	struct xe_guc_db_mgr *dbm = &gt->uc.guc.dbm;
+
+	if (num)
+		xe_guc_db_mgr_release_range(dbm, start, num);
+}
+
+static void pf_release_config_dbs(struct xe_gt *gt, struct xe_gt_sriov_config *config)
+{
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	pf_release_dbs(gt, config->begin_db, config->num_dbs);
+	config->begin_db = 0;
+	config->num_dbs = 0;
+}
+
+static int pf_provision_vf_dbs(struct xe_gt *gt, unsigned int vfid, u32 num_dbs)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+	int ret;
+
+	xe_gt_assert(gt, vfid);
+
+	if (num_dbs > GUC_NUM_DOORBELLS)
+		return -EINVAL;
+
+	if (config->num_dbs) {
+		ret = pf_push_vf_cfg_dbs(gt, vfid, 0, 0);
+		if (unlikely(ret))
+			return ret;
+
+		pf_release_config_dbs(gt, config);
+	}
+
+	if (!num_dbs)
+		return 0;
+
+	ret = pf_reserve_dbs(gt, num_dbs);
+	if (unlikely(ret < 0))
+		return ret;
+
+	config->begin_db = ret;
+	config->num_dbs = num_dbs;
+
+	ret = pf_push_vf_cfg_dbs(gt, vfid, config->begin_db, config->num_dbs);
+	if (unlikely(ret)) {
+		pf_release_config_dbs(gt, config);
+		return ret;
+	}
+
+	xe_gt_sriov_dbg_verbose(gt, "VF%u doorbells %u-%u\n",
+				vfid, config->begin_db, config->begin_db + config->num_dbs - 1);
+	return 0;
+}
+
+static u32 pf_get_vf_config_dbs(struct xe_gt *gt, unsigned int vfid)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+
+	return config->num_dbs;
+}
+
+/**
+ * xe_gt_sriov_pf_config_get_dbs - Get VF's GuC doorbells IDs quota.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ *
+ * This function can only be called on PF.
+ * If &vfid represents a PF then number of PF's spare GuC doorbells IDs is returned.
+ *
+ * Return: VF's quota (or PF's spare).
+ */
+u32 xe_gt_sriov_pf_config_get_dbs(struct xe_gt *gt, unsigned int vfid)
+{
+	u32 num_dbs;
+
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt)));
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	if (vfid)
+		num_dbs = pf_get_vf_config_dbs(gt, vfid);
+	else
+		num_dbs = pf_get_spare_dbs(gt);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return num_dbs;
+}
+
+/**
+ * xe_gt_sriov_pf_config_set_dbs - Configure GuC doorbells IDs quota for the VF.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ * @num_dbs: requested number of GuC doorbells IDs (0 to release)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_dbs(struct xe_gt *gt, unsigned int vfid, u32 num_dbs)
+{
+	int err;
+
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt)));
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	if (vfid)
+		err = pf_provision_vf_dbs(gt, vfid, num_dbs);
+	else
+		err = pf_set_spare_dbs(gt, num_dbs);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_config_set_u32_done(gt, vfid, num_dbs,
+				      xe_gt_sriov_pf_config_get_dbs(gt, vfid),
+				      "GuC doorbell IDs", vfid ? no_unit : spare_unit, err);
+}
+
+/**
+ * xe_gt_sriov_pf_config_bulk_set_dbs - Provision many VFs with GuC context IDs.
+ * @gt: the &xe_gt
+ * @vfid: starting VF identifier (can't be 0)
+ * @num_vfs: number of VFs to provision
+ * @num_dbs: requested number of GuC doorbell IDs (0 to release)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_bulk_set_dbs(struct xe_gt *gt, unsigned int vfid,
+				       unsigned int num_vfs, u32 num_dbs)
+{
+	unsigned int n;
+	int err = 0;
+
+	xe_gt_assert(gt, vfid);
+
+	if (!num_vfs)
+		return 0;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	for (n = vfid; n < vfid + num_vfs; n++) {
+		err = pf_provision_vf_dbs(gt, n, num_dbs);
+		if (err)
+			break;
+	}
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_config_bulk_set_u32_done(gt, vfid, num_vfs, num_dbs,
+					   xe_gt_sriov_pf_config_get_dbs,
+					   "GuC doorbell IDs", no_unit, n, err);
+}
+
+static u32 pf_estimate_fair_dbs(struct xe_gt *gt, unsigned int num_vfs)
+{
+	struct xe_guc_db_mgr *dbm = &gt->uc.guc.dbm;
+	u32 spare = pf_get_spare_dbs(gt);
+	u32 fair = (GUC_NUM_DOORBELLS - spare) / num_vfs;
+	int ret;
+
+	for (; fair; --fair) {
+		ret = xe_guc_db_mgr_reserve_range(dbm, fair * num_vfs, spare);
+		if (ret < 0)
+			continue;
+		xe_guc_db_mgr_release_range(dbm, ret, fair * num_vfs);
+		break;
+	}
+
+	xe_gt_sriov_dbg_verbose(gt, "doorbells fair(%u x %u)\n", num_vfs, fair);
+	return fair;
+}
+
+/**
+ * xe_gt_sriov_pf_config_set_fair_dbs - Provision many VFs with fair GuC doorbell  IDs.
+ * @gt: the &xe_gt
+ * @vfid: starting VF identifier (can't be 0)
+ * @num_vfs: number of VFs to provision (can't be 0)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_fair_dbs(struct xe_gt *gt, unsigned int vfid,
+				       unsigned int num_vfs)
+{
+	u32 fair;
+
+	xe_gt_assert(gt, vfid);
+	xe_gt_assert(gt, num_vfs);
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	fair = pf_estimate_fair_dbs(gt, num_vfs);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	if (!fair)
+		return -ENOSPC;
+
+	return xe_gt_sriov_pf_config_bulk_set_dbs(gt, vfid, num_vfs, fair);
+}
+
+static u64 pf_get_lmem_alignment(struct xe_gt *gt)
+{
+	/* this might be platform dependent */
+	return SZ_2M;
+}
+
+static u64 pf_get_min_spare_lmem(struct xe_gt *gt)
+{
+	/* this might be platform dependent */
+	return SZ_128M; /* XXX: preliminary */
+}
+
+static u64 pf_get_spare_lmem(struct xe_gt *gt)
+{
+	u64 spare;
+
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	spare = gt->sriov.pf.spare.lmem_size;
+	spare = max_t(u64, spare, pf_get_min_spare_lmem(gt));
+
+	return spare;
+}
+
+static int pf_set_spare_lmem(struct xe_gt *gt, u64 size)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	if (size && size < pf_get_min_spare_lmem(gt))
+		return -EINVAL;
+
+	gt->sriov.pf.spare.lmem_size = size;
+	return 0;
+}
+
+static u64 pf_get_vf_config_lmem(struct xe_gt *gt, unsigned int vfid)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+	struct xe_bo *bo;
+
+	bo = config->lmem_obj;
+	return bo ? bo->size : 0;
+}
+
+static int pf_distribute_config_lmem(struct xe_gt *gt, unsigned int vfid, u64 size)
+{
+	struct xe_device *xe = gt_to_xe(gt);
+	struct xe_tile *tile;
+	unsigned int tid;
+	int err;
+
+	for_each_tile(tile, xe, tid) {
+		if (tile->primary_gt == gt) {
+			err = pf_push_vf_cfg_lmem(gt, vfid, size);
+		} else {
+			u64 lmem = pf_get_vf_config_lmem(tile->primary_gt, vfid);
+
+			if (!lmem)
+				continue;
+			err = pf_push_vf_cfg_lmem(gt, vfid, lmem);
+		}
+		if (unlikely(err))
+			return err;
+	}
+	return 0;
+}
+
+static void pf_force_lmtt_invalidate(struct xe_device *xe)
+{
+	/* TODO */
+}
+
+static void pf_reset_vf_lmtt(struct xe_device *xe, unsigned int vfid)
+{
+	struct xe_lmtt *lmtt;
+	struct xe_tile *tile;
+	unsigned int tid;
+
+	for_each_tile(tile, xe, tid) {
+		lmtt = &tile->sriov.pf.lmtt;
+		xe_lmtt_drop_pages(lmtt, vfid);
+	}
+}
+
+static int pf_update_vf_lmtt(struct xe_device *xe, unsigned int vfid)
+{
+	struct xe_gt_sriov_config *config;
+	struct xe_tile *tile;
+	struct xe_lmtt *lmtt;
+	struct xe_bo *bo;
+	struct xe_gt *gt;
+	u64 total, offset;
+	unsigned int gtid;
+	unsigned int tid;
+	int err;
+
+	total = 0;
+	for_each_tile(tile, xe, tid)
+		total += pf_get_vf_config_lmem(tile->primary_gt, vfid);
+
+	for_each_tile(tile, xe, tid) {
+		lmtt = &tile->sriov.pf.lmtt;
+
+		xe_lmtt_drop_pages(lmtt, vfid);
+		if (!total)
+			continue;
+
+		err  = xe_lmtt_prepare_pages(lmtt, vfid, total);
+		if (err)
+			goto fail;
+
+		offset = 0;
+		for_each_gt(gt, xe, gtid) {
+			if (xe_gt_is_media_type(gt))
+				continue;
+
+			config = pf_pick_vf_config(gt, vfid);
+			bo = config->lmem_obj;
+			if (!bo)
+				continue;
+
+			err = xe_lmtt_populate_pages(lmtt, vfid, bo, offset);
+			if (err)
+				goto fail;
+			offset += bo->size;
+		}
+	}
+
+	pf_force_lmtt_invalidate(xe);
+	return 0;
+
+fail:
+	for_each_tile(tile, xe, tid) {
+		lmtt = &tile->sriov.pf.lmtt;
+		xe_lmtt_drop_pages(lmtt, vfid);
+	}
+	return err;
+}
+
+static void pf_release_vf_config_lmem(struct xe_gt *gt, struct xe_gt_sriov_config *config)
+{
+	xe_gt_assert(gt, !xe_gt_is_media_type(gt));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	if (config->lmem_obj) {
+		xe_bo_unpin_map_no_vm(config->lmem_obj);
+		config->lmem_obj = NULL;
+	}
+}
+
+static int pf_provision_vf_lmem(struct xe_gt *gt, unsigned int vfid, u64 size)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+	struct xe_device *xe = gt_to_xe(gt);
+	struct xe_tile *tile = gt_to_tile(gt);
+	struct xe_bo *bo;
+	int err;
+
+	xe_gt_assert(gt, vfid);
+	xe_gt_assert(gt, !xe_gt_is_media_type(gt));
+
+	size = round_up(size, pf_get_lmem_alignment(gt));
+
+	if (config->lmem_obj) {
+		err = pf_distribute_config_lmem(gt, vfid, 0);
+		if (unlikely(err))
+			return err;
+
+		pf_reset_vf_lmtt(xe, vfid);
+		pf_release_vf_config_lmem(gt, config);
+	}
+	xe_gt_assert(gt, !config->lmem_obj);
+
+	if (!size)
+		return 0;
+
+	xe_gt_assert(gt, pf_get_lmem_alignment(gt) == SZ_2M);
+	bo = xe_bo_create_pin_map(xe, tile, NULL,
+				  ALIGN(size, PAGE_SIZE),
+				  ttm_bo_type_kernel,
+				  XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+				  XE_BO_FLAG_PINNED);
+	if (IS_ERR(bo))
+		return PTR_ERR(bo);
+
+	config->lmem_obj = bo;
+
+	err = pf_update_vf_lmtt(xe, vfid);
+	if (unlikely(err))
+		goto release;
+
+	err = pf_push_vf_cfg_lmem(gt, vfid, bo->size);
+	if (unlikely(err))
+		goto reset_lmtt;
+
+	xe_gt_sriov_dbg_verbose(gt, "VF%u LMEM %zu (%zuM)\n",
+				vfid, bo->size, bo->size / SZ_1M);
+	return 0;
+
+reset_lmtt:
+	pf_reset_vf_lmtt(xe, vfid);
+release:
+	pf_release_vf_config_lmem(gt, config);
+	return err;
+}
+
+/**
+ * xe_gt_sriov_pf_config_get_lmem - Get VF's LMEM quota.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ *
+ * This function can only be called on PF.
+ *
+ * Return: VF's (or PF's spare) LMEM quota.
+ */
+u64 xe_gt_sriov_pf_config_get_lmem(struct xe_gt *gt, unsigned int vfid)
+{
+	u64 size;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	if (vfid)
+		size = pf_get_vf_config_lmem(gt, vfid);
+	else
+		size = pf_get_spare_lmem(gt);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return size;
+}
+
+/**
+ * xe_gt_sriov_pf_config_set_lmem - Provision VF with LMEM.
+ * @gt: the &xe_gt (can't be media)
+ * @vfid: the VF identifier
+ * @size: requested LMEM size
+ *
+ * This function can only be called on PF.
+ */
+int xe_gt_sriov_pf_config_set_lmem(struct xe_gt *gt, unsigned int vfid, u64 size)
+{
+	int err;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	if (vfid)
+		err = pf_provision_vf_lmem(gt, vfid, size);
+	else
+		err = pf_set_spare_lmem(gt, size);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_config_set_u64_done(gt, vfid, size,
+				      xe_gt_sriov_pf_config_get_lmem(gt, vfid),
+				      vfid ? "LMEM" : "spare LMEM", err);
+}
+
+/**
+ * xe_gt_sriov_pf_config_bulk_set_lmem - Provision many VFs with LMEM.
+ * @gt: the &xe_gt (can't be media)
+ * @vfid: starting VF identifier (can't be 0)
+ * @num_vfs: number of VFs to provision
+ * @size: requested LMEM size
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_bulk_set_lmem(struct xe_gt *gt, unsigned int vfid,
+					unsigned int num_vfs, u64 size)
+{
+	unsigned int n;
+	int err = 0;
+
+	xe_gt_assert(gt, vfid);
+	xe_gt_assert(gt, !xe_gt_is_media_type(gt));
+
+	if (!num_vfs)
+		return 0;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	for (n = vfid; n < vfid + num_vfs; n++) {
+		err = pf_provision_vf_lmem(gt, n, size);
+		if (err)
+			break;
+	}
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_config_bulk_set_u64_done(gt, vfid, num_vfs, size,
+					   xe_gt_sriov_pf_config_get_lmem,
+					   "LMEM", n, err);
+}
+
+static u64 pf_query_free_lmem(struct xe_gt *gt)
+{
+	struct xe_tile *tile = gt->tile;
+
+	return xe_ttm_vram_get_avail(&tile->mem.vram_mgr->manager);
+}
+
+static u64 pf_query_max_lmem(struct xe_gt *gt)
+{
+	u64 alignment = pf_get_lmem_alignment(gt);
+	u64 spare = pf_get_spare_lmem(gt);
+	u64 free = pf_query_free_lmem(gt);
+	u64 avail;
+
+	/* XXX: need to account for 2MB blocks only */
+	avail = free > spare ? free - spare : 0;
+	avail = round_down(avail, alignment);
+
+	return avail;
+}
+
+#ifdef CONFIG_DRM_XE_DEBUG_SRIOV
+#define MAX_FAIR_LMEM	SZ_128M	/* XXX: make it small for the driver bringup */
+#else
+#define MAX_FAIR_LMEM	SZ_2G	/* XXX: known issue with allocating BO over 2GiB */
+#endif
+
+static u64 pf_estimate_fair_lmem(struct xe_gt *gt, unsigned int num_vfs)
+{
+	u64 available = pf_query_max_lmem(gt);
+	u64 alignment = pf_get_lmem_alignment(gt);
+	u64 fair;
+
+	fair = div_u64(available, num_vfs);
+	fair = ALIGN_DOWN(fair, alignment);
+#ifdef MAX_FAIR_LMEM
+	fair = min_t(u64, MAX_FAIR_LMEM, fair);
+#endif
+	xe_gt_sriov_dbg_verbose(gt, "LMEM available(%lluM) fair(%u x %lluM)\n",
+				available / SZ_1M, num_vfs, fair / SZ_1M);
+	return fair;
+}
+
+/**
+ * xe_gt_sriov_pf_config_set_fair_lmem - Provision many VFs with fair LMEM.
+ * @gt: the &xe_gt (can't be media)
+ * @vfid: starting VF identifier (can't be 0)
+ * @num_vfs: number of VFs to provision (can't be 0)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_fair_lmem(struct xe_gt *gt, unsigned int vfid,
+					unsigned int num_vfs)
+{
+	u64 fair;
+
+	xe_gt_assert(gt, vfid);
+	xe_gt_assert(gt, num_vfs);
+	xe_gt_assert(gt, !xe_gt_is_media_type(gt));
+
+	if (!IS_DGFX(gt_to_xe(gt)))
+		return 0;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	fair = pf_estimate_fair_lmem(gt, num_vfs);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	if (!fair)
+		return -ENOSPC;
+
+	return xe_gt_sriov_pf_config_bulk_set_lmem(gt, vfid, num_vfs, fair);
+}
+
+/**
+ * xe_gt_sriov_pf_config_set_fair - Provision many VFs with fair resources.
+ * @gt: the &xe_gt
+ * @vfid: starting VF identifier (can't be 0)
+ * @num_vfs: number of VFs to provision (can't be 0)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_fair(struct xe_gt *gt, unsigned int vfid,
+				   unsigned int num_vfs)
+{
+	int result = 0;
+	int err;
+
+	xe_gt_assert(gt, vfid);
+	xe_gt_assert(gt, num_vfs);
+
+	if (!xe_gt_is_media_type(gt)) {
+		err = xe_gt_sriov_pf_config_set_fair_ggtt(gt, vfid, num_vfs);
+		result = result ?: err;
+		err = xe_gt_sriov_pf_config_set_fair_lmem(gt, vfid, num_vfs);
+		result = result ?: err;
+	}
+	err = xe_gt_sriov_pf_config_set_fair_ctxs(gt, vfid, num_vfs);
+	result = result ?: err;
+	err = xe_gt_sriov_pf_config_set_fair_dbs(gt, vfid, num_vfs);
+	result = result ?: err;
+
+	return result;
+}
+
+static const char *exec_quantum_unit(u32 exec_quantum)
+{
+	return exec_quantum ? "ms" : "(infinity)";
+}
+
+static int pf_provision_exec_quantum(struct xe_gt *gt, unsigned int vfid,
+				     u32 exec_quantum)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+	int err;
+
+	err = pf_push_vf_cfg_exec_quantum(gt, vfid, exec_quantum);
+	if (unlikely(err))
+		return err;
+
+	config->exec_quantum = exec_quantum;
+	return 0;
+}
+
+static int pf_get_exec_quantum(struct xe_gt *gt, unsigned int vfid)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+
+	return config->exec_quantum;
+}
+
+/**
+ * xe_gt_sriov_pf_config_set_exec_quantum - Configure execution quantum for the VF.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ * @exec_quantum: requested execution quantum in milliseconds (0 is infinity)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_exec_quantum(struct xe_gt *gt, unsigned int vfid,
+					   u32 exec_quantum)
+{
+	int err;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	err = pf_provision_exec_quantum(gt, vfid, exec_quantum);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_config_set_u32_done(gt, vfid, exec_quantum,
+				      xe_gt_sriov_pf_config_get_exec_quantum(gt, vfid),
+				      "execution quantum", exec_quantum_unit, err);
+}
+
+/**
+ * xe_gt_sriov_pf_config_get_exec_quantum - Get VF's execution quantum.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ *
+ * This function can only be called on PF.
+ *
+ * Return: VF's (or PF's) execution quantum in milliseconds.
+ */
+u32 xe_gt_sriov_pf_config_get_exec_quantum(struct xe_gt *gt, unsigned int vfid)
+{
+	u32 exec_quantum;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	exec_quantum = pf_get_exec_quantum(gt, vfid);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return exec_quantum;
+}
+
+static const char *preempt_timeout_unit(u32 preempt_timeout)
+{
+	return preempt_timeout ? "us" : "(infinity)";
+}
+
+static int pf_provision_preempt_timeout(struct xe_gt *gt, unsigned int vfid,
+					u32 preempt_timeout)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+	int err;
+
+	err = pf_push_vf_cfg_preempt_timeout(gt, vfid, preempt_timeout);
+	if (unlikely(err))
+		return err;
+
+	config->preempt_timeout = preempt_timeout;
+
+	return 0;
+}
+
+static int pf_get_preempt_timeout(struct xe_gt *gt, unsigned int vfid)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+
+	return config->preempt_timeout;
+}
+
+/**
+ * xe_gt_sriov_pf_config_set_preempt_timeout - Configure preemption timeout for the VF.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ * @preempt_timeout: requested preemption timeout in microseconds (0 is infinity)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_preempt_timeout(struct xe_gt *gt, unsigned int vfid,
+					      u32 preempt_timeout)
+{
+	int err;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	err = pf_provision_preempt_timeout(gt, vfid, preempt_timeout);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_config_set_u32_done(gt, vfid, preempt_timeout,
+				      xe_gt_sriov_pf_config_get_preempt_timeout(gt, vfid),
+				      "preemption timeout", preempt_timeout_unit, err);
+}
+
+/**
+ * xe_gt_sriov_pf_config_get_preempt_timeout - Get VF's preemption timeout.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ *
+ * This function can only be called on PF.
+ *
+ * Return: VF's (or PF's) preemption timeout in microseconds.
+ */
+u32 xe_gt_sriov_pf_config_get_preempt_timeout(struct xe_gt *gt, unsigned int vfid)
+{
+	u32 preempt_timeout;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	preempt_timeout = pf_get_preempt_timeout(gt, vfid);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return preempt_timeout;
+}
+
+static void pf_reset_config_sched(struct xe_gt *gt, struct xe_gt_sriov_config *config)
+{
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	config->exec_quantum = 0;
+	config->preempt_timeout = 0;
+}
+
+static void pf_release_vf_config(struct xe_gt *gt, unsigned int vfid)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+
+	if (!xe_gt_is_media_type(gt)) {
+		pf_release_vf_config_ggtt(gt, config);
+		pf_release_vf_config_lmem(gt, config);
+	}
+	pf_release_config_ctxs(gt, config);
+	pf_release_config_dbs(gt, config);
+	pf_reset_config_sched(gt, config);
+}
+
+/**
+ * xe_gt_sriov_pf_config_release - Release and reset VF configuration.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier (can't be PF)
+ * @force: force configuration release
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force)
+{
+	int err;
+
+	xe_gt_assert(gt, vfid);
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	err = pf_send_vf_cfg_reset(gt, vfid);
+	if (!err || force)
+		pf_release_vf_config(gt, vfid);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	if (unlikely(err)) {
+		xe_gt_sriov_notice(gt, "VF%u unprovisioning failed with error (%pe)%s\n",
+				   vfid, ERR_PTR(err),
+				   force ? " but all resources were released anyway!" : "");
+	}
+
+	return force ? 0 : err;
+}
+
+/**
+ * xe_gt_sriov_pf_config_push - Reprovision VF's configuration.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier (can't be PF)
+ * @refresh: explicit refresh
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_push(struct xe_gt *gt, unsigned int vfid, bool refresh)
+{
+	int err = 0;
+
+	xe_gt_assert(gt, vfid);
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	if (refresh)
+		err = pf_send_vf_cfg_reset(gt, vfid);
+	if (!err)
+		err = pf_push_full_vf_config(gt, vfid);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	if (unlikely(err)) {
+		xe_gt_sriov_notice(gt, "Failed to %s VF%u configuration (%pe)\n",
+				   refresh ? "refresh" : "push", vfid, ERR_PTR(err));
+	}
+
+	return err;
+}
+
+/**
+ * xe_gt_sriov_pf_config_print_ggtt - Print GGTT configurations.
+ * @gt: the &xe_gt
+ * @p: the &drm_printer
+ *
+ * Print GGTT configuration data for all VFs.
+ * VFs without provisioned GGTT are ignored.
+ *
+ * This function can only be called on PF.
+ */
+int xe_gt_sriov_pf_config_print_ggtt(struct xe_gt *gt, struct drm_printer *p)
+{
+	unsigned int n, total_vfs = xe_sriov_pf_get_totalvfs(gt_to_xe(gt));
+	const struct xe_gt_sriov_config *config;
+	char buf[10];
+
+	for (n = 1; n <= total_vfs; n++) {
+		config = &gt->sriov.pf.vfs[n].config;
+		if (!drm_mm_node_allocated(&config->ggtt_region))
+			continue;
+
+		string_get_size(config->ggtt_region.size, 1, STRING_UNITS_2, buf, sizeof(buf));
+		drm_printf(p, "VF%u:\t%#0llx-%#llx\t(%s)\n",
+			   n, config->ggtt_region.start,
+			   config->ggtt_region.start + config->ggtt_region.size - 1, buf);
+	}
+
+	return 0;
+}
+
+/**
+ * xe_gt_sriov_pf_config_print_ctxs - Print GuC context IDs configurations.
+ * @gt: the &xe_gt
+ * @p: the &drm_printer
+ *
+ * Print GuC context ID allocations across all VFs.
+ * VFs without GuC context IDs are skipped.
+ *
+ * This function can only be called on PF.
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_print_ctxs(struct xe_gt *gt, struct drm_printer *p)
+{
+	unsigned int n, total_vfs = xe_sriov_pf_get_totalvfs(gt_to_xe(gt));
+	const struct xe_gt_sriov_config *config;
+
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+
+	for (n = 1; n <= total_vfs; n++) {
+		config = &gt->sriov.pf.vfs[n].config;
+		if (!config->num_ctxs)
+			continue;
+
+		drm_printf(p, "VF%u:\t%u-%u\t(%u)\n",
+			   n,
+			   config->begin_ctx,
+			   config->begin_ctx + config->num_ctxs - 1,
+			   config->num_ctxs);
+	}
+
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+	return 0;
+}
+
+/**
+ * xe_gt_sriov_pf_config_print_dbs - Print GuC doorbell ID configurations.
+ * @gt: the &xe_gt
+ * @p: the &drm_printer
+ *
+ * Print GuC doorbell IDs allocations across all VFs.
+ * VFs without GuC doorbell IDs are skipped.
+ *
+ * This function can only be called on PF.
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_print_dbs(struct xe_gt *gt, struct drm_printer *p)
+{
+	unsigned int n, total_vfs = xe_sriov_pf_get_totalvfs(gt_to_xe(gt));
+	const struct xe_gt_sriov_config *config;
+
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+
+	for (n = 1; n <= total_vfs; n++) {
+		config = &gt->sriov.pf.vfs[n].config;
+		if (!config->num_dbs)
+			continue;
+
+		drm_printf(p, "VF%u:\t%u-%u\t(%u)\n",
+			   n,
+			   config->begin_db,
+			   config->begin_db + config->num_dbs - 1,
+			   config->num_dbs);
+	}
+
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+	return 0;
+}
+
+/**
+ * xe_gt_sriov_pf_config_print_available_ggtt - Print available GGTT ranges.
+ * @gt: the &xe_gt
+ * @p: the &drm_printer
+ *
+ * Print GGTT ranges that are available for the provisioning.
+ *
+ * This function can only be called on PF.
+ */
+int xe_gt_sriov_pf_config_print_available_ggtt(struct xe_gt *gt, struct drm_printer *p)
+{
+	struct xe_ggtt *ggtt = gt_to_tile(gt)->mem.ggtt;
+	const struct drm_mm *mm = &ggtt->mm;
+	const struct drm_mm_node *entry;
+	u64 alignment = pf_get_ggtt_alignment(gt);
+	u64 hole_min_start = xe_wopcm_size(gt_to_xe(gt));
+	u64 hole_start, hole_end, hole_size;
+	u64 spare, avail, total = 0;
+	char buf[10];
+
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+
+	spare = pf_get_spare_ggtt(gt);
+
+	mutex_lock(&ggtt->lock);
+
+	drm_mm_for_each_hole(entry, mm, hole_start, hole_end) {
+		hole_start = max(hole_start, hole_min_start);
+		hole_start = ALIGN(hole_start, alignment);
+		hole_end = ALIGN_DOWN(hole_end, alignment);
+		if (hole_start >= hole_end)
+			continue;
+		hole_size = hole_end - hole_start;
+		total += hole_size;
+
+		string_get_size(hole_size, 1, STRING_UNITS_2, buf, sizeof(buf));
+		drm_printf(p, "range:\t%#llx-%#llx\t(%s)\n",
+			   hole_start, hole_end - 1, buf);
+	}
+
+	mutex_unlock(&ggtt->lock);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	string_get_size(total, 1, STRING_UNITS_2, buf, sizeof(buf));
+	drm_printf(p, "total:\t%llu\t(%s)\n", total, buf);
+
+	string_get_size(spare, 1, STRING_UNITS_2, buf, sizeof(buf));
+	drm_printf(p, "spare:\t%llu\t(%s)\n", spare, buf);
+
+	avail = total > spare ? total - spare : 0;
+
+	string_get_size(avail, 1, STRING_UNITS_2, buf, sizeof(buf));
+	drm_printf(p, "avail:\t%llu\t(%s)\n", avail, buf);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h
new file mode 100644
index 000000000000..5e6b36f00b5b
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#ifndef _XE_GT_SRIOV_PF_CONFIG_H_
+#define _XE_GT_SRIOV_PF_CONFIG_H_
+
+#include <linux/types.h>
+
+struct drm_printer;
+struct xe_gt;
+
+u64 xe_gt_sriov_pf_config_get_ggtt(struct xe_gt *gt, unsigned int vfid);
+int xe_gt_sriov_pf_config_set_ggtt(struct xe_gt *gt, unsigned int vfid, u64 size);
+int xe_gt_sriov_pf_config_set_fair_ggtt(struct xe_gt *gt,
+					unsigned int vfid, unsigned int num_vfs);
+int xe_gt_sriov_pf_config_bulk_set_ggtt(struct xe_gt *gt,
+					unsigned int vfid, unsigned int num_vfs, u64 size);
+
+u32 xe_gt_sriov_pf_config_get_ctxs(struct xe_gt *gt, unsigned int vfid);
+int xe_gt_sriov_pf_config_set_ctxs(struct xe_gt *gt, unsigned int vfid, u32 num_ctxs);
+int xe_gt_sriov_pf_config_set_fair_ctxs(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs);
+int xe_gt_sriov_pf_config_bulk_set_ctxs(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs,
+					u32 num_ctxs);
+
+u32 xe_gt_sriov_pf_config_get_dbs(struct xe_gt *gt, unsigned int vfid);
+int xe_gt_sriov_pf_config_set_dbs(struct xe_gt *gt, unsigned int vfid, u32 num_dbs);
+int xe_gt_sriov_pf_config_set_fair_dbs(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs);
+int xe_gt_sriov_pf_config_bulk_set_dbs(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs,
+				       u32 num_dbs);
+
+u64 xe_gt_sriov_pf_config_get_lmem(struct xe_gt *gt, unsigned int vfid);
+int xe_gt_sriov_pf_config_set_lmem(struct xe_gt *gt, unsigned int vfid, u64 size);
+int xe_gt_sriov_pf_config_set_fair_lmem(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs);
+int xe_gt_sriov_pf_config_bulk_set_lmem(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs,
+					u64 size);
+
+u32 xe_gt_sriov_pf_config_get_exec_quantum(struct xe_gt *gt, unsigned int vfid);
+int xe_gt_sriov_pf_config_set_exec_quantum(struct xe_gt *gt, unsigned int vfid, u32 exec_quantum);
+
+u32 xe_gt_sriov_pf_config_get_preempt_timeout(struct xe_gt *gt, unsigned int vfid);
+int xe_gt_sriov_pf_config_set_preempt_timeout(struct xe_gt *gt, unsigned int vfid,
+					      u32 preempt_timeout);
+
+int xe_gt_sriov_pf_config_set_fair(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs);
+int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force);
+int xe_gt_sriov_pf_config_push(struct xe_gt *gt, unsigned int vfid, bool refresh);
+
+int xe_gt_sriov_pf_config_print_ggtt(struct xe_gt *gt, struct drm_printer *p);
+int xe_gt_sriov_pf_config_print_ctxs(struct xe_gt *gt, struct drm_printer *p);
+int xe_gt_sriov_pf_config_print_dbs(struct xe_gt *gt, struct drm_printer *p);
+
+int xe_gt_sriov_pf_config_print_available_ggtt(struct xe_gt *gt, struct drm_printer *p);
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config_types.h
new file mode 100644
index 000000000000..d3745c355957
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config_types.h
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#ifndef _XE_GT_SRIOV_PF_CONFIG_TYPES_H_
+#define _XE_GT_SRIOV_PF_CONFIG_TYPES_H_
+
+#include <drm/drm_mm.h>
+
+struct xe_bo;
+
+/**
+ * struct xe_gt_sriov_config - GT level per-VF configuration data.
+ *
+ * Used by the PF driver to maintain per-VF provisioning data.
+ */
+struct xe_gt_sriov_config {
+	/** @ggtt_region: GGTT region assigned to the VF. */
+	struct drm_mm_node ggtt_region;
+	/** @lmem_obj: LMEM allocation for use by the VF. */
+	struct xe_bo *lmem_obj;
+	/** @num_ctxs: number of GuC contexts IDs.  */
+	u16 num_ctxs;
+	/** @begin_ctx: start index of GuC context ID range. */
+	u16 begin_ctx;
+	/** @num_dbs: number of GuC doorbells IDs. */
+	u16 num_dbs;
+	/** @begin_db: start index of GuC doorbell ID range. */
+	u16 begin_db;
+	/** @exec_quantum: execution-quantum in milliseconds. */
+	u32 exec_quantum;
+	/** @preempt_timeout: preemption timeout in microseconds. */
+	u32 preempt_timeout;
+};
+
+/**
+ * struct xe_gt_sriov_spare_config - GT-level PF spare configuration data.
+ *
+ * Used by the PF driver to maintain it's own reserved (spare) provisioning
+ * data that is not applicable to be tracked in struct xe_gt_sriov_config.
+ */
+struct xe_gt_sriov_spare_config {
+	/** @ggtt_size: GGTT size. */
+	u64 ggtt_size;
+	/** @lmem_size: LMEM size. */
+	u64 lmem_size;
+	/** @num_ctxs: number of GuC submission contexts. */
+	u16 num_ctxs;
+	/** @num_dbs: number of GuC doorbells. */
+	u16 num_dbs;
+};
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c
new file mode 100644
index 000000000000..40b8f881fe04
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c
@@ -0,0 +1,257 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#include "abi/guc_actions_sriov_abi.h"
+
+#include "xe_device.h"
+#include "xe_gt.h"
+#include "xe_gt_sriov_pf_control.h"
+#include "xe_gt_sriov_printk.h"
+#include "xe_guc_ct.h"
+#include "xe_sriov.h"
+
+static const char *control_cmd_to_string(u32 cmd)
+{
+	switch (cmd) {
+	case GUC_PF_TRIGGER_VF_PAUSE:
+		return "PAUSE";
+	case GUC_PF_TRIGGER_VF_RESUME:
+		return "RESUME";
+	case GUC_PF_TRIGGER_VF_STOP:
+		return "STOP";
+	case GUC_PF_TRIGGER_VF_FLR_START:
+		return "FLR_START";
+	case GUC_PF_TRIGGER_VF_FLR_FINISH:
+		return "FLR_FINISH";
+	default:
+		return "<unknown>";
+	}
+}
+
+static int guc_action_vf_control_cmd(struct xe_guc *guc, u32 vfid, u32 cmd)
+{
+	u32 request[PF2GUC_VF_CONTROL_REQUEST_MSG_LEN] = {
+		FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
+		FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
+		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_PF2GUC_VF_CONTROL),
+		FIELD_PREP(PF2GUC_VF_CONTROL_REQUEST_MSG_1_VFID, vfid),
+		FIELD_PREP(PF2GUC_VF_CONTROL_REQUEST_MSG_2_COMMAND, cmd),
+	};
+	int ret;
+
+	/* XXX those two commands are now sent from the G2H handler */
+	if (cmd == GUC_PF_TRIGGER_VF_FLR_START || cmd == GUC_PF_TRIGGER_VF_FLR_FINISH)
+		return xe_guc_ct_send_g2h_handler(&guc->ct, request, ARRAY_SIZE(request));
+
+	ret = xe_guc_ct_send_block(&guc->ct, request, ARRAY_SIZE(request));
+	return ret > 0 ? -EPROTO : ret;
+}
+
+static int pf_send_vf_control_cmd(struct xe_gt *gt, unsigned int vfid, u32 cmd)
+{
+	int err;
+
+	xe_gt_assert(gt, vfid != PFID);
+
+	err = guc_action_vf_control_cmd(&gt->uc.guc, vfid, cmd);
+	if (unlikely(err))
+		xe_gt_sriov_err(gt, "VF%u control command %s failed (%pe)\n",
+				vfid, control_cmd_to_string(cmd), ERR_PTR(err));
+	return err;
+}
+
+static int pf_send_vf_pause(struct xe_gt *gt, unsigned int vfid)
+{
+	return pf_send_vf_control_cmd(gt, vfid, GUC_PF_TRIGGER_VF_PAUSE);
+}
+
+static int pf_send_vf_resume(struct xe_gt *gt, unsigned int vfid)
+{
+	return pf_send_vf_control_cmd(gt, vfid, GUC_PF_TRIGGER_VF_RESUME);
+}
+
+static int pf_send_vf_stop(struct xe_gt *gt, unsigned int vfid)
+{
+	return pf_send_vf_control_cmd(gt, vfid, GUC_PF_TRIGGER_VF_STOP);
+}
+
+static int pf_send_vf_flr_start(struct xe_gt *gt, unsigned int vfid)
+{
+	return pf_send_vf_control_cmd(gt, vfid, GUC_PF_TRIGGER_VF_FLR_START);
+}
+
+static int pf_send_vf_flr_finish(struct xe_gt *gt, unsigned int vfid)
+{
+	return pf_send_vf_control_cmd(gt, vfid, GUC_PF_TRIGGER_VF_FLR_FINISH);
+}
+
+/**
+ * xe_gt_sriov_pf_control_pause_vf - Pause a VF.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ *
+ * This function is for PF only.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_control_pause_vf(struct xe_gt *gt, unsigned int vfid)
+{
+	return pf_send_vf_pause(gt, vfid);
+}
+
+/**
+ * xe_gt_sriov_pf_control_resume_vf - Resume a VF.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ *
+ * This function is for PF only.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_control_resume_vf(struct xe_gt *gt, unsigned int vfid)
+{
+	return pf_send_vf_resume(gt, vfid);
+}
+
+/**
+ * xe_gt_sriov_pf_control_stop_vf - Stop a VF.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ *
+ * This function is for PF only.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_control_stop_vf(struct xe_gt *gt, unsigned int vfid)
+{
+	return pf_send_vf_stop(gt, vfid);
+}
+
+/**
+ * DOC: The VF FLR Flow with GuC
+ *
+ *          PF                        GUC             PCI
+ * ========================================================
+ *          |                          |               |
+ * (1)      |                         [ ] <----- FLR --|
+ *          |                         [ ]              :
+ * (2)     [ ] <-------- NOTIFY FLR --[ ]
+ *         [ ]                         |
+ * (3)     [ ]                         |
+ *         [ ]                         |
+ *         [ ]-- START FLR ---------> [ ]
+ *          |                         [ ]
+ * (4)      |                         [ ]
+ *          |                         [ ]
+ *         [ ] <--------- FLR DONE -- [ ]
+ *         [ ]                         |
+ * (5)     [ ]                         |
+ *         [ ]                         |
+ *         [ ]-- FINISH FLR --------> [ ]
+ *          |                          |
+ *
+ * Step 1: PCI HW generates interrupt to the GuC about VF FLR
+ * Step 2: GuC FW sends G2H notification to the PF about VF FLR
+ * Step 2a: on some platforms G2H is only received from root GuC
+ * Step 3: PF sends H2G request to the GuC to start VF FLR sequence
+ * Step 3a: on some platforms PF must send H2G to all other GuCs
+ * Step 4: GuC FW performs VF FLR cleanups and notifies the PF when done
+ * Step 5: PF performs VF FLR cleanups and notifies the GuC FW when finished
+ */
+
+static bool needs_dispatch_flr(struct xe_device *xe)
+{
+	return xe->info.platform == XE_PVC;
+}
+
+static void pf_handle_vf_flr(struct xe_gt *gt, u32 vfid)
+{
+	struct xe_device *xe = gt_to_xe(gt);
+	struct xe_gt *gtit;
+	unsigned int gtid;
+
+	xe_gt_sriov_info(gt, "VF%u FLR\n", vfid);
+
+	if (needs_dispatch_flr(xe)) {
+		for_each_gt(gtit, xe, gtid)
+			pf_send_vf_flr_start(gtit, vfid);
+	} else {
+		pf_send_vf_flr_start(gt, vfid);
+	}
+}
+
+static void pf_handle_vf_flr_done(struct xe_gt *gt, u32 vfid)
+{
+	pf_send_vf_flr_finish(gt, vfid);
+}
+
+static int pf_handle_vf_event(struct xe_gt *gt, u32 vfid, u32 eventid)
+{
+	switch (eventid) {
+	case GUC_PF_NOTIFY_VF_FLR:
+		pf_handle_vf_flr(gt, vfid);
+		break;
+	case GUC_PF_NOTIFY_VF_FLR_DONE:
+		pf_handle_vf_flr_done(gt, vfid);
+		break;
+	case GUC_PF_NOTIFY_VF_PAUSE_DONE:
+		break;
+	case GUC_PF_NOTIFY_VF_FIXUP_DONE:
+		break;
+	default:
+		return -ENOPKG;
+	}
+	return 0;
+}
+
+static int pf_handle_pf_event(struct xe_gt *gt, u32 eventid)
+{
+	switch (eventid) {
+	case GUC_PF_NOTIFY_VF_ENABLE:
+		xe_gt_sriov_dbg_verbose(gt, "VFs %s/%s\n",
+					str_enabled_disabled(true),
+					str_enabled_disabled(false));
+		break;
+	default:
+		return -ENOPKG;
+	}
+	return 0;
+}
+
+/**
+ * xe_gt_sriov_pf_control_process_guc2pf - Handle VF state notification from GuC.
+ * @gt: the &xe_gt
+ * @msg: the G2H message
+ * @len: the length of the G2H message
+ *
+ * This function is for PF only.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_control_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 len)
+{
+	u32 vfid;
+	u32 eventid;
+
+	xe_gt_assert(gt, len);
+	xe_gt_assert(gt, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC);
+	xe_gt_assert(gt, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT);
+	xe_gt_assert(gt, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) ==
+		     GUC_ACTION_GUC2PF_VF_STATE_NOTIFY);
+
+	if (unlikely(!xe_device_is_sriov_pf(gt_to_xe(gt))))
+		return -EPROTO;
+
+	if (unlikely(FIELD_GET(GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_0_MBZ, msg[0])))
+		return -EPFNOSUPPORT;
+
+	if (unlikely(len != GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_LEN))
+		return -EPROTO;
+
+	vfid = FIELD_GET(GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_1_VFID, msg[1]);
+	eventid = FIELD_GET(GUC2PF_VF_STATE_NOTIFY_EVENT_MSG_2_EVENT, msg[2]);
+
+	return vfid ? pf_handle_vf_event(gt, vfid, eventid) : pf_handle_pf_event(gt, eventid);
+}
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h
new file mode 100644
index 000000000000..850a3e37661f
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#ifndef _XE_GT_SRIOV_PF_CONTROL_H_
+#define _XE_GT_SRIOV_PF_CONTROL_H_
+
+#include <linux/errno.h>
+#include <linux/types.h>
+
+struct xe_gt;
+
+int xe_gt_sriov_pf_control_pause_vf(struct xe_gt *gt, unsigned int vfid);
+int xe_gt_sriov_pf_control_resume_vf(struct xe_gt *gt, unsigned int vfid);
+int xe_gt_sriov_pf_control_stop_vf(struct xe_gt *gt, unsigned int vfid);
+
+#ifdef CONFIG_PCI_IOV
+int xe_gt_sriov_pf_control_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 len);
+#else
+static inline int xe_gt_sriov_pf_control_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 len)
+{
+	return -EPROTO;
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_helpers.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_helpers.h
new file mode 100644
index 000000000000..0bf12d89ceb2
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_helpers.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#ifndef _XE_GT_SRIOV_PF_HELPERS_H_
+#define _XE_GT_SRIOV_PF_HELPERS_H_
+
+#include "xe_gt_types.h"
+#include "xe_sriov_pf_helpers.h"
+
+/**
+ * xe_gt_sriov_pf_assert_vfid() - warn if &id is not a supported VF number when debugging.
+ * @gt: the PF &xe_gt to assert on
+ * @vfid: the VF number to assert
+ *
+ * Assert that &gt belongs to the Physical Function (PF) device and provided &vfid
+ * is within a range of supported VF numbers (up to maximum number of VFs that
+ * driver can support, including VF0 that represents the PF itself).
+ *
+ * Note: Effective only on debug builds. See `Xe ASSERTs`_ for more information.
+ */
+#define xe_gt_sriov_pf_assert_vfid(gt, vfid)	xe_sriov_pf_assert_vfid(gt_to_xe(gt), (vfid))
+
+static inline int xe_gt_sriov_pf_get_totalvfs(struct xe_gt *gt)
+{
+	return xe_sriov_pf_get_totalvfs(gt_to_xe(gt));
+}
+
+static inline struct mutex *xe_gt_sriov_pf_master_mutex(struct xe_gt *gt)
+{
+	return xe_sriov_pf_master_mutex(gt_to_xe(gt));
+}
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c
new file mode 100644
index 000000000000..fae5be5a2a11
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c
@@ -0,0 +1,418 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#include "abi/guc_actions_sriov_abi.h"
+
+#include "xe_bo.h"
+#include "xe_gt.h"
+#include "xe_gt_sriov_pf_helpers.h"
+#include "xe_gt_sriov_pf_policy.h"
+#include "xe_gt_sriov_printk.h"
+#include "xe_guc_ct.h"
+#include "xe_guc_klv_helpers.h"
+#include "xe_pm.h"
+
+/*
+ * Return: number of KLVs that were successfully parsed and saved,
+ *         negative error code on failure.
+ */
+static int guc_action_update_vgt_policy(struct xe_guc *guc, u64 addr, u32 size)
+{
+	u32 request[] = {
+		GUC_ACTION_PF2GUC_UPDATE_VGT_POLICY,
+		lower_32_bits(addr),
+		upper_32_bits(addr),
+		size,
+	};
+
+	return xe_guc_ct_send_block(&guc->ct, request, ARRAY_SIZE(request));
+}
+
+/*
+ * Return: number of KLVs that were successfully parsed and saved,
+ *         negative error code on failure.
+ */
+static int pf_send_policy_klvs(struct xe_gt *gt, const u32 *klvs, u32 num_dwords)
+{
+	const u32 bytes = num_dwords * sizeof(u32);
+	struct xe_tile *tile = gt_to_tile(gt);
+	struct xe_device *xe = tile_to_xe(tile);
+	struct xe_guc *guc = &gt->uc.guc;
+	struct xe_bo *bo;
+	int ret;
+
+	bo = xe_bo_create_pin_map(xe, tile, NULL,
+				  ALIGN(bytes, PAGE_SIZE),
+				  ttm_bo_type_kernel,
+				  XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+				  XE_BO_FLAG_GGTT);
+	if (IS_ERR(bo))
+		return PTR_ERR(bo);
+
+	xe_map_memcpy_to(xe, &bo->vmap, 0, klvs, bytes);
+
+	ret = guc_action_update_vgt_policy(guc, xe_bo_ggtt_addr(bo), num_dwords);
+
+	xe_bo_unpin_map_no_vm(bo);
+
+	return ret;
+}
+
+/*
+ * Return: 0 on success, -ENOKEY if some KLVs were not updated, -EPROTO if reply was malformed,
+ *         negative error code on failure.
+ */
+static int pf_push_policy_klvs(struct xe_gt *gt, u32 num_klvs,
+			       const u32 *klvs, u32 num_dwords)
+{
+	int ret;
+
+	xe_gt_assert(gt, num_klvs == xe_guc_klv_count(klvs, num_dwords));
+
+	ret = pf_send_policy_klvs(gt, klvs, num_dwords);
+
+	if (ret != num_klvs) {
+		int err = ret < 0 ? ret : ret < num_klvs ? -ENOKEY : -EPROTO;
+		struct drm_printer p = xe_gt_info_printer(gt);
+
+		xe_gt_sriov_notice(gt, "Failed to push %u policy KLV%s (%pe)\n",
+				   num_klvs, str_plural(num_klvs), ERR_PTR(err));
+		xe_guc_klv_print(klvs, num_dwords, &p);
+		return err;
+	}
+
+	return 0;
+}
+
+static int pf_push_policy_u32(struct xe_gt *gt, u16 key, u32 value)
+{
+	u32 klv[] = {
+		PREP_GUC_KLV(key, 1),
+		value,
+	};
+
+	return pf_push_policy_klvs(gt, 1, klv, ARRAY_SIZE(klv));
+}
+
+static int pf_update_policy_bool(struct xe_gt *gt, u16 key, bool *policy, bool value)
+{
+	int err;
+
+	err = pf_push_policy_u32(gt, key, value);
+	if (unlikely(err)) {
+		xe_gt_sriov_notice(gt, "Failed to update policy %#x '%s' to '%s' (%pe)\n",
+				   key, xe_guc_klv_key_to_string(key),
+				   str_enabled_disabled(value), ERR_PTR(err));
+		return err;
+	}
+
+	xe_gt_sriov_dbg(gt, "policy key %#x '%s' updated to '%s'\n",
+			key, xe_guc_klv_key_to_string(key),
+			str_enabled_disabled(value));
+
+	*policy = value;
+	return 0;
+}
+
+static int pf_update_policy_u32(struct xe_gt *gt, u16 key, u32 *policy, u32 value)
+{
+	int err;
+
+	err = pf_push_policy_u32(gt, key, value);
+	if (unlikely(err)) {
+		xe_gt_sriov_notice(gt, "Failed to update policy %#x '%s' to '%s' (%pe)\n",
+				   key, xe_guc_klv_key_to_string(key),
+				   str_enabled_disabled(value), ERR_PTR(err));
+		return err;
+	}
+
+	xe_gt_sriov_dbg(gt, "policy key %#x '%s' updated to %u\n",
+			key, xe_guc_klv_key_to_string(key), value);
+
+	*policy = value;
+	return 0;
+}
+
+static int pf_provision_sched_if_idle(struct xe_gt *gt, bool enable)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_update_policy_bool(gt, GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY,
+				     &gt->sriov.pf.policy.guc.sched_if_idle,
+				     enable);
+}
+
+static int pf_reprovision_sched_if_idle(struct xe_gt *gt)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_provision_sched_if_idle(gt, gt->sriov.pf.policy.guc.sched_if_idle);
+}
+
+static void pf_sanitize_sched_if_idle(struct xe_gt *gt)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	gt->sriov.pf.policy.guc.sched_if_idle = false;
+}
+
+/**
+ * xe_gt_sriov_pf_policy_set_sched_if_idle - Control the 'sched_if_idle' policy.
+ * @gt: the &xe_gt where to apply the policy
+ * @enable: the value of the 'sched_if_idle' policy
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable)
+{
+	int err;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	err = pf_provision_sched_if_idle(gt, enable);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return err;
+}
+
+/**
+ * xe_gt_sriov_pf_policy_get_sched_if_idle - Retrieve value of 'sched_if_idle' policy.
+ * @gt: the &xe_gt where to read the policy from
+ *
+ * This function can only be called on PF.
+ *
+ * Return: value of 'sched_if_idle' policy.
+ */
+bool xe_gt_sriov_pf_policy_get_sched_if_idle(struct xe_gt *gt)
+{
+	bool enable;
+
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	enable = gt->sriov.pf.policy.guc.sched_if_idle;
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return enable;
+}
+
+static int pf_provision_reset_engine(struct xe_gt *gt, bool enable)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_update_policy_bool(gt, GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_KEY,
+				     &gt->sriov.pf.policy.guc.reset_engine, enable);
+}
+
+static int pf_reprovision_reset_engine(struct xe_gt *gt)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_provision_reset_engine(gt, gt->sriov.pf.policy.guc.reset_engine);
+}
+
+static void pf_sanitize_reset_engine(struct xe_gt *gt)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	gt->sriov.pf.policy.guc.reset_engine = false;
+}
+
+/**
+ * xe_gt_sriov_pf_policy_set_reset_engine - Control the 'reset_engine' policy.
+ * @gt: the &xe_gt where to apply the policy
+ * @enable: the value of the 'reset_engine' policy
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_policy_set_reset_engine(struct xe_gt *gt, bool enable)
+{
+	int err;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	err = pf_provision_reset_engine(gt, enable);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return err;
+}
+
+/**
+ * xe_gt_sriov_pf_policy_get_reset_engine - Retrieve value of 'reset_engine' policy.
+ * @gt: the &xe_gt where to read the policy from
+ *
+ * This function can only be called on PF.
+ *
+ * Return: value of 'reset_engine' policy.
+ */
+bool xe_gt_sriov_pf_policy_get_reset_engine(struct xe_gt *gt)
+{
+	bool enable;
+
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	enable = gt->sriov.pf.policy.guc.reset_engine;
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return enable;
+}
+
+static int pf_provision_sample_period(struct xe_gt *gt, u32 value)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_update_policy_u32(gt, GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_KEY,
+				    &gt->sriov.pf.policy.guc.sample_period, value);
+}
+
+static int pf_reprovision_sample_period(struct xe_gt *gt)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	return pf_provision_sample_period(gt, gt->sriov.pf.policy.guc.sample_period);
+}
+
+static void pf_sanitize_sample_period(struct xe_gt *gt)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	gt->sriov.pf.policy.guc.sample_period = 0;
+}
+
+/**
+ * xe_gt_sriov_pf_policy_set_sample_period - Control the 'sample_period' policy.
+ * @gt: the &xe_gt where to apply the policy
+ * @value: the value of the 'sample_period' policy
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_policy_set_sample_period(struct xe_gt *gt, u32 value)
+{
+	int err;
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	err = pf_provision_sample_period(gt, value);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return err;
+}
+
+/**
+ * xe_gt_sriov_pf_policy_get_sample_period - Retrieve value of 'sample_period' policy.
+ * @gt: the &xe_gt where to read the policy from
+ *
+ * This function can only be called on PF.
+ *
+ * Return: value of 'sample_period' policy.
+ */
+u32 xe_gt_sriov_pf_policy_get_sample_period(struct xe_gt *gt)
+{
+	u32 value;
+
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	value = gt->sriov.pf.policy.guc.sample_period;
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return value;
+}
+
+static void pf_sanitize_guc_policies(struct xe_gt *gt)
+{
+	pf_sanitize_sched_if_idle(gt);
+	pf_sanitize_reset_engine(gt);
+	pf_sanitize_sample_period(gt);
+}
+
+/**
+ * xe_gt_sriov_pf_policy_sanitize - Reset policy settings.
+ * @gt: the &xe_gt
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+void xe_gt_sriov_pf_policy_sanitize(struct xe_gt *gt)
+{
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	pf_sanitize_guc_policies(gt);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+}
+
+/**
+ * xe_gt_sriov_pf_policy_reprovision - Reprovision (and optionally reset) policy settings.
+ * @gt: the &xe_gt
+ * @reset: if true will reprovision using default values instead of latest
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_policy_reprovision(struct xe_gt *gt, bool reset)
+{
+	int err = 0;
+
+	xe_pm_runtime_get_noresume(gt_to_xe(gt));
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	if (reset)
+		pf_sanitize_guc_policies(gt);
+	err |= pf_reprovision_sched_if_idle(gt);
+	err |= pf_reprovision_reset_engine(gt);
+	err |= pf_reprovision_sample_period(gt);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	xe_pm_runtime_put(gt_to_xe(gt));
+
+	return err ? -ENXIO : 0;
+}
+
+static void print_guc_policies(struct drm_printer *p, struct xe_gt_sriov_guc_policies *policy)
+{
+	drm_printf(p, "%s:\t%s\n",
+		   xe_guc_klv_key_to_string(GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY),
+		   str_enabled_disabled(policy->sched_if_idle));
+	drm_printf(p, "%s:\t%s\n",
+		   xe_guc_klv_key_to_string(GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_KEY),
+		   str_enabled_disabled(policy->reset_engine));
+	drm_printf(p, "%s:\t%u %s\n",
+		   xe_guc_klv_key_to_string(GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_KEY),
+		   policy->sample_period, policy->sample_period ? "ms" : "(disabled)");
+}
+
+/**
+ * xe_gt_sriov_pf_policy_print - Dump actual policy values.
+ * @gt: the &xe_gt where to read the policy from
+ * @p: the &drm_printer
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_policy_print(struct xe_gt *gt, struct drm_printer *p)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+
+	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	print_guc_policies(p, &gt->sriov.pf.policy.guc);
+	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h
new file mode 100644
index 000000000000..2a5dc33dc6d7
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#ifndef _XE_GT_SRIOV_PF_POLICY_H_
+#define _XE_GT_SRIOV_PF_POLICY_H_
+
+#include <linux/types.h>
+
+struct drm_printer;
+struct xe_gt;
+
+int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable);
+bool xe_gt_sriov_pf_policy_get_sched_if_idle(struct xe_gt *gt);
+int xe_gt_sriov_pf_policy_set_reset_engine(struct xe_gt *gt, bool enable);
+bool xe_gt_sriov_pf_policy_get_reset_engine(struct xe_gt *gt);
+int xe_gt_sriov_pf_policy_set_sample_period(struct xe_gt *gt, u32 value);
+u32 xe_gt_sriov_pf_policy_get_sample_period(struct xe_gt *gt);
+
+void xe_gt_sriov_pf_policy_sanitize(struct xe_gt *gt);
+int xe_gt_sriov_pf_policy_reprovision(struct xe_gt *gt, bool reset);
+int xe_gt_sriov_pf_policy_print(struct xe_gt *gt, struct drm_printer *p);
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy_types.h
new file mode 100644
index 000000000000..4de532af135e
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#ifndef _XE_GT_SRIOV_PF_POLICY_TYPES_H_
+#define _XE_GT_SRIOV_PF_POLICY_TYPES_H_
+
+#include <linux/types.h>
+
+/**
+ * struct xe_gt_sriov_guc_policies - GuC SR-IOV policies.
+ * @sched_if_idle: controls strict scheduling policy.
+ * @reset_engine: controls engines reset on VF switch policy.
+ * @sample_period: adverse events sampling period (in milliseconds).
+ */
+struct xe_gt_sriov_guc_policies {
+	bool sched_if_idle;
+	bool reset_engine;
+	u32 sample_period;
+};
+
+/**
+ * struct xe_gt_sriov_pf_policy - PF policy data.
+ * @guc: GuC scheduling policies.
+ */
+struct xe_gt_sriov_pf_policy {
+	struct xe_gt_sriov_guc_policies guc;
+};
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h
new file mode 100644
index 000000000000..faf9ee8266ce
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#ifndef _XE_GT_SRIOV_PF_TYPES_H_
+#define _XE_GT_SRIOV_PF_TYPES_H_
+
+#include <linux/types.h>
+
+#include "xe_gt_sriov_pf_config_types.h"
+#include "xe_gt_sriov_pf_policy_types.h"
+
+/**
+ * struct xe_gt_sriov_metadata - GT level per-VF metadata.
+ */
+struct xe_gt_sriov_metadata {
+	/** @config: per-VF provisioning data. */
+	struct xe_gt_sriov_config config;
+};
+
+/**
+ * struct xe_gt_sriov_pf - GT level PF virtualization data.
+ * @policy: policy data.
+ * @spare: PF-only provisioning configuration.
+ * @vfs: metadata for all VFs.
+ */
+struct xe_gt_sriov_pf {
+	struct xe_gt_sriov_pf_policy policy;
+	struct xe_gt_sriov_spare_config spare;
+	struct xe_gt_sriov_metadata *vfs;
+};
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_gt_sysfs.c b/drivers/gpu/drm/xe/xe_gt_sysfs.c
index c69d2e8a0fe1..1e5971072bc8 100644
--- a/drivers/gpu/drm/xe/xe_gt_sysfs.c
+++ b/drivers/gpu/drm/xe/xe_gt_sysfs.c
@@ -29,7 +29,7 @@ static void gt_sysfs_fini(struct drm_device *drm, void *arg)
 	kobject_put(gt->sysfs);
 }
 
-void xe_gt_sysfs_init(struct xe_gt *gt)
+int xe_gt_sysfs_init(struct xe_gt *gt)
 {
 	struct xe_tile *tile = gt_to_tile(gt);
 	struct xe_device *xe = gt_to_xe(gt);
@@ -38,24 +38,18 @@ void xe_gt_sysfs_init(struct xe_gt *gt)
 
 	kg = kzalloc(sizeof(*kg), GFP_KERNEL);
 	if (!kg)
-		return;
+		return -ENOMEM;
 
 	kobject_init(&kg->base, &xe_gt_sysfs_kobj_type);
 	kg->gt = gt;
 
 	err = kobject_add(&kg->base, tile->sysfs, "gt%d", gt->info.id);
 	if (err) {
-		drm_warn(&xe->drm, "failed to add GT sysfs directory, err: %d\n", err);
 		kobject_put(&kg->base);
-		return;
+		return err;
 	}
 
 	gt->sysfs = &kg->base;
 
-	err = drmm_add_action_or_reset(&xe->drm, gt_sysfs_fini, gt);
-	if (err) {
-		drm_warn(&xe->drm, "%s: drmm_add_action_or_reset failed, err: %d\n",
-			 __func__, err);
-		return;
-	}
+	return drmm_add_action_or_reset(&xe->drm, gt_sysfs_fini, gt);
 }
diff --git a/drivers/gpu/drm/xe/xe_gt_sysfs.h b/drivers/gpu/drm/xe/xe_gt_sysfs.h
index e3ec278ca0be..ecbfcc5c7d42 100644
--- a/drivers/gpu/drm/xe/xe_gt_sysfs.h
+++ b/drivers/gpu/drm/xe/xe_gt_sysfs.h
@@ -8,7 +8,7 @@
 
 #include "xe_gt_sysfs_types.h"
 
-void xe_gt_sysfs_init(struct xe_gt *gt);
+int xe_gt_sysfs_init(struct xe_gt *gt);
 
 static inline struct xe_gt *
 kobj_to_gt(struct kobject *kobj)
diff --git a/drivers/gpu/drm/xe/xe_gt_throttle_sysfs.c b/drivers/gpu/drm/xe/xe_gt_throttle_sysfs.c
index 63d640591a52..fbe21a8599ca 100644
--- a/drivers/gpu/drm/xe/xe_gt_throttle_sysfs.c
+++ b/drivers/gpu/drm/xe/xe_gt_throttle_sysfs.c
@@ -11,6 +11,7 @@
 #include "xe_gt_sysfs.h"
 #include "xe_gt_throttle_sysfs.h"
 #include "xe_mmio.h"
+#include "xe_pm.h"
 
 /**
  * DOC: Xe GT Throttle
@@ -38,10 +39,12 @@ static u32 read_perf_limit_reasons(struct xe_gt *gt)
 {
 	u32 reg;
 
+	xe_pm_runtime_get(gt_to_xe(gt));
 	if (xe_gt_is_media_type(gt))
 		reg = xe_mmio_read32(gt, MTL_MEDIA_PERF_LIMIT_REASONS);
 	else
 		reg = xe_mmio_read32(gt, GT0_PERF_LIMIT_REASONS);
+	xe_pm_runtime_put(gt_to_xe(gt));
 
 	return reg;
 }
@@ -233,19 +236,14 @@ static void gt_throttle_sysfs_fini(struct drm_device *drm, void *arg)
 	sysfs_remove_group(gt->freq, &throttle_group_attrs);
 }
 
-void xe_gt_throttle_sysfs_init(struct xe_gt *gt)
+int xe_gt_throttle_sysfs_init(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 	int err;
 
 	err = sysfs_create_group(gt->freq, &throttle_group_attrs);
-	if (err) {
-		drm_warn(&xe->drm, "failed to register throttle sysfs, err: %d\n", err);
-		return;
-	}
-
-	err = drmm_add_action_or_reset(&xe->drm, gt_throttle_sysfs_fini, gt);
 	if (err)
-		drm_warn(&xe->drm, "%s: drmm_add_action_or_reset failed, err: %d\n",
-			 __func__, err);
+		return err;
+
+	return drmm_add_action_or_reset(&xe->drm, gt_throttle_sysfs_fini, gt);
 }
diff --git a/drivers/gpu/drm/xe/xe_gt_throttle_sysfs.h b/drivers/gpu/drm/xe/xe_gt_throttle_sysfs.h
index 3ecfd4beffe1..6c61e6f228a8 100644
--- a/drivers/gpu/drm/xe/xe_gt_throttle_sysfs.h
+++ b/drivers/gpu/drm/xe/xe_gt_throttle_sysfs.h
@@ -10,7 +10,7 @@
 
 struct xe_gt;
 
-void xe_gt_throttle_sysfs_init(struct xe_gt *gt);
+int xe_gt_throttle_sysfs_init(struct xe_gt *gt);
 
 #endif /* _XE_GT_THROTTLE_SYSFS_H_ */
 
diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c
index e598a4363d01..93df2d7969b3 100644
--- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c
+++ b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c
@@ -11,7 +11,9 @@
 #include "xe_gt_printk.h"
 #include "xe_guc.h"
 #include "xe_guc_ct.h"
+#include "xe_mmio.h"
 #include "xe_trace.h"
+#include "regs/xe_guc_regs.h"
 
 #define TLB_TIMEOUT	(HZ / 4)
 
@@ -209,7 +211,7 @@ static int send_tlb_invalidation(struct xe_guc *guc,
  * Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,
  * negative error code on error.
  */
-int xe_gt_tlb_invalidation_guc(struct xe_gt *gt)
+static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt)
 {
 	u32 action[] = {
 		XE_GUC_ACTION_TLB_INVALIDATION,
@@ -222,6 +224,45 @@ int xe_gt_tlb_invalidation_guc(struct xe_gt *gt)
 }
 
 /**
+ * xe_gt_tlb_invalidation_ggtt - Issue a TLB invalidation on this GT for the GGTT
+ * @gt: graphics tile
+ *
+ * Issue a TLB invalidation for the GGTT. Completion of TLB invalidation is
+ * synchronous.
+ *
+ * Return: 0 on success, negative error code on error
+ */
+int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt)
+{
+	struct xe_device *xe = gt_to_xe(gt);
+
+	if (xe_guc_ct_enabled(&gt->uc.guc.ct) &&
+	    gt->uc.guc.submission_state.enabled) {
+		int seqno;
+
+		seqno = xe_gt_tlb_invalidation_guc(gt);
+		if (seqno <= 0)
+			return seqno;
+
+		xe_gt_tlb_invalidation_wait(gt, seqno);
+	} else if (xe_device_uc_enabled(xe)) {
+		xe_gt_WARN_ON(gt, xe_force_wake_get(gt_to_fw(gt), XE_FW_GT));
+		if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) {
+			xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC1,
+					PVC_GUC_TLB_INV_DESC1_INVALIDATE);
+			xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC0,
+					PVC_GUC_TLB_INV_DESC0_VALID);
+		} else {
+			xe_mmio_write32(gt, GUC_TLB_INV_CR,
+					GUC_TLB_INV_CR_INVALIDATE);
+		}
+		xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
+	}
+
+	return 0;
+}
+
+/**
  * xe_gt_tlb_invalidation_vma - Issue a TLB invalidation on this GT for a VMA
  * @gt: graphics tile
  * @fence: invalidation fence which will be signal on TLB invalidation
diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h
index b333c1709397..fbb743d80d2c 100644
--- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h
+++ b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h
@@ -16,7 +16,7 @@ struct xe_vma;
 
 int xe_gt_tlb_invalidation_init(struct xe_gt *gt);
 void xe_gt_tlb_invalidation_reset(struct xe_gt *gt);
-int xe_gt_tlb_invalidation_guc(struct xe_gt *gt);
+int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt);
 int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
 			       struct xe_gt_tlb_invalidation_fence *fence,
 			       struct xe_vma *vma);
diff --git a/drivers/gpu/drm/xe/xe_gt_topology.c b/drivers/gpu/drm/xe/xe_gt_topology.c
index 5dc62fe1be49..3733e7a6860d 100644
--- a/drivers/gpu/drm/xe/xe_gt_topology.c
+++ b/drivers/gpu/drm/xe/xe_gt_topology.c
@@ -8,12 +8,10 @@
 #include <linux/bitmap.h>
 
 #include "regs/xe_gt_regs.h"
+#include "xe_assert.h"
 #include "xe_gt.h"
 #include "xe_mmio.h"
 
-#define XE_MAX_DSS_FUSE_BITS (32 * XE_MAX_DSS_FUSE_REGS)
-#define XE_MAX_EU_FUSE_BITS (32 * XE_MAX_EU_FUSE_REGS)
-
 static void
 load_dss_mask(struct xe_gt *gt, xe_dss_mask_t mask, int numregs, ...)
 {
@@ -62,6 +60,114 @@ load_eu_mask(struct xe_gt *gt, xe_eu_mask_t mask)
 	bitmap_from_arr32(mask, &val, XE_MAX_EU_FUSE_BITS);
 }
 
+/**
+ * gen_l3_mask_from_pattern - Replicate a bit pattern according to a mask
+ *
+ * It is used to compute the L3 bank masks in a generic format on
+ * various platforms where the internal representation of L3 node
+ * and masks from registers are different.
+ *
+ * @xe: device
+ * @dst: destination
+ * @pattern: pattern to replicate
+ * @patternbits: size of the pattern, in bits
+ * @mask: mask describing where to replicate the pattern
+ *
+ * Example 1:
+ * ----------
+ * @pattern =    0b1111
+ *                 └┬─┘
+ * @patternbits =   4 (bits)
+ * @mask = 0b0101
+ *           ││││
+ *           │││└────────────────── 0b1111 (=1×0b1111)
+ *           ││└──────────── 0b0000    │   (=0×0b1111)
+ *           │└────── 0b1111    │      │   (=1×0b1111)
+ *           └ 0b0000    │      │      │   (=0×0b1111)
+ *                │      │      │      │
+ * @dst =      0b0000 0b1111 0b0000 0b1111
+ *
+ * Example 2:
+ * ----------
+ * @pattern =    0b11111111
+ *                 └┬─────┘
+ * @patternbits =   8 (bits)
+ * @mask = 0b10
+ *           ││
+ *           ││
+ *           ││
+ *           │└────────── 0b00000000 (=0×0b11111111)
+ *           └ 0b11111111      │     (=1×0b11111111)
+ *                  │          │
+ * @dst =      0b11111111 0b00000000
+ */
+static void
+gen_l3_mask_from_pattern(struct xe_device *xe, xe_l3_bank_mask_t dst,
+			 xe_l3_bank_mask_t pattern, int patternbits,
+			 unsigned long mask)
+{
+	unsigned long bit;
+
+	xe_assert(xe, fls(mask) <= patternbits);
+	for_each_set_bit(bit, &mask, 32) {
+		xe_l3_bank_mask_t shifted_pattern = {};
+
+		bitmap_shift_left(shifted_pattern, pattern, bit * patternbits,
+				  XE_MAX_L3_BANK_MASK_BITS);
+		bitmap_or(dst, dst, shifted_pattern, XE_MAX_L3_BANK_MASK_BITS);
+	}
+}
+
+static void
+load_l3_bank_mask(struct xe_gt *gt, xe_l3_bank_mask_t l3_bank_mask)
+{
+	struct xe_device *xe = gt_to_xe(gt);
+	u32 fuse3 = xe_mmio_read32(gt, MIRROR_FUSE3);
+
+	if (GRAPHICS_VER(xe) >= 20) {
+		xe_l3_bank_mask_t per_node = {};
+		u32 meml3_en = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, fuse3);
+		u32 bank_val = REG_FIELD_GET(XE2_GT_L3_MODE_MASK, fuse3);
+
+		bitmap_from_arr32(per_node, &bank_val, 32);
+		gen_l3_mask_from_pattern(xe, l3_bank_mask, per_node, 4,
+					 meml3_en);
+	} else if (GRAPHICS_VERx100(xe) >= 1270) {
+		xe_l3_bank_mask_t per_node = {};
+		xe_l3_bank_mask_t per_mask_bit = {};
+		u32 meml3_en = REG_FIELD_GET(MEML3_EN_MASK, fuse3);
+		u32 fuse4 = xe_mmio_read32(gt, XEHP_FUSE4);
+		u32 bank_val = REG_FIELD_GET(GT_L3_EXC_MASK, fuse4);
+
+		bitmap_set_value8(per_mask_bit, 0x3, 0);
+		gen_l3_mask_from_pattern(xe, per_node, per_mask_bit, 2, bank_val);
+		gen_l3_mask_from_pattern(xe, l3_bank_mask, per_node, 4,
+					 meml3_en);
+	} else if (xe->info.platform == XE_PVC) {
+		xe_l3_bank_mask_t per_node = {};
+		xe_l3_bank_mask_t per_mask_bit = {};
+		u32 meml3_en = REG_FIELD_GET(MEML3_EN_MASK, fuse3);
+		u32 bank_val = REG_FIELD_GET(XEHPC_GT_L3_MODE_MASK, fuse3);
+
+		bitmap_set_value8(per_mask_bit, 0xf, 0);
+		gen_l3_mask_from_pattern(xe, per_node, per_mask_bit, 4,
+					 bank_val);
+		gen_l3_mask_from_pattern(xe, l3_bank_mask, per_node, 16,
+					 meml3_en);
+	} else if (xe->info.platform == XE_DG2) {
+		xe_l3_bank_mask_t per_node = {};
+		u32 mask = REG_FIELD_GET(MEML3_EN_MASK, fuse3);
+
+		bitmap_set_value8(per_node, 0xff, 0);
+		gen_l3_mask_from_pattern(xe, l3_bank_mask, per_node, 8, mask);
+	} else {
+		/* 1:1 register bit to mask bit (inverted register bits) */
+		u32 mask = REG_FIELD_GET(XELP_GT_L3_MODE_MASK, ~fuse3);
+
+		bitmap_from_arr32(l3_bank_mask, &mask, 32);
+	}
+}
+
 static void
 get_num_dss_regs(struct xe_device *xe, int *geometry_regs, int *compute_regs)
 {
@@ -106,6 +212,7 @@ xe_gt_topology_init(struct xe_gt *gt)
 		      XEHPC_GT_COMPUTE_DSS_ENABLE_EXT,
 		      XE2_GT_COMPUTE_DSS_2);
 	load_eu_mask(gt, gt->fuse_topo.eu_mask_per_dss);
+	load_l3_bank_mask(gt, gt->fuse_topo.l3_bank_mask);
 
 	p = drm_dbg_printer(&gt_to_xe(gt)->drm, DRM_UT_DRIVER, "GT topology");
 
@@ -123,6 +230,8 @@ xe_gt_topology_dump(struct xe_gt *gt, struct drm_printer *p)
 	drm_printf(p, "EU mask per DSS:     %*pb\n", XE_MAX_EU_FUSE_BITS,
 		   gt->fuse_topo.eu_mask_per_dss);
 
+	drm_printf(p, "L3 bank mask:        %*pb\n", XE_MAX_L3_BANK_MASK_BITS,
+		   gt->fuse_topo.l3_bank_mask);
 }
 
 /*
diff --git a/drivers/gpu/drm/xe/xe_gt_topology.h b/drivers/gpu/drm/xe/xe_gt_topology.h
index d1b54fb52ea6..b3e357777a6e 100644
--- a/drivers/gpu/drm/xe/xe_gt_topology.h
+++ b/drivers/gpu/drm/xe/xe_gt_topology.h
@@ -8,6 +8,17 @@
 
 #include "xe_gt_types.h"
 
+/*
+ * Loop over each DSS with the bit is 1 in geometry or compute mask
+ * @dss: iterated DSS bit from the DSS mask
+ * @gt: GT structure
+ */
+#define for_each_dss(dss, gt) \
+	for_each_or_bit((dss), \
+			(gt)->fuse_topo.g_dss_mask, \
+			(gt)->fuse_topo.c_dss_mask, \
+			XE_MAX_DSS_FUSE_BITS)
+
 struct drm_printer;
 
 void xe_gt_topology_init(struct xe_gt *gt);
diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h
index 07b2f724ec45..cfdc761ff7f4 100644
--- a/drivers/gpu/drm/xe/xe_gt_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_types.h
@@ -8,6 +8,7 @@
 
 #include "xe_force_wake_types.h"
 #include "xe_gt_idle_types.h"
+#include "xe_gt_sriov_pf_types.h"
 #include "xe_hw_engine_types.h"
 #include "xe_hw_fence_types.h"
 #include "xe_reg_sr_types.h"
@@ -24,11 +25,15 @@ enum xe_gt_type {
 	XE_GT_TYPE_MEDIA,
 };
 
-#define XE_MAX_DSS_FUSE_REGS	3
-#define XE_MAX_EU_FUSE_REGS	1
+#define XE_MAX_DSS_FUSE_REGS		3
+#define XE_MAX_DSS_FUSE_BITS		(32 * XE_MAX_DSS_FUSE_REGS)
+#define XE_MAX_EU_FUSE_REGS		1
+#define XE_MAX_EU_FUSE_BITS		(32 * XE_MAX_EU_FUSE_REGS)
+#define XE_MAX_L3_BANK_MASK_BITS	64
 
-typedef unsigned long xe_dss_mask_t[BITS_TO_LONGS(32 * XE_MAX_DSS_FUSE_REGS)];
-typedef unsigned long xe_eu_mask_t[BITS_TO_LONGS(32 * XE_MAX_EU_FUSE_REGS)];
+typedef unsigned long xe_dss_mask_t[BITS_TO_LONGS(XE_MAX_DSS_FUSE_BITS)];
+typedef unsigned long xe_eu_mask_t[BITS_TO_LONGS(XE_MAX_EU_FUSE_BITS)];
+typedef unsigned long xe_l3_bank_mask_t[BITS_TO_LONGS(XE_MAX_L3_BANK_MASK_BITS)];
 
 struct xe_mmio_range {
 	u32 start;
@@ -138,6 +143,12 @@ struct xe_gt {
 		u32 adj_offset;
 	} mmio;
 
+	/** @sriov: virtualization data related to GT */
+	union {
+		/** @sriov.pf: PF data. Valid only if driver is running as PF */
+		struct xe_gt_sriov_pf pf;
+	} sriov;
+
 	/**
 	 * @reg_sr: table with registers to be restored on GT init/resume/reset
 	 */
@@ -325,6 +336,9 @@ struct xe_gt {
 
 		/** @fuse_topo.eu_mask_per_dss: EU mask per DSS*/
 		xe_eu_mask_t eu_mask_per_dss;
+
+		/** @fuse_topo.l3_bank_mask: L3 bank mask */
+		xe_l3_bank_mask_t l3_bank_mask;
 	} fuse_topo;
 
 	/** @steering: register steering for individual HW units */
diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c
index 0d2a2dd13f11..240e7a4bbff1 100644
--- a/drivers/gpu/drm/xe/xe_guc.c
+++ b/drivers/gpu/drm/xe/xe_guc.c
@@ -12,11 +12,13 @@
 #include "abi/guc_actions_abi.h"
 #include "abi/guc_errors_abi.h"
 #include "regs/xe_gt_regs.h"
+#include "regs/xe_gtt_defs.h"
 #include "regs/xe_guc_regs.h"
 #include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_force_wake.h"
 #include "xe_gt.h"
+#include "xe_gt_printk.h"
 #include "xe_guc_ads.h"
 #include "xe_guc_ct.h"
 #include "xe_guc_hwconfig.h"
@@ -33,14 +35,13 @@
 #include "xe_wa.h"
 #include "xe_wopcm.h"
 
-/* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */
-#define GUC_GGTT_TOP    0xFEE00000
 static u32 guc_bo_ggtt_addr(struct xe_guc *guc,
 			    struct xe_bo *bo)
 {
 	struct xe_device *xe = guc_to_xe(guc);
 	u32 addr = xe_bo_ggtt_addr(bo);
 
+	/* GuC addresses above GUC_GGTT_TOP don't map through the GTT */
 	xe_assert(xe, addr >= xe_wopcm_size(guc_to_xe(guc)));
 	xe_assert(xe, addr < GUC_GGTT_TOP);
 	xe_assert(xe, bo->size <= GUC_GGTT_TOP - addr);
@@ -133,15 +134,10 @@ static u32 guc_ctl_ads_flags(struct xe_guc *guc)
 	return flags;
 }
 
-#define GUC_VER(maj, min, pat)	(((maj) << 16) | ((min) << 8) | (pat))
-
 static u32 guc_ctl_wa_flags(struct xe_guc *guc)
 {
 	struct xe_device *xe = guc_to_xe(guc);
 	struct xe_gt *gt = guc_to_gt(guc);
-	struct xe_uc_fw *uc_fw = &guc->fw;
-	struct xe_uc_fw_version *version = &uc_fw->versions.found[XE_UC_FW_VER_RELEASE];
-
 	u32 flags = 0;
 
 	if (XE_WA(gt, 22012773006))
@@ -164,20 +160,15 @@ static u32 guc_ctl_wa_flags(struct xe_guc *guc)
 	if (XE_WA(gt, 22012727170) || XE_WA(gt, 22012727685))
 		flags |= GUC_WA_CONTEXT_ISOLATION;
 
-	if ((XE_WA(gt, 16015675438) || XE_WA(gt, 18020744125)) &&
+	if (XE_WA(gt, 18020744125) &&
 	    !xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_RENDER))
 		flags |= GUC_WA_RCS_REGS_IN_CCS_REGS_LIST;
 
 	if (XE_WA(gt, 1509372804))
 		flags |= GUC_WA_RENDER_RST_RC6_EXIT;
 
-	if (XE_WA(gt, 14018913170)) {
-		if (GUC_VER(version->major, version->minor, version->patch) >= GUC_VER(70, 7, 0))
-			flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6;
-		else
-			drm_dbg(&xe->drm, "Skip WA 14018913170: GUC version expected >= 70.7.0, found %u.%u.%u\n",
-				version->major, version->minor, version->patch);
-	}
+	if (XE_WA(gt, 14018913170))
+		flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6;
 
 	return flags;
 }
@@ -189,15 +180,23 @@ static u32 guc_ctl_devid(struct xe_guc *guc)
 	return (((u32)xe->info.devid) << 16) | xe->info.revid;
 }
 
-static void guc_init_params(struct xe_guc *guc)
+static void guc_print_params(struct xe_guc *guc)
 {
-	struct xe_device *xe = guc_to_xe(guc);
+	struct xe_gt *gt = guc_to_gt(guc);
 	u32 *params = guc->params;
 	int i;
 
 	BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32));
 	BUILD_BUG_ON(GUC_CTL_MAX_DWORDS + 2 != SOFT_SCRATCH_COUNT);
 
+	for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
+		xe_gt_dbg(gt, "GuC param[%2d] = 0x%08x\n", i, params[i]);
+}
+
+static void guc_init_params(struct xe_guc *guc)
+{
+	u32 *params = guc->params;
+
 	params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc);
 	params[GUC_CTL_FEATURE] = 0;
 	params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
@@ -205,18 +204,12 @@ static void guc_init_params(struct xe_guc *guc)
 	params[GUC_CTL_WA] = 0;
 	params[GUC_CTL_DEVID] = guc_ctl_devid(guc);
 
-	for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
-		drm_dbg(&xe->drm, "GuC param[%2d] = 0x%08x\n", i, params[i]);
+	guc_print_params(guc);
 }
 
 static void guc_init_params_post_hwconfig(struct xe_guc *guc)
 {
-	struct xe_device *xe = guc_to_xe(guc);
 	u32 *params = guc->params;
-	int i;
-
-	BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32));
-	BUILD_BUG_ON(GUC_CTL_MAX_DWORDS + 2 != SOFT_SCRATCH_COUNT);
 
 	params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc);
 	params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc);
@@ -225,8 +218,7 @@ static void guc_init_params_post_hwconfig(struct xe_guc *guc)
 	params[GUC_CTL_WA] = guc_ctl_wa_flags(guc);
 	params[GUC_CTL_DEVID] = guc_ctl_devid(guc);
 
-	for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
-		drm_dbg(&xe->drm, "GuC param[%2d] = 0x%08x\n", i, params[i]);
+	guc_print_params(guc);
 }
 
 /*
@@ -250,10 +242,11 @@ static void guc_write_params(struct xe_guc *guc)
 static void guc_fini(struct drm_device *drm, void *arg)
 {
 	struct xe_guc *guc = arg;
+	struct xe_gt *gt = guc_to_gt(guc);
 
-	xe_force_wake_get(gt_to_fw(guc_to_gt(guc)), XE_FORCEWAKE_ALL);
+	xe_gt_WARN_ON(gt, xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL));
 	xe_uc_fini_hw(&guc_to_gt(guc)->uc);
-	xe_force_wake_put(gt_to_fw(guc_to_gt(guc)), XE_FORCEWAKE_ALL);
+	xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 }
 
 /**
@@ -330,7 +323,7 @@ int xe_guc_init(struct xe_guc *guc)
 	if (ret)
 		goto out;
 
-	ret = drmm_add_action_or_reset(&gt_to_xe(gt)->drm, guc_fini, guc);
+	ret = drmm_add_action_or_reset(&xe->drm, guc_fini, guc);
 	if (ret)
 		goto out;
 
@@ -343,7 +336,7 @@ int xe_guc_init(struct xe_guc *guc)
 	return 0;
 
 out:
-	drm_err(&xe->drm, "GuC init failed with %d", ret);
+	xe_gt_err(gt, "GuC init failed with %pe\n", ERR_PTR(ret));
 	return ret;
 }
 
@@ -380,7 +373,6 @@ int xe_guc_post_load_init(struct xe_guc *guc)
 
 int xe_guc_reset(struct xe_guc *guc)
 {
-	struct xe_device *xe = guc_to_xe(guc);
 	struct xe_gt *gt = guc_to_gt(guc);
 	u32 guc_status, gdrst;
 	int ret;
@@ -391,16 +383,14 @@ int xe_guc_reset(struct xe_guc *guc)
 
 	ret = xe_mmio_wait32(gt, GDRST, GRDOM_GUC, 0, 5000, &gdrst, false);
 	if (ret) {
-		drm_err(&xe->drm, "GuC reset timed out, GDRST=0x%8x\n",
-			gdrst);
+		xe_gt_err(gt, "GuC reset timed out, GDRST=%#x\n", gdrst);
 		goto err_out;
 	}
 
 	guc_status = xe_mmio_read32(gt, GUC_STATUS);
 	if (!(guc_status & GS_MIA_IN_RESET)) {
-		drm_err(&xe->drm,
-			"GuC status: 0x%x, MIA core expected to be in reset\n",
-			guc_status);
+		xe_gt_err(gt, "GuC status: %#x, MIA core expected to be in reset\n",
+			  guc_status);
 		ret = -EIO;
 		goto err_out;
 	}
@@ -463,7 +453,7 @@ static int guc_xfer_rsa(struct xe_guc *guc)
 
 static int guc_wait_ucode(struct xe_guc *guc)
 {
-	struct xe_device *xe = guc_to_xe(guc);
+	struct xe_gt *gt = guc_to_gt(guc);
 	u32 status;
 	int ret;
 
@@ -484,35 +474,32 @@ static int guc_wait_ucode(struct xe_guc *guc)
 	 * 200ms. Even at slowest clock, this should be sufficient. And
 	 * in the working case, a larger timeout makes no difference.
 	 */
-	ret = xe_mmio_wait32(guc_to_gt(guc), GUC_STATUS, GS_UKERNEL_MASK,
+	ret = xe_mmio_wait32(gt, GUC_STATUS, GS_UKERNEL_MASK,
 			     FIELD_PREP(GS_UKERNEL_MASK, XE_GUC_LOAD_STATUS_READY),
 			     200000, &status, false);
 
 	if (ret) {
-		struct drm_device *drm = &xe->drm;
-
-		drm_info(drm, "GuC load failed: status = 0x%08X\n", status);
-		drm_info(drm, "GuC load failed: status: Reset = %d, BootROM = 0x%02X, UKernel = 0x%02X, MIA = 0x%02X, Auth = 0x%02X\n",
-			 REG_FIELD_GET(GS_MIA_IN_RESET, status),
-			 REG_FIELD_GET(GS_BOOTROM_MASK, status),
-			 REG_FIELD_GET(GS_UKERNEL_MASK, status),
-			 REG_FIELD_GET(GS_MIA_MASK, status),
-			 REG_FIELD_GET(GS_AUTH_STATUS_MASK, status));
+		xe_gt_info(gt, "GuC load failed: status = 0x%08X\n", status);
+		xe_gt_info(gt, "GuC status: Reset = %u, BootROM = %#X, UKernel = %#X, MIA = %#X, Auth = %#X\n",
+			   REG_FIELD_GET(GS_MIA_IN_RESET, status),
+			   REG_FIELD_GET(GS_BOOTROM_MASK, status),
+			   REG_FIELD_GET(GS_UKERNEL_MASK, status),
+			   REG_FIELD_GET(GS_MIA_MASK, status),
+			   REG_FIELD_GET(GS_AUTH_STATUS_MASK, status));
 
 		if ((status & GS_BOOTROM_MASK) == GS_BOOTROM_RSA_FAILED) {
-			drm_info(drm, "GuC firmware signature verification failed\n");
+			xe_gt_info(gt, "GuC firmware signature verification failed\n");
 			ret = -ENOEXEC;
 		}
 
 		if (REG_FIELD_GET(GS_UKERNEL_MASK, status) ==
 		    XE_GUC_LOAD_STATUS_EXCEPTION) {
-			drm_info(drm, "GuC firmware exception. EIP: %#x\n",
-				 xe_mmio_read32(guc_to_gt(guc),
-						SOFT_SCRATCH(13)));
+			xe_gt_info(gt, "GuC firmware exception. EIP: %#x\n",
+				   xe_mmio_read32(gt, SOFT_SCRATCH(13)));
 			ret = -ENXIO;
 		}
 	} else {
-		drm_dbg(&xe->drm, "GuC successfully loaded");
+		xe_gt_dbg(gt, "GuC successfully loaded\n");
 	}
 
 	return ret;
@@ -604,6 +591,9 @@ static void guc_handle_mmio_msg(struct xe_guc *guc)
 	struct xe_gt *gt = guc_to_gt(guc);
 	u32 msg;
 
+	if (IS_SRIOV_VF(guc_to_xe(guc)))
+		return;
+
 	xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
 
 	msg = xe_mmio_read32(gt, SOFT_SCRATCH(15));
@@ -612,12 +602,10 @@ static void guc_handle_mmio_msg(struct xe_guc *guc)
 	xe_mmio_write32(gt, SOFT_SCRATCH(15), 0);
 
 	if (msg & XE_GUC_RECV_MSG_CRASH_DUMP_POSTED)
-		drm_err(&guc_to_xe(guc)->drm,
-			"Received early GuC crash dump notification!\n");
+		xe_gt_err(gt, "Received early GuC crash dump notification!\n");
 
 	if (msg & XE_GUC_RECV_MSG_EXCEPTION)
-		drm_err(&guc_to_xe(guc)->drm,
-			"Received early GuC exception notification!\n");
+		xe_gt_err(gt, "Received early GuC exception notification!\n");
 }
 
 static void guc_enable_irq(struct xe_guc *guc)
@@ -668,15 +656,15 @@ int xe_guc_enable_communication(struct xe_guc *guc)
 
 int xe_guc_suspend(struct xe_guc *guc)
 {
-	int ret;
+	struct xe_gt *gt = guc_to_gt(guc);
 	u32 action[] = {
 		XE_GUC_ACTION_CLIENT_SOFT_RESET,
 	};
+	int ret;
 
 	ret = xe_guc_mmio_send(guc, action, ARRAY_SIZE(action));
 	if (ret) {
-		drm_err(&guc_to_xe(guc)->drm,
-			"GuC suspend: CLIENT_SOFT_RESET fail: %d!\n", ret);
+		xe_gt_err(gt, "GuC suspend failed: %pe\n", ERR_PTR(ret));
 		return ret;
 	}
 
@@ -751,8 +739,8 @@ retry:
 			     50000, &reply, false);
 	if (ret) {
 timeout:
-		drm_err(&xe->drm, "mmio request %#x: no reply %#x\n",
-			request[0], reply);
+		xe_gt_err(gt, "GuC mmio request %#x: no reply %#x\n",
+			  request[0], reply);
 		return ret;
 	}
 
@@ -790,8 +778,8 @@ timeout:
 	    GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
 		u32 reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, header);
 
-		drm_dbg(&xe->drm, "mmio request %#x: retrying, reason %#x\n",
-			request[0], reason);
+		xe_gt_dbg(gt, "GuC mmio request %#x: retrying, reason %#x\n",
+			  request[0], reason);
 		goto retry;
 	}
 
@@ -800,16 +788,16 @@ timeout:
 		u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header);
 		u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header);
 
-		drm_err(&xe->drm, "mmio request %#x: failure %#x/%#x\n",
-			request[0], error, hint);
+		xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n",
+			  request[0], error, hint);
 		return -ENXIO;
 	}
 
 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) !=
 	    GUC_HXG_TYPE_RESPONSE_SUCCESS) {
 proto:
-		drm_err(&xe->drm, "mmio request %#x: unexpected reply %#x\n",
-			request[0], header);
+		xe_gt_err(gt, "GuC mmio request %#x: unexpected reply %#x\n",
+			  request[0], header);
 		return -EPROTO;
 	}
 
diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c
index 6ad4c1a90a78..7f5a523795c8 100644
--- a/drivers/gpu/drm/xe/xe_guc_ads.c
+++ b/drivers/gpu/drm/xe/xe_guc_ads.c
@@ -7,6 +7,8 @@
 
 #include <drm/drm_managed.h>
 
+#include <generated/xe_wa_oob.h>
+
 #include "regs/xe_engine_regs.h"
 #include "regs/xe_gt_regs.h"
 #include "regs/xe_guc_regs.h"
@@ -19,6 +21,7 @@
 #include "xe_map.h"
 #include "xe_mmio.h"
 #include "xe_platform_types.h"
+#include "xe_wa.h"
 
 /* Slack of a few additional entries per engine */
 #define ADS_REGSET_EXTRA_MAX	8
@@ -80,6 +83,10 @@ ads_to_map(struct xe_guc_ads *ads)
  *      +---------------------------------------+
  *      | padding                               |
  *      +---------------------------------------+ <== 4K aligned
+ *      | w/a KLVs                              |
+ *      +---------------------------------------+
+ *      | padding                               |
+ *      +---------------------------------------+ <== 4K aligned
  *      | capture lists                         |
  *      +---------------------------------------+
  *      | padding                               |
@@ -100,7 +107,7 @@ struct __guc_ads_blob {
 	struct guc_engine_usage engine_usage;
 	struct guc_um_init_params um_init_params;
 	/* From here on, location is dynamic! Refer to above diagram. */
-	struct guc_mmio_reg regset[0];
+	struct guc_mmio_reg regset[];
 } __packed;
 
 #define ads_blob_read(ads_, field_) \
@@ -131,6 +138,11 @@ static size_t guc_ads_golden_lrc_size(struct xe_guc_ads *ads)
 	return PAGE_ALIGN(ads->golden_lrc_size);
 }
 
+static u32 guc_ads_waklv_size(struct xe_guc_ads *ads)
+{
+	return PAGE_ALIGN(ads->ads_waklv_size);
+}
+
 static size_t guc_ads_capture_size(struct xe_guc_ads *ads)
 {
 	/* FIXME: Allocate a proper capture list */
@@ -167,12 +179,22 @@ static size_t guc_ads_golden_lrc_offset(struct xe_guc_ads *ads)
 	return PAGE_ALIGN(offset);
 }
 
+static size_t guc_ads_waklv_offset(struct xe_guc_ads *ads)
+{
+	u32 offset;
+
+	offset = guc_ads_golden_lrc_offset(ads) +
+		 guc_ads_golden_lrc_size(ads);
+
+	return PAGE_ALIGN(offset);
+}
+
 static size_t guc_ads_capture_offset(struct xe_guc_ads *ads)
 {
 	size_t offset;
 
-	offset = guc_ads_golden_lrc_offset(ads) +
-		guc_ads_golden_lrc_size(ads);
+	offset = guc_ads_waklv_offset(ads) +
+		 guc_ads_waklv_size(ads);
 
 	return PAGE_ALIGN(offset);
 }
@@ -260,6 +282,110 @@ static size_t calculate_golden_lrc_size(struct xe_guc_ads *ads)
 	return total_size;
 }
 
+static void guc_waklv_enable_one_word(struct xe_guc_ads *ads,
+				      enum xe_guc_klv_ids klv_id,
+				      u32 value,
+				      u32 *offset, u32 *remain)
+{
+	u32 size;
+	u32 klv_entry[] = {
+		/* 16:16 key/length */
+		FIELD_PREP(GUC_KLV_0_KEY, klv_id) |
+		FIELD_PREP(GUC_KLV_0_LEN, 1),
+		value,
+		/* 1 dword data */
+	};
+
+	size = sizeof(klv_entry);
+
+	if (*remain < size) {
+		drm_warn(&ads_to_xe(ads)->drm,
+			 "w/a klv buffer too small to add klv id %d\n", klv_id);
+	} else {
+		xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset,
+				 klv_entry, size);
+		*offset += size;
+		*remain -= size;
+	}
+}
+
+static void guc_waklv_enable_simple(struct xe_guc_ads *ads,
+				    enum xe_guc_klv_ids klv_id, u32 *offset, u32 *remain)
+{
+	u32 klv_entry[] = {
+		/* 16:16 key/length */
+		FIELD_PREP(GUC_KLV_0_KEY, klv_id) |
+		FIELD_PREP(GUC_KLV_0_LEN, 0),
+		/* 0 dwords data */
+	};
+	u32 size;
+
+	size = sizeof(klv_entry);
+
+	if (xe_gt_WARN(ads_to_gt(ads), *remain < size,
+		       "w/a klv buffer too small to add klv id %d\n", klv_id))
+		return;
+
+	xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset,
+			 klv_entry, size);
+	*offset += size;
+	*remain -= size;
+}
+
+static void guc_waklv_init(struct xe_guc_ads *ads)
+{
+	struct xe_gt *gt = ads_to_gt(ads);
+	u64 addr_ggtt;
+	u32 offset, remain, size;
+
+	offset = guc_ads_waklv_offset(ads);
+	remain = guc_ads_waklv_size(ads);
+
+	if (XE_WA(gt, 14019882105))
+		guc_waklv_enable_simple(ads,
+					GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED,
+					&offset, &remain);
+	if (XE_WA(gt, 18024947630))
+		guc_waklv_enable_simple(ads,
+					GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING,
+					&offset, &remain);
+	if (XE_WA(gt, 16022287689))
+		guc_waklv_enable_simple(ads,
+					GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE,
+					&offset, &remain);
+
+	/*
+	 * On RC6 exit, GuC will write register 0xB04 with the default value provided. As of now,
+	 * the default value for this register is determined to be 0xC40. This could change in the
+	 * future, so GuC depends on KMD to send it the correct value.
+	 */
+	if (XE_WA(gt, 13011645652))
+		guc_waklv_enable_one_word(ads,
+					  GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE,
+					  0xC40,
+					  &offset, &remain);
+
+	size = guc_ads_waklv_size(ads) - remain;
+	if (!size)
+		return;
+
+	offset = guc_ads_waklv_offset(ads);
+	addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset;
+
+	ads_blob_write(ads, ads.wa_klv_addr_lo, lower_32_bits(addr_ggtt));
+	ads_blob_write(ads, ads.wa_klv_addr_hi, upper_32_bits(addr_ggtt));
+	ads_blob_write(ads, ads.wa_klv_size, size);
+}
+
+static int calculate_waklv_size(struct xe_guc_ads *ads)
+{
+	/*
+	 * A single page is both the minimum size possible and
+	 * is sufficiently large enough for all current platforms.
+	 */
+	return SZ_4K;
+}
+
 #define MAX_GOLDEN_LRC_SIZE	(SZ_4K * 64)
 
 int xe_guc_ads_init(struct xe_guc_ads *ads)
@@ -271,10 +397,12 @@ int xe_guc_ads_init(struct xe_guc_ads *ads)
 
 	ads->golden_lrc_size = calculate_golden_lrc_size(ads);
 	ads->regset_size = calculate_regset_size(gt);
+	ads->ads_waklv_size = calculate_waklv_size(ads);
 
 	bo = xe_managed_bo_create_pin_map(xe, tile, guc_ads_size(ads) + MAX_GOLDEN_LRC_SIZE,
-					  XE_BO_CREATE_SYSTEM_BIT |
-					  XE_BO_CREATE_GGTT_BIT);
+					  XE_BO_FLAG_SYSTEM |
+					  XE_BO_FLAG_GGTT |
+					  XE_BO_FLAG_GGTT_INVALIDATE);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
@@ -597,6 +725,7 @@ void xe_guc_ads_populate(struct xe_guc_ads *ads)
 	guc_mapping_table_init(gt, &info_map);
 	guc_capture_list_init(ads);
 	guc_doorbell_init(ads);
+	guc_waklv_init(ads);
 
 	if (xe->info.has_usm) {
 		guc_um_init_params(ads);
diff --git a/drivers/gpu/drm/xe/xe_guc_ads_types.h b/drivers/gpu/drm/xe/xe_guc_ads_types.h
index 4afe44bece4b..2de5decfe0fd 100644
--- a/drivers/gpu/drm/xe/xe_guc_ads_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_ads_types.h
@@ -20,6 +20,8 @@ struct xe_guc_ads {
 	size_t golden_lrc_size;
 	/** @regset_size: size of register set passed to GuC for save/restore */
 	u32 regset_size;
+	/** @ads_waklv_size: total waklv size supported by platform */
+	u32 ads_waklv_size;
 };
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c
index 355edd4d758a..0151d29b3c58 100644
--- a/drivers/gpu/drm/xe/xe_guc_ct.c
+++ b/drivers/gpu/drm/xe/xe_guc_ct.c
@@ -21,6 +21,7 @@
 #include "xe_gt.h"
 #include "xe_gt_pagefault.h"
 #include "xe_gt_printk.h"
+#include "xe_gt_sriov_pf_control.h"
 #include "xe_gt_tlb_invalidation.h"
 #include "xe_guc.h"
 #include "xe_guc_relay.h"
@@ -120,6 +121,7 @@ static void guc_ct_fini(struct drm_device *drm, void *arg)
 {
 	struct xe_guc_ct *ct = arg;
 
+	destroy_workqueue(ct->g2h_wq);
 	xa_destroy(&ct->fence_lookup);
 }
 
@@ -143,20 +145,28 @@ int xe_guc_ct_init(struct xe_guc_ct *ct)
 	struct xe_bo *bo;
 	int err;
 
-	xe_assert(xe, !(guc_ct_size() % PAGE_SIZE));
+	xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE));
+
+	ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", 0);
+	if (!ct->g2h_wq)
+		return -ENOMEM;
 
-	drmm_mutex_init(&xe->drm, &ct->lock);
 	spin_lock_init(&ct->fast_lock);
 	xa_init(&ct->fence_lookup);
 	INIT_WORK(&ct->g2h_worker, g2h_worker_func);
 	init_waitqueue_head(&ct->wq);
 	init_waitqueue_head(&ct->g2h_fence_wq);
 
+	err = drmm_mutex_init(&xe->drm, &ct->lock);
+	if (err)
+		return err;
+
 	primelockdep(ct);
 
 	bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(),
-					  XE_BO_CREATE_SYSTEM_BIT |
-					  XE_BO_CREATE_GGTT_BIT);
+					  XE_BO_FLAG_SYSTEM |
+					  XE_BO_FLAG_GGTT |
+					  XE_BO_FLAG_GGTT_INVALIDATE);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
@@ -166,7 +176,7 @@ int xe_guc_ct_init(struct xe_guc_ct *ct)
 	if (err)
 		return err;
 
-	xe_assert(xe, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED);
+	xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED);
 	ct->state = XE_GUC_CT_STATE_DISABLED;
 	return 0;
 }
@@ -313,9 +323,10 @@ static void xe_guc_ct_set_state(struct xe_guc_ct *ct,
 int xe_guc_ct_enable(struct xe_guc_ct *ct)
 {
 	struct xe_device *xe = ct_to_xe(ct);
+	struct xe_gt *gt = ct_to_gt(ct);
 	int err;
 
-	xe_assert(xe, !xe_guc_ct_enabled(ct));
+	xe_gt_assert(gt, !xe_guc_ct_enabled(ct));
 
 	guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap);
 	guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap);
@@ -336,12 +347,12 @@ int xe_guc_ct_enable(struct xe_guc_ct *ct)
 
 	smp_mb();
 	wake_up_all(&ct->wq);
-	drm_dbg(&xe->drm, "GuC CT communication channel enabled\n");
+	xe_gt_dbg(gt, "GuC CT communication channel enabled\n");
 
 	return 0;
 
 err_out:
-	drm_err(&xe->drm, "Failed to enable CT (%d)\n", err);
+	xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err));
 
 	return err;
 }
@@ -422,7 +433,7 @@ static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len)
 
 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h)
 {
-	xe_assert(ct_to_xe(ct), g2h_len <= ct->ctbs.g2h.info.space);
+	xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space);
 
 	if (g2h_len) {
 		lockdep_assert_held(&ct->fast_lock);
@@ -435,8 +446,8 @@ static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h)
 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
 {
 	lockdep_assert_held(&ct->fast_lock);
-	xe_assert(ct_to_xe(ct), ct->ctbs.g2h.info.space + g2h_len <=
-		  ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space);
+	xe_gt_assert(ct_to_gt(ct), ct->ctbs.g2h.info.space + g2h_len <=
+		     ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space);
 
 	ct->ctbs.g2h.info.space += g2h_len;
 	--ct->g2h_outstanding;
@@ -455,6 +466,7 @@ static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
 		     u32 ct_fence_value, bool want_response)
 {
 	struct xe_device *xe = ct_to_xe(ct);
+	struct xe_gt *gt = ct_to_gt(ct);
 	struct guc_ctb *h2g = &ct->ctbs.h2g;
 	u32 cmd[H2G_CT_HEADERS];
 	u32 tail = h2g->info.tail;
@@ -465,8 +477,8 @@ static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
 	full_len = len + GUC_CTB_HDR_LEN;
 
 	lockdep_assert_held(&ct->lock);
-	xe_assert(xe, full_len <= GUC_CTB_MSG_MAX_LEN);
-	xe_assert(xe, tail <= h2g->info.size);
+	xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN);
+	xe_gt_assert(gt, tail <= h2g->info.size);
 
 	/* Command will wrap, zero fill (NOPs), return and check credits again */
 	if (tail + full_len > h2g->info.size) {
@@ -515,7 +527,7 @@ static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
 	/* Update descriptor */
 	desc_write(xe, h2g, tail, h2g->info.tail);
 
-	trace_xe_guc_ctb_h2g(ct_to_gt(ct)->info.id, *(action - 1), full_len,
+	trace_xe_guc_ctb_h2g(gt->info.id, *(action - 1), full_len,
 			     desc_read(xe, h2g, head), h2g->info.tail);
 
 	return 0;
@@ -544,15 +556,15 @@ static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,
 				u32 len, u32 g2h_len, u32 num_g2h,
 				struct g2h_fence *g2h_fence)
 {
-	struct xe_device *xe = ct_to_xe(ct);
+	struct xe_gt *gt __maybe_unused = ct_to_gt(ct);
 	u16 seqno;
 	int ret;
 
-	xe_assert(xe, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED);
-	xe_assert(xe, !g2h_len || !g2h_fence);
-	xe_assert(xe, !num_g2h || !g2h_fence);
-	xe_assert(xe, !g2h_len || num_g2h);
-	xe_assert(xe, g2h_len || !num_g2h);
+	xe_gt_assert(gt, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED);
+	xe_gt_assert(gt, !g2h_len || !g2h_fence);
+	xe_gt_assert(gt, !num_g2h || !g2h_fence);
+	xe_gt_assert(gt, !g2h_len || num_g2h);
+	xe_gt_assert(gt, g2h_len || !num_g2h);
 	lockdep_assert_held(&ct->lock);
 
 	if (unlikely(ct->ctbs.h2g.info.broken)) {
@@ -570,7 +582,7 @@ static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,
 		goto out;
 	}
 
-	xe_assert(xe, xe_guc_ct_enabled(ct));
+	xe_gt_assert(gt, xe_guc_ct_enabled(ct));
 
 	if (g2h_fence) {
 		g2h_len = GUC_CTB_HXG_MSG_MAX_LEN;
@@ -628,12 +640,12 @@ static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
 			      u32 g2h_len, u32 num_g2h,
 			      struct g2h_fence *g2h_fence)
 {
-	struct drm_device *drm = &ct_to_xe(ct)->drm;
-	struct drm_printer p = drm_info_printer(drm->dev);
+	struct xe_gt *gt = ct_to_gt(ct);
+	struct drm_printer p = xe_gt_info_printer(gt);
 	unsigned int sleep_period_ms = 1;
 	int ret;
 
-	xe_assert(ct_to_xe(ct), !g2h_len || !g2h_fence);
+	xe_gt_assert(gt, !g2h_len || !g2h_fence);
 	lockdep_assert_held(&ct->lock);
 	xe_device_assert_mem_access(ct_to_xe(ct));
 
@@ -691,7 +703,7 @@ try_again:
 	return ret;
 
 broken:
-	drm_err(drm, "No forward process on H2G, reset required");
+	xe_gt_err(gt, "No forward process on H2G, reset required\n");
 	xe_guc_ct_print(ct, &p, true);
 	ct->ctbs.h2g.info.broken = true;
 
@@ -703,7 +715,7 @@ static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
 {
 	int ret;
 
-	xe_assert(ct_to_xe(ct), !g2h_len || !g2h_fence);
+	xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence);
 
 	mutex_lock(&ct->lock);
 	ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence);
@@ -771,7 +783,7 @@ static bool retry_failure(struct xe_guc_ct *ct, int ret)
 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
 			    u32 *response_buffer, bool no_fail)
 {
-	struct xe_device *xe = ct_to_xe(ct);
+	struct xe_gt *gt = ct_to_gt(ct);
 	struct g2h_fence g2h_fence;
 	int ret = 0;
 
@@ -813,20 +825,20 @@ retry_same_fence:
 
 	ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ);
 	if (!ret) {
-		drm_err(&xe->drm, "Timed out wait for G2H, fence %u, action %04x",
-			g2h_fence.seqno, action[0]);
+		xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x",
+			  g2h_fence.seqno, action[0]);
 		xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno);
 		return -ETIME;
 	}
 
 	if (g2h_fence.retry) {
-		drm_warn(&xe->drm, "Send retry, action 0x%04x, reason %d",
-			 action[0], g2h_fence.reason);
+		xe_gt_warn(gt, "H2G retry, action 0x%04x, reason %u",
+			   action[0], g2h_fence.reason);
 		goto retry;
 	}
 	if (g2h_fence.fail) {
-		drm_err(&xe->drm, "Send failed, action 0x%04x, error %d, hint %d",
-			action[0], g2h_fence.error, g2h_fence.hint);
+		xe_gt_err(gt, "H2G send failed, action 0x%04x, error %d, hint %u",
+			  action[0], g2h_fence.error, g2h_fence.hint);
 		ret = -EIO;
 	}
 
@@ -895,7 +907,6 @@ static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len)
 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
 {
 	struct xe_gt *gt =  ct_to_gt(ct);
-	struct xe_device *xe = gt_to_xe(gt);
 	u32 *hxg = msg_to_hxg(msg);
 	u32 hxg_len = msg_len_to_hxg_len(len);
 	u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]);
@@ -933,7 +944,7 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
 		return 0;
 	}
 
-	xe_assert(xe, fence == g2h_fence->seqno);
+	xe_gt_assert(gt, fence == g2h_fence->seqno);
 
 	if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) {
 		g2h_fence->fail = true;
@@ -961,7 +972,7 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
 
 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
 {
-	struct xe_device *xe = ct_to_xe(ct);
+	struct xe_gt *gt = ct_to_gt(ct);
 	u32 *hxg = msg_to_hxg(msg);
 	u32 origin, type;
 	int ret;
@@ -970,9 +981,8 @@ static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
 
 	origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);
 	if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
-		drm_err(&xe->drm,
-			"G2H channel broken on read, origin=%d, reset required\n",
-			origin);
+		xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n",
+			  origin);
 		ct->ctbs.g2h.info.broken = true;
 
 		return -EPROTO;
@@ -989,9 +999,8 @@ static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
 		ret = parse_g2h_response(ct, msg, len);
 		break;
 	default:
-		drm_err(&xe->drm,
-			"G2H channel broken on read, type=%d, reset required\n",
-			type);
+		xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n",
+			  type);
 		ct->ctbs.g2h.info.broken = true;
 
 		ret = -EOPNOTSUPP;
@@ -1002,8 +1011,8 @@ static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
 
 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
 {
-	struct xe_device *xe = ct_to_xe(ct);
 	struct xe_guc *guc = ct_to_guc(ct);
+	struct xe_gt *gt = ct_to_gt(ct);
 	u32 hxg_len = msg_len_to_hxg_len(len);
 	u32 *hxg = msg_to_hxg(msg);
 	u32 action, adj_len;
@@ -1054,18 +1063,21 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
 							   adj_len);
 		break;
 	case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:
-		ret = xe_guc_relay_process_guc2pf(&guc->relay, payload, adj_len);
+		ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len);
 		break;
 	case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF:
-		ret = xe_guc_relay_process_guc2vf(&guc->relay, payload, adj_len);
+		ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len);
+		break;
+	case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY:
+		ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len);
 		break;
 	default:
-		drm_err(&xe->drm, "unexpected action 0x%04x\n", action);
+		xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action);
 	}
 
 	if (ret)
-		drm_err(&xe->drm, "action 0x%04x failed processing, ret=%d\n",
-			action, ret);
+		xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n",
+			  action, ERR_PTR(ret));
 
 	return 0;
 }
@@ -1073,13 +1085,14 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)
 {
 	struct xe_device *xe = ct_to_xe(ct);
+	struct xe_gt *gt = ct_to_gt(ct);
 	struct guc_ctb *g2h = &ct->ctbs.g2h;
 	u32 tail, head, len;
 	s32 avail;
 	u32 action;
 	u32 *hxg;
 
-	xe_assert(xe, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED);
+	xe_gt_assert(gt, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED);
 	lockdep_assert_held(&ct->fast_lock);
 
 	if (ct->state == XE_GUC_CT_STATE_DISABLED)
@@ -1091,7 +1104,7 @@ static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)
 	if (g2h->info.broken)
 		return -EPIPE;
 
-	xe_assert(xe, xe_guc_ct_enabled(ct));
+	xe_gt_assert(gt, xe_guc_ct_enabled(ct));
 
 	/* Calculate DW available to read */
 	tail = desc_read(xe, g2h, tail);
@@ -1107,9 +1120,8 @@ static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)
 			   sizeof(u32));
 	len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN;
 	if (len > avail) {
-		drm_err(&xe->drm,
-			"G2H channel broken on read, avail=%d, len=%d, reset required\n",
-			avail, len);
+		xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n",
+			  avail, len);
 		g2h->info.broken = true;
 
 		return -EPROTO;
@@ -1162,7 +1174,7 @@ static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)
 
 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len)
 {
-	struct xe_device *xe = ct_to_xe(ct);
+	struct xe_gt *gt = ct_to_gt(ct);
 	struct xe_guc *guc = ct_to_guc(ct);
 	u32 hxg_len = msg_len_to_hxg_len(len);
 	u32 *hxg = msg_to_hxg(msg);
@@ -1181,12 +1193,12 @@ static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len)
 							   adj_len);
 		break;
 	default:
-		drm_warn(&xe->drm, "NOT_POSSIBLE");
+		xe_gt_warn(gt, "NOT_POSSIBLE");
 	}
 
 	if (ret)
-		drm_err(&xe->drm, "action 0x%04x failed processing, ret=%d\n",
-			action, ret);
+		xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n",
+			  action, ERR_PTR(ret));
 }
 
 /**
@@ -1203,7 +1215,7 @@ void xe_guc_ct_fast_path(struct xe_guc_ct *ct)
 	bool ongoing;
 	int len;
 
-	ongoing = xe_device_mem_access_get_if_ongoing(ct_to_xe(ct));
+	ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));
 	if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
 		return;
 
@@ -1216,7 +1228,7 @@ void xe_guc_ct_fast_path(struct xe_guc_ct *ct)
 	spin_unlock(&ct->fast_lock);
 
 	if (ongoing)
-		xe_device_mem_access_put(xe);
+		xe_pm_runtime_put(xe);
 }
 
 /* Returns less than zero on error, 0 on done, 1 on more available */
@@ -1247,6 +1259,7 @@ static int dequeue_one_g2h(struct xe_guc_ct *ct)
 static void g2h_worker_func(struct work_struct *w)
 {
 	struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker);
+	struct xe_gt *gt = ct_to_gt(ct);
 	bool ongoing;
 	int ret;
 
@@ -1273,7 +1286,7 @@ static void g2h_worker_func(struct work_struct *w)
 	 * responses, if the worker here is blocked on those callbacks
 	 * completing, creating a deadlock.
 	 */
-	ongoing = xe_device_mem_access_get_if_ongoing(ct_to_xe(ct));
+	ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));
 	if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
 		return;
 
@@ -1283,8 +1296,7 @@ static void g2h_worker_func(struct work_struct *w)
 		mutex_unlock(&ct->lock);
 
 		if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) {
-			struct drm_device *drm = &ct_to_xe(ct)->drm;
-			struct drm_printer p = drm_info_printer(drm->dev);
+			struct drm_printer p = xe_gt_info_printer(gt);
 
 			xe_guc_ct_print(ct, &p, false);
 			kick_reset(ct);
@@ -1292,7 +1304,7 @@ static void g2h_worker_func(struct work_struct *w)
 	} while (ret == 1);
 
 	if (ongoing)
-		xe_device_mem_access_put(ct_to_xe(ct));
+		xe_pm_runtime_put(ct_to_xe(ct));
 }
 
 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb,
@@ -1394,7 +1406,7 @@ struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct,
 		return NULL;
 	}
 
-	if (xe_guc_ct_enabled(ct)) {
+	if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) {
 		snapshot->ct_enabled = true;
 		snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding);
 		guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g,
diff --git a/drivers/gpu/drm/xe/xe_guc_ct.h b/drivers/gpu/drm/xe/xe_guc_ct.h
index 5083e099064f..105bb8e99a8d 100644
--- a/drivers/gpu/drm/xe/xe_guc_ct.h
+++ b/drivers/gpu/drm/xe/xe_guc_ct.h
@@ -34,7 +34,7 @@ static inline void xe_guc_ct_irq_handler(struct xe_guc_ct *ct)
 		return;
 
 	wake_up_all(&ct->wq);
-	queue_work(system_unbound_wq, &ct->g2h_worker);
+	queue_work(ct->g2h_wq, &ct->g2h_worker);
 	xe_guc_ct_fast_path(ct);
 }
 
diff --git a/drivers/gpu/drm/xe/xe_guc_ct_types.h b/drivers/gpu/drm/xe/xe_guc_ct_types.h
index d29144c9f20b..fede4c6e93cb 100644
--- a/drivers/gpu/drm/xe/xe_guc_ct_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_ct_types.h
@@ -120,6 +120,8 @@ struct xe_guc_ct {
 	wait_queue_head_t wq;
 	/** @g2h_fence_wq: wait queue used for G2H fencing */
 	wait_queue_head_t g2h_fence_wq;
+	/** @g2h_wq: used to process G2H */
+	struct workqueue_struct *g2h_wq;
 	/** @msg: Message buffer */
 	u32 msg[GUC_CTB_MSG_MAX_LEN];
 	/** @fast_msg: Message buffer */
diff --git a/drivers/gpu/drm/xe/xe_guc_debugfs.c b/drivers/gpu/drm/xe/xe_guc_debugfs.c
index ffd7d53bcc42..d3822cbea273 100644
--- a/drivers/gpu/drm/xe/xe_guc_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_guc_debugfs.c
@@ -14,6 +14,7 @@
 #include "xe_guc_ct.h"
 #include "xe_guc_log.h"
 #include "xe_macros.h"
+#include "xe_pm.h"
 
 static struct xe_guc *node_to_guc(struct drm_info_node *node)
 {
@@ -26,9 +27,9 @@ static int guc_info(struct seq_file *m, void *data)
 	struct xe_device *xe = guc_to_xe(guc);
 	struct drm_printer p = drm_seq_file_printer(m);
 
-	xe_device_mem_access_get(xe);
+	xe_pm_runtime_get(xe);
 	xe_guc_print_info(guc, &p);
-	xe_device_mem_access_put(xe);
+	xe_pm_runtime_put(xe);
 
 	return 0;
 }
@@ -39,9 +40,9 @@ static int guc_log(struct seq_file *m, void *data)
 	struct xe_device *xe = guc_to_xe(guc);
 	struct drm_printer p = drm_seq_file_printer(m);
 
-	xe_device_mem_access_get(xe);
+	xe_pm_runtime_get(xe);
 	xe_guc_log_print(&guc->log, &p);
-	xe_device_mem_access_put(xe);
+	xe_pm_runtime_put(xe);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h
index c281fdbfd2d6..19ee71aeaf17 100644
--- a/drivers/gpu/drm/xe/xe_guc_fwif.h
+++ b/drivers/gpu/drm/xe/xe_guc_fwif.h
@@ -14,6 +14,8 @@
 #define G2H_LEN_DW_DEREGISTER_CONTEXT		3
 #define G2H_LEN_DW_TLB_INVALIDATE		3
 
+#define GUC_ID_MAX			65535
+
 #define GUC_CONTEXT_DISABLE		0
 #define GUC_CONTEXT_ENABLE		1
 
@@ -207,7 +209,10 @@ struct guc_ads {
 	u32 capture_instance[GUC_CAPTURE_LIST_INDEX_MAX][GUC_MAX_ENGINE_CLASSES];
 	u32 capture_class[GUC_CAPTURE_LIST_INDEX_MAX][GUC_MAX_ENGINE_CLASSES];
 	u32 capture_global[GUC_CAPTURE_LIST_INDEX_MAX];
-	u32 reserved[14];
+	u32 wa_klv_addr_lo;
+	u32 wa_klv_addr_hi;
+	u32 wa_klv_size;
+	u32 reserved[11];
 } __packed;
 
 /* Engine usage stats */
diff --git a/drivers/gpu/drm/xe/xe_guc_hwconfig.c b/drivers/gpu/drm/xe/xe_guc_hwconfig.c
index ea49f3885c10..d9b570a154a2 100644
--- a/drivers/gpu/drm/xe/xe_guc_hwconfig.c
+++ b/drivers/gpu/drm/xe/xe_guc_hwconfig.c
@@ -14,7 +14,7 @@
 #include "xe_guc.h"
 #include "xe_map.h"
 
-static int send_get_hwconfig(struct xe_guc *guc, u32 ggtt_addr, u32 size)
+static int send_get_hwconfig(struct xe_guc *guc, u64 ggtt_addr, u32 size)
 {
 	u32 action[] = {
 		XE_GUC_ACTION_GET_HWCONFIG,
@@ -78,8 +78,9 @@ int xe_guc_hwconfig_init(struct xe_guc *guc)
 		return -EINVAL;
 
 	bo = xe_managed_bo_create_pin_map(xe, tile, PAGE_ALIGN(size),
-					  XE_BO_CREATE_SYSTEM_BIT |
-					  XE_BO_CREATE_GGTT_BIT);
+					  XE_BO_FLAG_SYSTEM |
+					  XE_BO_FLAG_GGTT |
+					  XE_BO_FLAG_GGTT_INVALIDATE);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 	guc->hwconfig.bo = bo;
diff --git a/drivers/gpu/drm/xe/xe_guc_id_mgr.c b/drivers/gpu/drm/xe/xe_guc_id_mgr.c
new file mode 100644
index 000000000000..0fb7c6b78c31
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_guc_id_mgr.c
@@ -0,0 +1,279 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#include <linux/bitmap.h>
+#include <linux/mutex.h>
+
+#include <drm/drm_managed.h>
+
+#include "xe_assert.h"
+#include "xe_gt_printk.h"
+#include "xe_guc.h"
+#include "xe_guc_id_mgr.h"
+#include "xe_guc_types.h"
+
+static struct xe_guc *idm_to_guc(struct xe_guc_id_mgr *idm)
+{
+	return container_of(idm, struct xe_guc, submission_state.idm);
+}
+
+static struct xe_gt *idm_to_gt(struct xe_guc_id_mgr *idm)
+{
+	return guc_to_gt(idm_to_guc(idm));
+}
+
+static struct xe_device *idm_to_xe(struct xe_guc_id_mgr *idm)
+{
+	return gt_to_xe(idm_to_gt(idm));
+}
+
+#define idm_assert(idm, cond)		xe_gt_assert(idm_to_gt(idm), cond)
+#define idm_mutex(idm)			(&idm_to_guc(idm)->submission_state.lock)
+
+static void idm_print_locked(struct xe_guc_id_mgr *idm, struct drm_printer *p, int indent);
+
+static void __fini_idm(struct drm_device *drm, void *arg)
+{
+	struct xe_guc_id_mgr *idm = arg;
+
+	mutex_lock(idm_mutex(idm));
+
+	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
+		unsigned int weight = bitmap_weight(idm->bitmap, idm->total);
+
+		if (weight) {
+			struct drm_printer p = xe_gt_info_printer(idm_to_gt(idm));
+
+			xe_gt_err(idm_to_gt(idm), "GUC ID manager unclean (%u/%u)\n",
+				  weight, idm->total);
+			idm_print_locked(idm, &p, 1);
+		}
+	}
+
+	bitmap_free(idm->bitmap);
+	idm->bitmap = NULL;
+	idm->total = 0;
+	idm->used = 0;
+
+	mutex_unlock(idm_mutex(idm));
+}
+
+/**
+ * xe_guc_id_mgr_init() - Initialize GuC context ID Manager.
+ * @idm: the &xe_guc_id_mgr to initialize
+ * @limit: number of IDs to manage
+ *
+ * The bare-metal or PF driver can pass ~0 as &limit to indicate that all
+ * context IDs supported by the GuC firmware are available for use.
+ *
+ * Only VF drivers will have to provide explicit number of context IDs
+ * that they can use.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_guc_id_mgr_init(struct xe_guc_id_mgr *idm, unsigned int limit)
+{
+	int ret;
+
+	idm_assert(idm, !idm->bitmap);
+	idm_assert(idm, !idm->total);
+	idm_assert(idm, !idm->used);
+
+	if (limit == ~0)
+		limit = GUC_ID_MAX;
+	else if (limit > GUC_ID_MAX)
+		return -ERANGE;
+	else if (!limit)
+		return -EINVAL;
+
+	idm->bitmap = bitmap_zalloc(limit, GFP_KERNEL);
+	if (!idm->bitmap)
+		return -ENOMEM;
+	idm->total = limit;
+
+	ret = drmm_add_action_or_reset(&idm_to_xe(idm)->drm, __fini_idm, idm);
+	if (ret)
+		return ret;
+
+	xe_gt_info(idm_to_gt(idm), "using %u GUC ID(s)\n", idm->total);
+	return 0;
+}
+
+static unsigned int find_last_zero_area(unsigned long *bitmap,
+					unsigned int total,
+					unsigned int count)
+{
+	unsigned int found = total;
+	unsigned int rs, re, range;
+
+	for_each_clear_bitrange(rs, re, bitmap, total) {
+		range = re - rs;
+		if (range < count)
+			continue;
+		found = rs + (range - count);
+	}
+	return found;
+}
+
+static int idm_reserve_chunk_locked(struct xe_guc_id_mgr *idm,
+				    unsigned int count, unsigned int retain)
+{
+	int id;
+
+	idm_assert(idm, count);
+	lockdep_assert_held(idm_mutex(idm));
+
+	if (!idm->total)
+		return -ENODATA;
+
+	if (retain) {
+		/*
+		 * For IDs reservations (used on PF for VFs) we want to make
+		 * sure there will be at least 'retain' available for the PF
+		 */
+		if (idm->used + count + retain > idm->total)
+			return -EDQUOT;
+		/*
+		 * ... and we want to reserve highest IDs close to the end.
+		 */
+		id = find_last_zero_area(idm->bitmap, idm->total, count);
+	} else {
+		/*
+		 * For regular IDs reservations (used by submission code)
+		 * we start searching from the lower range of IDs.
+		 */
+		id = bitmap_find_next_zero_area(idm->bitmap, idm->total, 0, count, 0);
+	}
+	if (id >= idm->total)
+		return -ENOSPC;
+
+	bitmap_set(idm->bitmap, id, count);
+	idm->used += count;
+
+	return id;
+}
+
+static void idm_release_chunk_locked(struct xe_guc_id_mgr *idm,
+				     unsigned int start, unsigned int count)
+{
+	idm_assert(idm, count);
+	idm_assert(idm, count <= idm->used);
+	idm_assert(idm, start < idm->total);
+	idm_assert(idm, start + count - 1 < idm->total);
+	lockdep_assert_held(idm_mutex(idm));
+
+	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
+		unsigned int n;
+
+		for (n = 0; n < count; n++)
+			idm_assert(idm, test_bit(start + n, idm->bitmap));
+	}
+	bitmap_clear(idm->bitmap, start, count);
+	idm->used -= count;
+}
+
+/**
+ * xe_guc_id_mgr_reserve_locked() - Reserve one or more GuC context IDs.
+ * @idm: the &xe_guc_id_mgr
+ * @count: number of IDs to allocate (can't be 0)
+ *
+ * This function is dedicated for the use by the GuC submission code,
+ * where submission lock is already taken.
+ *
+ * Return: ID of allocated GuC context or a negative error code on failure.
+ */
+int xe_guc_id_mgr_reserve_locked(struct xe_guc_id_mgr *idm, unsigned int count)
+{
+	return idm_reserve_chunk_locked(idm, count, 0);
+}
+
+/**
+ * xe_guc_id_mgr_release_locked() - Release one or more GuC context IDs.
+ * @idm: the &xe_guc_id_mgr
+ * @id: the GuC context ID to release
+ * @count: number of IDs to release (can't be 0)
+ *
+ * This function is dedicated for the use by the GuC submission code,
+ * where submission lock is already taken.
+ */
+void xe_guc_id_mgr_release_locked(struct xe_guc_id_mgr *idm, unsigned int id,
+				  unsigned int count)
+{
+	return idm_release_chunk_locked(idm, id, count);
+}
+
+/**
+ * xe_guc_id_mgr_reserve() - Reserve a range of GuC context IDs.
+ * @idm: the &xe_guc_id_mgr
+ * @count: number of GuC context IDs to reserve (can't be 0)
+ * @retain: number of GuC context IDs to keep available (can't be 0)
+ *
+ * This function is dedicated for the use by the PF driver which expects that
+ * reserved range of IDs will be contiguous and that there will be at least
+ * &retain IDs still available for the PF after this reservation.
+ *
+ * Return: starting ID of the allocated GuC context ID range or
+ *         a negative error code on failure.
+ */
+int xe_guc_id_mgr_reserve(struct xe_guc_id_mgr *idm,
+			  unsigned int count, unsigned int retain)
+{
+	int ret;
+
+	idm_assert(idm, count);
+	idm_assert(idm, retain);
+
+	mutex_lock(idm_mutex(idm));
+	ret = idm_reserve_chunk_locked(idm, count, retain);
+	mutex_unlock(idm_mutex(idm));
+
+	return ret;
+}
+
+/**
+ * xe_guc_id_mgr_release() - Release a range of GuC context IDs.
+ * @idm: the &xe_guc_id_mgr
+ * @start: the starting ID of GuC context range to release
+ * @count: number of GuC context IDs to release
+ */
+void xe_guc_id_mgr_release(struct xe_guc_id_mgr *idm,
+			   unsigned int start, unsigned int count)
+{
+	mutex_lock(idm_mutex(idm));
+	idm_release_chunk_locked(idm, start, count);
+	mutex_unlock(idm_mutex(idm));
+}
+
+static void idm_print_locked(struct xe_guc_id_mgr *idm, struct drm_printer *p, int indent)
+{
+	unsigned int rs, re;
+
+	lockdep_assert_held(idm_mutex(idm));
+
+	drm_printf_indent(p, indent, "total %u\n", idm->total);
+	if (!idm->bitmap)
+		return;
+
+	drm_printf_indent(p, indent, "used %u\n", idm->used);
+	for_each_set_bitrange(rs, re, idm->bitmap, idm->total)
+		drm_printf_indent(p, indent, "range %u..%u (%u)\n", rs, re - 1, re - rs);
+}
+
+/**
+ * xe_guc_id_mgr_print() - Print status of GuC ID Manager.
+ * @idm: the &xe_guc_id_mgr to print
+ * @p: the &drm_printer to print to
+ * @indent: tab indentation level
+ */
+void xe_guc_id_mgr_print(struct xe_guc_id_mgr *idm, struct drm_printer *p, int indent)
+{
+	mutex_lock(idm_mutex(idm));
+	idm_print_locked(idm, p, indent);
+	mutex_unlock(idm_mutex(idm));
+}
+
+#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST)
+#include "tests/xe_guc_id_mgr_test.c"
+#endif
diff --git a/drivers/gpu/drm/xe/xe_guc_id_mgr.h b/drivers/gpu/drm/xe/xe_guc_id_mgr.h
new file mode 100644
index 000000000000..368f8c80e4c7
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_guc_id_mgr.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#ifndef _XE_GUC_ID_MGR_H_
+#define _XE_GUC_ID_MGR_H_
+
+struct drm_printer;
+struct xe_guc_id_mgr;
+
+int xe_guc_id_mgr_init(struct xe_guc_id_mgr *idm, unsigned int count);
+
+int xe_guc_id_mgr_reserve_locked(struct xe_guc_id_mgr *idm, unsigned int count);
+void xe_guc_id_mgr_release_locked(struct xe_guc_id_mgr *idm, unsigned int id, unsigned int count);
+
+int xe_guc_id_mgr_reserve(struct xe_guc_id_mgr *idm, unsigned int count, unsigned int retain);
+void xe_guc_id_mgr_release(struct xe_guc_id_mgr *idm, unsigned int start, unsigned int count);
+
+void xe_guc_id_mgr_print(struct xe_guc_id_mgr *idm, struct drm_printer *p, int indent);
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_guc_klv_helpers.c b/drivers/gpu/drm/xe/xe_guc_klv_helpers.c
new file mode 100644
index 000000000000..ceca949932a0
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_guc_klv_helpers.c
@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#include <linux/bitfield.h>
+#include <drm/drm_print.h>
+
+#include "abi/guc_klvs_abi.h"
+#include "xe_guc_klv_helpers.h"
+
+#define make_u64(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo)))
+
+/**
+ * xe_guc_klv_key_to_string - Convert KLV key into friendly name.
+ * @key: the `GuC KLV`_ key
+ *
+ * Return: name of the KLV key.
+ */
+const char *xe_guc_klv_key_to_string(u16 key)
+{
+	switch (key) {
+	/* VGT POLICY keys */
+	case GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY:
+		return "sched_if_idle";
+	case GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_KEY:
+		return "sample_period";
+	case GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_KEY:
+		return "reset_engine";
+	/* VF CFG keys */
+	case GUC_KLV_VF_CFG_GGTT_START_KEY:
+		return "ggtt_start";
+	case GUC_KLV_VF_CFG_GGTT_SIZE_KEY:
+		return "ggtt_size";
+	case GUC_KLV_VF_CFG_LMEM_SIZE_KEY:
+		return "lmem_size";
+	case GUC_KLV_VF_CFG_NUM_CONTEXTS_KEY:
+		return "num_contexts";
+	case GUC_KLV_VF_CFG_TILE_MASK_KEY:
+		return "tile_mask";
+	case GUC_KLV_VF_CFG_NUM_DOORBELLS_KEY:
+		return "num_doorbells";
+	case GUC_KLV_VF_CFG_EXEC_QUANTUM_KEY:
+		return "exec_quantum";
+	case GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_KEY:
+		return "preempt_timeout";
+	case GUC_KLV_VF_CFG_BEGIN_DOORBELL_ID_KEY:
+		return "begin_db_id";
+	case GUC_KLV_VF_CFG_BEGIN_CONTEXT_ID_KEY:
+		return "begin_ctx_id";
+	default:
+		return "(unknown)";
+	}
+}
+
+/**
+ * xe_guc_klv_print - Print content of the buffer with `GuC KLV`_.
+ * @klvs: the buffer with KLVs
+ * @num_dwords: number of dwords (u32) available in the buffer
+ * @p: the &drm_printer
+ *
+ * The buffer may contain more than one KLV.
+ */
+void xe_guc_klv_print(const u32 *klvs, u32 num_dwords, struct drm_printer *p)
+{
+	while (num_dwords >= GUC_KLV_LEN_MIN) {
+		u32 key = FIELD_GET(GUC_KLV_0_KEY, klvs[0]);
+		u32 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]);
+
+		klvs += GUC_KLV_LEN_MIN;
+		num_dwords -= GUC_KLV_LEN_MIN;
+
+		if (num_dwords < len) {
+			drm_printf(p, "{ key %#06x : truncated %zu of %zu bytes %*ph } # %s\n",
+				   key, num_dwords * sizeof(u32), len * sizeof(u32),
+				   (int)(num_dwords * sizeof(u32)), klvs,
+				   xe_guc_klv_key_to_string(key));
+			return;
+		}
+
+		switch (len) {
+		case 0:
+			drm_printf(p, "{ key %#06x : no value } # %s\n",
+				   key, xe_guc_klv_key_to_string(key));
+			break;
+		case 1:
+			drm_printf(p, "{ key %#06x : 32b value %u } # %s\n",
+				   key, klvs[0], xe_guc_klv_key_to_string(key));
+			break;
+		case 2:
+			drm_printf(p, "{ key %#06x : 64b value %#llx } # %s\n",
+				   key, make_u64(klvs[1], klvs[0]),
+				   xe_guc_klv_key_to_string(key));
+			break;
+		default:
+			drm_printf(p, "{ key %#06x : %zu bytes %*ph } # %s\n",
+				   key, len * sizeof(u32), (int)(len * sizeof(u32)),
+				   klvs, xe_guc_klv_key_to_string(key));
+			break;
+		}
+
+		klvs += len;
+		num_dwords -= len;
+	}
+
+	/* we don't expect any leftovers, fix if KLV header is ever changed */
+	BUILD_BUG_ON(GUC_KLV_LEN_MIN > 1);
+}
+
+/**
+ * xe_guc_klv_count - Count KLVs present in the buffer.
+ * @klvs: the buffer with KLVs
+ * @num_dwords: number of dwords (u32) in the buffer
+ *
+ * Return: number of recognized KLVs or
+ *         a negative error code if KLV buffer is truncated.
+ */
+int xe_guc_klv_count(const u32 *klvs, u32 num_dwords)
+{
+	int num_klvs = 0;
+
+	while (num_dwords >= GUC_KLV_LEN_MIN) {
+		u32 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]);
+
+		if (num_dwords < len + GUC_KLV_LEN_MIN)
+			break;
+
+		klvs += GUC_KLV_LEN_MIN + len;
+		num_dwords -= GUC_KLV_LEN_MIN + len;
+		num_klvs++;
+	}
+
+	return num_dwords ? -ENODATA : num_klvs;
+}
diff --git a/drivers/gpu/drm/xe/xe_guc_klv_helpers.h b/drivers/gpu/drm/xe/xe_guc_klv_helpers.h
new file mode 100644
index 000000000000..b835e0ebe6db
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_guc_klv_helpers.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#ifndef _XE_GUC_KLV_HELPERS_H_
+#define _XE_GUC_KLV_HELPERS_H_
+
+#include <linux/types.h>
+
+struct drm_printer;
+
+const char *xe_guc_klv_key_to_string(u16 key);
+
+void xe_guc_klv_print(const u32 *klvs, u32 num_dwords, struct drm_printer *p);
+int xe_guc_klv_count(const u32 *klvs, u32 num_dwords);
+
+/**
+ * PREP_GUC_KLV - Prepare KLV header value based on provided key and len.
+ * @key: KLV key
+ * @len: KLV length
+ *
+ * Return: value of the KLV header (u32).
+ */
+#define PREP_GUC_KLV(key, len) \
+	(FIELD_PREP(GUC_KLV_0_KEY, (key)) | \
+	 FIELD_PREP(GUC_KLV_0_LEN, (len)))
+
+/**
+ * PREP_GUC_KLV_CONST - Prepare KLV header value based on const key and len.
+ * @key: const KLV key
+ * @len: const KLV length
+ *
+ * Return: value of the KLV header (u32).
+ */
+#define PREP_GUC_KLV_CONST(key, len) \
+	(FIELD_PREP_CONST(GUC_KLV_0_KEY, (key)) | \
+	 FIELD_PREP_CONST(GUC_KLV_0_LEN, (len)))
+
+/**
+ * PREP_GUC_KLV_TAG - Prepare KLV header value based on unique KLV definition tag.
+ * @TAG: unique tag of the KLV definition
+ *
+ * Combine separate KEY and LEN definitions of the KLV identified by the TAG.
+ *
+ * Return: value of the KLV header (u32).
+ */
+#define PREP_GUC_KLV_TAG(TAG) \
+	PREP_GUC_KLV_CONST(GUC_KLV_##TAG##_KEY, GUC_KLV_##TAG##_LEN)
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c
index 45135c3520e5..a37ee3419428 100644
--- a/drivers/gpu/drm/xe/xe_guc_log.c
+++ b/drivers/gpu/drm/xe/xe_guc_log.c
@@ -84,8 +84,9 @@ int xe_guc_log_init(struct xe_guc_log *log)
 	struct xe_bo *bo;
 
 	bo = xe_managed_bo_create_pin_map(xe, tile, guc_log_size(),
-					  XE_BO_CREATE_SYSTEM_BIT |
-					  XE_BO_CREATE_GGTT_BIT);
+					  XE_BO_FLAG_SYSTEM |
+					  XE_BO_FLAG_GGTT |
+					  XE_BO_FLAG_GGTT_INVALIDATE);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c
index 2839d685631b..509649d0e65e 100644
--- a/drivers/gpu/drm/xe/xe_guc_pc.c
+++ b/drivers/gpu/drm/xe/xe_guc_pc.c
@@ -145,25 +145,6 @@ static int pc_action_reset(struct xe_guc_pc *pc)
 	return ret;
 }
 
-static int pc_action_shutdown(struct xe_guc_pc *pc)
-{
-	struct  xe_guc_ct *ct = &pc_to_guc(pc)->ct;
-	int ret;
-	u32 action[] = {
-		GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST,
-		SLPC_EVENT(SLPC_EVENT_SHUTDOWN, 2),
-		xe_bo_ggtt_addr(pc->bo),
-		0,
-	};
-
-	ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
-	if (ret)
-		drm_err(&pc_to_xe(pc)->drm, "GuC PC shutdown %pe",
-			ERR_PTR(ret));
-
-	return ret;
-}
-
 static int pc_action_query_task_state(struct xe_guc_pc *pc)
 {
 	struct xe_guc_ct *ct = &pc_to_guc(pc)->ct;
@@ -381,8 +362,6 @@ u32 xe_guc_pc_get_act_freq(struct xe_guc_pc *pc)
 	struct xe_device *xe = gt_to_xe(gt);
 	u32 freq;
 
-	xe_device_mem_access_get(gt_to_xe(gt));
-
 	/* When in RC6, actual frequency reported will be 0. */
 	if (GRAPHICS_VERx100(xe) >= 1270) {
 		freq = xe_mmio_read32(gt, MTL_MIRROR_TARGET_WP1);
@@ -394,8 +373,6 @@ u32 xe_guc_pc_get_act_freq(struct xe_guc_pc *pc)
 
 	freq = decode_freq(freq);
 
-	xe_device_mem_access_put(gt_to_xe(gt));
-
 	return freq;
 }
 
@@ -412,14 +389,13 @@ int xe_guc_pc_get_cur_freq(struct xe_guc_pc *pc, u32 *freq)
 	struct xe_gt *gt = pc_to_gt(pc);
 	int ret;
 
-	xe_device_mem_access_get(gt_to_xe(gt));
 	/*
 	 * GuC SLPC plays with cur freq request when GuCRC is enabled
 	 * Block RC6 for a more reliable read.
 	 */
 	ret = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	if (ret)
-		goto out;
+		return ret;
 
 	*freq = xe_mmio_read32(gt, RPNSWREQ);
 
@@ -427,9 +403,7 @@ int xe_guc_pc_get_cur_freq(struct xe_guc_pc *pc, u32 *freq)
 	*freq = decode_freq(*freq);
 
 	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
-out:
-	xe_device_mem_access_put(gt_to_xe(gt));
-	return ret;
+	return 0;
 }
 
 /**
@@ -451,12 +425,7 @@ u32 xe_guc_pc_get_rp0_freq(struct xe_guc_pc *pc)
  */
 u32 xe_guc_pc_get_rpe_freq(struct xe_guc_pc *pc)
 {
-	struct xe_gt *gt = pc_to_gt(pc);
-	struct xe_device *xe = gt_to_xe(gt);
-
-	xe_device_mem_access_get(xe);
 	pc_update_rp_values(pc);
-	xe_device_mem_access_put(xe);
 
 	return pc->rpe_freq;
 }
@@ -485,7 +454,6 @@ int xe_guc_pc_get_min_freq(struct xe_guc_pc *pc, u32 *freq)
 	struct xe_gt *gt = pc_to_gt(pc);
 	int ret;
 
-	xe_device_mem_access_get(pc_to_xe(pc));
 	mutex_lock(&pc->freq_lock);
 	if (!pc->freq_ready) {
 		/* Might be in the middle of a gt reset */
@@ -511,7 +479,6 @@ fw:
 	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
 out:
 	mutex_unlock(&pc->freq_lock);
-	xe_device_mem_access_put(pc_to_xe(pc));
 	return ret;
 }
 
@@ -528,7 +495,6 @@ int xe_guc_pc_set_min_freq(struct xe_guc_pc *pc, u32 freq)
 {
 	int ret;
 
-	xe_device_mem_access_get(pc_to_xe(pc));
 	mutex_lock(&pc->freq_lock);
 	if (!pc->freq_ready) {
 		/* Might be in the middle of a gt reset */
@@ -544,8 +510,6 @@ int xe_guc_pc_set_min_freq(struct xe_guc_pc *pc, u32 freq)
 
 out:
 	mutex_unlock(&pc->freq_lock);
-	xe_device_mem_access_put(pc_to_xe(pc));
-
 	return ret;
 }
 
@@ -561,7 +525,6 @@ int xe_guc_pc_get_max_freq(struct xe_guc_pc *pc, u32 *freq)
 {
 	int ret;
 
-	xe_device_mem_access_get(pc_to_xe(pc));
 	mutex_lock(&pc->freq_lock);
 	if (!pc->freq_ready) {
 		/* Might be in the middle of a gt reset */
@@ -577,7 +540,6 @@ int xe_guc_pc_get_max_freq(struct xe_guc_pc *pc, u32 *freq)
 
 out:
 	mutex_unlock(&pc->freq_lock);
-	xe_device_mem_access_put(pc_to_xe(pc));
 	return ret;
 }
 
@@ -594,7 +556,6 @@ int xe_guc_pc_set_max_freq(struct xe_guc_pc *pc, u32 freq)
 {
 	int ret;
 
-	xe_device_mem_access_get(pc_to_xe(pc));
 	mutex_lock(&pc->freq_lock);
 	if (!pc->freq_ready) {
 		/* Might be in the middle of a gt reset */
@@ -610,7 +571,6 @@ int xe_guc_pc_set_max_freq(struct xe_guc_pc *pc, u32 freq)
 
 out:
 	mutex_unlock(&pc->freq_lock);
-	xe_device_mem_access_put(pc_to_xe(pc));
 	return ret;
 }
 
@@ -623,8 +583,6 @@ enum xe_gt_idle_state xe_guc_pc_c_status(struct xe_guc_pc *pc)
 	struct xe_gt *gt = pc_to_gt(pc);
 	u32 reg, gt_c_state;
 
-	xe_device_mem_access_get(gt_to_xe(gt));
-
 	if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
 		reg = xe_mmio_read32(gt, MTL_MIRROR_TARGET_WP1);
 		gt_c_state = REG_FIELD_GET(MTL_CC_MASK, reg);
@@ -633,8 +591,6 @@ enum xe_gt_idle_state xe_guc_pc_c_status(struct xe_guc_pc *pc)
 		gt_c_state = REG_FIELD_GET(RCN_MASK, reg);
 	}
 
-	xe_device_mem_access_put(gt_to_xe(gt));
-
 	switch (gt_c_state) {
 	case GT_C6:
 		return GT_IDLE_C6;
@@ -654,9 +610,7 @@ u64 xe_guc_pc_rc6_residency(struct xe_guc_pc *pc)
 	struct xe_gt *gt = pc_to_gt(pc);
 	u32 reg;
 
-	xe_device_mem_access_get(gt_to_xe(gt));
 	reg = xe_mmio_read32(gt, GT_GFX_RC6);
-	xe_device_mem_access_put(gt_to_xe(gt));
 
 	return reg;
 }
@@ -670,9 +624,7 @@ u64 xe_guc_pc_mc6_residency(struct xe_guc_pc *pc)
 	struct xe_gt *gt = pc_to_gt(pc);
 	u64 reg;
 
-	xe_device_mem_access_get(gt_to_xe(gt));
 	reg = xe_mmio_read32(gt, MTL_MEDIA_MC6);
-	xe_device_mem_access_put(gt_to_xe(gt));
 
 	return reg;
 }
@@ -743,24 +695,28 @@ static int pc_adjust_freq_bounds(struct xe_guc_pc *pc)
 
 	ret = pc_action_query_task_state(pc);
 	if (ret)
-		return ret;
+		goto out;
 
 	/*
 	 * GuC defaults to some RPmax that is not actually achievable without
 	 * overclocking. Let's adjust it to the Hardware RP0, which is the
 	 * regular maximum
 	 */
-	if (pc_get_max_freq(pc) > pc->rp0_freq)
-		pc_set_max_freq(pc, pc->rp0_freq);
+	if (pc_get_max_freq(pc) > pc->rp0_freq) {
+		ret = pc_set_max_freq(pc, pc->rp0_freq);
+		if (ret)
+			goto out;
+	}
 
 	/*
 	 * Same thing happens for Server platforms where min is listed as
 	 * RPMax
 	 */
 	if (pc_get_min_freq(pc) > pc->rp0_freq)
-		pc_set_min_freq(pc, pc->rp0_freq);
+		ret = pc_set_min_freq(pc, pc->rp0_freq);
 
-	return 0;
+out:
+	return ret;
 }
 
 static int pc_adjust_requested_freq(struct xe_guc_pc *pc)
@@ -801,23 +757,19 @@ int xe_guc_pc_gucrc_disable(struct xe_guc_pc *pc)
 	if (xe->info.skip_guc_pc)
 		return 0;
 
-	xe_device_mem_access_get(pc_to_xe(pc));
-
 	ret = pc_action_setup_gucrc(pc, XE_GUCRC_HOST_CONTROL);
 	if (ret)
-		goto out;
+		return ret;
 
 	ret = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	if (ret)
-		goto out;
+		return ret;
 
 	xe_gt_idle_disable_c6(gt);
 
 	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
 
-out:
-	xe_device_mem_access_put(pc_to_xe(pc));
-	return ret;
+	return 0;
 }
 
 static void pc_init_pcode_freq(struct xe_guc_pc *pc)
@@ -870,11 +822,9 @@ int xe_guc_pc_start(struct xe_guc_pc *pc)
 
 	xe_gt_assert(gt, xe_device_uc_enabled(xe));
 
-	xe_device_mem_access_get(pc_to_xe(pc));
-
 	ret = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	if (ret)
-		goto out_fail_force_wake;
+		return ret;
 
 	if (xe->info.skip_guc_pc) {
 		if (xe->info.platform != XE_PVC)
@@ -914,8 +864,6 @@ int xe_guc_pc_start(struct xe_guc_pc *pc)
 
 out:
 	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
-out_fail_force_wake:
-	xe_device_mem_access_put(pc_to_xe(pc));
 	return ret;
 }
 
@@ -926,32 +874,17 @@ out_fail_force_wake:
 int xe_guc_pc_stop(struct xe_guc_pc *pc)
 {
 	struct xe_device *xe = pc_to_xe(pc);
-	int ret;
-
-	xe_device_mem_access_get(pc_to_xe(pc));
 
 	if (xe->info.skip_guc_pc) {
 		xe_gt_idle_disable_c6(pc_to_gt(pc));
-		ret = 0;
-		goto out;
+		return 0;
 	}
 
 	mutex_lock(&pc->freq_lock);
 	pc->freq_ready = false;
 	mutex_unlock(&pc->freq_lock);
 
-	ret = pc_action_shutdown(pc);
-	if (ret)
-		goto out;
-
-	if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_NOT_RUNNING)) {
-		drm_err(&pc_to_xe(pc)->drm, "GuC PC Shutdown failed\n");
-		ret = -EIO;
-	}
-
-out:
-	xe_device_mem_access_put(pc_to_xe(pc));
-	return ret;
+	return 0;
 }
 
 /**
@@ -965,13 +898,11 @@ static void xe_guc_pc_fini(struct drm_device *drm, void *arg)
 	struct xe_device *xe = pc_to_xe(pc);
 
 	if (xe->info.skip_guc_pc) {
-		xe_device_mem_access_get(xe);
 		xe_gt_idle_disable_c6(pc_to_gt(pc));
-		xe_device_mem_access_put(xe);
 		return;
 	}
 
-	xe_force_wake_get(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL);
+	XE_WARN_ON(xe_force_wake_get(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL));
 	XE_WARN_ON(xe_guc_pc_gucrc_disable(pc));
 	XE_WARN_ON(xe_guc_pc_stop(pc));
 	xe_force_wake_put(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL);
@@ -998,16 +929,13 @@ int xe_guc_pc_init(struct xe_guc_pc *pc)
 		return err;
 
 	bo = xe_managed_bo_create_pin_map(xe, tile, size,
-					  XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-					  XE_BO_CREATE_GGTT_BIT);
+					  XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+					  XE_BO_FLAG_GGTT |
+					  XE_BO_FLAG_GGTT_INVALIDATE);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
 	pc->bo = bo;
 
-	err = drmm_add_action_or_reset(&xe->drm, xe_guc_pc_fini, pc);
-	if (err)
-		return err;
-
-	return 0;
+	return drmm_add_action_or_reset(&xe->drm, xe_guc_pc_fini, pc);
 }
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c
index e2a4c3b5e9ff..e4e3658e6a13 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.c
+++ b/drivers/gpu/drm/xe/xe_guc_submit.c
@@ -27,6 +27,7 @@
 #include "xe_guc.h"
 #include "xe_guc_ct.h"
 #include "xe_guc_exec_queue_types.h"
+#include "xe_guc_id_mgr.h"
 #include "xe_guc_submit_types.h"
 #include "xe_hw_engine.h"
 #include "xe_hw_fence.h"
@@ -236,17 +237,9 @@ static void guc_submit_fini(struct drm_device *drm, void *arg)
 	struct xe_guc *guc = arg;
 
 	xa_destroy(&guc->submission_state.exec_queue_lookup);
-	ida_destroy(&guc->submission_state.guc_ids);
-	bitmap_free(guc->submission_state.guc_ids_bitmap);
 	free_submit_wq(guc);
-	mutex_destroy(&guc->submission_state.lock);
 }
 
-#define GUC_ID_MAX		65535
-#define GUC_ID_NUMBER_MLRC	4096
-#define GUC_ID_NUMBER_SLRC	(GUC_ID_MAX - GUC_ID_NUMBER_MLRC)
-#define GUC_ID_START_MLRC	GUC_ID_NUMBER_SLRC
-
 static const struct xe_exec_queue_ops guc_exec_queue_ops;
 
 static void primelockdep(struct xe_guc *guc)
@@ -269,33 +262,28 @@ int xe_guc_submit_init(struct xe_guc *guc)
 	struct xe_gt *gt = guc_to_gt(guc);
 	int err;
 
-	guc->submission_state.guc_ids_bitmap =
-		bitmap_zalloc(GUC_ID_NUMBER_MLRC, GFP_KERNEL);
-	if (!guc->submission_state.guc_ids_bitmap)
-		return -ENOMEM;
+	err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock);
+	if (err)
+		return err;
+
+	err = xe_guc_id_mgr_init(&guc->submission_state.idm, ~0);
+	if (err)
+		return err;
 
 	err = alloc_submit_wq(guc);
-	if (err) {
-		bitmap_free(guc->submission_state.guc_ids_bitmap);
+	if (err)
 		return err;
-	}
 
 	gt->exec_queue_ops = &guc_exec_queue_ops;
 
-	mutex_init(&guc->submission_state.lock);
 	xa_init(&guc->submission_state.exec_queue_lookup);
-	ida_init(&guc->submission_state.guc_ids);
 
 	spin_lock_init(&guc->submission_state.suspend.lock);
 	guc->submission_state.suspend.context = dma_fence_context_alloc(1);
 
 	primelockdep(guc);
 
-	err = drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc);
-	if (err)
-		return err;
-
-	return 0;
+	return drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc);
 }
 
 static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa_count)
@@ -307,12 +295,8 @@ static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa
 	for (i = 0; i < xa_count; ++i)
 		xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id + i);
 
-	if (xe_exec_queue_is_parallel(q))
-		bitmap_release_region(guc->submission_state.guc_ids_bitmap,
-				      q->guc->id - GUC_ID_START_MLRC,
-				      order_base_2(q->width));
-	else
-		ida_free(&guc->submission_state.guc_ids, q->guc->id);
+	xe_guc_id_mgr_release_locked(&guc->submission_state.idm,
+				     q->guc->id, q->width);
 }
 
 static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)
@@ -330,21 +314,12 @@ static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)
 	 */
 	lockdep_assert_held(&guc->submission_state.lock);
 
-	if (xe_exec_queue_is_parallel(q)) {
-		void *bitmap = guc->submission_state.guc_ids_bitmap;
-
-		ret = bitmap_find_free_region(bitmap, GUC_ID_NUMBER_MLRC,
-					      order_base_2(q->width));
-	} else {
-		ret = ida_alloc_max(&guc->submission_state.guc_ids,
-				    GUC_ID_NUMBER_SLRC - 1, GFP_NOWAIT);
-	}
+	ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm,
+					   q->width);
 	if (ret < 0)
 		return ret;
 
 	q->guc->id = ret;
-	if (xe_exec_queue_is_parallel(q))
-		q->guc->id += GUC_ID_START_MLRC;
 
 	for (i = 0; i < q->width; ++i) {
 		ptr = xa_store(&guc->submission_state.exec_queue_lookup,
@@ -533,7 +508,7 @@ static void register_engine(struct xe_exec_queue *q)
 	info.flags = CONTEXT_REGISTRATION_FLAG_KMD;
 
 	if (xe_exec_queue_is_parallel(q)) {
-		u32 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc);
+		u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc);
 		struct iosys_map map = xe_lrc_parallel_map(lrc);
 
 		info.wq_desc_lo = lower_32_bits(ggtt_addr +
@@ -833,7 +808,9 @@ static void simple_error_capture(struct xe_exec_queue *q)
 			}
 		}
 
-		xe_force_wake_get(gt_to_fw(guc_to_gt(guc)), XE_FORCEWAKE_ALL);
+		if (xe_force_wake_get(gt_to_fw(guc_to_gt(guc)), XE_FORCEWAKE_ALL))
+			xe_gt_info(guc_to_gt(guc),
+				   "failed to get forcewake for error capture");
 		xe_guc_ct_print(&guc->ct, &p, true);
 		guc_exec_queue_print(q, &p);
 		for_each_hw_engine(hwe, guc_to_gt(guc), id) {
@@ -929,20 +906,26 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 	int err = -ETIME;
 	int i = 0;
 
-	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags)) {
-		drm_notice(&xe->drm, "Timedout job: seqno=%u, guc_id=%d, flags=0x%lx",
-			   xe_sched_job_seqno(job), q->guc->id, q->flags);
-		xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL,
-			   "Kernel-submitted job timed out\n");
-		xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q),
-			   "VM job timed out on non-killed execqueue\n");
+	/*
+	 * TDR has fired before free job worker. Common if exec queue
+	 * immediately closed after last fence signaled.
+	 */
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags)) {
+		guc_exec_queue_free_job(drm_job);
 
-		simple_error_capture(q);
-		xe_devcoredump(job);
-	} else {
-		drm_dbg(&xe->drm, "Timedout signaled job: seqno=%u, guc_id=%d, flags=0x%lx",
-			 xe_sched_job_seqno(job), q->guc->id, q->flags);
+		return DRM_GPU_SCHED_STAT_NOMINAL;
 	}
+
+	drm_notice(&xe->drm, "Timedout job: seqno=%u, guc_id=%d, flags=0x%lx",
+		   xe_sched_job_seqno(job), q->guc->id, q->flags);
+	xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL,
+		   "Kernel-submitted job timed out\n");
+	xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q),
+		   "VM job timed out on non-killed execqueue\n");
+
+	simple_error_capture(q);
+	xe_devcoredump(job);
+
 	trace_xe_sched_job_timedout(job);
 
 	/* Kill the run_job entry point */
@@ -1257,6 +1240,7 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 	return 0;
 
 err_entity:
+	mutex_unlock(&guc->submission_state.lock);
 	xe_sched_entity_fini(&ge->entity);
 err_sched:
 	xe_sched_fini(&ge->sched);
@@ -1568,28 +1552,8 @@ static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q)
 	xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action));
 }
 
-int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
+static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q)
 {
-	struct xe_device *xe = guc_to_xe(guc);
-	struct xe_exec_queue *q;
-	u32 guc_id = msg[0];
-
-	if (unlikely(len < 2)) {
-		drm_err(&xe->drm, "Invalid length %u", len);
-		return -EPROTO;
-	}
-
-	q = g2h_exec_queue_lookup(guc, guc_id);
-	if (unlikely(!q))
-		return -EPROTO;
-
-	if (unlikely(!exec_queue_pending_enable(q) &&
-		     !exec_queue_pending_disable(q))) {
-		drm_err(&xe->drm, "Unexpected engine state 0x%04x",
-			atomic_read(&q->guc->state));
-		return -EPROTO;
-	}
-
 	trace_xe_exec_queue_scheduling_done(q);
 
 	if (exec_queue_pending_enable(q)) {
@@ -1609,17 +1573,15 @@ int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
 			deregister_exec_queue(guc, q);
 		}
 	}
-
-	return 0;
 }
 
-int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
+int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
 {
 	struct xe_device *xe = guc_to_xe(guc);
 	struct xe_exec_queue *q;
 	u32 guc_id = msg[0];
 
-	if (unlikely(len < 1)) {
+	if (unlikely(len < 2)) {
 		drm_err(&xe->drm, "Invalid length %u", len);
 		return -EPROTO;
 	}
@@ -1628,13 +1590,20 @@ int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
 	if (unlikely(!q))
 		return -EPROTO;
 
-	if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) ||
-	    exec_queue_pending_enable(q) || exec_queue_enabled(q)) {
+	if (unlikely(!exec_queue_pending_enable(q) &&
+		     !exec_queue_pending_disable(q))) {
 		drm_err(&xe->drm, "Unexpected engine state 0x%04x",
 			atomic_read(&q->guc->state));
 		return -EPROTO;
 	}
 
+	handle_sched_done(guc, q);
+
+	return 0;
+}
+
+static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q)
+{
 	trace_xe_exec_queue_deregister_done(q);
 
 	clear_exec_queue_registered(q);
@@ -1643,6 +1612,31 @@ int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
 		xe_exec_queue_put(q);
 	else
 		__guc_exec_queue_fini(guc, q);
+}
+
+int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
+{
+	struct xe_device *xe = guc_to_xe(guc);
+	struct xe_exec_queue *q;
+	u32 guc_id = msg[0];
+
+	if (unlikely(len < 1)) {
+		drm_err(&xe->drm, "Invalid length %u", len);
+		return -EPROTO;
+	}
+
+	q = g2h_exec_queue_lookup(guc, guc_id);
+	if (unlikely(!q))
+		return -EPROTO;
+
+	if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) ||
+	    exec_queue_pending_enable(q) || exec_queue_enabled(q)) {
+		drm_err(&xe->drm, "Unexpected engine state 0x%04x",
+			atomic_read(&q->guc->state));
+		return -EPROTO;
+	}
+
+	handle_deregister_done(guc, q);
 
 	return 0;
 }
@@ -1782,7 +1776,7 @@ guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snaps
 
 /**
  * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine.
- * @job: faulty Xe scheduled job.
+ * @q: faulty exec queue
  *
  * This can be printed out in a later stage like during dev_coredump
  * analysis.
@@ -1791,9 +1785,8 @@ guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snaps
  * caller, using `xe_guc_exec_queue_snapshot_free`.
  */
 struct xe_guc_submit_exec_queue_snapshot *
-xe_guc_exec_queue_snapshot_capture(struct xe_sched_job *job)
+xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q)
 {
-	struct xe_exec_queue *q = job->q;
 	struct xe_gpu_scheduler *sched = &q->guc->sched;
 	struct xe_guc_submit_exec_queue_snapshot *snapshot;
 	int i;
@@ -1814,21 +1807,14 @@ xe_guc_exec_queue_snapshot_capture(struct xe_sched_job *job)
 	snapshot->sched_props.preempt_timeout_us =
 		q->sched_props.preempt_timeout_us;
 
-	snapshot->lrc = kmalloc_array(q->width, sizeof(struct lrc_snapshot),
+	snapshot->lrc = kmalloc_array(q->width, sizeof(struct xe_lrc_snapshot *),
 				      GFP_ATOMIC);
 
 	if (snapshot->lrc) {
 		for (i = 0; i < q->width; ++i) {
 			struct xe_lrc *lrc = q->lrc + i;
 
-			snapshot->lrc[i].context_desc =
-				lower_32_bits(xe_lrc_ggtt_addr(lrc));
-			snapshot->lrc[i].head = xe_lrc_ring_head(lrc);
-			snapshot->lrc[i].tail.internal = lrc->ring.tail;
-			snapshot->lrc[i].tail.memory =
-				xe_lrc_read_ctx_reg(lrc, CTX_RING_TAIL);
-			snapshot->lrc[i].start_seqno = xe_lrc_start_seqno(lrc);
-			snapshot->lrc[i].seqno = xe_lrc_seqno(lrc);
+			snapshot->lrc[i] = xe_lrc_snapshot_capture(lrc);
 		}
 	}
 
@@ -1867,6 +1853,24 @@ xe_guc_exec_queue_snapshot_capture(struct xe_sched_job *job)
 }
 
 /**
+ * xe_guc_exec_queue_snapshot_capture_delayed - Take delayed part of snapshot of the GuC Engine.
+ * @snapshot: Previously captured snapshot of job.
+ *
+ * This captures some data that requires taking some locks, so it cannot be done in signaling path.
+ */
+void
+xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot)
+{
+	int i;
+
+	if (!snapshot || !snapshot->lrc)
+		return;
+
+	for (i = 0; i < snapshot->width; ++i)
+		xe_lrc_snapshot_capture_delayed(snapshot->lrc[i]);
+}
+
+/**
  * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot.
  * @snapshot: GuC Submit Engine snapshot object.
  * @p: drm_printer where it will be printed out.
@@ -1894,18 +1898,9 @@ xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snaps
 	drm_printf(p, "\tPreempt timeout: %u (us)\n",
 		   snapshot->sched_props.preempt_timeout_us);
 
-	for (i = 0; snapshot->lrc && i < snapshot->width; ++i) {
-		drm_printf(p, "\tHW Context Desc: 0x%08x\n",
-			   snapshot->lrc[i].context_desc);
-		drm_printf(p, "\tLRC Head: (memory) %u\n",
-			   snapshot->lrc[i].head);
-		drm_printf(p, "\tLRC Tail: (internal) %u, (memory) %u\n",
-			   snapshot->lrc[i].tail.internal,
-			   snapshot->lrc[i].tail.memory);
-		drm_printf(p, "\tStart seqno: (memory) %d\n",
-			   snapshot->lrc[i].start_seqno);
-		drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->lrc[i].seqno);
-	}
+	for (i = 0; snapshot->lrc && i < snapshot->width; ++i)
+		xe_lrc_snapshot_print(snapshot->lrc[i], p);
+
 	drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state);
 	drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags);
 
@@ -1930,10 +1925,16 @@ xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snaps
  */
 void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot)
 {
+	int i;
+
 	if (!snapshot)
 		return;
 
-	kfree(snapshot->lrc);
+	if (snapshot->lrc) {
+		for (i = 0; i < snapshot->width; i++)
+			xe_lrc_snapshot_free(snapshot->lrc[i]);
+		kfree(snapshot->lrc);
+	}
 	kfree(snapshot->pending_list);
 	kfree(snapshot);
 }
@@ -1941,28 +1942,10 @@ void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *s
 static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p)
 {
 	struct xe_guc_submit_exec_queue_snapshot *snapshot;
-	struct xe_gpu_scheduler *sched = &q->guc->sched;
-	struct xe_sched_job *job;
-	bool found = false;
-
-	spin_lock(&sched->base.job_list_lock);
-	list_for_each_entry(job, &sched->base.pending_list, drm.list) {
-		if (job->q == q) {
-			xe_sched_job_get(job);
-			found = true;
-			break;
-		}
-	}
-	spin_unlock(&sched->base.job_list_lock);
 
-	if (!found)
-		return;
-
-	snapshot = xe_guc_exec_queue_snapshot_capture(job);
+	snapshot = xe_guc_exec_queue_snapshot_capture(q);
 	xe_guc_exec_queue_snapshot_print(snapshot, p);
 	xe_guc_exec_queue_snapshot_free(snapshot);
-
-	xe_sched_job_put(job);
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h
index 723dc2bd8df9..fad0421ead36 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.h
+++ b/drivers/gpu/drm/xe/xe_guc_submit.h
@@ -9,8 +9,8 @@
 #include <linux/types.h>
 
 struct drm_printer;
+struct xe_exec_queue;
 struct xe_guc;
-struct xe_sched_job;
 
 int xe_guc_submit_init(struct xe_guc *guc);
 
@@ -27,7 +27,9 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg,
 int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len);
 
 struct xe_guc_submit_exec_queue_snapshot *
-xe_guc_exec_queue_snapshot_capture(struct xe_sched_job *job);
+xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q);
+void
+xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot);
 void
 xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot,
 				 struct drm_printer *p);
diff --git a/drivers/gpu/drm/xe/xe_guc_submit_types.h b/drivers/gpu/drm/xe/xe_guc_submit_types.h
index 72fc0f42b0a5..dc7456c34583 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_submit_types.h
@@ -61,17 +61,6 @@ struct guc_submit_parallel_scratch {
 	u32 wq[WQ_SIZE / sizeof(u32)];
 };
 
-struct lrc_snapshot {
-	u32 context_desc;
-	u32 head;
-	struct {
-		u32 internal;
-		u32 memory;
-	} tail;
-	u32 start_seqno;
-	u32 seqno;
-};
-
 struct pending_list_snapshot {
 	u32 seqno;
 	bool fence;
@@ -109,7 +98,7 @@ struct xe_guc_submit_exec_queue_snapshot {
 	} sched_props;
 
 	/** @lrc: LRC Snapshot */
-	struct lrc_snapshot *lrc;
+	struct xe_lrc_snapshot **lrc;
 
 	/** @schedule_state: Schedule State at the moment of Crash */
 	u32 schedule_state;
diff --git a/drivers/gpu/drm/xe/xe_guc_types.h b/drivers/gpu/drm/xe/xe_guc_types.h
index edcd1a950bd3..82bd93f7867d 100644
--- a/drivers/gpu/drm/xe/xe_guc_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_types.h
@@ -32,6 +32,21 @@ struct xe_guc_db_mgr {
 };
 
 /**
+ * struct xe_guc_id_mgr - GuC context ID Manager.
+ *
+ * Note: GuC context ID Manager is relying on &xe_guc::submission_state.lock
+ * to protect its members.
+ */
+struct xe_guc_id_mgr {
+	/** @bitmap: bitmap to track allocated IDs */
+	unsigned long *bitmap;
+	/** @total: total number of IDs being managed */
+	unsigned int total;
+	/** @used: number of IDs currently in use */
+	unsigned int used;
+};
+
+/**
  * struct xe_guc - Graphic micro controller
  */
 struct xe_guc {
@@ -49,12 +64,10 @@ struct xe_guc {
 	struct xe_guc_db_mgr dbm;
 	/** @submission_state: GuC submission state */
 	struct {
+		/** @submission_state.idm: GuC context ID Manager */
+		struct xe_guc_id_mgr idm;
 		/** @submission_state.exec_queue_lookup: Lookup an xe_engine from guc_id */
 		struct xarray exec_queue_lookup;
-		/** @submission_state.guc_ids: used to allocate new guc_ids, single-lrc */
-		struct ida guc_ids;
-		/** @submission_state.guc_ids_bitmap: used to allocate new guc_ids, multi-lrc */
-		unsigned long *guc_ids_bitmap;
 		/** @submission_state.stopped: submissions are stopped */
 		atomic_t stopped;
 		/** @submission_state.lock: protects submission state */
diff --git a/drivers/gpu/drm/xe/xe_hmm.c b/drivers/gpu/drm/xe/xe_hmm.c
new file mode 100644
index 000000000000..2c32dc46f7d4
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_hmm.c
@@ -0,0 +1,253 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#include <linux/scatterlist.h>
+#include <linux/mmu_notifier.h>
+#include <linux/dma-mapping.h>
+#include <linux/memremap.h>
+#include <linux/swap.h>
+#include <linux/hmm.h>
+#include <linux/mm.h>
+#include "xe_hmm.h"
+#include "xe_vm.h"
+#include "xe_bo.h"
+
+static u64 xe_npages_in_range(unsigned long start, unsigned long end)
+{
+	return (end - start) >> PAGE_SHIFT;
+}
+
+/*
+ * xe_mark_range_accessed() - mark a range is accessed, so core mm
+ * have such information for memory eviction or write back to
+ * hard disk
+ *
+ * @range: the range to mark
+ * @write: if write to this range, we mark pages in this range
+ * as dirty
+ */
+static void xe_mark_range_accessed(struct hmm_range *range, bool write)
+{
+	struct page *page;
+	u64 i, npages;
+
+	npages = xe_npages_in_range(range->start, range->end);
+	for (i = 0; i < npages; i++) {
+		page = hmm_pfn_to_page(range->hmm_pfns[i]);
+		if (write)
+			set_page_dirty_lock(page);
+
+		mark_page_accessed(page);
+	}
+}
+
+/*
+ * xe_build_sg() - build a scatter gather table for all the physical pages/pfn
+ * in a hmm_range. dma-map pages if necessary. dma-address is save in sg table
+ * and will be used to program GPU page table later.
+ *
+ * @xe: the xe device who will access the dma-address in sg table
+ * @range: the hmm range that we build the sg table from. range->hmm_pfns[]
+ * has the pfn numbers of pages that back up this hmm address range.
+ * @st: pointer to the sg table.
+ * @write: whether we write to this range. This decides dma map direction
+ * for system pages. If write we map it bi-diretional; otherwise
+ * DMA_TO_DEVICE
+ *
+ * All the contiguous pfns will be collapsed into one entry in
+ * the scatter gather table. This is for the purpose of efficiently
+ * programming GPU page table.
+ *
+ * The dma_address in the sg table will later be used by GPU to
+ * access memory. So if the memory is system memory, we need to
+ * do a dma-mapping so it can be accessed by GPU/DMA.
+ *
+ * FIXME: This function currently only support pages in system
+ * memory. If the memory is GPU local memory (of the GPU who
+ * is going to access memory), we need gpu dpa (device physical
+ * address), and there is no need of dma-mapping. This is TBD.
+ *
+ * FIXME: dma-mapping for peer gpu device to access remote gpu's
+ * memory. Add this when you support p2p
+ *
+ * This function allocates the storage of the sg table. It is
+ * caller's responsibility to free it calling sg_free_table.
+ *
+ * Returns 0 if successful; -ENOMEM if fails to allocate memory
+ */
+static int xe_build_sg(struct xe_device *xe, struct hmm_range *range,
+		       struct sg_table *st, bool write)
+{
+	struct device *dev = xe->drm.dev;
+	struct page **pages;
+	u64 i, npages;
+	int ret;
+
+	npages = xe_npages_in_range(range->start, range->end);
+	pages = kvmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
+	if (!pages)
+		return -ENOMEM;
+
+	for (i = 0; i < npages; i++) {
+		pages[i] = hmm_pfn_to_page(range->hmm_pfns[i]);
+		xe_assert(xe, !is_device_private_page(pages[i]));
+	}
+
+	ret = sg_alloc_table_from_pages_segment(st, pages, npages, 0, npages << PAGE_SHIFT,
+						xe_sg_segment_size(dev), GFP_KERNEL);
+	if (ret)
+		goto free_pages;
+
+	ret = dma_map_sgtable(dev, st, write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE,
+			      DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_NO_KERNEL_MAPPING);
+	if (ret) {
+		sg_free_table(st);
+		st = NULL;
+	}
+
+free_pages:
+	kvfree(pages);
+	return ret;
+}
+
+/*
+ * xe_hmm_userptr_free_sg() - Free the scatter gather table of userptr
+ *
+ * @uvma: the userptr vma which hold the scatter gather table
+ *
+ * With function xe_userptr_populate_range, we allocate storage of
+ * the userptr sg table. This is a helper function to free this
+ * sg table, and dma unmap the address in the table.
+ */
+void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma)
+{
+	struct xe_userptr *userptr = &uvma->userptr;
+	struct xe_vma *vma = &uvma->vma;
+	bool write = !xe_vma_read_only(vma);
+	struct xe_vm *vm = xe_vma_vm(vma);
+	struct xe_device *xe = vm->xe;
+	struct device *dev = xe->drm.dev;
+
+	xe_assert(xe, userptr->sg);
+	dma_unmap_sgtable(dev, userptr->sg,
+			  write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, 0);
+
+	sg_free_table(userptr->sg);
+	userptr->sg = NULL;
+}
+
+/**
+ * xe_hmm_userptr_populate_range() - Populate physical pages of a virtual
+ * address range
+ *
+ * @uvma: userptr vma which has information of the range to populate.
+ * @is_mm_mmap_locked: True if mmap_read_lock is already acquired by caller.
+ *
+ * This function populate the physical pages of a virtual
+ * address range. The populated physical pages is saved in
+ * userptr's sg table. It is similar to get_user_pages but call
+ * hmm_range_fault.
+ *
+ * This function also read mmu notifier sequence # (
+ * mmu_interval_read_begin), for the purpose of later
+ * comparison (through mmu_interval_read_retry).
+ *
+ * This must be called with mmap read or write lock held.
+ *
+ * This function allocates the storage of the userptr sg table.
+ * It is caller's responsibility to free it calling sg_free_table.
+ *
+ * returns: 0 for succuss; negative error no on failure
+ */
+int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma,
+				  bool is_mm_mmap_locked)
+{
+	unsigned long timeout =
+		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
+	unsigned long *pfns, flags = HMM_PFN_REQ_FAULT;
+	struct xe_userptr *userptr;
+	struct xe_vma *vma = &uvma->vma;
+	u64 userptr_start = xe_vma_userptr(vma);
+	u64 userptr_end = userptr_start + xe_vma_size(vma);
+	struct xe_vm *vm = xe_vma_vm(vma);
+	struct hmm_range hmm_range;
+	bool write = !xe_vma_read_only(vma);
+	unsigned long notifier_seq;
+	u64 npages;
+	int ret;
+
+	userptr = &uvma->userptr;
+
+	if (is_mm_mmap_locked)
+		mmap_assert_locked(userptr->notifier.mm);
+
+	if (vma->gpuva.flags & XE_VMA_DESTROYED)
+		return 0;
+
+	notifier_seq = mmu_interval_read_begin(&userptr->notifier);
+	if (notifier_seq == userptr->notifier_seq)
+		return 0;
+
+	if (userptr->sg)
+		xe_hmm_userptr_free_sg(uvma);
+
+	npages = xe_npages_in_range(userptr_start, userptr_end);
+	pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
+	if (unlikely(!pfns))
+		return -ENOMEM;
+
+	if (write)
+		flags |= HMM_PFN_REQ_WRITE;
+
+	if (!mmget_not_zero(userptr->notifier.mm)) {
+		ret = -EFAULT;
+		goto free_pfns;
+	}
+
+	hmm_range.default_flags = flags;
+	hmm_range.hmm_pfns = pfns;
+	hmm_range.notifier = &userptr->notifier;
+	hmm_range.start = userptr_start;
+	hmm_range.end = userptr_end;
+	hmm_range.dev_private_owner = vm->xe;
+
+	while (true) {
+		hmm_range.notifier_seq = mmu_interval_read_begin(&userptr->notifier);
+
+		if (!is_mm_mmap_locked)
+			mmap_read_lock(userptr->notifier.mm);
+
+		ret = hmm_range_fault(&hmm_range);
+
+		if (!is_mm_mmap_locked)
+			mmap_read_unlock(userptr->notifier.mm);
+
+		if (ret == -EBUSY) {
+			if (time_after(jiffies, timeout))
+				break;
+
+			continue;
+		}
+		break;
+	}
+
+	mmput(userptr->notifier.mm);
+
+	if (ret)
+		goto free_pfns;
+
+	ret = xe_build_sg(vm->xe, &hmm_range, &userptr->sgt, write);
+	if (ret)
+		goto free_pfns;
+
+	xe_mark_range_accessed(&hmm_range, write);
+	userptr->sg = &userptr->sgt;
+	userptr->notifier_seq = hmm_range.notifier_seq;
+
+free_pfns:
+	kvfree(pfns);
+	return ret;
+}
+
diff --git a/drivers/gpu/drm/xe/xe_hmm.h b/drivers/gpu/drm/xe/xe_hmm.h
new file mode 100644
index 000000000000..909dc2bdcd97
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_hmm.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2024 Intel Corporation
+ */
+
+#include <linux/types.h>
+
+struct xe_userptr_vma;
+
+int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma, bool is_mm_mmap_locked);
+void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma);
diff --git a/drivers/gpu/drm/xe/xe_huc.c b/drivers/gpu/drm/xe/xe_huc.c
index b545f850087c..39a484a57585 100644
--- a/drivers/gpu/drm/xe/xe_huc.c
+++ b/drivers/gpu/drm/xe/xe_huc.c
@@ -53,26 +53,19 @@ static int huc_alloc_gsc_pkt(struct xe_huc *huc)
 	struct xe_gt *gt = huc_to_gt(huc);
 	struct xe_device *xe = gt_to_xe(gt);
 	struct xe_bo *bo;
-	int err;
 
 	/* we use a single object for both input and output */
 	bo = xe_bo_create_pin_map(xe, gt_to_tile(gt), NULL,
 				  PXP43_HUC_AUTH_INOUT_SIZE * 2,
 				  ttm_bo_type_kernel,
-				  XE_BO_CREATE_SYSTEM_BIT |
-				  XE_BO_CREATE_GGTT_BIT);
+				  XE_BO_FLAG_SYSTEM |
+				  XE_BO_FLAG_GGTT);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
 	huc->gsc_pkt = bo;
 
-	err = drmm_add_action_or_reset(&xe->drm, free_gsc_pkt, huc);
-	if (err) {
-		free_gsc_pkt(&xe->drm, huc);
-		return err;
-	}
-
-	return 0;
+	return drmm_add_action_or_reset(&xe->drm, free_gsc_pkt, huc);
 }
 
 int xe_huc_init(struct xe_huc *huc)
diff --git a/drivers/gpu/drm/xe/xe_huc_debugfs.c b/drivers/gpu/drm/xe/xe_huc_debugfs.c
index 18585a7eeb9d..3a888a40188b 100644
--- a/drivers/gpu/drm/xe/xe_huc_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_huc_debugfs.c
@@ -12,6 +12,7 @@
 #include "xe_gt.h"
 #include "xe_huc.h"
 #include "xe_macros.h"
+#include "xe_pm.h"
 
 static struct xe_gt *
 huc_to_gt(struct xe_huc *huc)
@@ -36,9 +37,9 @@ static int huc_info(struct seq_file *m, void *data)
 	struct xe_device *xe = huc_to_xe(huc);
 	struct drm_printer p = drm_seq_file_printer(m);
 
-	xe_device_mem_access_get(xe);
+	xe_pm_runtime_get(xe);
 	xe_huc_print_info(huc, &p);
-	xe_device_mem_access_put(xe);
+	xe_pm_runtime_put(xe);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c
index b5e83ea172f3..455f375c1cbd 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine.c
+++ b/drivers/gpu/drm/xe/xe_hw_engine.c
@@ -14,8 +14,10 @@
 #include "xe_device.h"
 #include "xe_execlist.h"
 #include "xe_force_wake.h"
+#include "xe_gsc.h"
 #include "xe_gt.h"
 #include "xe_gt_ccs_mode.h"
+#include "xe_gt_printk.h"
 #include "xe_gt_topology.h"
 #include "xe_hw_fence.h"
 #include "xe_irq.h"
@@ -463,6 +465,32 @@ static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe,
 		hwe->eclass->sched_props.preempt_timeout_us = XE_HW_ENGINE_PREEMPT_TIMEOUT;
 		hwe->eclass->sched_props.preempt_timeout_min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN;
 		hwe->eclass->sched_props.preempt_timeout_max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX;
+
+		/*
+		 * The GSC engine can accept submissions while the GSC shim is
+		 * being reset, during which time the submission is stalled. In
+		 * the worst case, the shim reset can take up to the maximum GSC
+		 * command execution time (250ms), so the request start can be
+		 * delayed by that much; the request itself can take that long
+		 * without being preemptible, which means worst case it can
+		 * theoretically take up to 500ms for a preemption to go through
+		 * on the GSC engine. Adding to that an extra 100ms as a safety
+		 * margin, we get a minimum recommended timeout of 600ms.
+		 * The preempt_timeout value can't be tuned for OTHER_CLASS
+		 * because the class is reserved for kernel usage, so we just
+		 * need to make sure that the starting value is above that
+		 * threshold; since our default value (640ms) is greater than
+		 * 600ms, the only way we can go below is via a kconfig setting.
+		 * If that happens, log it in dmesg and update the value.
+		 */
+		if (hwe->class == XE_ENGINE_CLASS_OTHER) {
+			const u32 min_preempt_timeout = 600 * 1000;
+			if (hwe->eclass->sched_props.preempt_timeout_us < min_preempt_timeout) {
+				hwe->eclass->sched_props.preempt_timeout_us = min_preempt_timeout;
+				xe_gt_notice(gt, "Increasing preempt_timeout for GSC to 600ms\n");
+			}
+		}
+
 		/* Record default props */
 		hwe->eclass->defaults = hwe->eclass->sched_props;
 	}
@@ -490,8 +518,9 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe,
 	xe_reg_sr_apply_whitelist(hwe);
 
 	hwe->hwsp = xe_managed_bo_create_pin_map(xe, tile, SZ_4K,
-						 XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-						 XE_BO_CREATE_GGTT_BIT);
+						 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+						 XE_BO_FLAG_GGTT |
+						 XE_BO_FLAG_GGTT_INVALIDATE);
 	if (IS_ERR(hwe->hwsp)) {
 		err = PTR_ERR(hwe->hwsp);
 		goto err_name;
@@ -509,18 +538,19 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe,
 		}
 	}
 
-	if (xe_device_uc_enabled(xe))
+	if (xe_device_uc_enabled(xe)) {
+		/* GSCCS has a special interrupt for reset */
+		if (hwe->class == XE_ENGINE_CLASS_OTHER)
+			hwe->irq_handler = xe_gsc_hwe_irq_handler;
+
 		xe_hw_engine_enable_ring(hwe);
+	}
 
 	/* We reserve the highest BCS instance for USM */
 	if (xe->info.has_usm && hwe->class == XE_ENGINE_CLASS_COPY)
 		gt->usm.reserved_bcs_instance = hwe->instance;
 
-	err = drmm_add_action_or_reset(&xe->drm, hw_engine_fini, hwe);
-	if (err)
-		return err;
-
-	return 0;
+	return drmm_add_action_or_reset(&xe->drm, hw_engine_fini, hwe);
 
 err_kernel_lrc:
 	xe_lrc_finish(&hwe->kernel_lrc);
diff --git a/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c b/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c
index 2345fb42fa39..844ec68cbbb8 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c
+++ b/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c
@@ -7,8 +7,10 @@
 #include <linux/kobject.h>
 #include <linux/sysfs.h>
 
+#include "xe_device.h"
 #include "xe_gt.h"
 #include "xe_hw_engine_class_sysfs.h"
+#include "xe_pm.h"
 
 #define MAX_ENGINE_CLASS_NAME_LEN    16
 static int xe_add_hw_engine_class_defaults(struct xe_device *xe,
@@ -70,7 +72,7 @@ static ssize_t job_timeout_max_show(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj);
 
-	return sprintf(buf, "%u\n", eclass->sched_props.job_timeout_max);
+	return sysfs_emit(buf, "%u\n", eclass->sched_props.job_timeout_max);
 }
 
 static const struct kobj_attribute job_timeout_max_attr =
@@ -106,7 +108,7 @@ static ssize_t job_timeout_min_show(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj);
 
-	return sprintf(buf, "%u\n", eclass->sched_props.job_timeout_min);
+	return sysfs_emit(buf, "%u\n", eclass->sched_props.job_timeout_min);
 }
 
 static const struct kobj_attribute job_timeout_min_attr =
@@ -139,7 +141,7 @@ static ssize_t job_timeout_show(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj);
 
-	return sprintf(buf, "%u\n", eclass->sched_props.job_timeout_ms);
+	return sysfs_emit(buf, "%u\n", eclass->sched_props.job_timeout_ms);
 }
 
 static const struct kobj_attribute job_timeout_attr =
@@ -150,7 +152,7 @@ static ssize_t job_timeout_default(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent);
 
-	return sprintf(buf, "%u\n", eclass->defaults.job_timeout_ms);
+	return sysfs_emit(buf, "%u\n", eclass->defaults.job_timeout_ms);
 }
 
 static const struct kobj_attribute job_timeout_def =
@@ -161,7 +163,7 @@ static ssize_t job_timeout_min_default(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent);
 
-	return sprintf(buf, "%u\n", eclass->defaults.job_timeout_min);
+	return sysfs_emit(buf, "%u\n", eclass->defaults.job_timeout_min);
 }
 
 static const struct kobj_attribute job_timeout_min_def =
@@ -172,7 +174,7 @@ static ssize_t job_timeout_max_default(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent);
 
-	return sprintf(buf, "%u\n", eclass->defaults.job_timeout_max);
+	return sysfs_emit(buf, "%u\n", eclass->defaults.job_timeout_max);
 }
 
 static const struct kobj_attribute job_timeout_max_def =
@@ -231,7 +233,7 @@ static ssize_t timeslice_duration_max_show(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj);
 
-	return sprintf(buf, "%u\n", eclass->sched_props.timeslice_max);
+	return sysfs_emit(buf, "%u\n", eclass->sched_props.timeslice_max);
 }
 
 static const struct kobj_attribute timeslice_duration_max_attr =
@@ -269,7 +271,7 @@ static ssize_t timeslice_duration_min_show(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj);
 
-	return sprintf(buf, "%u\n", eclass->sched_props.timeslice_min);
+	return sysfs_emit(buf, "%u\n", eclass->sched_props.timeslice_min);
 }
 
 static const struct kobj_attribute timeslice_duration_min_attr =
@@ -281,7 +283,7 @@ static ssize_t timeslice_duration_show(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj);
 
-	return sprintf(buf, "%u\n", eclass->sched_props.timeslice_us);
+	return sysfs_emit(buf, "%u\n", eclass->sched_props.timeslice_us);
 }
 
 static const struct kobj_attribute timeslice_duration_attr =
@@ -293,7 +295,7 @@ static ssize_t timeslice_default(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent);
 
-	return sprintf(buf, "%u\n", eclass->defaults.timeslice_us);
+	return sysfs_emit(buf, "%u\n", eclass->defaults.timeslice_us);
 }
 
 static const struct kobj_attribute timeslice_duration_def =
@@ -304,7 +306,7 @@ static ssize_t timeslice_min_default(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent);
 
-	return sprintf(buf, "%u\n", eclass->defaults.timeslice_min);
+	return sysfs_emit(buf, "%u\n", eclass->defaults.timeslice_min);
 }
 
 static const struct kobj_attribute timeslice_duration_min_def =
@@ -315,7 +317,7 @@ static ssize_t timeslice_max_default(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent);
 
-	return sprintf(buf, "%u\n", eclass->defaults.timeslice_max);
+	return sysfs_emit(buf, "%u\n", eclass->defaults.timeslice_max);
 }
 
 static const struct kobj_attribute timeslice_duration_max_def =
@@ -348,7 +350,7 @@ static ssize_t preempt_timeout_show(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj);
 
-	return sprintf(buf, "%u\n", eclass->sched_props.preempt_timeout_us);
+	return sysfs_emit(buf, "%u\n", eclass->sched_props.preempt_timeout_us);
 }
 
 static const struct kobj_attribute preempt_timeout_attr =
@@ -360,7 +362,7 @@ static ssize_t preempt_timeout_default(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent);
 
-	return sprintf(buf, "%u\n", eclass->defaults.preempt_timeout_us);
+	return sysfs_emit(buf, "%u\n", eclass->defaults.preempt_timeout_us);
 }
 
 static const struct kobj_attribute preempt_timeout_def =
@@ -372,7 +374,7 @@ static ssize_t preempt_timeout_min_default(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent);
 
-	return sprintf(buf, "%u\n", eclass->defaults.preempt_timeout_min);
+	return sysfs_emit(buf, "%u\n", eclass->defaults.preempt_timeout_min);
 }
 
 static const struct kobj_attribute preempt_timeout_min_def =
@@ -384,7 +386,7 @@ static ssize_t preempt_timeout_max_default(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj->parent);
 
-	return sprintf(buf, "%u\n", eclass->defaults.preempt_timeout_max);
+	return sysfs_emit(buf, "%u\n", eclass->defaults.preempt_timeout_max);
 }
 
 static const struct kobj_attribute preempt_timeout_max_def =
@@ -420,7 +422,7 @@ static ssize_t preempt_timeout_max_show(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj);
 
-	return sprintf(buf, "%u\n", eclass->sched_props.preempt_timeout_max);
+	return sysfs_emit(buf, "%u\n", eclass->sched_props.preempt_timeout_max);
 }
 
 static const struct kobj_attribute preempt_timeout_max_attr =
@@ -457,7 +459,7 @@ static ssize_t preempt_timeout_min_show(struct kobject *kobj,
 {
 	struct xe_hw_engine_class_intf *eclass = kobj_to_eclass(kobj);
 
-	return sprintf(buf, "%u\n", eclass->sched_props.preempt_timeout_min);
+	return sysfs_emit(buf, "%u\n", eclass->sched_props.preempt_timeout_min);
 }
 
 static const struct kobj_attribute preempt_timeout_min_attr =
@@ -498,8 +500,8 @@ static void kobj_xe_hw_engine_class_fini(struct drm_device *drm, void *arg)
 	kobject_put(kobj);
 }
 
-	static struct kobj_eclass *
-kobj_xe_hw_engine_class(struct xe_device *xe, struct kobject *parent, char *name)
+static struct kobj_eclass *
+kobj_xe_hw_engine_class(struct xe_device *xe, struct kobject *parent, const char *name)
 {
 	struct kobj_eclass *keclass;
 	int err = 0;
@@ -513,13 +515,13 @@ kobj_xe_hw_engine_class(struct xe_device *xe, struct kobject *parent, char *name
 		kobject_put(&keclass->base);
 		return NULL;
 	}
+	keclass->xe = xe;
 
 	err = drmm_add_action_or_reset(&xe->drm, kobj_xe_hw_engine_class_fini,
 				       &keclass->base);
 	if (err)
-		drm_warn(&xe->drm,
-			 "%s: drmm_add_action_or_reset failed, err: %d\n",
-			 __func__, err);
+		return NULL;
+
 	return keclass;
 }
 
@@ -550,13 +552,8 @@ static int xe_add_hw_engine_class_defaults(struct xe_device *xe,
 	if (err)
 		goto err_object;
 
-	err = drmm_add_action_or_reset(&xe->drm, hw_engine_class_defaults_fini,
-				       kobj);
-	if (err)
-		drm_warn(&xe->drm,
-			 "%s: drmm_add_action_or_reset failed, err: %d\n",
-			 __func__, err);
-	return err;
+	return drmm_add_action_or_reset(&xe->drm, hw_engine_class_defaults_fini, kobj);
+
 err_object:
 	kobject_put(kobj);
 	return err;
@@ -567,9 +564,51 @@ static void xe_hw_engine_sysfs_kobj_release(struct kobject *kobj)
 	kfree(kobj);
 }
 
+static ssize_t xe_hw_engine_class_sysfs_attr_show(struct kobject *kobj,
+						  struct attribute *attr,
+						  char *buf)
+{
+	struct xe_device *xe = kobj_to_xe(kobj);
+	struct kobj_attribute *kattr;
+	ssize_t ret = -EIO;
+
+	kattr = container_of(attr, struct kobj_attribute, attr);
+	if (kattr->show) {
+		xe_pm_runtime_get(xe);
+		ret = kattr->show(kobj, kattr, buf);
+		xe_pm_runtime_put(xe);
+	}
+
+	return ret;
+}
+
+static ssize_t xe_hw_engine_class_sysfs_attr_store(struct kobject *kobj,
+						   struct attribute *attr,
+						   const char *buf,
+						   size_t count)
+{
+	struct xe_device *xe = kobj_to_xe(kobj);
+	struct kobj_attribute *kattr;
+	ssize_t ret = -EIO;
+
+	kattr = container_of(attr, struct kobj_attribute, attr);
+	if (kattr->store) {
+		xe_pm_runtime_get(xe);
+		ret = kattr->store(kobj, kattr, buf, count);
+		xe_pm_runtime_put(xe);
+	}
+
+	return ret;
+}
+
+static const struct sysfs_ops xe_hw_engine_class_sysfs_ops = {
+	.show = xe_hw_engine_class_sysfs_attr_show,
+	.store = xe_hw_engine_class_sysfs_attr_store,
+};
+
 static const struct kobj_type xe_hw_engine_sysfs_kobj_type = {
 	.release = xe_hw_engine_sysfs_kobj_release,
-	.sysfs_ops = &kobj_sysfs_ops,
+	.sysfs_ops = &xe_hw_engine_class_sysfs_ops,
 };
 
 static void hw_engine_class_sysfs_fini(struct drm_device *drm, void *arg)
@@ -579,6 +618,24 @@ static void hw_engine_class_sysfs_fini(struct drm_device *drm, void *arg)
 	kobject_put(kobj);
 }
 
+static const char *xe_hw_engine_class_to_str(enum xe_engine_class class)
+{
+	switch (class) {
+	case XE_ENGINE_CLASS_RENDER:
+		return "rcs";
+	case XE_ENGINE_CLASS_VIDEO_DECODE:
+		return "vcs";
+	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
+		return "vecs";
+	case XE_ENGINE_CLASS_COPY:
+		return "bcs";
+	case XE_ENGINE_CLASS_COMPUTE:
+		return "ccs";
+	default:
+		return NULL;
+	}
+}
+
 /**
  * xe_hw_engine_class_sysfs_init - Init HW engine classes on GT.
  * @gt: Xe GT.
@@ -608,7 +665,7 @@ int xe_hw_engine_class_sysfs_init(struct xe_gt *gt)
 		goto err_object;
 
 	for_each_hw_engine(hwe, gt, id) {
-		char name[MAX_ENGINE_CLASS_NAME_LEN];
+		const char *name;
 		struct kobj_eclass *keclass;
 
 		if (hwe->class == XE_ENGINE_CLASS_OTHER ||
@@ -619,24 +676,8 @@ int xe_hw_engine_class_sysfs_init(struct xe_gt *gt)
 			continue;
 
 		class_mask |= 1 << hwe->class;
-
-		switch (hwe->class) {
-		case XE_ENGINE_CLASS_RENDER:
-			strcpy(name, "rcs");
-			break;
-		case XE_ENGINE_CLASS_VIDEO_DECODE:
-			strcpy(name, "vcs");
-			break;
-		case XE_ENGINE_CLASS_VIDEO_ENHANCE:
-			strcpy(name, "vecs");
-			break;
-		case XE_ENGINE_CLASS_COPY:
-			strcpy(name, "bcs");
-			break;
-		case XE_ENGINE_CLASS_COMPUTE:
-			strcpy(name, "ccs");
-			break;
-		default:
+		name = xe_hw_engine_class_to_str(hwe->class);
+		if (!name) {
 			err = -EINVAL;
 			goto err_object;
 		}
@@ -649,26 +690,16 @@ int xe_hw_engine_class_sysfs_init(struct xe_gt *gt)
 
 		keclass->eclass = hwe->eclass;
 		err = xe_add_hw_engine_class_defaults(xe, &keclass->base);
-		if (err) {
-			drm_warn(&xe->drm,
-				 "Add .defaults to engines failed!, err: %d\n",
-				 err);
+		if (err)
 			goto err_object;
-		}
 
 		err = sysfs_create_files(&keclass->base, files);
 		if (err)
 			goto err_object;
 	}
 
-	err = drmm_add_action_or_reset(&xe->drm, hw_engine_class_sysfs_fini,
-				       kobj);
-	if (err)
-		drm_warn(&xe->drm,
-			 "%s: drmm_add_action_or_reset failed, err: %d\n",
-			 __func__, err);
+	return drmm_add_action_or_reset(&xe->drm, hw_engine_class_sysfs_fini, kobj);
 
-	return err;
 err_object:
 	kobject_put(kobj);
 	return err;
diff --git a/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.h b/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.h
index ec5ba673b314..28a0d7c909c0 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.h
+++ b/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.h
@@ -26,6 +26,8 @@ struct kobj_eclass {
 	struct kobject base;
 	/** @eclass: A pointer to the hw engine class interface */
 	struct xe_hw_engine_class_intf *eclass;
+	/** @xe: A pointer to the xe device */
+	struct xe_device *xe;
 };
 
 static inline struct xe_hw_engine_class_intf *kobj_to_eclass(struct kobject *kobj)
@@ -33,4 +35,9 @@ static inline struct xe_hw_engine_class_intf *kobj_to_eclass(struct kobject *kob
 	return container_of(kobj, struct kobj_eclass, base)->eclass;
 }
 
+static inline struct xe_device *kobj_to_xe(struct kobject *kobj)
+{
+	return container_of(kobj, struct kobj_eclass, base)->xe;
+}
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_hw_fence.c b/drivers/gpu/drm/xe/xe_hw_fence.c
index a5de3e7b0bd6..f872ef103127 100644
--- a/drivers/gpu/drm/xe/xe_hw_fence.c
+++ b/drivers/gpu/drm/xe/xe_hw_fence.c
@@ -130,7 +130,7 @@ void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt,
 	ctx->irq = irq;
 	ctx->dma_fence_ctx = dma_fence_context_alloc(1);
 	ctx->next_seqno = XE_FENCE_INITIAL_SEQNO;
-	sprintf(ctx->name, "%s", name);
+	snprintf(ctx->name, sizeof(ctx->name), "%s", name);
 }
 
 void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx)
diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c
index 9ac7fbe201b3..453e601ddd5e 100644
--- a/drivers/gpu/drm/xe/xe_hwmon.c
+++ b/drivers/gpu/drm/xe/xe_hwmon.c
@@ -18,6 +18,7 @@
 #include "xe_pcode.h"
 #include "xe_pcode_api.h"
 #include "xe_sriov.h"
+#include "xe_pm.h"
 
 enum xe_hwmon_reg {
 	REG_PKG_RAPL_LIMIT,
@@ -33,6 +34,12 @@ enum xe_hwmon_reg_operation {
 	REG_READ64,
 };
 
+enum xe_hwmon_channel {
+	CHANNEL_CARD,
+	CHANNEL_PKG,
+	CHANNEL_MAX,
+};
+
 /*
  * SF_* - scale factors for particular quantities according to hwmon spec.
  */
@@ -68,61 +75,61 @@ struct xe_hwmon {
 	int scl_shift_energy;
 	/** @scl_shift_time: pkg time unit */
 	int scl_shift_time;
-	/** @ei: Energy info for energy1_input */
-	struct xe_hwmon_energy_info ei;
+	/** @ei: Energy info for energyN_input */
+	struct xe_hwmon_energy_info ei[CHANNEL_MAX];
 };
 
-static u32 xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg hwmon_reg)
+static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg hwmon_reg,
+				      int channel)
 {
 	struct xe_device *xe = gt_to_xe(hwmon->gt);
-	struct xe_reg reg = XE_REG(0);
 
 	switch (hwmon_reg) {
 	case REG_PKG_RAPL_LIMIT:
-		if (xe->info.platform == XE_PVC)
-			reg = PVC_GT0_PACKAGE_RAPL_LIMIT;
-		else if (xe->info.platform == XE_DG2)
-			reg = PCU_CR_PACKAGE_RAPL_LIMIT;
+		if (xe->info.platform == XE_PVC && channel == CHANNEL_PKG)
+			return PVC_GT0_PACKAGE_RAPL_LIMIT;
+		else if ((xe->info.platform == XE_DG2) && (channel == CHANNEL_PKG))
+			return PCU_CR_PACKAGE_RAPL_LIMIT;
 		break;
 	case REG_PKG_POWER_SKU:
-		if (xe->info.platform == XE_PVC)
-			reg = PVC_GT0_PACKAGE_POWER_SKU;
-		else if (xe->info.platform == XE_DG2)
-			reg = PCU_CR_PACKAGE_POWER_SKU;
+		if (xe->info.platform == XE_PVC && channel == CHANNEL_PKG)
+			return PVC_GT0_PACKAGE_POWER_SKU;
+		else if ((xe->info.platform == XE_DG2) && (channel == CHANNEL_PKG))
+			return PCU_CR_PACKAGE_POWER_SKU;
 		break;
 	case REG_PKG_POWER_SKU_UNIT:
 		if (xe->info.platform == XE_PVC)
-			reg = PVC_GT0_PACKAGE_POWER_SKU_UNIT;
+			return PVC_GT0_PACKAGE_POWER_SKU_UNIT;
 		else if (xe->info.platform == XE_DG2)
-			reg = PCU_CR_PACKAGE_POWER_SKU_UNIT;
+			return PCU_CR_PACKAGE_POWER_SKU_UNIT;
 		break;
 	case REG_GT_PERF_STATUS:
-		if (xe->info.platform == XE_DG2)
-			reg = GT_PERF_STATUS;
+		if (xe->info.platform == XE_DG2 && channel == CHANNEL_PKG)
+			return GT_PERF_STATUS;
 		break;
 	case REG_PKG_ENERGY_STATUS:
-		if (xe->info.platform == XE_PVC)
-			reg = PVC_GT0_PLATFORM_ENERGY_STATUS;
-		else if (xe->info.platform == XE_DG2)
-			reg = PCU_CR_PACKAGE_ENERGY_STATUS;
+		if (xe->info.platform == XE_PVC && channel == CHANNEL_PKG)
+			return PVC_GT0_PLATFORM_ENERGY_STATUS;
+		else if ((xe->info.platform == XE_DG2) && (channel == CHANNEL_PKG))
+			return PCU_CR_PACKAGE_ENERGY_STATUS;
 		break;
 	default:
 		drm_warn(&xe->drm, "Unknown xe hwmon reg id: %d\n", hwmon_reg);
 		break;
 	}
 
-	return reg.raw;
+	return XE_REG(0);
 }
 
 static void xe_hwmon_process_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg hwmon_reg,
 				 enum xe_hwmon_reg_operation operation, u64 *value,
-				 u32 clr, u32 set)
+				 u32 clr, u32 set, int channel)
 {
 	struct xe_reg reg;
 
-	reg.raw = xe_hwmon_get_reg(hwmon, hwmon_reg);
+	reg = xe_hwmon_get_reg(hwmon, hwmon_reg, channel);
 
-	if (!reg.raw)
+	if (!xe_reg_is_valid(reg))
 		return;
 
 	switch (operation) {
@@ -150,13 +157,13 @@ static void xe_hwmon_process_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg hwmon
  * same pattern for sysfs, allow arbitrary PL1 limits to be set but display
  * clamped values when read.
  */
-static void xe_hwmon_power_max_read(struct xe_hwmon *hwmon, long *value)
+static void xe_hwmon_power_max_read(struct xe_hwmon *hwmon, int channel, long *value)
 {
 	u64 reg_val, min, max;
 
 	mutex_lock(&hwmon->hwmon_lock);
 
-	xe_hwmon_process_reg(hwmon, REG_PKG_RAPL_LIMIT, REG_READ32, &reg_val, 0, 0);
+	xe_hwmon_process_reg(hwmon, REG_PKG_RAPL_LIMIT, REG_READ32, &reg_val, 0, 0, channel);
 	/* Check if PL1 limit is disabled */
 	if (!(reg_val & PKG_PWR_LIM_1_EN)) {
 		*value = PL1_DISABLE;
@@ -166,7 +173,7 @@ static void xe_hwmon_power_max_read(struct xe_hwmon *hwmon, long *value)
 	reg_val = REG_FIELD_GET(PKG_PWR_LIM_1, reg_val);
 	*value = mul_u64_u32_shr(reg_val, SF_POWER, hwmon->scl_shift_power);
 
-	xe_hwmon_process_reg(hwmon, REG_PKG_POWER_SKU, REG_READ64, &reg_val, 0, 0);
+	xe_hwmon_process_reg(hwmon, REG_PKG_POWER_SKU, REG_READ64, &reg_val, 0, 0, channel);
 	min = REG_FIELD_GET(PKG_MIN_PWR, reg_val);
 	min = mul_u64_u32_shr(min, SF_POWER, hwmon->scl_shift_power);
 	max = REG_FIELD_GET(PKG_MAX_PWR, reg_val);
@@ -178,7 +185,7 @@ unlock:
 	mutex_unlock(&hwmon->hwmon_lock);
 }
 
-static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, long value)
+static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, int channel, long value)
 {
 	int ret = 0;
 	u64 reg_val;
@@ -188,9 +195,9 @@ static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, long value)
 	/* Disable PL1 limit and verify, as limit cannot be disabled on all platforms */
 	if (value == PL1_DISABLE) {
 		xe_hwmon_process_reg(hwmon, REG_PKG_RAPL_LIMIT, REG_RMW32, &reg_val,
-				     PKG_PWR_LIM_1_EN, 0);
+				     PKG_PWR_LIM_1_EN, 0, channel);
 		xe_hwmon_process_reg(hwmon, REG_PKG_RAPL_LIMIT, REG_READ32, &reg_val,
-				     PKG_PWR_LIM_1_EN, 0);
+				     PKG_PWR_LIM_1_EN, 0, channel);
 
 		if (reg_val & PKG_PWR_LIM_1_EN) {
 			ret = -EOPNOTSUPP;
@@ -203,17 +210,17 @@ static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, long value)
 	reg_val = PKG_PWR_LIM_1_EN | REG_FIELD_PREP(PKG_PWR_LIM_1, reg_val);
 
 	xe_hwmon_process_reg(hwmon, REG_PKG_RAPL_LIMIT, REG_RMW32, &reg_val,
-			     PKG_PWR_LIM_1_EN | PKG_PWR_LIM_1, reg_val);
+			     PKG_PWR_LIM_1_EN | PKG_PWR_LIM_1, reg_val, channel);
 unlock:
 	mutex_unlock(&hwmon->hwmon_lock);
 	return ret;
 }
 
-static void xe_hwmon_power_rated_max_read(struct xe_hwmon *hwmon, long *value)
+static void xe_hwmon_power_rated_max_read(struct xe_hwmon *hwmon, int channel, long *value)
 {
 	u64 reg_val;
 
-	xe_hwmon_process_reg(hwmon, REG_PKG_POWER_SKU, REG_READ32, &reg_val, 0, 0);
+	xe_hwmon_process_reg(hwmon, REG_PKG_POWER_SKU, REG_READ32, &reg_val, 0, 0, channel);
 	reg_val = REG_FIELD_GET(PKG_TDP, reg_val);
 	*value = mul_u64_u32_shr(reg_val, SF_POWER, hwmon->scl_shift_power);
 }
@@ -236,16 +243,16 @@ static void xe_hwmon_power_rated_max_read(struct xe_hwmon *hwmon, long *value)
  * the hwmon API. Using x86_64 128 bit arithmetic (see mul_u64_u32_shr()),
  * a 'long' of 63 bits, SF_ENERGY of 1e6 (~20 bits) and
  * hwmon->scl_shift_energy of 14 bits we have 57 (63 - 20 + 14) bits before
- * energy1_input overflows. This at 1000 W is an overflow duration of 278 years.
+ * energyN_input overflows. This at 1000 W is an overflow duration of 278 years.
  */
 static void
-xe_hwmon_energy_get(struct xe_hwmon *hwmon, long *energy)
+xe_hwmon_energy_get(struct xe_hwmon *hwmon, int channel, long *energy)
 {
-	struct xe_hwmon_energy_info *ei = &hwmon->ei;
+	struct xe_hwmon_energy_info *ei = &hwmon->ei[channel];
 	u64 reg_val;
 
 	xe_hwmon_process_reg(hwmon, REG_PKG_ENERGY_STATUS, REG_READ32,
-			     &reg_val, 0, 0);
+			     &reg_val, 0, 0, channel);
 
 	if (reg_val >= ei->reg_val_prev)
 		ei->accum_energy += reg_val - ei->reg_val_prev;
@@ -259,23 +266,24 @@ xe_hwmon_energy_get(struct xe_hwmon *hwmon, long *energy)
 }
 
 static ssize_t
-xe_hwmon_power1_max_interval_show(struct device *dev, struct device_attribute *attr,
-				  char *buf)
+xe_hwmon_power_max_interval_show(struct device *dev, struct device_attribute *attr,
+				 char *buf)
 {
 	struct xe_hwmon *hwmon = dev_get_drvdata(dev);
 	u32 x, y, x_w = 2; /* 2 bits */
 	u64 r, tau4, out;
+	int sensor_index = to_sensor_dev_attr(attr)->index;
 
-	xe_device_mem_access_get(gt_to_xe(hwmon->gt));
+	xe_pm_runtime_get(gt_to_xe(hwmon->gt));
 
 	mutex_lock(&hwmon->hwmon_lock);
 
 	xe_hwmon_process_reg(hwmon, REG_PKG_RAPL_LIMIT,
-			     REG_READ32, &r, 0, 0);
+			     REG_READ32, &r, 0, 0, sensor_index);
 
 	mutex_unlock(&hwmon->hwmon_lock);
 
-	xe_device_mem_access_put(gt_to_xe(hwmon->gt));
+	xe_pm_runtime_put(gt_to_xe(hwmon->gt));
 
 	x = REG_FIELD_GET(PKG_PWR_LIM_1_TIME_X, r);
 	y = REG_FIELD_GET(PKG_PWR_LIM_1_TIME_Y, r);
@@ -299,14 +307,15 @@ xe_hwmon_power1_max_interval_show(struct device *dev, struct device_attribute *a
 }
 
 static ssize_t
-xe_hwmon_power1_max_interval_store(struct device *dev, struct device_attribute *attr,
-				   const char *buf, size_t count)
+xe_hwmon_power_max_interval_store(struct device *dev, struct device_attribute *attr,
+				  const char *buf, size_t count)
 {
 	struct xe_hwmon *hwmon = dev_get_drvdata(dev);
 	u32 x, y, rxy, x_w = 2; /* 2 bits */
 	u64 tau4, r, max_win;
 	unsigned long val;
 	int ret;
+	int sensor_index = to_sensor_dev_attr(attr)->index;
 
 	ret = kstrtoul(buf, 0, &val);
 	if (ret)
@@ -325,7 +334,7 @@ xe_hwmon_power1_max_interval_store(struct device *dev, struct device_attribute *
 
 	/*
 	 * val must be < max in hwmon interface units. The steps below are
-	 * explained in xe_hwmon_power1_max_interval_show()
+	 * explained in xe_hwmon_power_max_interval_show()
 	 */
 	r = FIELD_PREP(PKG_MAX_WIN, PKG_MAX_WIN_DEFAULT);
 	x = REG_FIELD_GET(PKG_MAX_WIN_X, r);
@@ -354,26 +363,31 @@ xe_hwmon_power1_max_interval_store(struct device *dev, struct device_attribute *
 
 	rxy = REG_FIELD_PREP(PKG_PWR_LIM_1_TIME_X, x) | REG_FIELD_PREP(PKG_PWR_LIM_1_TIME_Y, y);
 
-	xe_device_mem_access_get(gt_to_xe(hwmon->gt));
+	xe_pm_runtime_get(gt_to_xe(hwmon->gt));
 
 	mutex_lock(&hwmon->hwmon_lock);
 
 	xe_hwmon_process_reg(hwmon, REG_PKG_RAPL_LIMIT, REG_RMW32, (u64 *)&r,
-			     PKG_PWR_LIM_1_TIME, rxy);
+			     PKG_PWR_LIM_1_TIME, rxy, sensor_index);
 
 	mutex_unlock(&hwmon->hwmon_lock);
 
-	xe_device_mem_access_put(gt_to_xe(hwmon->gt));
+	xe_pm_runtime_put(gt_to_xe(hwmon->gt));
 
 	return count;
 }
 
 static SENSOR_DEVICE_ATTR(power1_max_interval, 0664,
-			  xe_hwmon_power1_max_interval_show,
-			  xe_hwmon_power1_max_interval_store, 0);
+			  xe_hwmon_power_max_interval_show,
+			  xe_hwmon_power_max_interval_store, CHANNEL_CARD);
+
+static SENSOR_DEVICE_ATTR(power2_max_interval, 0664,
+			  xe_hwmon_power_max_interval_show,
+			  xe_hwmon_power_max_interval_store, CHANNEL_PKG);
 
 static struct attribute *hwmon_attributes[] = {
 	&sensor_dev_attr_power1_max_interval.dev_attr.attr,
+	&sensor_dev_attr_power2_max_interval.dev_attr.attr,
 	NULL
 };
 
@@ -384,12 +398,11 @@ static umode_t xe_hwmon_attributes_visible(struct kobject *kobj,
 	struct xe_hwmon *hwmon = dev_get_drvdata(dev);
 	int ret = 0;
 
-	xe_device_mem_access_get(gt_to_xe(hwmon->gt));
+	xe_pm_runtime_get(gt_to_xe(hwmon->gt));
 
-	if (attr == &sensor_dev_attr_power1_max_interval.dev_attr.attr)
-		ret = xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT) ? attr->mode : 0;
+	ret = xe_reg_is_valid(xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, index)) ? attr->mode : 0;
 
-	xe_device_mem_access_put(gt_to_xe(hwmon->gt));
+	xe_pm_runtime_put(gt_to_xe(hwmon->gt));
 
 	return ret;
 }
@@ -405,10 +418,11 @@ static const struct attribute_group *hwmon_groups[] = {
 };
 
 static const struct hwmon_channel_info * const hwmon_info[] = {
-	HWMON_CHANNEL_INFO(power, HWMON_P_MAX | HWMON_P_RATED_MAX | HWMON_P_CRIT),
-	HWMON_CHANNEL_INFO(curr, HWMON_C_CRIT),
-	HWMON_CHANNEL_INFO(in, HWMON_I_INPUT),
-	HWMON_CHANNEL_INFO(energy, HWMON_E_INPUT),
+	HWMON_CHANNEL_INFO(power, HWMON_P_MAX | HWMON_P_RATED_MAX | HWMON_P_LABEL,
+			   HWMON_P_MAX | HWMON_P_RATED_MAX | HWMON_P_CRIT | HWMON_P_LABEL),
+	HWMON_CHANNEL_INFO(curr, HWMON_C_LABEL, HWMON_C_CRIT | HWMON_C_LABEL),
+	HWMON_CHANNEL_INFO(in, HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL),
+	HWMON_CHANNEL_INFO(energy, HWMON_E_INPUT | HWMON_E_LABEL, HWMON_E_INPUT | HWMON_E_LABEL),
 	NULL
 };
 
@@ -431,7 +445,8 @@ static int xe_hwmon_pcode_write_i1(struct xe_gt *gt, u32 uval)
 			      uval);
 }
 
-static int xe_hwmon_power_curr_crit_read(struct xe_hwmon *hwmon, long *value, u32 scale_factor)
+static int xe_hwmon_power_curr_crit_read(struct xe_hwmon *hwmon, int channel,
+					 long *value, u32 scale_factor)
 {
 	int ret;
 	u32 uval;
@@ -449,7 +464,8 @@ unlock:
 	return ret;
 }
 
-static int xe_hwmon_power_curr_crit_write(struct xe_hwmon *hwmon, long value, u32 scale_factor)
+static int xe_hwmon_power_curr_crit_write(struct xe_hwmon *hwmon, int channel,
+					  long value, u32 scale_factor)
 {
 	int ret;
 	u32 uval;
@@ -463,117 +479,131 @@ static int xe_hwmon_power_curr_crit_write(struct xe_hwmon *hwmon, long value, u3
 	return ret;
 }
 
-static void xe_hwmon_get_voltage(struct xe_hwmon *hwmon, long *value)
+static void xe_hwmon_get_voltage(struct xe_hwmon *hwmon, int channel, long *value)
 {
 	u64 reg_val;
 
 	xe_hwmon_process_reg(hwmon, REG_GT_PERF_STATUS,
-			     REG_READ32, &reg_val, 0, 0);
+			     REG_READ32, &reg_val, 0, 0, channel);
 	/* HW register value in units of 2.5 millivolt */
 	*value = DIV_ROUND_CLOSEST(REG_FIELD_GET(VOLTAGE_MASK, reg_val) * 2500, SF_VOLTAGE);
 }
 
 static umode_t
-xe_hwmon_power_is_visible(struct xe_hwmon *hwmon, u32 attr, int chan)
+xe_hwmon_power_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel)
 {
 	u32 uval;
 
 	switch (attr) {
 	case hwmon_power_max:
-		return xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT) ? 0664 : 0;
+		return xe_reg_is_valid(xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT,
+				       channel)) ? 0664 : 0;
 	case hwmon_power_rated_max:
-		return xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU) ? 0444 : 0;
+		return xe_reg_is_valid(xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU,
+				       channel)) ? 0444 : 0;
 	case hwmon_power_crit:
-		return (xe_hwmon_pcode_read_i1(hwmon->gt, &uval) ||
-			!(uval & POWER_SETUP_I1_WATTS)) ? 0 : 0644;
+		if (channel == CHANNEL_PKG)
+			return (xe_hwmon_pcode_read_i1(hwmon->gt, &uval) ||
+				!(uval & POWER_SETUP_I1_WATTS)) ? 0 : 0644;
+		break;
+	case hwmon_power_label:
+		return xe_reg_is_valid(xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU_UNIT,
+				       channel)) ? 0444 : 0;
 	default:
 		return 0;
 	}
+	return 0;
 }
 
 static int
-xe_hwmon_power_read(struct xe_hwmon *hwmon, u32 attr, int chan, long *val)
+xe_hwmon_power_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val)
 {
 	switch (attr) {
 	case hwmon_power_max:
-		xe_hwmon_power_max_read(hwmon, val);
+		xe_hwmon_power_max_read(hwmon, channel, val);
 		return 0;
 	case hwmon_power_rated_max:
-		xe_hwmon_power_rated_max_read(hwmon, val);
+		xe_hwmon_power_rated_max_read(hwmon, channel, val);
 		return 0;
 	case hwmon_power_crit:
-		return xe_hwmon_power_curr_crit_read(hwmon, val, SF_POWER);
+		return xe_hwmon_power_curr_crit_read(hwmon, channel, val, SF_POWER);
 	default:
 		return -EOPNOTSUPP;
 	}
 }
 
 static int
-xe_hwmon_power_write(struct xe_hwmon *hwmon, u32 attr, int chan, long val)
+xe_hwmon_power_write(struct xe_hwmon *hwmon, u32 attr, int channel, long val)
 {
 	switch (attr) {
 	case hwmon_power_max:
-		return xe_hwmon_power_max_write(hwmon, val);
+		return xe_hwmon_power_max_write(hwmon, channel, val);
 	case hwmon_power_crit:
-		return xe_hwmon_power_curr_crit_write(hwmon, val, SF_POWER);
+		return xe_hwmon_power_curr_crit_write(hwmon, channel, val, SF_POWER);
 	default:
 		return -EOPNOTSUPP;
 	}
 }
 
 static umode_t
-xe_hwmon_curr_is_visible(const struct xe_hwmon *hwmon, u32 attr)
+xe_hwmon_curr_is_visible(const struct xe_hwmon *hwmon, u32 attr, int channel)
 {
 	u32 uval;
 
 	switch (attr) {
 	case hwmon_curr_crit:
-		return (xe_hwmon_pcode_read_i1(hwmon->gt, &uval) ||
-			(uval & POWER_SETUP_I1_WATTS)) ? 0 : 0644;
+	case hwmon_curr_label:
+		if (channel == CHANNEL_PKG)
+			return (xe_hwmon_pcode_read_i1(hwmon->gt, &uval) ||
+				(uval & POWER_SETUP_I1_WATTS)) ? 0 : 0644;
+		break;
 	default:
 		return 0;
 	}
+	return 0;
 }
 
 static int
-xe_hwmon_curr_read(struct xe_hwmon *hwmon, u32 attr, long *val)
+xe_hwmon_curr_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val)
 {
 	switch (attr) {
 	case hwmon_curr_crit:
-		return xe_hwmon_power_curr_crit_read(hwmon, val, SF_CURR);
+		return xe_hwmon_power_curr_crit_read(hwmon, channel, val, SF_CURR);
 	default:
 		return -EOPNOTSUPP;
 	}
 }
 
 static int
-xe_hwmon_curr_write(struct xe_hwmon *hwmon, u32 attr, long val)
+xe_hwmon_curr_write(struct xe_hwmon *hwmon, u32 attr, int channel, long val)
 {
 	switch (attr) {
 	case hwmon_curr_crit:
-		return xe_hwmon_power_curr_crit_write(hwmon, val, SF_CURR);
+		return xe_hwmon_power_curr_crit_write(hwmon, channel, val, SF_CURR);
 	default:
 		return -EOPNOTSUPP;
 	}
 }
 
 static umode_t
-xe_hwmon_in_is_visible(struct xe_hwmon *hwmon, u32 attr)
+xe_hwmon_in_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel)
 {
 	switch (attr) {
 	case hwmon_in_input:
-		return xe_hwmon_get_reg(hwmon, REG_GT_PERF_STATUS) ? 0444 : 0;
+	case hwmon_in_label:
+		return xe_reg_is_valid(xe_hwmon_get_reg(hwmon, REG_GT_PERF_STATUS,
+				       channel)) ? 0444 : 0;
 	default:
 		return 0;
 	}
 }
 
 static int
-xe_hwmon_in_read(struct xe_hwmon *hwmon, u32 attr, long *val)
+xe_hwmon_in_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val)
 {
 	switch (attr) {
 	case hwmon_in_input:
-		xe_hwmon_get_voltage(hwmon, val);
+		xe_hwmon_get_voltage(hwmon, channel, val);
 		return 0;
 	default:
 		return -EOPNOTSUPP;
@@ -581,22 +611,24 @@ xe_hwmon_in_read(struct xe_hwmon *hwmon, u32 attr, long *val)
 }
 
 static umode_t
-xe_hwmon_energy_is_visible(struct xe_hwmon *hwmon, u32 attr)
+xe_hwmon_energy_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel)
 {
 	switch (attr) {
 	case hwmon_energy_input:
-		return xe_hwmon_get_reg(hwmon, REG_PKG_ENERGY_STATUS) ? 0444 : 0;
+	case hwmon_energy_label:
+		return xe_reg_is_valid(xe_hwmon_get_reg(hwmon, REG_PKG_ENERGY_STATUS,
+				       channel)) ? 0444 : 0;
 	default:
 		return 0;
 	}
 }
 
 static int
-xe_hwmon_energy_read(struct xe_hwmon *hwmon, u32 attr, long *val)
+xe_hwmon_energy_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val)
 {
 	switch (attr) {
 	case hwmon_energy_input:
-		xe_hwmon_energy_get(hwmon, val);
+		xe_hwmon_energy_get(hwmon, channel, val);
 		return 0;
 	default:
 		return -EOPNOTSUPP;
@@ -610,27 +642,27 @@ xe_hwmon_is_visible(const void *drvdata, enum hwmon_sensor_types type,
 	struct xe_hwmon *hwmon = (struct xe_hwmon *)drvdata;
 	int ret;
 
-	xe_device_mem_access_get(gt_to_xe(hwmon->gt));
+	xe_pm_runtime_get(gt_to_xe(hwmon->gt));
 
 	switch (type) {
 	case hwmon_power:
 		ret = xe_hwmon_power_is_visible(hwmon, attr, channel);
 		break;
 	case hwmon_curr:
-		ret = xe_hwmon_curr_is_visible(hwmon, attr);
+		ret = xe_hwmon_curr_is_visible(hwmon, attr, channel);
 		break;
 	case hwmon_in:
-		ret = xe_hwmon_in_is_visible(hwmon, attr);
+		ret = xe_hwmon_in_is_visible(hwmon, attr, channel);
 		break;
 	case hwmon_energy:
-		ret = xe_hwmon_energy_is_visible(hwmon, attr);
+		ret = xe_hwmon_energy_is_visible(hwmon, attr, channel);
 		break;
 	default:
 		ret = 0;
 		break;
 	}
 
-	xe_device_mem_access_put(gt_to_xe(hwmon->gt));
+	xe_pm_runtime_put(gt_to_xe(hwmon->gt));
 
 	return ret;
 }
@@ -642,27 +674,27 @@ xe_hwmon_read(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 	struct xe_hwmon *hwmon = dev_get_drvdata(dev);
 	int ret;
 
-	xe_device_mem_access_get(gt_to_xe(hwmon->gt));
+	xe_pm_runtime_get(gt_to_xe(hwmon->gt));
 
 	switch (type) {
 	case hwmon_power:
 		ret = xe_hwmon_power_read(hwmon, attr, channel, val);
 		break;
 	case hwmon_curr:
-		ret = xe_hwmon_curr_read(hwmon, attr, val);
+		ret = xe_hwmon_curr_read(hwmon, attr, channel, val);
 		break;
 	case hwmon_in:
-		ret = xe_hwmon_in_read(hwmon, attr, val);
+		ret = xe_hwmon_in_read(hwmon, attr, channel, val);
 		break;
 	case hwmon_energy:
-		ret = xe_hwmon_energy_read(hwmon, attr, val);
+		ret = xe_hwmon_energy_read(hwmon, attr, channel, val);
 		break;
 	default:
 		ret = -EOPNOTSUPP;
 		break;
 	}
 
-	xe_device_mem_access_put(gt_to_xe(hwmon->gt));
+	xe_pm_runtime_put(gt_to_xe(hwmon->gt));
 
 	return ret;
 }
@@ -674,29 +706,49 @@ xe_hwmon_write(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 	struct xe_hwmon *hwmon = dev_get_drvdata(dev);
 	int ret;
 
-	xe_device_mem_access_get(gt_to_xe(hwmon->gt));
+	xe_pm_runtime_get(gt_to_xe(hwmon->gt));
 
 	switch (type) {
 	case hwmon_power:
 		ret = xe_hwmon_power_write(hwmon, attr, channel, val);
 		break;
 	case hwmon_curr:
-		ret = xe_hwmon_curr_write(hwmon, attr, val);
+		ret = xe_hwmon_curr_write(hwmon, attr, channel, val);
 		break;
 	default:
 		ret = -EOPNOTSUPP;
 		break;
 	}
 
-	xe_device_mem_access_put(gt_to_xe(hwmon->gt));
+	xe_pm_runtime_put(gt_to_xe(hwmon->gt));
 
 	return ret;
 }
 
+static int xe_hwmon_read_label(struct device *dev,
+			       enum hwmon_sensor_types type,
+			       u32 attr, int channel, const char **str)
+{
+	switch (type) {
+	case hwmon_power:
+	case hwmon_energy:
+	case hwmon_curr:
+	case hwmon_in:
+		if (channel == CHANNEL_CARD)
+			*str = "card";
+		else if (channel == CHANNEL_PKG)
+			*str = "pkg";
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
 static const struct hwmon_ops hwmon_ops = {
 	.is_visible = xe_hwmon_is_visible,
 	.read = xe_hwmon_read,
 	.write = xe_hwmon_write,
+	.read_string = xe_hwmon_read_label,
 };
 
 static const struct hwmon_chip_info hwmon_chip_info = {
@@ -710,14 +762,15 @@ xe_hwmon_get_preregistration_info(struct xe_device *xe)
 	struct xe_hwmon *hwmon = xe->hwmon;
 	long energy;
 	u64 val_sku_unit = 0;
+	int channel;
 
 	/*
 	 * The contents of register PKG_POWER_SKU_UNIT do not change,
 	 * so read it once and store the shift values.
 	 */
-	if (xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU_UNIT)) {
+	if (xe_reg_is_valid(xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU_UNIT, 0))) {
 		xe_hwmon_process_reg(hwmon, REG_PKG_POWER_SKU_UNIT,
-				     REG_READ32, &val_sku_unit, 0, 0);
+				     REG_READ32, &val_sku_unit, 0, 0, 0);
 		hwmon->scl_shift_power = REG_FIELD_GET(PKG_PWR_UNIT, val_sku_unit);
 		hwmon->scl_shift_energy = REG_FIELD_GET(PKG_ENERGY_UNIT, val_sku_unit);
 		hwmon->scl_shift_time = REG_FIELD_GET(PKG_TIME_UNIT, val_sku_unit);
@@ -727,8 +780,9 @@ xe_hwmon_get_preregistration_info(struct xe_device *xe)
 	 * Initialize 'struct xe_hwmon_energy_info', i.e. set fields to the
 	 * first value of the energy register read
 	 */
-	if (xe_hwmon_is_visible(hwmon, hwmon_energy, hwmon_energy_input, 0))
-		xe_hwmon_energy_get(hwmon, &energy);
+	for (channel = 0; channel < CHANNEL_MAX; channel++)
+		if (xe_hwmon_is_visible(hwmon, hwmon_energy, hwmon_energy_input, channel))
+			xe_hwmon_energy_get(hwmon, channel, &energy);
 }
 
 static void xe_hwmon_mutex_destroy(void *arg)
diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c
index 2f5d179e0d00..996806353171 100644
--- a/drivers/gpu/drm/xe/xe_irq.c
+++ b/drivers/gpu/drm/xe/xe_irq.c
@@ -187,7 +187,7 @@ void xe_irq_enable_hwe(struct xe_gt *gt)
 		 * GSCCS interrupts, but it has its own mask register.
 		 */
 		if (xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_OTHER)) {
-			gsc_mask = irqs;
+			gsc_mask = irqs | GSC_ER_COMPLETE;
 			heci_mask = GSC_IRQ_INTF(1);
 		} else if (HAS_HECI_GSCFI(xe)) {
 			gsc_mask = GSC_IRQ_INTF(1);
@@ -326,7 +326,6 @@ static void gt_irq_handler(struct xe_tile *tile,
 					xe_heci_gsc_irq_handler(xe, intr_vec);
 				else
 					gt_other_irq_handler(engine_gt, instance, intr_vec);
-				continue;
 			}
 		}
 	}
diff --git a/drivers/gpu/drm/xe/xe_lmtt.c b/drivers/gpu/drm/xe/xe_lmtt.c
index 0d7c5514e092..418661a88918 100644
--- a/drivers/gpu/drm/xe/xe_lmtt.c
+++ b/drivers/gpu/drm/xe/xe_lmtt.c
@@ -35,7 +35,7 @@
 
 static bool xe_has_multi_level_lmtt(struct xe_device *xe)
 {
-	return xe->info.platform == XE_PVC;
+	return GRAPHICS_VERx100(xe) >= 1260;
 }
 
 static struct xe_tile *lmtt_to_tile(struct xe_lmtt *lmtt)
@@ -70,8 +70,8 @@ static struct xe_lmtt_pt *lmtt_pt_alloc(struct xe_lmtt *lmtt, unsigned int level
 				  PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) *
 					     lmtt->ops->lmtt_pte_num(level)),
 				  ttm_bo_type_kernel,
-				  XE_BO_CREATE_VRAM_IF_DGFX(lmtt_to_tile(lmtt)) |
-				  XE_BO_CREATE_PINNED_BIT);
+				  XE_BO_FLAG_VRAM_IF_DGFX(lmtt_to_tile(lmtt)) |
+				  XE_BO_NEEDS_64K | XE_BO_FLAG_PINNED);
 	if (IS_ERR(bo)) {
 		err = PTR_ERR(bo);
 		goto out_free_pt;
diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c
index 57066faf575e..615bbc372ac6 100644
--- a/drivers/gpu/drm/xe/xe_lrc.c
+++ b/drivers/gpu/drm/xe/xe_lrc.c
@@ -5,8 +5,11 @@
 
 #include "xe_lrc.h"
 
+#include <linux/ascii85.h>
+
 #include "instructions/xe_mi_commands.h"
 #include "instructions/xe_gfxpipe_commands.h"
+#include "instructions/xe_gfx_state_commands.h"
 #include "regs/xe_engine_regs.h"
 #include "regs/xe_gpu_commands.h"
 #include "regs/xe_lrc_layout.h"
@@ -23,13 +26,28 @@
 #include "xe_sriov.h"
 #include "xe_vm.h"
 
-#define LRC_VALID				(1 << 0)
-#define LRC_PRIVILEGE				(1 << 8)
-#define LRC_ADDRESSING_MODE_SHIFT		3
+#define LRC_VALID				BIT_ULL(0)
+#define LRC_PRIVILEGE				BIT_ULL(8)
+#define LRC_ADDRESSING_MODE			GENMASK_ULL(4, 3)
 #define LRC_LEGACY_64B_CONTEXT			3
 
-#define ENGINE_CLASS_SHIFT			61
-#define ENGINE_INSTANCE_SHIFT			48
+#define LRC_ENGINE_CLASS			GENMASK_ULL(63, 61)
+#define LRC_ENGINE_INSTANCE			GENMASK_ULL(53, 48)
+
+struct xe_lrc_snapshot {
+	struct xe_bo *lrc_bo;
+	void *lrc_snapshot;
+	unsigned long lrc_size, lrc_offset;
+
+	u32 context_desc;
+	u32 head;
+	struct {
+		u32 internal;
+		u32 memory;
+	} tail;
+	u32 start_seqno;
+	u32 seqno;
+};
 
 static struct xe_device *
 lrc_to_xe(struct xe_lrc *lrc)
@@ -634,7 +652,7 @@ static inline struct iosys_map __xe_lrc_##elem##_map(struct xe_lrc *lrc) \
 	iosys_map_incr(&map, __xe_lrc_##elem##_offset(lrc)); \
 	return map; \
 } \
-static inline u32 __xe_lrc_##elem##_ggtt_addr(struct xe_lrc *lrc) \
+static inline u32 __maybe_unused __xe_lrc_##elem##_ggtt_addr(struct xe_lrc *lrc) \
 { \
 	return xe_bo_ggtt_addr(lrc->bo) + __xe_lrc_##elem##_offset(lrc); \
 } \
@@ -724,8 +742,9 @@ int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 	lrc->bo = xe_bo_create_pin_map(xe, tile, vm,
 				      ring_size + xe_lrc_size(xe, hwe->class),
 				      ttm_bo_type_kernel,
-				      XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-				      XE_BO_CREATE_GGTT_BIT);
+				      XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+				      XE_BO_FLAG_GGTT |
+				      XE_BO_FLAG_GGTT_INVALIDATE);
 	if (IS_ERR(lrc->bo))
 		return PTR_ERR(lrc->bo);
 
@@ -776,7 +795,7 @@ int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 		xe_lrc_write_ctx_reg(lrc, PVC_CTX_ASID, vm->usm.asid);
 
 	lrc->desc = LRC_VALID;
-	lrc->desc |= LRC_LEGACY_64B_CONTEXT << LRC_ADDRESSING_MODE_SHIFT;
+	lrc->desc |= FIELD_PREP(LRC_ADDRESSING_MODE, LRC_LEGACY_64B_CONTEXT);
 	/* TODO: Priority */
 
 	/* While this appears to have something about privileged batches or
@@ -786,8 +805,8 @@ int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 		lrc->desc |= LRC_PRIVILEGE;
 
 	if (GRAPHICS_VERx100(xe) < 1250) {
-		lrc->desc |= (u64)hwe->instance << ENGINE_INSTANCE_SHIFT;
-		lrc->desc |= (u64)hwe->class << ENGINE_CLASS_SHIFT;
+		lrc->desc |= FIELD_PREP(LRC_ENGINE_INSTANCE, hwe->instance);
+		lrc->desc |= FIELD_PREP(LRC_ENGINE_CLASS, hwe->class);
 	}
 
 	arb_enable = MI_ARB_ON_OFF | MI_ARB_ENABLE;
@@ -1034,6 +1053,8 @@ static int dump_gfxpipe_command(struct drm_printer *p,
 	MATCH(GPGPU_CSR_BASE_ADDRESS);
 	MATCH(STATE_COMPUTE_MODE);
 	MATCH3D(3DSTATE_BTD);
+	MATCH(STATE_SYSTEM_MEM_FENCE_ADDRESS);
+	MATCH(STATE_CONTEXT_DATA_BASE_ADDRESS);
 
 	MATCH3D(3DSTATE_VF_STATISTICS);
 
@@ -1058,6 +1079,7 @@ static int dump_gfxpipe_command(struct drm_printer *p,
 	MATCH3D(3DSTATE_WM);
 	MATCH3D(3DSTATE_CONSTANT_VS);
 	MATCH3D(3DSTATE_CONSTANT_GS);
+	MATCH3D(3DSTATE_CONSTANT_PS);
 	MATCH3D(3DSTATE_SAMPLE_MASK);
 	MATCH3D(3DSTATE_CONSTANT_HS);
 	MATCH3D(3DSTATE_CONSTANT_DS);
@@ -1150,6 +1172,31 @@ static int dump_gfxpipe_command(struct drm_printer *p,
 	}
 }
 
+static int dump_gfx_state_command(struct drm_printer *p,
+				  struct xe_gt *gt,
+				  u32 *dw,
+				  int remaining_dw)
+{
+	u32 numdw = instr_dw(*dw);
+	u32 opcode = REG_FIELD_GET(GFX_STATE_OPCODE, *dw);
+
+	/*
+	 * Make sure we haven't mis-parsed a number of dwords that exceeds the
+	 * remaining size of the LRC.
+	 */
+	if (xe_gt_WARN_ON(gt, numdw > remaining_dw))
+		numdw = remaining_dw;
+
+	switch (*dw & (XE_INSTR_GFX_STATE | GFX_STATE_OPCODE)) {
+	MATCH(STATE_WRITE_INLINE);
+
+	default:
+		drm_printf(p, "[%#010x] unknown GFX_STATE command (opcode=%#x), likely %d dwords\n",
+			   *dw, opcode, numdw);
+		return numdw;
+	}
+}
+
 void xe_lrc_dump_default(struct drm_printer *p,
 			 struct xe_gt *gt,
 			 enum xe_engine_class hwe_class)
@@ -1174,6 +1221,8 @@ void xe_lrc_dump_default(struct drm_printer *p,
 			num_dw = dump_mi_command(p, gt, dw, remaining_dw);
 		} else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFXPIPE) {
 			num_dw = dump_gfxpipe_command(p, gt, dw, remaining_dw);
+		} else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFX_STATE) {
+			num_dw = dump_gfx_state_command(p, gt, dw, remaining_dw);
 		} else {
 			num_dw = min(instr_dw(*dw), remaining_dw);
 			drm_printf(p, "[%#10x] Unknown instruction of type %#x, likely %d dwords\n",
@@ -1297,3 +1346,101 @@ void xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, struct xe_bb *b
 		bb->len += num_dw;
 	}
 }
+
+struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc)
+{
+	struct xe_lrc_snapshot *snapshot = kmalloc(sizeof(*snapshot), GFP_NOWAIT);
+
+	if (!snapshot)
+		return NULL;
+
+	snapshot->context_desc = lower_32_bits(xe_lrc_ggtt_addr(lrc));
+	snapshot->head = xe_lrc_ring_head(lrc);
+	snapshot->tail.internal = lrc->ring.tail;
+	snapshot->tail.memory = xe_lrc_read_ctx_reg(lrc, CTX_RING_TAIL);
+	snapshot->start_seqno = xe_lrc_start_seqno(lrc);
+	snapshot->seqno = xe_lrc_seqno(lrc);
+	snapshot->lrc_bo = xe_bo_get(lrc->bo);
+	snapshot->lrc_offset = xe_lrc_pphwsp_offset(lrc);
+	snapshot->lrc_size = lrc->bo->size - snapshot->lrc_offset;
+	snapshot->lrc_snapshot = NULL;
+	return snapshot;
+}
+
+void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot)
+{
+	struct xe_bo *bo;
+	struct iosys_map src;
+
+	if (!snapshot)
+		return;
+
+	bo = snapshot->lrc_bo;
+	snapshot->lrc_bo = NULL;
+
+	snapshot->lrc_snapshot = kvmalloc(snapshot->lrc_size, GFP_KERNEL);
+	if (!snapshot->lrc_snapshot)
+		goto put_bo;
+
+	dma_resv_lock(bo->ttm.base.resv, NULL);
+	if (!ttm_bo_vmap(&bo->ttm, &src)) {
+		xe_map_memcpy_from(xe_bo_device(bo),
+				   snapshot->lrc_snapshot, &src, snapshot->lrc_offset,
+				   snapshot->lrc_size);
+		ttm_bo_vunmap(&bo->ttm, &src);
+	} else {
+		kvfree(snapshot->lrc_snapshot);
+		snapshot->lrc_snapshot = NULL;
+	}
+	dma_resv_unlock(bo->ttm.base.resv);
+put_bo:
+	xe_bo_put(bo);
+}
+
+void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer *p)
+{
+	unsigned long i;
+
+	if (!snapshot)
+		return;
+
+	drm_printf(p, "\tHW Context Desc: 0x%08x\n", snapshot->context_desc);
+	drm_printf(p, "\tLRC Head: (memory) %u\n", snapshot->head);
+	drm_printf(p, "\tLRC Tail: (internal) %u, (memory) %u\n",
+		   snapshot->tail.internal, snapshot->tail.memory);
+	drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno);
+	drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno);
+
+	if (!snapshot->lrc_snapshot)
+		return;
+
+	drm_printf(p, "\t[HWSP].length: 0x%x\n", LRC_PPHWSP_SIZE);
+	drm_puts(p, "\t[HWSP].data: ");
+	for (i = 0; i < LRC_PPHWSP_SIZE; i += sizeof(u32)) {
+		u32 *val = snapshot->lrc_snapshot + i;
+		char dumped[ASCII85_BUFSZ];
+
+		drm_puts(p, ascii85_encode(*val, dumped));
+	}
+
+	drm_printf(p, "\n\t[HWCTX].length: 0x%lx\n", snapshot->lrc_size - LRC_PPHWSP_SIZE);
+	drm_puts(p, "\t[HWCTX].data: ");
+	for (; i < snapshot->lrc_size; i += sizeof(u32)) {
+		u32 *val = snapshot->lrc_snapshot + i;
+		char dumped[ASCII85_BUFSZ];
+
+		drm_puts(p, ascii85_encode(*val, dumped));
+	}
+	drm_puts(p, "\n");
+}
+
+void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot)
+{
+	if (!snapshot)
+		return;
+
+	kvfree(snapshot->lrc_snapshot);
+	if (snapshot->lrc_bo)
+		xe_bo_put(snapshot->lrc_bo);
+	kfree(snapshot);
+}
diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h
index 28b1d3f404d4..d32fa31faa2c 100644
--- a/drivers/gpu/drm/xe/xe_lrc.h
+++ b/drivers/gpu/drm/xe/xe_lrc.h
@@ -55,4 +55,9 @@ void xe_lrc_dump_default(struct drm_printer *p,
 
 void xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, struct xe_bb *bb);
 
+struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc);
+void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot);
+void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer *p);
+void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot);
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h
index 24f20ed66fd1..b716df0dfb4e 100644
--- a/drivers/gpu/drm/xe/xe_lrc_types.h
+++ b/drivers/gpu/drm/xe/xe_lrc_types.h
@@ -43,4 +43,6 @@ struct xe_lrc {
 	struct xe_hw_fence_ctx fence_ctx;
 };
 
+struct xe_lrc_snapshot;
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_memirq.c b/drivers/gpu/drm/xe/xe_memirq.c
index 76e95535d7f6..95b6e9d7b7db 100644
--- a/drivers/gpu/drm/xe/xe_memirq.c
+++ b/drivers/gpu/drm/xe/xe_memirq.c
@@ -127,10 +127,11 @@ static int memirq_alloc_pages(struct xe_memirq *memirq)
 	/* XXX: convert to managed bo */
 	bo = xe_bo_create_pin_map(xe, tile, NULL, SZ_4K,
 				  ttm_bo_type_kernel,
-				  XE_BO_CREATE_SYSTEM_BIT |
-				  XE_BO_CREATE_GGTT_BIT |
-				  XE_BO_NEEDS_UC |
-				  XE_BO_NEEDS_CPU_ACCESS);
+				  XE_BO_FLAG_SYSTEM |
+				  XE_BO_FLAG_GGTT |
+				  XE_BO_FLAG_GGTT_INVALIDATE |
+				  XE_BO_FLAG_NEEDS_UC |
+				  XE_BO_FLAG_NEEDS_CPU_ACCESS);
 	if (IS_ERR(bo)) {
 		err = PTR_ERR(bo);
 		goto out;
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index 2ba4fb9511f6..65e5a3f4c340 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -16,6 +16,7 @@
 
 #include "instructions/xe_mi_commands.h"
 #include "regs/xe_gpu_commands.h"
+#include "regs/xe_gtt_defs.h"
 #include "tests/xe_test.h"
 #include "xe_assert.h"
 #include "xe_bb.h"
@@ -33,7 +34,6 @@
 #include "xe_sync.h"
 #include "xe_trace.h"
 #include "xe_vm.h"
-#include "xe_wa.h"
 
 /**
  * struct xe_migrate - migrate context.
@@ -155,8 +155,8 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
 	bo = xe_bo_create_pin_map(vm->xe, tile, vm,
 				  num_entries * XE_PAGE_SIZE,
 				  ttm_bo_type_kernel,
-				  XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-				  XE_BO_CREATE_PINNED_BIT);
+				  XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+				  XE_BO_FLAG_PINNED);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
@@ -299,10 +299,6 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
 }
 
 /*
- * Due to workaround 16017236439, odd instance hardware copy engines are
- * faster than even instance ones.
- * This function returns the mask involving all fast copy engines and the
- * reserved copy engine to be used as logical mask for migrate engine.
  * Including the reserved copy engine is required to avoid deadlocks due to
  * migrate jobs servicing the faults gets stuck behind the job that faulted.
  */
@@ -316,8 +312,7 @@ static u32 xe_migrate_usm_logical_mask(struct xe_gt *gt)
 		if (hwe->class != XE_ENGINE_CLASS_COPY)
 			continue;
 
-		if (!XE_WA(gt, 16017236439) ||
-		    xe_gt_is_usm_hwe(gt, hwe) || hwe->instance & 1)
+		if (xe_gt_is_usm_hwe(gt, hwe))
 			logical_mask |= BIT(hwe->logical_instance);
 	}
 
@@ -368,6 +363,10 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile)
 		if (!hwe || !logical_mask)
 			return ERR_PTR(-EINVAL);
 
+		/*
+		 * XXX: Currently only reserving 1 (likely slow) BCS instance on
+		 * PVC, may want to revisit if performance is needed.
+		 */
 		m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe,
 					    EXEC_QUEUE_FLAG_KERNEL |
 					    EXEC_QUEUE_FLAG_PERMANENT |
@@ -984,7 +983,6 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
 	struct xe_res_cursor src_it;
 	struct ttm_resource *src = dst;
 	int err;
-	int pass = 0;
 
 	if (!clear_vram)
 		xe_res_first_sg(xe_bo_sg(bo), 0, bo->size, &src_it);
@@ -1005,8 +1003,6 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
 
 		clear_L0 = xe_migrate_res_sizes(m, &src_it);
 
-		drm_dbg(&xe->drm, "Pass %u, size: %llu\n", pass++, clear_L0);
-
 		/* Calculate final sizes and batch size.. */
 		batch_size = 2 +
 			pte_update_size(m, clear_vram, src, &src_it,
diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c
index 7ba2477452d7..334637511e75 100644
--- a/drivers/gpu/drm/xe/xe_mmio.c
+++ b/drivers/gpu/drm/xe/xe_mmio.c
@@ -163,6 +163,42 @@ static int xe_determine_lmem_bar_size(struct xe_device *xe)
 	return 0;
 }
 
+static inline u64 get_flat_ccs_offset(struct xe_gt *gt, u64 tile_size)
+{
+	struct xe_device *xe = gt_to_xe(gt);
+	u64 offset;
+	u32 reg;
+
+	if (GRAPHICS_VER(xe) >= 20) {
+		u64 ccs_size = tile_size / 512;
+		u64 offset_hi, offset_lo;
+		u32 nodes, num_enabled;
+
+		reg = xe_mmio_read32(gt, MIRROR_FUSE3);
+		nodes = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, reg);
+		num_enabled = hweight32(nodes); /* Number of enabled l3 nodes */
+
+		reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
+		offset_lo = REG_FIELD_GET(XE2_FLAT_CCS_BASE_LOWER_ADDR_MASK, reg);
+
+		reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_UPPER);
+		offset_hi = REG_FIELD_GET(XE2_FLAT_CCS_BASE_UPPER_ADDR_MASK, reg);
+
+		offset = offset_hi << 32; /* HW view bits 39:32 */
+		offset |= offset_lo << 6; /* HW view bits 31:6 */
+		offset *= num_enabled; /* convert to SW view */
+
+		/* We don't expect any holes */
+		xe_assert_msg(xe, offset == (xe_mmio_read64_2x32(gt, GSMBASE) - ccs_size),
+			      "Hole between CCS and GSM.\n");
+	} else {
+		reg = xe_gt_mcr_unicast_read_any(gt, XEHP_FLAT_CCS_BASE_ADDR);
+		offset = (u64)REG_FIELD_GET(XEHP_FLAT_CCS_PTR, reg) * SZ_64K;
+	}
+
+	return offset;
+}
+
 /**
  * xe_mmio_tile_vram_size() - Collect vram size and offset information
  * @tile: tile to get info for
@@ -207,8 +243,7 @@ static int xe_mmio_tile_vram_size(struct xe_tile *tile, u64 *vram_size,
 
 	/* minus device usage */
 	if (xe->info.has_flat_ccs) {
-		reg = xe_gt_mcr_unicast_read_any(gt, XEHP_FLAT_CCS_BASE_ADDR);
-		offset = (u64)REG_FIELD_GET(GENMASK(31, 8), reg) * SZ_64K;
+		offset = get_flat_ccs_offset(gt, *tile_size);
 	} else {
 		offset = xe_mmio_read64_2x32(gt, GSMBASE);
 	}
@@ -360,32 +395,9 @@ static void mmio_fini(struct drm_device *drm, void *arg)
 		iounmap(xe->mem.vram.mapping);
 }
 
-static int xe_verify_lmem_ready(struct xe_device *xe)
-{
-	struct xe_gt *gt = xe_root_mmio_gt(xe);
-
-	if (!IS_DGFX(xe))
-		return 0;
-
-	if (IS_SRIOV_VF(xe))
-		return 0;
-
-	/*
-	 * The boot firmware initializes local memory and assesses its health.
-	 * If memory training fails, the punit will have been instructed to
-	 * keep the GT powered down; we won't be able to communicate with it
-	 * and we should not continue with driver initialization.
-	 */
-	if (!(xe_mmio_read32(gt, GU_CNTL) & LMEM_INIT)) {
-		drm_err(&xe->drm, "VRAM not initialized by firmware\n");
-		return -ENODEV;
-	}
-
-	return 0;
-}
-
 int xe_mmio_init(struct xe_device *xe)
 {
+	struct xe_tile *root_tile = xe_device_get_root_tile(xe);
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
 	const int mmio_bar = 0;
 
@@ -401,23 +413,83 @@ int xe_mmio_init(struct xe_device *xe)
 		return -EIO;
 	}
 
+	/* Setup first tile; other tiles (if present) will be setup later. */
+	root_tile->mmio.size = SZ_16M;
+	root_tile->mmio.regs = xe->mmio.regs;
+
 	return drmm_add_action_or_reset(&xe->drm, mmio_fini, xe);
 }
 
-int xe_mmio_root_tile_init(struct xe_device *xe)
+u8 xe_mmio_read8(struct xe_gt *gt, struct xe_reg reg)
 {
-	struct xe_tile *root_tile = xe_device_get_root_tile(xe);
-	int err;
+	struct xe_tile *tile = gt_to_tile(gt);
 
-	/* Setup first tile; other tiles (if present) will be setup later. */
-	root_tile->mmio.size = SZ_16M;
-	root_tile->mmio.regs = xe->mmio.regs;
+	if (reg.addr < gt->mmio.adj_limit)
+		reg.addr += gt->mmio.adj_offset;
 
-	err = xe_verify_lmem_ready(xe);
-	if (err)
-		return err;
+	return readb((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + reg.addr);
+}
 
-	return 0;
+u16 xe_mmio_read16(struct xe_gt *gt, struct xe_reg reg)
+{
+	struct xe_tile *tile = gt_to_tile(gt);
+
+	if (reg.addr < gt->mmio.adj_limit)
+		reg.addr += gt->mmio.adj_offset;
+
+	return readw((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + reg.addr);
+}
+
+void xe_mmio_write32(struct xe_gt *gt, struct xe_reg reg, u32 val)
+{
+	struct xe_tile *tile = gt_to_tile(gt);
+
+	if (reg.addr < gt->mmio.adj_limit)
+		reg.addr += gt->mmio.adj_offset;
+
+	writel(val, (reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + reg.addr);
+}
+
+u32 xe_mmio_read32(struct xe_gt *gt, struct xe_reg reg)
+{
+	struct xe_tile *tile = gt_to_tile(gt);
+
+	if (reg.addr < gt->mmio.adj_limit)
+		reg.addr += gt->mmio.adj_offset;
+
+	return readl((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + reg.addr);
+}
+
+u32 xe_mmio_rmw32(struct xe_gt *gt, struct xe_reg reg, u32 clr, u32 set)
+{
+	u32 old, reg_val;
+
+	old = xe_mmio_read32(gt, reg);
+	reg_val = (old & ~clr) | set;
+	xe_mmio_write32(gt, reg, reg_val);
+
+	return old;
+}
+
+int xe_mmio_write32_and_verify(struct xe_gt *gt,
+			       struct xe_reg reg, u32 val, u32 mask, u32 eval)
+{
+	u32 reg_val;
+
+	xe_mmio_write32(gt, reg, val);
+	reg_val = xe_mmio_read32(gt, reg);
+
+	return (reg_val & mask) != eval ? -EINVAL : 0;
+}
+
+bool xe_mmio_in_range(const struct xe_gt *gt,
+		      const struct xe_mmio_range *range,
+		      struct xe_reg reg)
+{
+	if (reg.addr < gt->mmio.adj_limit)
+		reg.addr += gt->mmio.adj_offset;
+
+	return range && reg.addr >= range->start && reg.addr <= range->end;
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_mmio.h b/drivers/gpu/drm/xe/xe_mmio.h
index 98de5c13c89b..a3cd7b3036c7 100644
--- a/drivers/gpu/drm/xe/xe_mmio.h
+++ b/drivers/gpu/drm/xe/xe_mmio.h
@@ -21,83 +21,15 @@ struct xe_device;
 #define LMEM_BAR		2
 
 int xe_mmio_init(struct xe_device *xe);
-int xe_mmio_root_tile_init(struct xe_device *xe);
 void xe_mmio_probe_tiles(struct xe_device *xe);
 
-static inline u8 xe_mmio_read8(struct xe_gt *gt, struct xe_reg reg)
-{
-	struct xe_tile *tile = gt_to_tile(gt);
-
-	if (reg.addr < gt->mmio.adj_limit)
-		reg.addr += gt->mmio.adj_offset;
-
-	return readb((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + reg.addr);
-}
-
-static inline u16 xe_mmio_read16(struct xe_gt *gt, struct xe_reg reg)
-{
-	struct xe_tile *tile = gt_to_tile(gt);
-
-	if (reg.addr < gt->mmio.adj_limit)
-		reg.addr += gt->mmio.adj_offset;
-
-	return readw((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + reg.addr);
-}
-
-static inline void xe_mmio_write32(struct xe_gt *gt,
-				   struct xe_reg reg, u32 val)
-{
-	struct xe_tile *tile = gt_to_tile(gt);
-
-	if (reg.addr < gt->mmio.adj_limit)
-		reg.addr += gt->mmio.adj_offset;
-
-	writel(val, (reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + reg.addr);
-}
-
-static inline u32 xe_mmio_read32(struct xe_gt *gt, struct xe_reg reg)
-{
-	struct xe_tile *tile = gt_to_tile(gt);
-
-	if (reg.addr < gt->mmio.adj_limit)
-		reg.addr += gt->mmio.adj_offset;
-
-	return readl((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + reg.addr);
-}
-
-static inline u32 xe_mmio_rmw32(struct xe_gt *gt, struct xe_reg reg, u32 clr,
-				u32 set)
-{
-	u32 old, reg_val;
-
-	old = xe_mmio_read32(gt, reg);
-	reg_val = (old & ~clr) | set;
-	xe_mmio_write32(gt, reg, reg_val);
-
-	return old;
-}
-
-static inline int xe_mmio_write32_and_verify(struct xe_gt *gt,
-					     struct xe_reg reg, u32 val,
-					     u32 mask, u32 eval)
-{
-	u32 reg_val;
-
-	xe_mmio_write32(gt, reg, val);
-	reg_val = xe_mmio_read32(gt, reg);
-
-	return (reg_val & mask) != eval ? -EINVAL : 0;
-}
-
-static inline bool xe_mmio_in_range(const struct xe_gt *gt,
-				    const struct xe_mmio_range *range,
-				    struct xe_reg reg)
-{
-	if (reg.addr < gt->mmio.adj_limit)
-		reg.addr += gt->mmio.adj_offset;
-
-	return range && reg.addr >= range->start && reg.addr <= range->end;
-}
+u8 xe_mmio_read8(struct xe_gt *gt, struct xe_reg reg);
+u16 xe_mmio_read16(struct xe_gt *gt, struct xe_reg reg);
+void xe_mmio_write32(struct xe_gt *gt, struct xe_reg reg, u32 val);
+u32 xe_mmio_read32(struct xe_gt *gt, struct xe_reg reg);
+u32 xe_mmio_rmw32(struct xe_gt *gt, struct xe_reg reg, u32 clr, u32 set);
+int xe_mmio_write32_and_verify(struct xe_gt *gt, struct xe_reg reg, u32 val, u32 mask, u32 eval);
+bool xe_mmio_in_range(const struct xe_gt *gt, const struct xe_mmio_range *range, struct xe_reg reg);
 
 int xe_mmio_probe_vram(struct xe_device *xe);
 u64 xe_mmio_read64_2x32(struct xe_gt *gt, struct xe_reg reg);
diff --git a/drivers/gpu/drm/xe/xe_mocs.c b/drivers/gpu/drm/xe/xe_mocs.c
index 609d997b3e9b..1e92f8ee07ba 100644
--- a/drivers/gpu/drm/xe/xe_mocs.c
+++ b/drivers/gpu/drm/xe/xe_mocs.c
@@ -17,10 +17,10 @@
 #include "xe_step_types.h"
 
 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
-#define mocs_dbg drm_dbg
+#define mocs_dbg xe_gt_dbg
 #else
 __printf(2, 3)
-static inline void mocs_dbg(const struct drm_device *dev,
+static inline void mocs_dbg(const struct xe_gt *gt,
 			    const char *format, ...)
 { /* noop */ }
 #endif
@@ -72,7 +72,7 @@ struct xe_mocs_info {
 /* Helper defines */
 #define XELP_NUM_MOCS_ENTRIES	64  /* 63-64 are reserved, but configured. */
 #define PVC_NUM_MOCS_ENTRIES	3
-#define MTL_NUM_MOCS_ENTRIES    16
+#define MTL_NUM_MOCS_ENTRIES	16
 #define XE2_NUM_MOCS_ENTRIES	16
 
 /* (e)LLC caching options */
@@ -375,6 +375,7 @@ static unsigned int get_mocs_settings(struct xe_device *xe,
 
 	switch (xe->info.platform) {
 	case XE_LUNARLAKE:
+	case XE_BATTLEMAGE:
 		info->size = ARRAY_SIZE(xe2_mocs_table);
 		info->table = xe2_mocs_table;
 		info->n_entries = XE2_NUM_MOCS_ENTRIES;
@@ -401,7 +402,11 @@ static unsigned int get_mocs_settings(struct xe_device *xe,
 		info->size = ARRAY_SIZE(dg2_mocs_desc);
 		info->table = dg2_mocs_desc;
 		info->uc_index = 1;
-		info->n_entries = XELP_NUM_MOCS_ENTRIES;
+		/*
+		 * Last entry is RO on hardware, don't bother with what was
+		 * written when checking later
+		 */
+		info->n_entries = XELP_NUM_MOCS_ENTRIES - 1;
 		info->unused_entries_index = 3;
 		break;
 	case XE_DG1:
@@ -462,24 +467,34 @@ static u32 get_entry_control(const struct xe_mocs_info *info,
 	return info->table[info->unused_entries_index].control_value;
 }
 
-static void __init_mocs_table(struct xe_gt *gt,
-			      const struct xe_mocs_info *info)
+static bool regs_are_mcr(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 
+	if (xe_gt_is_media_type(gt))
+		return MEDIA_VER(xe) >= 20;
+	else
+		return GRAPHICS_VERx100(xe) >= 1250;
+}
+
+static void __init_mocs_table(struct xe_gt *gt,
+			      const struct xe_mocs_info *info)
+{
 	unsigned int i;
 	u32 mocs;
 
-	mocs_dbg(&gt_to_xe(gt)->drm, "entries:%d\n", info->n_entries);
-	drm_WARN_ONCE(&xe->drm, !info->unused_entries_index,
-		      "Unused entries index should have been defined\n");
-	for (i = 0;
-	     i < info->n_entries ? (mocs = get_entry_control(info, i)), 1 : 0;
-	     i++) {
-		mocs_dbg(&gt_to_xe(gt)->drm, "GLOB_MOCS[%d] 0x%x 0x%x\n", i,
+	xe_gt_WARN_ONCE(gt, !info->unused_entries_index,
+			"Unused entries index should have been defined\n");
+
+	mocs_dbg(gt, "mocs entries: %d\n", info->n_entries);
+
+	for (i = 0; i < info->n_entries; i++) {
+		mocs = get_entry_control(info, i);
+
+		mocs_dbg(gt, "GLOB_MOCS[%d] 0x%x 0x%x\n", i,
 			 XELP_GLOBAL_MOCS(i).addr, mocs);
 
-		if (GRAPHICS_VERx100(gt_to_xe(gt)) > 1250)
+		if (regs_are_mcr(gt))
 			xe_gt_mcr_multicast_write(gt, XEHP_GLOBAL_MOCS(i), mocs);
 		else
 			xe_mmio_write32(gt, XELP_GLOBAL_MOCS(i), mocs);
@@ -510,16 +525,16 @@ static void init_l3cc_table(struct xe_gt *gt,
 	unsigned int i;
 	u32 l3cc;
 
-	mocs_dbg(&gt_to_xe(gt)->drm, "entries:%d\n", info->n_entries);
-	for (i = 0;
-	     i < (info->n_entries + 1) / 2 ?
-	     (l3cc = l3cc_combine(get_entry_l3cc(info, 2 * i),
-				  get_entry_l3cc(info, 2 * i + 1))), 1 : 0;
-	     i++) {
-		mocs_dbg(&gt_to_xe(gt)->drm, "LNCFCMOCS[%d] 0x%x 0x%x\n", i, XELP_LNCFCMOCS(i).addr,
-			 l3cc);
+	mocs_dbg(gt, "l3cc entries: %d\n", info->n_entries);
+
+	for (i = 0; i < (info->n_entries + 1) / 2; i++) {
+		l3cc = l3cc_combine(get_entry_l3cc(info, 2 * i),
+				    get_entry_l3cc(info, 2 * i + 1));
 
-		if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250)
+		mocs_dbg(gt, "LNCFCMOCS[%d] 0x%x 0x%x\n", i,
+			 XELP_LNCFCMOCS(i).addr, l3cc);
+
+		if (regs_are_mcr(gt))
 			xe_gt_mcr_multicast_write(gt, XEHP_LNCFCMOCS(i), l3cc);
 		else
 			xe_mmio_write32(gt, XELP_LNCFCMOCS(i), l3cc);
@@ -552,7 +567,10 @@ void xe_mocs_init(struct xe_gt *gt)
 	 * performed by the GuC.
 	 */
 	flags = get_mocs_settings(gt_to_xe(gt), &table);
-	mocs_dbg(&gt_to_xe(gt)->drm, "flag:0x%x\n", flags);
+	mocs_dbg(gt, "flag:0x%x\n", flags);
+
+	if (IS_SRIOV_VF(gt_to_xe(gt)))
+		return;
 
 	if (flags & HAS_GLOBAL_MOCS)
 		__init_mocs_table(gt, &table);
diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c
index 110b69864656..ceb8345cbca6 100644
--- a/drivers/gpu/drm/xe/xe_module.c
+++ b/drivers/gpu/drm/xe/xe_module.c
@@ -48,6 +48,13 @@ module_param_named_unsafe(force_probe, xe_modparam.force_probe, charp, 0400);
 MODULE_PARM_DESC(force_probe,
 		 "Force probe options for specified devices. See CONFIG_DRM_XE_FORCE_PROBE for details.");
 
+#ifdef CONFIG_PCI_IOV
+module_param_named(max_vfs, xe_modparam.max_vfs, uint, 0400);
+MODULE_PARM_DESC(max_vfs,
+		 "Limit number of Virtual Functions (VFs) that could be managed. "
+		 "(0 = no VFs [default]; N = allow up to N VFs)");
+#endif
+
 struct init_funcs {
 	int (*init)(void);
 	void (*exit)(void);
diff --git a/drivers/gpu/drm/xe/xe_module.h b/drivers/gpu/drm/xe/xe_module.h
index 88ef0e8b2bfd..b369984f08ec 100644
--- a/drivers/gpu/drm/xe/xe_module.h
+++ b/drivers/gpu/drm/xe/xe_module.h
@@ -18,6 +18,9 @@ struct xe_modparam {
 	char *huc_firmware_path;
 	char *gsc_firmware_path;
 	char *force_probe;
+#ifdef CONFIG_PCI_IOV
+	unsigned int max_vfs;
+#endif
 };
 
 extern struct xe_modparam xe_modparam;
diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c
index e148934d554b..d5b516f115ad 100644
--- a/drivers/gpu/drm/xe/xe_pat.c
+++ b/drivers/gpu/drm/xe/xe_pat.c
@@ -142,6 +142,7 @@ static const struct xe_pat_table_entry xe2_pat_table[] = {
 
 /* Special PAT values programmed outside the main table */
 static const struct xe_pat_table_entry xe2_pat_ats = XE2_PAT( 0, 0, 0, 0, 3, 3 );
+static const struct xe_pat_table_entry xe2_pat_pta = XE2_PAT( 0, 0, 0, 0, 3, 0 );
 
 u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index)
 {
@@ -174,7 +175,6 @@ static void xelp_dump(struct xe_gt *gt, struct drm_printer *p)
 	struct xe_device *xe = gt_to_xe(gt);
 	int i, err;
 
-	xe_device_mem_access_get(xe);
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 	if (err)
 		goto err_fw;
@@ -192,7 +192,6 @@ static void xelp_dump(struct xe_gt *gt, struct drm_printer *p)
 	err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
 err_fw:
 	xe_assert(xe, !err);
-	xe_device_mem_access_put(xe);
 }
 
 static const struct xe_pat_ops xelp_pat_ops = {
@@ -205,7 +204,6 @@ static void xehp_dump(struct xe_gt *gt, struct drm_printer *p)
 	struct xe_device *xe = gt_to_xe(gt);
 	int i, err;
 
-	xe_device_mem_access_get(xe);
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 	if (err)
 		goto err_fw;
@@ -225,7 +223,6 @@ static void xehp_dump(struct xe_gt *gt, struct drm_printer *p)
 	err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
 err_fw:
 	xe_assert(xe, !err);
-	xe_device_mem_access_put(xe);
 }
 
 static const struct xe_pat_ops xehp_pat_ops = {
@@ -238,7 +235,6 @@ static void xehpc_dump(struct xe_gt *gt, struct drm_printer *p)
 	struct xe_device *xe = gt_to_xe(gt);
 	int i, err;
 
-	xe_device_mem_access_get(xe);
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 	if (err)
 		goto err_fw;
@@ -256,7 +252,6 @@ static void xehpc_dump(struct xe_gt *gt, struct drm_printer *p)
 	err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
 err_fw:
 	xe_assert(xe, !err);
-	xe_device_mem_access_put(xe);
 }
 
 static const struct xe_pat_ops xehpc_pat_ops = {
@@ -269,7 +264,6 @@ static void xelpg_dump(struct xe_gt *gt, struct drm_printer *p)
 	struct xe_device *xe = gt_to_xe(gt);
 	int i, err;
 
-	xe_device_mem_access_get(xe);
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 	if (err)
 		goto err_fw;
@@ -292,7 +286,6 @@ static void xelpg_dump(struct xe_gt *gt, struct drm_printer *p)
 	err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
 err_fw:
 	xe_assert(xe, !err);
-	xe_device_mem_access_put(xe);
 }
 
 /*
@@ -310,6 +303,9 @@ static void xe2lpg_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry
 {
 	program_pat_mcr(gt, table, n_entries);
 	xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe2_pat_ats.value);
+
+	if (IS_DGFX(gt_to_xe(gt)))
+		xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), xe2_pat_pta.value);
 }
 
 static void xe2lpm_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
@@ -317,6 +313,9 @@ static void xe2lpm_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry
 {
 	program_pat(gt, table, n_entries);
 	xe_mmio_write32(gt, XE_REG(_PAT_ATS), xe2_pat_ats.value);
+
+	if (IS_DGFX(gt_to_xe(gt)))
+		xe_mmio_write32(gt, XE_REG(_PAT_PTA), xe2_pat_pta.value);
 }
 
 static void xe2_dump(struct xe_gt *gt, struct drm_printer *p)
@@ -325,7 +324,6 @@ static void xe2_dump(struct xe_gt *gt, struct drm_printer *p)
 	int i, err;
 	u32 pat;
 
-	xe_device_mem_access_get(xe);
 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 	if (err)
 		goto err_fw;
@@ -370,7 +368,6 @@ static void xe2_dump(struct xe_gt *gt, struct drm_printer *p)
 	err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
 err_fw:
 	xe_assert(xe, !err);
-	xe_device_mem_access_put(xe);
 }
 
 static const struct xe_pat_ops xe2_pat_ops = {
@@ -438,6 +435,10 @@ void xe_pat_init_early(struct xe_device *xe)
 	/* VFs can't program nor dump PAT settings */
 	if (IS_SRIOV_VF(xe))
 		xe->pat.ops = NULL;
+
+	xe_assert(xe, !xe->pat.ops || xe->pat.ops->dump);
+	xe_assert(xe, !xe->pat.ops || xe->pat.ops->program_graphics);
+	xe_assert(xe, !xe->pat.ops || MEDIA_VER(xe) < 13 || xe->pat.ops->program_media);
 }
 
 void xe_pat_init(struct xe_gt *gt)
@@ -457,7 +458,7 @@ void xe_pat_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 
-	if (!xe->pat.ops->dump)
+	if (!xe->pat.ops)
 		return;
 
 	xe->pat.ops->dump(gt, p);
diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
index 557f2d88a8c1..f326dbb1cecd 100644
--- a/drivers/gpu/drm/xe/xe_pci.c
+++ b/drivers/gpu/drm/xe/xe_pci.c
@@ -174,7 +174,7 @@ static const struct xe_graphics_desc graphics_xelpg = {
 		GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0)
 
 static const struct xe_graphics_desc graphics_xe2 = {
-	.name = "Xe2_LPG",
+	.name = "Xe2_LPG / Xe2_HPG",
 
 	XE2_GFX_FEATURES,
 };
@@ -185,8 +185,8 @@ static const struct xe_media_desc media_xem = {
 	.rel = 0,
 
 	.hw_engine_mask =
-		BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS2) |
-		BIT(XE_HW_ENGINE_VECS0),
+		GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) |
+		GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0),
 };
 
 static const struct xe_media_desc media_xehpm = {
@@ -195,21 +195,23 @@ static const struct xe_media_desc media_xehpm = {
 	.rel = 55,
 
 	.hw_engine_mask =
-		BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS2) |
-		BIT(XE_HW_ENGINE_VECS0) | BIT(XE_HW_ENGINE_VECS1),
+		GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) |
+		GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0),
 };
 
 static const struct xe_media_desc media_xelpmp = {
 	.name = "Xe_LPM+",
 	.hw_engine_mask =
-		BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS2) |
-		BIT(XE_HW_ENGINE_VECS0) | BIT(XE_HW_ENGINE_GSCCS0)
+		GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) |
+		GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0) |
+		BIT(XE_HW_ENGINE_GSCCS0)
 };
 
 static const struct xe_media_desc media_xe2 = {
-	.name = "Xe2_LPM",
+	.name = "Xe2_LPM / Xe2_HPM",
 	.hw_engine_mask =
-		BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VECS0), /* TODO: GSC0 */
+		GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) |
+		GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0), /* TODO: GSC0 */
 };
 
 static const struct xe_device_desc tgl_desc = {
@@ -333,6 +335,13 @@ static const struct xe_device_desc mtl_desc = {
 
 static const struct xe_device_desc lnl_desc = {
 	PLATFORM(XE_LUNARLAKE),
+	.has_display = true,
+	.require_force_probe = true,
+};
+
+static const struct xe_device_desc bmg_desc __maybe_unused = {
+	DGFX_FEATURES,
+	PLATFORM(XE_BATTLEMAGE),
 	.require_force_probe = true,
 };
 
@@ -343,12 +352,15 @@ __diag_pop();
 static const struct gmdid_map graphics_ip_map[] = {
 	{ 1270, &graphics_xelpg },
 	{ 1271, &graphics_xelpg },
+	{ 1274, &graphics_xelpg },	/* Xe_LPG+ */
+	{ 2001, &graphics_xe2 },
 	{ 2004, &graphics_xe2 },
 };
 
 /* Map of GMD_ID values to media IP */
 static const struct gmdid_map media_ip_map[] = {
 	{ 1300, &media_xelpmp },
+	{ 1301, &media_xe2 },
 	{ 2000, &media_xe2 },
 };
 
@@ -737,8 +749,6 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (err)
 		return err;
 
-	xe_sriov_probe_early(xe, desc->has_sriov);
-
 	err = xe_device_probe_early(xe);
 	if (err)
 		return err;
@@ -774,18 +784,26 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		str_yes_no(xe_device_has_sriov(xe)),
 		xe_sriov_mode_to_string(xe_device_sriov_mode(xe)));
 
-	xe_pm_init_early(xe);
+	err = xe_pm_init_early(xe);
+	if (err)
+		return err;
 
 	err = xe_device_probe(xe);
 	if (err)
 		return err;
 
-	xe_pm_init(xe);
+	err = xe_pm_init(xe);
+	if (err)
+		goto err_driver_cleanup;
 
 	drm_dbg(&xe->drm, "d3cold: capable=%s\n",
 		str_yes_no(xe->d3cold.capable));
 
 	return 0;
+
+err_driver_cleanup:
+	xe_pci_remove(pdev);
+	return err;
 }
 
 static void xe_pci_shutdown(struct pci_dev *pdev)
diff --git a/drivers/gpu/drm/xe/xe_pcode.c b/drivers/gpu/drm/xe/xe_pcode.c
index b324dc2a5deb..a5e7da8cf944 100644
--- a/drivers/gpu/drm/xe/xe_pcode.c
+++ b/drivers/gpu/drm/xe/xe_pcode.c
@@ -10,6 +10,7 @@
 
 #include <drm/drm_managed.h>
 
+#include "xe_device.h"
 #include "xe_gt.h"
 #include "xe_mmio.h"
 #include "xe_pcode_api.h"
@@ -43,8 +44,6 @@ static int pcode_mailbox_status(struct xe_gt *gt)
 		[PCODE_ERROR_MASK] = {-EPROTO, "Unknown"},
 	};
 
-	lockdep_assert_held(&gt->pcode.lock);
-
 	err = xe_mmio_read32(gt, PCODE_MAILBOX) & PCODE_ERROR_MASK;
 	if (err) {
 		drm_err(&gt_to_xe(gt)->drm, "PCODE Mailbox failed: %d %s", err,
@@ -55,17 +54,15 @@ static int pcode_mailbox_status(struct xe_gt *gt)
 	return 0;
 }
 
-static int pcode_mailbox_rw(struct xe_gt *gt, u32 mbox, u32 *data0, u32 *data1,
-			    unsigned int timeout_ms, bool return_data,
-			    bool atomic)
+static int __pcode_mailbox_rw(struct xe_gt *gt, u32 mbox, u32 *data0, u32 *data1,
+			      unsigned int timeout_ms, bool return_data,
+			      bool atomic)
 {
 	int err;
 
 	if (gt_to_xe(gt)->info.skip_pcode)
 		return 0;
 
-	lockdep_assert_held(&gt->pcode.lock);
-
 	if ((xe_mmio_read32(gt, PCODE_MAILBOX) & PCODE_READY) != 0)
 		return -EAGAIN;
 
@@ -74,7 +71,7 @@ static int pcode_mailbox_rw(struct xe_gt *gt, u32 mbox, u32 *data0, u32 *data1,
 	xe_mmio_write32(gt, PCODE_MAILBOX, PCODE_READY | mbox);
 
 	err = xe_mmio_wait32(gt, PCODE_MAILBOX, PCODE_READY, 0,
-			     timeout_ms * 1000, NULL, atomic);
+			     timeout_ms * USEC_PER_MSEC, NULL, atomic);
 	if (err)
 		return err;
 
@@ -87,6 +84,18 @@ static int pcode_mailbox_rw(struct xe_gt *gt, u32 mbox, u32 *data0, u32 *data1,
 	return pcode_mailbox_status(gt);
 }
 
+static int pcode_mailbox_rw(struct xe_gt *gt, u32 mbox, u32 *data0, u32 *data1,
+			    unsigned int timeout_ms, bool return_data,
+			    bool atomic)
+{
+	if (gt_to_xe(gt)->info.skip_pcode)
+		return 0;
+
+	lockdep_assert_held(&gt->pcode.lock);
+
+	return __pcode_mailbox_rw(gt, mbox, data0, data1, timeout_ms, return_data, atomic);
+}
+
 int xe_pcode_write_timeout(struct xe_gt *gt, u32 mbox, u32 data, int timeout)
 {
 	int err;
@@ -109,15 +118,19 @@ int xe_pcode_read(struct xe_gt *gt, u32 mbox, u32 *val, u32 *val1)
 	return err;
 }
 
-static int xe_pcode_try_request(struct xe_gt *gt, u32 mbox,
-				u32 request, u32 reply_mask, u32 reply,
-				u32 *status, bool atomic, int timeout_us)
+static int pcode_try_request(struct xe_gt *gt, u32 mbox,
+			     u32 request, u32 reply_mask, u32 reply,
+			     u32 *status, bool atomic, int timeout_us, bool locked)
 {
 	int slept, wait = 10;
 
 	for (slept = 0; slept < timeout_us; slept += wait) {
-		*status = pcode_mailbox_rw(gt, mbox, &request, NULL, 1, true,
-					   atomic);
+		if (locked)
+			*status = pcode_mailbox_rw(gt, mbox, &request, NULL, 1, true,
+						   atomic);
+		else
+			*status = __pcode_mailbox_rw(gt, mbox, &request, NULL, 1, true,
+						     atomic);
 		if ((*status == 0) && ((request & reply_mask) == reply))
 			return 0;
 
@@ -158,8 +171,8 @@ int xe_pcode_request(struct xe_gt *gt, u32 mbox, u32 request,
 
 	mutex_lock(&gt->pcode.lock);
 
-	ret = xe_pcode_try_request(gt, mbox, request, reply_mask, reply, &status,
-				   false, timeout_base_ms * 1000);
+	ret = pcode_try_request(gt, mbox, request, reply_mask, reply, &status,
+				false, timeout_base_ms * 1000, true);
 	if (!ret)
 		goto out;
 
@@ -177,8 +190,8 @@ int xe_pcode_request(struct xe_gt *gt, u32 mbox, u32 request,
 		"PCODE timeout, retrying with preemption disabled\n");
 	drm_WARN_ON_ONCE(&gt_to_xe(gt)->drm, timeout_base_ms > 1);
 	preempt_disable();
-	ret = xe_pcode_try_request(gt, mbox, request, reply_mask, reply, &status,
-				   true, timeout_base_ms * 1000);
+	ret = pcode_try_request(gt, mbox, request, reply_mask, reply, &status,
+				true, 50 * 1000, true);
 	preempt_enable();
 
 out:
@@ -238,59 +251,71 @@ unlock:
 }
 
 /**
- * xe_pcode_init - Ensure PCODE is initialized
- * @gt: gt instance
+ * xe_pcode_ready - Ensure PCODE is initialized
+ * @xe: xe instance
+ * @locked: true if lock held, false otherwise
  *
- * This function ensures that PCODE is properly initialized. To be called during
- * probe and resume paths.
+ * PCODE init mailbox is polled only on root gt of root tile
+ * as the root tile provides the initialization is complete only
+ * after all the tiles have completed the initialization.
+ * Called only on early probe without locks and with locks in
+ * resume path.
  *
- * It returns 0 on success, and -error number on failure.
+ * Returns 0 on success, and -error number on failure.
  */
-int xe_pcode_init(struct xe_gt *gt)
+int xe_pcode_ready(struct xe_device *xe, bool locked)
 {
 	u32 status, request = DGFX_GET_INIT_STATUS;
+	struct xe_gt *gt = xe_root_mmio_gt(xe);
 	int timeout_us = 180000000; /* 3 min */
 	int ret;
 
-	if (gt_to_xe(gt)->info.skip_pcode)
+	if (xe->info.skip_pcode)
 		return 0;
 
-	if (!IS_DGFX(gt_to_xe(gt)))
+	if (!IS_DGFX(xe))
 		return 0;
 
-	mutex_lock(&gt->pcode.lock);
-	ret = xe_pcode_try_request(gt, DGFX_PCODE_STATUS, request,
-				   DGFX_INIT_STATUS_COMPLETE,
-				   DGFX_INIT_STATUS_COMPLETE,
-				   &status, false, timeout_us);
-	mutex_unlock(&gt->pcode.lock);
+	if (locked)
+		mutex_lock(&gt->pcode.lock);
+
+	ret = pcode_try_request(gt, DGFX_PCODE_STATUS, request,
+				DGFX_INIT_STATUS_COMPLETE,
+				DGFX_INIT_STATUS_COMPLETE,
+				&status, false, timeout_us, locked);
+
+	if (locked)
+		mutex_unlock(&gt->pcode.lock);
 
 	if (ret)
-		drm_err(&gt_to_xe(gt)->drm,
+		drm_err(&xe->drm,
 			"PCODE initialization timedout after: 3 min\n");
 
 	return ret;
 }
 
 /**
- * xe_pcode_probe - Prepare xe_pcode and also ensure PCODE is initialized.
+ * xe_pcode_init: initialize components of PCODE
  * @gt: gt instance
  *
- * This function initializes the xe_pcode component, and when needed, it ensures
- * that PCODE has properly performed its initialization and it is really ready
- * to go. To be called once only during probe.
- *
- * It returns 0 on success, and -error number on failure.
+ * This function initializes the xe_pcode component.
+ * To be called once only during probe.
  */
-int xe_pcode_probe(struct xe_gt *gt)
+void xe_pcode_init(struct xe_gt *gt)
 {
 	drmm_mutex_init(&gt_to_xe(gt)->drm, &gt->pcode.lock);
+}
 
-	if (gt_to_xe(gt)->info.skip_pcode)
-		return 0;
-
-	if (!IS_DGFX(gt_to_xe(gt)))
-		return 0;
-
-	return xe_pcode_init(gt);
+/**
+ * xe_pcode_probe_early: initializes PCODE
+ * @xe: xe instance
+ *
+ * This function checks the initialization status of PCODE
+ * To be called once only during early probe without locks.
+ *
+ * Returns 0 on success, error code otherwise
+ */
+int xe_pcode_probe_early(struct xe_device *xe)
+{
+	return xe_pcode_ready(xe, false);
 }
diff --git a/drivers/gpu/drm/xe/xe_pcode.h b/drivers/gpu/drm/xe/xe_pcode.h
index 08cb1d047cba..3f54c6d2a57d 100644
--- a/drivers/gpu/drm/xe/xe_pcode.h
+++ b/drivers/gpu/drm/xe/xe_pcode.h
@@ -8,9 +8,11 @@
 
 #include <linux/types.h>
 struct xe_gt;
+struct xe_device;
 
-int xe_pcode_probe(struct xe_gt *gt);
-int xe_pcode_init(struct xe_gt *gt);
+void xe_pcode_init(struct xe_gt *gt);
+int xe_pcode_probe_early(struct xe_device *xe);
+int xe_pcode_ready(struct xe_device *xe, bool locked);
 int xe_pcode_init_min_freq_table(struct xe_gt *gt, u32 min_gt_freq,
 				 u32 max_gt_freq);
 int xe_pcode_read(struct xe_gt *gt, u32 mbox, u32 *val, u32 *val1);
diff --git a/drivers/gpu/drm/xe/xe_platform_types.h b/drivers/gpu/drm/xe/xe_platform_types.h
index 553f53dbd093..79b7042c4534 100644
--- a/drivers/gpu/drm/xe/xe_platform_types.h
+++ b/drivers/gpu/drm/xe/xe_platform_types.h
@@ -22,6 +22,7 @@ enum xe_platform {
 	XE_PVC,
 	XE_METEORLAKE,
 	XE_LUNARLAKE,
+	XE_BATTLEMAGE,
 };
 
 enum xe_subplatform {
diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c
index 53b3b0b019ac..37fbeda12d3b 100644
--- a/drivers/gpu/drm/xe/xe_pm.c
+++ b/drivers/gpu/drm/xe/xe_pm.c
@@ -25,23 +25,55 @@
 /**
  * DOC: Xe Power Management
  *
- * Xe PM shall be guided by the simplicity.
- * Use the simplest hook options whenever possible.
- * Let's not reinvent the runtime_pm references and hooks.
- * Shall have a clear separation of display and gt underneath this component.
+ * Xe PM implements the main routines for both system level suspend states and
+ * for the opportunistic runtime suspend states.
  *
- * What's next:
+ * System Level Suspend (S-States) - In general this is OS initiated suspend
+ * driven by ACPI for achieving S0ix (a.k.a. S2idle, freeze), S3 (suspend to ram),
+ * S4 (disk). The main functions here are `xe_pm_suspend` and `xe_pm_resume`. They
+ * are the main point for the suspend to and resume from these states.
  *
- * For now s2idle and s3 are only working in integrated devices. The next step
- * is to iterate through all VRAM's BO backing them up into the system memory
- * before allowing the system suspend.
+ * PCI Device Suspend (D-States) - This is the opportunistic PCIe device low power
+ * state D3, controlled by the PCI subsystem and ACPI with the help from the
+ * runtime_pm infrastructure.
+ * PCI D3 is special and can mean D3hot, where Vcc power is on for keeping memory
+ * alive and quicker low latency resume or D3Cold where Vcc power is off for
+ * better power savings.
+ * The Vcc control of PCI hierarchy can only be controlled at the PCI root port
+ * level, while the device driver can be behind multiple bridges/switches and
+ * paired with other devices. For this reason, the PCI subsystem cannot perform
+ * the transition towards D3Cold. The lowest runtime PM possible from the PCI
+ * subsystem is D3hot. Then, if all these paired devices in the same root port
+ * are in D3hot, ACPI will assist here and run its own methods (_PR3 and _OFF)
+ * to perform the transition from D3hot to D3cold. Xe may disallow this
+ * transition by calling pci_d3cold_disable(root_pdev) before going to runtime
+ * suspend. It will be based on runtime conditions such as VRAM usage for a
+ * quick and low latency resume for instance.
  *
- * Also runtime_pm needs to be here from the beginning.
+ * Runtime PM - This infrastructure provided by the Linux kernel allows the
+ * device drivers to indicate when the can be runtime suspended, so the device
+ * could be put at D3 (if supported), or allow deeper package sleep states
+ * (PC-states), and/or other low level power states. Xe PM component provides
+ * `xe_pm_runtime_suspend` and `xe_pm_runtime_resume` functions that PCI
+ * subsystem will call before transition to/from runtime suspend.
  *
- * RC6/RPS are also critical PM features. Let's start with GuCRC and GuC SLPC
- * and no wait boost. Frequency optimizations should come on a next stage.
+ * Also, Xe PM provides get and put functions that Xe driver will use to
+ * indicate activity. In order to avoid locking complications with the memory
+ * management, whenever possible, these get and put functions needs to be called
+ * from the higher/outer levels.
+ * The main cases that need to be protected from the outer levels are: IOCTL,
+ * sysfs, debugfs, dma-buf sharing, GPU execution.
+ *
+ * This component is not responsible for GT idleness (RC6) nor GT frequency
+ * management (RPS).
  */
 
+#ifdef CONFIG_LOCKDEP
+struct lockdep_map xe_pm_runtime_lockdep_map = {
+	.name = "xe_pm_runtime_lockdep_map"
+};
+#endif
+
 /**
  * xe_pm_suspend - Helper for System suspend, i.e. S0->S3 / S0->S2idle
  * @xe: xe device instance
@@ -54,13 +86,15 @@ int xe_pm_suspend(struct xe_device *xe)
 	u8 id;
 	int err;
 
+	drm_dbg(&xe->drm, "Suspending device\n");
+
 	for_each_gt(gt, xe, id)
 		xe_gt_suspend_prepare(gt);
 
 	/* FIXME: Super racey... */
 	err = xe_bo_evict_all(xe);
 	if (err)
-		return err;
+		goto err;
 
 	xe_display_pm_suspend(xe);
 
@@ -68,7 +102,7 @@ int xe_pm_suspend(struct xe_device *xe)
 		err = xe_gt_suspend(gt);
 		if (err) {
 			xe_display_pm_resume(xe);
-			return err;
+			goto err;
 		}
 	}
 
@@ -76,7 +110,11 @@ int xe_pm_suspend(struct xe_device *xe)
 
 	xe_display_pm_suspend_late(xe);
 
+	drm_dbg(&xe->drm, "Device suspended\n");
 	return 0;
+err:
+	drm_dbg(&xe->drm, "Device suspend failed %d\n", err);
+	return err;
 }
 
 /**
@@ -92,14 +130,14 @@ int xe_pm_resume(struct xe_device *xe)
 	u8 id;
 	int err;
 
+	drm_dbg(&xe->drm, "Resuming device\n");
+
 	for_each_tile(tile, xe, id)
 		xe_wa_apply_tile_workarounds(tile);
 
-	for_each_gt(gt, xe, id) {
-		err = xe_pcode_init(gt);
-		if (err)
-			return err;
-	}
+	err = xe_pcode_ready(xe, true);
+	if (err)
+		return err;
 
 	xe_display_pm_resume_early(xe);
 
@@ -109,7 +147,7 @@ int xe_pm_resume(struct xe_device *xe)
 	 */
 	err = xe_bo_restore_kernel(xe);
 	if (err)
-		return err;
+		goto err;
 
 	xe_irq_resume(xe);
 
@@ -120,9 +158,13 @@ int xe_pm_resume(struct xe_device *xe)
 
 	err = xe_bo_restore_user(xe);
 	if (err)
-		return err;
+		goto err;
 
+	drm_dbg(&xe->drm, "Device resumed\n");
 	return 0;
+err:
+	drm_dbg(&xe->drm, "Device resume failed %d\n", err);
+	return err;
 }
 
 static bool xe_pm_pci_d3cold_capable(struct xe_device *xe)
@@ -172,30 +214,60 @@ static void xe_pm_runtime_init(struct xe_device *xe)
 	pm_runtime_put(dev);
 }
 
-void xe_pm_init_early(struct xe_device *xe)
+int xe_pm_init_early(struct xe_device *xe)
 {
+	int err;
+
 	INIT_LIST_HEAD(&xe->mem_access.vram_userfault.list);
-	drmm_mutex_init(&xe->drm, &xe->mem_access.vram_userfault.lock);
+
+	err = drmm_mutex_init(&xe->drm, &xe->mem_access.vram_userfault.lock);
+	if (err)
+		return err;
+
+	err = drmm_mutex_init(&xe->drm, &xe->d3cold.lock);
+	if (err)
+		return err;
+
+	return 0;
 }
 
-void xe_pm_init(struct xe_device *xe)
+/**
+ * xe_pm_init - Initialize Xe Power Management
+ * @xe: xe device instance
+ *
+ * This component is responsible for System and Device sleep states.
+ *
+ * Returns 0 for success, negative error code otherwise.
+ */
+int xe_pm_init(struct xe_device *xe)
 {
+	int err;
+
 	/* For now suspend/resume is only allowed with GuC */
 	if (!xe_device_uc_enabled(xe))
-		return;
-
-	drmm_mutex_init(&xe->drm, &xe->d3cold.lock);
+		return 0;
 
 	xe->d3cold.capable = xe_pm_pci_d3cold_capable(xe);
 
 	if (xe->d3cold.capable) {
-		xe_device_sysfs_init(xe);
-		xe_pm_set_vram_threshold(xe, DEFAULT_VRAM_THRESHOLD);
+		err = xe_device_sysfs_init(xe);
+		if (err)
+			return err;
+
+		err = xe_pm_set_vram_threshold(xe, DEFAULT_VRAM_THRESHOLD);
+		if (err)
+			return err;
 	}
 
 	xe_pm_runtime_init(xe);
+
+	return 0;
 }
 
+/**
+ * xe_pm_runtime_fini - Finalize Runtime PM
+ * @xe: xe device instance
+ */
 void xe_pm_runtime_fini(struct xe_device *xe)
 {
 	struct device *dev = xe->drm.dev;
@@ -225,6 +297,28 @@ struct task_struct *xe_pm_read_callback_task(struct xe_device *xe)
 	return READ_ONCE(xe->pm_callback_task);
 }
 
+/**
+ * xe_pm_runtime_suspended - Check if runtime_pm state is suspended
+ * @xe: xe device instance
+ *
+ * This does not provide any guarantee that the device is going to remain
+ * suspended as it might be racing with the runtime state transitions.
+ * It can be used only as a non-reliable assertion, to ensure that we are not in
+ * the sleep state while trying to access some memory for instance.
+ *
+ * Returns true if PCI device is suspended, false otherwise.
+ */
+bool xe_pm_runtime_suspended(struct xe_device *xe)
+{
+	return pm_runtime_suspended(xe->drm.dev);
+}
+
+/**
+ * xe_pm_runtime_suspend - Prepare our device for D3hot/D3Cold
+ * @xe: xe device instance
+ *
+ * Returns 0 for success, negative error code otherwise.
+ */
 int xe_pm_runtime_suspend(struct xe_device *xe)
 {
 	struct xe_bo *bo, *on;
@@ -232,18 +326,15 @@ int xe_pm_runtime_suspend(struct xe_device *xe)
 	u8 id;
 	int err = 0;
 
-	if (xe->d3cold.allowed && xe_device_mem_access_ongoing(xe))
-		return -EBUSY;
-
 	/* Disable access_ongoing asserts and prevent recursive pm calls */
 	xe_pm_write_callback_task(xe, current);
 
 	/*
-	 * The actual xe_device_mem_access_put() is always async underneath, so
+	 * The actual xe_pm_runtime_put() is always async underneath, so
 	 * exactly where that is called should makes no difference to us. However
 	 * we still need to be very careful with the locks that this callback
 	 * acquires and the locks that are acquired and held by any callers of
-	 * xe_device_mem_access_get(). We already have the matching annotation
+	 * xe_runtime_pm_get(). We already have the matching annotation
 	 * on that side, but we also need it here. For example lockdep should be
 	 * able to tell us if the following scenario is in theory possible:
 	 *
@@ -251,15 +342,15 @@ int xe_pm_runtime_suspend(struct xe_device *xe)
 	 * lock(A)                       |
 	 *                               | xe_pm_runtime_suspend()
 	 *                               |      lock(A)
-	 * xe_device_mem_access_get()    |
+	 * xe_pm_runtime_get()           |
 	 *
 	 * This will clearly deadlock since rpm core needs to wait for
 	 * xe_pm_runtime_suspend() to complete, but here we are holding lock(A)
 	 * on CPU0 which prevents CPU1 making forward progress.  With the
-	 * annotation here and in xe_device_mem_access_get() lockdep will see
+	 * annotation here and in xe_pm_runtime_get() lockdep will see
 	 * the potential lock inversion and give us a nice splat.
 	 */
-	lock_map_acquire(&xe_device_mem_access_lockdep_map);
+	lock_map_acquire(&xe_pm_runtime_lockdep_map);
 
 	/*
 	 * Applying lock for entire list op as xe_ttm_bo_destroy and xe_bo_move_notify
@@ -285,11 +376,17 @@ int xe_pm_runtime_suspend(struct xe_device *xe)
 
 	xe_irq_suspend(xe);
 out:
-	lock_map_release(&xe_device_mem_access_lockdep_map);
+	lock_map_release(&xe_pm_runtime_lockdep_map);
 	xe_pm_write_callback_task(xe, NULL);
 	return err;
 }
 
+/**
+ * xe_pm_runtime_resume - Waking up from D3hot/D3Cold
+ * @xe: xe device instance
+ *
+ * Returns 0 for success, negative error code otherwise.
+ */
 int xe_pm_runtime_resume(struct xe_device *xe)
 {
 	struct xe_gt *gt;
@@ -299,7 +396,7 @@ int xe_pm_runtime_resume(struct xe_device *xe)
 	/* Disable access_ongoing asserts and prevent recursive pm calls */
 	xe_pm_write_callback_task(xe, current);
 
-	lock_map_acquire(&xe_device_mem_access_lockdep_map);
+	lock_map_acquire(&xe_pm_runtime_lockdep_map);
 
 	/*
 	 * It can be possible that xe has allowed d3cold but other pcie devices
@@ -310,11 +407,9 @@ int xe_pm_runtime_resume(struct xe_device *xe)
 	xe->d3cold.power_lost = xe_guc_in_reset(&gt->uc.guc);
 
 	if (xe->d3cold.allowed && xe->d3cold.power_lost) {
-		for_each_gt(gt, xe, id) {
-			err = xe_pcode_init(gt);
-			if (err)
-				goto out;
-		}
+		err = xe_pcode_ready(xe, true);
+		if (err)
+			goto out;
 
 		/*
 		 * This only restores pinned memory which is the memory
@@ -336,27 +431,147 @@ int xe_pm_runtime_resume(struct xe_device *xe)
 			goto out;
 	}
 out:
-	lock_map_release(&xe_device_mem_access_lockdep_map);
+	lock_map_release(&xe_pm_runtime_lockdep_map);
 	xe_pm_write_callback_task(xe, NULL);
 	return err;
 }
 
-int xe_pm_runtime_get(struct xe_device *xe)
+/*
+ * For places where resume is synchronous it can be quite easy to deadlock
+ * if we are not careful. Also in practice it might be quite timing
+ * sensitive to ever see the 0 -> 1 transition with the callers locks
+ * held, so deadlocks might exist but are hard for lockdep to ever see.
+ * With this in mind, help lockdep learn about the potentially scary
+ * stuff that can happen inside the runtime_resume callback by acquiring
+ * a dummy lock (it doesn't protect anything and gets compiled out on
+ * non-debug builds).  Lockdep then only needs to see the
+ * xe_pm_runtime_lockdep_map -> runtime_resume callback once, and then can
+ * hopefully validate all the (callers_locks) -> xe_pm_runtime_lockdep_map.
+ * For example if the (callers_locks) are ever grabbed in the
+ * runtime_resume callback, lockdep should give us a nice splat.
+ */
+static void pm_runtime_lockdep_prime(void)
+{
+	lock_map_acquire(&xe_pm_runtime_lockdep_map);
+	lock_map_release(&xe_pm_runtime_lockdep_map);
+}
+
+/**
+ * xe_pm_runtime_get - Get a runtime_pm reference and resume synchronously
+ * @xe: xe device instance
+ */
+void xe_pm_runtime_get(struct xe_device *xe)
 {
-	return pm_runtime_get_sync(xe->drm.dev);
+	pm_runtime_get_noresume(xe->drm.dev);
+
+	if (xe_pm_read_callback_task(xe) == current)
+		return;
+
+	pm_runtime_lockdep_prime();
+	pm_runtime_resume(xe->drm.dev);
 }
 
-int xe_pm_runtime_put(struct xe_device *xe)
+/**
+ * xe_pm_runtime_put - Put the runtime_pm reference back and mark as idle
+ * @xe: xe device instance
+ */
+void xe_pm_runtime_put(struct xe_device *xe)
 {
-	pm_runtime_mark_last_busy(xe->drm.dev);
-	return pm_runtime_put(xe->drm.dev);
+	if (xe_pm_read_callback_task(xe) == current) {
+		pm_runtime_put_noidle(xe->drm.dev);
+	} else {
+		pm_runtime_mark_last_busy(xe->drm.dev);
+		pm_runtime_put(xe->drm.dev);
+	}
 }
 
+/**
+ * xe_pm_runtime_get_ioctl - Get a runtime_pm reference before ioctl
+ * @xe: xe device instance
+ *
+ * Returns: Any number greater than or equal to 0 for success, negative error
+ * code otherwise.
+ */
+int xe_pm_runtime_get_ioctl(struct xe_device *xe)
+{
+	if (WARN_ON(xe_pm_read_callback_task(xe) == current))
+		return -ELOOP;
+
+	pm_runtime_lockdep_prime();
+	return pm_runtime_get_sync(xe->drm.dev);
+}
+
+/**
+ * xe_pm_runtime_get_if_active - Get a runtime_pm reference if device active
+ * @xe: xe device instance
+ *
+ * Returns: Any number greater than or equal to 0 for success, negative error
+ * code otherwise.
+ */
 int xe_pm_runtime_get_if_active(struct xe_device *xe)
 {
 	return pm_runtime_get_if_active(xe->drm.dev);
 }
 
+/**
+ * xe_pm_runtime_get_if_in_use - Get a runtime_pm reference and resume if needed
+ * @xe: xe device instance
+ *
+ * Returns: True if device is awake and the reference was taken, false otherwise.
+ */
+bool xe_pm_runtime_get_if_in_use(struct xe_device *xe)
+{
+	if (xe_pm_read_callback_task(xe) == current) {
+		/* The device is awake, grab the ref and move on */
+		pm_runtime_get_noresume(xe->drm.dev);
+		return true;
+	}
+
+	return pm_runtime_get_if_in_use(xe->drm.dev) > 0;
+}
+
+/**
+ * xe_pm_runtime_get_noresume - Bump runtime PM usage counter without resuming
+ * @xe: xe device instance
+ *
+ * This function should be used in inner places where it is surely already
+ * protected by outer-bound callers of `xe_pm_runtime_get`.
+ * It will warn if not protected.
+ * The reference should be put back after this function regardless, since it
+ * will always bump the usage counter, regardless.
+ */
+void xe_pm_runtime_get_noresume(struct xe_device *xe)
+{
+	bool ref;
+
+	ref = xe_pm_runtime_get_if_in_use(xe);
+
+	if (drm_WARN(&xe->drm, !ref, "Missing outer runtime PM protection\n"))
+		pm_runtime_get_noresume(xe->drm.dev);
+}
+
+/**
+ * xe_pm_runtime_resume_and_get - Resume, then get a runtime_pm ref if awake.
+ * @xe: xe device instance
+ *
+ * Returns: True if device is awake and the reference was taken, false otherwise.
+ */
+bool xe_pm_runtime_resume_and_get(struct xe_device *xe)
+{
+	if (xe_pm_read_callback_task(xe) == current) {
+		/* The device is awake, grab the ref and move on */
+		pm_runtime_get_noresume(xe->drm.dev);
+		return true;
+	}
+
+	pm_runtime_lockdep_prime();
+	return pm_runtime_resume_and_get(xe->drm.dev) >= 0;
+}
+
+/**
+ * xe_pm_assert_unbounded_bridge - Disable PM on unbounded pcie parent bridge
+ * @xe: xe device instance
+ */
 void xe_pm_assert_unbounded_bridge(struct xe_device *xe)
 {
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
@@ -371,6 +586,13 @@ void xe_pm_assert_unbounded_bridge(struct xe_device *xe)
 	}
 }
 
+/**
+ * xe_pm_set_vram_threshold - Set a vram threshold for allowing/blocking D3Cold
+ * @xe: xe device instance
+ * @threshold: VRAM size in bites for the D3cold threshold
+ *
+ * Returns 0 for success, negative error code otherwise.
+ */
 int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold)
 {
 	struct ttm_resource_manager *man;
@@ -395,6 +617,13 @@ int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold)
 	return 0;
 }
 
+/**
+ * xe_pm_d3cold_allowed_toggle - Check conditions to toggle d3cold.allowed
+ * @xe: xe device instance
+ *
+ * To be called during runtime_pm idle callback.
+ * Check for all the D3Cold conditions ahead of runtime suspend.
+ */
 void xe_pm_d3cold_allowed_toggle(struct xe_device *xe)
 {
 	struct ttm_resource_manager *man;
diff --git a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h
index 64a97c6726a7..18b0613fe57b 100644
--- a/drivers/gpu/drm/xe/xe_pm.h
+++ b/drivers/gpu/drm/xe/xe_pm.h
@@ -20,14 +20,19 @@ struct xe_device;
 int xe_pm_suspend(struct xe_device *xe);
 int xe_pm_resume(struct xe_device *xe);
 
-void xe_pm_init_early(struct xe_device *xe);
-void xe_pm_init(struct xe_device *xe);
+int xe_pm_init_early(struct xe_device *xe);
+int xe_pm_init(struct xe_device *xe);
 void xe_pm_runtime_fini(struct xe_device *xe);
+bool xe_pm_runtime_suspended(struct xe_device *xe);
 int xe_pm_runtime_suspend(struct xe_device *xe);
 int xe_pm_runtime_resume(struct xe_device *xe);
-int xe_pm_runtime_get(struct xe_device *xe);
-int xe_pm_runtime_put(struct xe_device *xe);
+void xe_pm_runtime_get(struct xe_device *xe);
+int xe_pm_runtime_get_ioctl(struct xe_device *xe);
+void xe_pm_runtime_put(struct xe_device *xe);
 int xe_pm_runtime_get_if_active(struct xe_device *xe);
+bool xe_pm_runtime_get_if_in_use(struct xe_device *xe);
+void xe_pm_runtime_get_noresume(struct xe_device *xe);
+bool xe_pm_runtime_resume_and_get(struct xe_device *xe);
 void xe_pm_assert_unbounded_bridge(struct xe_device *xe);
 int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold);
 void xe_pm_d3cold_allowed_toggle(struct xe_device *xe);
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index 4efc8c1a3d7a..5b7930f46cf3 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -5,6 +5,7 @@
 
 #include "xe_pt.h"
 
+#include "regs/xe_gtt_defs.h"
 #include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_drm_client.h"
@@ -108,11 +109,11 @@ struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,
 	pt->level = level;
 	bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K,
 				  ttm_bo_type_kernel,
-				  XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-				  XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT |
-				  XE_BO_CREATE_PINNED_BIT |
-				  XE_BO_CREATE_NO_RESV_EVICT |
-				  XE_BO_PAGETABLE);
+				  XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+				  XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE |
+				  XE_BO_FLAG_PINNED |
+				  XE_BO_FLAG_NO_RESV_EVICT |
+				  XE_BO_FLAG_PAGETABLE);
 	if (IS_ERR(bo)) {
 		err = PTR_ERR(bo);
 		goto err_kfree;
@@ -618,7 +619,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
 	struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];
 	int ret;
 
-	if (vma && (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) &&
+	if ((vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) &&
 	    (is_devmem || !IS_DGFX(xe)))
 		xe_walk.default_pte |= XE_USM_PPGTT_PTE_AE;
 
diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c
index 075f9eaef031..df407d73e5f5 100644
--- a/drivers/gpu/drm/xe/xe_query.c
+++ b/drivers/gpu/drm/xe/xe_query.c
@@ -12,6 +12,7 @@
 #include <drm/xe_drm.h>
 
 #include "regs/xe_engine_regs.h"
+#include "regs/xe_gt_regs.h"
 #include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_exec_queue.h"
@@ -147,8 +148,8 @@ query_engine_cycles(struct xe_device *xe,
 	if (!hwe)
 		return -EINVAL;
 
-	xe_device_mem_access_get(xe);
-	xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	if (xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL))
+		return -EIO;
 
 	__read_timestamps(gt,
 			  RING_TIMESTAMP(hwe->mmio_base),
@@ -159,7 +160,6 @@ query_engine_cycles(struct xe_device *xe,
 			  cpu_clock);
 
 	xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	xe_device_mem_access_put(xe);
 	resp.width = 36;
 
 	/* Only write to the output fields of user query */
@@ -403,6 +403,13 @@ static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query
 				BIT(gt_to_tile(gt)->id) << 1;
 		gt_list->gt_list[id].far_mem_regions = xe->info.mem_region_mask ^
 			gt_list->gt_list[id].near_mem_regions;
+
+		gt_list->gt_list[id].ip_ver_major =
+			REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid);
+		gt_list->gt_list[id].ip_ver_minor =
+			REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid);
+		gt_list->gt_list[id].ip_ver_rev =
+			REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid);
 	}
 
 	if (copy_to_user(query_ptr, gt_list, size)) {
@@ -433,9 +440,7 @@ static int query_hwconfig(struct xe_device *xe,
 	if (!hwconfig)
 		return -ENOMEM;
 
-	xe_device_mem_access_get(xe);
 	xe_guc_hwconfig_copy(&gt->uc.guc, hwconfig);
-	xe_device_mem_access_put(xe);
 
 	if (copy_to_user(query_ptr, hwconfig, size)) {
 		kfree(hwconfig);
@@ -544,14 +549,44 @@ query_uc_fw_version(struct xe_device *xe, struct drm_xe_device_query *query)
 		version = &guc->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY];
 		break;
 	}
+	case XE_QUERY_UC_TYPE_HUC: {
+		struct xe_gt *media_gt = NULL;
+		struct xe_huc *huc;
+
+		if (MEDIA_VER(xe) >= 13) {
+			struct xe_tile *tile;
+			u8 gt_id;
+
+			for_each_tile(tile, xe, gt_id) {
+				if (tile->media_gt) {
+					media_gt = tile->media_gt;
+					break;
+				}
+			}
+		} else {
+			media_gt = xe->tiles[0].primary_gt;
+		}
+
+		if (!media_gt)
+			break;
+
+		huc = &media_gt->uc.huc;
+		if (huc->fw.status == XE_UC_FIRMWARE_RUNNING)
+			version = &huc->fw.versions.found[XE_UC_FW_VER_RELEASE];
+		break;
+	}
 	default:
 		return -EINVAL;
 	}
 
-	resp.branch_ver = 0;
-	resp.major_ver = version->major;
-	resp.minor_ver = version->minor;
-	resp.patch_ver = version->patch;
+	if (version) {
+		resp.branch_ver = 0;
+		resp.major_ver = version->major;
+		resp.minor_ver = version->minor;
+		resp.patch_ver = version->patch;
+	} else {
+		return -ENODEV;
+	}
 
 	if (copy_to_user(query_ptr, &resp, size))
 		return -EFAULT;
diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c
index 5b2b37b59813..d42b3f33bd7a 100644
--- a/drivers/gpu/drm/xe/xe_ring_ops.c
+++ b/drivers/gpu/drm/xe/xe_ring_ops.c
@@ -17,6 +17,7 @@
 #include "xe_lrc.h"
 #include "xe_macros.h"
 #include "xe_sched_job.h"
+#include "xe_sriov.h"
 #include "xe_vm_types.h"
 #include "xe_vm.h"
 #include "xe_wa.h"
@@ -367,10 +368,12 @@ static void emit_migration_job_gen12(struct xe_sched_job *job,
 
 	i = emit_bb_start(job->batch_addr[0], BIT(8), dw, i);
 
-	/* XXX: Do we need this? Leaving for now. */
-	dw[i++] = preparser_disable(true);
-	i = emit_flush_invalidate(0, dw, i);
-	dw[i++] = preparser_disable(false);
+	if (!IS_SRIOV_VF(gt_to_xe(job->q->gt))) {
+		/* XXX: Do we need this? Leaving for now. */
+		dw[i++] = preparser_disable(true);
+		i = emit_flush_invalidate(0, dw, i);
+		dw[i++] = preparser_disable(false);
+	}
 
 	i = emit_bb_start(job->batch_addr[1], BIT(8), dw, i);
 
diff --git a/drivers/gpu/drm/xe/xe_sa.c b/drivers/gpu/drm/xe/xe_sa.c
index 2c4632259edd..8941522b7705 100644
--- a/drivers/gpu/drm/xe/xe_sa.c
+++ b/drivers/gpu/drm/xe/xe_sa.c
@@ -48,8 +48,9 @@ struct xe_sa_manager *xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32
 	sa_manager->bo = NULL;
 
 	bo = xe_bo_create_pin_map(xe, tile, NULL, size, ttm_bo_type_kernel,
-				  XE_BO_CREATE_VRAM_IF_DGFX(tile) |
-				  XE_BO_CREATE_GGTT_BIT);
+				  XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+				  XE_BO_FLAG_GGTT |
+				  XE_BO_FLAG_GGTT_INVALIDATE);
 	if (IS_ERR(bo)) {
 		drm_err(&xe->drm, "failed to allocate bo for sa manager: %ld\n",
 			PTR_ERR(bo));
diff --git a/drivers/gpu/drm/xe/xe_sched_job.c b/drivers/gpu/drm/xe/xe_sched_job.c
index b0c7fa4693cf..cd8a2fba5438 100644
--- a/drivers/gpu/drm/xe/xe_sched_job.c
+++ b/drivers/gpu/drm/xe/xe_sched_job.c
@@ -5,6 +5,7 @@
 
 #include "xe_sched_job.h"
 
+#include <drm/xe_drm.h>
 #include <linux/dma-fence-array.h>
 #include <linux/slab.h>
 
@@ -15,6 +16,8 @@
 #include "xe_hw_fence.h"
 #include "xe_lrc.h"
 #include "xe_macros.h"
+#include "xe_pm.h"
+#include "xe_sync_types.h"
 #include "xe_trace.h"
 #include "xe_vm.h"
 
@@ -157,7 +160,7 @@ struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q,
 
 	/* All other jobs require a VM to be open which has a ref */
 	if (unlikely(q->flags & EXEC_QUEUE_FLAG_KERNEL))
-		xe_device_mem_access_get(job_to_xe(job));
+		xe_pm_runtime_get_noresume(job_to_xe(job));
 	xe_device_assert_mem_access(job_to_xe(job));
 
 	trace_xe_sched_job_create(job);
@@ -190,7 +193,7 @@ void xe_sched_job_destroy(struct kref *ref)
 		container_of(ref, struct xe_sched_job, refcount);
 
 	if (unlikely(job->q->flags & EXEC_QUEUE_FLAG_KERNEL))
-		xe_device_mem_access_put(job_to_xe(job));
+		xe_pm_runtime_put(job_to_xe(job));
 	xe_exec_queue_put(job->q);
 	dma_fence_put(job->fence);
 	drm_sched_job_cleanup(&job->drm);
@@ -288,6 +291,22 @@ int xe_sched_job_last_fence_add_dep(struct xe_sched_job *job, struct xe_vm *vm)
 	return drm_sched_job_add_dependency(&job->drm, fence);
 }
 
+/**
+ * xe_sched_job_init_user_fence - Initialize user_fence for the job
+ * @job: job whose user_fence needs an init
+ * @sync: sync to be use to init user_fence
+ */
+void xe_sched_job_init_user_fence(struct xe_sched_job *job,
+				  struct xe_sync_entry *sync)
+{
+	if (sync->type != DRM_XE_SYNC_TYPE_USER_FENCE)
+		return;
+
+	job->user_fence.used = true;
+	job->user_fence.addr = sync->addr;
+	job->user_fence.value = sync->timeline_value;
+}
+
 struct xe_sched_job_snapshot *
 xe_sched_job_snapshot_capture(struct xe_sched_job *job)
 {
diff --git a/drivers/gpu/drm/xe/xe_sched_job.h b/drivers/gpu/drm/xe/xe_sched_job.h
index f1a660648cf0..c75018f4660d 100644
--- a/drivers/gpu/drm/xe/xe_sched_job.h
+++ b/drivers/gpu/drm/xe/xe_sched_job.h
@@ -10,6 +10,7 @@
 
 struct drm_printer;
 struct xe_vm;
+struct xe_sync_entry;
 
 #define XE_SCHED_HANG_LIMIT 1
 #define XE_SCHED_JOB_TIMEOUT LONG_MAX
@@ -58,6 +59,8 @@ void xe_sched_job_arm(struct xe_sched_job *job);
 void xe_sched_job_push(struct xe_sched_job *job);
 
 int xe_sched_job_last_fence_add_dep(struct xe_sched_job *job, struct xe_vm *vm);
+void xe_sched_job_init_user_fence(struct xe_sched_job *job,
+				  struct xe_sync_entry *sync);
 
 static inline struct xe_sched_job *
 to_xe_sched_job(struct drm_sched_job *drm)
diff --git a/drivers/gpu/drm/xe/xe_sriov.c b/drivers/gpu/drm/xe/xe_sriov.c
index f295d91886b1..1c3fa84b6adb 100644
--- a/drivers/gpu/drm/xe/xe_sriov.c
+++ b/drivers/gpu/drm/xe/xe_sriov.c
@@ -5,8 +5,13 @@
 
 #include <drm/drm_managed.h>
 
+#include "regs/xe_sriov_regs.h"
+
 #include "xe_assert.h"
+#include "xe_device.h"
+#include "xe_mmio.h"
 #include "xe_sriov.h"
+#include "xe_sriov_pf.h"
 
 /**
  * xe_sriov_mode_to_string - Convert enum value to string.
@@ -28,10 +33,16 @@ const char *xe_sriov_mode_to_string(enum xe_sriov_mode mode)
 	}
 }
 
+static bool test_is_vf(struct xe_device *xe)
+{
+	u32 value = xe_mmio_read32(xe_root_mmio_gt(xe), VF_CAP_REG);
+
+	return value & VF_CAP;
+}
+
 /**
  * xe_sriov_probe_early - Probe a SR-IOV mode.
  * @xe: the &xe_device to probe mode on
- * @has_sriov: flag indicating hardware support for SR-IOV
  *
  * This function should be called only once and as soon as possible during
  * driver probe to detect whether we are running a SR-IOV Physical Function
@@ -40,12 +51,17 @@ const char *xe_sriov_mode_to_string(enum xe_sriov_mode mode)
  * SR-IOV PF mode detection is based on PCI @dev_is_pf() function.
  * SR-IOV VF mode detection is based on dedicated MMIO register read.
  */
-void xe_sriov_probe_early(struct xe_device *xe, bool has_sriov)
+void xe_sriov_probe_early(struct xe_device *xe)
 {
 	enum xe_sriov_mode mode = XE_SRIOV_MODE_NONE;
+	bool has_sriov = xe->info.has_sriov;
 
-	/* TODO: replace with proper mode detection */
-	xe_assert(xe, !has_sriov);
+	if (has_sriov) {
+		if (test_is_vf(xe))
+			mode = XE_SRIOV_MODE_VF;
+		else if (xe_sriov_pf_readiness(xe))
+			mode = XE_SRIOV_MODE_PF;
+	}
 
 	xe_assert(xe, !xe->sriov.__mode);
 	xe->sriov.__mode = mode;
@@ -78,6 +94,13 @@ int xe_sriov_init(struct xe_device *xe)
 	if (!IS_SRIOV(xe))
 		return 0;
 
+	if (IS_SRIOV_PF(xe)) {
+		int err = xe_sriov_pf_init_early(xe);
+
+		if (err)
+			return err;
+	}
+
 	xe_assert(xe, !xe->sriov.wq);
 	xe->sriov.wq = alloc_workqueue("xe-sriov-wq", 0, 0);
 	if (!xe->sriov.wq)
@@ -85,3 +108,34 @@ int xe_sriov_init(struct xe_device *xe)
 
 	return drmm_add_action_or_reset(&xe->drm, fini_sriov, xe);
 }
+
+/**
+ * xe_sriov_print_info - Print basic SR-IOV information.
+ * @xe: the &xe_device to print info from
+ * @p: the &drm_printer
+ *
+ * Print SR-IOV related information into provided DRM printer.
+ */
+void xe_sriov_print_info(struct xe_device *xe, struct drm_printer *p)
+{
+	drm_printf(p, "supported: %s\n", str_yes_no(xe_device_has_sriov(xe)));
+	drm_printf(p, "enabled: %s\n", str_yes_no(IS_SRIOV(xe)));
+	drm_printf(p, "mode: %s\n", xe_sriov_mode_to_string(xe_device_sriov_mode(xe)));
+}
+
+/**
+ * xe_sriov_function_name() - Get SR-IOV Function name.
+ * @n: the Function number (identifier) to get name of
+ * @buf: the buffer to format to
+ * @size: size of the buffer (shall be at least 5 bytes)
+ *
+ * Return: formatted function name ("PF" or "VF%u").
+ */
+const char *xe_sriov_function_name(unsigned int n, char *buf, size_t size)
+{
+	if (n)
+		snprintf(buf, size, "VF%u", n);
+	else
+		strscpy(buf, "PF", size);
+	return buf;
+}
diff --git a/drivers/gpu/drm/xe/xe_sriov.h b/drivers/gpu/drm/xe/xe_sriov.h
index 1545552162c9..486bb21c3256 100644
--- a/drivers/gpu/drm/xe/xe_sriov.h
+++ b/drivers/gpu/drm/xe/xe_sriov.h
@@ -10,9 +10,13 @@
 #include "xe_device_types.h"
 #include "xe_sriov_types.h"
 
+struct drm_printer;
+
 const char *xe_sriov_mode_to_string(enum xe_sriov_mode mode);
+const char *xe_sriov_function_name(unsigned int n, char *buf, size_t len);
 
-void xe_sriov_probe_early(struct xe_device *xe, bool has_sriov);
+void xe_sriov_probe_early(struct xe_device *xe);
+void xe_sriov_print_info(struct xe_device *xe, struct drm_printer *p);
 int xe_sriov_init(struct xe_device *xe);
 
 static inline enum xe_sriov_mode xe_device_sriov_mode(struct xe_device *xe)
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.c b/drivers/gpu/drm/xe/xe_sriov_pf.c
new file mode 100644
index 000000000000..0f721ae17b26
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_sriov_pf.c
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#include <drm/drm_managed.h>
+
+#include "xe_assert.h"
+#include "xe_device.h"
+#include "xe_module.h"
+#include "xe_sriov.h"
+#include "xe_sriov_pf.h"
+#include "xe_sriov_printk.h"
+
+static unsigned int wanted_max_vfs(struct xe_device *xe)
+{
+	return xe_modparam.max_vfs;
+}
+
+static int pf_reduce_totalvfs(struct xe_device *xe, int limit)
+{
+	struct device *dev = xe->drm.dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+	int err;
+
+	err = pci_sriov_set_totalvfs(pdev, limit);
+	if (err)
+		xe_sriov_notice(xe, "Failed to set number of VFs to %d (%pe)\n",
+				limit, ERR_PTR(err));
+	return err;
+}
+
+static bool pf_continue_as_native(struct xe_device *xe, const char *why)
+{
+	xe_sriov_dbg(xe, "%s, continuing as native\n", why);
+	pf_reduce_totalvfs(xe, 0);
+	return false;
+}
+
+/**
+ * xe_sriov_pf_readiness - Check if PF functionality can be enabled.
+ * @xe: the &xe_device to check
+ *
+ * This function is called as part of the SR-IOV probe to validate if all
+ * PF prerequisites are satisfied and we can continue with enabling PF mode.
+ *
+ * Return: true if the PF mode can be turned on.
+ */
+bool xe_sriov_pf_readiness(struct xe_device *xe)
+{
+	struct device *dev = xe->drm.dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+	int totalvfs = pci_sriov_get_totalvfs(pdev);
+	int newlimit = min_t(u16, wanted_max_vfs(xe), totalvfs);
+
+	xe_assert(xe, totalvfs <= U16_MAX);
+
+	if (!dev_is_pf(dev))
+		return false;
+
+	if (!xe_device_uc_enabled(xe))
+		return pf_continue_as_native(xe, "Guc submission disabled");
+
+	if (!newlimit)
+		return pf_continue_as_native(xe, "all VFs disabled");
+
+	pf_reduce_totalvfs(xe, newlimit);
+
+	xe->sriov.pf.device_total_vfs = totalvfs;
+	xe->sriov.pf.driver_max_vfs = newlimit;
+
+	return true;
+}
+
+/**
+ * xe_sriov_pf_init_early - Initialize SR-IOV PF specific data.
+ * @xe: the &xe_device to initialize
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_sriov_pf_init_early(struct xe_device *xe)
+{
+	xe_assert(xe, IS_SRIOV_PF(xe));
+
+	return drmm_mutex_init(&xe->drm, &xe->sriov.pf.master_lock);
+}
+
+/**
+ * xe_sriov_pf_print_vfs_summary - Print SR-IOV PF information.
+ * @xe: the &xe_device to print info from
+ * @p: the &drm_printer
+ *
+ * Print SR-IOV PF related information into provided DRM printer.
+ */
+void xe_sriov_pf_print_vfs_summary(struct xe_device *xe, struct drm_printer *p)
+{
+	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+
+	xe_assert(xe, IS_SRIOV_PF(xe));
+
+	drm_printf(p, "total: %u\n", xe->sriov.pf.device_total_vfs);
+	drm_printf(p, "supported: %u\n", xe->sriov.pf.driver_max_vfs);
+	drm_printf(p, "enabled: %u\n", pci_num_vf(pdev));
+}
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.h b/drivers/gpu/drm/xe/xe_sriov_pf.h
new file mode 100644
index 000000000000..d1220e70e1c0
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_sriov_pf.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#ifndef _XE_SRIOV_PF_H_
+#define _XE_SRIOV_PF_H_
+
+#include <linux/types.h>
+
+struct drm_printer;
+struct xe_device;
+
+#ifdef CONFIG_PCI_IOV
+bool xe_sriov_pf_readiness(struct xe_device *xe);
+int xe_sriov_pf_init_early(struct xe_device *xe);
+void xe_sriov_pf_print_vfs_summary(struct xe_device *xe, struct drm_printer *p);
+#else
+static inline bool xe_sriov_pf_readiness(struct xe_device *xe)
+{
+	return false;
+}
+
+static inline int xe_sriov_pf_init_early(struct xe_device *xe)
+{
+	return 0;
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h b/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h
new file mode 100644
index 000000000000..7d156ba82479
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023-2024 Intel Corporation
+ */
+
+#ifndef _XE_SRIOV_PF_HELPERS_H_
+#define _XE_SRIOV_PF_HELPERS_H_
+
+#include "xe_assert.h"
+#include "xe_device_types.h"
+#include "xe_sriov.h"
+#include "xe_sriov_types.h"
+
+/**
+ * xe_sriov_pf_assert_vfid() - warn if &id is not a supported VF number when debugging.
+ * @xe: the PF &xe_device to assert on
+ * @vfid: the VF number to assert
+ *
+ * Assert that &xe represents the Physical Function (PF) device and provided &vfid
+ * is within a range of supported VF numbers (up to maximum number of VFs that
+ * driver can support, including VF0 that represents the PF itself).
+ *
+ * Note: Effective only on debug builds. See `Xe ASSERTs`_ for more information.
+ */
+#define xe_sriov_pf_assert_vfid(xe, vfid) \
+	xe_assert((xe), (vfid) <= xe_sriov_pf_get_totalvfs(xe))
+
+/**
+ * xe_sriov_pf_get_totalvfs() - Get maximum number of VFs that driver can support.
+ * @xe: the &xe_device to query (shall be PF)
+ *
+ * Return: Maximum number of VFs that this PF driver supports.
+ */
+static inline int xe_sriov_pf_get_totalvfs(struct xe_device *xe)
+{
+	xe_assert(xe, IS_SRIOV_PF(xe));
+	return xe->sriov.pf.driver_max_vfs;
+}
+
+static inline struct mutex *xe_sriov_pf_master_mutex(struct xe_device *xe)
+{
+	xe_assert(xe, IS_SRIOV_PF(xe));
+	return &xe->sriov.pf.master_lock;
+}
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_sriov_types.h b/drivers/gpu/drm/xe/xe_sriov_types.h
index 1a138108d139..c7b7ad4af5c8 100644
--- a/drivers/gpu/drm/xe/xe_sriov_types.h
+++ b/drivers/gpu/drm/xe/xe_sriov_types.h
@@ -7,6 +7,8 @@
 #define _XE_SRIOV_TYPES_H_
 
 #include <linux/build_bug.h>
+#include <linux/mutex.h>
+#include <linux/types.h>
 
 /**
  * VFID - Virtual Function Identifier
@@ -37,4 +39,21 @@ enum xe_sriov_mode {
 };
 static_assert(XE_SRIOV_MODE_NONE);
 
+/**
+ * struct xe_device_pf - Xe PF related data
+ *
+ * The data in this structure is valid only if driver is running in the
+ * @XE_SRIOV_MODE_PF mode.
+ */
+struct xe_device_pf {
+	/** @device_total_vfs: Maximum number of VFs supported by the device. */
+	u16 device_total_vfs;
+
+	/** @driver_max_vfs: Maximum number of VFs supported by the driver. */
+	u16 driver_max_vfs;
+
+	/** @master_lock: protects all VFs configurations across GTs */
+	struct mutex master_lock;
+};
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c
index 02c9577fe418..65f1f1628235 100644
--- a/drivers/gpu/drm/xe/xe_sync.c
+++ b/drivers/gpu/drm/xe/xe_sync.c
@@ -224,8 +224,7 @@ int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job)
 	return 0;
 }
 
-void xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job,
-			  struct dma_fence *fence)
+void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence)
 {
 	if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))
 		return;
@@ -254,10 +253,6 @@ void xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job,
 			user_fence_put(sync->ufence);
 			dma_fence_put(fence);
 		}
-	} else if (sync->type == DRM_XE_SYNC_TYPE_USER_FENCE) {
-		job->user_fence.used = true;
-		job->user_fence.addr = sync->addr;
-		job->user_fence.value = sync->timeline_value;
 	}
 }
 
diff --git a/drivers/gpu/drm/xe/xe_sync.h b/drivers/gpu/drm/xe/xe_sync.h
index 0fd0d51208e6..3e03396af2c6 100644
--- a/drivers/gpu/drm/xe/xe_sync.h
+++ b/drivers/gpu/drm/xe/xe_sync.h
@@ -26,7 +26,6 @@ int xe_sync_entry_wait(struct xe_sync_entry *sync);
 int xe_sync_entry_add_deps(struct xe_sync_entry *sync,
 			   struct xe_sched_job *job);
 void xe_sync_entry_signal(struct xe_sync_entry *sync,
-			  struct xe_sched_job *job,
 			  struct dma_fence *fence);
 void xe_sync_entry_cleanup(struct xe_sync_entry *sync);
 struct dma_fence *
diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c
index 0650b2fa75ef..15ea0a942f67 100644
--- a/drivers/gpu/drm/xe/xe_tile.c
+++ b/drivers/gpu/drm/xe/xe_tile.c
@@ -160,24 +160,19 @@ int xe_tile_init_noalloc(struct xe_tile *tile)
 {
 	int err;
 
-	xe_device_mem_access_get(tile_to_xe(tile));
-
 	err = tile_ttm_mgr_init(tile);
 	if (err)
-		goto err_mem_access;
+		return err;
 
 	tile->mem.kernel_bb_pool = xe_sa_bo_manager_init(tile, SZ_1M, 16);
-	if (IS_ERR(tile->mem.kernel_bb_pool)) {
-		err = PTR_ERR(tile->mem.kernel_bb_pool);
-		goto err_mem_access;
-	}
+	if (IS_ERR(tile->mem.kernel_bb_pool))
+		return PTR_ERR(tile->mem.kernel_bb_pool);
+
 	xe_wa_apply_tile_workarounds(tile);
 
-	xe_tile_sysfs_init(tile);
+	err = xe_tile_sysfs_init(tile);
 
-err_mem_access:
-	xe_device_mem_access_put(tile_to_xe(tile));
-	return err;
+	return 0;
 }
 
 void xe_tile_migrate_wait(struct xe_tile *tile)
diff --git a/drivers/gpu/drm/xe/xe_tile_sysfs.c b/drivers/gpu/drm/xe/xe_tile_sysfs.c
index 0662968d7bcb..64661403afcd 100644
--- a/drivers/gpu/drm/xe/xe_tile_sysfs.c
+++ b/drivers/gpu/drm/xe/xe_tile_sysfs.c
@@ -7,6 +7,7 @@
 #include <linux/sysfs.h>
 #include <drm/drm_managed.h>
 
+#include "xe_pm.h"
 #include "xe_tile.h"
 #include "xe_tile_sysfs.h"
 #include "xe_vram_freq.h"
@@ -28,7 +29,7 @@ static void tile_sysfs_fini(struct drm_device *drm, void *arg)
 	kobject_put(tile->sysfs);
 }
 
-void xe_tile_sysfs_init(struct xe_tile *tile)
+int xe_tile_sysfs_init(struct xe_tile *tile)
 {
 	struct xe_device *xe = tile_to_xe(tile);
 	struct device *dev = xe->drm.dev;
@@ -37,7 +38,7 @@ void xe_tile_sysfs_init(struct xe_tile *tile)
 
 	kt = kzalloc(sizeof(*kt), GFP_KERNEL);
 	if (!kt)
-		return;
+		return -ENOMEM;
 
 	kobject_init(&kt->base, &xe_tile_sysfs_kobj_type);
 	kt->tile = tile;
@@ -45,16 +46,14 @@ void xe_tile_sysfs_init(struct xe_tile *tile)
 	err = kobject_add(&kt->base, &dev->kobj, "tile%d", tile->id);
 	if (err) {
 		kobject_put(&kt->base);
-		drm_warn(&xe->drm, "failed to register TILE sysfs directory, err: %d\n", err);
-		return;
+		return err;
 	}
 
 	tile->sysfs = &kt->base;
 
-	xe_vram_freq_sysfs_init(tile);
-
-	err = drmm_add_action_or_reset(&xe->drm, tile_sysfs_fini, tile);
+	err = xe_vram_freq_sysfs_init(tile);
 	if (err)
-		drm_warn(&xe->drm, "%s: drmm_add_action_or_reset failed, err: %d\n",
-			 __func__, err);
+		return err;
+
+	return drmm_add_action_or_reset(&xe->drm, tile_sysfs_fini, tile);
 }
diff --git a/drivers/gpu/drm/xe/xe_tile_sysfs.h b/drivers/gpu/drm/xe/xe_tile_sysfs.h
index e4f065039eba..54a2ba8ba533 100644
--- a/drivers/gpu/drm/xe/xe_tile_sysfs.h
+++ b/drivers/gpu/drm/xe/xe_tile_sysfs.h
@@ -8,7 +8,7 @@
 
 #include "xe_tile_sysfs_types.h"
 
-void xe_tile_sysfs_init(struct xe_tile *tile);
+int xe_tile_sysfs_init(struct xe_tile *tile);
 
 static inline struct xe_tile *
 kobj_to_tile(struct kobject *kobj)
diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h
index 846f14507d5f..2d56cfc09e42 100644
--- a/drivers/gpu/drm/xe/xe_trace.h
+++ b/drivers/gpu/drm/xe/xe_trace.h
@@ -258,7 +258,7 @@ DECLARE_EVENT_CLASS(xe_sched_job,
 			     __field(u32, guc_state)
 			     __field(u32, flags)
 			     __field(int, error)
-			     __field(u64, fence)
+			     __field(struct dma_fence *, fence)
 			     __field(u64, batch_addr)
 			     ),
 
@@ -269,11 +269,11 @@ DECLARE_EVENT_CLASS(xe_sched_job,
 			   atomic_read(&job->q->guc->state);
 			   __entry->flags = job->q->flags;
 			   __entry->error = job->fence->error;
-			   __entry->fence = (unsigned long)job->fence;
+			   __entry->fence = job->fence;
 			   __entry->batch_addr = (u64)job->batch_addr[0];
 			   ),
 
-		    TP_printk("fence=0x%016llx, seqno=%u, guc_id=%d, batch_addr=0x%012llx, guc_state=0x%x, flags=0x%x, error=%d",
+		    TP_printk("fence=%p, seqno=%u, guc_id=%d, batch_addr=0x%012llx, guc_state=0x%x, flags=0x%x, error=%d",
 			      __entry->fence, __entry->seqno, __entry->guc_id,
 			      __entry->batch_addr, __entry->guc_state,
 			      __entry->flags, __entry->error)
diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c
index 3107d2a12426..f77367329760 100644
--- a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c
@@ -204,9 +204,14 @@ void xe_ttm_stolen_mgr_init(struct xe_device *xe)
 {
 	struct xe_ttm_stolen_mgr *mgr = drmm_kzalloc(&xe->drm, sizeof(*mgr), GFP_KERNEL);
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
-	u64 stolen_size, io_size, pgsize;
+	u64 stolen_size, io_size;
 	int err;
 
+	if (!mgr) {
+		drm_dbg_kms(&xe->drm, "Stolen mgr init failed\n");
+		return;
+	}
+
 	if (IS_SRIOV_VF(xe))
 		stolen_size = 0;
 	else if (IS_DGFX(xe))
@@ -221,10 +226,6 @@ void xe_ttm_stolen_mgr_init(struct xe_device *xe)
 		return;
 	}
 
-	pgsize = xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K;
-	if (pgsize < PAGE_SIZE)
-		pgsize = PAGE_SIZE;
-
 	/*
 	 * We don't try to attempt partial visible support for stolen vram,
 	 * since stolen is always at the end of vram, and the BAR size is pretty
@@ -235,7 +236,7 @@ void xe_ttm_stolen_mgr_init(struct xe_device *xe)
 		io_size = stolen_size;
 
 	err = __xe_ttm_vram_mgr_init(xe, &mgr->base, XE_PL_STOLEN, stolen_size,
-				     io_size, pgsize);
+				     io_size, PAGE_SIZE);
 	if (err) {
 		drm_dbg_kms(&xe->drm, "Stolen mgr init failed: %i\n", err);
 		return;
@@ -298,7 +299,7 @@ static int __xe_ttm_stolen_io_mem_reserve_stolen(struct xe_device *xe,
 	XE_WARN_ON(IS_DGFX(xe));
 
 	/* XXX: Require BO to be mapped to GGTT? */
-	if (drm_WARN_ON(&xe->drm, !(bo->flags & XE_BO_CREATE_GGTT_BIT)))
+	if (drm_WARN_ON(&xe->drm, !(bo->flags & XE_BO_FLAG_GGTT)))
 		return -EIO;
 
 	/* GGTT is always contiguously mapped */
diff --git a/drivers/gpu/drm/xe/xe_ttm_sys_mgr.c b/drivers/gpu/drm/xe/xe_ttm_sys_mgr.c
index 3e1fa0c832ca..9844a8edbfe1 100644
--- a/drivers/gpu/drm/xe/xe_ttm_sys_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_sys_mgr.c
@@ -73,7 +73,10 @@ static void xe_ttm_sys_mgr_del(struct ttm_resource_manager *man,
 static void xe_ttm_sys_mgr_debug(struct ttm_resource_manager *man,
 				 struct drm_printer *printer)
 {
-
+	/*
+	 * This function is called by debugfs entry and would require
+	 * pm_runtime_{get,put} wrappers around any operation.
+	 */
 }
 
 static const struct ttm_resource_manager_func xe_ttm_sys_mgr_func = {
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
index 115ec745e502..fe3779fdba2c 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
@@ -91,7 +91,7 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
 
 	min_page_size = mgr->default_page_size;
 	if (tbo->page_alignment)
-		min_page_size = tbo->page_alignment << PAGE_SHIFT;
+		min_page_size = (u64)tbo->page_alignment << PAGE_SHIFT;
 
 	if (WARN_ON(min_page_size < mm->chunk_size)) {
 		err = -EINVAL;
@@ -196,7 +196,7 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
 	return 0;
 
 error_free_blocks:
-	drm_buddy_free_list(mm, &vres->blocks);
+	drm_buddy_free_list(mm, &vres->blocks, 0);
 	mutex_unlock(&mgr->lock);
 error_fini:
 	ttm_resource_fini(man, &vres->base);
@@ -214,7 +214,7 @@ static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man,
 	struct drm_buddy *mm = &mgr->mm;
 
 	mutex_lock(&mgr->lock);
-	drm_buddy_free_list(mm, &vres->blocks);
+	drm_buddy_free_list(mm, &vres->blocks, 0);
 	mgr->visible_avail += vres->used_visible_size;
 	mutex_unlock(&mgr->lock);
 
@@ -478,3 +478,15 @@ void xe_ttm_vram_get_used(struct ttm_resource_manager *man,
 	*used_visible = mgr->visible_size - mgr->visible_avail;
 	mutex_unlock(&mgr->lock);
 }
+
+u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man)
+{
+	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
+	u64 avail;
+
+	mutex_lock(&mgr->lock);
+	avail =  mgr->mm.avail;
+	mutex_unlock(&mgr->lock);
+
+	return avail;
+}
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h
index d184e19a9230..cc76050e376d 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h
@@ -25,6 +25,7 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
 void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir,
 			      struct sg_table *sgt);
 
+u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man);
 u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man);
 void xe_ttm_vram_get_used(struct ttm_resource_manager *man,
 			  u64 *used, u64 *used_visible);
diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c
index 5c83c75bc497..d4e6fa918942 100644
--- a/drivers/gpu/drm/xe/xe_tuning.c
+++ b/drivers/gpu/drm/xe/xe_tuning.c
@@ -28,7 +28,7 @@ static const struct xe_rtp_entry_sr gt_tunings[] = {
 	/* Xe2 */
 
 	{ XE_RTP_NAME("Tuning: L3 cache"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004)),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED)),
 	  XE_RTP_ACTIONS(FIELD_SET(XEHP_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK,
 				   REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f)))
 	},
@@ -38,11 +38,11 @@ static const struct xe_rtp_entry_sr gt_tunings[] = {
 				   REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f)))
 	},
 	{ XE_RTP_NAME("Tuning: Compression Overfetch"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2004, XE_RTP_END_VERSION_UNDEFINED)),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED)),
 	  XE_RTP_ACTIONS(CLR(CCCHKNREG1, ENCOMPPERFFIX)),
 	},
 	{ XE_RTP_NAME("Tuning: Enable compressible partial write overfetch in L3"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2004, XE_RTP_END_VERSION_UNDEFINED)),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED)),
 	  XE_RTP_ACTIONS(SET(L3SQCREG3, COMPPWOVERFETCHEN))
 	},
 	{}
@@ -50,7 +50,7 @@ static const struct xe_rtp_entry_sr gt_tunings[] = {
 
 static const struct xe_rtp_entry_sr engine_tunings[] = {
 	{ XE_RTP_NAME("Tuning: Set Indirect State Override"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1271),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1274),
 		       ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(SAMPLER_MODE, INDIRECT_STATE_BASE_ADDR_OVERRIDE))
 	},
@@ -88,7 +88,7 @@ static const struct xe_rtp_entry_sr lrc_tunings[] = {
 	/* Xe_LPG */
 
 	{ XE_RTP_NAME("Tuning: L3 cache"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1271), ENGINE_CLASS(RENDER)),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(FIELD_SET(XEHP_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK,
 				   REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f)))
 	},
diff --git a/drivers/gpu/drm/xe/xe_uc.c b/drivers/gpu/drm/xe/xe_uc.c
index 7033f8c1b431..4feb35c95a1c 100644
--- a/drivers/gpu/drm/xe/xe_uc.c
+++ b/drivers/gpu/drm/xe/xe_uc.c
@@ -32,11 +32,8 @@ uc_to_xe(struct xe_uc *uc)
 /* Should be called once at driver load only */
 int xe_uc_init(struct xe_uc *uc)
 {
-	struct xe_device *xe = uc_to_xe(uc);
 	int ret;
 
-	xe_device_mem_access_get(xe);
-
 	/*
 	 * We call the GuC/HuC/GSC init functions even if GuC submission is off
 	 * to correctly move our tracking of the FW state to "disabled".
@@ -65,16 +62,8 @@ int xe_uc_init(struct xe_uc *uc)
 		goto err;
 
 	ret = xe_guc_db_mgr_init(&uc->guc.dbm, ~0);
-	if (ret)
-		goto err;
-
-	xe_device_mem_access_put(xe);
-
-	return 0;
 
 err:
-	xe_device_mem_access_put(xe);
-
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/xe/xe_uc_debugfs.c b/drivers/gpu/drm/xe/xe_uc_debugfs.c
index 0a39ec5a6e99..78eb8db73791 100644
--- a/drivers/gpu/drm/xe/xe_uc_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_uc_debugfs.c
@@ -3,6 +3,8 @@
  * Copyright © 2022 Intel Corporation
  */
 
+#include <linux/debugfs.h>
+
 #include <drm/drm_debugfs.h>
 
 #include "xe_gt.h"
diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c
index a9d25b3fa67c..186f81640cef 100644
--- a/drivers/gpu/drm/xe/xe_uc_fw.c
+++ b/drivers/gpu/drm/xe/xe_uc_fw.c
@@ -17,6 +17,7 @@
 #include "xe_map.h"
 #include "xe_mmio.h"
 #include "xe_module.h"
+#include "xe_sriov.h"
 #include "xe_uc_fw.h"
 
 /*
@@ -296,36 +297,28 @@ static void uc_fw_fini(struct drm_device *drm, void *arg)
 	xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_SELECTED);
 }
 
-static void guc_read_css_info(struct xe_uc_fw *uc_fw, struct uc_css_header *css)
+static int guc_read_css_info(struct xe_uc_fw *uc_fw, struct uc_css_header *css)
 {
 	struct xe_gt *gt = uc_fw_to_gt(uc_fw);
 	struct xe_uc_fw_version *release = &uc_fw->versions.found[XE_UC_FW_VER_RELEASE];
 	struct xe_uc_fw_version *compatibility = &uc_fw->versions.found[XE_UC_FW_VER_COMPATIBILITY];
 
 	xe_gt_assert(gt, uc_fw->type == XE_UC_FW_TYPE_GUC);
-	xe_gt_assert(gt, release->major >= 70);
-
-	if (release->major > 70 || release->minor >= 6) {
-		/* v70.6.0 adds CSS header support */
-		compatibility->major = FIELD_GET(CSS_SW_VERSION_UC_MAJOR,
-						 css->submission_version);
-		compatibility->minor = FIELD_GET(CSS_SW_VERSION_UC_MINOR,
-						 css->submission_version);
-		compatibility->patch = FIELD_GET(CSS_SW_VERSION_UC_PATCH,
-						 css->submission_version);
-	} else if (release->minor >= 3) {
-		/* v70.3.0 introduced v1.1.0 */
-		compatibility->major = 1;
-		compatibility->minor = 1;
-		compatibility->patch = 0;
-	} else {
-		/* v70.0.0 introduced v1.0.0 */
-		compatibility->major = 1;
-		compatibility->minor = 0;
-		compatibility->patch = 0;
+
+	/* We don't support GuC releases older than 70.19 */
+	if (release->major < 70 || (release->major == 70 && release->minor < 19)) {
+		xe_gt_err(gt, "Unsupported GuC v%u.%u! v70.19 or newer is required\n",
+			  release->major, release->minor);
+		return -EINVAL;
 	}
 
+	compatibility->major = FIELD_GET(CSS_SW_VERSION_UC_MAJOR, css->submission_version);
+	compatibility->minor = FIELD_GET(CSS_SW_VERSION_UC_MINOR, css->submission_version);
+	compatibility->patch = FIELD_GET(CSS_SW_VERSION_UC_PATCH, css->submission_version);
+
 	uc_fw->private_data_size = css->private_data_size;
+
+	return 0;
 }
 
 int xe_uc_fw_check_version_requirements(struct xe_uc_fw *uc_fw)
@@ -424,7 +417,7 @@ static int parse_css_header(struct xe_uc_fw *uc_fw, const void *fw_data, size_t
 	release->patch = FIELD_GET(CSS_SW_VERSION_UC_PATCH, css->sw_version);
 
 	if (uc_fw->type == XE_UC_FW_TYPE_GUC)
-		guc_read_css_info(uc_fw, css);
+		return guc_read_css_info(uc_fw, css);
 
 	return 0;
 }
@@ -658,7 +651,17 @@ static int uc_fw_request(struct xe_uc_fw *uc_fw, const struct firmware **firmwar
 	xe_assert(xe, !uc_fw->path);
 
 	uc_fw_auto_select(xe, uc_fw);
+
+	if (IS_SRIOV_VF(xe)) {
+		/* VF will support only firmwares that driver can autoselect */
+		xe_uc_fw_change_status(uc_fw, uc_fw->path ?
+				       XE_UC_FIRMWARE_PRELOADED :
+				       XE_UC_FIRMWARE_NOT_SUPPORTED);
+		return 0;
+	}
+
 	uc_fw_override(uc_fw);
+
 	xe_uc_fw_change_status(uc_fw, uc_fw->path ?
 			       XE_UC_FIRMWARE_SELECTED :
 			       XE_UC_FIRMWARE_NOT_SUPPORTED);
@@ -771,7 +774,8 @@ int xe_uc_fw_init(struct xe_uc_fw *uc_fw)
 		return 0;
 
 	err = uc_fw_copy(uc_fw, fw->data, fw->size,
-			 XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_GGTT_BIT);
+			 XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT |
+			 XE_BO_FLAG_GGTT_INVALIDATE);
 
 	uc_fw_release(fw);
 
@@ -787,7 +791,8 @@ static int uc_fw_xfer(struct xe_uc_fw *uc_fw, u32 offset, u32 dma_flags)
 {
 	struct xe_device *xe = uc_fw_to_xe(uc_fw);
 	struct xe_gt *gt = uc_fw_to_gt(uc_fw);
-	u32 src_offset, dma_ctrl;
+	u64 src_offset;
+	u32 dma_ctrl;
 	int ret;
 
 	xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
diff --git a/drivers/gpu/drm/xe/xe_uc_fw.h b/drivers/gpu/drm/xe/xe_uc_fw.h
index 85c20795d1f8..35078038797e 100644
--- a/drivers/gpu/drm/xe/xe_uc_fw.h
+++ b/drivers/gpu/drm/xe/xe_uc_fw.h
@@ -59,6 +59,8 @@ const char *xe_uc_fw_status_repr(enum xe_uc_fw_status status)
 		return "TRANSFERRED";
 	case XE_UC_FIRMWARE_RUNNING:
 		return "RUNNING";
+	case XE_UC_FIRMWARE_PRELOADED:
+		return "PRELOADED";
 	}
 	return "<invalid>";
 }
@@ -85,6 +87,7 @@ static inline int xe_uc_fw_status_to_error(enum xe_uc_fw_status status)
 	case XE_UC_FIRMWARE_LOADABLE:
 	case XE_UC_FIRMWARE_TRANSFERRED:
 	case XE_UC_FIRMWARE_RUNNING:
+	case XE_UC_FIRMWARE_PRELOADED:
 		return 0;
 	}
 	return -EINVAL;
@@ -134,7 +137,8 @@ static inline bool xe_uc_fw_is_available(struct xe_uc_fw *uc_fw)
 
 static inline bool xe_uc_fw_is_loadable(struct xe_uc_fw *uc_fw)
 {
-	return __xe_uc_fw_status(uc_fw) >= XE_UC_FIRMWARE_LOADABLE;
+	return __xe_uc_fw_status(uc_fw) >= XE_UC_FIRMWARE_LOADABLE &&
+		__xe_uc_fw_status(uc_fw) != XE_UC_FIRMWARE_PRELOADED;
 }
 
 static inline bool xe_uc_fw_is_loaded(struct xe_uc_fw *uc_fw)
@@ -144,7 +148,7 @@ static inline bool xe_uc_fw_is_loaded(struct xe_uc_fw *uc_fw)
 
 static inline bool xe_uc_fw_is_running(struct xe_uc_fw *uc_fw)
 {
-	return __xe_uc_fw_status(uc_fw) == XE_UC_FIRMWARE_RUNNING;
+	return __xe_uc_fw_status(uc_fw) >= XE_UC_FIRMWARE_RUNNING;
 }
 
 static inline bool xe_uc_fw_is_overridden(const struct xe_uc_fw *uc_fw)
diff --git a/drivers/gpu/drm/xe/xe_uc_fw_types.h b/drivers/gpu/drm/xe/xe_uc_fw_types.h
index bc800b696866..0d8caa0e7354 100644
--- a/drivers/gpu/drm/xe/xe_uc_fw_types.h
+++ b/drivers/gpu/drm/xe/xe_uc_fw_types.h
@@ -50,7 +50,8 @@ enum xe_uc_fw_status {
 	XE_UC_FIRMWARE_LOADABLE, /* all fw-required objects are ready */
 	XE_UC_FIRMWARE_LOAD_FAIL, /* failed to xfer or init/auth the fw */
 	XE_UC_FIRMWARE_TRANSFERRED, /* dma xfer done */
-	XE_UC_FIRMWARE_RUNNING /* init/auth done */
+	XE_UC_FIRMWARE_RUNNING, /* init/auth done */
+	XE_UC_FIRMWARE_PRELOADED, /* preloaded by the PF driver */
 };
 
 enum xe_uc_fw_type {
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 62d1ef8867a8..4aa3943e6f29 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -21,12 +21,12 @@
 
 #include <generated/xe_wa_oob.h>
 
+#include "regs/xe_gtt_defs.h"
 #include "xe_assert.h"
 #include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_drm_client.h"
 #include "xe_exec_queue.h"
-#include "xe_gt.h"
 #include "xe_gt_pagefault.h"
 #include "xe_gt_tlb_invalidation.h"
 #include "xe_migrate.h"
@@ -38,6 +38,7 @@
 #include "xe_sync.h"
 #include "xe_trace.h"
 #include "xe_wa.h"
+#include "xe_hmm.h"
 
 static struct drm_gem_object *xe_vm_obj(struct xe_vm *vm)
 {
@@ -65,113 +66,14 @@ int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma)
 
 int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma)
 {
-	struct xe_userptr *userptr = &uvma->userptr;
 	struct xe_vma *vma = &uvma->vma;
 	struct xe_vm *vm = xe_vma_vm(vma);
 	struct xe_device *xe = vm->xe;
-	const unsigned long num_pages = xe_vma_size(vma) >> PAGE_SHIFT;
-	struct page **pages;
-	bool in_kthread = !current->mm;
-	unsigned long notifier_seq;
-	int pinned, ret, i;
-	bool read_only = xe_vma_read_only(vma);
 
 	lockdep_assert_held(&vm->lock);
 	xe_assert(xe, xe_vma_is_userptr(vma));
-retry:
-	if (vma->gpuva.flags & XE_VMA_DESTROYED)
-		return 0;
-
-	notifier_seq = mmu_interval_read_begin(&userptr->notifier);
-	if (notifier_seq == userptr->notifier_seq)
-		return 0;
-
-	pages = kvmalloc_array(num_pages, sizeof(*pages), GFP_KERNEL);
-	if (!pages)
-		return -ENOMEM;
-
-	if (userptr->sg) {
-		dma_unmap_sgtable(xe->drm.dev,
-				  userptr->sg,
-				  read_only ? DMA_TO_DEVICE :
-				  DMA_BIDIRECTIONAL, 0);
-		sg_free_table(userptr->sg);
-		userptr->sg = NULL;
-	}
-
-	pinned = ret = 0;
-	if (in_kthread) {
-		if (!mmget_not_zero(userptr->notifier.mm)) {
-			ret = -EFAULT;
-			goto mm_closed;
-		}
-		kthread_use_mm(userptr->notifier.mm);
-	}
-
-	while (pinned < num_pages) {
-		ret = get_user_pages_fast(xe_vma_userptr(vma) +
-					  pinned * PAGE_SIZE,
-					  num_pages - pinned,
-					  read_only ? 0 : FOLL_WRITE,
-					  &pages[pinned]);
-		if (ret < 0)
-			break;
-
-		pinned += ret;
-		ret = 0;
-	}
-
-	if (in_kthread) {
-		kthread_unuse_mm(userptr->notifier.mm);
-		mmput(userptr->notifier.mm);
-	}
-mm_closed:
-	if (ret)
-		goto out;
 
-	ret = sg_alloc_table_from_pages_segment(&userptr->sgt, pages,
-						pinned, 0,
-						(u64)pinned << PAGE_SHIFT,
-						xe_sg_segment_size(xe->drm.dev),
-						GFP_KERNEL);
-	if (ret) {
-		userptr->sg = NULL;
-		goto out;
-	}
-	userptr->sg = &userptr->sgt;
-
-	ret = dma_map_sgtable(xe->drm.dev, userptr->sg,
-			      read_only ? DMA_TO_DEVICE :
-			      DMA_BIDIRECTIONAL,
-			      DMA_ATTR_SKIP_CPU_SYNC |
-			      DMA_ATTR_NO_KERNEL_MAPPING);
-	if (ret) {
-		sg_free_table(userptr->sg);
-		userptr->sg = NULL;
-		goto out;
-	}
-
-	for (i = 0; i < pinned; ++i) {
-		if (!read_only) {
-			lock_page(pages[i]);
-			set_page_dirty(pages[i]);
-			unlock_page(pages[i]);
-		}
-
-		mark_page_accessed(pages[i]);
-	}
-
-out:
-	release_pages(pages, pinned);
-	kvfree(pages);
-
-	if (!(ret < 0)) {
-		userptr->notifier_seq = notifier_seq;
-		if (xe_vma_userptr_check_repin(uvma) == -EAGAIN)
-			goto retry;
-	}
-
-	return ret < 0 ? ret : 0;
+	return xe_hmm_userptr_populate_range(uvma, false);
 }
 
 static bool preempt_fences_waiting(struct xe_vm *vm)
@@ -682,6 +584,10 @@ static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
 	if (!mmu_notifier_range_blockable(range))
 		return false;
 
+	vm_dbg(&xe_vma_vm(vma)->xe->drm,
+	       "NOTIFIER: addr=0x%016llx, range=0x%016llx",
+		xe_vma_start(vma), xe_vma_size(vma));
+
 	down_write(&vm->userptr.notifier_lock);
 	mmu_interval_set_seq(mni, cur_seq);
 
@@ -951,8 +857,6 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
 static void xe_vma_destroy_late(struct xe_vma *vma)
 {
 	struct xe_vm *vm = xe_vma_vm(vma);
-	struct xe_device *xe = vm->xe;
-	bool read_only = xe_vma_read_only(vma);
 
 	if (vma->ufence) {
 		xe_sync_ufence_put(vma->ufence);
@@ -960,16 +864,11 @@ static void xe_vma_destroy_late(struct xe_vma *vma)
 	}
 
 	if (xe_vma_is_userptr(vma)) {
-		struct xe_userptr *userptr = &to_userptr_vma(vma)->userptr;
+		struct xe_userptr_vma *uvma = to_userptr_vma(vma);
+		struct xe_userptr *userptr = &uvma->userptr;
 
-		if (userptr->sg) {
-			dma_unmap_sgtable(xe->drm.dev,
-					  userptr->sg,
-					  read_only ? DMA_TO_DEVICE :
-					  DMA_BIDIRECTIONAL, 0);
-			sg_free_table(userptr->sg);
-			userptr->sg = NULL;
-		}
+		if (userptr->sg)
+			xe_hmm_userptr_free_sg(uvma);
 
 		/*
 		 * Since userptr pages are not pinned, we can't remove
@@ -1274,8 +1173,6 @@ static const struct xe_pt_ops xelp_pt_ops = {
 	.pde_encode_bo = xelp_pde_encode_bo,
 };
 
-static void vm_destroy_work_func(struct work_struct *w);
-
 /**
  * xe_vm_create_scratch() - Setup a scratch memory pagetable tree for the
  * given tile and vm.
@@ -1355,8 +1252,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
 	init_rwsem(&vm->userptr.notifier_lock);
 	spin_lock_init(&vm->userptr.invalidated_lock);
 
-	INIT_WORK(&vm->destroy_work, vm_destroy_work_func);
-
 	INIT_LIST_HEAD(&vm->preempt.exec_queues);
 	vm->preempt.min_run_period_ms = 10;	/* FIXME: Wire up to uAPI */
 
@@ -1366,7 +1261,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
 	vm->pt_ops = &xelp_pt_ops;
 
 	if (!(flags & XE_VM_FLAG_MIGRATION))
-		xe_device_mem_access_get(xe);
+		xe_pm_runtime_get_noresume(xe);
 
 	vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm);
 	if (!vm_resv_obj) {
@@ -1411,9 +1306,8 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
 		vm->batch_invalidate_tlb = true;
 	}
 
-	if (flags & XE_VM_FLAG_LR_MODE) {
+	if (vm->flags & XE_VM_FLAG_LR_MODE) {
 		INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func);
-		vm->flags |= XE_VM_FLAG_LR_MODE;
 		vm->batch_invalidate_tlb = false;
 	}
 
@@ -1477,7 +1371,7 @@ err_no_resv:
 		xe_range_fence_tree_fini(&vm->rftree[id]);
 	kfree(vm);
 	if (!(flags & XE_VM_FLAG_MIGRATION))
-		xe_device_mem_access_put(xe);
+		xe_pm_runtime_put(xe);
 	return ERR_PTR(err);
 }
 
@@ -1577,6 +1471,16 @@ void xe_vm_close_and_put(struct xe_vm *vm)
 		xe->usm.num_vm_in_fault_mode--;
 	else if (!(vm->flags & XE_VM_FLAG_MIGRATION))
 		xe->usm.num_vm_in_non_fault_mode--;
+
+	if (vm->usm.asid) {
+		void *lookup;
+
+		xe_assert(xe, xe->info.has_asid);
+		xe_assert(xe, !(vm->flags & XE_VM_FLAG_MIGRATION));
+
+		lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid);
+		xe_assert(xe, lookup == vm);
+	}
 	mutex_unlock(&xe->usm.lock);
 
 	for_each_tile(tile, xe, id)
@@ -1585,30 +1489,23 @@ void xe_vm_close_and_put(struct xe_vm *vm)
 	xe_vm_put(vm);
 }
 
-static void vm_destroy_work_func(struct work_struct *w)
+static void xe_vm_free(struct drm_gpuvm *gpuvm)
 {
-	struct xe_vm *vm =
-		container_of(w, struct xe_vm, destroy_work);
+	struct xe_vm *vm = container_of(gpuvm, struct xe_vm, gpuvm);
 	struct xe_device *xe = vm->xe;
 	struct xe_tile *tile;
 	u8 id;
-	void *lookup;
 
 	/* xe_vm_close_and_put was not called? */
 	xe_assert(xe, !vm->size);
 
-	mutex_destroy(&vm->snap_mutex);
+	if (xe_vm_in_preempt_fence_mode(vm))
+		flush_work(&vm->preempt.rebind_work);
 
-	if (!(vm->flags & XE_VM_FLAG_MIGRATION)) {
-		xe_device_mem_access_put(xe);
+	mutex_destroy(&vm->snap_mutex);
 
-		if (xe->info.has_asid && vm->usm.asid) {
-			mutex_lock(&xe->usm.lock);
-			lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid);
-			xe_assert(xe, lookup == vm);
-			mutex_unlock(&xe->usm.lock);
-		}
-	}
+	if (!(vm->flags & XE_VM_FLAG_MIGRATION))
+		xe_pm_runtime_put(xe);
 
 	for_each_tile(tile, xe, id)
 		XE_WARN_ON(vm->pt_root[id]);
@@ -1617,14 +1514,6 @@ static void vm_destroy_work_func(struct work_struct *w)
 	kfree(vm);
 }
 
-static void xe_vm_free(struct drm_gpuvm *gpuvm)
-{
-	struct xe_vm *vm = container_of(gpuvm, struct xe_vm, gpuvm);
-
-	/* To destroy the VM we need to be able to sleep */
-	queue_work(system_unbound_wq, &vm->destroy_work);
-}
-
 struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id)
 {
 	struct xe_vm *vm;
@@ -1721,7 +1610,7 @@ next:
 		xe_exec_queue_last_fence_get(wait_exec_queue, vm) : fence;
 	if (last_op) {
 		for (i = 0; i < num_syncs; i++)
-			xe_sync_entry_signal(&syncs[i], NULL, fence);
+			xe_sync_entry_signal(&syncs[i], fence);
 	}
 
 	return fence;
@@ -1795,7 +1684,7 @@ next:
 
 	if (last_op) {
 		for (i = 0; i < num_syncs; i++)
-			xe_sync_entry_signal(&syncs[i], NULL,
+			xe_sync_entry_signal(&syncs[i],
 					     cf ? &cf->base : fence);
 	}
 
@@ -1856,7 +1745,7 @@ static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma,
 		fence = xe_exec_queue_last_fence_get(wait_exec_queue, vm);
 		if (last_op) {
 			for (i = 0; i < num_syncs; i++)
-				xe_sync_entry_signal(&syncs[i], NULL, fence);
+				xe_sync_entry_signal(&syncs[i], fence);
 		}
 	}
 
@@ -2057,7 +1946,7 @@ static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
 	struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
 	int err;
 
-	xe_assert(vm->xe, region <= ARRAY_SIZE(region_to_mem_type));
+	xe_assert(vm->xe, region < ARRAY_SIZE(region_to_mem_type));
 
 	if (!xe_vma_has_no_bo(vma)) {
 		err = xe_bo_migrate(xe_vma_bo(vma), region_to_mem_type[region]);
@@ -2077,7 +1966,7 @@ static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
 				struct dma_fence *fence =
 					xe_exec_queue_last_fence_get(wait_exec_queue, vm);
 
-				xe_sync_entry_signal(&syncs[i], NULL, fence);
+				xe_sync_entry_signal(&syncs[i], fence);
 				dma_fence_put(fence);
 			}
 		}
@@ -2209,6 +2098,10 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo,
 		struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
 
 		if (__op->op == DRM_GPUVA_OP_MAP) {
+			op->map.immediate =
+				flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE;
+			op->map.read_only =
+				flags & DRM_XE_VM_BIND_FLAG_READONLY;
 			op->map.is_null = flags & DRM_XE_VM_BIND_FLAG_NULL;
 			op->map.dumpable = flags & DRM_XE_VM_BIND_FLAG_DUMPABLE;
 			op->map.pat_index = pat_index;
@@ -2403,6 +2296,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
 		switch (op->base.op) {
 		case DRM_GPUVA_OP_MAP:
 		{
+			flags |= op->map.read_only ?
+				VMA_CREATE_FLAG_READ_ONLY : 0;
 			flags |= op->map.is_null ?
 				VMA_CREATE_FLAG_IS_NULL : 0;
 			flags |= op->map.dumpable ?
@@ -2547,7 +2442,7 @@ static int op_execute(struct drm_exec *exec, struct xe_vm *vm,
 	case DRM_GPUVA_OP_MAP:
 		err = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma),
 				 op->syncs, op->num_syncs,
-				 !xe_vm_in_fault_mode(vm),
+				 op->map.immediate || !xe_vm_in_fault_mode(vm),
 				 op->flags & XE_VMA_OP_FIRST,
 				 op->flags & XE_VMA_OP_LAST);
 		break;
@@ -2822,7 +2717,10 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
 	return 0;
 }
 
-#define SUPPORTED_FLAGS	(DRM_XE_VM_BIND_FLAG_NULL | \
+#define SUPPORTED_FLAGS	\
+	(DRM_XE_VM_BIND_FLAG_READONLY | \
+	 DRM_XE_VM_BIND_FLAG_IMMEDIATE | \
+	 DRM_XE_VM_BIND_FLAG_NULL | \
 	 DRM_XE_VM_BIND_FLAG_DUMPABLE)
 #define XE_64K_PAGE_MASK 0xffffull
 #define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP)
@@ -2955,7 +2853,7 @@ static int vm_bind_ioctl_signal_fences(struct xe_vm *vm,
 		return PTR_ERR(fence);
 
 	for (i = 0; i < num_syncs; i++)
-		xe_sync_entry_signal(&syncs[i], NULL, fence);
+		xe_sync_entry_signal(&syncs[i], fence);
 
 	xe_exec_queue_last_fence_set(to_wait_exec_queue(vm, q), vm,
 				     fence);
@@ -3066,7 +2964,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 			goto put_obj;
 		}
 
-		if (bos[i]->flags & XE_BO_INTERNAL_64K) {
+		if (bos[i]->flags & XE_BO_FLAG_INTERNAL_64K) {
 			if (XE_IOCTL_DBG(xe, obj_offset &
 					 XE_64K_PAGE_MASK) ||
 			    XE_IOCTL_DBG(xe, addr & XE_64K_PAGE_MASK) ||
@@ -3258,6 +3156,10 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)
 	xe_assert(xe, !xe_vma_is_null(vma));
 	trace_xe_vma_invalidate(vma);
 
+	vm_dbg(&xe_vma_vm(vma)->xe->drm,
+	       "INVALIDATE: addr=0x%016llx, range=0x%016llx",
+		xe_vma_start(vma), xe_vma_size(vma));
+
 	/* Check that we don't race with page-table updates */
 	if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
 		if (xe_vma_is_userptr(vma)) {
@@ -3376,8 +3278,10 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm)
 
 	if (num_snaps)
 		snap = kvzalloc(offsetof(struct xe_vm_snapshot, snap[num_snaps]), GFP_NOWAIT);
-	if (!snap)
+	if (!snap) {
+		snap = num_snaps ? ERR_PTR(-ENOMEM) : ERR_PTR(-ENODEV);
 		goto out_unlock;
+	}
 
 	snap->num_snaps = num_snaps;
 	i = 0;
@@ -3417,6 +3321,9 @@ out_unlock:
 
 void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap)
 {
+	if (IS_ERR_OR_NULL(snap))
+		return;
+
 	for (int i = 0; i < snap->num_snaps; i++) {
 		struct xe_bo *bo = snap->snap[i].bo;
 		struct iosys_map src;
@@ -3471,13 +3378,21 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p)
 {
 	unsigned long i, j;
 
-	for (i = 0; i < snap->num_snaps; i++) {
-		if (IS_ERR(snap->snap[i].data))
-			goto uncaptured;
+	if (IS_ERR_OR_NULL(snap)) {
+		drm_printf(p, "[0].error: %li\n", PTR_ERR(snap));
+		return;
+	}
 
+	for (i = 0; i < snap->num_snaps; i++) {
 		drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len);
-		drm_printf(p, "[%llx].data: ",
-			   snap->snap[i].ofs);
+
+		if (IS_ERR(snap->snap[i].data)) {
+			drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs,
+				   PTR_ERR(snap->snap[i].data));
+			continue;
+		}
+
+		drm_printf(p, "[%llx].data: ", snap->snap[i].ofs);
 
 		for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) {
 			u32 *val = snap->snap[i].data + j;
@@ -3487,12 +3402,6 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p)
 		}
 
 		drm_puts(p, "\n");
-		continue;
-
-uncaptured:
-		drm_printf(p, "Unable to capture range [%llx-%llx]: %li\n",
-			   snap->snap[i].ofs, snap->snap[i].ofs + snap->snap[i].len - 1,
-			   PTR_ERR(snap->snap[i].data));
 	}
 }
 
@@ -3500,7 +3409,7 @@ void xe_vm_snapshot_free(struct xe_vm_snapshot *snap)
 {
 	unsigned long i;
 
-	if (!snap)
+	if (IS_ERR_OR_NULL(snap))
 		return;
 
 	for (i = 0; i < snap->num_snaps; i++) {
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index badf3945083d..72a100671e5d 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -178,13 +178,6 @@ struct xe_vm {
 	struct list_head rebind_list;
 
 	/**
-	 * @destroy_work: worker to destroy VM, needed as a dma_fence signaling
-	 * from an irq context can be last put and the destroy needs to be able
-	 * to sleep.
-	 */
-	struct work_struct destroy_work;
-
-	/**
 	 * @rftree: range fence tree to track updates to page table structure.
 	 * Used to implement conflict tracking between independent bind engines.
 	 */
@@ -276,6 +269,10 @@ struct xe_vm {
 struct xe_vma_op_map {
 	/** @vma: VMA to map */
 	struct xe_vma *vma;
+	/** @immediate: Immediate bind */
+	bool immediate;
+	/** @read_only: Read only */
+	bool read_only;
 	/** @is_null: is NULL binding */
 	bool is_null;
 	/** @dumpable: whether BO is dumped on GPU hang */
diff --git a/drivers/gpu/drm/xe/xe_vram_freq.c b/drivers/gpu/drm/xe/xe_vram_freq.c
index c5f6b5a5d117..3e21ddc6e60c 100644
--- a/drivers/gpu/drm/xe/xe_vram_freq.c
+++ b/drivers/gpu/drm/xe/xe_vram_freq.c
@@ -100,31 +100,27 @@ static void vram_freq_sysfs_fini(struct drm_device *drm, void *arg)
  * @tile: Xe Tile object
  *
  * It needs to be initialized after the main tile component is ready
+ *
+ * Returns: 0 on success, negative error code on error.
  */
-void xe_vram_freq_sysfs_init(struct xe_tile *tile)
+int xe_vram_freq_sysfs_init(struct xe_tile *tile)
 {
 	struct xe_device *xe = tile_to_xe(tile);
 	struct kobject *kobj;
 	int err;
 
 	if (xe->info.platform != XE_PVC)
-		return;
+		return 0;
 
 	kobj = kobject_create_and_add("memory", tile->sysfs);
-	if (!kobj) {
-		drm_warn(&xe->drm, "failed to add memory directory, err: %d\n", -ENOMEM);
-		return;
-	}
+	if (!kobj)
+		return -ENOMEM;
 
 	err = sysfs_create_group(kobj, &freq_group_attrs);
 	if (err) {
 		kobject_put(kobj);
-		drm_warn(&xe->drm, "failed to register vram freq sysfs, err: %d\n", err);
-		return;
+		return err;
 	}
 
-	err = drmm_add_action_or_reset(&xe->drm, vram_freq_sysfs_fini, kobj);
-	if (err)
-		drm_warn(&xe->drm, "%s: drmm_add_action_or_reset failed, err: %d\n",
-			 __func__, err);
+	return drmm_add_action_or_reset(&xe->drm, vram_freq_sysfs_fini, kobj);
 }
diff --git a/drivers/gpu/drm/xe/xe_vram_freq.h b/drivers/gpu/drm/xe/xe_vram_freq.h
index cbe8c12fbd64..bf726bc5881f 100644
--- a/drivers/gpu/drm/xe/xe_vram_freq.h
+++ b/drivers/gpu/drm/xe/xe_vram_freq.h
@@ -8,6 +8,6 @@
 
 struct xe_tile;
 
-void xe_vram_freq_sysfs_init(struct xe_tile *tile);
+int xe_vram_freq_sysfs_init(struct xe_tile *tile);
 
 #endif /* _XE_VRAM_FREQ_H_ */
diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c
index a0264eedd443..dd214d95e4b6 100644
--- a/drivers/gpu/drm/xe/xe_wa.c
+++ b/drivers/gpu/drm/xe/xe_wa.c
@@ -173,11 +173,11 @@ static const struct xe_rtp_entry_sr gt_was[] = {
 	  XE_RTP_ACTIONS(CLR(MISCCPCTL, DOP_CLOCK_GATE_RENDER_ENABLE))
 	},
 	{ XE_RTP_NAME("14018575942"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1271)),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274)),
 	  XE_RTP_ACTIONS(SET(COMP_MOD_CTRL, FORCE_MISS_FTLB))
 	},
 	{ XE_RTP_NAME("22016670082"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1271)),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274)),
 	  XE_RTP_ACTIONS(SET(SQCNT1, ENFORCE_RAR))
 	},
 
@@ -228,6 +228,28 @@ static const struct xe_rtp_entry_sr gt_was[] = {
 	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
 	},
 
+	/* Xe2_HPM */
+
+	{ XE_RTP_NAME("16021867713"),
+	  XE_RTP_RULES(MEDIA_VERSION(1301),
+		       ENGINE_CLASS(VIDEO_DECODE)),
+	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)),
+	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
+	},
+	{ XE_RTP_NAME("14020316580"),
+	  XE_RTP_RULES(MEDIA_VERSION(1301)),
+	  XE_RTP_ACTIONS(CLR(PG_ENABLE,
+			     VD0_HCP_POWERGATE_ENABLE |
+			     VD0_MFXVDENC_POWERGATE_ENABLE |
+			     VD2_HCP_POWERGATE_ENABLE |
+			     VD2_MFXVDENC_POWERGATE_ENABLE)),
+	},
+	{ XE_RTP_NAME("14019449301"),
+	  XE_RTP_RULES(MEDIA_VERSION(1301), ENGINE_CLASS(VIDEO_DECODE)),
+	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F08(0), CG3DDISHRS_CLKGATE_DIS)),
+	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
+	},
+
 	{}
 };
 
@@ -328,12 +350,6 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE))
 	},
-	{ XE_RTP_NAME("16015675438"),
-	  XE_RTP_RULES(PLATFORM(DG2),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(FF_SLICE_CS_CHICKEN2(RENDER_RING_BASE),
-			     PERF_FIX_BALANCING_CFE_DISABLE))
-	},
 	{ XE_RTP_NAME("18028616096"),
 	  XE_RTP_RULES(PLATFORM(DG2),
 		       FUNC(xe_rtp_match_first_render_or_compute)),
@@ -383,10 +399,10 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 	  XE_RTP_RULES(PLATFORM(PVC), FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE))
 	},
-	{ XE_RTP_NAME("16015675438"),
-	  XE_RTP_RULES(PLATFORM(PVC), FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(FF_SLICE_CS_CHICKEN2(RENDER_RING_BASE),
-			     PERF_FIX_BALANCING_CFE_DISABLE))
+	{ XE_RTP_NAME("18020744125"),
+	  XE_RTP_RULES(PLATFORM(PVC), FUNC(xe_rtp_match_first_render_or_compute),
+		       ENGINE_CLASS(COMPUTE)),
+	  XE_RTP_ACTIONS(SET(RING_HWSTAM(RENDER_RING_BASE), ~0))
 	},
 	{ XE_RTP_NAME("14014999345"),
 	  XE_RTP_RULES(PLATFORM(PVC), ENGINE_CLASS(COMPUTE),
@@ -397,7 +413,7 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 	/* Xe_LPG */
 
 	{ XE_RTP_NAME("14017856879"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1271),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274),
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(ROW_CHICKEN3, DIS_FIX_EOT1_FLUSH))
 	},
@@ -407,6 +423,11 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 	  XE_RTP_ACTIONS(SET(XEHP_HDC_CHICKEN0, DIS_ATOMIC_CHAINING_TYPED_WRITES,
 			     XE_RTP_NOCHECK))
 	},
+	{ XE_RTP_NAME("14020495402"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(ROW_CHICKEN2, DISABLE_TDL_SVHS_GATING))
+	},
 
 	/* Xe2_LPG */
 
@@ -424,8 +445,12 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN5, DISABLE_SAMPLE_G_PERFORMANCE))
 	},
-	{ XE_RTP_NAME("16021540221"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
+	{ XE_RTP_NAME("14020338487"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(ROW_CHICKEN3, XE2_EUPEND_CHK_FLUSH_DIS))
+	},
+	{ XE_RTP_NAME("18034896535, 16021540221"), /* 16021540221: GRAPHICS_STEP(A0, B0) */
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004),
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(ROW_CHICKEN4, DISABLE_TDL_PUSH))
 	},
@@ -460,6 +485,65 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, SLM_WMTP_RESTORE))
 	},
+
+	/* Xe2_HPG */
+
+	{ XE_RTP_NAME("16018712365"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, XE2_ALLOC_DPA_STARVE_FIX_DIS))
+	},
+	{ XE_RTP_NAME("16018737384"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(ROW_CHICKEN, EARLY_EOT_DIS))
+	},
+	{ XE_RTP_NAME("14019988906"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD))
+	},
+	{ XE_RTP_NAME("14019877138"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))
+	},
+	{ XE_RTP_NAME("14020338487"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(ROW_CHICKEN3, XE2_EUPEND_CHK_FLUSH_DIS))
+	},
+	{ XE_RTP_NAME("18032247524"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, SEQUENTIAL_ACCESS_UPGRADE_DISABLE))
+	},
+	{ XE_RTP_NAME("14018471104"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2001), FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, ENABLE_SMP_LD_RENDER_SURFACE_CONTROL))
+	},
+	/*
+	 * Although this workaround isn't required for the RCS, disabling these
+	 * reports has no impact for our driver or the GuC, so we go ahead and
+	 * apply this to all engines for simplicity.
+	 */
+	{ XE_RTP_NAME("16021639441"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2001)),
+	  XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),
+			     GHWSP_CSB_REPORT_DIS |
+			     PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS,
+			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
+	{ XE_RTP_NAME("14019811474"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2001),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, WR_REQ_CHAINING_DIS))
+	},
+
+	/* Xe2_HPM */
+
+	{ XE_RTP_NAME("16021639441"),
+	  XE_RTP_RULES(MEDIA_VERSION(1301)),
+	  XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),
+			     GHWSP_CSB_REPORT_DIS |
+			     PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS,
+			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
+
 	{}
 };
 
@@ -537,7 +621,7 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
 	/* Xe_LPG */
 
 	{ XE_RTP_NAME("18019271663"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1271)),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274)),
 	  XE_RTP_ACTIONS(SET(CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE))
 	},
 	{ XE_RTP_NAME("14019877138"),
@@ -580,6 +664,24 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
 		       ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(INSTPM(RENDER_RING_BASE), ENABLE_SEMAPHORE_POLL_BIT))
 	},
+	{ XE_RTP_NAME("18033852989"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST))
+	},
+
+	/* Xe2_HPG */
+	{ XE_RTP_NAME("15010599737"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_SF_ROUND_NEAREST_EVEN))
+	},
+	{ XE_RTP_NAME("14019386621"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE))
+	},
+	{ XE_RTP_NAME("14020756599"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS))
+	},
 
 	{}
 };
diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules
index b138cbd51bdb..12fe88796a49 100644
--- a/drivers/gpu/drm/xe/xe_wa_oob.rules
+++ b/drivers/gpu/drm/xe/xe_wa_oob.rules
@@ -4,9 +4,6 @@
 22011391025	PLATFORM(DG2)
 22012727170	SUBPLATFORM(DG2, G11)
 22012727685	SUBPLATFORM(DG2, G11)
-16015675438	PLATFORM(PVC)
-		SUBPLATFORM(DG2, G10)
-		SUBPLATFORM(DG2, G12)
 18020744125	PLATFORM(PVC)
 1509372804	PLATFORM(PVC), GRAPHICS_STEP(A0, C0)
 1409600907	GRAPHICS_VERSION_RANGE(1200, 1250)
@@ -22,3 +19,11 @@
 		GRAPHICS_VERSION_RANGE(1270, 1274)
 		MEDIA_VERSION(1300)
 		PLATFORM(DG2)
+14018094691	GRAPHICS_VERSION(2004)
+14019882105	GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)
+18024947630	GRAPHICS_VERSION(2001)
+		GRAPHICS_VERSION(2004)
+		MEDIA_VERSION(2000)
+16022287689	GRAPHICS_VERSION(2001)
+		GRAPHICS_VERSION(2004)
+13011645652	GRAPHICS_VERSION(2004)
diff --git a/drivers/gpu/drm/xen/xen_drm_front_gem.c b/drivers/gpu/drm/xen/xen_drm_front_gem.c
index 3ad2b4cfd1f0..63112ed975c4 100644
--- a/drivers/gpu/drm/xen/xen_drm_front_gem.c
+++ b/drivers/gpu/drm/xen/xen_drm_front_gem.c
@@ -11,6 +11,7 @@
 #include <linux/dma-buf.h>
 #include <linux/scatterlist.h>
 #include <linux/shmem_fs.h>
+#include <linux/vmalloc.h>
 
 #include <drm/drm_gem.h>
 #include <drm/drm_prime.h>
diff --git a/drivers/gpu/drm/xlnx/zynqmp_disp.c b/drivers/gpu/drm/xlnx/zynqmp_disp.c
index 8a39b3accce5..c9fb432d4cbd 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_disp.c
+++ b/drivers/gpu/drm/xlnx/zynqmp_disp.c
@@ -18,6 +18,7 @@
 #include <linux/dma/xilinx_dpdma.h>
 #include <linux/dma-mapping.h>
 #include <linux/dmaengine.h>
+#include <linux/media-bus-format.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
@@ -65,14 +66,26 @@
 #define ZYNQMP_DISP_MAX_NUM_SUB_PLANES			3
 
 /**
+ * enum zynqmp_dpsub_layer_mode - Layer mode
+ * @ZYNQMP_DPSUB_LAYER_NONLIVE: non-live (memory) mode
+ * @ZYNQMP_DPSUB_LAYER_LIVE: live (stream) mode
+ */
+enum zynqmp_dpsub_layer_mode {
+	ZYNQMP_DPSUB_LAYER_NONLIVE,
+	ZYNQMP_DPSUB_LAYER_LIVE,
+};
+
+/**
  * struct zynqmp_disp_format - Display subsystem format information
  * @drm_fmt: DRM format (4CC)
+ * @bus_fmt: Media bus format
  * @buf_fmt: AV buffer format
  * @swap: Flag to swap R & B for RGB formats, and U & V for YUV formats
  * @sf: Scaling factors for color components
  */
 struct zynqmp_disp_format {
 	u32 drm_fmt;
+	u32 bus_fmt;
 	u32 buf_fmt;
 	bool swap;
 	const u32 *sf;
@@ -172,6 +185,12 @@ static const u32 scaling_factors_565[] = {
 	ZYNQMP_DISP_AV_BUF_5BIT_SF,
 };
 
+static const u32 scaling_factors_666[] = {
+	ZYNQMP_DISP_AV_BUF_6BIT_SF,
+	ZYNQMP_DISP_AV_BUF_6BIT_SF,
+	ZYNQMP_DISP_AV_BUF_6BIT_SF,
+};
+
 static const u32 scaling_factors_888[] = {
 	ZYNQMP_DISP_AV_BUF_8BIT_SF,
 	ZYNQMP_DISP_AV_BUF_8BIT_SF,
@@ -354,6 +373,41 @@ static const struct zynqmp_disp_format avbuf_gfx_fmts[] = {
 	},
 };
 
+/* List of live video layer formats */
+static const struct zynqmp_disp_format avbuf_live_fmts[] = {
+	{
+		.drm_fmt	= DRM_FORMAT_RGB565,
+		.bus_fmt	= MEDIA_BUS_FMT_RGB666_1X18,
+		.buf_fmt	= ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_BPC_6 |
+				  ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_RGB,
+		.sf		= scaling_factors_666,
+	}, {
+		.drm_fmt	= DRM_FORMAT_RGB888,
+		.bus_fmt	= MEDIA_BUS_FMT_RGB888_1X24,
+		.buf_fmt	= ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_BPC_8 |
+				  ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_RGB,
+		.sf		= scaling_factors_888,
+	}, {
+		.drm_fmt	= DRM_FORMAT_YUV422,
+		.bus_fmt	= MEDIA_BUS_FMT_UYVY8_1X16,
+		.buf_fmt	= ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_BPC_8 |
+				  ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_YUV422,
+		.sf		= scaling_factors_888,
+	}, {
+		.drm_fmt	= DRM_FORMAT_YUV444,
+		.bus_fmt	= MEDIA_BUS_FMT_VUY8_1X24,
+		.buf_fmt	= ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_BPC_8 |
+				  ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_YUV444,
+		.sf		= scaling_factors_888,
+	}, {
+		.drm_fmt	= DRM_FORMAT_P210,
+		.bus_fmt	= MEDIA_BUS_FMT_UYVY10_1X20,
+		.buf_fmt	= ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_BPC_10 |
+				  ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_YUV422,
+		.sf		= scaling_factors_101010,
+	},
+};
+
 static u32 zynqmp_disp_avbuf_read(struct zynqmp_disp *disp, int reg)
 {
 	return readl(disp->avbuf.base + reg);
@@ -382,19 +436,29 @@ static void zynqmp_disp_avbuf_set_format(struct zynqmp_disp *disp,
 					 const struct zynqmp_disp_format *fmt)
 {
 	unsigned int i;
-	u32 val;
-
-	val = zynqmp_disp_avbuf_read(disp, ZYNQMP_DISP_AV_BUF_FMT);
-	val &= zynqmp_disp_layer_is_video(layer)
-	    ? ~ZYNQMP_DISP_AV_BUF_FMT_NL_VID_MASK
-	    : ~ZYNQMP_DISP_AV_BUF_FMT_NL_GFX_MASK;
-	val |= fmt->buf_fmt;
-	zynqmp_disp_avbuf_write(disp, ZYNQMP_DISP_AV_BUF_FMT, val);
+	u32 val, reg;
+
+	layer->disp_fmt = fmt;
+	if (layer->mode == ZYNQMP_DPSUB_LAYER_NONLIVE) {
+		reg = ZYNQMP_DISP_AV_BUF_FMT;
+		val = zynqmp_disp_avbuf_read(disp, ZYNQMP_DISP_AV_BUF_FMT);
+		val &= zynqmp_disp_layer_is_video(layer)
+		    ? ~ZYNQMP_DISP_AV_BUF_FMT_NL_VID_MASK
+		    : ~ZYNQMP_DISP_AV_BUF_FMT_NL_GFX_MASK;
+		val |= fmt->buf_fmt;
+		zynqmp_disp_avbuf_write(disp, reg, val);
+	} else {
+		reg = zynqmp_disp_layer_is_video(layer)
+		    ? ZYNQMP_DISP_AV_BUF_LIVE_VID_CONFIG
+		    : ZYNQMP_DISP_AV_BUF_LIVE_GFX_CONFIG;
+		val = fmt->buf_fmt;
+		zynqmp_disp_avbuf_write(disp, reg, val);
+	}
 
 	for (i = 0; i < ZYNQMP_DISP_AV_BUF_NUM_SF; i++) {
-		unsigned int reg = zynqmp_disp_layer_is_video(layer)
-				 ? ZYNQMP_DISP_AV_BUF_VID_COMP_SF(i)
-				 : ZYNQMP_DISP_AV_BUF_GFX_COMP_SF(i);
+		reg = zynqmp_disp_layer_is_video(layer)
+		    ? ZYNQMP_DISP_AV_BUF_VID_COMP_SF(i)
+		    : ZYNQMP_DISP_AV_BUF_GFX_COMP_SF(i);
 
 		zynqmp_disp_avbuf_write(disp, reg, fmt->sf[i]);
 	}
@@ -873,10 +937,40 @@ zynqmp_disp_layer_find_format(struct zynqmp_disp_layer *layer,
 }
 
 /**
+ * zynqmp_disp_layer_find_live_format - Find format information for given
+ * media bus format
+ * @layer: The layer
+ * @media_bus_format: Media bus format to search
+ *
+ * Search display subsystem format information corresponding to the given media
+ * bus format @media_bus_format for the @layer, and return a pointer to the
+ * format descriptor.
+ *
+ * Return: A pointer to the format descriptor if found, NULL otherwise
+ */
+static const struct zynqmp_disp_format *
+zynqmp_disp_layer_find_live_format(struct zynqmp_disp_layer *layer,
+				   u32 media_bus_format)
+{
+	unsigned int i;
+
+	for (i = 0; i < layer->info->num_formats; i++)
+		if (layer->info->formats[i].bus_fmt == media_bus_format)
+			return &layer->info->formats[i];
+
+	return NULL;
+}
+
+/**
  * zynqmp_disp_layer_drm_formats - Return the DRM formats supported by the layer
  * @layer: The layer
  * @num_formats: Pointer to the returned number of formats
  *
+ * NOTE: This function doesn't make sense for live video layers and will
+ * always return an empty list in such cases. zynqmp_disp_live_layer_formats()
+ * should be used to query a list of media bus formats supported by the live
+ * video input layer.
+ *
  * Return: A newly allocated u32 array that stores all the DRM formats
  * supported by the layer. The number of formats in the array is returned
  * through the num_formats argument.
@@ -887,10 +981,17 @@ u32 *zynqmp_disp_layer_drm_formats(struct zynqmp_disp_layer *layer,
 	unsigned int i;
 	u32 *formats;
 
+	if (WARN_ON(layer->mode != ZYNQMP_DPSUB_LAYER_NONLIVE)) {
+		*num_formats = 0;
+		return NULL;
+	}
+
 	formats = kcalloc(layer->info->num_formats, sizeof(*formats),
 			  GFP_KERNEL);
-	if (!formats)
+	if (!formats) {
+		*num_formats = 0;
 		return NULL;
+	}
 
 	for (i = 0; i < layer->info->num_formats; ++i)
 		formats[i] = layer->info->formats[i].drm_fmt;
@@ -900,17 +1001,51 @@ u32 *zynqmp_disp_layer_drm_formats(struct zynqmp_disp_layer *layer,
 }
 
 /**
+ * zynqmp_disp_live_layer_formats - Return the media bus formats supported by
+ * the live video layer
+ * @layer: The layer
+ * @num_formats: Pointer to the returned number of formats
+ *
+ * NOTE: This function should be used only for live video input layers.
+ *
+ * Return: A newly allocated u32 array of media bus formats supported by the
+ * layer. The number of formats in the array is returned through the
+ * @num_formats argument.
+ */
+u32 *zynqmp_disp_live_layer_formats(struct zynqmp_disp_layer *layer,
+				    unsigned int *num_formats)
+{
+	unsigned int i;
+	u32 *formats;
+
+	if (WARN_ON(layer->mode != ZYNQMP_DPSUB_LAYER_LIVE)) {
+		*num_formats = 0;
+		return NULL;
+	}
+
+	formats = kcalloc(layer->info->num_formats, sizeof(*formats),
+			  GFP_KERNEL);
+	if (!formats) {
+		*num_formats = 0;
+		return NULL;
+	}
+
+	for (i = 0; i < layer->info->num_formats; ++i)
+		formats[i] = layer->info->formats[i].bus_fmt;
+
+	*num_formats = layer->info->num_formats;
+	return formats;
+}
+
+/**
  * zynqmp_disp_layer_enable - Enable a layer
  * @layer: The layer
- * @mode: Operating mode of layer
  *
  * Enable the @layer in the audio/video buffer manager and the blender. DMA
  * channels are started separately by zynqmp_disp_layer_update().
  */
-void zynqmp_disp_layer_enable(struct zynqmp_disp_layer *layer,
-			      enum zynqmp_dpsub_layer_mode mode)
+void zynqmp_disp_layer_enable(struct zynqmp_disp_layer *layer)
 {
-	layer->mode = mode;
 	zynqmp_disp_avbuf_enable_video(layer->disp, layer);
 	zynqmp_disp_blend_layer_enable(layer->disp, layer);
 }
@@ -926,7 +1061,7 @@ void zynqmp_disp_layer_disable(struct zynqmp_disp_layer *layer)
 {
 	unsigned int i;
 
-	if (layer->disp->dpsub->dma_enabled) {
+	if (layer->mode == ZYNQMP_DPSUB_LAYER_NONLIVE) {
 		for (i = 0; i < layer->drm_fmt->num_planes; i++)
 			dmaengine_terminate_sync(layer->dmas[i].chan);
 	}
@@ -940,6 +1075,9 @@ void zynqmp_disp_layer_disable(struct zynqmp_disp_layer *layer)
  * @layer: The layer
  * @info: The format info
  *
+ * NOTE: Use zynqmp_disp_layer_set_live_format() to set media bus format for
+ * live video layers.
+ *
  * Set the format for @layer to @info. The layer must be disabled.
  */
 void zynqmp_disp_layer_set_format(struct zynqmp_disp_layer *layer,
@@ -947,14 +1085,16 @@ void zynqmp_disp_layer_set_format(struct zynqmp_disp_layer *layer,
 {
 	unsigned int i;
 
+	if (WARN_ON(layer->mode != ZYNQMP_DPSUB_LAYER_NONLIVE))
+		return;
+
 	layer->disp_fmt = zynqmp_disp_layer_find_format(layer, info->format);
+	if (WARN_ON(!layer->disp_fmt))
+		return;
 	layer->drm_fmt = info;
 
 	zynqmp_disp_avbuf_set_format(layer->disp, layer, layer->disp_fmt);
 
-	if (!layer->disp->dpsub->dma_enabled)
-		return;
-
 	/*
 	 * Set pconfig for each DMA channel to indicate they're part of a
 	 * video group.
@@ -975,6 +1115,32 @@ void zynqmp_disp_layer_set_format(struct zynqmp_disp_layer *layer,
 }
 
 /**
+ * zynqmp_disp_layer_set_live_format - Set the live video layer format
+ * @layer: The layer
+ * @media_bus_format: Media bus format to set
+ *
+ * NOTE: This function should not be used to set format for non-live video
+ * layer. Use zynqmp_disp_layer_set_format() instead.
+ *
+ * Set the display format for the live @layer. The layer must be disabled.
+ */
+void zynqmp_disp_layer_set_live_format(struct zynqmp_disp_layer *layer,
+				       u32 media_bus_format)
+{
+	if (WARN_ON(layer->mode != ZYNQMP_DPSUB_LAYER_LIVE))
+		return;
+
+	layer->disp_fmt = zynqmp_disp_layer_find_live_format(layer,
+							     media_bus_format);
+	if (WARN_ON(!layer->disp_fmt))
+		return;
+
+	zynqmp_disp_avbuf_set_format(layer->disp, layer, layer->disp_fmt);
+
+	layer->drm_fmt = drm_format_info(layer->disp_fmt->drm_fmt);
+}
+
+/**
  * zynqmp_disp_layer_update - Update the layer framebuffer
  * @layer: The layer
  * @state: The plane state
@@ -990,7 +1156,7 @@ int zynqmp_disp_layer_update(struct zynqmp_disp_layer *layer,
 	const struct drm_format_info *info = layer->drm_fmt;
 	unsigned int i;
 
-	if (!layer->disp->dpsub->dma_enabled)
+	if (layer->mode == ZYNQMP_DPSUB_LAYER_LIVE)
 		return 0;
 
 	for (i = 0; i < info->num_planes; i++) {
@@ -1040,9 +1206,6 @@ static void zynqmp_disp_layer_release_dma(struct zynqmp_disp *disp,
 {
 	unsigned int i;
 
-	if (!layer->info || !disp->dpsub->dma_enabled)
-		return;
-
 	for (i = 0; i < layer->info->num_channels; i++) {
 		struct zynqmp_disp_layer_dma *dma = &layer->dmas[i];
 
@@ -1083,9 +1246,6 @@ static int zynqmp_disp_layer_request_dma(struct zynqmp_disp *disp,
 	unsigned int i;
 	int ret;
 
-	if (!disp->dpsub->dma_enabled)
-		return 0;
-
 	for (i = 0; i < layer->info->num_channels; i++) {
 		struct zynqmp_disp_layer_dma *dma = &layer->dmas[i];
 		char dma_channel_name[16];
@@ -1124,6 +1284,11 @@ static int zynqmp_disp_create_layers(struct zynqmp_disp *disp)
 			.num_channels = 1,
 		},
 	};
+	static const struct zynqmp_disp_layer_info live_layer_info = {
+		.formats = avbuf_live_fmts,
+		.num_formats = ARRAY_SIZE(avbuf_live_fmts),
+		.num_channels = 0,
+	};
 
 	unsigned int i;
 	int ret;
@@ -1133,7 +1298,17 @@ static int zynqmp_disp_create_layers(struct zynqmp_disp *disp)
 
 		layer->id = i;
 		layer->disp = disp;
-		layer->info = &layer_info[i];
+		/*
+		 * For now assume dpsub works in either live or non-live mode for both layers.
+		 * Hybrid mode is not supported yet.
+		 */
+		if (disp->dpsub->dma_enabled) {
+			layer->mode = ZYNQMP_DPSUB_LAYER_NONLIVE;
+			layer->info = &layer_info[i];
+		} else {
+			layer->mode = ZYNQMP_DPSUB_LAYER_LIVE;
+			layer->info = &live_layer_info;
+		}
 
 		ret = zynqmp_disp_layer_request_dma(disp, layer);
 		if (ret)
diff --git a/drivers/gpu/drm/xlnx/zynqmp_disp.h b/drivers/gpu/drm/xlnx/zynqmp_disp.h
index 123cffac08be..fa545533c9d1 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_disp.h
+++ b/drivers/gpu/drm/xlnx/zynqmp_disp.h
@@ -42,16 +42,6 @@ enum zynqmp_dpsub_layer_id {
 	ZYNQMP_DPSUB_LAYER_GFX,
 };
 
-/**
- * enum zynqmp_dpsub_layer_mode - Layer mode
- * @ZYNQMP_DPSUB_LAYER_NONLIVE: non-live (memory) mode
- * @ZYNQMP_DPSUB_LAYER_LIVE: live (stream) mode
- */
-enum zynqmp_dpsub_layer_mode {
-	ZYNQMP_DPSUB_LAYER_NONLIVE,
-	ZYNQMP_DPSUB_LAYER_LIVE,
-};
-
 void zynqmp_disp_enable(struct zynqmp_disp *disp);
 void zynqmp_disp_disable(struct zynqmp_disp *disp);
 int zynqmp_disp_setup_clock(struct zynqmp_disp *disp,
@@ -62,11 +52,14 @@ void zynqmp_disp_blend_set_global_alpha(struct zynqmp_disp *disp,
 
 u32 *zynqmp_disp_layer_drm_formats(struct zynqmp_disp_layer *layer,
 				   unsigned int *num_formats);
-void zynqmp_disp_layer_enable(struct zynqmp_disp_layer *layer,
-			      enum zynqmp_dpsub_layer_mode mode);
+u32 *zynqmp_disp_live_layer_formats(struct zynqmp_disp_layer *layer,
+				    unsigned int *num_formats);
+void zynqmp_disp_layer_enable(struct zynqmp_disp_layer *layer);
 void zynqmp_disp_layer_disable(struct zynqmp_disp_layer *layer);
 void zynqmp_disp_layer_set_format(struct zynqmp_disp_layer *layer,
 				  const struct drm_format_info *info);
+void zynqmp_disp_layer_set_live_format(struct zynqmp_disp_layer *layer,
+				       u32 media_bus_format);
 int zynqmp_disp_layer_update(struct zynqmp_disp_layer *layer,
 			     struct drm_plane_state *state);
 
diff --git a/drivers/gpu/drm/xlnx/zynqmp_disp_regs.h b/drivers/gpu/drm/xlnx/zynqmp_disp_regs.h
index f92a006d5070..fa3935384834 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_disp_regs.h
+++ b/drivers/gpu/drm/xlnx/zynqmp_disp_regs.h
@@ -165,10 +165,10 @@
 #define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_BPC_10		0x2
 #define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_BPC_12		0x3
 #define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_BPC_MASK		GENMASK(2, 0)
-#define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_RGB		0x0
-#define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_YUV444	0x1
-#define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_YUV422	0x2
-#define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_YONLY	0x3
+#define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_RGB		(0x0 << 4)
+#define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_YUV444	(0x1 << 4)
+#define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_YUV422	(0x2 << 4)
+#define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_YONLY	(0x3 << 4)
 #define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_FMT_MASK		GENMASK(5, 4)
 #define ZYNQMP_DISP_AV_BUF_LIVE_CONFIG_CB_FIRST		BIT(8)
 #define ZYNQMP_DISP_AV_BUF_PALETTE_MEMORY		0x400
diff --git a/drivers/gpu/drm/xlnx/zynqmp_dp.c b/drivers/gpu/drm/xlnx/zynqmp_dp.c
index 1846c4971fd8..8c2d24809014 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_dp.c
+++ b/drivers/gpu/drm/xlnx/zynqmp_dp.c
@@ -22,6 +22,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/io.h>
+#include <linux/media-bus-format.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
@@ -1276,28 +1277,45 @@ static void zynqmp_dp_encoder_mode_set_stream(struct zynqmp_dp *dp,
  * DISP Configuration
  */
 
+/**
+ * zynqmp_dp_disp_connected_live_layer - Return the first connected live layer
+ * @dp: DisplayPort IP core structure
+ *
+ * Return: The first connected live display layer or NULL if none of the live
+ * layers are connected.
+ */
+static struct zynqmp_disp_layer *
+zynqmp_dp_disp_connected_live_layer(struct zynqmp_dp *dp)
+{
+	if (dp->dpsub->connected_ports & BIT(ZYNQMP_DPSUB_PORT_LIVE_VIDEO))
+		return dp->dpsub->layers[ZYNQMP_DPSUB_LAYER_VID];
+	else if (dp->dpsub->connected_ports & BIT(ZYNQMP_DPSUB_PORT_LIVE_GFX))
+		return dp->dpsub->layers[ZYNQMP_DPSUB_LAYER_GFX];
+	else
+		return NULL;
+}
+
 static void zynqmp_dp_disp_enable(struct zynqmp_dp *dp,
 				  struct drm_bridge_state *old_bridge_state)
 {
-	enum zynqmp_dpsub_layer_id layer_id;
 	struct zynqmp_disp_layer *layer;
-	const struct drm_format_info *info;
+	struct drm_bridge_state *bridge_state;
+	u32 bus_fmt;
 
-	if (dp->dpsub->connected_ports & BIT(ZYNQMP_DPSUB_PORT_LIVE_VIDEO))
-		layer_id = ZYNQMP_DPSUB_LAYER_VID;
-	else if (dp->dpsub->connected_ports & BIT(ZYNQMP_DPSUB_PORT_LIVE_GFX))
-		layer_id = ZYNQMP_DPSUB_LAYER_GFX;
-	else
+	layer = zynqmp_dp_disp_connected_live_layer(dp);
+	if (!layer)
 		return;
 
-	layer = dp->dpsub->layers[layer_id];
+	bridge_state = drm_atomic_get_new_bridge_state(old_bridge_state->base.state,
+						       old_bridge_state->bridge);
+	if (WARN_ON(!bridge_state))
+		return;
 
-	/* TODO: Make the format configurable. */
-	info = drm_format_info(DRM_FORMAT_YUV422);
-	zynqmp_disp_layer_set_format(layer, info);
-	zynqmp_disp_layer_enable(layer, ZYNQMP_DPSUB_LAYER_LIVE);
+	bus_fmt = bridge_state->input_bus_cfg.format;
+	zynqmp_disp_layer_set_live_format(layer, bus_fmt);
+	zynqmp_disp_layer_enable(layer);
 
-	if (layer_id == ZYNQMP_DPSUB_LAYER_GFX)
+	if (layer == dp->dpsub->layers[ZYNQMP_DPSUB_LAYER_GFX])
 		zynqmp_disp_blend_set_global_alpha(dp->dpsub->disp, true, 255);
 	else
 		zynqmp_disp_blend_set_global_alpha(dp->dpsub->disp, false, 0);
@@ -1310,11 +1328,8 @@ static void zynqmp_dp_disp_disable(struct zynqmp_dp *dp,
 {
 	struct zynqmp_disp_layer *layer;
 
-	if (dp->dpsub->connected_ports & BIT(ZYNQMP_DPSUB_PORT_LIVE_VIDEO))
-		layer = dp->dpsub->layers[ZYNQMP_DPSUB_LAYER_VID];
-	else if (dp->dpsub->connected_ports & BIT(ZYNQMP_DPSUB_PORT_LIVE_GFX))
-		layer = dp->dpsub->layers[ZYNQMP_DPSUB_LAYER_GFX];
-	else
+	layer = zynqmp_dp_disp_connected_live_layer(dp);
+	if (!layer)
 		return;
 
 	zynqmp_disp_disable(dp->dpsub->disp);
@@ -1568,6 +1583,35 @@ static const struct drm_edid *zynqmp_dp_bridge_edid_read(struct drm_bridge *brid
 	return drm_edid_read_ddc(connector, &dp->aux.ddc);
 }
 
+static u32 *zynqmp_dp_bridge_default_bus_fmts(unsigned int *num_input_fmts)
+{
+	u32 *formats = kzalloc(sizeof(*formats), GFP_KERNEL);
+
+	if (formats)
+		*formats = MEDIA_BUS_FMT_FIXED;
+	*num_input_fmts = !!formats;
+
+	return formats;
+}
+
+static u32 *
+zynqmp_dp_bridge_get_input_bus_fmts(struct drm_bridge *bridge,
+				    struct drm_bridge_state *bridge_state,
+				    struct drm_crtc_state *crtc_state,
+				    struct drm_connector_state *conn_state,
+				    u32 output_fmt,
+				    unsigned int *num_input_fmts)
+{
+	struct zynqmp_dp *dp = bridge_to_dp(bridge);
+	struct zynqmp_disp_layer *layer;
+
+	layer = zynqmp_dp_disp_connected_live_layer(dp);
+	if (layer)
+		return zynqmp_disp_live_layer_formats(layer, num_input_fmts);
+	else
+		return zynqmp_dp_bridge_default_bus_fmts(num_input_fmts);
+}
+
 static const struct drm_bridge_funcs zynqmp_dp_bridge_funcs = {
 	.attach = zynqmp_dp_bridge_attach,
 	.detach = zynqmp_dp_bridge_detach,
@@ -1580,6 +1624,7 @@ static const struct drm_bridge_funcs zynqmp_dp_bridge_funcs = {
 	.atomic_check = zynqmp_dp_bridge_atomic_check,
 	.detect = zynqmp_dp_bridge_detect,
 	.edid_read = zynqmp_dp_bridge_edid_read,
+	.atomic_get_input_bus_fmts = zynqmp_dp_bridge_get_input_bus_fmts,
 };
 
 /* -----------------------------------------------------------------------------
@@ -1714,6 +1759,10 @@ int zynqmp_dp_probe(struct zynqmp_dpsub *dpsub)
 		goto err_free;
 	}
 
+	ret = zynqmp_dp_reset(dp, true);
+	if (ret < 0)
+		goto err_free;
+
 	ret = zynqmp_dp_reset(dp, false);
 	if (ret < 0)
 		goto err_free;
diff --git a/drivers/gpu/drm/xlnx/zynqmp_dpsub.c b/drivers/gpu/drm/xlnx/zynqmp_dpsub.c
index 88eb33acd5f0..face8d6b2a6f 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_dpsub.c
+++ b/drivers/gpu/drm/xlnx/zynqmp_dpsub.c
@@ -256,12 +256,12 @@ static int zynqmp_dpsub_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_dp;
 
+	drm_bridge_add(dpsub->bridge);
+
 	if (dpsub->dma_enabled) {
 		ret = zynqmp_dpsub_drm_init(dpsub);
 		if (ret)
 			goto err_disp;
-	} else {
-		drm_bridge_add(dpsub->bridge);
 	}
 
 	dev_info(&pdev->dev, "ZynqMP DisplayPort Subsystem driver probed");
@@ -288,9 +288,8 @@ static void zynqmp_dpsub_remove(struct platform_device *pdev)
 
 	if (dpsub->drm)
 		zynqmp_dpsub_drm_cleanup(dpsub);
-	else
-		drm_bridge_remove(dpsub->bridge);
 
+	drm_bridge_remove(dpsub->bridge);
 	zynqmp_disp_remove(dpsub);
 	zynqmp_dp_remove(dpsub);
 
diff --git a/drivers/gpu/drm/xlnx/zynqmp_kms.c b/drivers/gpu/drm/xlnx/zynqmp_kms.c
index db3bb4afbfc4..43bf416b33d5 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_kms.c
+++ b/drivers/gpu/drm/xlnx/zynqmp_kms.c
@@ -122,7 +122,7 @@ static void zynqmp_dpsub_plane_atomic_update(struct drm_plane *plane,
 
 	/* Enable or re-enable the plane if the format has changed. */
 	if (format_changed)
-		zynqmp_disp_layer_enable(layer, ZYNQMP_DPSUB_LAYER_NONLIVE);
+		zynqmp_disp_layer_enable(layer);
 }
 
 static const struct drm_plane_helper_funcs zynqmp_dpsub_plane_helper_funcs = {
diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c
index 89983d7d73ca..3a0aaa68ac8d 100644
--- a/drivers/gpu/host1x/dev.c
+++ b/drivers/gpu/host1x/dev.c
@@ -216,6 +216,30 @@ static const struct host1x_info host1x07_info = {
  */
 static const struct host1x_sid_entry tegra234_sid_table[] = {
 	{
+		/* SE2 MMIO */
+		.base = 0x1658,
+		.offset = 0x90,
+		.limit = 0x90
+	},
+	{
+		/* SE4 MMIO */
+		.base = 0x1660,
+		.offset = 0x90,
+		.limit = 0x90
+	},
+	{
+		/* SE2 channel */
+		.base = 0x1738,
+		.offset = 0x90,
+		.limit = 0x90
+	},
+	{
+		/* SE4 channel */
+		.base = 0x1740,
+		.offset = 0x90,
+		.limit = 0x90
+	},
+	{
 		/* VIC channel */
 		.base = 0x17b8,
 		.offset = 0x30,